Repository: gristlabs/grist-core Branch: main Commit: c2f6e1fc8184 Files: 2151 Total size: 19.8 MB Directory structure: gitextract_r_s8ij64/ ├── .dockerignore ├── .editorconfig ├── .git-blame-ignore-revs ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── 00-bug-issue.yml │ │ ├── 10-installation-issue.yml │ │ ├── 20-feature-request.yml │ │ └── config.yml │ ├── PULL_REQUEST_TEMPLATE.md │ ├── cla/ │ │ ├── individual-cla.md │ │ └── signatures.json │ └── workflows/ │ ├── cla.yml │ ├── docker.yml │ ├── docker_latest.yml │ ├── fly-build.yml │ ├── fly-cleanup.yml │ ├── fly-deploy.yml │ ├── fly-destroy.yml │ ├── main.yml │ ├── self-hosted.yml │ └── translation_keys.yml ├── .gitignore ├── .nvmrc ├── .yarnrc ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── SECURITY.md ├── app/ │ ├── cli.sh │ ├── client/ │ │ ├── DefaultHooks.ts │ │ ├── Hooks.ts │ │ ├── aclui/ │ │ │ ├── ACLColumnList.ts │ │ │ ├── ACLFormulaEditor.ts │ │ │ ├── ACLMemoEditor.ts │ │ │ ├── ACLSelect.ts │ │ │ ├── ACLUsers.ts │ │ │ ├── AccessRules.ts │ │ │ └── PermissionsWidget.ts │ │ ├── apiconsole.ts │ │ ├── app.css │ │ ├── app.js │ │ ├── billingMain.ts │ │ ├── browserCheck.ts │ │ ├── components/ │ │ │ ├── AceEditor.css │ │ │ ├── AceEditor.js │ │ │ ├── AceEditorCompletions.ts │ │ │ ├── ActionCounter.ts │ │ │ ├── ActionLog.css │ │ │ ├── ActionLog.ts │ │ │ ├── Banner.ts │ │ │ ├── BaseView.ts │ │ │ ├── BaseView2.ts │ │ │ ├── BehavioralPromptsManager.ts │ │ │ ├── CellPosition.ts │ │ │ ├── CellSelector.ts │ │ │ ├── ChartView.css │ │ │ ├── ChartView.ts │ │ │ ├── ClientScope.ts │ │ │ ├── Clipboard.css │ │ │ ├── Clipboard.ts │ │ │ ├── CodeEditorPanel.css │ │ │ ├── CodeEditorPanel.ts │ │ │ ├── ColumnFilters.css │ │ │ ├── ColumnTransform.ts │ │ │ ├── Comm.ts │ │ │ ├── CopySelection.ts │ │ │ ├── CoreBanners.ts │ │ │ ├── Cursor.ts │ │ │ ├── CursorMonitor.ts │ │ │ ├── CustomCalendarView.ts │ │ │ ├── CustomView.css │ │ │ ├── CustomView.ts │ │ │ ├── DataTables.ts │ │ │ ├── DetailView.css │ │ │ ├── DetailView.ts │ │ │ ├── DocComm.ts │ │ │ ├── DocumentUsage.ts │ │ │ ├── Drafts.ts │ │ │ ├── DropdownConditionConfig.ts │ │ │ ├── DropdownConditionEditor.ts │ │ │ ├── EditorMonitor.ts │ │ │ ├── EmbedForm.css │ │ │ ├── ExternalAttachmentBanner.ts │ │ │ ├── FieldConfigTab.css │ │ │ ├── FormRenderer.ts │ │ │ ├── FormRendererCss.ts │ │ │ ├── Forms/ │ │ │ │ ├── Columns.ts │ │ │ │ ├── Editor.ts │ │ │ │ ├── Field.ts │ │ │ │ ├── FormConfig.ts │ │ │ │ ├── FormView.ts │ │ │ │ ├── MappedFieldsConfig.ts │ │ │ │ ├── Menu.ts │ │ │ │ ├── Model.ts │ │ │ │ ├── Paragraph.ts │ │ │ │ ├── Section.ts │ │ │ │ ├── Submit.ts │ │ │ │ ├── elements.ts │ │ │ │ └── styles.ts │ │ │ ├── FormulaTransform.ts │ │ │ ├── GridView.css │ │ │ ├── GridView.ts │ │ │ ├── GristClientSocket.ts │ │ │ ├── GristDoc.css │ │ │ ├── GristDoc.ts │ │ │ ├── GristWSConnection.ts │ │ │ ├── Importer.ts │ │ │ ├── KeyboardFocusHighlighter.ts │ │ │ ├── Layout.css │ │ │ ├── Layout.ts │ │ │ ├── LayoutEditor.css │ │ │ ├── LayoutEditor.ts │ │ │ ├── LayoutTray.ts │ │ │ ├── LinkingState.ts │ │ │ ├── Login.css │ │ │ ├── ParseOptions.ts │ │ │ ├── PluginScreen.ts │ │ │ ├── Printing.css │ │ │ ├── Printing.ts │ │ │ ├── RawDataPage.ts │ │ │ ├── RecordCardPopup.ts │ │ │ ├── RecordLayout.css │ │ │ ├── RecordLayout.js │ │ │ ├── RecordLayoutEditor.js │ │ │ ├── RefSelect.ts │ │ │ ├── RegionFocusSwitcher.ts │ │ │ ├── SearchBar.css │ │ │ ├── SelectionSummary.ts │ │ │ ├── TypeConversion.ts │ │ │ ├── TypeTransform.ts │ │ │ ├── UndoStack.ts │ │ │ ├── UnsavedChanges.ts │ │ │ ├── VersionUpdateBanner.ts │ │ │ ├── ViewAsBanner.ts │ │ │ ├── ViewConfigTab.css │ │ │ ├── ViewConfigTab.js │ │ │ ├── ViewLayout.css │ │ │ ├── ViewLayout.ts │ │ │ ├── ViewLinker.css │ │ │ ├── ViewPane.ts │ │ │ ├── VirtualDoc.ts │ │ │ ├── VirtualTable.ts │ │ │ ├── WidgetFrame.ts │ │ │ ├── buildViewSectionDom.ts │ │ │ ├── commandList.ts │ │ │ ├── commands.css │ │ │ ├── commands.ts │ │ │ ├── duplicatePage.ts │ │ │ ├── duplicateWidget.ts │ │ │ ├── modals.ts │ │ │ ├── viewCommon.css │ │ │ └── viewCommon.js │ │ ├── declarations.d.ts │ │ ├── errorMain.ts │ │ ├── exposeModulesForTests.js │ │ ├── formMain.ts │ │ ├── lib/ │ │ │ ├── ACIndex.ts │ │ │ ├── ACSelect.ts │ │ │ ├── ACUserManager.ts │ │ │ ├── BoxSpec.ts │ │ │ ├── CellDiffTool.ts │ │ │ ├── CustomSectionElement.ts │ │ │ ├── Delay.ts │ │ │ ├── DocPluginManager.ts │ │ │ ├── DocSchemaImport.ts │ │ │ ├── FocusLayer.ts │ │ │ ├── GristWindow.ts │ │ │ ├── HomePluginManager.ts │ │ │ ├── ImportSourceElement.ts │ │ │ ├── Mousetrap.js │ │ │ ├── MultiUserManager.ts │ │ │ ├── ObservableMap.js │ │ │ ├── ObservableSet.js │ │ │ ├── ReferenceUtils.ts │ │ │ ├── SafeBrowser.ts │ │ │ ├── SafeBrowserProcess.css │ │ │ ├── Signal.ts │ │ │ ├── Suggestions.ts │ │ │ ├── TokenField.ts │ │ │ ├── UrlState.ts │ │ │ ├── Validator.ts │ │ │ ├── airtable/ │ │ │ │ ├── AirtableImportUI.ts │ │ │ │ ├── AirtableImporter.ts │ │ │ │ ├── startDocAirtableImport.ts │ │ │ │ └── startHomeAirtableImport.ts │ │ │ ├── autocomplete.ts │ │ │ ├── browserGlobals.ts │ │ │ ├── browserInfo.ts │ │ │ ├── chartUtil.ts │ │ │ ├── clipboardUtils.ts │ │ │ ├── dblclick.ts │ │ │ ├── dispose.d.ts │ │ │ ├── dispose.js │ │ │ ├── dom.js │ │ │ ├── domAsync.ts │ │ │ ├── domUtils.ts │ │ │ ├── download.js │ │ │ ├── formUtils.ts │ │ │ ├── formatUtils.ts │ │ │ ├── fromKoSave.ts │ │ │ ├── getOrCreateStyleElement.ts │ │ │ ├── guessTimezone.ts │ │ │ ├── hashUtils.ts │ │ │ ├── helpScout.ts │ │ │ ├── imports.d.ts │ │ │ ├── imports.js │ │ │ ├── isFocusable.ts │ │ │ ├── koArray.d.ts │ │ │ ├── koArray.js │ │ │ ├── koArrayWrap.ts │ │ │ ├── koDom.js │ │ │ ├── koDomScrolly.css │ │ │ ├── koDomScrolly.js │ │ │ ├── koForm.css │ │ │ ├── koForm.js │ │ │ ├── koUtil.js │ │ │ ├── loadScript.ts │ │ │ ├── localStorageObs.ts │ │ │ ├── localization.ts │ │ │ ├── log.ts │ │ │ ├── markdown.ts │ │ │ ├── nameUtils.ts │ │ │ ├── pausableObs.ts │ │ │ ├── popupControl.ts │ │ │ ├── popupUtils.ts │ │ │ ├── sanitizeUrl.ts │ │ │ ├── sessionObs.ts │ │ │ ├── simpleList.ts │ │ │ ├── sortUtil.ts │ │ │ ├── storage.ts │ │ │ ├── tableUtil.ts │ │ │ ├── telemetry.ts │ │ │ ├── testState.ts │ │ │ ├── textUtils.ts │ │ │ ├── timeUtils.ts │ │ │ ├── trapTabKey.ts │ │ │ ├── uploads.ts │ │ │ └── urlUtils.ts │ │ ├── logo.css │ │ ├── models/ │ │ │ ├── AdminChecks.ts │ │ │ ├── AppModel.ts │ │ │ ├── AuditLogsModel.ts │ │ │ ├── BaseRowModel.js │ │ │ ├── ChatHistory.ts │ │ │ ├── ClientColumnGetters.ts │ │ │ ├── ColumnACIndexes.ts │ │ │ ├── ColumnCache.ts │ │ │ ├── ColumnFilter.ts │ │ │ ├── ColumnFilterMenuModel.ts │ │ │ ├── ColumnToMap.ts │ │ │ ├── ConnectState.ts │ │ │ ├── DataRowModel.ts │ │ │ ├── DataTableModel.js │ │ │ ├── DataTableModelWithDiff.ts │ │ │ ├── DocData.ts │ │ │ ├── DocModel.ts │ │ │ ├── DocPageModel.ts │ │ │ ├── FormModel.ts │ │ │ ├── HomeModel.ts │ │ │ ├── MetaRowModel.js │ │ │ ├── MetaTableModel.js │ │ │ ├── NotifyModel.ts │ │ │ ├── QuerySet.ts │ │ │ ├── RuleOwner.ts │ │ │ ├── SearchModel.ts │ │ │ ├── SectionFilter.ts │ │ │ ├── Styles.ts │ │ │ ├── TableData.ts │ │ │ ├── TableModel.js │ │ │ ├── TelemetryModel.ts │ │ │ ├── TimeQuery.ts │ │ │ ├── ToggleEnterpriseModel.ts │ │ │ ├── TreeModel.ts │ │ │ ├── UnionRowSource.ts │ │ │ ├── UserManagerModel.ts │ │ │ ├── UserPrefs.ts │ │ │ ├── UserPresenceModel.ts │ │ │ ├── ViewFieldConfig.ts │ │ │ ├── VirtualTable.ts │ │ │ ├── VirtualTableMeta.ts │ │ │ ├── WorkspaceInfo.ts │ │ │ ├── entities/ │ │ │ │ ├── ACLRuleRec.ts │ │ │ │ ├── CellRec.ts │ │ │ │ ├── ColumnRec.ts │ │ │ │ ├── DocInfoRec.ts │ │ │ │ ├── FilterRec.ts │ │ │ │ ├── PageRec.ts │ │ │ │ ├── ShareRec.ts │ │ │ │ ├── TabBarRec.ts │ │ │ │ ├── TableRec.ts │ │ │ │ ├── ValidationRec.ts │ │ │ │ ├── ViewFieldRec.ts │ │ │ │ ├── ViewRec.ts │ │ │ │ └── ViewSectionRec.ts │ │ │ ├── errors.ts │ │ │ ├── features.ts │ │ │ ├── gristConfigCache.ts │ │ │ ├── gristUrlState.ts │ │ │ ├── homeUrl.ts │ │ │ ├── modelUtil.js │ │ │ ├── rowset.ts │ │ │ └── rowuid.js │ │ ├── tsconfig.json │ │ ├── ui/ │ │ │ ├── AccountPage.ts │ │ │ ├── AccountPageCss.ts │ │ │ ├── AccountWidget.ts │ │ │ ├── AccountWidgetCss.ts │ │ │ ├── ActiveUserList.ts │ │ │ ├── AddNewButton.ts │ │ │ ├── AddNewTip.ts │ │ │ ├── AdminLeftPanel.ts │ │ │ ├── AdminPanel.ts │ │ │ ├── AdminPanelCss.ts │ │ │ ├── AdminPanelName.ts │ │ │ ├── AdminTogglesCss.ts │ │ │ ├── ApiKey.ts │ │ │ ├── App.css │ │ │ ├── App.ts │ │ │ ├── AppHeader.ts │ │ │ ├── AppUI.ts │ │ │ ├── AuditLogStreamingConfig.ts │ │ │ ├── AuditLogsPage.ts │ │ │ ├── AuthenticationSection.ts │ │ │ ├── BottomBar.ts │ │ │ ├── CardContextMenu.ts │ │ │ ├── CellContextMenu.ts │ │ │ ├── ChangeAdminModal.ts │ │ │ ├── CodeHighlight.ts │ │ │ ├── ColumnFilterCalendarView.ts │ │ │ ├── ColumnFilterMenu.ts │ │ │ ├── ColumnFilterMenuUtils.ts │ │ │ ├── ColumnTitle.ts │ │ │ ├── ConfigsAPI.ts │ │ │ ├── CoreHomeImports.ts │ │ │ ├── CoreNewDocMethods.ts │ │ │ ├── CreateTeamModal.ts │ │ │ ├── CustomSectionConfig.ts │ │ │ ├── CustomThemes.ts │ │ │ ├── CustomWidgetGallery.ts │ │ │ ├── DateRangeOptions.ts │ │ │ ├── DefaultActivationPage.ts │ │ │ ├── DescriptionConfig.ts │ │ │ ├── DocHistory.ts │ │ │ ├── DocIcon.ts │ │ │ ├── DocList.ts │ │ │ ├── DocMenu.ts │ │ │ ├── DocMenuCss.ts │ │ │ ├── DocTour.ts │ │ │ ├── DocTutorial.css │ │ │ ├── DocTutorial.ts │ │ │ ├── DocTutorialRenderer.ts │ │ │ ├── DocumentSettings.ts │ │ │ ├── DuplicateTable.ts │ │ │ ├── EmojiPicker.ts │ │ │ ├── ExampleCard.ts │ │ │ ├── ExampleInfo.ts │ │ │ ├── Experiments.ts │ │ │ ├── FieldConfig.ts │ │ │ ├── FieldContextMenu.ts │ │ │ ├── FieldMenus.ts │ │ │ ├── FileDialog.ts │ │ │ ├── FilterBar.ts │ │ │ ├── FilterConfig.ts │ │ │ ├── FloatingPopup.ts │ │ │ ├── FormAPI.ts │ │ │ ├── FormContainer.ts │ │ │ ├── FormErrorPage.ts │ │ │ ├── FormPage.ts │ │ │ ├── FormSuccessPage.ts │ │ │ ├── GetGristComProvider.ts │ │ │ ├── GridOptions.ts │ │ │ ├── GridViewMenus.ts │ │ │ ├── GridViewMenusDateHelpers.ts │ │ │ ├── GristTooltips.ts │ │ │ ├── HomeIntro.ts │ │ │ ├── HomeIntroCards.ts │ │ │ ├── HomeLeftPane.ts │ │ │ ├── IAssistantPopup.ts │ │ │ ├── ImportProgress.ts │ │ │ ├── LanguageMenu.ts │ │ │ ├── LeftPanelCommon.ts │ │ │ ├── LinkConfig.ts │ │ │ ├── LoginPagesCss.ts │ │ │ ├── MakeCopyMenu.ts │ │ │ ├── MarkdownCellRenderer.ts │ │ │ ├── MenuToggle.ts │ │ │ ├── MultiSelector.ts │ │ │ ├── NewRecordButton.ts │ │ │ ├── NotifyUI.ts │ │ │ ├── OnBoardingPopups.ts │ │ │ ├── OnboardingPage.ts │ │ │ ├── OpenAccessibilityModal.ts │ │ │ ├── OpenUserManager.ts │ │ │ ├── OpenVideoTour.ts │ │ │ ├── PagePanels.ts │ │ │ ├── PageWidgetPicker.ts │ │ │ ├── Pages.ts │ │ │ ├── PinnedDocs.ts │ │ │ ├── PredefinedCustomSectionConfig.ts │ │ │ ├── ProposedChangesPage.ts │ │ │ ├── RelativeDatesOptions.ts │ │ │ ├── RenameDocModal.ts │ │ │ ├── RenamePopupStyles.ts │ │ │ ├── RightPanel.ts │ │ │ ├── RightPanelStyles.ts │ │ │ ├── RightPanelUtils.ts │ │ │ ├── RowContextMenu.ts │ │ │ ├── RowHeightConfig.ts │ │ │ ├── ShareMenu.ts │ │ │ ├── ShortcutKey.ts │ │ │ ├── SiteSwitcher.ts │ │ │ ├── SortConfig.ts │ │ │ ├── SortFilterConfig.ts │ │ │ ├── SupportGristButton.ts │ │ │ ├── SupportGristPage.ts │ │ │ ├── TemplateDocs.ts │ │ │ ├── ThemeConfig.ts │ │ │ ├── TimingPage.ts │ │ │ ├── ToggleEnterpriseWidget.ts │ │ │ ├── Tools.ts │ │ │ ├── TopBar.ts │ │ │ ├── TopBarCss.ts │ │ │ ├── TreeViewComponent.ts │ │ │ ├── TreeViewComponentCss.ts │ │ │ ├── TriggerFormulas.ts │ │ │ ├── UserImage.ts │ │ │ ├── UserItem.ts │ │ │ ├── UserManager.ts │ │ │ ├── ViewLayoutMenu.ts │ │ │ ├── ViewSectionMenu.ts │ │ │ ├── VisibleFieldsConfig.ts │ │ │ ├── WebhookPage.ts │ │ │ ├── WelcomeCoachingCall.ts │ │ │ ├── WelcomePage.ts │ │ │ ├── WelcomeSitePicker.ts │ │ │ ├── WelcomeTour.ts │ │ │ ├── WidgetTitle.ts │ │ │ ├── YouTubePlayer.ts │ │ │ ├── buildReassignModal.ts │ │ │ ├── buttons.ts │ │ │ ├── contextMenu.ts │ │ │ ├── createAppPage.ts │ │ │ ├── createPage.ts │ │ │ ├── cssInput.ts │ │ │ ├── errorPages.ts │ │ │ ├── forms.ts │ │ │ ├── googleAuth.ts │ │ │ ├── inputs.ts │ │ │ ├── mouseDrag.ts │ │ │ ├── resizeHandle.ts │ │ │ ├── sanitizeHTML.ts │ │ │ ├── searchDropdown.ts │ │ │ ├── selectBy.ts │ │ │ ├── sendToDrive.ts │ │ │ ├── shadowScroll.ts │ │ │ ├── tooltips.ts │ │ │ ├── transientInput.ts │ │ │ ├── transitions.ts │ │ │ ├── userTrustsCustomWidget.ts │ │ │ ├── viewport.ts │ │ │ └── widgetTypesMap.ts │ │ ├── ui2018/ │ │ │ ├── ColorPalette.ts │ │ │ ├── ColorSelect.ts │ │ │ ├── IconList.ts │ │ │ ├── alerts.ts │ │ │ ├── ariaTabs.ts │ │ │ ├── breadcrumbs.ts │ │ │ ├── buttonSelect.ts │ │ │ ├── buttons.ts │ │ │ ├── checkbox.ts │ │ │ ├── cssVars.ts │ │ │ ├── draggableList.ts │ │ │ ├── editableLabel.ts │ │ │ ├── icons.ts │ │ │ ├── links.ts │ │ │ ├── loaders.ts │ │ │ ├── menus.ts │ │ │ ├── modals.ts │ │ │ ├── pages.ts │ │ │ ├── popups.ts │ │ │ ├── radio.ts │ │ │ ├── search.ts │ │ │ ├── select.ts │ │ │ ├── stretchedLink.ts │ │ │ ├── tabs.ts │ │ │ ├── theme.ts │ │ │ ├── toggleSwitch.ts │ │ │ ├── unstyled.ts │ │ │ └── visuallyHidden.ts │ │ └── widgets/ │ │ ├── AbstractWidget.js │ │ ├── Assistant.ts │ │ ├── AttachmentsEditor.ts │ │ ├── AttachmentsWidget.ts │ │ ├── BaseEditor.js │ │ ├── CellStyle.ts │ │ ├── CheckBox.css │ │ ├── CheckBoxEditor.js │ │ ├── ChoiceEditor.js │ │ ├── ChoiceListCell.ts │ │ ├── ChoiceListEditor.ts │ │ ├── ChoiceListEntry.ts │ │ ├── ChoiceTextBox.ts │ │ ├── ChoiceToken.ts │ │ ├── ConditionalStyle.ts │ │ ├── CurrencyPicker.ts │ │ ├── DateEditor.ts │ │ ├── DateTextBox.js │ │ ├── DateTimeEditor.css │ │ ├── DateTimeEditor.ts │ │ ├── DateTimeTextBox.js │ │ ├── DiffBox.ts │ │ ├── DiscussionEditor.ts │ │ ├── EditorButtons.ts │ │ ├── EditorPlacement.ts │ │ ├── EditorTooltip.ts │ │ ├── ErrorDom.ts │ │ ├── FieldBuilder.css │ │ ├── FieldBuilder.ts │ │ ├── FieldEditor.ts │ │ ├── FloatingEditor.ts │ │ ├── FormulaAssistant.ts │ │ ├── FormulaEditor.ts │ │ ├── HyperLinkEditor.ts │ │ ├── HyperLinkTextBox.ts │ │ ├── MarkdownTextBox.ts │ │ ├── MentionTextBox.ts │ │ ├── NTextBox.ts │ │ ├── NTextEditor.ts │ │ ├── NewAbstractWidget.ts │ │ ├── NewBaseEditor.ts │ │ ├── NumericEditor.ts │ │ ├── NumericSpinner.ts │ │ ├── NumericTextBox.ts │ │ ├── Reference.css │ │ ├── Reference.ts │ │ ├── ReferenceEditor.ts │ │ ├── ReferenceList.ts │ │ ├── ReferenceListEditor.ts │ │ ├── ReverseReferenceConfig.ts │ │ ├── Spinner.css │ │ ├── Spinner.ts │ │ ├── TZAutocomplete.ts │ │ ├── TextBox.css │ │ ├── TextEditor.css │ │ ├── TextEditor.js │ │ ├── Toggle.ts │ │ ├── UserType.ts │ │ └── UserTypeImpl.ts │ ├── common/ │ │ ├── ACLPermissions.ts │ │ ├── ACLRuleCollection.ts │ │ ├── ACLRulesReader.ts │ │ ├── ActionBundle.ts │ │ ├── ActionDispatcher.ts │ │ ├── ActionGroup.ts │ │ ├── ActionRouter.ts │ │ ├── ActionSummarizer.ts │ │ ├── ActionSummary.ts │ │ ├── ActivationAPI.ts │ │ ├── ActiveDocAPI.ts │ │ ├── AlternateActions.ts │ │ ├── ApiError.ts │ │ ├── Assistance.ts │ │ ├── Assistant.ts │ │ ├── AsyncCreate.ts │ │ ├── AsyncFlow.ts │ │ ├── AttachmentColumns.ts │ │ ├── BaseAPI.ts │ │ ├── BasketClientAPI.ts │ │ ├── BigInt.ts │ │ ├── BillingAPI.ts │ │ ├── BinaryIndexedTree.js │ │ ├── BootProbe.ts │ │ ├── BrowserSettings.ts │ │ ├── CircularArray.js │ │ ├── ColumnFilterFunc.ts │ │ ├── ColumnGetters.ts │ │ ├── CommTypes.ts │ │ ├── Config-ti.ts │ │ ├── Config.ts │ │ ├── ConfigAPI.ts │ │ ├── CssCustomProp.ts │ │ ├── CustomWidget.ts │ │ ├── DisposableWithEvents.ts │ │ ├── DocActions.ts │ │ ├── DocComments.ts │ │ ├── DocData.ts │ │ ├── DocDataCache.ts │ │ ├── DocLimits.ts │ │ ├── DocListAPI.ts │ │ ├── DocSchemaImport.ts │ │ ├── DocSchemaImportTypes-ti.ts │ │ ├── DocSchemaImportTypes.ts │ │ ├── DocSnapshot.ts │ │ ├── DocState.ts │ │ ├── DocUsage.ts │ │ ├── DocumentSettings-ti.ts │ │ ├── DocumentSettings.ts │ │ ├── DropdownCondition.ts │ │ ├── EncActionBundle.ts │ │ ├── ErrorWithCode.ts │ │ ├── Features-ti.ts │ │ ├── Features.ts │ │ ├── FilterState.ts │ │ ├── Forms.ts │ │ ├── Formula.ts │ │ ├── GranularAccessClause.ts │ │ ├── GristServerAPI.ts │ │ ├── ICommonUrls-ti.ts │ │ ├── ICommonUrls.ts │ │ ├── InactivityTimer.ts │ │ ├── Install.ts │ │ ├── InstallAPI.ts │ │ ├── Interval.ts │ │ ├── KeyedMutex.ts │ │ ├── KeyedOps.ts │ │ ├── Limits.ts │ │ ├── LinkNode.ts │ │ ├── LocaleCodes.ts │ │ ├── Locales.ts │ │ ├── LoginSessionAPI.ts │ │ ├── MemBuffer.js │ │ ├── NumberFormat.ts │ │ ├── NumberParse.ts │ │ ├── PluginInstance.ts │ │ ├── PredicateFormula.ts │ │ ├── Prefs.ts │ │ ├── RecentItems.js │ │ ├── RecordView.ts │ │ ├── RefCountMap.ts │ │ ├── RelativeDates.ts │ │ ├── RowFilterFunc.ts │ │ ├── SandboxInfo.ts │ │ ├── ServiceAccountTypes-ti.ts │ │ ├── ServiceAccountTypes.ts │ │ ├── ShareAnnotator.ts │ │ ├── ShareOptions.ts │ │ ├── SortFunc.ts │ │ ├── SortSpec.ts │ │ ├── StringUnion.ts │ │ ├── TableData.ts │ │ ├── TabularDiff.ts │ │ ├── Telemetry.ts │ │ ├── TestState.ts │ │ ├── ThemePrefs.ts │ │ ├── Themes.ts │ │ ├── TimeQuery.ts │ │ ├── Triggers-ti.ts │ │ ├── Triggers.ts │ │ ├── User.ts │ │ ├── UserAPI.ts │ │ ├── UserConfig.ts │ │ ├── ValueConverter.ts │ │ ├── ValueFormatter.ts │ │ ├── ValueGuesser.ts │ │ ├── ValueParser.ts │ │ ├── WidgetOptions.ts │ │ ├── airtable/ │ │ │ ├── AirtableAPI.ts │ │ │ ├── AirtableAPITypes-ti.ts │ │ │ ├── AirtableAPITypes.ts │ │ │ ├── AirtableAttachmentTracker.ts │ │ │ ├── AirtableCrosswalk.ts │ │ │ ├── AirtableDataImporter.ts │ │ │ ├── AirtableDataImporterTypes.ts │ │ │ ├── AirtableReferenceTracker.ts │ │ │ └── AirtableSchemaImporter.ts │ │ ├── arrayToString.ts │ │ ├── asyncIterators.ts │ │ ├── csvFormat.ts │ │ ├── declarations.d.ts │ │ ├── delay.ts │ │ ├── emails.ts │ │ ├── getCurrentTime.ts │ │ ├── gristTypes.ts │ │ ├── gristUrls.ts │ │ ├── gutil.ts │ │ ├── isHiddenTable.ts │ │ ├── loginProviders.ts │ │ ├── marshal.ts │ │ ├── normalizedDateTimeString.ts │ │ ├── orgNameUtils.ts │ │ ├── parseDate.ts │ │ ├── plugin.ts │ │ ├── resetOrg.ts │ │ ├── roles.ts │ │ ├── schema.ts │ │ ├── tagManager.ts │ │ ├── tbind.ts │ │ ├── themes/ │ │ │ ├── Base.ts │ │ │ ├── GristDark.ts │ │ │ ├── GristLight.ts │ │ │ └── HighContrastLight.ts │ │ ├── timeFormat.ts │ │ ├── tpromisified.ts │ │ ├── tsconfig.json │ │ ├── tsvFormat.ts │ │ ├── uploads.ts │ │ ├── urlUtils.ts │ │ └── widgetTypes.ts │ ├── gen-server/ │ │ ├── ApiServer.ts │ │ ├── entity/ │ │ │ ├── AclRule.ts │ │ │ ├── Activation.ts │ │ │ ├── Alias.ts │ │ │ ├── BillingAccount.ts │ │ │ ├── BillingAccountManager.ts │ │ │ ├── Config.ts │ │ │ ├── DocPref.ts │ │ │ ├── Document.ts │ │ │ ├── Group.ts │ │ │ ├── Limit.ts │ │ │ ├── Login.ts │ │ │ ├── OAuthClient.ts │ │ │ ├── OAuthGrant.ts │ │ │ ├── Organization.ts │ │ │ ├── Pref.ts │ │ │ ├── Product.ts │ │ │ ├── Proposal.ts │ │ │ ├── Resource.ts │ │ │ ├── Secret.ts │ │ │ ├── ServiceAccount.ts │ │ │ ├── Share.ts │ │ │ ├── User.ts │ │ │ └── Workspace.ts │ │ ├── lib/ │ │ │ ├── ActivationsManager.ts │ │ │ ├── DocApiForwarder.ts │ │ │ ├── DocWorkerMap.ts │ │ │ ├── Doom.ts │ │ │ ├── Housekeeper.ts │ │ │ ├── NotifierTypes.ts │ │ │ ├── Permissions.ts │ │ │ ├── TypeORMPatches.ts │ │ │ ├── Usage.ts │ │ │ ├── homedb/ │ │ │ │ ├── Caches.ts │ │ │ │ ├── GroupsManager.ts │ │ │ │ ├── HomeDBManager.ts │ │ │ │ ├── Interfaces.ts │ │ │ │ ├── ServiceAccountsManager.ts │ │ │ │ └── UsersManager.ts │ │ │ ├── scrubUserFromOrg.ts │ │ │ └── values.ts │ │ ├── migration/ │ │ │ ├── 1536634251710-Initial.ts │ │ │ ├── 1539031763952-Login.ts │ │ │ ├── 1549313797109-PinDocs.ts │ │ │ ├── 1549381727494-UserPicture.ts │ │ │ ├── 1551805156919-LoginDisplayEmail.ts │ │ │ ├── 1552416614755-LoginDisplayEmailNonNull.ts │ │ │ ├── 1553016106336-Indexes.ts │ │ │ ├── 1556726945436-Billing.ts │ │ │ ├── 1557157922339-OrgDomainUnique.ts │ │ │ ├── 1561589211752-Aliases.ts │ │ │ ├── 1568238234987-TeamMembers.ts │ │ │ ├── 1569593726320-FirstLogin.ts │ │ │ ├── 1569946508569-FirstTimeUser.ts │ │ │ ├── 1573569442552-CustomerIndex.ts │ │ │ ├── 1579559983067-ExtraIndexes.ts │ │ │ ├── 1591755411755-OrgHost.ts │ │ │ ├── 1592261300044-DocRemovedAt.ts │ │ │ ├── 1596456522124-Prefs.ts │ │ │ ├── 1623871765992-ExternalBilling.ts │ │ │ ├── 1626369037484-DocOptions.ts │ │ │ ├── 1631286208009-Secret.ts │ │ │ ├── 1644363380225-UserOptions.ts │ │ │ ├── 1647883793388-GracePeriodStart.ts │ │ │ ├── 1651469582887-DocumentUsage.ts │ │ │ ├── 1652273656610-Activations.ts │ │ │ ├── 1652277549983-UserConnectId.ts │ │ │ ├── 1663851423064-UserUUID.ts │ │ │ ├── 1664528376930-UserRefUnique.ts │ │ │ ├── 1673051005072-Forks.ts │ │ │ ├── 1678737195050-ForkIndexes.ts │ │ │ ├── 1682636695021-ActivationPrefs.ts │ │ │ ├── 1685343047786-AssistantLimit.ts │ │ │ ├── 1701557445716-Shares.ts │ │ │ ├── 1711557445716-Billing.ts │ │ │ ├── 1713186031023-UserLastConnection.ts │ │ │ ├── 1722529827161-Activation-Enabled.ts │ │ │ ├── 1727747249153-Configs.ts │ │ │ ├── 1729754662550-LoginsEmailIndex.ts │ │ │ ├── 1732103776245-GracePeriod.ts │ │ │ ├── 1738912357827-UserCreatedAt.ts │ │ │ ├── 1746246433628-DocPref.ts │ │ │ ├── 1749454162428-GroupUsersCreatedAt.ts │ │ │ ├── 1753088213255-GroupTypes.ts │ │ │ ├── 1754077317821-UserDisabledAt.ts │ │ │ ├── 1756799894986-UserUnsubscribeKey.ts │ │ │ ├── 1756918816559-ServiceAccounts.ts │ │ │ ├── 1759256005608-Proposals.ts │ │ │ ├── 1759434763338-DocDisabledAt.ts │ │ │ ├── 1764872085347-OAuthClientsAndGrants.ts │ │ │ └── README.md │ │ └── sqlUtils.ts │ ├── plugin/ │ │ ├── CustomSectionAPI-ti.ts │ │ ├── CustomSectionAPI.ts │ │ ├── DocApiTypes-ti.ts │ │ ├── DocApiTypes.ts │ │ ├── FileParserAPI-ti.ts │ │ ├── FileParserAPI.ts │ │ ├── GristAPI-ti.ts │ │ ├── GristAPI.ts │ │ ├── GristData-ti.ts │ │ ├── GristData.ts │ │ ├── GristTable-ti.ts │ │ ├── GristTable.ts │ │ ├── ImportSourceAPI-ti.ts │ │ ├── ImportSourceAPI.ts │ │ ├── InternalImportSourceAPI-ti.ts │ │ ├── InternalImportSourceAPI.ts │ │ ├── PluginManifest-ti.ts │ │ ├── PluginManifest.ts │ │ ├── README.md │ │ ├── RenderOptions-ti.ts │ │ ├── RenderOptions.ts │ │ ├── StorageAPI-ti.ts │ │ ├── StorageAPI.ts │ │ ├── TableOperations.ts │ │ ├── TableOperationsImpl.ts │ │ ├── TypeCheckers.ts │ │ ├── WidgetAPI-ti.ts │ │ ├── WidgetAPI.ts │ │ ├── grist-plugin-api.ts │ │ ├── gutil.ts │ │ ├── objtypes.ts │ │ └── tsconfig.json │ ├── server/ │ │ ├── MergedServer.ts │ │ ├── companion.ts │ │ ├── declarations.d.ts │ │ ├── devServerMain.ts │ │ ├── generateCheckpoint.ts │ │ ├── generateInitialDocSql.ts │ │ ├── lib/ │ │ │ ├── AccessTokens.ts │ │ │ ├── ActionHistory.ts │ │ │ ├── ActionHistoryImpl.ts │ │ │ ├── ActiveDoc.ts │ │ │ ├── ActiveDocImport.ts │ │ │ ├── ActiveDocUtils.ts │ │ │ ├── AppEndpoint.ts │ │ │ ├── AppSettings.ts │ │ │ ├── Archive.ts │ │ │ ├── Assistant.ts │ │ │ ├── AssistantStatePermit.ts │ │ │ ├── AttachmentFileManager.ts │ │ │ ├── AttachmentStore.ts │ │ │ ├── AttachmentStoreProvider.ts │ │ │ ├── AuditEvent.ts │ │ │ ├── AuthSession.ts │ │ │ ├── Authorizer.ts │ │ │ ├── BootProbes.ts │ │ │ ├── BrowserSession.ts │ │ │ ├── CellDataAccess.ts │ │ │ ├── Client.ts │ │ │ ├── Comm.ts │ │ │ ├── ConfigBackendAPI.ts │ │ │ ├── DiscourseConnect.ts │ │ │ ├── DocApi.ts │ │ │ ├── DocApiTriggers.ts │ │ │ ├── DocApiUtils.ts │ │ │ ├── DocAuthorizer.ts │ │ │ ├── DocClients.ts │ │ │ ├── DocManager.ts │ │ │ ├── DocPluginData.ts │ │ │ ├── DocPluginManager.ts │ │ │ ├── DocSession.ts │ │ │ ├── DocSnapshots.ts │ │ │ ├── DocStorage.ts │ │ │ ├── DocStorageManager.ts │ │ │ ├── DocWorker.ts │ │ │ ├── DocWorkerLoadTracker.ts │ │ │ ├── DocWorkerMap.ts │ │ │ ├── DocWorkerUtils.ts │ │ │ ├── ExcelFormatter.ts │ │ │ ├── ExpandedQuery.ts │ │ │ ├── Export.ts │ │ │ ├── ExportDSV.ts │ │ │ ├── ExportTableSchema.ts │ │ │ ├── ExportXLSX.ts │ │ │ ├── ExternalStorage.ts │ │ │ ├── FileParserElement.ts │ │ │ ├── FlexServer.ts │ │ │ ├── ForwardAuthLogin.ts │ │ │ ├── GetGristComConfig.ts │ │ │ ├── GoogleAuth.ts │ │ │ ├── GoogleExport.ts │ │ │ ├── GoogleImport.ts │ │ │ ├── GranularAccess.ts │ │ │ ├── GristJobs.ts │ │ │ ├── GristServer.ts │ │ │ ├── GristServerSocket.ts │ │ │ ├── GristSocketServer.ts │ │ │ ├── HashUtil.ts │ │ │ ├── HostedMetadataManager.ts │ │ │ ├── HostedStorageManager.ts │ │ │ ├── IAssistant.ts │ │ │ ├── IAuditLogger.ts │ │ │ ├── IBilling.ts │ │ │ ├── IChecksumStore.ts │ │ │ ├── ICreate.ts │ │ │ ├── IDocNotificationManager.ts │ │ │ ├── IDocStorageManager.ts │ │ │ ├── IElectionStore.ts │ │ │ ├── INotifier.ts │ │ │ ├── ISandbox.ts │ │ │ ├── IShell.ts │ │ │ ├── ITestingHooks-ti.ts │ │ │ ├── ITestingHooks.ts │ │ │ ├── InsightLog.ts │ │ │ ├── InstallAdmin.ts │ │ │ ├── LogMethods.ts │ │ │ ├── LoginSystemConfig.ts │ │ │ ├── MemoryPool.ts │ │ │ ├── MinIOExternalStorage.ts │ │ │ ├── MinimalLogin.ts │ │ │ ├── NSandbox.ts │ │ │ ├── NullSandbox.ts │ │ │ ├── OAuth2Clients.ts │ │ │ ├── OIDCConfig.ts │ │ │ ├── OnDemandActions.ts │ │ │ ├── OpenAIAssistantV1.ts │ │ │ ├── Patch.ts │ │ │ ├── PermissionInfo.ts │ │ │ ├── Permit.ts │ │ │ ├── PluginEndpoint.ts │ │ │ ├── PluginManager.ts │ │ │ ├── ProcessMonitor.ts │ │ │ ├── ProxyAgent.ts │ │ │ ├── PubSubCache.ts │ │ │ ├── PubSubManager.ts │ │ │ ├── Requests.ts │ │ │ ├── RowAccess.ts │ │ │ ├── SQLiteDB.ts │ │ │ ├── SafePythonComponent.ts │ │ │ ├── SamlConfig.ts │ │ │ ├── SandboxControl.ts │ │ │ ├── SandboxPyodide.ts │ │ │ ├── ServerColumnGetters.ts │ │ │ ├── ServerLocale.ts │ │ │ ├── Sessions.ts │ │ │ ├── Sharing.ts │ │ │ ├── SqliteCommon.ts │ │ │ ├── SqliteNode.ts │ │ │ ├── TableMetadataLoader.ts │ │ │ ├── TagChecker.ts │ │ │ ├── Telemetry.ts │ │ │ ├── TestLogin.ts │ │ │ ├── TestingHooks.ts │ │ │ ├── Throttle.ts │ │ │ ├── TimeQuery.ts │ │ │ ├── Triggers.ts │ │ │ ├── UnsafeNodeComponent.ts │ │ │ ├── UpdateManager.ts │ │ │ ├── UserPresence.ts │ │ │ ├── WebhookQueue.ts │ │ │ ├── WidgetRepository.ts │ │ │ ├── attachEarlyEndpoints.ts │ │ │ ├── backupSqliteDatabase.ts │ │ │ ├── checksumFile.ts │ │ │ ├── config.ts │ │ │ ├── configCore.ts │ │ │ ├── configCoreFileFormats-ti.ts │ │ │ ├── configCoreFileFormats.ts │ │ │ ├── configureMinIOExternalStorage.ts │ │ │ ├── configureOpenAIAssistantV1.ts │ │ │ ├── cookieUtils.ts │ │ │ ├── coreCreator.ts │ │ │ ├── coreLogins.ts │ │ │ ├── createSavedDoc.ts │ │ │ ├── dbUtils.ts │ │ │ ├── describeDocActions.ts │ │ │ ├── docUtils.d.ts │ │ │ ├── docUtils.js │ │ │ ├── expressWrap.ts │ │ │ ├── extractOrg.ts │ │ │ ├── filterUtils.ts │ │ │ ├── gristSessions.ts │ │ │ ├── gristSettings.ts │ │ │ ├── guessExt.ts │ │ │ ├── hashingUtils.ts │ │ │ ├── httpEncoding.ts │ │ │ ├── idUtils.ts │ │ │ ├── initialDocSql.ts │ │ │ ├── log.ts │ │ │ ├── loginSystemHelpers.ts │ │ │ ├── manifest.ts │ │ │ ├── middleware.ts │ │ │ ├── oidc/ │ │ │ │ └── Protections.ts │ │ │ ├── places.ts │ │ │ ├── reportTimeTaken.ts │ │ │ ├── requestUtils.ts │ │ │ ├── runSQLQuery.ts │ │ │ ├── sandboxUtil.ts │ │ │ ├── scim/ │ │ │ │ ├── index.ts │ │ │ │ └── v2/ │ │ │ │ ├── BaseController.ts │ │ │ │ ├── ScimGroupController.ts │ │ │ │ ├── ScimRoleController.ts │ │ │ │ ├── ScimTypes.ts │ │ │ │ ├── ScimUserController.ts │ │ │ │ ├── ScimUtils.ts │ │ │ │ ├── ScimV2Api.ts │ │ │ │ └── roles/ │ │ │ │ ├── SCIMMYRoleResource.ts │ │ │ │ └── SCIMMYRoleSchema.ts │ │ │ ├── selectBy.ts │ │ │ ├── sendAppPage.ts │ │ │ ├── serverUtils.ts │ │ │ ├── sessionUtils.ts │ │ │ ├── shortDesc.ts │ │ │ ├── shutdown.js │ │ │ ├── updateChecker.ts │ │ │ ├── uploads.ts │ │ │ └── workerExporter.ts │ │ ├── localization.ts │ │ ├── tsconfig.json │ │ └── utils/ │ │ ├── LogSanitizer.ts │ │ ├── gristify.ts │ │ ├── pruneActionHistory.ts │ │ ├── showAuditLogEvents.ts │ │ └── streams.ts │ └── tsconfig.json ├── buildtools/ │ ├── .grist-ee-version │ ├── build.sh │ ├── checkout-ext-directory.sh │ ├── fly-deploy.js │ ├── fly-template.env │ ├── fly-template.toml │ ├── genIconCSS.ts │ ├── generate_locale_list.js │ ├── generate_translation_keys.js │ ├── install_chrome_for_tests.sh │ ├── prepare_ee.sh │ ├── prepare_python.sh │ ├── sanitize_translations.js │ ├── tsconfig-base-ext.json │ ├── tsconfig-base.json │ ├── update_schema.sh │ ├── update_type_info.sh │ ├── webpack.api.config.js │ ├── webpack.check.js │ └── webpack.config.js ├── crowdin.yml ├── docker-compose-examples/ │ ├── grist-local-testing/ │ │ ├── README.md │ │ └── docker-compose.yml │ ├── grist-traefik-basic-auth/ │ │ ├── README.md │ │ ├── configs/ │ │ │ ├── traefik-config.yml │ │ │ └── traefik-dynamic-config.yml │ │ └── docker-compose.yml │ ├── grist-traefik-oidc-auth/ │ │ ├── README.md │ │ ├── configs/ │ │ │ ├── authelia/ │ │ │ │ ├── configuration.yml │ │ │ │ └── users_database.yml │ │ │ └── traefik/ │ │ │ └── config.yml │ │ ├── docker-compose.yml │ │ ├── env-template │ │ ├── generateSecureSecrets.sh │ │ └── secrets_template/ │ │ ├── GRIST_CLIENT_SECRET_DIGEST │ │ ├── HMAC_SECRET │ │ ├── JWT_SECRET │ │ ├── SESSION_SECRET │ │ ├── STORAGE_ENCRYPTION_KEY │ │ └── certs/ │ │ └── private.pem │ ├── grist-with-keycloak-postgres-redis-minio/ │ │ ├── README.md │ │ └── docker-compose.yml │ └── grist-with-postgres-redis-minio/ │ ├── README.md │ └── docker-compose.yml ├── documentation/ │ ├── database.md │ ├── develop.md │ ├── disposal.md │ ├── grainjs.md │ ├── grist-data-format.md │ ├── images/ │ │ └── BDD.drawio │ ├── migrations.md │ ├── overview.md │ ├── translations.md │ └── urls.md ├── eslint.config.js ├── package.json ├── plugins/ │ └── core/ │ └── manifest.yml ├── publiccode.yml ├── sandbox/ │ ├── MANIFEST.in │ ├── bundle_as_wheel.sh │ ├── docker/ │ │ ├── Dockerfile │ │ └── Makefile │ ├── docker_entrypoint.sh │ ├── gen_js_schema.py │ ├── grist/ │ │ ├── acl.py │ │ ├── action_obj.py │ │ ├── action_summary.py │ │ ├── actions.py │ │ ├── attribute_recorder.py │ │ ├── autocomplete_context.py │ │ ├── codebuilder.py │ │ ├── column.py │ │ ├── csv_patch.py │ │ ├── depend.py │ │ ├── docactions.py │ │ ├── docmodel.py │ │ ├── dropdown_condition.py │ │ ├── engine.py │ │ ├── fake_std_streams.py │ │ ├── formula_prompt.py │ │ ├── friendly_errors.py │ │ ├── functions/ │ │ │ ├── __init__.py │ │ │ ├── date.py │ │ │ ├── info.py │ │ │ ├── logical.py │ │ │ ├── lookup.py │ │ │ ├── math.py │ │ │ ├── prevnext.py │ │ │ ├── schedule.py │ │ │ ├── stats.py │ │ │ ├── test_schedule.py │ │ │ ├── text.py │ │ │ └── unimplemented.py │ │ ├── gencode.py │ │ ├── grist.py │ │ ├── identifiers.py │ │ ├── import_actions.py │ │ ├── imports/ │ │ │ ├── __init__.py │ │ │ ├── fixtures/ │ │ │ │ ├── nyc_schools_progress_report_ec_2013.xlsx │ │ │ │ ├── strange_dates.xlsx │ │ │ │ ├── test_boolean.xlsx │ │ │ │ ├── test_empty_rows.xlsx │ │ │ │ ├── test_encoding_utf8.csv │ │ │ │ ├── test_excel.xlsx │ │ │ │ ├── test_excel_numeric_gs.xlsx │ │ │ │ ├── test_excel_types.csv │ │ │ │ ├── test_excel_types.xlsx │ │ │ │ ├── test_falsy_cells.xlsx │ │ │ │ ├── test_headers_with_none_cell.xlsx │ │ │ │ ├── test_import_csv.csv │ │ │ │ ├── test_invalid_dimensions.xlsx │ │ │ │ ├── test_isdigit.csv │ │ │ │ ├── test_long_cell.csv │ │ │ │ └── test_single_merged_cell.xlsx │ │ │ ├── import_csv.py │ │ │ ├── import_csv_test.py │ │ │ ├── import_json.py │ │ │ ├── import_json_test.py │ │ │ ├── import_utils.py │ │ │ ├── import_xls.py │ │ │ ├── import_xls_test.py │ │ │ ├── register.py │ │ │ └── test_imports.py │ │ ├── lookup.py │ │ ├── main.py │ │ ├── match_counter.py │ │ ├── migrations.py │ │ ├── moment.py │ │ ├── objtypes.py │ │ ├── parse_data.py │ │ ├── predicate_formula.py │ │ ├── records.py │ │ ├── relabeling.py │ │ ├── relation.py │ │ ├── reverse_references.py │ │ ├── runtests.py │ │ ├── sandbox.py │ │ ├── schema.py │ │ ├── sort_key.py │ │ ├── sort_specs.py │ │ ├── summary.py │ │ ├── table.py │ │ ├── table_data_set.py │ │ ├── test_acl_formula.py │ │ ├── test_acl_renames.py │ │ ├── test_actions.py │ │ ├── test_codebuilder.py │ │ ├── test_column_actions.py │ │ ├── test_completion.py │ │ ├── test_date_types.py │ │ ├── test_default_formulas.py │ │ ├── test_depend.py │ │ ├── test_derived.py │ │ ├── test_display_cols.py │ │ ├── test_docmodel.py │ │ ├── test_dropdown_condition.py │ │ ├── test_dropdown_condition_renames.py │ │ ├── test_engine.py │ │ ├── test_find_col.py │ │ ├── test_formula_error.py │ │ ├── test_formula_prompt.py │ │ ├── test_formula_undo.py │ │ ├── test_functions.py │ │ ├── test_gencode.py │ │ ├── test_import_actions.py │ │ ├── test_lookup_find.py │ │ ├── test_lookup_perf.py │ │ ├── test_lookup_sort.py │ │ ├── test_lookups.py │ │ ├── test_match_counter.py │ │ ├── test_migrations.py │ │ ├── test_moment.py │ │ ├── test_objtypes.py │ │ ├── test_predicate_formula.py │ │ ├── test_prevnext.py │ │ ├── test_record_func.py │ │ ├── test_recordlist.py │ │ ├── test_reflist_rel.py │ │ ├── test_relabeling.py │ │ ├── test_renames.py │ │ ├── test_renames2.py │ │ ├── test_replace_table_data.py │ │ ├── test_replay.py │ │ ├── test_requests.py │ │ ├── test_rules.py │ │ ├── test_rules_grid.py │ │ ├── test_side_effects.py │ │ ├── test_sort_key.py │ │ ├── test_sort_spec.py │ │ ├── test_summary.py │ │ ├── test_summary2.py │ │ ├── test_summary_choicelist.py │ │ ├── test_summary_undo.py │ │ ├── test_table_actions.py │ │ ├── test_table_data_set.py │ │ ├── test_temp_rowids.py │ │ ├── test_textbuilder.py │ │ ├── test_treeview.py │ │ ├── test_trigger_expression.py │ │ ├── test_trigger_formulas.py │ │ ├── test_twoway_refs.py │ │ ├── test_twowaymap.py │ │ ├── test_types.py │ │ ├── test_undo.py │ │ ├── test_urllib_patch.py │ │ ├── test_user.py │ │ ├── test_useractions.py │ │ ├── testsamples.py │ │ ├── testscript.json │ │ ├── testutil.py │ │ ├── textbuilder.py │ │ ├── timing.py │ │ ├── treeview.py │ │ ├── trigger_expression.py │ │ ├── twowaymap.py │ │ ├── tzdata.data │ │ ├── urllib_patch.py │ │ ├── user.py │ │ ├── useractions.py │ │ ├── usercode.py │ │ ├── usertypes.py │ │ └── xmlrunner.py │ ├── gvisor/ │ │ ├── get_checkpoint_path.sh │ │ ├── run.py │ │ └── update_engine_checkpoint.sh │ ├── install_tz.js │ ├── pyodide/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── build_packages.sh │ │ ├── package.json │ │ ├── package_filenames.json │ │ ├── packages.js │ │ ├── pipe.js │ │ ├── preparePackages.js │ │ └── setup.sh │ ├── requirements.in │ ├── requirements.txt │ ├── run.sh │ ├── setup.py │ ├── supervisor.mjs │ └── watch.sh ├── static/ │ ├── apiconsole.html │ ├── app.html │ ├── custom-widget.html │ ├── custom.css │ ├── error.html │ ├── form.html │ ├── icons/ │ │ ├── grist.icns │ │ ├── gristdoc.icns │ │ ├── icons.css │ │ └── locales/ │ │ └── LICENSE │ ├── index.html │ ├── locales/ │ │ ├── ar.client.json │ │ ├── ar.server.json │ │ ├── bci.client.json │ │ ├── bci.server.json │ │ ├── bg.client.json │ │ ├── bg.server.json │ │ ├── ca.client.json │ │ ├── ca.server.json │ │ ├── cs.client.json │ │ ├── cs.server.json │ │ ├── de.client.json │ │ ├── de.server.json │ │ ├── el.client.json │ │ ├── el.server.json │ │ ├── en.client.json │ │ ├── en.server.json │ │ ├── en_GB.client.json │ │ ├── en_GB.server.json │ │ ├── es.client.json │ │ ├── es.server.json │ │ ├── eu.client.json │ │ ├── eu.server.json │ │ ├── fa.client.json │ │ ├── fa.server.json │ │ ├── fi.client.json │ │ ├── fi.server.json │ │ ├── fr.client.json │ │ ├── fr.server.json │ │ ├── hu.client.json │ │ ├── hu.server.json │ │ ├── id.client.json │ │ ├── id.server.json │ │ ├── ig.client.json │ │ ├── ig.server.json │ │ ├── it.client.json │ │ ├── it.server.json │ │ ├── ja.client.json │ │ ├── ja.server.json │ │ ├── ko.client.json │ │ ├── ko.server.json │ │ ├── nb_NO.client.json │ │ ├── nb_NO.server.json │ │ ├── nl.client.json │ │ ├── nl.server.json │ │ ├── pl.client.json │ │ ├── pl.server.json │ │ ├── pt.client.json │ │ ├── pt.server.json │ │ ├── pt_BR.client.json │ │ ├── pt_BR.server.json │ │ ├── ro.client.json │ │ ├── ro.server.json │ │ ├── ru.client.json │ │ ├── ru.server.json │ │ ├── sk.client.json │ │ ├── sk.server.json │ │ ├── sl.client.json │ │ ├── sl.server.json │ │ ├── sv.client.json │ │ ├── sv.server.json │ │ ├── ta.client.json │ │ ├── ta.server.json │ │ ├── th.client.json │ │ ├── th.server.json │ │ ├── tr.client.json │ │ ├── tr.server.json │ │ ├── uk.client.json │ │ ├── uk.server.json │ │ ├── ur.client.json │ │ ├── ur.server.json │ │ ├── vi.client.json │ │ ├── vi.server.json │ │ ├── zh_Hans.client.json │ │ ├── zh_Hans.server.json │ │ ├── zh_Hant.client.json │ │ ├── zh_Hant.server.json │ │ ├── zun.client.json │ │ └── zun.server.json │ ├── message.html │ ├── swagger-ui-dark.css │ ├── test.html │ └── testWebdriverJQuery.html ├── stubs/ │ └── app/ │ ├── client/ │ │ ├── components/ │ │ │ └── Banners.ts │ │ ├── ui/ │ │ │ ├── ActivationPage.ts │ │ │ ├── AdminControls.ts │ │ │ ├── BillingPage.ts │ │ │ ├── ChangePasswordDialog.ts │ │ │ ├── CustomThemes.ts │ │ │ ├── DeleteAccountDialog.ts │ │ │ ├── HomeImports.ts │ │ │ ├── MFAConfig.ts │ │ │ ├── NewDocMethods.ts │ │ │ ├── Notifications.ts │ │ │ └── ProductUpgrades.ts │ │ └── widgets/ │ │ └── AssistantPopup.ts │ ├── common/ │ │ └── version.ts │ ├── server/ │ │ ├── declarations.d.ts │ │ ├── lib/ │ │ │ ├── create.ts │ │ │ ├── globalConfig.ts │ │ │ └── loginSystems.ts │ │ ├── prometheus-exporter.ts │ │ └── server.ts │ └── tsconfig.json ├── test/ │ ├── assistant/ │ │ ├── data/ │ │ │ └── formula-dataset-index.csv │ │ └── v1/ │ │ ├── runCompletion.js │ │ └── runCompletion_impl.ts │ ├── chai-as-promised.js │ ├── client/ │ │ ├── clientUtil.js │ │ ├── components/ │ │ │ ├── Layout.js │ │ │ ├── WidgetFrame.ts │ │ │ ├── commands.js │ │ │ └── sampleLayout.js │ │ ├── lib/ │ │ │ ├── ACIndex.ts │ │ │ ├── Delay.js │ │ │ ├── DocSchemaImport.ts │ │ │ ├── ImportSourceElement.ts │ │ │ ├── ObservableMap.js │ │ │ ├── ObservableSet.js │ │ │ ├── PluginApi.ts │ │ │ ├── SafeBrowser.ts │ │ │ ├── Signal.ts │ │ │ ├── UrlState.ts │ │ │ ├── chartUtil.ts │ │ │ ├── dispose.js │ │ │ ├── dom.js │ │ │ ├── domAsync.ts │ │ │ ├── koArray.js │ │ │ ├── koArrayWrap.ts │ │ │ ├── koDom.js │ │ │ ├── koDomScrolly.js │ │ │ ├── koForm.js │ │ │ ├── koUtil.js │ │ │ ├── localStorageObs.ts │ │ │ ├── localization.ts │ │ │ ├── nameUtils.ts │ │ │ ├── sanitizeUrl.ts │ │ │ ├── sortUtil.ts │ │ │ ├── textUtils.ts │ │ │ ├── timeUtils.ts │ │ │ └── urlUtils.ts │ │ ├── models/ │ │ │ ├── ColumnFilter.ts │ │ │ ├── TreeModel.ts │ │ │ ├── gristUrlState.ts │ │ │ ├── modelUtil.js │ │ │ ├── rowset.js │ │ │ └── rowuid.js │ │ ├── shortcuts/ │ │ │ ├── excel.js │ │ │ └── gsMac.js │ │ ├── ui/ │ │ │ ├── DocumentSettings.ts │ │ │ ├── RelativeDatesOptions.ts │ │ │ └── UserImage.ts │ │ └── ui2018/ │ │ └── cssVars.ts │ ├── client-harness/ │ │ └── client.js │ ├── common/ │ │ ├── ACLPermissions.ts │ │ ├── AsyncCreate.ts │ │ ├── BigInt.ts │ │ ├── BinaryIndexedTree.js │ │ ├── ChoiceListParser.ts │ │ ├── CircularArray.js │ │ ├── ColumnFilterFunc.ts │ │ ├── DocActions.ts │ │ ├── DocSchemaImport.ts │ │ ├── InactivityTimer.ts │ │ ├── Interval.ts │ │ ├── KeyedMutex.ts │ │ ├── MemBuffer.js │ │ ├── NumberFormat.ts │ │ ├── NumberParse.ts │ │ ├── PluginInstance.ts │ │ ├── RecentItems.js │ │ ├── RefCountMap.ts │ │ ├── RelativeDates.ts │ │ ├── SortFunc.ts │ │ ├── StringUnion.ts │ │ ├── TableData.ts │ │ ├── Telemetry.ts │ │ ├── ThemePrefs.ts │ │ ├── ValueFormatter.ts │ │ ├── ValueGuesser.ts │ │ ├── airtable/ │ │ │ ├── AirtableAPI.ts │ │ │ ├── AirtableDataImporter.ts │ │ │ └── AirtableSchemaImporter.ts │ │ ├── arraySplice.js │ │ ├── csvFormat.ts │ │ ├── getTableTitle.ts │ │ ├── gristUrls.ts │ │ ├── gutil.js │ │ ├── gutil2.ts │ │ ├── marshal.js │ │ ├── parseDate.ts │ │ ├── promises.js │ │ ├── roles.ts │ │ ├── serializeTiming.js │ │ ├── sortTiming.js │ │ ├── timeFormat.js │ │ └── tsvFormat.ts │ ├── declarations.d.ts │ ├── deployment/ │ │ ├── ActionLog.ts │ │ ├── ChoiceList.ts │ │ ├── DuplicateDocument.ts │ │ ├── Fork.ts │ │ ├── HomeIntro.ts │ │ ├── Pages.ts │ │ ├── README.md │ │ ├── ReferenceColumns.ts │ │ ├── ReferenceList.ts │ │ └── Smoke.ts │ ├── fixtures/ │ │ ├── docs/ │ │ │ ├── ACL-Test.grist │ │ │ ├── ActiveDoc-sqlite.grist │ │ │ ├── AllColumns.grist │ │ │ ├── ApiDataRecordsTest.grist │ │ │ ├── AttachmentsJsonMigration.grist │ │ │ ├── BadRules.grist │ │ │ ├── BlobMigrationV1.grist │ │ │ ├── BlobMigrationV16.grist │ │ │ ├── BlobMigrationV17.grist │ │ │ ├── BlobMigrationV2.grist │ │ │ ├── BlobMigrationV3.grist │ │ │ ├── BlobMigrationV4.grist │ │ │ ├── BlobMigrationV5.grist │ │ │ ├── BlobMigrationV6.grist │ │ │ ├── BlobMigrationV7.grist │ │ │ ├── BlobMigrationV8.grist │ │ │ ├── BlobMigrationV9.grist │ │ │ ├── CCTransactions.grist │ │ │ ├── CC_Statement.grist │ │ │ ├── CC_Summaries-v2.grist │ │ │ ├── CC_Summaries-v6.grist │ │ │ ├── CC_Summaries.grist │ │ │ ├── CardView.grist │ │ │ ├── ChartData.grist │ │ │ ├── Class Enrollment.grist │ │ │ ├── Comments_44.grist │ │ │ ├── CopyOptions.grist │ │ │ ├── CopyPaste.grist │ │ │ ├── CopyPaste2.grist │ │ │ ├── Countries-Print.grist │ │ │ ├── Covid-19.grist │ │ │ ├── Currencies.grist │ │ │ ├── CursorWithRefLists1.grist │ │ │ ├── CustomWidget.grist │ │ │ ├── DefaultValuesV5.grist │ │ │ ├── DefaultValuesV6.grist │ │ │ ├── DefaultValuesV7.grist │ │ │ ├── DefaultValuesV8.grist │ │ │ ├── DefaultValuesV9.grist │ │ │ ├── DeleteColumnsUndo.grist │ │ │ ├── DownmigrateTest.grist │ │ │ ├── DropdownCondition.grist │ │ │ ├── Excel.grist │ │ │ ├── ExemptFromFilterBug.grist │ │ │ ├── Exports.grist │ │ │ ├── ExternalAttachmentsInvalidStoreId.grist │ │ │ ├── Favorite_Films.grist │ │ │ ├── Favorite_Films_Raw.grist │ │ │ ├── Favorite_Films_With_Linked_Ref.grist │ │ │ ├── FetchSelectedOptions.grist │ │ │ ├── FieldSettings.grist │ │ │ ├── FilmsWithImages.grist │ │ │ ├── FilterByComplexCellValues.grist │ │ │ ├── FilterLinkChain.grist │ │ │ ├── FilterTest.grist │ │ │ ├── Grist Basics.grist │ │ │ ├── GristNewUserInfo.grist │ │ │ ├── Hello.grist │ │ │ ├── Hooks-v37.grist │ │ │ ├── ImportReferences.grist │ │ │ ├── InvalidValues.grist │ │ │ ├── Investment Research (smaller).grist │ │ │ ├── Investment Research.grist │ │ │ ├── Landlord.grist │ │ │ ├── LastPosition.grist │ │ │ ├── LinkChain.grist │ │ │ ├── LongList.grist │ │ │ ├── ManyRefs.grist │ │ │ ├── Memos-v34.grist │ │ │ ├── NumericFormatting.grist │ │ │ ├── Pages-v19.grist │ │ │ ├── Pages.grist │ │ │ ├── PasteParsing.grist │ │ │ ├── RawSummaryTables.grist │ │ │ ├── Ref-AC-Test.grist │ │ │ ├── Ref-List-AC-Test.grist │ │ │ ├── RemoveTransformColumns.grist │ │ │ ├── SchoolsSample.grist │ │ │ ├── SelectByRefList.grist │ │ │ ├── SelectBySummary.grist │ │ │ ├── SelectBySummaryRef.grist │ │ │ ├── SelectionSummary.grist │ │ │ ├── ShiftSelection.grist │ │ │ ├── SortDates.grist │ │ │ ├── SortFilterIconTest.grist │ │ │ ├── SummarizeByRef.grist │ │ │ ├── SummaryRulesBug.grist │ │ │ ├── SummaryTableFormula.grist │ │ │ ├── TabBar.grist │ │ │ ├── Teams.grist │ │ │ ├── TypeConversions.grist │ │ │ ├── TypeEncoding.grist │ │ │ ├── Widgets.grist │ │ │ ├── World-v0.grist │ │ │ ├── World-v1.grist │ │ │ ├── World-v10.grist │ │ │ ├── World-v11.grist │ │ │ ├── World-v12.grist │ │ │ ├── World-v13.grist │ │ │ ├── World-v14.grist │ │ │ ├── World-v15.grist │ │ │ ├── World-v18.grist │ │ │ ├── World-v20.grist │ │ │ ├── World-v24.grist │ │ │ ├── World-v25.grist │ │ │ ├── World-v3.grist │ │ │ ├── World-v33.grist │ │ │ ├── World-v39.grist │ │ │ ├── World-v8.grist │ │ │ ├── World.grist │ │ │ ├── WorldSQLDB.grist │ │ │ ├── WorldUndo.grist │ │ │ ├── doctour.grist │ │ │ ├── selectBy.grist │ │ │ └── video/ │ │ │ ├── ACME Orders.grist │ │ │ ├── Afterschool Program.grist │ │ │ ├── Candidates.grist │ │ │ ├── Employees HomePage.grist │ │ │ ├── Employees.grist │ │ │ ├── Leases.grist │ │ │ └── Lightweight CRM.grist │ │ ├── export-csv/ │ │ │ ├── CCTransactions-DBA-desc.csv │ │ │ ├── CCTransactions.csv │ │ │ ├── choice.csv │ │ │ ├── date.csv │ │ │ ├── datetime.csv │ │ │ ├── field-options.csv │ │ │ ├── filtered-ref-list.csv │ │ │ ├── filters-manual.csv │ │ │ ├── filters-saved.csv │ │ │ ├── hidden-text.csv │ │ │ ├── integer.csv │ │ │ ├── many-rows.csv │ │ │ ├── numeric.csv │ │ │ ├── order-color-desc.csv │ │ │ ├── order-color-manual.csv │ │ │ ├── order-color-place.csv │ │ │ ├── order-manual.csv │ │ │ ├── reference.csv │ │ │ ├── text.csv │ │ │ └── toggle.csv │ │ ├── export-dsv/ │ │ │ ├── CCTransactions.dsv │ │ │ └── text.dsv │ │ ├── export-tsv/ │ │ │ ├── CCTransactions.tsv │ │ │ └── text.tsv │ │ ├── export-xlsx/ │ │ │ ├── CC_Statement.xlsx │ │ │ ├── CC_Summaries.xlsx │ │ │ ├── Currencies.xlsx │ │ │ ├── Excel.xlsx │ │ │ ├── Exports.xlsx │ │ │ └── World-v0.xlsx │ │ ├── plugins/ │ │ │ ├── .jshintrc │ │ │ ├── browserInstalledPlugins/ │ │ │ │ └── plugins/ │ │ │ │ ├── browser-GristDocAPI/ │ │ │ │ │ ├── main.js │ │ │ │ │ └── manifest.yml │ │ │ │ ├── custom-section/ │ │ │ │ │ ├── index-bis.html │ │ │ │ │ ├── index.html │ │ │ │ │ ├── main.js │ │ │ │ │ ├── manifest.yml │ │ │ │ │ ├── test-subscribe-api.html │ │ │ │ │ └── test-subscribe-api.js │ │ │ │ └── dummy-importer/ │ │ │ │ ├── index.html │ │ │ │ ├── main.js │ │ │ │ ├── manifest.yml │ │ │ │ ├── node/ │ │ │ │ │ └── main.js │ │ │ │ ├── sandbox/ │ │ │ │ │ └── main.py │ │ │ │ └── script.js │ │ │ ├── builtInPlugins/ │ │ │ │ └── plugins/ │ │ │ │ ├── 2/ │ │ │ │ │ └── manifest.yml │ │ │ │ ├── experimental-plugin/ │ │ │ │ │ ├── manifest.yml │ │ │ │ │ └── sandbox/ │ │ │ │ │ └── main.py │ │ │ │ ├── invalid-contrib-point/ │ │ │ │ │ └── manifest.yml │ │ │ │ ├── long-call/ │ │ │ │ │ ├── manifest.yml │ │ │ │ │ └── sandbox/ │ │ │ │ │ └── main.py │ │ │ │ ├── missing-component/ │ │ │ │ │ └── manifest.yml │ │ │ │ ├── missing-safePython/ │ │ │ │ │ └── manifest.yml │ │ │ │ ├── safePython-deactivate-fast/ │ │ │ │ │ ├── manifest.yml │ │ │ │ │ └── sandbox/ │ │ │ │ │ └── main.py │ │ │ │ ├── testing-function-call-plugin/ │ │ │ │ │ ├── backend.js │ │ │ │ │ ├── manifest.yml │ │ │ │ │ └── sandbox/ │ │ │ │ │ └── main.py │ │ │ │ ├── valid-file-parser/ │ │ │ │ │ ├── manifest.yml │ │ │ │ │ └── sandbox/ │ │ │ │ │ └── main.py │ │ │ │ ├── valid-import-source/ │ │ │ │ │ └── manifest.yml │ │ │ │ ├── wrong-json/ │ │ │ │ │ └── manifest.json │ │ │ │ └── wrong-yaml/ │ │ │ │ └── manifest.yml │ │ │ └── installedPlugins/ │ │ │ └── plugins/ │ │ │ ├── node-GristDocAPI/ │ │ │ │ ├── TestSubscribe.js │ │ │ │ ├── main.js │ │ │ │ └── manifest.yml │ │ │ ├── node-fail/ │ │ │ │ ├── main.js │ │ │ │ └── manifest.yml │ │ │ ├── node-mini-csv/ │ │ │ │ ├── manifest.yml │ │ │ │ └── nodebox/ │ │ │ │ └── main.js │ │ │ ├── node-wrong-message/ │ │ │ │ ├── main.js │ │ │ │ └── manifest.yml │ │ │ └── valid-import-source/ │ │ │ └── manifest.yml │ │ ├── projects/ │ │ │ ├── AddNewButton.ts │ │ │ ├── ApiKey.ts │ │ │ ├── ColorSelect.ts │ │ │ ├── ColumnFilterMenu.ts │ │ │ ├── DocMenu.ts │ │ │ ├── DocumentSettings.ts │ │ │ ├── ErrorNotify.ts │ │ │ ├── Icons.ts │ │ │ ├── Importer.ts │ │ │ ├── Mentions.ts │ │ │ ├── MultiSelector.ts │ │ │ ├── OnBoardingPopups.ts │ │ │ ├── PagePanels.ts │ │ │ ├── PageWidgetPicker.ts │ │ │ ├── PagesComponent.ts │ │ │ ├── ParseOptions.ts │ │ │ ├── ProgressIndicator.ts │ │ │ ├── Selects.ts │ │ │ ├── TreeViewComponent.ts │ │ │ ├── UI2018.ts │ │ │ ├── UserImage.ts │ │ │ ├── UserManager.ts │ │ │ ├── contextMenu.ts │ │ │ ├── editableLabel.ts │ │ │ ├── forms.ts │ │ │ ├── helpers/ │ │ │ │ ├── MockUserAPI.ts │ │ │ │ ├── Pages.ts │ │ │ │ ├── ParseOptionsData.ts │ │ │ │ ├── States.ts │ │ │ │ ├── gristStyles.ts │ │ │ │ ├── widgetPicker.ts │ │ │ │ └── withLocale.ts │ │ │ ├── modals.ts │ │ │ ├── mouseDrag.ts │ │ │ ├── resizeHandle.ts │ │ │ ├── searchDropdown.ts │ │ │ ├── sessionObs.ts │ │ │ ├── simpleList.ts │ │ │ ├── template.html │ │ │ ├── tokenfield.ts │ │ │ ├── tooltips.ts │ │ │ ├── transitions.ts │ │ │ ├── webpack-test-server.ts │ │ │ └── webpack.config.js │ │ ├── saml/ │ │ │ ├── keycloak.pem │ │ │ ├── saml-login │ │ │ ├── saml-logout │ │ │ ├── saml.crt │ │ │ └── saml.key │ │ ├── sites/ │ │ │ ├── config/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ ├── deferred-ready/ │ │ │ │ └── index.html │ │ │ ├── embed/ │ │ │ │ └── embed.html │ │ │ ├── fetchSelectedOptions/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ ├── filter/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ ├── hello/ │ │ │ │ └── index.html │ │ │ ├── paste/ │ │ │ │ └── paste.html │ │ │ ├── probe/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ ├── readout/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ ├── types/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ ├── types-raw-refs/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ ├── types-rest-api/ │ │ │ │ ├── index.html │ │ │ │ └── page.js │ │ │ └── zap/ │ │ │ ├── index.html │ │ │ └── page.js │ │ └── uploads/ │ │ ├── BooleanData.xlsx │ │ ├── CCTransactions.csv │ │ ├── ChartData-Sort_Test.csv │ │ ├── ChartData.csv │ │ ├── Cities.csv │ │ ├── CodeEditor.test.csv │ │ ├── ColumnFilterData_A.csv │ │ ├── ColumnFilterData_B.csv │ │ ├── DateTimeData.xlsx │ │ ├── EmptyDate.csv │ │ ├── FileUploadData.csv │ │ ├── ImportReferences-Tasks.csv │ │ ├── SchoolData.csv │ │ ├── StudentData.csv │ │ ├── UploadedData1.csv │ │ ├── UploadedData1Extended.csv │ │ ├── UploadedData2.csv │ │ ├── UploadedData2Extended.csv │ │ ├── UploadedData3.csv │ │ ├── UploadedDataEmpty.csv │ │ ├── World-v0.xlsx │ │ ├── World-v1.xlsx │ │ ├── cities.jgrist │ │ ├── cities_broken.jgrist │ │ ├── dirtyNames.json │ │ ├── empty_data.jgrist │ │ ├── empty_excel.xlsx │ │ ├── formatted_numbers.csv │ │ ├── homicide_rates.xlsx │ │ ├── htmlfile.html │ │ ├── mixed_dates.csv │ │ ├── name_references.csv │ │ ├── names.json │ │ ├── simple_array.json │ │ ├── spotifyGetSeveralAlbums.json │ │ ├── unicode_headers.csv │ │ ├── unicode_headers.xlsx │ │ └── video/ │ │ └── investment-data.xlsx │ ├── gen-server/ │ │ ├── ApiServer.ts │ │ ├── ApiServerAccess.ts │ │ ├── ApiServerBenchmark.ts │ │ ├── ApiServerBugs.ts │ │ ├── ApiSession.ts │ │ ├── AuthCaching.ts │ │ ├── SqliteSettings.ts │ │ ├── UpdateChecks.ts │ │ ├── apiUtils.ts │ │ ├── lib/ │ │ │ ├── DocApiForwarder.ts │ │ │ ├── DocPrefs.ts │ │ │ ├── DocWorkerMap.ts │ │ │ ├── HealthCheck.ts │ │ │ ├── HomeDBCaches.ts │ │ │ ├── HomeDBManager.ts │ │ │ ├── Housekeeper.ts │ │ │ ├── emails.ts │ │ │ ├── everyone.ts │ │ │ ├── homedb/ │ │ │ │ ├── GroupsManager.ts │ │ │ │ └── UsersManager.ts │ │ │ ├── limits.ts │ │ │ ├── listing.ts │ │ │ ├── mergedOrgs.ts │ │ │ ├── prefs.ts │ │ │ ├── previewer.ts │ │ │ ├── removedAt.ts │ │ │ ├── scrubUserFromOrg.ts │ │ │ ├── suspension.ts │ │ │ └── urlIds.ts │ │ ├── migrations.ts │ │ ├── seed.ts │ │ └── testUtils.ts │ ├── init-mocha-webdriver.js │ ├── nbrowser/ │ │ ├── AccessRules1.ts │ │ ├── AccessRules2.ts │ │ ├── AccessRules3.ts │ │ ├── AccessRules4.ts │ │ ├── AccessRulesAttrs.ts │ │ ├── AccessRulesIntro.ts │ │ ├── AccessRulesSchemaEdit.ts │ │ ├── AccessibilityModal.ts │ │ ├── ActionLog.ts │ │ ├── ActiveUserList.ts │ │ ├── AdminPanel.ts │ │ ├── AdminPanelTools.ts │ │ ├── AirtableImport.ts │ │ ├── ApiConsole.ts │ │ ├── AttachedCustomWidget.ts │ │ ├── AttachmentsLinking.ts │ │ ├── AttachmentsTransfer.ts │ │ ├── AttachmentsWidget.ts │ │ ├── AuthProvider.ts │ │ ├── AuthProviderGetGrist.ts │ │ ├── BehavioralPrompts.ts │ │ ├── Boot.ts │ │ ├── BundleActions.ts │ │ ├── CardView.ts │ │ ├── CellColor.ts │ │ ├── CellFormat.ts │ │ ├── ChartView1.ts │ │ ├── Choice.ts │ │ ├── ChoiceList.ts │ │ ├── ClientUnitTests.ntest.js │ │ ├── CodeEditor.ntest.js │ │ ├── ColumnFilterMenu.ts │ │ ├── ColumnFilterMenu2.ts │ │ ├── ColumnFilterMenu3.ts │ │ ├── ColumnOps.ntest.js │ │ ├── ColumnTransform.ts │ │ ├── Comments.ts │ │ ├── Comparison.ts │ │ ├── CopyPaste.ts │ │ ├── CopyPaste2.ntest.js │ │ ├── CopyPasteColumnOptions.ts │ │ ├── CopyPasteFiles.ts │ │ ├── CopyPasteLinked.ts │ │ ├── CopyWithHeaders.ts │ │ ├── CursorSaving.ts │ │ ├── CustomView.ts │ │ ├── CustomWidgets.ts │ │ ├── CustomWidgetsConfig.ts │ │ ├── DateEditor.ts │ │ ├── Dates.ntest.js │ │ ├── DeleteColumnsUndo.ts │ │ ├── DescriptionColumn.ts │ │ ├── DescriptionWidget.ts │ │ ├── DetailView.ntest.js │ │ ├── DetailView.ts │ │ ├── DocTour.ts │ │ ├── DocTutorial.ts │ │ ├── DocTypeConversion.ts │ │ ├── DocUsageTracking.ts │ │ ├── DropdownConditionEditor.ts │ │ ├── DuplicateDocument.ts │ │ ├── DuplicatePage.ts │ │ ├── Export.ntest.js │ │ ├── ExportSection.ts │ │ ├── Features.ts │ │ ├── FieldConfigTab.ntest.js │ │ ├── FieldEditor.ts │ │ ├── FieldSettings.ntest.js │ │ ├── FieldSettings2.ts │ │ ├── FillLinkedRecords.ntest.js │ │ ├── FillSelectionDown.ts │ │ ├── FilterLinkChain.ts │ │ ├── FilteringBugs.ts │ │ ├── Fork.ts │ │ ├── FormView1.ts │ │ ├── FormView2.ts │ │ ├── FormsUrlValues.ts │ │ ├── FormulaAutocomplete.ts │ │ ├── Formulas.ts │ │ ├── GridOptions.ntest.js │ │ ├── GridViewBugs.ts │ │ ├── GridViewNewColumnMenu.ts │ │ ├── GridViewNewColumnMenuDateHelpers.ts │ │ ├── GridViewNewColumnMenuUtils.ts │ │ ├── HeaderColor.ts │ │ ├── Health.ntest.js │ │ ├── HomeIntro.ts │ │ ├── HomeIntroWithoutPlaygound.ts │ │ ├── ImportReferences.ts │ │ ├── Importer.ts │ │ ├── Importer2.ts │ │ ├── LanguageSettings.ts │ │ ├── LazyLoad.ts │ │ ├── LeftPanel.ts │ │ ├── LinkingBidirectional.ts │ │ ├── LinkingErrors.ts │ │ ├── LinkingSelector.ts │ │ ├── Localization.ts │ │ ├── MultiColumn.ts │ │ ├── NewDocument.ntest.js │ │ ├── NumericEditor.ts │ │ ├── OnDemand.ts │ │ ├── Pages.ts │ │ ├── Printing.ts │ │ ├── Properties.ntest.js │ │ ├── ProposedChangesPage.ts │ │ ├── RawData.ts │ │ ├── RecordCards.ts │ │ ├── RecordLayout.ts │ │ ├── RefNumericChange.ts │ │ ├── RefTransforms.ts │ │ ├── ReferenceColumns.ts │ │ ├── ReferenceList.ts │ │ ├── RegionFocusSwitcher.ts │ │ ├── RemoveTransformColumns.ts │ │ ├── RightPanel.ts │ │ ├── RightPanelSelectBy.ts │ │ ├── RowHeights.ts │ │ ├── RowMenu.ts │ │ ├── SavePosition.ntest.js │ │ ├── Search.ts │ │ ├── Search2.ts │ │ ├── Search3.ts │ │ ├── SearchBar.ntest.ts │ │ ├── SectionFilter.ts │ │ ├── SelectBy.ts │ │ ├── SelectByRefList.ts │ │ ├── SelectByRightPanel.ts │ │ ├── SelectBySummary.ts │ │ ├── SelectBySummaryRef.ts │ │ ├── SelectionSummary.ts │ │ ├── ShiftSelection.ts │ │ ├── Smoke.ts │ │ ├── SortDates.ntest.js │ │ ├── SortEditSave.ntest.js │ │ ├── SortFilterSectionOptions.ts │ │ ├── SortPositions.ts │ │ ├── Summaries.ntest.js │ │ ├── SupportGrist.ts │ │ ├── TermsOfService.ts │ │ ├── TextEditor.ntest.js │ │ ├── Themes.ts │ │ ├── Timing.ts │ │ ├── ToggleColumns.ts │ │ ├── TokenField.ts │ │ ├── TwoWayReference.ts │ │ ├── TypeChange.ntest.js │ │ ├── UndoJumps.ntest.js │ │ ├── UploadLimits.ts │ │ ├── UserManager.ts │ │ ├── UserManager2.ts │ │ ├── VersionUpdateBanner.ts │ │ ├── ViewConfigTab.ntest.js │ │ ├── ViewLayout.ts │ │ ├── ViewLayoutCollapse.ts │ │ ├── ViewLayoutUtils.ts │ │ ├── Views.ntest.js │ │ ├── VisibleFieldsConfig.ts │ │ ├── WebhookOverflow.ts │ │ ├── WebhookPage.ts │ │ ├── aclTestUtils.ts │ │ ├── chartViewTestUtils.ts │ │ ├── customUtil.ts │ │ ├── disabledAt.ts │ │ ├── duplicateWidget.ts │ │ ├── elementUtils.ts │ │ ├── externalAttachmentsHelpers.ts │ │ ├── formTools.ts │ │ ├── gristUtil-nbrowser.js │ │ ├── gristUtils.ts │ │ ├── gristWebDriverUtils.ts │ │ ├── homeUtil.ts │ │ ├── importerTestUtils.ts │ │ ├── links.ts │ │ ├── saveViewSection.ts │ │ ├── testServer.ts │ │ ├── testUtils.ts │ │ ├── webdriverUtils.ts │ │ ├── webdriverjq-nbrowser.js │ │ └── webdriverjq.ntest.js │ ├── nbrowser_with_stubs/ │ │ ├── CreateTeamSite.ts │ │ └── CustomWidgets.ts │ ├── projects/ │ │ ├── AccountWidget.ts │ │ ├── ApiKey.ts │ │ ├── ColorSelect.ts │ │ ├── ColumnFilterMenu.ts │ │ ├── ColumnFilterMenu2.ts │ │ ├── DateRangeFilter.ts │ │ ├── DocMenu.ts │ │ ├── DocumentSettings.ts │ │ ├── Icons.ts │ │ ├── Mentions.ts │ │ ├── MultiSelector.ts │ │ ├── NotifyBar.ts │ │ ├── OnBoardingPopups.ts │ │ ├── PagePanels.ts │ │ ├── PageWidgetPicker.ts │ │ ├── PagesComponent.ts │ │ ├── RangeFilter.ts │ │ ├── TreeViewComponent.ts │ │ ├── UI2018.ts │ │ ├── UserManager.ts │ │ ├── contextMenu.ts │ │ ├── editableLabel.ts │ │ ├── errorPages.ts │ │ ├── filterUtils.ts │ │ ├── modals.ts │ │ ├── mouseDrag.ts │ │ ├── resizeHandle.ts │ │ ├── searchDropdown.ts │ │ ├── sessionObs.ts │ │ ├── simpleList.ts │ │ ├── testUtils.ts │ │ ├── tokenfield.ts │ │ ├── tooltips.ts │ │ └── transitions.ts │ ├── report-why-tests-hang.js │ ├── server/ │ │ ├── Comm.ts │ │ ├── PyMomentTest.ts │ │ ├── Sandbox.ts │ │ ├── customUtil.ts │ │ ├── docTools.ts │ │ ├── generateInitialDocSql.ts │ │ ├── gristClient.ts │ │ ├── lib/ │ │ │ ├── ACLFormula.ts │ │ │ ├── ACLRulesReader.ts │ │ │ ├── AccessTokens.ts │ │ │ ├── ActionHistory.ts │ │ │ ├── ActionHistoryMemory.ts │ │ │ ├── ActionSummary.ts │ │ │ ├── ActiveDoc.ts │ │ │ ├── ActiveDocImport.js │ │ │ ├── ActiveDocShutdown.ts │ │ │ ├── AppSettings.ts │ │ │ ├── Archive.ts │ │ │ ├── AttachmentFileManager.ts │ │ │ ├── AttachmentStoreProvider.ts │ │ │ ├── Authorizer.ts │ │ │ ├── BundleActions.ts │ │ │ ├── CommentAccess.ts │ │ │ ├── CommentAccess2.ts │ │ │ ├── ConfigBackendAPI.ts │ │ │ ├── DocSnapshots.ts │ │ │ ├── DocStorage.js │ │ │ ├── DocStorageManager.ts │ │ │ ├── DocStorageMigrations.ts │ │ │ ├── DocStorageQuery.ts │ │ │ ├── DocWorkerLoadTracker.ts │ │ │ ├── ExportsAccessRules.ts │ │ │ ├── ExternalStorageAttachmentStore.ts │ │ │ ├── FilesystemAttachmentStore.ts │ │ │ ├── GranularAccess.ts │ │ │ ├── GristJobs.ts │ │ │ ├── GristSockets.ts │ │ │ ├── HashUtil.ts │ │ │ ├── HostedMetadataManager.ts │ │ │ ├── HostedStorageManager.ts │ │ │ ├── ManyFetches.ts │ │ │ ├── MemoryPool.ts │ │ │ ├── MinIOExternalStorage.ts │ │ │ ├── OIDCConfig.ts │ │ │ ├── OnDemandActions.ts │ │ │ ├── OpenAIAssistantV1.ts │ │ │ ├── Proposals.ts │ │ │ ├── ProxyAgent.ts │ │ │ ├── PubSubCache.ts │ │ │ ├── PubSubManager.ts │ │ │ ├── RowAccess.ts │ │ │ ├── SQLiteDB.ts │ │ │ ├── SamlConfig.ts │ │ │ ├── Scim.ts │ │ │ ├── TableMetadataLoader.ts │ │ │ ├── Telemetry.ts │ │ │ ├── TestingHooks.ts │ │ │ ├── Throttle.ts │ │ │ ├── TimeQuery.ts │ │ │ ├── Triggers.ts │ │ │ ├── UnhandledErrors.ts │ │ │ ├── UserAttributes.ts │ │ │ ├── UserPresence.ts │ │ │ ├── Webhooks-Proxy.ts │ │ │ ├── checksumFile.ts │ │ │ ├── config.ts │ │ │ ├── configCore.ts │ │ │ ├── configCoreFileFormats.ts │ │ │ ├── docapi/ │ │ │ │ ├── DocApiAnonPlayground.ts │ │ │ │ ├── DocApiAttachments.ts │ │ │ │ ├── DocApiBugsAndFixes.ts │ │ │ │ ├── DocApiColumns.ts │ │ │ │ ├── DocApiCreation.ts │ │ │ │ ├── DocApiDocuments.ts │ │ │ │ ├── DocApiDownloads.ts │ │ │ │ ├── DocApiMisc.ts │ │ │ │ ├── DocApiOrgLimitFlags.ts │ │ │ │ ├── DocApiPermissions.ts │ │ │ │ ├── DocApiQueryParameters.ts │ │ │ │ ├── DocApiRecords.ts │ │ │ │ ├── DocApiReverseProxy.ts │ │ │ │ ├── DocApiSql.ts │ │ │ │ ├── DocApiTables.ts │ │ │ │ ├── DocApiWebhooks.ts │ │ │ │ └── helpers.ts │ │ │ ├── extractOrg.ts │ │ │ ├── helpers/ │ │ │ │ ├── PrepareDatabase.ts │ │ │ │ ├── PrepareFilesystemDirectoryForTests.ts │ │ │ │ ├── Signal.ts │ │ │ │ ├── TestProxyServer.ts │ │ │ │ └── TestServer.ts │ │ │ ├── idUtils.ts │ │ │ ├── requestUtils.ts │ │ │ ├── sandboxUtil.ts │ │ │ ├── serverUtils.js │ │ │ ├── serverUtils2.ts │ │ │ ├── shortDesc.js │ │ │ ├── updateChecker.ts │ │ │ └── uploads.ts │ │ ├── tcpForwarder.ts │ │ ├── testCleanup.ts │ │ ├── testUtils.ts │ │ ├── utils/ │ │ │ ├── CachedFetcher.ts │ │ │ ├── LogSanitizer.ts │ │ │ └── streams.ts │ │ └── wait.ts │ ├── setupPaths.js │ ├── split-tests.js │ ├── testUtils.ts │ ├── test_env.sh │ ├── test_under_docker.sh │ ├── timings/ │ │ ├── nbrowser.txt │ │ └── server.txt │ ├── tsconfig.json │ ├── upgradeDocument │ ├── upgradeDocumentImpl.ts │ ├── utils.js │ └── xunit-file.js ├── tsconfig-ext.json ├── tsconfig-prod.json ├── tsconfig.eslint.json └── tsconfig.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ # explicitly list the files needed by docker. * !package.json !yarn.lock !tsconfig-ext.json !tsconfig-prod.json !tsconfig.json !stubs !app !buildtools !static !bower_components !sandbox !plugins !test !ext **/_build ================================================ FILE: .editorconfig ================================================ # EditorConfig is awesome: https://EditorConfig.org # top-most EditorConfig file root = true # Unix-style newlines with a newline ending every file [*.{ts,js,py}] end_of_line = lf insert_final_newline = true charset = utf-8 # indent with 2 spaces indent_style = space indent_size = 2 trim_trailing_whitespace = true ================================================ FILE: .git-blame-ignore-revs ================================================ a7eb4d6e60c50375a835a5300b8e6beebcfb8422 3a859ee5454d9c2b9600c9d9a3bd859d65d496bf 9c9c45a6894e80748f0e337d4f164d40b503cae5 a3082e1f8b59b36154e721eb3d17c6504cf63bb8 29541707086d4de9fbe8773f42e0f5f0f845ea54 4faa64a1c0c194b32a804b2e562361ba197d8699 e44e6217bd223873aef2f1aa2df32dc9c17a3abf e75c8ce28d9470ba5e92ee25aec65869a7051668 b7e9ebed9dddd931a9bf33ae57dccb6423a7b6ae a893706c5b9659157878318513f6d6cbe025e51d 2f2cd7d60144486e67fc2fe0eaf20fcafcd50f21 ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: gristlabs ================================================ FILE: .github/ISSUE_TEMPLATE/00-bug-issue.yml ================================================ # Inspired by PeerTube templates: # https://github.com/Chocobozzz/PeerTube/blob/3d4d49a23eae71f3ce62cbbd7d93f07336a106b7/.github/ISSUE_TEMPLATE/00-bug-issue.yml name: 🐛 Bug Report description: Use this template for reporting a bug body: - type: markdown attributes: value: | Thanks for taking time to fill out this bug report! Please search among past open/closed issues for a similar one beforehand: - https://github.com/gristlabs/grist-core/issues?q= - https://community.getgrist.com/ - type: textarea attributes: label: Describe the current behavior - type: textarea attributes: label: Steps to reproduce value: | 1. 2. 3. - type: textarea attributes: label: Describe the expected behavior - type: checkboxes attributes: label: Where have you encountered this bug? options: - label: On [docs.getgrist.com](https://docs.getgrist.com) - label: On a self-hosted instance validations: required: true - type: textarea attributes: label: Instance information (when self-hosting only) description: In case you self-host, please share information above. You can discard any question you don't know the answer. value: | * Grist instance: * Version: * URL (if it's OK for you to share it): * Installation mode: docker/kubernetes/... * Architecture: single-worker/multi-workers * Browser name, version and platforms on which you could reproduce the bug: * Link to browser console log if relevant: * Link to server log if relevant: ================================================ FILE: .github/ISSUE_TEMPLATE/10-installation-issue.yml ================================================ # Inspired by PeerTube templates: # https://github.com/Chocobozzz/PeerTube/blob/master/.github/ISSUE_TEMPLATE/10-installation-issue.yml name: 🛠️ Installation/Upgrade Issue description: Use this template for installation/upgrade issues body: - type: markdown attributes: value: | Please check first the official documentation for self-hosting: https://support.getgrist.com/self-managed/ - type: markdown attributes: value: | Please search among past open/closed issues for a similar one beforehand: - https://github.com/gristlabs/grist-core/issues?q= - https://community.getgrist.com/ - type: textarea attributes: label: Describe the problem - type: textarea attributes: label: Additional information value: | * Grist version: * Grist instance URL: * SSO solution used and its version (if relevant): * S3 storage solution and its version (if relevant): * Docker version (if relevant): * NodeJS version (if relevant): * Redis version (if relevant): * PostgreSQL version (if relevant): ================================================ FILE: .github/ISSUE_TEMPLATE/20-feature-request.yml ================================================ # Inspired by PeerTube templates: # https://github.com/Chocobozzz/PeerTube/blob/master/.github/ISSUE_TEMPLATE/30-feature-request.yml --- name: ✨ Feature Request description: Use this template to ask for new features and suggest new ideas 💡 body: - type: markdown attributes: value: | Thanks for taking time to share your ideas! Please search among past open/closed issues for a similar one beforehand: - https://github.com/gristlabs/grist-core/issues?q= - https://community.getgrist.com/ - type: textarea attributes: label: Describe the problem to be solved description: Provide a clear and concise description of what the problem is - type: textarea attributes: label: Describe the solution you would like description: Provide a clear and concise description of what you want to happen ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: 🤷💻🤦 Question/Forum url: https://community.getgrist.com/ about: You can ask and answer other questions here - name: 💬 Discord url: https://discord.com/invite/MYKpYQ3fbP about: Chat with us via Discord for quick Q/A here and sharing tips ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ ## Context ## Proposed solution ## Related issues ## Has this been tested? - [ ] 👍 yes, I added tests to the test suite - [ ] 💭 no, because this PR is a draft and still needs work - [ ] 🙅 no, because this is not relevant here - [ ] 🙋 no, because I need help ## Screenshots / Screencasts ================================================ FILE: .github/cla/individual-cla.md ================================================ # Individual Contributor License Agreement ("Agreement"), v1.0 (Based on https://www.apache.org/licenses/icla.pdf by the Apache Foundation. Contact support@getgrist.com if you wish to execute a Corporate CLA.) Thank you for your interest in contributing to software projects made available by Grist Labs Inc ("Grist"). To clarify the intellectual property license granted with Contributions from any person or entity, Grist must have on file a signed Contributor License Agreement ("CLA") from each Contributor, indicating agreement with the license terms below. This agreement is for your protection as a Contributor as well as the protection of Grist and its users. It does not change your rights to use your own Contributions for any other purpose. Contributions entirely composed of commits with authorship at `*.gouv.fr` domains fall outside the scope of this agreement. You accept and agree to the following terms and conditions for Your Contributions (present and future) that you submit to Grist. Except for the license granted herein to Grist and recipients of software distributed by Grist, You reserve all right, title, and interest in and to Your Contributions. 1. Definitions. "You" (or "Your") shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Grist. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "Contribution" shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to Grist for inclusion in, or documentation of, any of the products owned or managed by Grist (the "Work"). For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to Grist or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Grist for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution." 2. Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to Grist and to recipients of software distributed by Grist a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works. 3. Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to Grist and to recipients of software distributed by Grist a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed. 4. You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Grist, or that your employer has executed a separate Corporate CLA with Grist. 5. You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions. 6. You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. 7. Should You wish to submit work that is not Your original creation, You may submit it to Grist separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]". 8. You agree to notify Grist of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect. ================================================ FILE: .github/cla/signatures.json ================================================ { "signedContributors": [ { "name": "jordigh", "id": 260143, "comment_id": 3053400623, "created_at": "2025-07-09T17:12:17Z", "repoId": 266033395, "pullRequestNo": 1682 }, { "name": "georgegevoian", "id": 85144792, "comment_id": 3070684406, "created_at": "2025-07-14T19:16:24Z", "repoId": 266033395, "pullRequestNo": 1705 }, { "name": "scytacki", "id": 86016, "comment_id": 3071744517, "created_at": "2025-07-15T03:10:33Z", "repoId": 266033395, "pullRequestNo": 1661 }, { "name": "fflorent", "id": 371705, "comment_id": 3079056343, "created_at": "2025-07-16T15:06:22Z", "repoId": 266033395, "pullRequestNo": 1683 }, { "name": "manuhabitela", "id": 221253, "comment_id": 3083417362, "created_at": "2025-07-17T09:55:14Z", "repoId": 266033395, "pullRequestNo": 1431 }, { "name": "hexaltation", "id": 31125573, "comment_id": 3083469227, "created_at": "2025-07-17T10:11:19Z", "repoId": 266033395, "pullRequestNo": 1699 }, { "name": "paulfitz", "id": 118367, "comment_id": 3084758085, "created_at": "2025-07-17T16:53:25Z", "repoId": 266033395, "pullRequestNo": 1711 }, { "name": "Spoffy", "id": 4805393, "comment_id": 3090253280, "created_at": "2025-07-18T18:01:45Z", "repoId": 266033395, "pullRequestNo": 1670 }, { "name": "Anany-k", "id": 13112955, "comment_id": 3095980233, "created_at": "2025-07-21T09:55:52Z", "repoId": 266033395, "pullRequestNo": 1716 }, { "name": "vviers", "id": 30295971, "comment_id": 3097269575, "created_at": "2025-07-21T15:34:37Z", "repoId": 266033395, "pullRequestNo": 1719 }, { "name": "guillett", "id": 1410356, "comment_id": 3113331623, "created_at": "2025-07-24T12:43:18Z", "repoId": 266033395, "pullRequestNo": 1691 }, { "name": "ogui11aume", "id": 31072389, "comment_id": 3118009823, "created_at": "2025-07-25T14:21:07Z", "repoId": 266033395, "pullRequestNo": 1653 }, { "name": "mrdev023", "id": 11292703, "comment_id": 3155373604, "created_at": "2025-08-05T14:02:58Z", "repoId": 266033395, "pullRequestNo": 1750 }, { "name": "jonathanperret", "id": 300823, "comment_id": 3233984142, "created_at": "2025-08-28T15:30:02Z", "repoId": 266033395, "pullRequestNo": 1778 }, { "name": "berhalak", "id": 11277225, "comment_id": 3249600375, "created_at": "2025-09-03T14:53:49Z", "repoId": 266033395, "pullRequestNo": 1807 }, { "name": "Ajay-Satish-01", "id": 71289526, "comment_id": 3260958924, "created_at": "2025-09-06T05:28:40Z", "repoId": 266033395, "pullRequestNo": 1818 }, { "name": "dsagal", "id": 1091143, "comment_id": 3293302321, "created_at": "2025-09-15T17:52:16Z", "repoId": 266033395, "pullRequestNo": 1841 }, { "name": "tristanrobert", "id": 19711088, "comment_id": 3297472676, "created_at": "2025-09-16T10:20:26Z", "repoId": 266033395, "pullRequestNo": 1840 }, { "name": "ohemelaar", "id": 19656762, "comment_id": 3326880813, "created_at": "2025-09-24T07:05:33Z", "repoId": 266033395, "pullRequestNo": 1830 }, { "name": "nbush", "id": 3422005, "comment_id": 3467956044, "created_at": "2025-10-30T13:18:24Z", "repoId": 266033395, "pullRequestNo": 1906 }, { "name": "SimLV", "id": 68837817, "comment_id": 3485625062, "created_at": "2025-11-04T11:57:57Z", "repoId": 266033395, "pullRequestNo": 1921 }, { "name": "SleepyLeslie", "id": 142967379, "comment_id": 3487547371, "created_at": "2025-11-04T18:46:35Z", "repoId": 266033395, "pullRequestNo": 1922 }, { "name": "samchencode", "id": 62081196, "comment_id": 3522061093, "created_at": "2025-11-12T13:53:43Z", "repoId": 266033395, "pullRequestNo": 1935 }, { "name": "RapidShade", "id": 20725534, "comment_id": 3533412593, "created_at": "2025-11-14T15:54:39Z", "repoId": 266033395, "pullRequestNo": 1945 }, { "name": "cfpwastaken", "id": 44261356, "comment_id": 3694780658, "created_at": "2025-12-28T14:18:03Z", "repoId": 266033395, "pullRequestNo": 2023 }, { "name": "unknownconstant", "id": 14999931, "comment_id": 3791801541, "created_at": "2026-01-23T18:54:49Z", "repoId": 266033395, "pullRequestNo": 2067 }, { "name": "mikhailbogdan-droid", "id": 238033187, "comment_id": 3823827850, "created_at": "2026-01-30T13:42:28Z", "repoId": 266033395, "pullRequestNo": 2085 }, { "name": "Vortezz", "id": 61989315, "comment_id": 3861607282, "created_at": "2026-02-06T17:08:18Z", "repoId": 266033395, "pullRequestNo": 2096 }, { "name": "kosssi", "id": 1135513, "comment_id": 4004185474, "created_at": "2026-03-05T10:52:21Z", "repoId": 266033395, "pullRequestNo": 2151 }, { "name": "webash", "id": 6205899, "comment_id": 4026041376, "created_at": "2026-03-09T18:57:39Z", "repoId": 266033395, "pullRequestNo": 2161 }, { "name": "Gwabix", "id": 50367170, "comment_id": 4060017206, "created_at": "2026-03-14T08:23:49Z", "repoId": 266033395, "pullRequestNo": 2178 } ] } ================================================ FILE: .github/workflows/cla.yml ================================================ # Workflow body from https://github.com/contributor-assistant/github-action name: "CLA Assistant" on: issue_comment: types: [created] pull_request_target: types: [opened,closed,synchronize] permissions: actions: write contents: write pull-requests: write statuses: write jobs: CLAAssistant: runs-on: ubuntu-latest steps: - name: "CLA Assistant" if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' uses: contributor-assistant/github-action@v2.6.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: path-to-signatures: '.github/cla/signatures.json' path-to-document: 'https://github.com/gristlabs/grist-core/blob/main/.github/cla/individual-cla.md' branch: 'main' allowlist: github-actions[bot],dependabot[bot] lock-pullrequest-aftermerge: false ================================================ FILE: .github/workflows/docker.yml ================================================ name: Push Docker image on: release: types: [published] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: inputs: tag: description: "Tag for the resulting images" type: string required: True default: 'stable' env: TAG: ${{ inputs.tag || 'stable' }} DOCKER_HUB_OWNER: ${{ vars.DOCKER_HUB_OWNER || github.repository_owner }} PLATFORMS: ${{ vars.PLATFORMS || 'linux/amd64,linux/arm64/v8' }} jobs: push_to_registry: name: Push Docker images to Docker Hub runs-on: ubuntu-22.04 strategy: matrix: image: # We build two images, `grist-oss` and `grist`. # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images - name: "grist-oss" repo: "grist-core" - name: "grist" repo: "grist-ee" steps: - name: Free some space run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /usr/local/.ghcup sudo rm -rf /usr/share/swift - name: Check out the repo uses: actions/checkout@v3 - name: Add a dummy ext/ directory run: mkdir ext && touch ext/dummy - name: Check out the ext/ directory if: matrix.image.name != 'grist-oss' run: buildtools/checkout-ext-directory.sh ${{ matrix.image.repo }} - name: Generate metadata tag input id: meta_input run: | { echo "tags<> $GITHUB_OUTPUT - name: Docker meta id: meta uses: docker/metadata-action@v4 with: images: | ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }} tags: | ${{ steps.meta_input.outputs.tags }} - name: Docker meta (EE) if: ${{ matrix.image.name == 'grist' }} id: meta_ee uses: docker/metadata-action@v4 with: images: | ${{ env.DOCKER_HUB_OWNER }}/grist-ee tags: | ${{ steps.meta_input.outputs.tags }} - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Log in to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Push to Docker Hub uses: docker/build-push-action@v2 with: context: . build-args: GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=${{ matrix.image.name == 'grist-oss' && 'false' || 'true' }} push: true platforms: ${{ env.PLATFORMS }} tags: ${{ steps.meta.outputs.tags }} cache-from: type=gha cache-to: type=gha,mode=max build-contexts: ext=ext - name: Push Enterprise to Docker Hub if: ${{ matrix.image.name == 'grist' }} uses: docker/build-push-action@v2 with: context: . build-args: | BASE_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name}} BASE_VERSION=${{ env.TAG }} GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=true file: ext/Dockerfile platforms: ${{ env.PLATFORMS }} push: true tags: ${{ steps.meta_ee.outputs.tags }} cache-from: type=gha cache-to: type=gha,mode=max ================================================ FILE: .github/workflows/docker_latest.yml ================================================ name: Push latest Docker image on: push: # Trigger if latest_candidate updates. This is automatically done by another # workflow whenever tests pass on main - but events don't chain without using # personal access tokens so we just use a cron job. branches: [ latest_candidate ] schedule: # Run at 5:41 UTC daily - cron: '41 5 * * *' workflow_dispatch: inputs: branch: description: "Branch from which to create the latest Docker image (default: latest_candidate)" type: string required: true default: latest_candidate disable_tests: description: "Should the tests be skipped?" type: boolean required: True default: False platforms: description: "Platforms to build" type: choice required: True options: - linux/amd64 - linux/arm64/v8 - linux/amd64,linux/arm64/v8 default: linux/amd64,linux/arm64/v8 tag: description: "Tag for the resulting images" type: string required: True default: 'latest' env: BRANCH: ${{ inputs.branch || 'latest_candidate' }} PLATFORMS: ${{ inputs.platforms || 'linux/amd64,linux/arm64/v8' }} TAG: ${{ inputs.tag || 'latest' }} DOCKER_HUB_OWNER: ${{ vars.DOCKER_HUB_OWNER || github.repository_owner }} jobs: push_to_registry: name: Push latest Docker image to Docker Hub runs-on: ubuntu-22.04 if: ${{ vars.RUN_DAILY_BUILD }} strategy: matrix: python-version: [3.11] node-version: [22.x] image: # We build two images, `grist-oss` and `grist`. # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images - name: "grist-oss" repo: "grist-core" - name: "grist" repo: "grist-ee" steps: - name: Build settings run: | echo "Branch: $BRANCH" echo "Platforms: $PLATFORMS" echo "Docker Hub Owner: $DOCKER_HUB_OWNER" echo "Tag: $TAG" - name: Free disk space run: | echo "Disk space before cleanup:" df -h / echo "Removing Android SDK..." sudo rm -rf /usr/local/lib/android df -h / echo "Removing .NET..." sudo rm -rf /usr/share/dotnet df -h / echo "Removing Haskell..." sudo rm -rf /usr/local/.ghcup df -h / echo "Removing Swift..." sudo rm -rf /usr/share/swift df -h / echo "Final disk space:" df -h / - name: Check out the repo uses: actions/checkout@v4 with: ref: ${{ env.BRANCH }} - name: Add a dummy ext/ directory run: mkdir ext && touch ext/dummy - name: Check out the ext/ directory if: matrix.image.name != 'grist-oss' run: buildtools/checkout-ext-directory.sh ${{ matrix.image.repo }} - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Prepare image but do not push it yet uses: docker/build-push-action@v2 with: context: . load: true tags: ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} cache-from: type=gha build-contexts: ext=ext - name: Use Node.js ${{ matrix.node-version }} for testing if: ${{ !inputs.disable_tests }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - name: Set up Python ${{ matrix.python-version }} for testing - maybe not needed if: ${{ !inputs.disable_tests }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install Python packages if: ${{ !inputs.disable_tests }} run: | pip install virtualenv yarn run install:python - name: Install Node.js packages if: ${{ !inputs.disable_tests }} run: yarn install - name: Disable the ext/ directory if: ${{ !inputs.disable_tests }} run: mv ext/ ext-disabled/ - name: Build Node.js code if: ${{ !inputs.disable_tests }} run: yarn run build - name: Install Google Chrome and chromedriver run: buildtools/install_chrome_for_tests.sh -y - name: Run tests with default settings if: ${{ !inputs.disable_tests }} run: | export TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} export VERBOSE=1 export DEBUG=1 export MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:docker - name: Run some tests with gvisor and python if: ${{ !inputs.disable_tests }} run: | export TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} export VERBOSE=1 export DEBUG=1 export MOCHA_WEBDRIVER_HEADLESS=1 export GREP_TESTS='should support basic editing' export TEST_DOCKER_OPTIONS='-e GRIST_SANDBOX_FLAVOR=gvisor -e PYTHON_VERSION_ON_CREATION=3' yarn run test:docker - name: Re-enable the ext/ directory if: ${{ !inputs.disable_tests }} run: mv ext-disabled/ ext/ - name: Log in to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Push to Docker Hub uses: docker/build-push-action@v2 with: context: . build-args: GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=${{ matrix.image.name == 'grist-oss' && 'false' || 'true' }} platforms: ${{ env.PLATFORMS }} push: true tags: ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} cache-from: type=gha cache-to: type=gha,mode=max build-contexts: ext=ext - name: Push Enterprise to Docker Hub if: ${{ matrix.image.name == 'grist' }} uses: docker/build-push-action@v2 with: context: . build-args: | BASE_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name}} BASE_VERSION=${{ env.TAG }} GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=true file: ext/Dockerfile platforms: ${{ env.PLATFORMS }} push: true tags: ${{ env.DOCKER_HUB_OWNER }}/grist-ee:${{ env.TAG }} cache-from: type=gha cache-to: type=gha,mode=max update_latest_branch: name: Update latest branch runs-on: ubuntu-22.04 needs: push_to_registry steps: - name: Check out the repo uses: actions/checkout@v2 with: ref: ${{ inputs.latest_branch }} - name: Update latest branch uses: ad-m/github-push-action@8407731efefc0d8f72af254c74276b7a90be36e1 with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: latest force: true ================================================ FILE: .github/workflows/fly-build.yml ================================================ # fly-deploy will be triggered on completion of this workflow to actually deploy the code to fly.io. name: fly.io Build on: pull_request: branches: [ main ] types: [labeled, opened, synchronize, reopened] # Allows running this workflow manually from the Actions tab workflow_dispatch: jobs: build: name: Build Docker image runs-on: ubuntu-22.04 # Build when the 'preview' label is added, or when PR is updated with this label present. if: > github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'preview')) steps: - uses: actions/checkout@v4 - name: Build and export Docker image id: docker-build run: > ./buildtools/checkout-ext-directory.sh grist-ee && docker build -t grist-core:preview . --build-context ext=ext && docker image save grist-core:preview -o grist-core.tar - name: Save PR information run: | echo PR_NUMBER=${{ github.event.number }} >> ./pr-info.txt echo PR_SOURCE=${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }} >> ./pr-info.txt echo PR_SHASUM=${{ github.event.pull_request.head.sha }} >> ./pr-info.txt # PR_SOURCE looks like /-. # For example, if the GitHub user "foo" forked grist-core as "grist-bar", and makes a PR from their branch named "baz", # it will be "foo/grist-bar-baz". deploy.js later replaces "/" with "-", making it "foo-grist-bar-baz". - name: Upload artifact uses: actions/upload-artifact@v4 with: name: docker-image path: | ./grist-core.tar ./pr-info.txt ./buildtools/fly-template.env if-no-files-found: "error" ================================================ FILE: .github/workflows/fly-cleanup.yml ================================================ name: fly.io Cleanup on: schedule: # Once a day, clean up jobs marked as expired - cron: '50 12 * * *' # Allows running this workflow manually from the Actions tab workflow_dispatch: env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} jobs: clean: name: Clean stale deployed apps runs-on: ubuntu-22.04 if: github.repository_owner == 'gristlabs' steps: - uses: actions/checkout@v3 - uses: superfly/flyctl-actions/setup-flyctl@master with: version: 0.2.72 - run: node buildtools/fly-deploy.js clean ================================================ FILE: .github/workflows/fly-deploy.yml ================================================ # Follow-up of fly-build, with access to secrets for making deployments. # This workflow runs in the target repo context. It does not, and should never execute user-supplied code. # See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ name: fly.io Deploy on: workflow_run: workflows: ["fly.io Build"] types: - completed jobs: deploy: name: Deploy app to fly.io runs-on: ubuntu-22.04 if: | github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' steps: - uses: actions/checkout@v4 - name: Set up flyctl uses: superfly/flyctl-actions/setup-flyctl@master with: version: 0.2.72 - name: Download artifacts uses: actions/github-script@v7 with: script: | var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, run_id: ${{ github.event.workflow_run.id }}, }); var matchArtifact = artifacts.data.artifacts.filter((artifact) => { return artifact.name == "docker-image" })[0]; var download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, artifact_id: matchArtifact.id, archive_format: 'zip', }); var fs = require('fs'); fs.writeFileSync('${{github.workspace}}/docker-image.zip', Buffer.from(download.data)); await github.rest.actions.deleteArtifact({ owner: context.repo.owner, repo: context.repo.repo, artifact_id: matchArtifact.id, }); - name: Extract artifacts id: extract_artifacts run: | unzip -o docker-image.zip grist-core.tar pr-info.txt buildtools/fly-template.env cat ./pr-info.txt >> $GITHUB_OUTPUT - name: Load Docker image run: docker load --input grist-core.tar - name: Deploy to fly.io id: fly_deploy env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} BRANCH_NAME: ${{ steps.extract_artifacts.outputs.PR_SOURCE }} run: | node buildtools/fly-deploy.js deploy flyctl config -c ./fly.toml env | awk '/APP_HOME_URL/{print "DEPLOY_URL=" $2}' >> $GITHUB_OUTPUT flyctl config -c ./fly.toml env | awk '/FLY_DEPLOY_EXPIRATION/{print "EXPIRES=" $2}' >> $GITHUB_OUTPUT - name: Comment on PR uses: actions/github-script@v7 with: script: | github.rest.issues.createComment({ issue_number: ${{ steps.extract_artifacts.outputs.PR_NUMBER }}, owner: context.repo.owner, repo: context.repo.repo, body: `Deployed commit \`${{ steps.extract_artifacts.outputs.PR_SHASUM }}\` as ${{ steps.fly_deploy.outputs.DEPLOY_URL }} (until ${{ steps.fly_deploy.outputs.EXPIRES }})` }) ================================================ FILE: .github/workflows/fly-destroy.yml ================================================ # This workflow runs in the target repo context, as it is triggered via pull_request_target. # It does not, and should not have access to code in the PR. # See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ name: fly.io Destroy on: pull_request_target: branches: [ main ] types: [unlabeled, closed] # Allows running this workflow manually from the Actions tab workflow_dispatch: jobs: destroy: name: Remove app from fly.io runs-on: ubuntu-22.04 # Remove the deployment when 'preview' label is removed, or the PR is closed. if: | github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && (github.event.action == 'closed' || (github.event.action == 'unlabeled' && github.event.label.name == 'preview'))) steps: - uses: actions/checkout@v4 - name: Set up flyctl uses: superfly/flyctl-actions/setup-flyctl@master with: version: 0.2.72 - name: Destroy fly.io app env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} BRANCH_NAME: ${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }} # See fly-build for what BRANCH_NAME looks like. id: fly_destroy run: node buildtools/fly-deploy.js destroy ================================================ FILE: .github/workflows/main.yml ================================================ name: CI on: push: branches: [ main ] pull_request: branches: [ main ] # Allows running this workflow manually from the Actions tab workflow_dispatch: jobs: build_and_test: runs-on: ${{ matrix.os }} strategy: # it is helpful to know which sets of tests would have succeeded, # even when there is a failure. fail-fast: false matrix: os: ['ubuntu-24.04'] python-version: [3.11] node-version: [22.x] tests: - ':lint:python:client:common:smoke:stubs:pyodide:' - ':server-1-of-2:' - ':server-2-of-2:' - ':gen-server:' - ':nbrowser-^[A-D]:' - ':nbrowser-^[E-L]:' - ':nbrowser-^[M-N]:' - ':nbrowser-^[O-R]:' - ':nbrowser-^[^A-R]:' - ':projects:' include: - tests: ':lint:python:client:common:smoke:' node-version: 22.x python-version: '3.10' os: ubuntu-24.04 - tests: ':pyodide:macsandbox:' node-version: 22.x python-version: '3.11' os: macos-latest steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} cache: 'yarn' - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install Python packages run: | pip install virtualenv yarn run install:python - name: Install Node.js packages run: yarn install - name: Install gvisor if: contains(matrix.os, 'ubuntu') run: | docker create --name temp-runsc gristlabs/gvisor-unprivileged:buster /bin/true sudo docker cp temp-runsc:/runsc /usr/bin/runsc docker rm temp-runsc - name: Run eslint if: contains(matrix.tests, ':lint:') run: yarn run lint:ci - name: Make sure bucket is versioned if: contains(matrix.os, 'ubuntu') && contains(matrix.tests, ':server-') || contains(matrix.os, 'ubuntu') && contains(matrix.tests, ':gen-server:') env: AWS_ACCESS_KEY_ID: administrator AWS_SECRET_ACCESS_KEY: administrator run: aws --region us-east-1 --endpoint-url http://localhost:9000 s3api put-bucket-versioning --bucket grist-docs-test --versioning-configuration Status=Enabled - name: Build Node.js code run: yarn run build - name: Install Google Chrome and chromedriver if: contains(matrix.tests, ':nbrowser-') || contains(matrix.tests, ':smoke:') || contains(matrix.tests, ':stubs:') || contains(matrix.tests, ':projects:') run: buildtools/install_chrome_for_tests.sh -y - name: Run smoke test if: contains(matrix.tests, ':smoke:') run: VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:smoke - name: Run python tests if: contains(matrix.tests, ':python:') run: yarn run test:python - name: Run client tests if: contains(matrix.tests, ':client:') run: yarn run test:client - name: Run common tests if: contains(matrix.tests, ':common:') run: yarn run test:common - name: Run stubs tests if: contains(matrix.tests, ':stubs:') run: MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:stubs - name: Run gen-server tests with sqlite, minio and redis if: contains(matrix.tests, ':gen-server:') run: | yarn run test:gen-server # Anchors should be used once available. Not supported yet as of December 2024. # https://github.com/actions/runner/issues/1182 env: MOCHA_WEBDRIVER_HEADLESS: 1 TESTS: ${{ matrix.tests }} GRIST_DOCS_MINIO_ACCESS_KEY: administrator GRIST_DOCS_MINIO_SECRET_KEY: administrator TEST_REDIS_URL: "redis://localhost/11" GRIST_DOCS_MINIO_USE_SSL: 0 GRIST_DOCS_MINIO_ENDPOINT: localhost GRIST_DOCS_MINIO_PORT: 9000 GRIST_DOCS_MINIO_BUCKET: grist-docs-test - name: Run a couple of tests using pyodide if: contains(matrix.tests, ':pyodide:') run: | cd sandbox/pyodide make setup cd ../.. yarn run test:server -g 'ActiveDoc.useQuerySet|Sandbox' yarn run test:nbrowser -g 'Importer.*should.show.correct.preview' env: MOCHA_WEBDRIVER_HEADLESS: 1 GRIST_SANDBOX_FLAVOR: pyodide - name: Run a couple of tests using macSandboxExec if: contains(matrix.tests, ':macsandbox:') run: | yarn run test:server -g Sandbox env: MOCHA_WEBDRIVER_HEADLESS: 1 GRIST_SANDBOX_FLAVOR: macSandboxExec - name: Run gen-server tests with postgres, minio and redis if: contains(matrix.tests, ':gen-server:') run: | PGPASSWORD=$TYPEORM_PASSWORD psql -h $TYPEORM_HOST -U $TYPEORM_USERNAME -w $TYPEORM_DATABASE -c "SHOW ALL;" | grep ' jit ' yarn run test:gen-server env: MOCHA_WEBDRIVER_HEADLESS: 1 TESTS: ${{ matrix.tests }} GRIST_DOCS_MINIO_ACCESS_KEY: administrator GRIST_DOCS_MINIO_SECRET_KEY: administrator TEST_REDIS_URL: "redis://localhost/11" GRIST_DOCS_MINIO_USE_SSL: 0 GRIST_DOCS_MINIO_ENDPOINT: localhost GRIST_DOCS_MINIO_PORT: 9000 GRIST_DOCS_MINIO_BUCKET: grist-docs-test TYPEORM_TYPE: postgres TYPEORM_HOST: localhost TYPEORM_DATABASE: db_name TYPEORM_USERNAME: db_user TYPEORM_PASSWORD: db_password - name: Run server tests with minio and redis if: contains(matrix.tests, ':server-') run: | export TEST_SPLITS=$(echo $TESTS | sed "s/.*:server-\([^:]*\).*/\1/") yarn run test:server env: MOCHA_WEBDRIVER_HEADLESS: 1 TESTS: ${{ matrix.tests }} GRIST_DOCS_MINIO_ACCESS_KEY: administrator GRIST_DOCS_MINIO_SECRET_KEY: administrator TEST_REDIS_URL: "redis://localhost/11" GVISOR_FLAGS: "-unprivileged -ignore-cgroups" GVISOR_EXTRA_DIRS: /opt GRIST_DOCS_MINIO_USE_SSL: 0 GRIST_DOCS_MINIO_ENDPOINT: localhost GRIST_DOCS_MINIO_PORT: 9000 GRIST_DOCS_MINIO_BUCKET: grist-docs-test - name: Run main tests without minio and redis if: contains(matrix.tests, ':nbrowser-') run: | mkdir -p $MOCHA_WEBDRIVER_LOGDIR export GREP_TESTS=$(echo $TESTS | sed "s/.*:nbrowser-\([^:]*\).*/\1/") MOCHA_WEBDRIVER_SKIP_CLEANUP=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:nbrowser --parallel --jobs 3 env: TESTS: ${{ matrix.tests }} MOCHA_WEBDRIVER_LOGDIR: ${{ runner.temp }}/test-logs/webdriver GVISOR_FLAGS: "-unprivileged -ignore-cgroups" GVISOR_EXTRA_DIRS: /opt TESTDIR: ${{ runner.temp }}/test-logs - name: Run projects tests if: contains(matrix.tests, ':projects:') run: | mkdir -p $MOCHA_WEBDRIVER_LOGDIR yarn run test:projects env: MOCHA_WEBDRIVER_LOGDIR: ${{ runner.temp }}/test-logs/webdriver TESTDIR: ${{ runner.temp }}/test-logs MOCHA_WEBDRIVER_HEADLESS: 1 - name: Prepare for saving artifact if: failure() run: | ARTIFACT_NAME=logs-$(echo $TESTS | sed 's/[^-a-zA-Z0-9]/_/g') echo "Artifact name is '$ARTIFACT_NAME'" echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV mkdir -p $TESTDIR find $TESTDIR -iname "*.socket" -exec rm {} \; env: TESTS: ${{ matrix.tests }} TESTDIR: ${{ runner.temp }}/test-logs - name: Save artifacts on failure if: failure() uses: actions/upload-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} path: ${{ runner.temp }}/test-logs # only exists for webdriver tests services: # https://github.com/bitnami/containers/issues/83267 minio: image: ${{ matrix.os == 'ubuntu-24.04' && 'bitnamilegacy/minio:2025.4.22' || '' }} env: MINIO_DEFAULT_BUCKETS: "grist-docs-test:public" MINIO_ROOT_USER: administrator MINIO_ROOT_PASSWORD: administrator ports: - 9000:9000 options: >- --health-cmd "curl -f http://localhost:9000/minio/health/ready" --health-interval 10s --health-timeout 5s --health-retries 5 redis: image: ${{ matrix.os == 'ubuntu-24.04' && 'redis' || '' }} ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 postgresql: image: ${{ matrix.os == 'ubuntu-24.04' && 'postgres:latest' || '' }} env: POSTGRES_USER: db_user POSTGRES_PASSWORD: db_password POSTGRES_DB: db_name # JIT is enabled by default since Postgres 17 and has a huge negative impact on performance, # making many tests timeout. # https://support.getgrist.com/self-managed/#what-is-a-home-database POSTGRES_INITDB_ARGS: "-c jit=off" ports: - 5432:5432 options: >- --health-cmd "pg_isready -U db_user" --health-interval 10s --health-timeout 5s --health-retries 5 candidate: needs: build_and_test if: ${{ success() && github.event_name == 'push' }} runs-on: ubuntu-22.04 steps: - name: Fetch new candidate branch uses: actions/checkout@v3 - name: Update candidate branch uses: ad-m/github-push-action@8407731efefc0d8f72af254c74276b7a90be36e1 with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: latest_candidate force: true ================================================ FILE: .github/workflows/self-hosted.yml ================================================ name: Add self-hosting issues to the self-hosting project on: issues: types: - opened - labeled jobs: add-to-project: name: Add issue to project runs-on: ubuntu-22.04 steps: - uses: actions/add-to-project@v1.0.1 with: project-url: https://github.com/orgs/gristlabs/projects/2 github-token: ${{ secrets.SELF_HOSTED_PROJECT }} labeled: self-hosting ================================================ FILE: .github/workflows/translation_keys.yml ================================================ name: Translation keys on: push: branches: [ main ] workflow_dispatch: permissions: pull-requests: write contents: write jobs: build: if: github.repository_owner == 'gristlabs' runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 with: fetch-depth: 0 # Let's get all the branches - name: Use Node.js uses: actions/setup-node@v1 with: node-version: 22 - name: Install Node.js packages run: yarn install - name: Build code run: yarn run build - name: Scan for keys id: scan-keys run: | git checkout -b translation-keys yarn run generate:translation 2>&1 | tee /tmp/scan-output.txt git status --porcelain if [[ $(git status --porcelain | wc -l) -eq "0" ]]; then echo "No changes" echo "CHANGED=false" >> $GITHUB_ENV else echo "Changes detected" echo "CHANGED=true" >> $GITHUB_ENV fi - name: setup git config run: | git config user.name "Paul's Grist Bot" git config user.email "" - name: Create PR and merge if: env.CHANGED == 'true' run: | git commit -m "automated update to translation keys" -a git push --set-upstream origin HEAD:translation-keys -f num=$(gh pr list --search "automated update to translation keys" --json number -q ".[].number") if [[ "$num" != "" ]]; then echo "Existing translation keys PR #$num is open, skipping" exit 0 fi sed -n '/TRANSLATION_SUMMARY_START/,/TRANSLATION_SUMMARY_END/{//d;p;}' /tmp/scan-output.txt > /tmp/pr-body.txt gh pr create --title "automated update to translation keys" --body-file /tmp/pr-body.txt gh pr merge --merge --delete-branch env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .gitignore ================================================ /node_modules/ /_build/ /static/*.bundle.js /static/*.bundle.js.map /static/grist-plugin-api* /static/bundle.css /static/browser-check.js /static/*.bundle.js.*.txt /grist-sessions.db /landing.db /docs/ /sandbox_venv* /.vscode/ # Files created by grist-desktop setup /cpython.tar.gz /python /static_ext # Build helper files. /.build* *.swp *.pyc *.bak .DS_Store # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage # nyc test coverage .nyc_output # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) .grunt # node-waf configuration .lock-wscript # Compiled binary addons (http://nodejs.org/api/addons.html) build/Release # Dependency directories /node_modules/ jspm_packages/ /docs # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variables file .env # Test timings.txt xunit.xml .clipboard.lock **/_build # ext directory can be overwritten /ext /ext/** # Docker compose examples - persistent values and secrets /docker-compose-examples/*/persist /docker-compose-examples/*/secrets /docker-compose-examples/grist-traefik-oidc-auth/.env # Sample grist documents /samples/ /test/assistant/data/cache/ /test/assistant/data/results/ /test/assistant/data/templates/ ================================================ FILE: .nvmrc ================================================ v22.12.0 ================================================ FILE: .yarnrc ================================================ # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. # yarn lockfile v1 yarn-offline-mirror false ================================================ FILE: CONTRIBUTING.md ================================================ # Welcome to the contribution guide for Grist! You are eager to contribute to Grist? That's awesome! See below some contributions you can make: - [translate](/documentation/translations.md) - [write tutorials and user documentation](https://github.com/gristlabs/grist-help?tab=readme-ov-file#grist-help-center) - [develop](/documentation/develop.md) - [report issues or suggest enhancement](https://github.com/gristlabs/grist-core/issues/new/choose) ================================================ FILE: Dockerfile ================================================ ################################################################################ ## The Grist source can be extended. This is a stub that can be overridden ## from command line, as: ## docker buildx build -t ... --build-context=ext= . ## The code in will then be built along with the rest of Grist. ################################################################################ FROM scratch AS ext ################################################################################ ## Javascript build stage ################################################################################ FROM node:22-trixie AS prod-builder # Install all node dependencies. WORKDIR /grist COPY package.json yarn.lock /grist/ RUN \ yarn install --prod --frozen-lockfile --verbose --network-timeout 600000 FROM prod-builder AS builder # Create node_modules with devDependencies to be able to build the app # Add at global level gyp deps to build sqlite3 for prod # then create node_modules_prod that will be the node_modules of final image RUN \ yarn install --frozen-lockfile --verbose --network-timeout 600000 # Install any extra node dependencies (at root level, to avoid having to wrestle # with merging them). COPY --from=ext / /grist/ext RUN \ mkdir /node_modules && \ cd /grist/ext && \ { if [ -e package.json ] ; then yarn install --frozen-lockfile --modules-folder=/node_modules --verbose --network-timeout 600000 ; fi } # Build node code. COPY tsconfig.json /grist COPY tsconfig-ext.json /grist COPY tsconfig-prod.json /grist COPY test/tsconfig.json /grist/test/tsconfig.json COPY test/chai-as-promised.js /grist/test/chai-as-promised.js COPY app /grist/app COPY stubs /grist/stubs COPY buildtools /grist/buildtools # Copy locales files early. During build process they are validated. COPY static/locales /grist/static/locales RUN WEBPACK_EXTRA_MODULE_PATHS=/node_modules yarn run build:prod # We don't need them anymore, they will by copied to the final image. RUN rm -rf /grist/static/locales # Prepare material for optional pyodide sandbox COPY sandbox/pyodide /grist/sandbox/pyodide COPY sandbox/requirements.txt /grist/sandbox/requirements.txt RUN \ cd /grist/sandbox/pyodide && make setup ################################################################################ ## Python collection stage ################################################################################ # Fetch python3.11 FROM python:3.11-slim-trixie AS collector-py3 COPY sandbox/requirements.txt requirements.txt RUN \ pip3 install -r requirements.txt ################################################################################ ## Sandbox collection stage ################################################################################ # Fetch gvisor-based sandbox. Note, to enable it to run within default # unprivileged docker, layers of protection that require privilege have # been stripped away, see https://github.com/google/gvisor/issues/4371 # The standalone sandbox binary is built on buster, but remains compatible # with recent Debian. # If you'd like to use unmodified gvisor, you should be able to just drop # in the standard runsc binary and run the container with any extra permissions # it needs. FROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox ################################################################################ ## Run-time stage ################################################################################ # Now, start preparing final image. FROM node:22-trixie-slim ARG GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=false # Install curl for docker healthchecks, libexpat1 and libsqlite3-0 for python3 # library binary dependencies, and procps for managing gvisor processes. RUN \ apt-get update && \ apt-get install -y --no-install-recommends curl libexpat1 libsqlite3-0 procps tini && \ rm -rf /var/lib/apt/lists/* # Keep all storage user may want to persist in a distinct directory RUN mkdir -p /persist/docs # Copy node files. COPY --from=builder /node_modules /node_modules COPY --from=prod-builder /grist/node_modules /grist/node_modules COPY --from=builder /grist/_build /grist/_build COPY --from=builder /grist/static /grist/static-built COPY --from=builder /grist/app/cli.sh /grist/cli # Patterm match here is to copy assets only if it exists in the # builder stage, otherwise matches nothing. # https://stackoverflow.com/a/70096420/11352427 COPY --from=builder /grist/ext/asset[s] /grist/ext/assets # Copy python3 files. COPY --from=collector-py3 /usr/local/bin/python3.11 /usr/bin/python3.11 COPY --from=collector-py3 /usr/local/lib/python3.11 /usr/local/lib/python3.11 COPY --from=collector-py3 /usr/local/lib/libpython3.11.* /usr/local/lib/ # Set default to python3 RUN \ ln -s /usr/bin/python3.11 /usr/bin/python && \ ln -s /usr/bin/python3.11 /usr/bin/python3 && \ ldconfig # Copy runsc. COPY --from=sandbox /runsc /usr/bin/runsc # Add files needed for running server. COPY package.json /grist/package.json COPY bower_components /grist/bower_components COPY sandbox /grist/sandbox COPY plugins /grist/plugins COPY static /grist/static # Make optional pyodide sandbox available COPY --from=builder /grist/sandbox/pyodide /grist/sandbox/pyodide # Finalize static directory RUN \ mv /grist/static-built/* /grist/static && \ rmdir /grist/static-built # To ensure non-root users can run grist, 'other' users need read access (and execute on directories) # This should be the case by default when copying files in. # Only uncomment this if running into permissions issues, as it takes a long time to execute on some systems. # RUN chmod -R o+rX /grist # Add a user to allow de-escalating from root on startup RUN useradd -ms /bin/bash grist ENV GRIST_DOCKER_USER=grist \ GRIST_DOCKER_GROUP=grist WORKDIR /grist # Set some default environment variables to give a setup that works out of the box when # started as: # docker run -p 8484:8484 -it # Variables will need to be overridden for other setups. # # GRIST_SANDBOX_FLAVOR is set to unsandboxed by default, because it # appears that the services people use to run docker containers have # a wide variety of security settings and the functionality needed for # sandboxing may not be possible in every case. For default docker # settings, you can get sandboxing as follows: # docker run --env GRIST_SANDBOX_FLAVOR=gvisor -p 8484:8484 -it # # "NODE_OPTIONS=--no-deprecation" is set because there is a punycode # deprecation nag that is relevant to developers but not to users. # TODO: upgrade package.json to avoid using all package versions # using the punycode functionality that may be removed in future # versions of node. # # "NODE_ENV=production" gives ActiveDoc operations more time to # complete, and the express webserver also does some streamlining # with this setting. If you don't want these, set NODE_ENV to # development. # ENV \ GRIST_ORG_IN_PATH=true \ GRIST_HOST=0.0.0.0 \ GRIST_SINGLE_PORT=true \ GRIST_SERVE_SAME_ORIGIN=true \ GRIST_DATA_DIR=/persist/docs \ GRIST_INST_DIR=/persist \ GRIST_SESSION_COOKIE=grist_core \ GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=${GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING} \ GVISOR_FLAGS="-unprivileged -ignore-cgroups" \ GRIST_SANDBOX_FLAVOR=unsandboxed \ NODE_OPTIONS="--no-deprecation" \ NODE_ENV=production \ TYPEORM_DATABASE=/persist/home.sqlite3 EXPOSE 8484 # When run without any arguments, we run the Grist server within # a simple supervisor. # When arguments are supplied they are treated as a command to run, # as is default for docker. We arrange to have a "cli" command that # is the same as "yarn cli" run from the source code repo. # So you can do things like: # docker run --rm -v $PWD:$PWD -it gristlabs/grist \ # cli sqlite query $PWD/docs/4gtUhAEGbGAdsGNc52k4H6.grist \ # --json "select * from _gristsys_ActionHistory" ENTRYPOINT ["./sandbox/docker_entrypoint.sh"] CMD ["node", "./sandbox/supervisor.mjs"] ================================================ FILE: LICENSE.txt ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2014-2022 Grist Labs Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: NOTICE.txt ================================================ Grist Software Copyright 2014-2022 Grist Labs Inc. This product includes software developed at Grist Labs Inc. (https://www.getgrist.com/). ================================================ FILE: README.md ================================================ # Grist Grist is a modern relational spreadsheet. It combines the flexibility of a spreadsheet with the robustness of a database. * `grist-core` (this repo) has what you need to run a powerful server for hosting spreadsheets. * [`grist-desktop`](https://github.com/gristlabs/grist-desktop) is a Linux/macOS/Windows desktop app for viewing and editing spreadsheets stored locally. * [`grist-static`](https://github.com/gristlabs/grist-static) is a fully in-browser build of Grist for displaying spreadsheets on a website without back-end support. Grist is developed by [Grist Labs](https://www.linkedin.com/company/grist-labs/), an NYC-based company 🇺🇸🗽. The French government 🇫🇷 organizations [ANCT Données et Territoires](https://donnees.incubateur.anct.gouv.fr/toolbox/grist) and [DINUM (Direction Interministérielle du Numérique)](https://www.numerique.gouv.fr/dinum/) have also made significant contributions to the codebase. The `grist-core`, `grist-desktop`, and `grist-static` repositories are all open source (Apache License, Version 2.0). Grist Labs offers free and paid hosted services at [getgrist.com](https://getgrist.com), sells an Enterprise product, and offers [cloud packaging](https://support.getgrist.com/install/grist-builder-edition/). > Questions? Feedback? Want to share what you're building with Grist? Join our [official Discord server](https://discord.gg/MYKpYQ3fbP) or visit our [Community forum](https://community.getgrist.com/). > > To keep up-to-date with everything that's going on, you can [sign up for Grist's monthly newsletter](https://www.getgrist.com/newsletter/). https://github.com/user-attachments/assets/fe152f60-3d15-4b11-8cb2-05731a90d273 ## Features in `grist-core` To see exactly what is present in `grist-core`, you can run the [desktop app](https://github.com/gristlabs/grist-desktop), or use [`docker`](#using-grist). The absolute fastest way to try Grist out is to visit [docs.getgrist.com](https://docs.getgrist.com) and play with a spreadsheet there immediately – though if you do, please read the list of [extra extensions](#features-not-in-grist-core) that are not in `grist-core`. However you try it, you'll quickly see that Grist is a hybrid database/spreadsheet, meaning that: - Columns work like they do in databases: they are named, and they hold one kind of data. - Columns can be filled by formula, spreadsheet-style, with automatic updates when referenced cells change. This difference can confuse people coming directly from Excel or Google Sheets. Give it a chance! There's also a [Grist for Spreadsheet Users](https://www.getgrist.com/blog/grist-for-spreadsheet-users/) article to help get you oriented. If you're coming from Airtable, you'll find the model familiar (and there's also our [Grist vs Airtable](https://www.getgrist.com/blog/grist-v-airtable/) article for a direct comparison). Here are some specific feature highlights of Grist: * Python formulas. - Full [Python syntax is supported](https://support.getgrist.com/formulas/#python), including the standard library. - Many [Excel functions](https://support.getgrist.com/functions/) also available. - An [AI Assistant](https://www.getgrist.com/ai-formula-assistant/) specifically tuned for formula generation (using OpenAI gpt-3.5-turbo or [Llama](https://ai.meta.com/llama/) via llama-cpp-python). * A portable, self-contained format. - Based on SQLite, the most widely deployed database engine. - Any tool that can read SQLite can read numeric and text data from a Grist file. - Enables [backups](https://support.getgrist.com/exports/#backing-up-an-entire-document) that you can confidently restore in full. - Great for moving between different hosts. * Can be displayed on a static website with [`grist-static`](https://github.com/gristlabs/grist-static) – no special server needed. * A self-contained desktop app for viewing and editing locally: [`grist-desktop`](https://github.com/gristlabs/grist-desktop). * Convenient editing and formatting features. - Choices and [choice lists](https://support.getgrist.com/col-types/#choice-list-columns), for adding colorful tags to records. - [References](https://support.getgrist.com/col-refs/#creating-a-new-reference-list-column) and reference lists, for cross-referencing records in other tables. - [Attachments](https://support.getgrist.com/col-types/#attachment-columns), to include media or document files in records. - Dates and times, toggles, and special numerics such as currency all have specialized editors and formatting options. - [Conditional Formatting](https://support.getgrist.com/conditional-formatting/), letting you control the style of cells with formulas to draw attention to important information. * Drag-and-drop dashboards. - [Charts](https://support.getgrist.com/widget-chart/), [card views](https://support.getgrist.com/widget-card/) and a [calendar widget](https://support.getgrist.com/widget-calendar/) for visualization. - [Summary tables](https://support.getgrist.com/summary-tables/) for summing and counting across groups. - [Widget linking](https://support.getgrist.com/linking-widgets/) streamlines filtering and editing data. Grist has a unique approach to visualization, where you can lay out and link distinct widgets to show together, without cramming mixed material into a table. - [Filter bar](https://support.getgrist.com/search-sort-filter/#filter-buttons) for quick slicing and dicing. * [Incremental imports](https://support.getgrist.com/imports/#updating-existing-records). - Import a CSV of the last three months activity from your bank... - ...and import new activity a month later without fuss or duplication. * [Native forms](https://support.getgrist.com/widget-form/). Create forms that feed directly into your spreadsheet without fuss. * Integrations. - A [REST API](https://support.getgrist.com/api/), [Zapier actions/triggers](https://support.getgrist.com/integrators/#integrations-via-zapier), and support from similar [integrators](https://support.getgrist.com/integrators/). - Import/export to Google drive, Excel format, CSV. - Link data with [custom widgets](https://support.getgrist.com/widget-custom/#_top), hosted externally. - Configurable outgoing webhooks. * [Many templates](https://templates.getgrist.com/) to get you started, from investment research to organizing treasure hunts. * Access control options. - (You'll need SSO logins set up to make use of these options; [`grist-omnibus`](https://github.com/gristlabs/grist-omnibus) has a prepackaged solution if configuring this feels daunting) - Share [individual documents](https://support.getgrist.com/sharing/), workspaces, or [team sites](https://support.getgrist.com/team-sharing/). - Control access to [individual rows, columns, and tables](https://support.getgrist.com/access-rules/). - Control access based on cell values and user attributes. * Self-maintainable. - Useful for intranet operation and specific compliance requirements. * Sandboxing options for untrusted documents. - On Linux or with Docker, you can enable [gVisor](https://github.com/google/gvisor) sandboxing at the individual document level. - On macOS, you can use native sandboxing. - On any OS, including Windows, you can use a wasm-based sandbox. * Translated to many languages. * `F1` key brings up some quick help. This used to go without saying, but in general Grist has good keyboard support. * We post progress on [𝕏 or Twitter or whatever](https://twitter.com/getgrist) and publish [monthly newsletters](https://support.getgrist.com/newsletters/). If you are curious about where Grist is heading, see [our roadmap](https://github.com/gristlabs/grist-core/projects/1), drop a question in [our forum](https://community.getgrist.com), or browse [our extensive documentation](https://support.getgrist.com). ## Features not in `grist-core` If you evaluate Grist by using the hosted version at [getgrist.com](https://getgrist.com), be aware that it includes some extensions to Grist that aren't present in `grist-core`. To be sure you're seeing exactly what is present in `grist-core`, you can run the [desktop app](https://github.com/gristlabs/grist-desktop), or use [`docker`](#using-grist). Here is a list of features you may see in Grist Labs' hosting or Enterprise offerings that are not in `grist-core`, in chronological order of creation. If self-hosting, you can get access to a free trial of all of them using the Enterprise toggle on the [Admin Panel](https://support.getgrist.com/admin-panel/). * [GristConnect](https://support.getgrist.com/install/grist-connect/) (2022) - Any site that has plugins for letting Discourse use its logins (such as WordPress) can also let Grist use its logins. - GristConnect is a niche feature built for a specific client which you probably don't care about – `OIDC` and `SAML` support *is* part of `grist-core` and covers most authentication use cases. * [Azure back-end for document storage](https://support.getgrist.com/install/cloud-storage/#azure) (2022) - With `grist-core` you can store document versions in anything S3-compatible, which covers a lot of services, but not Azure specifically. The Azure back-end fills that gap. - Unless you are a Microsoft shop you probably don't care about this. * [Audit log streaming](https://support.getgrist.com/install/audit-log-streaming/) (2024) - With `grist-core` a lot of useful information is logged, but not organized specifically with auditing in mind. Audit log streaming supplies that organization, and a UI for setting things up. - Enterprises may care about this. * [Advanced Admin Controls](https://support.getgrist.com/admin-controls/) (2025) - This is a special page for a Grist installation administrator to monitor and edit user access to resources. - It uses a special set of administrative endpoints not present on `grist-core`. - If you're going to be running a large Grist installation, with employees coming and going, you may care about this. * [Grist Assistant](https://support.getgrist.com/assistant/#assistant) (2025) - An AI Formula Assistant - limited to working with formulas - is present in `grist-core`, but the newer Assistant can help with a wider range of tasks like building tables and dashboards and modifying data. - If you have many users who need help building documents or working with data, you may care about this one. * [Invite Notifications](https://support.getgrist.com/self-managed/#how-do-i-set-up-email-notifications) (2025) - When a user is added to a document, or a workspace, or a site, with email notifications they will get emailed a link to access the resource. - This link isn't special, with `grist-core` you can just send a link yourself or a colleague. - For a big Grist installation with users who aren't in close communication, emails might be nice? Hard to guess if you'll care about this one. * [Document Change and Comment Notifications](https://support.getgrist.com/document-settings/#notifications) (2025) - You can achieve change notifications in `grist-core` using webhooks, but it is less convenient. - People have been asking for this one for years. If you need an excuse to get your boss to pay for Grist, this might finally be the one that works? ## Using Grist To get the default version of `grist-core` running on your computer with [Docker](https://www.docker.com/get-started), do: ```sh docker pull gristlabs/grist docker run -p 8484:8484 -it gristlabs/grist ``` Then visit `http://localhost:8484` in your browser. You'll be able to create, edit, import, and export documents. To preserve your work across docker runs, share a directory as `/persist`: ```sh docker run -p 8484:8484 -v $PWD/persist:/persist -it gristlabs/grist ``` Get templates at [templates.getgrist.com](https://templates.getgrist.com) for payroll, inventory management, invoicing, D&D encounter tracking, and a lot more, or use any document you've created on [docs.getgrist.com](https://docs.getgrist.com). If you need to change the port Grist runs on, set a `PORT` variable, don't just change the port mapping: ``` docker run --env PORT=9999 -p 9999:9999 -v $PWD/persist:/persist -it gristlabs/grist ``` To enable gVisor sandboxing, set `--env GRIST_SANDBOX_FLAVOR=gvisor`. This should work with default docker settings, but may not work in all environments. You can find a lot more about configuring Grist, setting up authentication, and running it on a public server in our [Self-Managed Grist](https://support.getgrist.com/self-managed/) handbook. ## Using Grist with OpenRouter for Model Agnostic and Claude Support (Instructions contributed by @lshalon) Grist's AI Formula Assistant can be configured to use OpenRouter instead of connecting directly to OpenAI, allowing you to access a wide range of AI models including Anthropic's Claude models. This isn't the only way to use Claude models, but it's a good option if you want to use Claude models with Grist or intend to use other cheaper, faster, or potentially newer models. That's because this configuration gives you more flexibility in choosing the AI model that works best for your formula generation needs. To set up OpenRouter integration, configure the following environment variables: ### Required: Set the endpoint to OpenRouter's API ``` ASSISTANT_CHAT_COMPLETION_ENDPOINT=https://openrouter.ai/api/v1/chat/completions ``` ### Required: Your OpenRouter API key ``` ASSISTANT_API_KEY=your_openrouter_api_key_here ``` Sign up for an OpenRouter API key at ### Optional: Specify which model to use (examples below) ``` ASSISTANT_MODEL=anthropic/claude-3.7-sonnet ``` ### or other options like ``` ASSISTANT_MODEL=deepseek/deepseek-r1-zero:free ``` ``` ASSISTANT_MODEL=qwen/qwq-32b:free ``` ``` ASSISTANT_MODEL=mistralai/mistral-saba ``` ### Optional: Set a larger context model for fallback ``` ASSISTANT_LONGER_CONTEXT_MODEL=anthropic/claude-3-opus-20240229 ``` With this configuration, Grist's AI Formula Assistant will route requests through OpenRouter to your specified model. This allows you to: Access Anthropic's Claude models which excel at understanding context and generating accurate formulas Switch between different AI models without changing your Grist configuration Take advantage of OpenRouter's routing capabilities to optimize for cost, speed, or quality You can find the available models and their identifiers on the OpenRouter website. Note: Make sure not to set the OPENAI_API_KEY variable when using OpenRouter, as this would override the OpenRouter configuration. ## Available Docker images The default Docker image is `gristlabs/grist`. This contains all of the standard Grist functionality, as well as extra source-available code for enterprise customers taken from the [grist-ee](https://github.com/gristlabs/grist-ee) repository. This extra code is not under a free or open source license. By default, however, the code from the `grist-ee` repository is completely inert and inactive. This code becomes active only when enabled from the administrator panel. If you would rather use an image that contains exclusively free and open source code, the `gristlabs/grist-oss` Docker image is available for this purpose. It is by default functionally equivalent to the `gristlabs/grist` image. ## The administrator panel You can turn on a special admininistrator panel to inspect the status of your installation. Just visit `/admin` on your Grist server for instructions. Since it is useful for the admin panel to be available even when authentication isn't set up, you can give it a special access key by setting `GRIST_BOOT_KEY`. ``` docker run -p 8484:8484 -e GRIST_BOOT_KEY=secret -it gristlabs/grist ``` The boot page should then be available at `/admin?boot-key=`. We are collecting probes for common problems there. If you hit a problem that isn't covered, it would be great if you could add a probe for it in [BootProbes](https://github.com/gristlabs/grist-core/blob/main/app/server/lib/BootProbes.ts). You may instead file an issue so someone else can add it. ## Building from source To build Grist from source, follow these steps: yarn install yarn install:python yarn build yarn start # Grist will be available at http://localhost:8484/ Grist formulas in documents will be run using Python executed directly on your machine. You can configure sandboxing using a `GRIST_SANDBOX_FLAVOR` environment variable. * On macOS, `export GRIST_SANDBOX_FLAVOR=macSandboxExec` uses the native `sandbox-exec` command for sandboxing. * On Linux with [gVisor's runsc](https://github.com/google/gvisor) installed, `export GRIST_SANDBOX_FLAVOR=gvisor` is an option. * On any OS including Windows, `export GRIST_SANDBOX_FLAVOR=pyodide` is available. These sandboxing methods have been written for our own use at Grist Labs and may need tweaking to work in your own environment - pull requests very welcome here! If you wish to include Grist Labs enterprise extensions in your build, the steps are as follows. Note that this will add non-OSS code to your build. It will also place a directory called `node_modules` one level up, at the same level as the Grist repo. If that is a problem for you, just move everything into a subdirectory first. yarn install yarn install:ee yarn install:python yarn build yarn start # Grist will be available at http://localhost:8484/ The enterprise code will by default not be used. You need to explicitly enable it in the [Admin Panel](https://support.getgrist.com/self-managed/#how-do-i-enable-grist-enterprise). ## Logins Like git, Grist has features to track document revision history. So for full operation, Grist expects to know who the user modifying a document is. Until it does, it operates in a limited anonymous mode. To get you going, the docker image is configured so that when you click on the "sign in" button Grist will attribute your work to `you@example.com`. Change this by setting `GRIST_DEFAULT_EMAIL`: ``` docker run --env GRIST_DEFAULT_EMAIL=my@email -p 8484:8484 -v $PWD/persist:/persist -it gristlabs/grist ``` You can change your name in `Profile Settings` in the [User Menu](https://support.getgrist.com/glossary/#user-menu). For multi-user operation, or if you wish to access Grist across the public internet, you'll want to connect it to your own Single Sign-On service. There are a lot of ways to do this, including [SAML and forward authentication](https://support.getgrist.com/self-managed/#how-do-i-set-up-authentication). Grist has been tested with [Authentik](https://goauthentik.io/), [Auth0](https://auth0.com/), and Google/Microsoft sign-ins via [Dex](https://dexidp.io/). ## Translations We use [Weblate](https://hosted.weblate.org/engage/grist/) to manage translations. Thanks to everyone who is pitching in. Thanks especially to the ANCT developers who did the hard work of making a good chunk of the application localizable. Merci beaucoup ! Translation status [![Translation detail](https://hosted.weblate.org/widgets/grist/-/multi-green.svg)](https://hosted.weblate.org/engage/grist/) ## Why free and open source software This repository, `grist-core`, is maintained by Grist Labs. Our flagship product available at [getgrist.com](https://www.getgrist.com) is built from the code you see here, combined with business-specific software designed to scale to many users, handle billing, etc. Grist Labs is an open-core company. We offer Grist hosting as a service, with free and paid plans. We also develop and sell features related to Grist using a proprietary license, targeted at the needs of enterprises with large self-managed installations. We see data portability and autonomy as a key value, and `grist-core` is an essential part of that. We are committed to maintaining and improving the `grist-core` codebase, and to be thoughtful about how proprietary offerings impact data portability and autonomy. By opening its source code and offering an [OSI](https://opensource.org/)-approved free license, Grist benefits its users: - **Developer community.** The freedom to examine source code, make bug fixes, and develop new features is a big deal for a general-purpose spreadsheet-like product, where there is a very long tail of features vital to someone somewhere. - **Increased trust.** Because anyone can examine the source code, “security by obscurity” is not an option. Vulnerabilities in the code can be found by others and reported before they cause damage. - **Independence.** Grist is available to you regardless of the fortunes of the Grist Labs business, since it is open source and can be self-hosted. Using our hosted solution is convenient, but you are not locked in. - **Price flexibility.** If you are low on funds but have time to invest, self-hosting is a great option to have. And DIY users may have the technical savvy and motivation to delve in and make improvements, which can benefit all users of Grist. - **Extensibility.** For developers, having the source open makes it easier to build extensions (such as [Custom Widgets](https://support.getgrist.com/widget-custom/)). You can more easily include Grist in your pipeline. And if a feature is missing, you can just take the source code and build on top of it. For more on Grist Labs' history and principles, see our [About Us](https://www.getgrist.com/about/) page. ## Sponsors

## Reviews * [Grist on ProductHunt](https://www.producthunt.com/posts/grist-2) * [Grist on AppSumo](https://appsumo.com/products/grist/) (life-time deal is sold out) * [Capterra](https://www.capterra.com/p/232821/Grist/#reviews), [G2](https://www.g2.com/products/grist/reviews), [TrustRadius](https://www.trustradius.com/products/grist/reviews) ## Environment variables Grist can be configured in many ways. Here are the main environment variables it is sensitive to: | Variable | Purpose | | -------- | ------- | | ALLOWED_WEBHOOK_DOMAINS | comma-separated list of permitted domains to use in webhooks (e.g. webhook.site,zapier.com). You can set this to `*` to allow all domains, but if doing so, we recommend using a carefully locked-down proxy (see `GRIST_PROXY_FOR_UNTRUSTED_URLS`) if you do not entirely trust users. Otherwise services on your internal network may become vulnerable to manipulation. | | APP_DOC_URL | doc worker url, set when starting an individual doc worker (other servers will find doc worker urls via redis) | | APP_DOC_INTERNAL_URL | like `APP_DOC_URL` but used by the home server to reach the server using an internal domain name resolution (like in a docker environment). It only makes sense to define this value in the doc worker. Defaults to `APP_DOC_URL`. | | APP_HOME_URL | url prefix for home api (home and doc servers need this) | | APP_HOME_INTERNAL_URL | like `APP_HOME_URL` but used by the home and the doc servers to reach any home workers using an internal domain name resolution (like in a docker environment). Defaults to `APP_HOME_URL` | | APP_STATIC_URL | url prefix for static resources | | APP_STATIC_INCLUDE_CUSTOM_CSS | set to "true" to include custom.css (from APP_STATIC_URL) in static pages | | APP_UNTRUSTED_URL | URL at which to serve/expect plugin content. | | GRIST_ACTION_HISTORY_MAX_ROWS | Maximum number of rows allowed in ActionHistory before pruning (up to a 1.25 grace factor). Defaults to 1000. ⚠️ A too low value may make the "[Work on a copy](https://support.getgrist.com/newsletters/2021-06/#work-on-a-copy)" feature [malfunction](https://github.com/gristlabs/grist-core/issues/1121#issuecomment-2248112023) | | GRIST_ACTION_HISTORY_MAX_BYTES | Maximum number of rows allowed in ActionHistory before pruning (up to a 1.25 grace factor). Defaults to 1Gb. ⚠️ A too low value may make the "[Work on a copy](https://support.getgrist.com/newsletters/2021-06/#work-on-a-copy)" feature [malfunction](https://github.com/gristlabs/grist-core/issues/1121#issuecomment-2248112023) | | GRIST_ADAPT_DOMAIN | set to "true" to support multiple base domains (careful, host header should be trustworthy) | | GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING | Whether Grist is allowed to automatically check if a newer Grist version is available. Defaults to "true" on the default `grist` and `grist-ee` Docker images. Defaults false in `grist-oss` and everywhere else. | | GRIST_ALLOW_DEPRECATED_BARE_ORG_DELETE | If set, the deprecated DELETE /api/orgs/:orgId endpoint is available. | | GRIST_APP_ROOT | directory containing Grist sandbox and assets (specifically the sandbox and static subdirectories). | | GRIST_ATTACHMENT_THRESHOLD_MB | attachment storage limit per document beyond which Grist will recommend external storage (if available). Defaults to 50MB. | | GRIST_BACKUP_DELAY_SECS | wait this long after a doc change before making a backup | | GRIST_BOOT_KEY | if set, offer diagnostics at /boot/GRIST_BOOT_KEY | | GRIST_BROADCAST_TIMEOUT_MS | Set the maximum time a web client has to accept a broadcast message about a document before being disconnected (default: 1 minute). | | GRIST_DATA_DIR | Directory in which to store documents. Defaults to `docs/` relative to the Grist application directory. In Grist's default Docker image, its default value is /persist/docs so that it will be used as a mounted volume. | | GRIST_DEFAULT_EMAIL | if set, login as this user if no other credentials presented | | GRIST_DEFAULT_PRODUCT | if set, this controls enabled features and limits of new sites. See names of PRODUCTS in Product.ts. | | GRIST_DEFAULT_LOCALE | Locale to use as fallback when Grist cannot honour the browser locale. | | GRIST_DOMAIN | in hosted Grist, Grist is served from subdomains of this domain. Defaults to "getgrist.com". | | GRIST_EXPERIMENTAL_PLUGINS | enables experimental plugins | | GRIST_EXTERNAL_ATTACHMENTS_MODE | required to enable external storage for attachments. Set to "snapshots" to enable external storage. Default value is "none". Note that when enabled, a [snapshot storage has to be configured](https://support.getgrist.com/self-managed/#how-do-i-set-up-snapshots) as well. | | GRIST_ENABLE_SERVICE_ACCOUNTS | enables the `service accounts` feature. This feature allows users to create special service accounts that they can manage and to whom they can grant restricted access to chosen resources. Useful as a way to get fine-grained api keys for use with third party automations. Unset by default | | GRIST_ENABLE_REQUEST_FUNCTION | enables the REQUEST function. This function performs HTTP requests in a similar way to `requests.request`. This function presents a significant security risk, since it can let users call internal endpoints when Grist is available publicly. This function can also cause performance issues. Unset by default. | | GRIST_HEADERS_TIMEOUT_MS | if set, override nodes's server.headersTimeout flag. | | GRIST_HIDE_UI_ELEMENTS | comma-separated list of UI features to disable. Allowed names of parts: `helpCenter`, `billing`, `templates`, `createSite`, `multiSite`, `multiAccounts`, `importFromAirtable`, `sendToDrive`, `tutorials`, `supportGrist`, `themes`. If a part also exists in GRIST_UI_FEATURES, it will still be disabled. | | GRIST_HOST | hostname to use when listening on a port. | | GRIST_PROXY_FOR_UNTRUSTED_URLS | Full URL of proxy for delivering webhook payloads. Default value is `direct` for delivering payloads without proxying. | | HTTPS_PROXY or https_proxy | Full URL of reverse web proxy (corporate proxy) for fetching the custom widgets repository or the OIDC config from the issuer. | | GRIST_ID_PREFIX | for subdomains of form o-*, expect or produce o-${GRIST_ID_PREFIX}*. | | GRIST_IGNORE_SESSION | if set, Grist will not use a session for authentication. | | GRIST_INCLUDE_CUSTOM_SCRIPT_URL | if set, will load the referenced URL in a ` `; } ================================================ FILE: app/common/TimeQuery.ts ================================================ import { concatenateSummaries } from "app/common/ActionSummarizer"; import { ActionSummary, ColumnDelta, createEmptyActionSummary, createEmptyTableDelta } from "app/common/ActionSummary"; import { CellDelta } from "app/common/TabularDiff"; import keyBy from "lodash/keyBy"; import matches from "lodash/matches"; import sortBy from "lodash/sortBy"; import toPairs from "lodash/toPairs"; /** * We can combine an ActionSummary with the current state of the database * to answer questions about the state of the database in the past. This * is particularly useful for Grist metadata tables, which are needed to * interpret the content of user tables fully. * - TimeCursor is a simple container for the db and an ActionSummary * - TimeQuery offers a db-like interface for a given table and set of columns * - TimeLayout answers a couple of concrete questions about table meta-data using a * set of TimeQuery objects hooked up to _grist_* tables. */ export interface ResultRow { [column: string]: any; } export interface ITimeData { fetch(tableId: string, colIds: string[], rowIds?: number[]): Promise; getColIds(tableId: string): Promise; } /** Track the state of the database at a particular time. */ export class TimeCursor { public summary: ActionSummary; constructor(public db: ITimeData) { this.summary = createEmptyActionSummary(); } /** * Add a summary of an action just before the last action applied to * the TimeCursor, so we stretch further back in time. */ public prepend(prevSummary: ActionSummary) { this.summary = concatenateSummaries([prevSummary, this.summary]); } /** * Add a summary of an action just after the last action applied to * the TimeCursor, going one step closer to current time. When the * cursor is used, the summary is assumed to extend right up to * current time. */ public append(nextSummary: ActionSummary) { // TODO: concatenation appears to modify its inputs, so we // need to clone to avoid propagating that. Look to see if // a safe version of concatenation could be written to save // cloning. this.summary = concatenateSummaries([this.summary, nextSummary]); } } /** internal class for storing a ResultRow dictionary, keyed by rowId */ interface ResultRows { [rowId: number]: ResultRow; } /** * Query the state of a particular table in the past, given a TimeCursor holding the * current db and a summary of all changes between that past time and now. * For the moment, for simplicity, names of tables and columns are assumed not to * change, and TimeQuery should only be used for _grist_* tables. */ export class TimeQuery { private _currentRows: ResultRow[]; private _pastRows: ResultRow[]; constructor(public tc: TimeCursor, public tableId: string, public colIds: string[] | "*", public rowIds?: number[]) { } public reset(tableId: string, colIds: string[] | "*", rowIds?: number[]) { this.tableId = tableId; this.colIds = colIds; this.rowIds = rowIds; this._currentRows = []; this._pastRows = []; } /** * Get fresh data from DB and overlay with any past data. * TODO: optimize. */ public async update(): Promise { this._currentRows = []; this._pastRows = []; const tableRenameDelta = this.tc.summary.tableRenames.find( delta => delta[0] === this.tableId, ); const tableRenamed = tableRenameDelta ? tableRenameDelta[1] : this.tableId; // Table no longer exists. if (!tableRenamed) { return []; } // Let's see everything the summary has accumulated about the table back then. const td = this.tc.summary.tableDeltas[tableRenamed] || createEmptyTableDelta(); const columnForwardRenames: Record = Object.fromEntries(td.columnRenames.filter(delta => delta[0])); const columnBackwardRenames: Record = Object.fromEntries(td.columnRenames.map(([a, b]) => [b, a]).filter(delta => delta[0])); const colIdsExpanded = this.colIds === "*" ? (await this.tc.db.getColIds(tableRenamed)).map(colId => columnBackwardRenames[colId] ?? colId) : this.colIds; const colIdsRenamed = colIdsExpanded.map(colId => columnForwardRenames[colId] ?? colId).filter(colId => colId); this._currentRows = await this.tc.db.fetch( tableRenamed, ["id", ...colIdsRenamed], this.rowIds, ); // Now rewrite the summary as a ResultRow dictionary, to make it comparable // with database. const summaryRows: ResultRows = {}; for (const [colId, columns] of toPairs(td.columnDeltas)) { for (const [rowId, cell] of toPairs(columns) as unknown as [keyof ColumnDelta, CellDelta][]) { if (!summaryRows[rowId]) { summaryRows[rowId] = {}; } const val = cell[0]; summaryRows[rowId][colId] = (val !== null && typeof val === "object") ? val[0] : null; } } // Prepare to access the current database state by rowId. const rowsById = keyBy(this._currentRows, r => (r.id as number)); // Prepare a list of rowIds at the time of interest. // The past rows are whatever the db has now, omitting rows that were added // since the past time, and adding back any rows that were removed since then. // Careful about the order of this, since rows could be replaced. const additions = new Set(td.addRows); const pastRowIds = new Set([...this._currentRows.map(r => r.id as number).filter(r => !additions.has(r)), ...td.removeRows]); // Now prepare a row for every expected rowId, using current db data if available // and relevant, and overlaying past data when available. this._pastRows = new Array(); const colIdsOfInterest = new Set(colIdsExpanded); for (const id of Array.from(pastRowIds).sort()) { const rowCurrent: ResultRow = rowsById[id] || { id }; const row: ResultRow = {}; for (const colId of ["id", ...colIdsExpanded]) { const colIdRenamed = columnForwardRenames[colId] ?? colId; if (!colIdRenamed) { continue; } row[colId] = rowCurrent[colIdRenamed]; } if (summaryRows[id] && !additions.has(id)) { for (const [colId, val] of toPairs(summaryRows[id])) { const colIdRenamed = columnBackwardRenames[colId] ?? colId; if (colIdsOfInterest.has(colIdRenamed)) { row[colIdRenamed] = val; } } } this._pastRows.push(row); } return this._pastRows; } /** * Do a query with a single result, specifying any desired filters. Exception thrown * if there is no result. */ public one(args: { [name: string]: any }): ResultRow { const result = this._pastRows.find(matches(args)); if (!result) { throw new Error(`could not find: ${JSON.stringify(args)} for ${this.tableId}`); } return result; } /** Get all results for a query. */ public all(args?: { [name: string]: any }): ResultRow[] { if (!args) { return this._pastRows; } return this._pastRows.filter(matches(args)); } } /** * Put some TimeQuery queries to work answering questions about column order and * user-facing name of tables. */ export class TimeLayout { public tables: TimeQuery; public fields: TimeQuery; public columns: TimeQuery; public views: TimeQuery; public sections: TimeQuery; constructor(public tc: TimeCursor) { this.tables = new TimeQuery(tc, "_grist_Tables", ["tableId", "primaryViewId", "rawViewSectionRef"]); this.fields = new TimeQuery(tc, "_grist_Views_section_field", ["parentId", "parentPos", "colRef"]); this.columns = new TimeQuery(tc, "_grist_Tables_column", ["parentId", "colId"]); this.views = new TimeQuery(tc, "_grist_Views", ["id", "name"]); this.sections = new TimeQuery(tc, "_grist_Views_section", ["id", "title"]); } /** update from TimeCursor */ public async update() { await this.tables.update(); await this.columns.update(); await this.fields.update(); await this.views.update(); await this.sections.update(); } public getColumnOrder(tableId: string): string[] { const primaryViewId = this.tables.one({ tableId }).primaryViewId; const preorder = this.fields.all({ parentId: primaryViewId }); const precol = keyBy(this.columns.all(), "id"); const ordered = sortBy(preorder, "parentPos"); const names = ordered.map(r => precol[r.colRef].colId); return names; } public getTableName(tableId: string): string { const rawViewSectionRef = this.tables.one({ tableId }).rawViewSectionRef; return this.sections.one({ id: rawViewSectionRef }).title; } } ================================================ FILE: app/common/Triggers-ti.ts ================================================ /** * This module was automatically generated by `ts-interface-builder` */ import * as t from "ts-interface-checker"; // tslint:disable:object-literal-key-quotes export const WebhookSubscribeCollection = t.iface([], { "webhooks": t.array("Webhook"), }); export const Webhook = t.iface([], { "fields": "WebhookFields", }); export const WebhookFields = t.iface([], { "url": "string", "authorization": t.opt("string"), "eventTypes": t.array(t.union(t.lit("add"), t.lit("update"))), "tableId": "string", "watchedColIds": t.opt(t.array("string")), "enabled": t.opt("boolean"), "isReadyColumn": t.opt(t.union("string", "null")), "condition": t.opt("string"), "name": t.opt("string"), "memo": t.opt("string"), }); export const WebhookBatchStatus = t.union(t.lit("success"), t.lit("failure"), t.lit("rejected")); export const WebhookStatus = t.union(t.lit("idle"), t.lit("sending"), t.lit("retrying"), t.lit("postponed"), t.lit("error"), t.lit("invalid")); export const WebHookSecret = t.iface([], { "url": "string", "unsubscribeKey": "string", "authorization": t.opt("string"), }); export const WebhookSubscribe = t.iface([], { "url": "string", "authorization": t.opt("string"), "eventTypes": t.array(t.union(t.lit("add"), t.lit("update"))), "watchedColIds": t.opt(t.array("string")), "enabled": t.opt("boolean"), "condition": t.opt("string"), "isReadyColumn": t.opt(t.union("string", "null")), "name": t.opt("string"), "memo": t.opt("string"), }); export const WebhookSummaryCollection = t.iface([], { "webhooks": t.array("WebhookSummary"), }); export const WebhookSummary = t.iface([], { "id": "string", "fields": t.iface([], { "url": "string", "authorization": t.opt("string"), "unsubscribeKey": "string", "eventTypes": t.array("string"), "isReadyColumn": t.union("string", "null"), "tableId": "string", "watchedColIds": t.opt(t.array("string")), "enabled": "boolean", "name": "string", "memo": "string", }), "usage": t.union("WebhookUsage", "null"), }); export const WebhookUpdate = t.iface([], { "id": "string", "fields": "WebhookPatch", }); export const WebhookPatch = t.iface([], { "url": t.opt("string"), "authorization": t.opt("string"), "eventTypes": t.opt(t.array(t.union(t.lit("add"), t.lit("update")))), "tableId": t.opt("string"), "watchedColIds": t.opt(t.array("string")), "enabled": t.opt("boolean"), "isReadyColumn": t.opt(t.union("string", "null")), "name": t.opt("string"), "memo": t.opt("string"), }); export const WebhookUsage = t.iface([], { "numWaiting": "number", "status": "WebhookStatus", "updatedTime": t.opt(t.union("number", "null")), "lastSuccessTime": t.opt(t.union("number", "null")), "lastFailureTime": t.opt(t.union("number", "null")), "lastErrorMessage": t.opt(t.union("string", "null")), "lastHttpStatus": t.opt(t.union("number", "null")), "lastEventBatch": t.opt(t.union("null", t.iface([], { "size": "number", "errorMessage": t.union("string", "null"), "httpStatus": t.union("number", "null"), "status": "WebhookBatchStatus", "attempts": "number", }))), "numSuccess": t.opt(t.iface([], { "pastHour": "number", "past24Hours": "number", })), }); export const TriggerAction = t.union("WebhookAction", "EmailAction"); export const WebhookAction = t.iface([], { "type": t.lit("webhook"), "id": "string", }); export const EmailAction = t.iface([], { "id": "string", "type": t.lit("email"), "to": "string", "subject": "string", "body": "string", }); const exportedTypeSuite: t.ITypeSuite = { WebhookSubscribeCollection, Webhook, WebhookFields, WebhookBatchStatus, WebhookStatus, WebHookSecret, WebhookSubscribe, WebhookSummaryCollection, WebhookSummary, WebhookUpdate, WebhookPatch, WebhookUsage, TriggerAction, WebhookAction, EmailAction, }; export default exportedTypeSuite; ================================================ FILE: app/common/Triggers.ts ================================================ export interface WebhookSubscribeCollection { webhooks: Webhook[] } export interface Webhook { fields: WebhookFields; } export interface WebhookFields { url: string; authorization?: string; eventTypes: ("add" | "update")[]; tableId: string; watchedColIds?: string[]; enabled?: boolean; isReadyColumn?: string | null; condition?: string; name?: string; memo?: string; } // Union discriminated by type export type WebhookBatchStatus = "success" | "failure" | "rejected"; export type WebhookStatus = "idle" | "sending" | "retrying" | "postponed" | "error" | "invalid"; /** Secrets for webhook stored outside the document in home db */ export interface WebHookSecret { url: string; unsubscribeKey: string; authorization?: string; } // WebhookSubscribe should be `Omit` (because subscribe endpoint read // tableId from the url) but generics are not yet supported by ts-interface-builder export interface WebhookSubscribe { url: string; authorization?: string; eventTypes: ("add" | "update")[]; watchedColIds?: string[]; enabled?: boolean; condition?: string; isReadyColumn?: string | null; name?: string; memo?: string; } export interface WebhookSummaryCollection { webhooks: WebhookSummary[]; } export interface WebhookSummary { id: string; fields: { url: string; authorization?: string; unsubscribeKey: string; eventTypes: string[]; isReadyColumn: string | null; tableId: string; watchedColIds?: string[]; enabled: boolean; name: string; memo: string; }, usage: WebhookUsage | null, } // Describes fields to update a webhook export interface WebhookUpdate { id: string; fields: WebhookPatch; } // WebhookPatch should be `Partial` but generics are not yet supported by // ts-interface-builder export interface WebhookPatch { url?: string; authorization?: string; eventTypes?: ("add" | "update")[]; tableId?: string; watchedColIds?: string[]; enabled?: boolean; isReadyColumn?: string | null; name?: string; memo?: string; } export interface WebhookUsage { // As minimum we need number of waiting events and status (by default pending). numWaiting: number, status: WebhookStatus; updatedTime?: number | null; lastSuccessTime?: number | null; lastFailureTime?: number | null; lastErrorMessage?: string | null; lastHttpStatus?: number | null; lastEventBatch?: null | { size: number; errorMessage: string | null; httpStatus: number | null; status: WebhookBatchStatus; attempts: number; }, numSuccess?: { pastHour: number; past24Hours: number; }, } // Union type for trigger actions. Currently only WebhookAction is supported, but this is // designed as a discriminated union to support additional action types in the future (e.g., emails). export type TriggerAction = WebhookAction | EmailAction; export interface WebhookAction { // The type field is used to discriminate between different action types. // For now we have only webhook, but next types in the pipeline are emails. type: "webhook"; id: string; // Unique id of the action, used as a key in homeDB secrets for webhooks } export interface EmailAction { id: string; type: "email"; to: string; // Comma-separated list of email addresses, user refs. subject: string; body: string; } ================================================ FILE: app/common/User.ts ================================================ import { getTableId } from "app/common/DocActions"; import { EmptyRecordView, RecordView } from "app/common/RecordView"; import { Role } from "app/common/roles"; /** * User type to distinguish between Users and service accounts */ export type UserType = "login" | "service"; /** * Information about a user, including any user attributes. */ export interface UserInfo { Name: string | null; Email: string | null; Access: Role | null; Origin: string | null; LinkKey: Record; UserID: number | null; UserRef: string | null; SessionID: string | null; /** * This is a rowId in the _grist_Shares table, if the user is accessing a document * via a share. Otherwise null. */ ShareRef: number | null; Type: UserType | null; [attributes: string]: unknown; } /** * Wrapper class for `UserInfo`. * * Contains methods for converting itself to different representations. */ export class User implements UserInfo { public Name: string | null = null; public UserID: number | null = null; public Access: Role | null = null; public Origin: string | null = null; public LinkKey: Record = {}; public Email: string | null = null; public SessionID: string | null = null; public UserRef: string | null = null; public ShareRef: number | null = null; public Type: UserType | null = null; [attribute: string]: any; constructor(info: Record = {}) { Object.assign(this, info); } /** * Returns a JSON representation of this class that excludes full row data, * only keeping user info and table/row ids for any user attributes. * * Used by the sandbox to support `user` variables in formulas (see `user.py`). */ public toJSON() { return this._toObject((value) => { if (value instanceof RecordView) { return [getTableId(value.data), value.get("id")]; } else if (value instanceof EmptyRecordView) { return null; } else { return value; } }); } /** * Returns a record representation of this class, with all user attributes * converted from `RecordView` instances to their JSON representations. * * Used by the client to support `user` variables in dropdown conditions. */ public toUserInfo(): UserInfo { return this._toObject((value) => { if (value instanceof RecordView) { return value.toJSON(); } else if (value instanceof EmptyRecordView) { return null; } else { return value; } }) as UserInfo; } private _toObject(mapValue: (value: unknown) => unknown) { const results: { [key: string]: any } = {}; for (const [key, value] of Object.entries(this)) { results[key] = mapValue(value); } return results; } } ================================================ FILE: app/common/UserAPI.ts ================================================ import { ApplyUAResult, ForkResult, FormulaTimingInfo, PermissionDataWithExtraUsers, QueryFilters, TimingStatus } from "app/common/ActiveDocAPI"; import { AssistanceRequest, AssistanceResponse } from "app/common/Assistance"; import { BaseAPI, IOptions } from "app/common/BaseAPI"; import { BillingAPI, BillingAPIImpl } from "app/common/BillingAPI"; import { BrowserSettings } from "app/common/BrowserSettings"; import { ICustomWidget } from "app/common/CustomWidget"; import { BulkColValues, TableColValues, TableRecordValue, TableRecordValues, TableRecordValuesWithoutIds, UserAction } from "app/common/DocActions"; import { DocCreationInfo, OpenDocMode } from "app/common/DocListAPI"; import { DocStateComparison, DocStates } from "app/common/DocState"; import { OrgUsageSummary } from "app/common/DocUsage"; import { Features, Product } from "app/common/Features"; import { isClient } from "app/common/gristUrls"; import { encodeQueryParams } from "app/common/gutil"; import { FullUser, UserProfile } from "app/common/LoginSessionAPI"; import { OrgPrefs, UserOrgPrefs, UserPrefs } from "app/common/Prefs"; import * as roles from "app/common/roles"; import { StringUnion } from "app/common/StringUnion"; import { WebhookFields, WebhookSubscribe, WebhookSummaryCollection, WebhookUpdate, } from "app/common/Triggers"; import { addCurrentOrgToPath, getGristConfig } from "app/common/urlUtils"; import { AttachmentStore, AttachmentStoreDesc, TablesGet } from "app/plugin/DocApiTypes"; import { AxiosProgressEvent } from "axios"; import omitBy from "lodash/omitBy"; export type { FullUser, UserProfile }; // Nominal email address of the anonymous user. export const ANONYMOUS_USER_EMAIL = "anon@getgrist.com"; // Nominal email address of a user who, if you share with them, everyone gets access. export const EVERYONE_EMAIL = "everyone@getgrist.com"; // Nominal email address of a user who can view anything (for thumbnails). export const PREVIEWER_EMAIL = "thumbnail@getgrist.com"; // A special 'docId' that means to create a new document. export const NEW_DOCUMENT_CODE = "new"; // Properties shared by org, workspace, and doc resources. export interface CommonProperties { name: string; createdAt: string; // ISO date string updatedAt: string; // ISO date string removedAt?: string; // ISO date string - only can appear on docs and workspaces currently disabledAt?: string; // ISO date string - only can appear on docs currently public?: boolean; // If set, resource is available to the public } export const commonPropertyKeys = ["createdAt", "name", "updatedAt"]; export interface OrganizationProperties extends CommonProperties { domain: string | null; // Organization includes preferences relevant to interacting with its content. userOrgPrefs?: UserOrgPrefs; // Preferences specific to user and org orgPrefs?: OrgPrefs; // Preferences specific to org (but not a particular user) userPrefs?: UserPrefs; // Preferences specific to user (but not a particular org) } export const organizationPropertyKeys = [...commonPropertyKeys, "domain", "orgPrefs", "userOrgPrefs", "userPrefs"]; // Basic information about an organization, excluding the user's access level export interface OrganizationWithoutAccessInfo extends OrganizationProperties { id: number; owner: FullUser | null; billingAccount?: BillingAccount; host: string | null; // if set, org's preferred domain (e.g. www.thing.com) } // Organization information plus the user's access level export interface Organization extends OrganizationWithoutAccessInfo { access: roles.Role; } // Basic information about a billing account associated with an org or orgs. export interface BillingAccount { id: number; individual: boolean; product: Product; stripePlanId: string; // Stripe price id. isManager: boolean; inGoodStanding: boolean; features?: Features; // Features override, not the final set of features. externalOptions?: { invoiceId?: string; }; } // The upload types vary based on which fetch implementation is in use. This is // an incomplete list. For example, node streaming types are supported by node-fetch. export type UploadType = string | Blob | Buffer; /** * Returns a user-friendly org name, which is either org.name, or "@User Name" for personal orgs. */ export function getOrgName(org: Organization): string { return org.owner ? `@` + org.owner.name : org.name; } /** * Returns whether the given org is the templates org, which contains the public * templates and tutorials. */ export function isTemplatesOrg(org: { domain: Organization["domain"] } | null): boolean { if (!org) { return false; } const { templateOrg } = getGristConfig(); return org.domain === templateOrg; } export type WorkspaceProperties = CommonProperties; export const workspacePropertyKeys = ["createdAt", "name", "updatedAt"]; export interface Workspace extends WorkspaceProperties { id: number; docs: Document[]; org: Organization; orgDomain?: string; access: roles.Role; owner?: FullUser; // Set when workspaces are in the "docs" pseudo-organization, // assembled from multiple personal organizations. // Not set when workspaces are all from the same organization. // Set when the workspace belongs to support@getgrist.com. We expect only one such workspace // ("Examples & Templates"), containing sample documents. isSupportWorkspace?: boolean; } // null stands for normal document type, the one set by default at document creation. export const DOCTYPE_NORMAL = null; export const DOCTYPE_TEMPLATE = "template"; export const DOCTYPE_TUTORIAL = "tutorial"; export type DocumentType = typeof DOCTYPE_NORMAL | typeof DOCTYPE_TEMPLATE | typeof DOCTYPE_TUTORIAL; // Non-core options for a document. // "Non-core" means bundled into a single options column in the database. // TODO: consider smoothing over this distinction in the API. export interface DocumentOptions { description?: string | null; icon?: string | null; openMode?: OpenDocMode | null; externalId?: string | null; // A slot for storing an externally maintained id. // Not used in grist-core, but handy for Electron app. tutorial?: TutorialMetadata | null; appearance?: DocumentAppearance | null; // Whether search engines should index this document. Defaults to `false`. allowIndex?: boolean; proposedChanges?: ProposedChanges | null; } export interface TutorialMetadata { lastSlideIndex?: number; percentComplete?: number; } interface DocumentAppearance { icon?: DocumentIcon | null; } interface DocumentIcon { backgroundColor?: string; color?: string; emoji?: string | null; } export interface DocumentProperties extends CommonProperties { isPinned: boolean; urlId: string | null; trunkId: string | null; type: DocumentType | null; options: DocumentOptions | null; } export interface ProposedChanges { mayHaveProposals?: boolean; acceptProposals?: boolean; } export const documentPropertyKeys = [ ...commonPropertyKeys, "isPinned", "urlId", "options", "type", "appearance", ]; export interface Document extends DocumentProperties { id: string; workspace: Workspace; access: roles.Role; trunkAccess?: roles.Role | null; forks?: Fork[]; } export interface Fork { id: string; trunkId: string; updatedAt: string; // ISO date string options: DocumentOptions | null; } export interface ProposalComparison { comparison?: DocStateComparison; } export interface ProposalStatus { status?: "applied" | "retracted" | "dismissed"; } export interface Proposal { shortId: number; comparison: ProposalComparison; status: ProposalStatus; createdAt: string; // ISO date string updatedAt: string; // ISO date string appliedAt: string | null; // ISO date string srcDocId: string; srcDoc: Document & { creator: FullUser }, destDocId: string; destDoc: Document & { creator: FullUser }, } // Non-core options for a user. export interface UserOptions { // Whether signing in with Google is allowed. Defaults to true if unset. allowGoogleLogin?: boolean; // The "sub" (subject) from the JWT issued by the password-based authentication provider. authSubject?: string; // Whether user is a consultant. Consultant users can be added to sites // without being counted for billing. Defaults to false if unset. isConsultant?: boolean; // Locale selected by the user. Defaults to 'en' if unset. locale?: string; ssoExtraInfo?: Record; // Extra fields from the user profile, e.g. from OIDC. // The first time the user logged in using getgrist.com auth. Only set in Grist SaaS. firstOAuthLoginAt?: Date; } export interface PermissionDelta { maxInheritedRole?: roles.BasicRole | null; users?: { // Maps from email to group name, or null to inherit. [email: string]: roles.NonGuestRole | null }; } export interface PermissionData { // True if permission data is restricted to current user. personal?: true; // True if current user is a public member. public?: boolean; maxInheritedRole?: roles.BasicRole | null; users: UserAccessData[]; } // A structure for modifying managers of a billing account. export interface ManagerDelta { users: { // To add a manager, link their email to 'managers'. // To remove a manager, link their email to null. // This format is used to rhyme with the ACL PermissionDelta format. [email: string]: "managers" | null }; } export interface UserAccess { // Represents the user's direct access to the resource of interest. Lack of access to a resource // is represented by a null value. access: roles.Role | null; // A user's parentAccess represent their effective inheritable access to the direct parent of the resource // of interest. The user's effective access to the resource of interest can be determined based // on the user's parentAccess, the maxInheritedRole setting of the resource and the user's direct // access to the resource. Lack of access to the parent resource is represented by a null value. // If parent has non-inheritable access, this should be null. parentAccess?: roles.BasicRole | null; } // Information about a user and their access to an unspecified resource of interest. export interface UserAccessData extends UserAccess { id: number; name: string; email: string; ref?: string | null; picture?: string | null; // When present, a url to a public image of unspecified dimensions. orgAccess?: roles.BasicRole | null; anonymous?: boolean; // If set to true, the user is the anonymous user. isMember?: boolean; disabledAt?: Date | null; // If not null, the user is disabled } /** * Combines access, parentAccess, and maxInheritedRole info into the resulting access role. */ export function getRealAccess( user: UserAccess, inherited: { maxInheritedRole?: roles.BasicRole | null }, ): roles.Role | null { const inheritedAccess = roles.getWeakestRole(user.parentAccess || null, inherited.maxInheritedRole || null); return roles.getStrongestRole(user.access, inheritedAccess); } const roleNames: { [role: string]: string } = { [roles.OWNER]: "Owner", [roles.EDITOR]: "Editor", [roles.VIEWER]: "Viewer", }; export function getUserRoleText(user: UserAccessData) { return roleNames[user.access!] || user.access || "no access"; } export interface ExtendedUser extends FullUser { helpScoutSignature?: string; isInstallAdmin?: boolean; // Set if user is allowed to manage this installation. } export interface ActiveSessionInfo { user: ExtendedUser; org: Organization | null; orgError?: OrgError; } export interface OrgError { error: string; status: number; } /** * Options to control the source of a document being replaced. For * example, a document could be initialized from another document * (e.g. a fork) or from a snapshot. */ export interface DocReplacementOptions { /** * The docId to copy from. */ sourceDocId?: string; /** * The s3 version ID. */ snapshotId?: string; /** * True if tutorial metadata should be reset. * * Metadata that's reset includes the doc (i.e. tutorial) name, and the * properties under options.tutorial (e.g. lastSlideIndex). */ resetTutorialMetadata?: boolean; } /** * Information about a single document snapshot/backup. */ export interface DocSnapshot { lastModified: string; // when the snapshot was made snapshotId: string; // the id of the snapshot in the underlying store docId: string; // an id for accessing the snapshot as a Grist document } /** * A list of document snapshots. */ export interface DocSnapshots { snapshots: DocSnapshot[]; // snapshots, freshest first. } export interface CopyDocOptions { documentName: string; asTemplate?: boolean; } export interface RenameDocOptions { icon?: DocumentIcon | null; } export interface UserAPI { getSessionActive(): Promise; setSessionActive(email: string, org?: string): Promise; getSessionAll(): Promise<{ users: FullUser[], orgs: Organization[] }>; getOrgs(merged?: boolean): Promise; getWorkspace(workspaceId: number): Promise; getOrg(orgId: number | string): Promise; getOrgWorkspaces(orgId: number | string, includeSupport?: boolean): Promise; getOrgUsageSummary(orgId: number | string): Promise; getTemplates(): Promise; getTemplate(docId: string): Promise; getDoc(docId: string): Promise; newOrg(props: Partial): Promise; newWorkspace(props: Partial, orgId: number | string): Promise; newDoc(props: Partial, workspaceId: number): Promise; newUnsavedDoc(options?: { timezone?: string }): Promise; copyDoc(sourceDocumentId: string, workspaceId: number, options: CopyDocOptions): Promise; renameOrg(orgId: number | string, name: string): Promise; renameWorkspace(workspaceId: number, name: string): Promise; renameDoc(docId: string, name: string, options?: RenameDocOptions): Promise; updateOrg(orgId: number | string, props: Partial): Promise; updateDoc(docId: string, props: Partial): Promise; deleteOrg(orgId: number | string): Promise; deleteWorkspace(workspaceId: number): Promise; // delete workspace permanently softDeleteWorkspace(workspaceId: number): Promise; // soft-delete workspace undeleteWorkspace(workspaceId: number): Promise; // recover soft-deleted workspace deleteDoc(docId: string): Promise; // delete doc permanently softDeleteDoc(docId: string): Promise; // soft-delete doc undeleteDoc(docId: string): Promise; // recover soft-deleted doc disableDoc(docId: string): Promise; // (admin-only) remove all access to doc except deletion enableDoc(docId: string): Promise; // (admin-only) recover disabled doc updateOrgPermissions(orgId: number | string, delta: PermissionDelta): Promise; updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise; updateDocPermissions(docId: string, delta: PermissionDelta): Promise; getOrgAccess(orgId: number | string): Promise; getWorkspaceAccess(workspaceId: number): Promise; getDocAccess(docId: string): Promise; pinDoc(docId: string): Promise; unpinDoc(docId: string): Promise; moveDoc(docId: string, workspaceId: number): Promise; getUserProfile(): Promise; updateUserName(name: string): Promise; updateUserLocale(locale: string | null): Promise; updateAllowGoogleLogin(allowGoogleLogin: boolean): Promise; disableUser(userId: number): Promise; enableUser(userId: number): Promise; updateIsConsultant(userId: number, isConsultant: boolean): Promise; getWorker(key: string): Promise; getWorkerFull(key: string): Promise; getWorkerAPI(key: string): Promise; getBillingAPI(): BillingAPI; getDocAPI(docId: string): DocAPI; fetchApiKey(): Promise; createApiKey(): Promise; deleteApiKey(): Promise; getTable(docId: string, tableName: string): Promise; applyUserActions(docId: string, actions: UserAction[]): Promise; importUnsavedDoc(material: UploadType, options?: { filename?: string, timezone?: string, onUploadProgress?: (ev: AxiosProgressEvent) => void, }): Promise; deleteUser(userId: number, name: string): Promise; getBaseUrl(): string; // Get the prefix for all the endpoints this object wraps. forRemoved(): UserAPI; // Get a version of the API that works on removed resources. getWidgets(): Promise; /** * Deletes account and personal org with all documents. Note: deleteUser doesn't clear documents, and this method * is specific to Grist installation, and might not be supported. Pass current user's id so that we can verify * that the user is deleting their own account. This is just to prevent accidental deletion from multiple tabs. * * @returns true if the account was deleted, false if there was a mismatch with the current user's id, and the * account was probably already deleted. */ closeAccount(userId: number): Promise; /** * Deletes current non personal org with all documents. Note: deleteOrg doesn't clear documents, and this method * is specific to Grist installation, and might not be supported. */ closeOrg(): Promise; } /** * Parameters for the download CSV and XLSX endpoint (/download/table-schema & /download/csv & /download/csv). */ export interface DownloadDocParams { tableId: string; viewSection?: number; activeSortSpec?: string; filters?: string; } export const CreatableArchiveFormats = StringUnion("zip", "tar"); export type CreatableArchiveFormats = typeof CreatableArchiveFormats.type; export interface AttachmentsArchiveParams { format?: CreatableArchiveFormats, } export interface ArchiveUploadResult { added: number; errored: number; unused: number; } interface GetRowsParams { filters?: QueryFilters; immediate?: boolean; } interface SqlResult extends TableRecordValuesWithoutIds { statement: string; } export const DocAttachmentsLocation = StringUnion( "none", "internal", "mixed", "external", ); export type DocAttachmentsLocation = typeof DocAttachmentsLocation.type; export const ExpandTableOption = StringUnion("column"); export type ExpandTableOption = typeof ExpandTableOption.type; interface GetTablesParams { expand?: ExpandTableOption[]; } /** * Collect endpoints related to the content of a single document that we've been thinking * of as the (restful) "Doc API". A few endpoints that could be here are not, for historical * reasons, such as downloads. */ export interface DocAPI { readonly options: IOptions; getBaseUrl(): string; getTables(options?: GetTablesParams): Promise; // Immediate flag is a currently not-advertised feature, allowing a query to proceed without // waiting for a document to be initialized. This is useful if the calculations done when // opening a document are irrelevant. getRows(tableId: string, options?: GetRowsParams): Promise; getRecords(tableId: string, options?: GetRowsParams): Promise; sql(sql: string, args?: any[]): Promise; updateRows(tableId: string, changes: TableColValues): Promise; addRows(tableId: string, additions: BulkColValues): Promise; removeRows(tableId: string, removals: number[]): Promise; fork(): Promise; replace(source: DocReplacementOptions): Promise; // Get list of document versions (specify raw to bypass caching, which should only make // a difference if snapshots have "leaked") getSnapshots(raw?: boolean): Promise; // remove selected snapshots, or all snapshots that have "leaked" from inventory (should // be empty), or all but the current snapshot. removeSnapshots(snapshotIds: string[] | "unlisted" | "past"): Promise<{ snapshotIds: string[] }>; getStates(): Promise; forceReload(): Promise; recover(recoveryMode: boolean): Promise; // Compare two documents, optionally including details of the changes. compareDoc( remoteDocId: string, options?: { detail?: boolean; maxRows?: number | null } ): Promise; // Compare two versions within a document, including details of the changes. // Versions are identified by action hashes, or aliases understood by HashUtil. // Currently, leftHash is expected to be an ancestor of rightHash. If rightHash // is HEAD, the result will contain a copy of any rows added or updated. compareVersion(leftHash: string, rightHash: string): Promise; getDownloadUrl(options: { template: boolean, removeHistory: boolean }): string; getDownloadXlsxUrl(params?: DownloadDocParams): string; getDownloadCsvUrl(params: DownloadDocParams): string; getDownloadTsvUrl(params: DownloadDocParams): string; getDownloadDsvUrl(params: DownloadDocParams): string; getDownloadTableSchemaUrl(params: DownloadDocParams): string; getDownloadAttachmentsArchiveUrl(params: AttachmentsArchiveParams): string; /** * Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first * acquire "code" via Google Auth Endpoint (see ShareMenu.ts for an example). * @param code Authorization code returned from Google (requested via Grist's Google Auth Endpoint) * @param title Name of the spreadsheet that will be created (should use a Grist document's title) */ sendToDrive(code: string, title: string): Promise<{ url: string }>; // Upload a single attachment and return the resulting metadata row ID. // The arguments are passed to FormData.append. uploadAttachment(value: string | Blob, filename?: string): Promise; uploadAttachmentArchive(archive: string | Blob, filename?: string): Promise; // Get users that are worth proposing to "View As" for access control purposes. getUsersForViewAs(): Promise; getWebhooks(): Promise; addWebhook(webhook: WebhookFields): Promise<{ webhookId: string }>; removeWebhook(webhookId: string, tableId: string): Promise; // Update webhook updateWebhook(webhook: WebhookUpdate): Promise; flushWebhooks(): Promise; flushWebhook(webhookId: string): Promise; getAssistance(params: AssistanceRequest): Promise; /** * Check if the document is currently in timing mode. * Status is either * - 'active' if timings are enabled. * - 'pending' if timings are enabled but we can't get the data yet (as engine is blocked) * - 'disabled' if timings are disabled. */ timing(): Promise; /** * Starts recording timing information for the document. Throws exception if timing is already * in progress or you don't have permission to start timing. */ startTiming(): Promise; stopTiming(): Promise; /** * Starts the transfer of all attachments from the old attachment storage to the new one. */ transferAllAttachments(): Promise; /** * Returns the status of the attachment transfer. */ getAttachmentTransferStatus(): Promise; /** * Retries type of attachment storage used by the document. */ getAttachmentStore(): Promise<{ type: AttachmentStore }>; /** * Sets the attachment storage used by the document. */ setAttachmentStore(type: AttachmentStore): Promise; /** * Lists available external attachment stores. For now it contains at most one store. * If there is one store available it means that external storage is configured and can be used by this document. */ getAttachmentStores(): Promise<{ stores: AttachmentStoreDesc[] }>; makeProposal(options?: { retracted?: boolean, }): Promise; getProposals(options?: { outgoing?: boolean }): Promise<{ proposals: Proposal[] }>; applyProposal(proposalId: number): Promise; applyUserActions(actions: UserAction[]): Promise; } // Operations that are supported by a doc worker. export interface DocWorkerAPI { readonly url: string; importDocToWorkspace(uploadId: number, workspaceId: number, settings?: BrowserSettings): Promise; upload(material: UploadType, filename?: string): Promise; downloadDoc(docId: string, template?: boolean): Promise; copyDoc(docId: string, template?: boolean, name?: string): Promise; } export class UserAPIImpl extends BaseAPI implements UserAPI { constructor(private _homeUrl: string, private _options: IOptions = {}) { super(_options); } public forRemoved(): UserAPI { const extraParameters = new Map([["showRemoved", "1"]]); return new UserAPIImpl(this._homeUrl, { ...this._options, extraParameters }); } public async getSessionActive(): Promise { return this.requestJson(`${this._url}/api/session/access/active`, { method: "GET" }); } public async setSessionActive(email: string, org?: string): Promise { const body = JSON.stringify({ email, org }); return this.requestJson(`${this._url}/api/session/access/active`, { method: "POST", body }); } public async getSessionAll(): Promise<{ users: FullUser[], orgs: Organization[] }> { return this.requestJson(`${this._url}/api/session/access/all`, { method: "GET" }); } public async getOrgs(merged: boolean = false): Promise { return this.requestJson(`${this._url}/api/orgs?merged=${merged ? 1 : 0}`, { method: "GET" }); } public async getWorkspace(workspaceId: number): Promise { return this.requestJson(`${this._url}/api/workspaces/${workspaceId}`, { method: "GET" }); } public async getOrg(orgId: number | string): Promise { return this.requestJson(`${this._url}/api/orgs/${orgId}`, { method: "GET" }); } public async getOrgWorkspaces(orgId: number | string, includeSupport = true): Promise { return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces?includeSupport=${includeSupport ? 1 : 0}`, { method: "GET" }); } public async getOrgUsageSummary(orgId: number | string): Promise { return this.requestJson(`${this._url}/api/orgs/${orgId}/usage`, { method: "GET" }); } public async getTemplates(): Promise { return this.requestJson(`${this._url}/api/templates`, { method: "GET" }); } public async getTemplate(docId: string): Promise { return this.requestJson(`${this._url}/api/templates/${docId}`, { method: "GET" }); } public async getWidgets(): Promise { return await this.requestJson(`${this._url}/api/widgets`, { method: "GET" }); } public async getDoc(docId: string): Promise { return this.requestJson(`${this._url}/api/docs/${docId}`, { method: "GET" }); } public async newOrg(props: Partial): Promise { return this.requestJson(`${this._url}/api/orgs`, { method: "POST", body: JSON.stringify(props), }); } public async newWorkspace(props: Partial, orgId: number | string): Promise { return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces`, { method: "POST", body: JSON.stringify(props), }); } public async newDoc(props: Partial, workspaceId: number): Promise { return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/docs`, { method: "POST", body: JSON.stringify(props), }); } public async newUnsavedDoc(options: { timezone?: string } = {}): Promise { return this.requestJson(`${this._url}/api/docs`, { method: "POST", body: JSON.stringify(options), }); } public async copyDoc( sourceDocumentId: string, workspaceId: number, options: CopyDocOptions, ): Promise { return this.requestJson(`${this._url}/api/docs`, { method: "POST", body: JSON.stringify({ sourceDocumentId, workspaceId, ...options, }), }); } public async renameOrg(orgId: number | string, name: string): Promise { await this.request(`${this._url}/api/orgs/${orgId}`, { method: "PATCH", body: JSON.stringify({ name }), }); } public async renameWorkspace(workspaceId: number, name: string): Promise { await this.request(`${this._url}/api/workspaces/${workspaceId}`, { method: "PATCH", body: JSON.stringify({ name }), }); } public async renameDoc(docId: string, name: string, { icon }: RenameDocOptions = {}): Promise { return this.updateDoc(docId, { name, ...(icon ? { options: { appearance: { icon } } } : undefined), }); } public async updateOrg(orgId: number | string, props: Partial): Promise { await this.request(`${this._url}/api/orgs/${orgId}`, { method: "PATCH", body: JSON.stringify(props), }); } public async updateDoc(docId: string, props: Partial): Promise { await this.request(`${this._url}/api/docs/${docId}`, { method: "PATCH", body: JSON.stringify(props), }); } public async deleteOrg(orgId: number | string): Promise { await this.request(`${this._url}/api/orgs/${orgId}/force-delete`, { method: "DELETE" }); } public async deleteWorkspace(workspaceId: number): Promise { await this.request(`${this._url}/api/workspaces/${workspaceId}`, { method: "DELETE" }); } public async softDeleteWorkspace(workspaceId: number): Promise { await this.request(`${this._url}/api/workspaces/${workspaceId}/remove`, { method: "POST" }); } public async undeleteWorkspace(workspaceId: number): Promise { await this.request(`${this._url}/api/workspaces/${workspaceId}/unremove`, { method: "POST" }); } public async deleteDoc(docId: string): Promise { await this.request(`${this._url}/api/docs/${docId}`, { method: "DELETE" }); } public async softDeleteDoc(docId: string): Promise { await this.request(`${this._url}/api/docs/${docId}/remove`, { method: "POST" }); } public async undeleteDoc(docId: string): Promise { await this.request(`${this._url}/api/docs/${docId}/unremove`, { method: "POST" }); } public async disableDoc(docId: string): Promise { await this.request(`${this._url}/api/docs/${docId}/disable`, { method: "POST" }); } public async enableDoc(docId: string): Promise { await this.request(`${this._url}/api/docs/${docId}/enable`, { method: "POST" }); } public async updateOrgPermissions(orgId: number | string, delta: PermissionDelta): Promise { await this.request(`${this._url}/api/orgs/${orgId}/access`, { method: "PATCH", body: JSON.stringify({ delta }), }); } public async updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise { await this.request(`${this._url}/api/workspaces/${workspaceId}/access`, { method: "PATCH", body: JSON.stringify({ delta }), }); } public async updateDocPermissions(docId: string, delta: PermissionDelta): Promise { await this.request(`${this._url}/api/docs/${docId}/access`, { method: "PATCH", body: JSON.stringify({ delta }), }); } public async getOrgAccess(orgId: number | string): Promise { return this.requestJson(`${this._url}/api/orgs/${orgId}/access`, { method: "GET" }); } public async getWorkspaceAccess(workspaceId: number): Promise { return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/access`, { method: "GET" }); } public async getDocAccess(docId: string): Promise { return this.requestJson(`${this._url}/api/docs/${docId}/access`, { method: "GET" }); } public async pinDoc(docId: string): Promise { await this.request(`${this._url}/api/docs/${docId}/pin`, { method: "PATCH", }); } public async unpinDoc(docId: string): Promise { await this.request(`${this._url}/api/docs/${docId}/unpin`, { method: "PATCH", }); } public async moveDoc(docId: string, workspaceId: number): Promise { await this.request(`${this._url}/api/docs/${docId}/move`, { method: "PATCH", body: JSON.stringify({ workspace: workspaceId }), }); } public async getUserProfile(): Promise { return this.requestJson(`${this._url}/api/profile/user`); } public async updateUserName(name: string): Promise { await this.request(`${this._url}/api/profile/user/name`, { method: "POST", body: JSON.stringify({ name }), }); } public async updateUserLocale(locale: string | null): Promise { await this.request(`${this._url}/api/profile/user/locale`, { method: "POST", body: JSON.stringify({ locale }), }); } public async updateAllowGoogleLogin(allowGoogleLogin: boolean): Promise { await this.request(`${this._url}/api/profile/allowGoogleLogin`, { method: "POST", body: JSON.stringify({ allowGoogleLogin }), }); } public async updateIsConsultant(userId: number, isConsultant: boolean): Promise { await this.request(`${this._url}/api/profile/isConsultant`, { method: "POST", body: JSON.stringify({ userId, isConsultant }), }); } public async disableUser(userId: number): Promise { await this.request(`${this._url}/api/users/${userId}/disable`, { method: "POST", }); } public async enableUser(userId: number): Promise { await this.request(`${this._url}/api/users/${userId}/enable`, { method: "POST", }); } public async getWorker(key: string): Promise { const full = await this.getWorkerFull(key); return getPublicDocWorkerUrl(this._homeUrl, full); } public async getWorkerFull(key: string): Promise { const json = (await this.requestJson(`${this._url}/api/worker/${key}`, { method: "GET", credentials: "include", })) as PublicDocWorkerUrlInfo; return json; } public async getWorkerAPI(key: string): Promise { const docUrl = this._urlWithOrg(await this.getWorker(key)); return new DocWorkerAPIImpl(docUrl, this._options); } public getBillingAPI(): BillingAPI { return new BillingAPIImpl(this._url, this._options); } public getDocAPI(docId: string): DocAPI { return new DocAPIImpl(this._url, docId, this._options); } public async fetchApiKey(): Promise { const resp = await this.request(`${this._url}/api/profile/apiKey`); return await resp.text(); } public async createApiKey(): Promise { const res = await this.request(`${this._url}/api/profile/apiKey`, { method: "POST", }); return await res.text(); } public async deleteApiKey(): Promise { await this.request(`${this._url}/api/profile/apiKey`, { method: "DELETE", }); } // This method is not strictly needed anymore, but is widely used by // tests so supporting as a handy shortcut for getDocAPI(docId).getRows(tableName) public async getTable(docId: string, tableName: string): Promise { return this.getDocAPI(docId).getRows(tableName); } public async applyUserActions(docId: string, actions: UserAction[]): Promise { return this.requestJson(`${this._url}/api/docs/${docId}/apply`, { method: "POST", body: JSON.stringify(actions), }); } public async importUnsavedDoc(material: UploadType, options?: { filename?: string, timezone?: string, onUploadProgress?: (ev: AxiosProgressEvent) => void, }): Promise { options = options || {}; const formData = this.newFormData(); formData.append("upload", material as any, options.filename); if (options.timezone) { formData.append("timezone", options.timezone); } const resp = await this.requestAxios(`${this._url}/api/docs`, { method: "POST", data: formData, onUploadProgress: options.onUploadProgress, // On browser, it is important not to set Content-Type so that the browser takes care // of setting HTTP headers appropriately. Outside browser, requestAxios has logic // for setting the HTTP headers. headers: { ...this.defaultHeadersWithoutContentType() }, }); return resp.data; } public async deleteUser(userId: number, name: string) { await this.request(`${this._url}/api/users/${userId}`, { method: "DELETE", body: JSON.stringify({ name }) }); } public async closeAccount(userId: number): Promise { return await this.requestJson(`${this._url}/api/doom/account?userid=` + userId, { method: "DELETE" }); } public async closeOrg() { await this.request(`${this._url}/api/doom/org`, { method: "DELETE" }); } public getBaseUrl(): string { return this._url; } // Recomputes the URL on every call to pick up changes in the URL when switching orgs. // (Feels inefficient, but probably doesn't matter, and it's simpler than the alternatives.) private get _url(): string { return this._urlWithOrg(this._homeUrl); } private _urlWithOrg(base: string): string { return isClient() ? addCurrentOrgToPath(base) : base.replace(/\/$/, ""); } } export class DocWorkerAPIImpl extends BaseAPI implements DocWorkerAPI { constructor(public readonly url: string, _options: IOptions = {}) { super(_options); } public async importDocToWorkspace(uploadId: number, workspaceId: number, browserSettings?: BrowserSettings): Promise { return this.requestJson(`${this.url}/api/workspaces/${workspaceId}/import`, { method: "POST", body: JSON.stringify({ uploadId, browserSettings }), }); } public async upload(material: UploadType, filename?: string): Promise { const formData = this.newFormData(); formData.append("upload", material as any, filename); const json = await this.requestJson(`${this.url}/uploads`, { // On browser, it is important not to set Content-Type so that the browser takes care // of setting HTTP headers appropriately. Outside of browser, node-fetch also appears // to take care of this - https://github.github.io/fetch/#request-body headers: { ...this.defaultHeadersWithoutContentType() }, method: "POST", body: formData, }); return json.uploadId; } public async downloadDoc(docId: string, template: boolean = false): Promise { const extra = template ? "?template=1" : ""; const result = await this.request(`${this.url}/api/docs/${docId}/download${extra}`, { method: "GET", }); if (!result.ok) { throw new Error(await result.text()); } return result; } public async copyDoc(docId: string, template: boolean = false, name?: string): Promise { const url = new URL(`${this.url}/copy?doc=${docId}`); if (template) { url.searchParams.append("template", "1"); } if (name) { url.searchParams.append("name", name); } const json = await this.requestJson(url.href, { method: "POST", }); return json.uploadId; } } export class DocAPIImpl extends BaseAPI implements DocAPI { private _url: string; constructor(url: string, public readonly docId: string, options: IOptions = {}) { super(options); this._url = `${url}/api/docs/${docId}`; } public getBaseUrl(): string { return this._url; } public async getTables(options?: GetTablesParams): Promise { const url = new URL(`${this._url}/tables`); if (options?.expand) { url.searchParams.set("expand", options.expand.join(",")); } return this.requestJson(url.href); } public async getRows(tableId: string, options?: GetRowsParams): Promise { return this._getRecords(tableId, "data", options); } public async getRecords(tableId: string, options?: GetRowsParams): Promise { const response: TableRecordValues = await this._getRecords(tableId, "records", options); return response.records; } public async sql(sql: string, args?: any[]): Promise { return this.requestJson(`${this._url}/sql`, { body: JSON.stringify({ sql, ...(args ? { args } : {}), }), method: "POST", }); } public async updateRows(tableId: string, changes: TableColValues): Promise { return this.requestJson(`${this._url}/tables/${tableId}/data`, { body: JSON.stringify(changes), method: "PATCH", }); } /** * Adds rows to a table. * * Example: * ```typescript * const newRowIds = await docApi.addRows("tableId", { * "Column1": [value1, value2], * "Column2": [value3, value4], * }); * ``` * @param tableId Table ID to add rows to. * @param additions JSON object with column values for the new rows. * @returns Array of new row IDs. */ public async addRows(tableId: string, additions: BulkColValues): Promise { return this.requestJson(`${this._url}/tables/${tableId}/data`, { body: JSON.stringify(additions), method: "POST", }); } public async removeRows(tableId: string, removals: number[]): Promise { return this.requestJson(`${this._url}/tables/${tableId}/records/delete`, { body: JSON.stringify(removals), method: "POST", }); } public async fork(): Promise { return this.requestJson(`${this._url}/fork`, { method: "POST", }); } public async replace(source: DocReplacementOptions): Promise { return this.requestJson(`${this._url}/replace`, { body: JSON.stringify(source), method: "POST", }); } public async getSnapshots(raw?: boolean): Promise { return this.requestJson(`${this._url}/snapshots?raw=${raw}`); } public async removeSnapshots(snapshotIds: string[] | "unlisted" | "past") { const body = typeof snapshotIds === "string" ? { select: snapshotIds } : { snapshotIds }; return await this.requestJson(`${this._url}/snapshots/remove`, { method: "POST", body: JSON.stringify(body), }); } public async getStates(): Promise { return this.requestJson(`${this._url}/states`); } public async getUsersForViewAs(): Promise { return this.requestJson(`${this._url}/usersForViewAs`); } public async getWebhooks(): Promise { return this.requestJson(`${this._url}/webhooks`); } public async addWebhook(webhook: WebhookSubscribe & { tableId: string }): Promise<{ webhookId: string }> { const { tableId } = webhook; return this.requestJson(`${this._url}/tables/${tableId}/_subscribe`, { method: "POST", body: JSON.stringify( omitBy(webhook, (val, key) => key === "tableId" || val === null)), }); } public async updateWebhook(webhook: WebhookUpdate): Promise { return this.requestJson(`${this._url}/webhooks/${webhook.id}`, { method: "PATCH", body: JSON.stringify(webhook.fields), }); } public removeWebhook(webhookId: string, tableId: string) { // unsubscribeKey is not required for owners const unsubscribeKey = ""; return this.requestJson(`${this._url}/tables/${tableId}/_unsubscribe`, { method: "POST", body: JSON.stringify({ webhookId, unsubscribeKey }), }); } public async flushWebhooks(): Promise { await this.request(`${this._url}/webhooks/queue`, { method: "DELETE", }); } public async flushWebhook(id: string): Promise { await this.request(`${this._url}/webhooks/queue/${id}`, { method: "DELETE", }); } public async forceReload(): Promise { await this.request(`${this._url}/force-reload`, { method: "POST", }); } public async recover(recoveryMode: boolean): Promise { await this.request(`${this._url}/recover`, { body: JSON.stringify({ recoveryMode }), method: "POST", }); } public async compareDoc( remoteDocId: string, options: { detail?: boolean; maxRows?: number | null; } = {}, ): Promise { const { detail, maxRows } = options; const url = new URL(`${this._url}/compare/${remoteDocId}`); if (detail) { url.searchParams.set("detail", "true"); } if (maxRows !== undefined) { url.searchParams.set("maxRows", String(maxRows)); } return this.requestJson(url.href); } public async copyDoc(workspaceId: number, options: CopyDocOptions): Promise { const { documentName, asTemplate } = options; return this.requestJson(`${this._url}/copy`, { body: JSON.stringify({ workspaceId, documentName, asTemplate }), method: "POST", }); } public async compareVersion(leftHash: string, rightHash: string): Promise { const url = new URL(`${this._url}/compare`); url.searchParams.append("left", leftHash); url.searchParams.append("right", rightHash); return this.requestJson(url.href); } public getDownloadUrl({ template, removeHistory}: { template: boolean, removeHistory: boolean }): string { return this._url + `/download?template=${template}&nohistory=${removeHistory}`; } public getDownloadXlsxUrl(params: DownloadDocParams) { return this._url + "/download/xlsx?" + encodeQueryParams({ ...params }); } public getDownloadCsvUrl(params: DownloadDocParams) { // We spread `params` to work around TypeScript being overly cautious. return this._url + "/download/csv?" + encodeQueryParams({ ...params }); } public getDownloadTsvUrl(params: DownloadDocParams) { return this._url + "/download/tsv?" + encodeQueryParams({ ...params }); } public getDownloadDsvUrl(params: DownloadDocParams) { return this._url + "/download/dsv?" + encodeQueryParams({ ...params }); } public getDownloadTableSchemaUrl(params: DownloadDocParams) { // We spread `params` to work around TypeScript being overly cautious. return this._url + "/download/table-schema?" + encodeQueryParams({ ...params }); } public getDownloadAttachmentsArchiveUrl(params: AttachmentsArchiveParams): string { return this._url + "/attachments/archive?" + encodeQueryParams({ ...params }); } public async sendToDrive(code: string, title: string): Promise<{ url: string }> { const url = new URL(`${this._url}/send-to-drive`); url.searchParams.append("title", title); url.searchParams.append("code", code); return this.requestJson(url.href); } public async uploadAttachment(value: string | Blob, filename?: string): Promise { const formData = this.newFormData(); formData.append("upload", value, filename); const response = await this.requestAxios(`${this._url}/attachments`, { method: "POST", data: formData, // On browser, it is important not to set Content-Type so that the browser takes care // of setting HTTP headers appropriately. Outside browser, requestAxios has logic // for setting the HTTP headers. headers: { ...this.defaultHeadersWithoutContentType() }, }); return response.data[0]; } public async uploadAttachmentArchive(archive: string | Blob, filename?: string): Promise { const formData = this.newFormData(); formData.append("upload", archive, filename); const response = await this.requestAxios(`${this._url}/attachments/archive`, { method: "POST", data: formData, // On the browser, Content-Type shouldn't be set as it prevents the browser from setting // Content-Type with the correct boundary expression to delimit form fields. // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest_API/Using_FormData_Objects#sending_files_using_a_formdata_object // Therefore we omit Content-Type, and allow Axios to handle it as it sees fit - which works // correctly in the browser and in Node. headers: { ...this.defaultHeadersWithoutContentType() }, }); return response.data; } public async getAssistance( params: AssistanceRequest, ): Promise { return await this.requestJson(`${this._url}/assistant`, { method: "POST", body: JSON.stringify(params), }); } public async timing(): Promise { return this.requestJson(`${this._url}/timing`); } public async startTiming(): Promise { await this.request(`${this._url}/timing/start`, { method: "POST" }); } public async stopTiming(): Promise { return await this.requestJson(`${this._url}/timing/stop`, { method: "POST" }); } public async transferAllAttachments(): Promise { await this.request(`${this._url}/attachments/transferAll`, { method: "POST" }); } public async getAttachmentTransferStatus(): Promise { return this.requestJson(`${this._url}/attachments/transferStatus`); } public async getAttachmentStore(): Promise<{ type: AttachmentStore }> { return this.requestJson(`${this._url}/attachments/store`); } public async getAttachmentStores(): Promise<{ stores: AttachmentStoreDesc[] }> { return this.requestJson(`${this._url}/attachments/stores`); } public async setAttachmentStore(type: AttachmentStore): Promise { await this.request(`${this._url}/attachments/store`, { method: "POST", body: JSON.stringify({ type }), }); } public async makeProposal(options?: { retracted?: boolean, }) { return this.requestJson(`${this._url}/propose`, { method: "POST", body: JSON.stringify(options || {}), }); } public async applyProposal(proposalId: number) { return this.requestJson(`${this._url}/proposals/${proposalId}/apply`, { method: "POST", }); } public async applyUserActions(actions: UserAction[]): Promise { return this.requestJson(`${this._url}/apply`, { method: "POST", body: JSON.stringify(actions), }); } public async getProposals(options?: { outgoing?: boolean, }) { const result = await this.requestJson(`${this._url}/proposals?outgoing=${Boolean(options?.outgoing)}`, { method: "GET", }); return result; } private _getRecords(tableId: string, endpoint: "data" | "records", options?: GetRowsParams): Promise { const url = new URL(`${this._url}/tables/${tableId}/${endpoint}`); if (options?.filters) { url.searchParams.append("filter", JSON.stringify(options.filters)); } if (options?.immediate) { url.searchParams.append("immediate", "true"); } return this.requestJson(url.href); } } export interface AttachmentTransferStatus { status: { pendingTransferCount: number; isRunning: boolean; // Count of failures and successes since starting a // transfer of all files. failures: number; successes: number; }; locationSummary: DocAttachmentsLocation; } /** * Represents information to build public doc worker url. * * Structure that may contain either **exclusively**: * - a selfPrefix when no pool of doc worker exist. * - a public doc worker url otherwise. */ export type PublicDocWorkerUrlInfo = { selfPrefix: string; docWorkerUrl: null; docWorkerId: null; } | { selfPrefix: null; docWorkerUrl: string; docWorkerId: string; }; export function getUrlFromPrefix(homeUrl: string, prefix: string) { const url = new URL(homeUrl); url.pathname = prefix + url.pathname; return url.href; } /** * Get a docWorkerUrl from information returned from backend. When the backend * is fully configured, and there is a pool of workers, this is straightforward, * just return the docWorkerUrl reported by the backend. For single-instance * installs, the backend returns a null docWorkerUrl, and a client can simply * use the homeUrl of the backend, with extra path prefix information * given by selfPrefix. At the time of writing, the selfPrefix contains a * doc-worker id, and a tag for the codebase (used in consistency checks). * * @param {string} homeUrl * @param {string} docWorkerInfo The information to build the public doc worker url * (result of the call to /api/worker/:docId) */ export function getPublicDocWorkerUrl(homeUrl: string, docWorkerInfo: PublicDocWorkerUrlInfo) { return docWorkerInfo.selfPrefix !== null ? getUrlFromPrefix(homeUrl, docWorkerInfo.selfPrefix) : docWorkerInfo.docWorkerUrl; } ================================================ FILE: app/common/UserConfig.ts ================================================ /* * Interface for the user's config found in config.json. */ export interface UserConfig { docListSortBy?: string; docListSortDir?: number; features?: ISupportedFeatures; /* * The host serving the untrusted content: on dev environment could be * "http://getgrist.localtest.me". Port is added at runtime and should not be included. */ untrustedContentOrigin?: string; } export interface ISupportedFeatures { signin?: boolean; sharing?: boolean; proxy?: boolean; // If true, Grist will accept login information via http headers // X-Forwarded-User and X-Forwarded-Email. Set to true only if // Grist is behind a reverse proxy that is managing those headers, // otherwise they could be spoofed. formulaBar?: boolean; // Plugin views, and REPL all need work, but are exposed here to allow existing // tests to continue running. These only affect client-side code. customViewPlugin?: boolean; } ================================================ FILE: app/common/ValueConverter.ts ================================================ import { TableDataActionSet } from "app/common/DocActions"; import { DocData } from "app/common/DocData"; import * as gristTypes from "app/common/gristTypes"; import { isList } from "app/common/gristTypes"; import { BaseFormatter, createFullFormatterFromDocData } from "app/common/ValueFormatter"; import { createParserOrFormatterArgumentsRaw, createParserRaw, ReferenceListParser, ReferenceParser, ValueParser, } from "app/common/ValueParser"; import { CellValue, GristObjCode } from "app/plugin/GristData"; /** * Base class for converting values from one type to another with the convert() method. * Has a formatter for the source column * and a parser for the destination column. * * The default convert() is for non-list destination types, so if the source value * is a list it only converts nicely if the list contains exactly one element. */ export class ValueConverter { private _isTargetText: boolean = ["Text", "Choice"].includes(this.parser.type); constructor(public formatter: BaseFormatter, public parser: ValueParser) { } public convert(value: any): any { if (isList(value)) { if (value.length === 1) { // Empty list: ['L'] return null; } else if (value.length > 2 || this._isTargetText) { // List with multiple values, or the target type is text. // Since we're converting to just one value, // format the whole thing as text, which is an error for most types. return this.formatter.formatAny(value); } else { // Singleton list: ['L', value] // Convert just that one value. value = value[1]; } } return this.convertInner(value); } protected convertInner(value: any): any { const formatted = this.formatter.formatAny(value); return this.parser.cleanParse(formatted); } } /** * Base class for converting to a list type (Reference List or Choice List). * * Wraps single values in a list, and converts lists elementwise. */ class ListConverter extends ValueConverter { // Don't parse strings like "Smith, John" which may look like lists but represent a single choice. // TODO this works when the source is a Choice column, but not when it's a Reference to a Choice column. // But the guessed choices are also broken in that case. private _choices = new Set((this.formatter.widgetOpts as any).choices || []); public convert(value: any): any { if (typeof value === "string" && !this._choices.has(value)) { // Parse CSV/JSON return this.parser.cleanParse(value); } const values = isList(value) ? value.slice(1) : [value]; if (!values.length || value == null) { return null; } return this.handleValues(value, values.map(v => this.convertInner(v))); } protected handleValues(originalValue: any, values: any[]) { return ["L", ...values]; } } class ChoiceListConverter extends ListConverter { /** * Convert each source value to a 'Choice' */ protected convertInner(value: any): any { return this.formatter.formatAny(value); } } class ReferenceListConverter extends ListConverter { private _innerConverter = new ReferenceConverter( this.formatter, new ReferenceParser("Ref", this.parser.widgetOpts, this.parser.docSettings), ); constructor(public formatter: BaseFormatter, public parser: ReferenceListParser) { super(formatter, parser); // Prevent the parser from looking up reference values in the frontend. // Leave it to the data engine which has a much more efficient algorithm for long lists of values. delete parser.tableData; } public handleValues(originalValue: any, values: any[]): any { const result = []; let lookupColumn: string = ""; const raw = this.formatter.formatAny(originalValue); // AltText if the reference lookup fails for (const value of values) { if (typeof value === "string") { // Failed to parse one of the references, so return a raw string for the whole thing return raw; } else { // value is a lookup tuple: ['l', value, options] result.push(value[1]); lookupColumn = value[2].column; } } return ["l", result, { column: lookupColumn, raw }]; } /** * Convert each source value to a 'Reference' */ protected convertInner(value: any): any { return this._innerConverter.convert(value); } } class ReferenceConverter extends ValueConverter { private _innerConverter: ValueConverter = createConverter(this.formatter, this.parser.visibleColParser); constructor(public formatter: BaseFormatter, public parser: ReferenceParser) { super(formatter, parser); // Prevent the parser from looking up reference values in the frontend. // Leave it to the data engine which has a much more efficient algorithm for long lists of values. delete parser.tableData; } protected convertInner(value: any): any { // Convert to the type of the visible column. const converted = this._innerConverter.convert(value); return this.parser.lookup(converted, this.formatter.formatAny(value)); } } class NumericConverter extends ValueConverter { protected convertInner(value: any): any { if (typeof value === "boolean") { return value ? 1 : 0; } return super.convertInner(value); } } class DateConverter extends ValueConverter { private _sourceType = gristTypes.extractInfoFromColType(this.formatter.type); protected convertInner(value: any): any { // When converting Date->DateTime, DateTime->Date, or between DateTime timezones, // it's important to send an encoded Date/DateTime object rather than just a timestamp number // so that the data engine knows what to do in do_convert, especially regarding timezones. // If the source column is a Reference to a Date/DateTime then `value` is already // an encoded object from the display column which has type Any. value = gristTypes.reencodeAsTypedCellValue(value, this._sourceType); if (Array.isArray(value) && ( value[0] === GristObjCode.Date || value[0] === GristObjCode.DateTime )) { return value; } return super.convertInner(value); } } export const valueConverterClasses: { [type: string]: typeof ValueConverter } = { Date: DateConverter, DateTime: DateConverter, ChoiceList: ChoiceListConverter, Ref: ReferenceConverter, RefList: ReferenceListConverter, Numeric: NumericConverter, Int: NumericConverter, }; export function createConverter(formatter: BaseFormatter, parser: ValueParser) { const cls = valueConverterClasses[gristTypes.extractTypeFromColType(parser.type)] || ValueConverter; return new cls(formatter, parser); } /** * Used by the ConvertFromColumn user action in the data engine. * The higher order function separates docData (passed by ActiveDoc) * from the arguments passed to call_external in Python. */ export function convertFromColumn( metaTables: TableDataActionSet, sourceColRef: number, type: string, widgetOpts: string, visibleColRef: number, values: readonly CellValue[], displayColValues?: readonly CellValue[], ): CellValue[] { const docData = new DocData( (_tableId) => { throw new Error("Unexpected DocData fetch"); }, metaTables, ); const formatter = createFullFormatterFromDocData(docData, sourceColRef); const parser = createParserRaw( ...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef), ); const converter = createConverter(formatter, parser); return convertValues(converter, values, displayColValues || values); } export function convertValues( converter: ValueConverter, // Raw values from the actual column, e.g. row IDs for reference columns values: readonly CellValue[], // Values from the display column, which is the same as the raw values for non-referencing columns. // In almost all cases these are the values that actually matter and get converted. displayColValues: readonly CellValue[], ): CellValue[] { // Converting Ref <-> RefList without changing the target table is a special case - see prepTransformColInfo. // In this case we deal with the actual row IDs stored in the real column, // whereas in all other cases we use display column values. const sourceType = gristTypes.extractInfoFromColType(converter.formatter.type); const targetType = gristTypes.extractInfoFromColType(converter.parser.type); const refToRefList = ( sourceType.type === "Ref" && targetType.type === "RefList" && sourceType.tableId === targetType.tableId ); const refListToRef = ( sourceType.type === "RefList" && targetType.type === "Ref" && sourceType.tableId === targetType.tableId ); return displayColValues.map((displayVal, i) => { const actualValue = values[i]; if (refToRefList && typeof actualValue === "number") { if (actualValue === 0) { return null; } else { return ["L", actualValue]; } } else if (refListToRef && isList(actualValue)) { if (actualValue.length === 1) { // Empty list: ['L'] return 0; } else if (actualValue.length === 2) { // Singleton list: ['L', rowId] return actualValue[1]; } } return converter.convert(displayVal); }); } ================================================ FILE: app/common/ValueFormatter.ts ================================================ import { csvEncodeRow } from "app/common/csvFormat"; import { CellValue } from "app/common/DocActions"; import { DocData } from "app/common/DocData"; import { DocumentSettings } from "app/common/DocumentSettings"; import * as gristTypes from "app/common/gristTypes"; import { getReferencedTableId, isList } from "app/common/gristTypes"; import * as gutil from "app/common/gutil"; import { isHiddenTable } from "app/common/isHiddenTable"; import { buildNumberFormat, NumberFormatOptions } from "app/common/NumberFormat"; import { createParserOrFormatterArguments, ReferenceParsingOptions } from "app/common/ValueParser"; import { GristObjCode } from "app/plugin/GristData"; import { decodeObject, GristDateTime } from "app/plugin/objtypes"; import isPlainObject from "lodash/isPlainObject"; import moment from "moment-timezone"; export { PENDING_DATA_PLACEHOLDER } from "app/plugin/objtypes"; export interface FormatOptions { [option: string]: any; } /** * Formats a value of any type generically (with no type-specific options). */ export function formatUnknown(value: CellValue): string { return formatDecoded(decodeObject(value)); } /** * Returns true if the array contains other arrays or structured objects, * indicating that the list should be formatted like JSON rather than CSV. */ function hasNestedObjects(value: any[]) { return value.some(v => typeof v === "object" && v && (Array.isArray(v) || isPlainObject(v))); } /** * Formats a decoded Grist value for displaying it. For top-level values, formats them the way we * like to see them in a cell or in, say, CSV export. * For top-level lists containing only simple values like strings and dates, formats them as a CSV row. * Nested lists and objects are formatted slightly differently, with quoted strings and ISO format for dates. */ export function formatDecoded(value: unknown, isTopLevel: boolean = true): string { if (typeof value === "object" && value) { if (Array.isArray(value)) { if (!isTopLevel || hasNestedObjects(value)) { return "[" + value.map(v => formatDecoded(v, false)).join(", ") + "]"; } else { return csvEncodeRow(value.map(v => formatDecoded(v, true)), { prettier: true }); } } else if (isPlainObject(value)) { const obj: any = value; const items = Object.keys(obj).map(k => `${JSON.stringify(k)}: ${formatDecoded(obj[k], false)}`); return "{" + items.join(", ") + "}"; } else if (isTopLevel && value instanceof GristDateTime) { return moment(value).tz(value.timezone).format("YYYY-MM-DD HH:mm:ssZ"); } return String(value); } if (isTopLevel) { return (value == null ? "" : String(value)); } return JSON.stringify(value); } export type IsRightTypeFunc = (value: CellValue) => boolean; export class BaseFormatter { protected isRightType: IsRightTypeFunc; constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) { this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) || gristTypes.isRightType("Any")!; } /** * Formats using this.format() if a value is of the right type for this formatter, or using * AnyFormatter otherwise. This method the recommended API. There is no need to override it. */ public formatAny(value: any, translate?: (val: string) => string): string { return this.isRightType(value) ? this.format(value, translate) : formatUnknown(value); } /** * Formats a value that matches the type of this formatter. This should be overridden by derived * classes to handle values in formatter-specific ways. */ protected format(value: any, _translate?: (val: string) => string): string { return String(value); } } export class BoolFormatter extends BaseFormatter { public format(value: boolean | 0 | 1, translate?: (val: string) => string): string { if (typeof value === "boolean" && translate) { return translate(String(value)); } return super.format(value, translate); } } class AnyFormatter extends BaseFormatter { public format(value: any): string { return formatUnknown(value); } } export class NumericFormatter extends BaseFormatter { private _numFormat: Intl.NumberFormat; private _formatter: (val: number) => string; constructor(type: string, options: NumberFormatOptions, docSettings: DocumentSettings) { super(type, options, docSettings); this._numFormat = buildNumberFormat(options, docSettings); this._formatter = (options.numSign === "parens") ? this._formatParens : this._formatPlain; } public format(value: any): string { return value === null ? "" : this._formatter(value); } public _formatPlain(value: number): string { return this._numFormat.format(value); } public _formatParens(value: number): string { // Surround positive numbers with spaces to align them visually to parenthesized numbers. return (value >= 0) ? ` ${this._numFormat.format(value)} ` : `(${this._numFormat.format(-value)})`; } } class IntFormatter extends NumericFormatter { constructor(type: string, opts: FormatOptions, docSettings: DocumentSettings) { super(type, { decimals: 0, ...opts }, docSettings); } } export interface DateFormatOptions { dateFormat?: string; } class DateFormatter extends BaseFormatter { protected _dateTimeFormat: string; private _timezone: string; constructor(type: string, widgetOpts: DateFormatOptions, docSettings: DocumentSettings, timezone: string = "UTC") { super(type, widgetOpts, docSettings); // Allow encoded dates/datetimes ([d, number] or [D, number, timezone]) // which are found in formula columns of type Any, // particularly reference display columns which are formatted here according to the visible column // which will have the correct column type and options. // Since these encoded objects are not expected in a Date/Datetime column and require // being handled differently from just a number, // we don't change `gristTypes.isRightType` which is used elsewhere. this.isRightType = (value: any) => ( value === null || typeof value === "number" || Array.isArray(value) && ( value[0] === GristObjCode.Date || value[0] === GristObjCode.DateTime ) ); this._dateTimeFormat = widgetOpts.dateFormat || "YYYY-MM-DD"; this._timezone = timezone; } public format(value: any): string { if (value === null) { return ""; } // For a DateTime object in an Any column, use the provided timezone (`value[2]`) // Otherwise use the timezone configured for a DateTime column. let timezone = this._timezone; if (Array.isArray(value)) { timezone = value[2] || timezone; value = value[1]; } // Now `value` is a number const time = moment.tz(value * 1000, timezone); return time.format(this._dateTimeFormat); } } export interface DateTimeFormatOptions extends DateFormatOptions { timeFormat?: string; } class DateTimeFormatter extends DateFormatter { constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) { const timezone = gutil.removePrefix(type, "DateTime:") || ""; // Pass up the original widgetOpts. It's helpful to have them available; e.g. ExcelFormatter // takes options from an initialized ValueFormatter. super(type, widgetOpts, docSettings, timezone); const timeFormat = widgetOpts.timeFormat === undefined ? "h:mma" : widgetOpts.timeFormat; this._dateTimeFormat = (widgetOpts.dateFormat || "YYYY-MM-DD") + " " + timeFormat; } } class RowIdFormatter extends BaseFormatter { public widgetOpts: { tableId: string }; public format(value: number): string { return value > 0 ? `${this.widgetOpts.tableId}[${value}]` : ""; } } interface ReferenceFormatOptions { visibleColFormatter?: BaseFormatter; } class ReferenceFormatter extends BaseFormatter { public widgetOpts: ReferenceFormatOptions; protected visibleColFormatter: BaseFormatter; constructor(type: string, widgetOpts: ReferenceFormatOptions, docSettings: DocumentSettings) { super(type, widgetOpts, docSettings); // widgetOpts.visibleColFormatter shouldn't be undefined, but it can be if a referencing column // is displaying another referencing column, which is partially prohibited in the UI but still possible. this.visibleColFormatter = widgetOpts.visibleColFormatter || createFormatter("Id", { tableId: getReferencedTableId(type) }, docSettings); } public formatAny(value: any): string { /* An invalid value in a referencing column is saved as a string and becomes AltText in the data engine. Then the display column formula (e.g. $person.first_name) raises an InvalidTypedValue trying to access an attribute of that AltText. This would normally lead to the formatter displaying `#Invalid Ref[List]: ` before the string value. That's inconsistent with how the cell is displayed (just the string value in pink) and with how invalid values in other columns are formatted (just the string). It's just a result of the formatter receiving a value from the display column, not the actual column. It's also likely to inconvenience users trying to import/migrate/convert data. So we suppress the error here and just show the text. It's still technically possible for the column to display an actual InvalidTypedValue exception from a formula and this will suppress that too, but this is unlikely and seems worth it. */ if ( Array.isArray(value) && value[0] === GristObjCode.Exception && value[1] === "InvalidTypedValue" && value[2]?.startsWith?.("Ref") ) { return value[3]; } return this.formatNotInvalidRef(value); } protected formatNotInvalidRef(value: any) { return this.visibleColFormatter.formatAny(value); } } class ReferenceListFormatter extends ReferenceFormatter { protected formatNotInvalidRef(value: any): string { // Part of this repeats the logic in BaseFormatter.formatAny which is overridden in ReferenceFormatter // It also ensures that complex lists (e.g. if this RefList is displaying a ChoiceList) // are formatted as JSON instead of CSV. if (!isList(value) || hasNestedObjects(decodeObject(value) as CellValue[])) { return formatUnknown(value); } // In the most common case, lists of simple objects like strings or dates // are formatted like a CSV. // This is similar to formatUnknown except the inner values are // formatted according to the visible column options. const formattedValues = value.slice(1).map(v => super.formatNotInvalidRef(v)); return csvEncodeRow(formattedValues, { prettier: true }); } } const formatters: { [name: string]: typeof BaseFormatter } = { Numeric: NumericFormatter, Int: IntFormatter, Bool: BoolFormatter, Date: DateFormatter, DateTime: DateTimeFormatter, Ref: ReferenceFormatter, RefList: ReferenceListFormatter, Id: RowIdFormatter, // We don't list anything that maps to AnyFormatter, since that's the default. }; /** * Takes column type, widget options and document settings, and returns a constructor * with a format function that can properly convert a value passed to it into the * right format for that column. */ export function createFormatter(type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings): BaseFormatter { const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter; return new ctor(type, widgetOpts, docSettings); } export interface FullFormatterArgs { docData: DocData; type: string; widgetOpts: FormatOptions; visibleColType: string; visibleColWidgetOpts: FormatOptions; docSettings: DocumentSettings; } /** * Returns a constructor * with a format function that can properly convert a value passed to it into the * right format for that column. * * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field * instead of the table column. */ export function createFullFormatterFromDocData( docData: DocData, colRef: number, fieldRef?: number, ): BaseFormatter { const [type, widgetOpts, docSettings] = createParserOrFormatterArguments(docData, colRef, fieldRef); const { visibleColType, visibleColWidgetOpts } = widgetOpts as ReferenceParsingOptions; return createFullFormatterRaw({ docData, type, widgetOpts, visibleColType, visibleColWidgetOpts, docSettings, }); } export function createFullFormatterRaw(args: FullFormatterArgs) { const { type, widgetOpts, docSettings } = args; const visibleColFormatter = createVisibleColFormatterRaw(args); return createFormatter(type, { ...widgetOpts, visibleColFormatter }, docSettings); } export function createVisibleColFormatterRaw( { docData, docSettings, type, visibleColType, visibleColWidgetOpts, widgetOpts, }: FullFormatterArgs, ): BaseFormatter { let referencedTableId = gristTypes.getReferencedTableId(type); if (!referencedTableId) { return createFormatter(type, widgetOpts, docSettings); } else if (visibleColType) { return createFormatter(visibleColType, visibleColWidgetOpts, docSettings); } else { // This column displays the Row ID, e.g. Table1[2] // Make referencedTableId empty if the table is hidden const tablesData = docData.getMetaTable("_grist_Tables"); const tableRef = tablesData.findRow("tableId", referencedTableId); if (isHiddenTable(tablesData, tableRef)) { referencedTableId = ""; } return createFormatter("Id", { tableId: referencedTableId }, docSettings); } } ================================================ FILE: app/common/ValueGuesser.ts ================================================ import { CellValue } from "app/common/DocActions"; import { DocData } from "app/common/DocData"; import { DocumentSettings } from "app/common/DocumentSettings"; import { isObject } from "app/common/gristTypes"; import { countIf } from "app/common/gutil"; import { NumberFormatOptions } from "app/common/NumberFormat"; import NumberParse from "app/common/NumberParse"; import { dateTimeWidgetOptions, guessDateFormat } from "app/common/parseDate"; import { MetaRowRecord } from "app/common/TableData"; import { createFormatter } from "app/common/ValueFormatter"; import { createParserRaw, ValueParser } from "app/common/ValueParser"; import * as moment from "moment-timezone"; interface GuessedColInfo { type: string; widgetOptions?: object; } export interface GuessResult { values?: CellValue[]; colInfo: GuessedColInfo; } type ColMetadata = Partial>; export interface GuessColMetadata { values: CellValue[]; colMetadata?: ColMetadata; // omitted if no changes are proposed. } /** * Class for guessing if an array of values should be interpreted as a specific column type. * T is the type of values that strings should be parsed to and is stored in the column. */ abstract class ValueGuesser { /** * Guessed column type and maybe widget options. */ public abstract colInfo(): GuessedColInfo; /** * Parse a single string to a typed value in such a way that formatting the value returns the original string. * If the string cannot be parsed, return the original string. */ public abstract parse(value: string): T | string; /** * Attempt to parse at least 90% the string values losslessly according to the guessed colInfo. * Return null if this cannot be done. */ public guess(values: (string | null)[], docSettings: DocumentSettings): GuessResult | null { const colInfo = this.colInfo(); const { type, widgetOptions } = colInfo; const formatter = createFormatter(type, widgetOptions || {}, docSettings); const result: any[] = []; // max number of non-parsed strings to allow before giving up const maxUnparsed = countIf(values, v => Boolean(v)) * 0.1; let unparsed = 0; for (const value of values) { if (!value) { if (this.allowBlank()) { result.push(null); continue; } else { return null; } } const parsed = this.parse(value); // Give up if too many strings failed to parse or if the parsed value changes when converted back to text if ((typeof parsed === "string" && ++unparsed > maxUnparsed) || !this.isEqualFormatted(formatter.formatAny(parsed), value)) { return null; } result.push(parsed); } return { values: result, colInfo }; } /** * Whether this type of column can store nulls directly. */ protected allowBlank(): boolean { return true; } protected isEqualFormatted(formatted1: string, formatted2: string): boolean { return formatted1 === formatted2; } } class BoolGuesser extends ValueGuesser { public colInfo(): GuessedColInfo { return { type: "Bool" }; } public parse(value: string): boolean | string { if (value === "true") { return true; } else if (value === "false") { return false; } else { return value; } } /** * This is the only type that can't store nulls, it converts them to false. */ protected allowBlank(): boolean { return false; } } class NumericGuesser extends ValueGuesser { private _parser: ValueParser; constructor(docSettings: DocumentSettings, private _options: NumberFormatOptions) { super(); this._parser = createParserRaw("Numeric", _options, docSettings); } public colInfo(): GuessedColInfo { const result: GuessedColInfo = { type: "Numeric" }; if (Object.keys(this._options).length) { result.widgetOptions = this._options; } return result; } public parse(value: string): number | string { return this._parser.cleanParse(value); } protected isEqualFormatted(formatted1: string, formatted2: string): boolean { // Consider format guessing successful if it returns the typed-in numeric value exactly or // differing only in whitespace. formatted1 = formatted1.replace(NumberParse.removeCharsRegex, ""); formatted2 = formatted2.replace(NumberParse.removeCharsRegex, ""); return formatted1 === formatted2; } } class DateGuesser extends ValueGuesser { // _format should be a full moment format string // _tz should be the document's default timezone constructor(private _format: string, private _tz: string) { super(); } public colInfo(): GuessedColInfo { const widgetOptions = dateTimeWidgetOptions(this._format, false); let type; if (widgetOptions.timeFormat) { type = "DateTime:" + this._tz; } else { type = "Date"; this._tz = "UTC"; } return { widgetOptions, type }; } // Note that this parsing is much stricter than parseDate to prevent loss of information. // Dates which can be parsed by parseDate based on the guessed widget options may not be parsed here. public parse(value: string): number | string { const m = moment.tz(value, this._format, true, this._tz); return m.isValid() ? m.valueOf() / 1000 : value; } } export function guessColInfoWithDocData(values: (string | null)[], docData: DocData) { return guessColInfo(values, docData.docSettings(), docData.docInfo().timezone); } export function guessColInfo( values: (string | null)[], docSettings: DocumentSettings, timezone: string, ): GuessResult { // Use short-circuiting of || to only do as much work as needed, // in particular not guessing date formats before trying other types. return ( new BoolGuesser() .guess(values, docSettings) || new NumericGuesser( docSettings, NumberParse.fromSettings(docSettings).guessOptions(values), ) .guess(values, docSettings) || new DateGuesser(guessDateFormat(values, timezone), timezone) .guess(values, docSettings) || // Don't return the same values back if there's no conversion to be done, // as they have to be serialized and transferred over a pipe to Python. { colInfo: { type: "Text" } } ); } /** * Guess column info for a new column, returning the metadata suitable for using with AddTable or * AddColumn user actions. In particular, widgetOptions, if any, are returned as a JSON string. * Will suggest turning the column to an empty one if all the values are empty (null or ""). */ export function guessColInfoForImports(values: CellValue[], docData: DocData): GuessColMetadata { if (values.every(v => (v === null || v === ""))) { // Suggest empty column. return { values, colMetadata: { type: "Any", isFormula: true, formula: "" } }; } if (values.some(isObject)) { // Suggest no changes. return { values }; } const strValues = values.map(v => (v === null || typeof v === "string" ? v : String(v))); const guessed = guessColInfoWithDocData(strValues, docData); values = guessed.values || values; const opts = guessed.colInfo.widgetOptions; const colMetadata: ColMetadata = { ...guessed.colInfo, widgetOptions: opts && JSON.stringify(opts) }; if (!colMetadata.widgetOptions) { delete colMetadata.widgetOptions; // Omit widgetOptions unless it is actually valid JSON. } return { values, colMetadata }; } ================================================ FILE: app/common/ValueParser.ts ================================================ import { csvDecodeRow } from "app/common/csvFormat"; import { BulkColValues, CellValue, ColValues, UserAction } from "app/common/DocActions"; import { DocData } from "app/common/DocData"; import { DocumentSettings } from "app/common/DocumentSettings"; import * as gristTypes from "app/common/gristTypes"; import { getReferencedTableId, isFullReferencingType } from "app/common/gristTypes"; import * as gutil from "app/common/gutil"; import { safeJsonParse } from "app/common/gutil"; import { NumberFormatOptions } from "app/common/NumberFormat"; import NumberParse from "app/common/NumberParse"; import { parseDateStrict, parseDateTime } from "app/common/parseDate"; import { MetaRowRecord, TableData } from "app/common/TableData"; import { DateFormatOptions, DateTimeFormatOptions, formatDecoded, FormatOptions } from "app/common/ValueFormatter"; import { encodeObject } from "app/plugin/objtypes"; import flatMap from "lodash/flatMap"; import mapValues from "lodash/mapValues"; export class ValueParser { constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) { } public cleanParse(value: string): any { if (!value) { return value; } return this.parse(value) ?? value; } public parse(value: string): any { return value; } } class IdentityParser extends ValueParser { } export class NumericParser extends ValueParser { private _parse: NumberParse; constructor(type: string, options: NumberFormatOptions, docSettings: DocumentSettings) { super(type, options, docSettings); this._parse = NumberParse.fromSettings(docSettings, options); } public parse(value: string): number | null { return this._parse.parse(value)?.result ?? null; } } class DateParser extends ValueParser { public parse(value: string): any { return parseDateStrict(value, (this.widgetOpts as DateFormatOptions).dateFormat!); } } class DateTimeParser extends ValueParser { constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) { super(type, widgetOpts, docSettings); const timezone = gutil.removePrefix(type, "DateTime:") || ""; this.widgetOpts = { ...widgetOpts, timezone }; } public parse(value: string): any { return parseDateTime(value, this.widgetOpts); } } class ChoiceListParser extends ValueParser { public cleanParse(value: string): string[] | null { value = value.trim(); const result = ( this._parseJson(value) || this._parseCsv(value) ).map(v => v.trim()) .filter(v => v); if (!result.length) { return null; } return ["L", ...result]; } private _parseJson(value: string): string[] | undefined { // Don't parse JSON non-arrays if (value.startsWith("[")) { const arr: unknown[] | null = safeJsonParse(value, null); return arr // Remove nulls and empty strings ?.filter(v => v || v === 0) // Convert values to strings, formatting nested JSON objects/arrays as JSON .map(v => formatDecoded(v)); } } private _parseCsv(value: string): string[] { // Split everything on newlines which are not allowed by the choice editor. return flatMap(value.split(/[\n\r]+/), (row) => { return csvDecodeRow(row) .map(v => v.trim()); }); } } /** * This is different from other widget options which are simple JSON * stored on the field. These have to be specially derived * for referencing columns. See createParser. */ export interface ReferenceParsingOptions { visibleColId: string; visibleColType: string; visibleColWidgetOpts: FormatOptions; // If this is provided and loaded, the ValueParser will look up values directly. // Otherwise an encoded lookup will be produced for the data engine to handle. tableData?: TableData; } export class ReferenceParser extends ValueParser { public widgetOpts: ReferenceParsingOptions; public tableData = this.widgetOpts.tableData; public visibleColParser = createParserRaw( this.widgetOpts.visibleColType, this.widgetOpts.visibleColWidgetOpts, this.docSettings, ); protected _visibleColId = this.widgetOpts.visibleColId; public parse(raw: string): any { const value = this.visibleColParser.cleanParse(raw); return this.lookup(value, raw); } public lookup(value: any, raw: string): any { if (value == null || value === "" || !raw) { return 0; // default value for a reference column } if (this._visibleColId === "id") { const n = Number(value); if (Number.isInteger(n)) { value = n; // Don't return yet because we need to check that this row ID exists } else { return raw; } } if (!this.tableData?.isLoaded) { const options: { column: string, raw?: string } = { column: this._visibleColId }; if (value !== raw) { options.raw = raw; } return ["l", value, options]; } return this.tableData.findMatchingRowId({ [this._visibleColId]: value }) || raw; } } export class ReferenceListParser extends ReferenceParser { public parse(raw: string): any { let values: any[] | null; try { values = JSON.parse(raw); } catch { values = null; } if (!Array.isArray(values)) { // csvDecodeRow should never raise an exception values = csvDecodeRow(raw); } values = values.map(v => typeof v === "string" ? this.visibleColParser.cleanParse(v) : encodeObject(v)); if (!values.length || !raw) { return null; // null is the default value for a reference list column } if (this._visibleColId === "id") { const numbers = values.map(Number); if (numbers.every(Number.isInteger)) { values = numbers; // Don't return yet because we need to check that these row IDs exist } else { return raw; } } if (!this.tableData?.isLoaded) { const options: { column: string, raw?: string } = { column: this._visibleColId }; if (!(values.length === 1 && values[0] === raw)) { options.raw = raw; } return ["l", values, options]; } const rowIds: number[] = []; for (const value of values) { const rowId = this.tableData.findMatchingRowId({ [this._visibleColId]: value }); if (rowId) { rowIds.push(rowId); } else { // There's no matching value in the visible column, i.e. this is not a valid reference. // We need to return a string which will become AltText. return raw; } } return ["L", ...rowIds]; } } export const valueParserClasses: { [type: string]: typeof ValueParser } = { Numeric: NumericParser, Int: NumericParser, Date: DateParser, DateTime: DateTimeParser, ChoiceList: ChoiceListParser, Ref: ReferenceParser, RefList: ReferenceListParser, Attachments: ReferenceListParser, }; /** * Returns a ValueParser which can parse strings into values appropriate for * a specific widget field or table column. * widgetOpts is usually the field/column's widgetOptions JSON * but referencing columns need more than that, see ReferenceParsingOptions above. */ export function createParserRaw( type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings, ): ValueParser { const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)] || IdentityParser; return new cls(type, widgetOpts, docSettings); } /** * Returns a ValueParser which can parse strings into values appropriate for * a specific widget field or table column. * * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field * instead of the table column. */ export function createParser( docData: DocData, colRef: number, fieldRef?: number, ): ValueParser { return createParserRaw(...createParserOrFormatterArguments(docData, colRef, fieldRef)); } /** * Returns arguments suitable for createParserRaw or createFormatter. Only for internal use. * * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field * instead of the table column. */ export function createParserOrFormatterArguments( docData: DocData, colRef: number, fieldRef?: number, ): [string, object, DocumentSettings] { const columnsTable = docData.getMetaTable("_grist_Tables_column"); const fieldsTable = docData.getMetaTable("_grist_Views_section_field"); const col = columnsTable.getRecord(colRef)!; let fieldOrCol: MetaRowRecord<"_grist_Tables_column" | "_grist_Views_section_field"> = col; if (fieldRef) { const field = fieldsTable.getRecord(fieldRef); fieldOrCol = field?.widgetOptions ? field : col; } return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol); } export function createParserOrFormatterArgumentsRaw( docData: DocData, type: string, widgetOptions: string, visibleColRef: number, ): [string, object, DocumentSettings] { const columnsTable = docData.getMetaTable("_grist_Tables_column"); const widgetOpts = safeJsonParse(widgetOptions, {}); if (isFullReferencingType(type)) { const vcol = columnsTable.getRecord(visibleColRef); widgetOpts.visibleColId = vcol?.colId || "id"; widgetOpts.visibleColType = vcol?.type; widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || "", {}); widgetOpts.tableData = docData.getTable(getReferencedTableId(type)!); } return [type, widgetOpts, docData.docSettings()]; } /** * Returns a copy of `colValues` with string values parsed according to the type and options of each column. * `bulk` should be `true` if `colValues` is of type `BulkColValues`. */ function parseColValues( tableId: string, colValues: T, docData: DocData, bulk: boolean, ): T { const columnsTable = docData.getMetaTable("_grist_Tables_column"); const tablesTable = docData.getMetaTable("_grist_Tables"); const tableRef = tablesTable.findRow("tableId", tableId); if (!tableRef) { return colValues; } return mapValues(colValues, (values, colId) => { const colRef = columnsTable.findMatchingRowId({ colId, parentId: tableRef }); if (!colRef) { // Column not found - let something else deal with that return values; } const parser = createParser(docData, colRef); // Optimisation: If there's no special parser for this column type, do nothing if (parser instanceof IdentityParser) { return values; } function parseIfString(val: any) { return typeof val === "string" ? parser.cleanParse(val) : val; } if (bulk) { if (!Array.isArray(values)) { // in case of bad input return values; } // `colValues` is of type `BulkColValues` return (values as CellValue[]).map(parseIfString); } else { // `colValues` is of type `ColValues`, `values` is just one value return parseIfString(values); } }); } export function parseUserAction(ua: UserAction, docData: DocData): UserAction { switch (ua[0]) { case "AddRecord": case "UpdateRecord": return _parseUserActionColValues(ua, docData, false); case "BulkAddRecord": case "BulkUpdateRecord": case "ReplaceTableData": return _parseUserActionColValues(ua, docData, true); case "AddOrUpdateRecord": // Parse `require` (2) and `col_values` (3). The action looks like: // ['AddOrUpdateRecord', table_id, require, col_values, options] // (`col_values` is called `fields` in the API) ua = _parseUserActionColValues(ua, docData, false, 2); ua = _parseUserActionColValues(ua, docData, false, 3); return ua; case "BulkAddOrUpdateRecord": ua = _parseUserActionColValues(ua, docData, true, 2); ua = _parseUserActionColValues(ua, docData, true, 3); return ua; default: return ua; } } // Returns a copy of the user action with one element parsed, by default the last one function _parseUserActionColValues(ua: UserAction, docData: DocData, parseBulk: boolean, index?: number, ): UserAction { ua = ua.slice(); const tableId = ua[1] as string; if (index === undefined) { index = ua.length - 1; } const colValues = ua[index] as ColValues | BulkColValues; ua[index] = parseColValues(tableId, colValues, docData, parseBulk); return ua; } ================================================ FILE: app/common/WidgetOptions.ts ================================================ import { NumberFormatOptions } from "app/common/NumberFormat"; export interface WidgetOptions extends NumberFormatOptions { textColor?: "string"; fillColor?: "string"; alignment?: "left" | "center" | "right"; dateFormat?: string; timeFormat?: string; widget?: "HyperLink"; choices?: string[]; } ================================================ FILE: app/common/airtable/AirtableAPI.ts ================================================ import { AirtableBaseSchema, AirtableFieldSchema, AirtableListBasesResponse, AirtableTableSchema, } from "app/common/airtable/AirtableAPITypes"; import AirtableSchemaTypeSuite from "app/common/airtable/AirtableAPITypes-ti"; import Airtable, { Record, SelectOptions as QueryParams } from "airtable"; import { CheckerT, createCheckers } from "ts-interface-checker"; export interface AirtableAPIOptions { apiKey: string; endpointUrl?: string; } // TODO - Improve error handling. Airtable's API throws if an error response is returned, // but we don't want to show that directly to users. /** * Simplifies access to Airtable's API. * - Allows easy access to meta methods (e.g. schema retrieval, listing bases) that aren't exposed * by the "airtable" package. * - Applies type checking and assertions to the responses */ export class AirtableAPI { private readonly _airtable: Airtable; private _metaRequester: Airtable.Base; constructor(_options: AirtableAPIOptions) { this._airtable = new Airtable(_options); // Airtable's JS library doesn't support fetching schemas, but by passing an empty baseId // we can still force it to request the URL we want, and re-use the library's backoff logic // to help with Airtable's rate limiting. this._metaRequester = this._airtable.base(""); } public base(baseId: string) { return this._airtable.base(baseId); } public async listBases(): Promise { // Technically there's pagination here - but each request returns 1000 bases, so it feels // premature to implement. const response = await this._metaRequester.makeRequest({ path: `meta/bases` }); const body = response.body as AirtableListBasesResponse; return body.bases; } public async getBaseSchema(baseId: string): Promise { const response = await this._metaRequester.makeRequest({ path: `meta/bases/${baseId}/tables`, }); const schema = response.body; if (!AirtableSchemaChecker.test(schema)) { throw new AirtableAPIError("unexpected response structure when fetching base schema"); } return schema; } } export class AirtableAPIError extends Error { constructor(message: string) { super(`Airtable API error: ${message}`); } } export interface ListAirtableRecordsResult { records: Airtable.Records, hasMoreRecords: boolean, fetchNextPage: FetchNextPageFunc } type FetchNextPageFunc = () => Promise; const fetchPageWhenNoMoreData: FetchNextPageFunc = () => Promise.resolve({ records: [], hasMoreRecords: false, fetchNextPage: fetchPageWhenNoMoreData, }); /** * Airtable's built-in record querying (base.table("MyTable").select().eachPage()) is prone * to hanging indefinitely when an error is thrown from the callback, or if the callback fails to * call `nextPage()` correctly. * * This re-implements the listRecords functionality, while keeping the error handling, * rate-limiting and auth logic from the Airtable library. */ export function listRecords( base: Airtable.Base, tableName: string, params: QueryParams, ): Promise { const table = base.table(tableName); const fetchNextPage = async (offset?: number): ReturnType => { const { body } = await base.makeRequest({ method: "GET", path: `/${encodeURIComponent(tableName)}`, qs: { ...params, offset, }, }); const records = body.records.map((recordJson: string) => new Record(table, "", recordJson)); const hasMoreRecords = body.offset !== undefined; return { records, hasMoreRecords, fetchNextPage: hasMoreRecords ? () => fetchNextPage(body.offset) : fetchPageWhenNoMoreData, }; }; return fetchNextPage(); } const checkers = createCheckers(AirtableSchemaTypeSuite); export const AirtableSchemaChecker = checkers.AirtableBaseSchema as CheckerT; export const AirtableSchemaTableChecker = checkers.AirtableSchemaTable as CheckerT; export const AirtableSchemaFieldChecker = checkers.AirtableSchemaField as CheckerT; ================================================ FILE: app/common/airtable/AirtableAPITypes-ti.ts ================================================ /** * This module was automatically generated by `ts-interface-builder` */ import * as t from "ts-interface-checker"; // tslint:disable:object-literal-key-quotes export const AirtableBaseId = t.name("string"); export const AirtableTableId = t.name("string"); export const AirtableFieldId = t.name("string"); export const AirtableFieldName = t.name("string"); export const AirtableBaseSchema = t.iface([], { "tables": t.array("AirtableTableSchema"), }); export const AirtableTableSchema = t.iface([], { "id": "AirtableTableId", "name": "string", "primaryFieldId": "string", "fields": t.array("AirtableFieldSchema"), }); export const AirtableFieldSchema = t.iface([], { "id": "AirtableFieldId", "name": "AirtableFieldName", "type": "string", "options": t.opt(t.iface([], { [t.indexKey]: "any", })), }); export const AirtableChoiceValue = t.iface([], { "id": "string", "name": "string", "color": "string", }); export const AirtableListBasesResponse = t.iface([], { "bases": t.array(t.iface([], { "id": "string", "name": "string", "permissionLevel": t.array(t.union(t.lit("none"), t.lit("read"), t.lit("comment"), t.lit("edit"), t.lit("create"))), })), "offset": t.opt("string"), }); const exportedTypeSuite: t.ITypeSuite = { AirtableBaseId, AirtableTableId, AirtableFieldId, AirtableFieldName, AirtableBaseSchema, AirtableTableSchema, AirtableFieldSchema, AirtableChoiceValue, AirtableListBasesResponse, }; export default exportedTypeSuite; ================================================ FILE: app/common/airtable/AirtableAPITypes.ts ================================================ // Aliases for the various Airtable IDs makes various Map type definitions clearer. export type AirtableBaseId = string; export type AirtableTableId = string; export type AirtableFieldId = string; export type AirtableFieldName = string; // Airtable schema response. Limit this to only needed fields to minimise chance of breakage. export interface AirtableBaseSchema { tables: AirtableTableSchema[]; } export interface AirtableTableSchema { id: AirtableTableId; name: string; primaryFieldId: string; fields: AirtableFieldSchema[]; } export interface AirtableFieldSchema { id: AirtableFieldId; name: AirtableFieldName; type: string; options?: { [key: string]: any }; } export interface AirtableChoiceValue { id: string; name: string; color: string; } export interface AirtableListBasesResponse { bases: { id: string, name: string, permissionLevel: ("none" | "read" | "comment" | "edit" | "create")[], }[], offset?: string, } ================================================ FILE: app/common/airtable/AirtableAttachmentTracker.ts ================================================ import { AirtableFieldSchema } from "app/common/airtable/AirtableAPITypes"; import { AirtableFieldMappingInfo, GristTableId } from "app/common/airtable/AirtableCrosswalk"; import { createEmptyBulkColValues } from "app/common/airtable/AirtableReferenceTracker"; import { TableColValues } from "app/common/DocActions"; import { getMaxUploadSizeAttachmentMB } from "app/common/gristUrls"; import { arrayRepeat, byteString } from "app/common/gutil"; import { GristObjCode } from "app/plugin/GristData"; import pick from "lodash/pick"; import pLimit from "p-limit"; export type AttachmentsByColumnId = Record; interface Attachment { filename: string; size: number; url: string; } interface AttachmentsForRecord { gristRecordId: number; attachmentsByColumnId: AttachmentsByColumnId; } export class AttachmentTracker { private _tableAttachmentTrackers = new Map(); public addTable(gristTableId: string, columnIdsToUpdate: string[]) { const tableTracker = new TableAttachmentTracker(gristTableId, columnIdsToUpdate); this._tableAttachmentTrackers.set(gristTableId, tableTracker); return tableTracker; } public getTables(): TableAttachmentTracker[] { return Array.from(this._tableAttachmentTrackers.values()); } public getRemainingAttachmentsCount() { let count = 0; for (const table of this.getTables()) { count += table.getRemainingAttachmentsCount(); } return count; } } export class TableAttachmentTracker { private _attachmentsForRecords: AttachmentsForRecord[] = []; public constructor(private _tableId: string, private _columnIds: string[]) { } public addRecord(attachmentsForRecord: AttachmentsForRecord) { this._attachmentsForRecords.push(attachmentsForRecord); } public async importAttachments( uploadAttachment: (value: string | Blob, filename?: string) => Promise, updateRows: (tableId: GristTableId, rows: TableColValues) => Promise, options: { maxConcurrentUploads?: number; updateRowsBatchSize?: number; onBatchComplete?(): void; } = {}, ) { const { maxConcurrentUploads = 5, updateRowsBatchSize = 25, onBatchComplete } = options; while (this._attachmentsForRecords.length > 0) { const attachmentsForRecords = this._attachmentsForRecords.splice(0, updateRowsBatchSize); const limit = pLimit(maxConcurrentUploads); const tableColValues: TableColValues = { id: [], ...createEmptyBulkColValues(this._columnIds) }; const uploads: Promise[] = []; for (let rowIdx = 0; rowIdx < attachmentsForRecords.length; rowIdx++) { const { gristRecordId, attachmentsByColumnId } = attachmentsForRecords[rowIdx]; tableColValues.id.push(gristRecordId); for (const colId of this._columnIds) { const attachments = attachmentsByColumnId[colId] ?? []; const cellValue: [GristObjCode.List, ...(number | undefined)[]] = [ GristObjCode.List, ...arrayRepeat(attachments.length, undefined)]; tableColValues[colId][rowIdx] = cellValue; attachments.forEach((attachment, index) => { uploads.push( limit(() => this._uploadAttachment(attachment, uploadAttachment)).then((id) => { cellValue[index + 1] = id; }), ); }); } } // TODO: Use a pipeline instead of batching for uploads. Batches are only as fast as the // slowest upload, and a particularly large attachment could hold up starting a new batch. // Also consider switching to allSettled and reporting any warnings/errors to the client. // Note that all errors are currently handled by _uploadAttachment, so this call shouldn't // throw. await Promise.all(uploads); for (const colId of this._columnIds) { for (let rowIdx = 0; rowIdx < attachmentsForRecords.length; rowIdx++) { const cellValue = tableColValues[colId][rowIdx] as [GristObjCode.List, ...(number | undefined)[]]; const attachmentIds = cellValue.slice(1) as (number | undefined)[]; tableColValues[colId][rowIdx] = [GristObjCode.List, ...attachmentIds.filter(id => id !== undefined)]; } } await updateRows(this._tableId, tableColValues); onBatchComplete?.(); } } public getRemainingAttachmentsCount() { let count = 0; for (const record of this._attachmentsForRecords) { for (const attachments of Object.values(record.attachmentsByColumnId)) { if (attachments) { count += attachments.length; } } } return count; } private async _uploadAttachment( { filename, size, url }: Attachment, uploadAttachment: (value: string | Blob, filename?: string) => Promise, ): Promise { try { const maxSize = getMaxUploadSizeAttachmentMB() * 1024 * 1024; if (size > maxSize) { throw new Error(`Attachments must not exceed ${byteString(maxSize)}`); } const response = await fetch(url); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const blob = await response.blob(); return await uploadAttachment(blob, filename); } catch (error) { console.error(`Failed to upload attachment "${filename}" (URL: ${url}):`, error); return undefined; } } } export function isAttachmentField({ type }: AirtableFieldSchema) { return type === "multipleAttachments"; } export function extractAttachmentsFromRecordField( fieldValue: any, fieldMapping: AirtableFieldMappingInfo, ): Attachment[] | undefined { if (fieldMapping.airtableField.type !== "multipleAttachments") { return undefined; } const attachments = Array.isArray(fieldValue) ? fieldValue : undefined; if (!attachments) { return undefined; } return attachments.map(a => pick(a, "filename", "size", "url")); } ================================================ FILE: app/common/airtable/AirtableCrosswalk.ts ================================================ import { AirtableBaseSchema, AirtableFieldName, AirtableFieldSchema, AirtableTableId, AirtableTableSchema, } from "app/common/airtable/AirtableAPITypes"; import { AirtableIdColumnLabel } from "app/common/airtable/AirtableSchemaImporter"; import { ExistingColumnSchema, ExistingDocSchema, ExistingTableSchema, } from "app/common/DocSchemaImportTypes"; export type GristTableId = string; export interface AirtableBaseSchemaCrosswalk { tables: Map } export interface AirtableTableCrosswalk { airtableTable: AirtableTableSchema; gristTable: ExistingTableSchema; fields: Map // Special case - ID isn't a field in Airtable, but it's useful to have a mapping if it exists. airtableIdColumn?: ExistingColumnSchema; } export interface AirtableFieldMappingInfo { airtableField: AirtableFieldSchema; gristColumn: ExistingColumnSchema; } /** * Creates a mapping from fields in an Airtable schema to fields in a Grist schema. * @param {AirtableBaseSchema} airtableSchema * @param {ExistingDocSchema} gristSchema * @param {Map} tableMap * @returns {{schemaCrosswalk: AirtableBaseSchemaCrosswalk, warnings: DocSchemaImportWarning[]}} */ export function createAirtableBaseToGristDocCrosswalk( airtableSchema: AirtableBaseSchema, gristSchema: ExistingDocSchema, tableMap: Map, ): { schemaCrosswalk: AirtableBaseSchemaCrosswalk, warnings: AirtableCrosswalkWarning[] } { const schemaCrosswalk: AirtableBaseSchemaCrosswalk = { tables: new Map(), }; const warnings: AirtableCrosswalkWarning[] = []; for (const [airtableTableId, gristTableId] of tableMap.entries()) { const airtableTableSchema = airtableSchema.tables.find(table => table.id === airtableTableId); const gristTableSchema = gristSchema.tables.find(table => table.id === gristTableId); if (!airtableTableSchema) { // Implementation error - this shouldn't be possible if the parameters are passed correctly. throw new Error(`No airtable table found with id '${airtableTableId}' when building crosswalk`); } if (!gristTableSchema) { warnings.push(new MissingGristTableWarning(gristTableId)); continue; } const { crosswalk: tableCrosswalk, warnings: tableWarnings } = createAirtableTableToGristTableCrosswalk(airtableTableSchema, gristTableSchema); warnings.push(...tableWarnings); schemaCrosswalk.tables.set(airtableTableId, tableCrosswalk); } return { schemaCrosswalk, warnings, }; } function createAirtableTableToGristTableCrosswalk( airtableTableSchema: AirtableTableSchema, gristTableSchema: ExistingTableSchema, ) { const warnings: AirtableCrosswalkWarning[] = []; const crosswalk: AirtableTableCrosswalk = { airtableTable: airtableTableSchema, gristTable: gristTableSchema, fields: new Map(), airtableIdColumn: gristTableSchema.columns.find(column => column.label === AirtableIdColumnLabel), }; for (const field of airtableTableSchema.fields) { // Match columns on label. It's the only reliable value we can automatically match on and the simplest to implement. const matchingColumn = findGristColumnForField(field, gristTableSchema); if (!matchingColumn) { warnings.push(new NoDestinationColumnWarning(gristTableSchema.id, field)); continue; } // Airtable record queries list fields by name (not id), which is guaranteed to be unique. crosswalk.fields.set(field.name, { airtableField: field, gristColumn: matchingColumn, }); } return { crosswalk, warnings }; } function findGristColumnForField(field: AirtableFieldSchema, gristSchema: ExistingTableSchema) { return gristSchema.columns.find(column => column.label === field.name); } export interface AirtableCrosswalkWarning { message: string; } class MissingGristTableWarning implements AirtableCrosswalkWarning { public readonly message; constructor(public readonly tableId: AirtableTableId) { this.message = `No Grist table found with id '${tableId}'`; } } class NoDestinationColumnWarning implements AirtableCrosswalkWarning { public readonly message: string; constructor(public readonly gristTableId: string, public readonly field: AirtableFieldSchema) { this.message = `No destination column in the Grist table '${gristTableId}' could be found for field '${field.name}'. A column with a matching label is required.`; } } ================================================ FILE: app/common/airtable/AirtableDataImporter.ts ================================================ import { AirtableFieldSchema } from "app/common/airtable/AirtableAPITypes"; import { AttachmentsByColumnId, AttachmentTracker, extractAttachmentsFromRecordField, isAttachmentField, TableAttachmentTracker, } from "app/common/airtable/AirtableAttachmentTracker"; import { AirtableDataImportParams } from "app/common/airtable/AirtableDataImporterTypes"; import { createEmptyBulkColValues, extractRefFromRecordField, isRefField, ReferenceTracker, RefValuesByColumnId, TableReferenceTracker, } from "app/common/airtable/AirtableReferenceTracker"; import { BulkColValues, CellValue, GristObjCode } from "app/plugin/GristData"; export async function importDataFromAirtableBase( { listRecords, addRows, updateRows, uploadAttachment, schemaCrosswalk, onProgress }: AirtableDataImportParams, ) { const referenceTracker = new ReferenceTracker(); const attachmentTracker = new AttachmentTracker(); const addRowsPromises: Promise[] = []; // TODO: Strings passed to onProgress calls in common code aren't translatable. onProgress?.({ percent: 0, status: "Importing records from Airtable..." }); for (const [tableId, tableCrosswalk] of schemaCrosswalk.tables.entries()) { // Filter out any formula columns early - Grist will error on any write to formula columns. const fieldMappings = Array.from(tableCrosswalk.fields.values()).filter(mapping => !mapping.gristColumn.isFormula); const gristColumnIds = fieldMappings.map(mapping => mapping.gristColumn.id); // Airtable ID needs to be handled separately to fields, as it's not stored as a field in Airtable if (tableCrosswalk.airtableIdColumn) { gristColumnIds.push(tableCrosswalk.airtableIdColumn.id); } const referenceColumnIds = Array.from(tableCrosswalk.fields.values()) .filter(mapping => isRefField(mapping.airtableField)) .map(mapping => mapping.gristColumn.id); let tableReferenceTracker: TableReferenceTracker | undefined; if (referenceColumnIds.length > 0) { tableReferenceTracker = referenceTracker.addTable(tableCrosswalk.gristTable.id, referenceColumnIds); } const attachmentColumnIds = Array.from(tableCrosswalk.fields.values()) .filter(mapping => isAttachmentField(mapping.airtableField)) .map(mapping => mapping.gristColumn.id); let tableAttachmentTracker: TableAttachmentTracker | undefined; if (attachmentColumnIds.length > 0) { tableAttachmentTracker = attachmentTracker.addTable(tableCrosswalk.gristTable.id, attachmentColumnIds); } let listRecordsResult = await listRecords(tableId); while (listRecordsResult.records.length > 0) { const { records } = listRecordsResult; const colValues: BulkColValues = createEmptyBulkColValues(gristColumnIds); const airtableRecordIds: string[] = []; const refsByColumnIdForRecords: RefValuesByColumnId[] = []; const attachmentsByColumnIdForRecords: AttachmentsByColumnId[] = []; for (const record of records) { const refsByColumnId: RefValuesByColumnId = {}; const attachmentsByColumnId: AttachmentsByColumnId = {}; airtableRecordIds.push(record.id); for (const fieldMapping of fieldMappings) { const { airtableField, gristColumn } = fieldMapping; const rawFieldValue = record.fields[airtableField.name]; if (isRefField(airtableField)) { refsByColumnId[gristColumn.id] = extractRefFromRecordField(rawFieldValue, fieldMapping); } if (isAttachmentField(airtableField)) { attachmentsByColumnId[gristColumn.id] = extractAttachmentsFromRecordField(rawFieldValue, fieldMapping); } if (isRefField(airtableField) || isAttachmentField(airtableField)) { // Column should remain blank until it's filled in by a later step. colValues[gristColumn.id].push(null); continue; } const converter = AirtableFieldValueConverters[fieldMapping.airtableField.type] ?? AirtableFieldValueConverters.identity; const value = converter(fieldMapping.airtableField, record.fields[fieldMapping.airtableField.name]); // Always push, even if the value is undefined, so that row values are always at the right index. colValues[fieldMapping.gristColumn.id].push(value ?? null); } if (tableCrosswalk.airtableIdColumn) { colValues[tableCrosswalk.airtableIdColumn.id].push(record.id); } refsByColumnIdForRecords.push(refsByColumnId); attachmentsByColumnIdForRecords.push(attachmentsByColumnId); } const addRowsPromise = addRows(tableCrosswalk.gristTable.id, colValues) .then((gristRowIds) => { airtableRecordIds.forEach((airtableRecordId, index) => { // Only add entries to the reference and attachment trackers once we know they're added to the table. referenceTracker.addRecordIdMapping(airtableRecordId, gristRowIds[index]); tableReferenceTracker?.addUnresolvedRecord({ gristRecordId: gristRowIds[index], refsByColumnId: refsByColumnIdForRecords[index], }); tableAttachmentTracker?.addRecord({ gristRecordId: gristRowIds[index], attachmentsByColumnId: attachmentsByColumnIdForRecords[index], }); }); }); addRowsPromises.push(addRowsPromise); listRecordsResult = await listRecordsResult.fetchNextPage(); } } // Future improvement - report all errors here using Promise.allSettled, or continue even if // a few sets of rows throw errors await Promise.all(addRowsPromises); for (const tableReferenceTracker of referenceTracker.getTables()) { await tableReferenceTracker.bulkUpdateRowsWithUnresolvedReferences(updateRows); } const totalAttachmentsCount = attachmentTracker.getRemainingAttachmentsCount(); for (const tableAttachmentTracker of attachmentTracker.getTables()) { await tableAttachmentTracker.importAttachments( uploadAttachment, updateRows, { onBatchComplete: () => { const remainingAttachmentsCount = attachmentTracker.getRemainingAttachmentsCount(); const uploadedAttachmentsCount = totalAttachmentsCount - remainingAttachmentsCount; const attachmentsPercent = (uploadedAttachmentsCount / totalAttachmentsCount) * 100; onProgress?.({ percent: 50 + (attachmentsPercent * 0.50), status: `Importing attachments from Airtable... (${remainingAttachmentsCount} remaining)`, }); }, }, ); } onProgress?.({ percent: 100 }); } type AirtableFieldValueConverter = (fieldSchema: AirtableFieldSchema, value: any) => CellValue | undefined; const AirtableFieldValueConverters: Record = { identity(fieldSchema, value) { return value; }, aiText(fieldSchema, aiTextState) { return aiTextState?.value; }, createdBy(fieldSchema, collaborator) { return formatCollaborator(collaborator); }, count(fieldSchema, collaborator) { throw new Error("Count is a formula column, and should not have data conversion run"); }, formula(fieldSchema, collaborator) { throw new Error("Formula is a formula column, and should not have data conversion run"); }, lastModifiedBy(fieldSchema, collaborator) { return formatCollaborator(collaborator); }, lookup(fieldSchema, value) { // Lookup fields fetch values from other columns. This should be a formula in Grist, no value needed. throw new Error("Lookup is a formula column, and should not have data conversion run"); }, multipleCollaborators(fieldSchema, collaborators) { const formattedCollaborators = collaborators?.map(formatCollaborator); if (!formattedCollaborators) { return null; } return formattedCollaborators.join(", "); }, singleCollaborator(fieldSchema, collaborator) { return formatCollaborator(collaborator); }, multipleSelects(fieldSchema, choices?: string[]) { if (!choices) { return null; } return [GristObjCode.List, ...choices]; }, rollup(fieldSchema, collaborator) { throw new Error("Rollup is a formula column, and should not have data conversion run"); }, }; const formatCollaborator = (collaborator: any) => collaborator?.name; ================================================ FILE: app/common/airtable/AirtableDataImporterTypes.ts ================================================ import { ListAirtableRecordsResult } from "app/common/airtable/AirtableAPI"; import { AirtableTableId } from "app/common/airtable/AirtableAPITypes"; import { AirtableBaseSchemaCrosswalk, GristTableId } from "app/common/airtable/AirtableCrosswalk"; import { BulkColValues, TableColValues } from "app/common/DocActions"; /** * Parameters for importing data from Airtable into Grist. */ export interface AirtableDataImportParams { // Airtable data API operations. Used to export data from Airtable. listRecords: ListRecordsFunc, // Grist data API operations. Used to import data into Grist. addRows: AddRowsFunc, updateRows: UpdateRowsFunc, uploadAttachment: UploadAttachmentFunc, // Mapping of Airtable tables to Grist tables. schemaCrosswalk: AirtableBaseSchemaCrosswalk, onProgress?(progress: AirtableImportProgress): void, } /** * The progress of an Airtable import. * * Used by the UI to show a progress bar with an optional status message communicating * the current operation in progress (e.g. importing attachments). */ export interface AirtableImportProgress { percent: number; status?: string; } /** * Function that fetches records from an Airtable table. */ export type ListRecordsFunc = (tableId: AirtableTableId) => Promise; /** * Function that adds rows to a Grist table. */ type AddRowsFunc = (tableId: GristTableId, rows: BulkColValues) => Promise; /** * Function that updates the column value(s) of a set of rows in a Grist table. */ export type UpdateRowsFunc = (tableId: GristTableId, rows: TableColValues) => Promise; /** * Function that uploads an attachment blob to Grist and returns the attachment ID. */ export type UploadAttachmentFunc = (value: string | Blob, filename?: string) => Promise; ================================================ FILE: app/common/airtable/AirtableReferenceTracker.ts ================================================ import { AirtableFieldSchema } from "app/common/airtable/AirtableAPITypes"; import { AirtableFieldMappingInfo } from "app/common/airtable/AirtableCrosswalk"; import { UpdateRowsFunc } from "app/common/airtable/AirtableDataImporterTypes"; import { TableColValues } from "app/common/DocActions"; import { isNonNullish } from "app/common/gutil"; import { BulkColValues, GristObjCode } from "app/plugin/GristData"; export type RefValuesByColumnId = Record; interface UnresolvedRefsForRecord { gristRecordId: number; refsByColumnId: RefValuesByColumnId; } export class ReferenceTracker { // Maps known airtable ids to their grist row ids to enable reference resolution. // Airtable row ids are guaranteed unique within a base. private _rowIdLookup = new Map(); // Group references by table and row to achieve bulk-updates and atomic resolutions for rows. private _tableReferenceTrackers = new Map(); public addRecordIdMapping(originalRecordId: string, gristRecordId: number) { this._rowIdLookup.set(originalRecordId, gristRecordId); } public resolve(originalRecordId: string): number | undefined { return this._rowIdLookup.get(originalRecordId); } public addTable(gristTableId: string, columnIdsToUpdate: string[]) { const tableTracker = new TableReferenceTracker(this, gristTableId, columnIdsToUpdate); this._tableReferenceTrackers.set(gristTableId, tableTracker); return tableTracker; } public getTables(): TableReferenceTracker[] { return Array.from(this._tableReferenceTrackers.values()); } } // Store and resolve references per-table to enable bulk updates. export class TableReferenceTracker { private _unresolvedRefsForRecords: UnresolvedRefsForRecord[] = []; // To perform bulk updates, all reference columns need updating at the same time. // Enforce this by explicitly listing the column ids to use during instantiation. public constructor(private _parent: ReferenceTracker, private _tableId: string, private _columnIds: string[]) { } public addUnresolvedRecord(unresolvedRefsForRecord: UnresolvedRefsForRecord) { this._unresolvedRefsForRecords.push(unresolvedRefsForRecord); } public async bulkUpdateRowsWithUnresolvedReferences( updateRows: UpdateRowsFunc, options?: { batchSize?: number }, ) { const batchSize = options?.batchSize ?? 100; let pendingUpdate: TableColValues = { id: [], ...createEmptyBulkColValues(this._columnIds) }; for (const unresolvedRefsForRecord of this._unresolvedRefsForRecords) { pendingUpdate.id.push(unresolvedRefsForRecord.gristRecordId); // Every row needs an entry in its respective column in the bulk update, so always loop through // the same columns for every row. for (const columnId of this._columnIds) { const references = unresolvedRefsForRecord.refsByColumnId[columnId]; // TODO - Unresolvable references are currently just skipped silently. Find a way to display // them in the cell / UI. const resolvedReferences = references ? references.map(originalRecordId => this._parent.resolve(originalRecordId)).filter(isNonNullish) : []; pendingUpdate[columnId].push( [GristObjCode.List, ...resolvedReferences], ); } if (pendingUpdate.id.length >= batchSize) { await updateRows(this._tableId, pendingUpdate); pendingUpdate = { id: [], ...createEmptyBulkColValues(this._columnIds) }; } } if (pendingUpdate.id.length > 0) { await updateRows(this._tableId, pendingUpdate); } } } export function isRefField(field: AirtableFieldSchema) { return field.type === "multipleRecordLinks"; } export function extractRefFromRecordField( fieldValue: any, fieldMapping: AirtableFieldMappingInfo, ): string[] | undefined { if (fieldMapping.airtableField.type === "multipleRecordLinks") { return fieldValue; } return undefined; } export function createEmptyBulkColValues(columnIds: string[]): BulkColValues { return Object.fromEntries(columnIds.map(id => [id, []])); } ================================================ FILE: app/common/airtable/AirtableSchemaImporter.ts ================================================ import { AirtableBaseSchema, AirtableChoiceValue, AirtableFieldSchema, AirtableTableSchema, } from "app/common/airtable/AirtableAPITypes"; import { ColumnImportSchema, DocSchemaImportWarning, FormulaTemplate, ImportSchema, OriginalTableRef, } from "app/common/DocSchemaImport"; import { RecalcWhen } from "app/common/gristTypes"; /** * Design note: this needs to be deterministic and based solely on the Airtable base schema, * it should not be based on the current state of the Grist doc or any other parameters passed to * the import. * * Other areas of the import code may transform the created schema * (e.g. skipping tables, resolving references). * If this schema changes based on the destination document state, a user-given parameter or anything * not directly derived from the Airtable schema, the remainder of the import code may not adapt the * schema properly for the target document. */ export function gristDocSchemaFromAirtableSchema( baseSchema: AirtableBaseSchema, ): { schema: ImportSchema; warnings: DocSchemaImportWarning[] } { const warnings: DocSchemaImportWarning[] = []; const schema: ImportSchema = { tables: baseSchema.tables.map((baseTable) => { const { columns, warnings: columnWarnings } = convertAirtableTableFieldsToColumnSchemas({ base: baseSchema, table: baseTable }); warnings.push(...columnWarnings); return { originalId: baseTable.id, desiredGristId: baseTable.name, columns: [createAirtableIdColumnSchema(), ...columns], }; }), }; return { schema, warnings }; } function convertAirtableTableFieldsToColumnSchemas( params: { base: AirtableBaseSchema, table: AirtableTableSchema }, ) { const { table } = params; const warnings: DocSchemaImportWarning[] = []; const columns = table.fields .map((field) => { const result = convertAirtableFieldToColumnSchema({ field, ...params }); if (result.warning) { warnings.push(result.warning); } return result.column; }) .filter((column): column is ColumnImportSchema => column !== undefined); return { columns, warnings }; } function convertAirtableFieldToColumnSchema( params: { base: AirtableBaseSchema, table: AirtableTableSchema, field: AirtableFieldSchema }, ): { column?: ColumnImportSchema, warning?: DocSchemaImportWarning } { const { field, table, base } = params; if (!AirtableFieldMappers[field.type]) { return { column: undefined, warning: new UnsupportedFieldTypeWarning(field.type, field.name, { originalTableId: table.id }), }; } return AirtableFieldMappers[field.type]({ field, table, getTableIdForField: (fieldId: string) => findTableIdForField(base, fieldId), }); } function findTableIdForField(baseSchema: AirtableBaseSchema, fieldId: string) { const tableId = baseSchema.tables.find(table => table.fields.find(field => field.id === fieldId))?.id; // Generally shouldn't happen - the schema should always have sufficient info to resolve a valid field id. if (tableId === undefined) { throw new Error(`Unable to resolve table id for Airtable field ${fieldId}`); } return tableId; } export const AirtableIdColumnLabel = "Airtable Id"; function createAirtableIdColumnSchema(): ColumnImportSchema { return { originalId: "airtableId", desiredGristId: "Airtable Id", type: "Text", label: AirtableIdColumnLabel, untieColIdFromLabel: true, }; } interface AirtableFieldMapperParams { field: AirtableFieldSchema, table: AirtableTableSchema, getTableIdForField: (fieldId: string) => string, } interface AirtableFieldMapperResult { column: ColumnImportSchema, warning?: DocSchemaImportWarning, } type AirtableFieldMapper = (params: AirtableFieldMapperParams) => AirtableFieldMapperResult; const AirtableFieldMappers: { [type: string]: AirtableFieldMapper } = { aiText({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", }, }; }, autoNumber({ field, table }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Numeric", formula: { formula: "MAX(PEEK([R0].all.[R1]))+1", replacements: [ { originalTableId: table.id }, { originalTableId: table.id, originalColId: field.id }, ], }, }, warning: new AutoNumberLimitationWarning(field.name, { originalTableId: table.id }), }; }, checkbox({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Bool", }, }; }, count({ field, table }) { let formula: FormulaTemplate = { formula: "", replacements: [] }; const fieldOptions = field.options; if (fieldOptions?.isValid && fieldOptions.recordLinkFieldId) { formula = { formula: "len($[R0])", replacements: [{ originalTableId: table.id, originalColId: fieldOptions.recordLinkFieldId }], }; } return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Numeric", isFormula: true, formula, }, warning: new CountLimitationWarning(field.name, { originalTableId: table.id }), }; }, createdBy({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", }, }; }, createdTime({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "DateTime", formula: { formula: "NOW()" }, recalcWhen: RecalcWhen.DEFAULT, }, }; }, currency({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Numeric", widgetOptions: { // Airtable only provides a currency symbol, which is pretty useless for setting this column up. // Instead of showing a wrong currency - omit currency formatting and just use precision. decimals: field.options?.precision ?? 2, maxDecimals: field.options?.precision ?? 2, }, }, }; }, date({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Date", widgetOptions: { isCustomDateFormat: true, // Airtable and Grist seem to share identical format syntax, based on limited testing dateFormat: field.options?.dateFormat?.format ?? "MM/DD/YYYY", }, }, }; }, dateTime({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "DateTime", widgetOptions: { isCustomDateFormat: true, // Airtable and Grist seem to share identical format syntax, based on limited testing dateFormat: field.options?.dateFormat?.format ?? "MM/DD/YYYY", isCustomTimeFormat: true, // Airtable and Grist seem to share identical format syntax, based on limited testing timeFormat: field.options?.timeFormat?.format ?? "h:mma", }, }, }; }, duration({ field, table }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Numeric", }, warning: new DurationFormatWarning(field.name, { originalTableId: table.id }), }; }, email({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", }, }; }, formula({ field }) { const formula = typeof field.options?.formula === "string" ? field.options?.formula : "No formula set"; // Store the formula as a comment to prevent it showing errors. const formattedFormula = formula.split("\n").map(line => `#${line.trim()}`).join("\n"); return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, // The field schema from Airtable has more information on what this should be, // such as field type, options and referenced fields. // The logic to implement that however doesn't seem worth the time investment. type: "Any", formula: { formula: formattedFormula }, isFormula: true, }, }; }, lastModifiedBy({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", formula: { formula: 'user and f"{user.Name}"' }, recalcWhen: 2, }, }; }, lastModifiedTime({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "DateTime", formula: { formula: "NOW()" }, recalcWhen: 2, widgetOptions: { isCustomDateFormat: true, dateFormat: field.options?.result?.dateFormat?.format ?? "MM/DD/YYYY", isCustomTimeFormat: true, timeFormat: field.options?.result?.timeFormat?.format ?? "h:mma", }, }, }; }, multilineText({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", }, }; }, multipleAttachments({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Attachments", }, }; }, multipleCollaborators({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", // Do we make a collaborators table and make this a reference instead? }, }; }, multipleLookupValues({ field, table, getTableIdForField }) { let formula: FormulaTemplate = { formula: "" }; const fieldOptions = field.options; if (fieldOptions?.recordLinkFieldId && fieldOptions.fieldIdInLinkedTable) { formula = { formula: "$[R0].[R1]", replacements: [ { originalTableId: table.id, originalColId: fieldOptions.recordLinkFieldId }, { originalTableId: getTableIdForField(fieldOptions.fieldIdInLinkedTable), originalColId: fieldOptions.fieldIdInLinkedTable, }, ], }; } return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Any", isFormula: true, formula, }, }; }, multipleRecordLinks({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: field.options?.prefersSingleRecordLink ? "Ref" : "RefList", ref: { originalTableId: field.options?.linkedTableId, }, }, }; }, multipleSelects({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "ChoiceList", widgetOptions: { choices: field.options?.choices.map((choice: AirtableChoiceValue) => choice.name), // We could import the color by mapping choice.color (e.g. tealLight2) to a hex color choiceOptions: {}, }, }, }; }, number({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Numeric", widgetOptions: { decimals: field.options?.precision, }, }, }; }, percent({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Numeric", widgetOptions: { decimals: field.options?.precision, numMode: "percent", }, }, }; }, phoneNumber({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", }, }; }, rating({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Int", // Consider setting up some nice conditional formatting. }, }; }, richText({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", widgetOptions: { widget: "Markdown", }, }, }; }, rollup({ field, table, getTableIdForField }) { let formula: FormulaTemplate = { formula: "" }; const fieldOptions = field.options; if (fieldOptions?.recordLinkFieldId && fieldOptions.fieldIdInLinkedTable) { formula = { formula: "$[R0].[R1]", replacements: [ { originalTableId: table.id, originalColId: fieldOptions.recordLinkFieldId }, { originalTableId: getTableIdForField(fieldOptions.fieldIdInLinkedTable), originalColId: fieldOptions.fieldIdInLinkedTable, }, ], }; } return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Any", isFormula: true, formula, }, warning: new RollupLimitationWarning(field.name, { originalTableId: table.id }), }; }, singleCollaborator({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", }, }; }, singleLineText({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", // We could potentially limit this to only a single line, but it's a view section option // which isn't (at the time of writing) supported by any of the import tools (which only deal // with structure, e.g. tables and columns). }, }; }, singleSelect({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Choice", widgetOptions: { choices: field.options?.choices.map((choice: AirtableChoiceValue) => choice.name), // We could import the color by mapping choice.color (e.g. tealLight2) to a hex color choiceOptions: {}, }, }, }; }, url({ field }) { return { column: { originalId: field.id, desiredGristId: field.name, label: field.name, type: "Text", widgetOptions: { widget: "HyperLink", }, }, }; }, }; class UnsupportedFieldTypeWarning implements DocSchemaImportWarning { public readonly message: string; constructor(fieldType: string, fieldName: string, public readonly ref: OriginalTableRef) { this.message = `Field "${fieldName}" has unsupported type "${fieldType}" and will be skipped`; } } class AutoNumberLimitationWarning implements DocSchemaImportWarning { public readonly message: string; constructor(fieldName: string, public readonly ref: OriginalTableRef) { this.message = `AutoNumber field "${fieldName}" behaviour will not be identical to Airtable's. Values may be re-used if rows are edited or deleted.`; } } class DurationFormatWarning implements DocSchemaImportWarning { public readonly message: string; constructor(fieldName: string, public readonly ref: OriginalTableRef) { this.message = `Duration field "${fieldName}" will be imported as a numeric duration in seconds. Duration formatting is not yet supported.`; } } class RollupLimitationWarning implements DocSchemaImportWarning { public readonly message: string; constructor(fieldName: string, public readonly ref: OriginalTableRef) { this.message = `Rollup field "${fieldName}" may not match Airtable. Summary parameters and filter conditions are not supported.`; } } class CountLimitationWarning implements DocSchemaImportWarning { public readonly message: string; constructor(fieldName: string, public readonly ref: OriginalTableRef) { this.message = `Count field "${fieldName}" may not match Airtable. Filter conditions are not supported.`; } } ================================================ FILE: app/common/arrayToString.ts ================================================ /** * Functions to convert between an array of bytes and a string. The implementations are * different for Node and for the browser. */ declare const TextDecoder: any, TextEncoder: any; export let arrayToString: (data: Uint8Array) => string; export let stringToArray: (data: string) => Uint8Array; if (typeof TextDecoder !== "undefined") { // Note that constructing a TextEncoder/Decoder takes time, so it's faster to reuse. const dec = new TextDecoder("utf8"); const enc = new TextEncoder("utf8"); arrayToString = function(uint8Array: Uint8Array): string { return dec.decode(uint8Array); }; stringToArray = function(str: string): Uint8Array { return enc.encode(str); }; } else { arrayToString = function(uint8Array: Uint8Array): string { return Buffer.from(uint8Array).toString("utf8"); }; stringToArray = function(str: string): Uint8Array { return new Uint8Array(Buffer.from(str, "utf8")); }; } ================================================ FILE: app/common/asyncIterators.ts ================================================ /** * Just some basic utilities for async generators that should really be part of the language or lodash or something. */ export async function* asyncFilter(it: AsyncIterableIterator, pred: (x: T) => boolean): AsyncIterableIterator { for await (const x of it) { if (pred(x)) { yield x; } } } export async function* asyncMap(it: AsyncIterableIterator, mapper: (x: T) => R): AsyncIterableIterator { for await (const x of it) { yield mapper(x); } } export async function toArray(it: AsyncIterableIterator): Promise { const result = []; for await (const x of it) { result.push(x); } return result; } ================================================ FILE: app/common/csvFormat.ts ================================================ /** * Simple utilities for escaping/quoting/parsing CSV data. * * This only supports the default Excel-like encoding, in which fields containing any separators * or quotes get quoted (using '"'), and quotes get doubled. * * Quoting is also applied when values contain leading or trailing whitespace, and on parsing, * leading or trailing whitespace in unquoted values is trimmed, so that "," or ", " may be used * as a separator. * * This is intended for copy-pasting multi-choice values, where plain comma-separated text is the * most user-friendly, and CSV encoding is used to ensure we can handle arbitrary values. */ // Encode a row. If {prettier: true} is set, separate output with ", ". Leading whitespace gets // encoded in any case. export function csvEncodeRow(values: string[], options: { prettier?: boolean } = {}): string { return values.map(csvEncodeCell).join(options.prettier ? ", " : ","); } export function csvDecodeRow(text: string): string[] { // Clever regexp from https://github.com/micnews/csv-line const parts = text.split(/((?:(?:"[^"]*")|[^,])*)/); const main = parts.filter((v, idx) => idx % 2).map(csvDecodeCell); // The "delimiter" (odd-numbered parts) is our content. If it's not at the start/end, it means // we have commas, and should include empty fields at those ends. if (parts[0]) { main.unshift(""); } if (parts[parts.length - 1]) { main.push(""); } return main; } export function csvEncodeCell(value: string): string { return /[,\r\n"]|^\s|\s$/.test(value) ? '"' + value.replace(/"/g, '""') + '"' : value; } export function csvDecodeCell(value: string): string { return value.trim().replace(/^"|"$/g, "").replace(/""/g, '"'); } ================================================ FILE: app/common/declarations.d.ts ================================================ declare module "app/common/MemBuffer" { const MemBuffer: any; type MemBuffer = any; export = MemBuffer; } declare module "locale-currency/map" { const Map: Record; type Map = Record; export = Map; } declare namespace Intl { class DisplayNames { public static supportedLocalesOf(locales: string | string[]): string[]; constructor(locales?: string, options?: object); public of(code: string): string; } class Locale { public region: string; public language: string; constructor(locale: string); } } declare module "@gristlabs/moment-guess/dist/bundle.js"; ================================================ FILE: app/common/delay.ts ================================================ /** * Returns a promise that resolves in the given number of milliseconds. * (A replica of bluebird.delay using native promises.) */ export function delay(msec: number): Promise { return new Promise(resolve => setTimeout(resolve, msec)); } ================================================ FILE: app/common/emails.ts ================================================ /** * * Utilities related to email normalization. Currently * trivial, but could potentially need special per-domain * rules in future. * * Email addresses are a bit slippery. Domain names are * case insensitive, but user names may or may not be, * depending on the mail server handling the domain. * Other special treatment of user names may also be in * place for particular domains (periods, plus sign, etc). * * We treat emails as case-insensitive for the purposes * of determining equality of emails, and indexing users * by email address. * */ /** * * Convert the supplied email address to a normalized form * that we will use for indexing and equality tests. * Many possible email addresses could map to the same * normalized result; as far as we are concerned those * addresses are equivalent. * * The normalization we do is a simple lowercase. This * means we won't be able to treat both Jane@x.y and * jane@x.y as separate email addresses, even through * they may in fact be separate mailboxes on x.y. * * The normalized email is not something we should show * the user in the UI, but is rather for internal purposes. * * The original non-normalized email is called a * "display email" to distinguish it from a "normalized * email" * */ export function normalizeEmail(displayEmail: string): string { // We take the lower case, without use of locale. return displayEmail.toLowerCase(); } ================================================ FILE: app/common/getCurrentTime.ts ================================================ import moment from "moment-timezone"; /** * Returns the current local time. Allows overriding via a "currentTime" URL parameter, for the sake * of tests. */ export default function getCurrentTime(): moment.Moment { const getDefault = () => moment(); if (typeof window === "undefined" || !window) { return getDefault(); } const searchParams = new URLSearchParams(window.location.search); return searchParams.has("currentTime") ? moment(searchParams.get("currentTime") || undefined) : getDefault(); } ================================================ FILE: app/common/gristTypes.ts ================================================ import { CellValue, CellVersions } from "app/common/DocActions"; import { removePrefix } from "app/common/gutil"; import { GristObjCode, GristType } from "app/plugin/GristData"; import isString from "lodash/isString"; export type GristTypeInfo = { type: "DateTime", timezone: string } | { type: "Ref", tableId: string } | { type: "RefList", tableId: string } | { type: Exclude }; export const MANUALSORT = "manualSort"; // Whether a column is internal and should be hidden. export function isHiddenCol(colId: string): boolean { return colId.startsWith("gristHelper_") || colId === MANUALSORT; } // This mapping includes both the default value, and its representation for SQLite. const _defaultValues: { [key in GristType]: [CellValue, string] } = { Any: [null, "NULL"], Attachments: [null, "NULL"], Blob: [null, "NULL"], // Bool is only supported by SQLite as 0 and 1 values. Bool: [false, "0"], Choice: ["", "''"], ChoiceList: [null, "NULL"], Date: [null, "NULL"], DateTime: [null, "NULL"], Id: [0, "0"], Int: [0, "0"], // Note that "1e999" is a way to store Infinity into SQLite. This is verified by "Defaults" // tests in DocStorage.js. See also http://sqlite.1065341.n5.nabble.com/Infinity-td55327.html. ManualSortPos: [Number.POSITIVE_INFINITY, "1e999"], Numeric: [0, "0"], PositionNumber: [Number.POSITIVE_INFINITY, "1e999"], Ref: [0, "0"], RefList: [null, "NULL"], Text: ["", "''"], }; /** * Given a grist column type (e.g Text, Numeric, ...) returns the default value for that type. * If options.sqlFormatted is true, returns the representation of the value for SQLite. */ export function getDefaultForType(colType: string, options: { sqlFormatted?: boolean } = {}) { const type = extractTypeFromColType(colType); return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0]; } /** * Convert a type like 'Numeric', 'DateTime:America/New_York', or 'Ref:Table1' to a GristTypeInfo * object. */ export function extractInfoFromColType(colType: string): GristTypeInfo { if (colType === "Attachments") { return { type: "RefList", tableId: "_grist_Attachments" }; } const colon = colType.indexOf(":"); const [type, arg] = (colon === -1) ? [colType] : [colType.slice(0, colon), colType.slice(colon + 1)]; return (type === "Ref") ? { type, tableId: String(arg) } : (type === "RefList") ? { type, tableId: String(arg) } : (type === "DateTime") ? { type, timezone: String(arg) } : { type } as GristTypeInfo; } /** * Re-encodes a CellValue of a given Grist type as a value suitable to use in an Any column. E.g. * reencodeAsTypedCellValue(123, {type: "Numeric"}) -> 123 * reencodeAsTypedCellValue(123, {type: "Date"}) -> ["d", 123] * reencodeAsTypedCellValue(123, {type: "Ref", tableId: "Table1"}) -> ["R", "Table1", 123] * reencodeAsTypedCellValue(["L", 123], {type: "RefList", tableId: "Table1"}) -> ["r", "Table1", [123]] * reencodeAsTypedCellValue(["L", 123], {type: "Attachments"}) -> ["r", "_grist_Attachments", [123]] */ export function reencodeAsTypedCellValue(value: CellValue, typeInfo: GristTypeInfo): CellValue { if (typeof value === "number") { switch (typeInfo.type) { case "Date": return [GristObjCode.Date, value]; case "DateTime": return [GristObjCode.DateTime, value, typeInfo.timezone]; case "Ref": return [GristObjCode.Reference, typeInfo.tableId, value]; } } else if (isList(value) || value === null) { const items = value ? value.slice(1) : []; switch (typeInfo.type) { case "ChoiceList": return [GristObjCode.List, ...items]; case "RefList": return [GristObjCode.ReferenceList, typeInfo.tableId, items]; case "Attachments": return [GristObjCode.ReferenceList, "_grist_Attachments", items]; } } return value; } /** * Returns whether a value (as received in a DocAction) represents a custom object. */ export function isObject(value: CellValue): value is [GristObjCode, any?] { return Array.isArray(value); } /** * Returns GristObjCode of the value if the value is an object, or null otherwise. * The return type includes any string, since we should not assume we can only get valid codes. */ export function getObjCode(value: CellValue): GristObjCode | string | null { return Array.isArray(value) ? value[0] : null; } /** * Returns whether a value (as received in a DocAction) represents a raised exception. */ export function isRaisedException(value: CellValue): boolean { return getObjCode(value) === GristObjCode.Exception; } /** * Returns whether a value (as received in a DocAction) represents a group of versions for * a comparison or conflict. */ export function isVersions(value: CellValue): value is [GristObjCode.Versions, CellVersions] { return getObjCode(value) === GristObjCode.Versions; } export function isSkip(value: CellValue): value is [GristObjCode.Skip] { return getObjCode(value) === GristObjCode.Skip; } export function isCensored(value: CellValue): value is [GristObjCode.Censored] { return getObjCode(value) === GristObjCode.Censored; } /** * Returns whether a value (as received in a DocAction) represents a list. */ export function isList(value: CellValue): value is [GristObjCode.List, ...CellValue[]] { return Array.isArray(value) && value[0] === GristObjCode.List; } /** * Returns whether a value (as received in a DocAction) represents a reference to a record. */ export function isReference(value: CellValue): value is [GristObjCode.Reference, string, number] { return Array.isArray(value) && value[0] === GristObjCode.Reference; } /** * Returns whether a value (as received in a DocAction) represents a reference list (RecordSet). */ export function isReferenceList(value: CellValue): value is [GristObjCode.ReferenceList, string, number[]] { return Array.isArray(value) && value[0] === GristObjCode.ReferenceList; } /** * Returns whether a value (as received in a DocAction) represents a reference or reference list. */ export function isReferencing(value: CellValue): value is [GristObjCode.ReferenceList | GristObjCode.Reference, string, number[] | number] { return Array.isArray(value) && (value[0] === GristObjCode.ReferenceList || value[0] === GristObjCode.Reference); } /** * Returns whether a value (as received in a DocAction) represents a list or is null, * which is a valid value for list types in grist. */ export function isListOrNull(value: CellValue): boolean { return value === null || isList(value); } /** * Returns whether a value (as received in a DocAction) represents an empty list. */ export function isEmptyList(value: CellValue): boolean { return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.List; } /** * Returns whether a value (as received in a DocAction) represents an empty reference list. */ export function isEmptyReferenceList(value: CellValue): boolean { return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.ReferenceList; } function isNumber(v: CellValue) { return typeof v === "number" || typeof v === "boolean"; } function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; } function isBoolean(v: CellValue) { return typeof v === "boolean" || v === 1 || v === 0; } function isBooleanOrNull(v: CellValue) { return isBoolean(v) || v === null; } // These values are not regular cell values, even in a column of type Any. const abnormalValueTypes: string[] = [GristObjCode.Exception, GristObjCode.Pending, GristObjCode.Skip, GristObjCode.Unmarshallable, GristObjCode.Versions]; function isNormalValue(value: CellValue) { return !abnormalValueTypes.includes(getObjCode(value)!); } /** * Map of Grist type to an "isRightType" checker function, which determines if a given values type * matches the declared type of the column. */ const rightType: { [key in GristType]: (value: CellValue) => boolean } = { Any: isNormalValue, Attachments: isListOrNull, Text: isString, Blob: isString, Int: isNumberOrNull, Bool: isBooleanOrNull, Date: isNumberOrNull, DateTime: isNumberOrNull, Numeric: isNumberOrNull, Id: isNumber, PositionNumber: isNumber, ManualSortPos: isNumber, Ref: isNumber, RefList: isListOrNull, Choice: isString, ChoiceList: isListOrNull, }; export function isRightType(type: string): undefined | ((value: CellValue, options?: any) => boolean) { return rightType[type as GristType]; } export function extractTypeFromColType(type: string): string { if (!type) { return type; } const colon = type.indexOf(":"); return (colon === -1 ? type : type.slice(0, colon)); } /** * Enum for values of columns' recalcWhen property, corresponding to Python definitions in * schema.py. */ export enum RecalcWhen { DEFAULT = 0, // Calculate on new records or when any field in recalcDeps changes. NEVER = 1, // Don't calculate automatically (but user can trigger manually) MANUAL_UPDATES = 2, // Calculate on new records and on manual updates to any data field. } /** * Converts SQL type strings produced by the Sequelize library into its corresponding * Grist type. The list of types is based on an analysis of SQL type string outputs * produced by the Sequelize library (mostly covered in lib/data-types.js). Some * additional engine/dialect specific types are detailed in dialect directories. * * TODO: A handful of exotic SQL types (mostly from PostgreSQL) will currently throw an * Error, rather than returning a type. Further testing is required to determine * whether Grist can manage those data types. * * @param {String} sqlType A string produced by Sequelize's describeTable query * @return {String} The corresponding Grist type string * @throws {Error} If the sqlType is unrecognized or unsupported */ export function sequelizeToGristType(sqlType: string): GristType { // Sequelize type strings can include parens (e.g., `CHAR(10)`). This function // ignores those additional details when determining the Grist type. let endMarker = sqlType.length; const parensMarker = sqlType.indexOf("("); endMarker = parensMarker > 0 ? parensMarker : endMarker; // Type strings might also include a space after the basic type description. // The type `DOUBLE PRECISION` is one such example, but modifiers or attributes // relevant to the type might also appear after the type itself (e.g., UNSIGNED, // NONZERO). These are ignored when determining the Grist type. const spaceMarker = sqlType.indexOf(" "); endMarker = spaceMarker > 0 && spaceMarker < endMarker ? spaceMarker : endMarker; switch (sqlType.substring(0, endMarker)) { case "INTEGER": case "BIGINT": case "SMALLINT": case "INT": return "Int"; case "NUMBER": case "FLOAT": case "DECIMAL": case "NUMERIC": case "REAL": case "DOUBLE": case "DOUBLE PRECISION": return "Numeric"; case "BOOLEAN": case "TINYINT": return "Bool"; case "STRING": case "CHAR": case "TEXT": case "UUID": case "UUIDV1": case "UUIDV4": case "VARCHAR": case "NVARCHAR": case "TINYTEXT": case "MEDIUMTEXT": case "LONGTEXT": case "ENUM": return "Text"; case "TIME": case "DATE": case "DATEONLY": case "DATETIME": case "NOW": return "Text"; case "BLOB": case "TINYBLOB": case "MEDIUMBLOB": case "LONGBLOB": // TODO: Passing binary data to the Sandbox is throwing Errors. Proper support // for these Blob data types requires some more investigation. throw new Error("SQL type: `" + sqlType + "` is currently unsupported"); case "NONE": case "HSTORE": case "JSON": case "JSONB": case "VIRTUAL": case "ARRAY": case "RANGE": case "GEOMETRY": throw new Error("SQL type: `" + sqlType + "` is currently untested"); default: throw new Error("Unrecognized datatype: `" + sqlType + "`"); } } export function getReferencedTableId(type: string) { if (type === "Attachments") { return "_grist_Attachments"; } return removePrefix(type, "Ref:") || removePrefix(type, "RefList:"); } export function isRefListType(type: string) { return type === "Attachments" || type?.startsWith("RefList:"); } export function isListType(type: string) { return type === "ChoiceList" || isRefListType(type); } export function isNumberType(type: string | undefined) { return ["Numeric", "Int"].includes(type || ""); } export function isDateLikeType(type: string) { return type === "Date" || type.startsWith("DateTime"); } export function isFullReferencingType(type: string) { return type.startsWith("Ref:") || isRefListType(type); } export function isValidRuleValue(value: CellValue | undefined) { // We want to strictly test if a value is boolean, when the value is 0 or 1 it might // indicate other number in the future. return value === null || typeof value === "boolean"; } /** * Returns true if `value` is blank. * * Blank values include `null`, (trimmed) empty string, and 0-length lists and * reference lists. */ export function isBlankValue(value: CellValue) { return ( value === null || (typeof value === "string" && value.trim().length === 0) || isEmptyList(value) || isEmptyReferenceList(value) ); } export type RefListValue = [GristObjCode.List, ...number[]] | null; /** * Type of cell metadata information. */ export enum CellInfoType { COMMENT = 1, } ================================================ FILE: app/common/gristUrls.ts ================================================ import { AssistantConfig } from "app/common/Assistant"; import { BillingPage, BillingSubPage, BillingTask } from "app/common/BillingAPI"; import { OpenDocMode } from "app/common/DocListAPI"; import { EngineCode } from "app/common/DocumentSettings"; import { Features as PlanFeatures } from "app/common/Features"; import { encodeQueryParams, isAffirmative, removePrefix } from "app/common/gutil"; import { ICommonUrls } from "app/common/ICommonUrls"; import ICommonUrlsTI from "app/common/ICommonUrls-ti"; import { LocalPlugin } from "app/common/plugin"; import { StringUnion } from "app/common/StringUnion"; import { TelemetryLevel } from "app/common/Telemetry"; import { ThemeAppearance, themeAppearances, ThemeName, themeNames } from "app/common/ThemePrefs"; import { getGristConfig } from "app/common/urlUtils"; import { Document } from "app/common/UserAPI"; import { IAttachedCustomWidget } from "app/common/widgetTypes"; import { UIRowId } from "app/plugin/GristAPI"; import clone from "lodash/clone"; import pickBy from "lodash/pickBy"; import slugify from "slugify"; import * as t from "ts-interface-checker"; const { ICommonUrls: ICommonUrlsChecker } = t.createCheckers(ICommonUrlsTI); export const SpecialDocPage = StringUnion( "code", "acl", "data", "GristDocTour", "settings", "suggestions", "webhook", "timing", ); type SpecialDocPage = typeof SpecialDocPage.type; export type IDocPage = number | SpecialDocPage; export type ViewDocPage = number | "data"; /** * ViewDocPage is a page that shows table data (either normal or raw data view). */ export function isViewDocPage(docPage: IDocPage): docPage is ViewDocPage { return typeof docPage === "number" || docPage === "data"; } // What page to show in the user's home area. Defaults to 'workspace' if a workspace is set, and // to 'all' otherwise. export const HomePage = StringUnion("all", "workspace", "templates", "trash"); export type IHomePage = typeof HomePage.type; export const HomePageTab = StringUnion("recent", "pinned", "all"); export type HomePageTab = typeof HomePageTab.type; export const WelcomePage = StringUnion("teams", "signup", "verify", "select-account"); export type WelcomePage = typeof WelcomePage.type; export const AccountPage = StringUnion("account"); export type AccountPage = typeof AccountPage.type; export const ActivationPage = StringUnion("activation"); export type ActivationPage = typeof ActivationPage.type; export const AuditLogsPage = StringUnion("audit-logs"); export type AuditLogsPage = typeof AuditLogsPage.type; export const LoginPage = StringUnion("signup", "login", "verified", "forgot-password"); export type LoginPage = typeof LoginPage.type; export const AdminPanelPage = StringUnion("admin", "docs", "users", "workspaces", "orgs"); export type AdminPanelPage = typeof AdminPanelPage.type; export const AdminPanelTab = StringUnion("users", "workspaces", "docs", "orgs", "details"); export type AdminPanelTab = typeof AdminPanelTab.type; export const PREFERRED_STORAGE_ANCHOR = "preferredStorage"; export const PersistentAnchor = StringUnion(PREFERRED_STORAGE_ANCHOR); export type PersistentAnchor = typeof PersistentAnchor.type; // Overall UI style. "full" is normal, "singlePage" is a single page focused, panels hidden experience. export const InterfaceStyle = StringUnion("singlePage", "full"); export type InterfaceStyle = typeof InterfaceStyle.type; export const CompareEmphasis = StringUnion("local", "remote"); export type CompareEmphasis = typeof CompareEmphasis.type; // Default subdomain for home api service if not otherwise specified. export const DEFAULT_HOME_SUBDOMAIN = "api"; // This is the minimum length a urlId may have if it is chosen // as a prefix of the docId. export const MIN_URLID_PREFIX_LENGTH = 12; // Values meeting MIN_URLID_PREFIX_LENGTH that appear in non-document URLs and // should not be recognized as urlId prefixes when decoding URLs. const RESERVED_URLID_PREFIXES = new Set(["forgot-password"]); // A prefix that identifies a urlId as a share key. // Important that this not be part of a valid docId. export const SHARE_KEY_PREFIX = "s."; /** * Form framing is used to control the way forms are rendered in Grist. * - 'border' adds a green border around the form, used to indicate that the forms can be created * by untrusted users and it makes phishing attacks harder. * - 'minimal' doesn't show the border, used for trusted users. * * The default value is 'border', and it can be controlled by GRIST_FEATURE_FORM_FRAMING environment * variable. */ export type FormFraming = "border" | "minimal"; /** * Special ways to open a document, based on what the user intends to do. * - view: Open document in read-only mode (even if user has edit rights) * - fork: Open document in fork-ready mode. This means that while edits are * permitted, those edits should go to a copy of the document rather than * the original. */ export const getCommonUrls = () => withAdminDefinedUrls({ help: getHelpCenterUrl(), helpAccessRules: "https://support.getgrist.com/access-rules", helpAssistant: "https://support.getgrist.com/assistant", helpAssistantDataUse: "https://support.getgrist.com/assistant/#data-use-policy", helpFormulaAssistantDataUse: "https://support.getgrist.com/ai-assistant/#data-use-policy", helpColRefs: "https://support.getgrist.com/col-refs", helpConditionalFormatting: "https://support.getgrist.com/conditional-formatting", helpFilterButtons: "https://support.getgrist.com/search-sort-filter/#pinning-filters", helpLinkingWidgets: "https://support.getgrist.com/linking-widgets", helpRawData: "https://support.getgrist.com/raw-data", helpSuggestions: "https://support.getgrist.com/sharing/#suggestions", helpUnderstandingReferenceColumns: "https://support.getgrist.com/col-refs/#understanding-reference-columns", helpTriggerFormulas: "https://support.getgrist.com/formulas/#trigger-formulas", helpTryingOutChanges: "https://support.getgrist.com/copying-docs/#trying-out-changes", helpWidgets: "https://support.getgrist.com/page-widgets/#widgets", helpCustomWidgets: "https://support.getgrist.com/widget-custom", helpInstallAuditLogs: "https://support.getgrist.com/install/audit-log-overview/", helpTeamAuditLogs: "https://support.getgrist.com/install/audit-log-overview/", helpTelemetryLimited: "https://support.getgrist.com/telemetry-limited", helpEnterpriseOptIn: "https://support.getgrist.com/self-managed/#how-do-i-enable-grist-enterprise", helpCalendarWidget: "https://support.getgrist.com/widget-calendar", helpLinkKeys: "https://support.getgrist.com/examples/2021-04-link-keys", helpFilteringReferenceChoices: "https://support.getgrist.com/col-refs/#filtering-reference-choices-in-dropdown-lists", helpSandboxing: "https://support.getgrist.com/self-managed/#how-do-i-sandbox-documents", helpAPI: "https://support.getgrist.com/api", helpStateStore: "https://support.getgrist.com/self-managed/#what-is-a-state-store", helpSummaryFormulas: "https://support.getgrist.com/summary-tables/#summary-formulas", helpAdminControls: "https://support.getgrist.com/admin-controls", helpFiddleMode: "https://support.getgrist.com/glossary/#fiddle-mode", helpSharing: "https://support.getgrist.com/sharing", helpFormUrlValues: "https://support.getgrist.com/widget-form/#accept-value-from-url", helpAirtableIntegration: "https://support.getgrist.com/install/integrations/airtable", freeCoachingCall: getFreeCoachingCallUrl(), contactSupport: getContactSupportUrl(), termsOfService: getTermsOfServiceUrl(), onboardingTutorialVideoId: getOnboardingVideoId(), plans: "https://www.getgrist.com/pricing", contact: "https://www.getgrist.com/contact", templates: "https://www.getgrist.com/templates", webinars: getWebinarsUrl(), community: "https://community.getgrist.com", functions: "https://support.getgrist.com/functions", formulaSheet: "https://support.getgrist.com/formula-cheat-sheet", formulas: "https://support.getgrist.com/formulas", forms: "https://www.getgrist.com/forms/?utm_source=grist-forms&utm_medium=grist-forms&utm_campaign=forms-footer", openGraphPreviewImage: "https://grist-static.com/icons/opengraph-preview-image.png", gristLabsCustomWidgets: "https://gristlabs.github.io/grist-widget/", gristLabsWidgetRepository: "https://github.com/gristlabs/grist-widget/releases/download/latest/manifest.json", githubGristCore: "https://github.com/gristlabs/grist-core", githubSponsorGristLabs: "https://github.com/sponsors/gristlabs", versionCheck: "https://api.getgrist.com/api/version", attachmentStorage: "https://support.getgrist.com/document-settings/#external-attachments", signInWithGristRegister: "https://login.getgrist.com/oauth/register", signInWithGristHelp: "https://support.getgrist.com/install/sign-in-with-grist", }); export const commonUrls = getCommonUrls(); /** * Values representable in a URL. The current state is available as urlState().state observable * in client. Updates to this state are expected by functions such as makeUrl() and setLinkUrl(). */ export interface IGristUrlState { org?: string; homePage?: IHomePage; homePageTab?: HomePageTab; ws?: number; doc?: string; slug?: string; // if present, this is based on the document title, and is not a stable id mode?: OpenDocMode; fork?: UrlIdParts; docPage?: IDocPage; account?: AccountPage; billing?: BillingPage; activation?: ActivationPage; auditLogs?: AuditLogsPage; login?: LoginPage; welcome?: WelcomePage; adminPanel?: AdminPanelPage; adminPanelTab?: AdminPanelTab; welcomeTour?: boolean; docTour?: boolean; manageUsers?: boolean; createTeam?: boolean; upgradeTeam?: boolean; params?: { billingPlan?: string; // priceId planType?: string; billingTask?: BillingTask; embed?: boolean; state?: string; srcDocId?: string; style?: InterfaceStyle; compare?: string; compareEmphasis?: CompareEmphasis; // which of local and remote changes should be emphasized visually. linkParameters?: Record; // Parameters to pass as 'user.Link' in granular ACLs. // Encoded in URL as query params with extra '_' suffix. themeSyncWithOs?: boolean; themeAppearance?: ThemeAppearance; themeName?: ThemeName; details?: boolean; // Used on admin pages to show details tab. assistantPrompt?: string; assistantState?: string; }; hash?: HashLink; // if present, this specifies an individual row within a section of a page. api?: boolean; // indicates that the URL should be encoded as an API URL, not as a landing page. // But this barely works, and is suitable only for documents. For decoding it // indicates that the URL probably points to an API endpoint. viaShare?: boolean; // Accessing document via a special share. form?: { vsId: number; // a view section id of a form. shareKey?: string; // only one of shareKey or doc should be set. }, } // Subset of GristLoadConfig used by getOrgUrlInfo(), which affects the interpretation of the // current URL. export interface OrgUrlOptions { // The org associated with the current URL. org?: string; // Base domain for constructing new URLs, should start with "." and not include port, e.g. // ".getgrist.com". It should be unset for localhost operation and in single-org mode. baseDomain?: string; // In single-org mode, this is the single well-known org. singleOrg?: string; // Base URL used for accessing plugin material. pluginUrl?: string; // If set, org is expected to be encoded in the path, not domain. pathOnly?: boolean; } // Result of getOrgUrlInfo(). export interface OrgUrlInfo { hostname?: string; // If hostname should be changed to access the requested org. orgInPath?: string; // If /o/{orgInPath} should be used to access the requested org. } function hostMatchesUrl(host?: string, url?: string) { return host !== undefined && url !== undefined && new URL(url).host === host; } /** * Returns true if: * - the server is a home worker and the host matches APP_HOME_INTERNAL_URL; * - or the server is a doc worker and the host matches APP_DOC_INTERNAL_URL; * * @param {string?} host The host to check */ function isOwnInternalUrlHost(host?: string) { // Note: APP_HOME_INTERNAL_URL may also be defined in doc worker as well as in home worker if (process.env.APP_HOME_INTERNAL_URL && hostMatchesUrl(host, process.env.APP_HOME_INTERNAL_URL)) { return true; } return Boolean(process.env.APP_DOC_INTERNAL_URL) && hostMatchesUrl(host, process.env.APP_DOC_INTERNAL_URL); } /** * Given host (optionally with port), baseDomain, and pluginUrl, determine whether to interpret host * as a custom domain, a native domain, or a plugin domain. */ export function getHostType(host: string, options: { baseDomain?: string, pluginUrl?: string }): "native" | "custom" | "plugin" { if (options.pluginUrl) { const url = new URL(options.pluginUrl); if (url.host.toLowerCase() === host.toLowerCase()) { return "plugin"; } } const hostname = host.split(":")[0]; if (!options.baseDomain) { return "native"; } if ( hostname === "localhost" || isOwnInternalUrlHost(host) || hostname.endsWith(options.baseDomain) ) { return "native"; } return "custom"; } export function getOrgUrlInfo(newOrg: string, currentHost: string, options: OrgUrlOptions): OrgUrlInfo { if (newOrg === options.singleOrg) { return {}; } if (options.pathOnly) { return { orgInPath: newOrg }; } const hostType = getHostType(currentHost, options); if (hostType !== "plugin") { const hostname = currentHost.split(":")[0]; if (!options.baseDomain || hostname === "localhost") { return { orgInPath: newOrg }; } } if (newOrg === options.org && hostType !== "native") { return {}; } return { hostname: newOrg + options.baseDomain }; } /** * The actual serialization of a url state into a URL. The URL has the form * / * /ws// * /doc/[/p/] * * where depends on whether subdomains are in use, e.g. * .getgrist.com * localhost:8080/o/ */ export function encodeUrl(gristConfig: Partial, state: IGristUrlState, baseLocation: Location | URL, options: { tweaks?: UrlTweaks, } = {}): string { const url = new URL(baseLocation.href); const parts = ["/"]; if (state.org) { // We figure out where to stick the org using the gristConfig and the current host. const { hostname, orgInPath } = getOrgUrlInfo(state.org, baseLocation.host, gristConfig); if (hostname) { url.hostname = hostname; } if (orgInPath) { parts.push(`o/${orgInPath}/`); } } if (state.api) { parts.push(`api/`); } if (state.ws) { parts.push(`ws/${state.ws}/`); } if (state.doc) { if (state.api) { parts.push(`docs/${encodeURIComponent(state.doc)}`); } else if (state.viaShare) { // Use a special path, and remove SHARE_KEY_PREFIX from id. let id = state.doc; if (id.startsWith(SHARE_KEY_PREFIX)) { id = id.substring(SHARE_KEY_PREFIX.length); } parts.push(`s/${encodeURIComponent(id)}`); } else if (state.slug) { parts.push(`${encodeURIComponent(state.doc)}/${encodeURIComponent(state.slug)}`); } else { parts.push(`doc/${encodeURIComponent(state.doc)}`); } if (state.mode && OpenDocMode.guard(state.mode)) { parts.push(`/m/${state.mode}`); } if (state.docPage) { parts.push(`/p/${state.docPage}`); } if (state.form) { parts.push(`/f/${state.form.vsId}`); } } else if (state.form?.shareKey) { parts.push(`forms/${encodeURIComponent(state.form.shareKey)}/${encodeURIComponent(state.form.vsId)}`); } else if (state.homePage === "trash" || state.homePage === "templates") { parts.push(`p/${state.homePage}`); } if (state.account) { parts.push(state.account === "account" ? "account" : `account/${state.account}`); } if (state.billing) { parts.push(state.billing === "billing" ? "billing" : `billing/${state.billing}`); } if (state.activation) { parts.push(state.activation); } if (state.auditLogs) { parts.push(state.auditLogs); } if (state.login) { parts.push(state.login); } if (state.welcome) { parts.push(`welcome/${state.welcome}`); } if (state.adminPanel) { parts.push(state.adminPanel === "admin" ? "admin" : `admin/${state.adminPanel}`); } const queryParams = pickBy(state.params, (v, k) => k !== "linkParameters") as { [key: string]: string }; for (const [k, v] of Object.entries(state.params?.linkParameters || {})) { queryParams[`${k}_`] = v; } if (state.params?.details) { queryParams.details = "true"; } const queryStr = encodeQueryParams(queryParams); url.pathname = parts.join(""); url.search = queryStr; if (state.homePageTab) { url.hash = state.homePageTab; } else if (state.hash?.anchor) { url.hash = state.hash.anchor; } else if (state.hash) { // Project tests use hashes, so only set hash if there is an anchor. url.hash = makeAnchorLinkValue(state.hash); } else if (state.welcomeTour) { url.hash = "repeat-welcome-tour"; } else if (state.docTour) { url.hash = "repeat-doc-tour"; } else if (state.manageUsers) { url.hash = "manage-users"; } else if (state.createTeam) { url.hash = "create-team"; } else if (state.upgradeTeam) { url.hash = "upgrade-team"; } else if (state.adminPanelTab) { url.hash = state.adminPanelTab; } else { url.hash = ""; } options.tweaks?.postEncode?.({ url, parts, state, baseLocation, }); return url.href; } /** * Parse a URL location into an IGristUrlState object. See encodeUrl() documentation. */ export function decodeUrl(gristConfig: Partial, location: Location | URL, options?: { tweaks?: UrlTweaks, }): IGristUrlState { location = new URL(location.href); // Make sure location is a URL. options?.tweaks?.preDecode?.({ url: location }); const parts = location.pathname.slice(1).split("/"); const state: IGristUrlState = {}; // Bare minimum we can do to detect API URLs: if it starts with /api/ or /o/{org}/api/... if (parts[0] === "api" || (parts[0] === "o" && parts[2] === "api")) { state.api = true; parts.splice(parts[0] === "api" ? 0 : 2, 1); } // Bare minimum we can do to detect form URLs with share keys: if it starts with /forms/ or /o/{org}/forms/... if (parts[0] === "forms" || (parts[0] === "o" && parts[2] === "forms")) { const startIndex = parts[0] === "forms" ? 0 : 2; // Form URLs have two parts to extract: the share key and the view section id. state.form = { shareKey: parts[startIndex + 1], vsId: parseInt(parts[startIndex + 2], 10), }; parts.splice(startIndex, 3); } const map = new Map(); for (let i = 0; i < parts.length; i += 2) { map.set(parts[i], decodeURIComponent(parts[i + 1])); } // For the API case, we need to map "docs" to "doc" (as this is what we did in encodeUrl and what API expects). if (state.api && map.has("docs")) { map.set("doc", map.get("docs")!); } // /s/ is accepted as another way to write -> /doc/ if (map.has("s")) { const key = map.get("s"); map.set("doc", `${SHARE_KEY_PREFIX}${key}`); state.viaShare = true; } // When the urlId is a prefix of the docId, documents are identified // as "/slug" instead of "doc/". We can detect that because // the minimum length of a urlId prefix is longer than the maximum length // of any of the valid keys in the url. for (const key of map.keys()) { if ( key.length >= MIN_URLID_PREFIX_LENGTH && !RESERVED_URLID_PREFIXES.has(key) ) { map.set("doc", key); map.set("slug", map.get(key)!); map.delete(key); break; } } const subdomain = parseSubdomain(location.host); if (gristConfig.org || gristConfig.singleOrg) { state.org = gristConfig.org || gristConfig.singleOrg; } else if (!gristConfig.pathOnly && subdomain.org) { state.org = subdomain.org; } const sp = new URLSearchParams(location.search); if (location.search) { state.params = {}; } if (map.has("o")) { state.org = map.get("o"); } if (map.has("ws")) { state.ws = parseInt(map.get("ws")!, 10); } if (map.has("doc")) { state.doc = map.get("doc"); const fork = parseUrlId(map.get("doc")!); if (fork.forkId) { state.fork = fork; } if (map.has("slug")) { state.slug = map.get("slug"); } if (map.has("p")) { state.docPage = parseDocPage(map.get("p")!); } if (map.has("f")) { state.form = { vsId: parseInt(map.get("f")!, 10) }; } } else { if (map.has("p")) { const p = map.get("p")!; state.homePage = HomePage.parse(p); } } if (map.has("m")) { state.mode = OpenDocMode.parse(map.get("m")); } if (map.has("account")) { state.account = AccountPage.parse(map.get("account")) || "account"; } if (map.has("billing")) { state.billing = BillingSubPage.parse(map.get("billing")) || "billing"; } if (map.has("activation")) { state.activation = ActivationPage.parse(map.get("activation")) || "activation"; } if (map.has("audit-logs")) { state.auditLogs = AuditLogsPage.parse(map.get("audit-logs")) || "audit-logs"; } if (map.has("welcome")) { state.welcome = WelcomePage.parse(map.get("welcome")); } if (map.has("admin")) { state.adminPanel = AdminPanelPage.parse(map.get("admin")) || "admin"; } if (sp.has("planType")) { state.params!.planType = sp.get("planType")!; } if (sp.has("billingPlan")) { state.params!.billingPlan = sp.get("billingPlan")!; } if (sp.has("billingTask")) { state.params!.billingTask = BillingTask.parse(sp.get("billingTask")); } if (map.has("signup")) { state.login = "signup"; } else if (map.has("login")) { state.login = "login"; } else if (map.has("verified")) { state.login = "verified"; } else if (map.has("forgot-password")) { state.login = "forgot-password"; } if (sp.has("state")) { state.params!.state = sp.get("state")!; } if (sp.has("srcDocId")) { state.params!.srcDocId = sp.get("srcDocId")!; } if (sp.has("style")) { let style = sp.get("style"); if (style === "light") { style = "singlePage"; } state.params!.style = InterfaceStyle.parse(style); } if (sp.has("embed")) { const embed = state.params!.embed = isAffirmative(sp.get("embed")); // Turn view mode on if no mode has been specified, and not a fork. if (embed && !state.mode && !state.fork) { state.mode = "view"; } // Turn on single page style if no style has been specified. if (embed && !state.params!.style) { state.params!.style = "singlePage"; } } // Theme overrides if (sp.has("themeSyncWithOs")) { state.params!.themeSyncWithOs = isAffirmative(sp.get("themeSyncWithOs")); } if (sp.has("themeAppearance")) { const appearance = sp.get("themeAppearance"); if (appearance && themeAppearances.includes(appearance as ThemeAppearance)) { state.params!.themeAppearance = appearance as ThemeAppearance; } } if (sp.has("themeName")) { const themeName = sp.get("themeName"); if (themeName && themeNames.includes(themeName as ThemeName)) { state.params!.themeName = themeName as ThemeName; } } if (sp.has("details")) { state.params!.details = isAffirmative(sp.get("details")); } if (sp.has("compare")) { state.params!.compare = sp.get("compare")!; } if (sp.has("compareEmphasis")) { const compareEmphasis = sp.get("compareEmphasis")!; if (compareEmphasis && CompareEmphasis.guard(compareEmphasis)) { state.params!.compareEmphasis = compareEmphasis; } } const linkParameters = decodeLinkParameters(sp); if (linkParameters) { state.params!.linkParameters = linkParameters; } if (sp.has("assistantPrompt")) { state.params!.assistantPrompt = sp.get("assistantPrompt")!; } if (sp.has("assistantState")) { state.params!.assistantState = sp.get("assistantState")!; } if (location.hash) { const hash = location.hash; const hashParts = hash.split("."); const hashMap = new Map(); for (const part of hashParts) { if (part.startsWith("rr")) { hashMap.set(part.slice(0, 2), part.slice(2)); } else { hashMap.set(part.slice(0, 1), part.slice(1)); } } state.homePageTab = HomePageTab.parse(hashMap.get("#")); state.adminPanelTab = AdminPanelTab.parse(hashMap.get("#")); const anchor = PersistentAnchor.parse(hashMap.get("#")); if (anchor) { state.hash = { anchor, }; } else if (hashMap.has("#") && ["a1", "a2", "a3", "a4"].includes(hashMap.get("#") || "")) { const link: HashLink = {}; const keys = [ "sectionId", "rowId", "colRef", ] as ("sectionId" | "rowId" | "colRef")[]; for (const key of keys) { let ch: string; if (key === "rowId" && hashMap.has("rr")) { ch = "rr"; link.rickRow = true; } else { ch = key.substr(0, 1); if (!hashMap.has(ch)) { continue; } } const value = hashMap.get(ch); if (key === "rowId" && value === "new") { link[key] = "new"; } else if (key === "rowId" && value?.includes("-")) { const rowIdParts = value.split("-").map(p => (p === "new" ? p : parseInt(p, 10))); link[key] = rowIdParts[0]; link.linkingRowIds = rowIdParts.slice(1); } else { link[key] = parseInt(value!, 10); } } if (hashMap.get("#") === "a2") { link.popup = true; } else if (hashMap.get("#") === "a3") { link.recordCard = true; } else if (hashMap.get("#") === "a4") { link.comments = true; } state.hash = link; } state.welcomeTour = hashMap.get("#") === "repeat-welcome-tour"; state.docTour = hashMap.get("#") === "repeat-doc-tour"; state.manageUsers = hashMap.get("#") === "manage-users"; state.createTeam = hashMap.get("#") === "create-team"; state.upgradeTeam = hashMap.get("#") === "upgrade-team"; } return state; } export function decodeLinkParameters(sp: URLSearchParams) { let linkParameters: Record | undefined = undefined; for (const [k, v] of sp.entries()) { if (k.endsWith("_")) { if (!linkParameters) { linkParameters = {}; } linkParameters[k.slice(0, k.length - 1)] = v; } } return linkParameters; } // Returns a function suitable for user with makeUrl/setHref/etc, which updates aclAsUser* // linkParameters in the current state, unsetting them if email is null. Optional extraState // allows setting other properties (e.g. 'docPage') at the same time. export function userOverrideParams(email: string | null, extraState?: IGristUrlState) { return function(prevState: IGristUrlState): IGristUrlState { const combined = { ...prevState, ...extraState }; const linkParameters = clone(combined.params?.linkParameters) || {}; if (email) { linkParameters.aclAsUser = email; } else { delete linkParameters.aclAsUser; } delete linkParameters.aclAsUserId; return { ...combined, params: { ...combined.params, linkParameters } }; }; } /** * parseDocPage is a noop for special pages, otherwise parse to integer */ function parseDocPage(p: string): IDocPage { if (SpecialDocPage.guard(p)) { return p; } return parseInt(p, 10); } /** * Parses the URL like "foo.bar.baz" into the pair {org: "foo", base: ".bar.baz"}. * Port is allowed and included into base. * * The "base" part is required to have at least two periods. The "org" part must pass * the subdomainRegex test. * * If there's no way to parse the URL into such a pair, then an empty object is returned. */ export function parseSubdomain(host: string | undefined): { org?: string, base?: string } { if (!host) { return {}; } const match = /^([^.]+)(\..+\..+)$/.exec(host.toLowerCase()); if (match) { const org = match[1]; const base = match[2]; if (subdomainRegex.exec(org)) { return { org, base }; } } // Host has nowhere to put a subdomain. return {}; } // Allowed localhost addresses. const localhostRegex = /^localhost(?::(\d+))?$/i; /** * Like parseSubdomain, but throws an error if neither of these cases apply: * - host can be parsed into a valid subdomain and a valid base domain. * - host is localhost:NNNN * An empty object is only returned when host is localhost:NNNN. */ export function parseSubdomainStrictly(host: string | undefined): { org?: string, base?: string } { if (!host) { throw new Error("host not known"); } const result = parseSubdomain(host); if (result.org) { return result; } if (!host.match(localhostRegex)) { throw new Error(`host not understood: ${host}`); } // Host is localhost[:NNNN], no org available. return {}; } /** * For a packaged version of Grist that requires activation, this * summarizes the current state. Not applicable to grist-core. * This is the thing that is send via sendAppPage (so this is embedded in HTML). */ export interface ActivationState { installationId: string; // Unique identifier for this installation. key?: { // Set when Grist is activated. expirationDate?: string; // ISO8601 date that Grist will need reactivation. daysLeft?: number; // Number of days until Grist will need reactivation. }, trial?: { // Present when installation has not yet been activated. days: number; // Max number of days allowed prior to activation. expirationDate: string; // ISO8601 date that Grist will get cranky. daysLeft: number; // Number of days left until Grist will get cranky. }, needKey?: boolean; // Set when Grist is cranky and demanding activation. error?: string; // Present when there is an error reading the key. features?: PlanFeatures; // Features available in this installation. current?: Partial; // Usage of features in this installation. grace?: { daysLeft: number; // Number of days left in grace period. graceStarted: string; // ISO8601 date when grace period started. } } export interface LatestVersionAvailable { version: string; isNewer: boolean; isCritical: boolean; dateChecked: number; releaseUrl?: string; } /** * These settings get sent to the client along with the loaded page. At the minimum, the browser * needs to know the URL of the home API server (e.g. api.getgrist.com). */ export interface GristLoadConfig { // URL of the Home API server for the browser client to use. homeUrl: string | null; // When loading /doc/{docId}, we include the id used to assign the document (this is the docId). assignmentId?: string; // Org or "subdomain". When present, this overrides org information from the hostname. We rely // on this for custom domains, but set it generally for all pages. org?: string; // Makes the Grist frontend access the Grist instance using its current URL in the browser, rather than APP_HOME_URL. // Used to simplify setup of single-domain (no subdomain / doc worker) installations. serveSameOrigin?: boolean; // Base domain for constructing new URLs, should start with "." and not include port, e.g. // ".getgrist.com". It should be unset for localhost operation and in single-org mode. baseDomain?: string; // In single-org mode, this is the single well-known org. Suppress any org selection UI. singleOrg?: string; // Url for support for the browser client to use. helpCenterUrl?: string; // Url for terms of service for the browser client to use termsOfServiceUrl?: string; // Url for free coaching call scheduling for the browser client to use. freeCoachingCallUrl?: string; // Url for "contact support" button on Grist's "not found" error page contactSupportUrl?: string; // Url for webinars. webinarsUrl?: string; // When set, this directs the client to encode org information in path, not in domain. pathOnly?: boolean; // Type of error page to show. This is used for pages such as "signed-out" and "not-found", // which don't include the full app. errPage?: string; // When errPage is a generic "other-error", this is the message to show. errMessage?: string; // When errPage is a generic page, not an error, this is additional details to show. errDetails?: Record; // When an error page is shown in response to a request for an URL, this is the URL that was // originally requested — this may not be the URL we're responding to, because of an // intermediate redirect in case of e.g. an OIDC sign-in. // The error page will set the browser's current URL to that, so that the user can // retry by simply refreshing the page. errTargetUrl?: string; // URL for client to use for untrusted content. pluginUrl?: string; // Stripe API key for use on the client. stripeAPIKey?: string; // BeaconID for the support widget from HelpScout. helpScoutBeaconId?: string; // If set, enable anonymous sharing UI elements. supportAnon?: boolean; // If set, enable anonymous playground. enableAnonPlayground?: boolean; // If set, allow non-admins to create new organizations canAnyoneCreateOrgs?: boolean; // If set, allow access to each user's personal organization enablePersonalOrgs?: boolean; // If set, allow selection of the specified engines. // TODO: move this list to a separate endpoint. supportEngines?: EngineCode[]; // Max upload allowed for imports (except .grist files), in bytes; 0 or omitted for unlimited. maxUploadSizeImport?: number; // Max upload allowed for attachments, in bytes; 0 or omitted for unlimited. maxUploadSizeAttachment?: number; // Pre-fetched call to getDoc for the doc being loaded. getDoc?: { [id: string]: Document }; // Pre-fetched call to getWorker for the doc being loaded. getWorker?: { [id: string]: string | null }; // The timestamp when this gristConfig was generated. timestampMs: number; // Google Client Id, used in Google integration (ex: Google Drive Plugin) googleClientId?: string; // Max scope we can request for accessing files from Google Drive. // Default used by Grist is https://www.googleapis.com/auth/drive.file: // View and manage Google Drive files and folders that you have opened or created with this app. // More on scopes: https://developers.google.com/identity/protocols/oauth2/scopes#drive googleDriveScope?: string; // List of registered plugins (used by HomePluginManager and DocPluginManager) plugins?: LocalPlugin[]; // If additional custom widgets (besides the Custom URL widget) should be shown in // the custom widget gallery. enableWidgetRepository?: boolean; // Whether there is somewhere for survey data to go. survey?: boolean; // Google Tag Manager id. Currently only used to load tag manager for reporting new sign-ups. tagManagerId?: string; activation?: ActivationState; // Latest Grist release available latestVersionAvailable?: LatestVersionAvailable; // Is automatic version checking allowed? automaticVersionCheckingAllowed?: boolean; // List of enabled features. features?: IFeature[]; // String to append to the end of the HTML document.title pageTitleSuffix?: string; // If custom CSS should be included in the head of each page. enableCustomCss?: boolean; // Supported languages for the UI. By default only english (en) is supported. supportedLngs?: readonly string[]; // Loaded namespaces for translations. namespaces?: readonly string[]; assistant?: AssistantConfig; permittedCustomWidgets?: IAttachedCustomWidget[]; // Email address of the support user. supportEmail?: string; // Current user locale, read from the user options; userLocale?: string; // Telemetry config. telemetry?: TelemetryConfig; // The Grist deployment type (e.g. core, enterprise). deploymentType?: GristDeploymentType; // Force enterprise deployment? For backwards compatibility with grist-ee Docker image forceEnableEnterprise?: boolean; // The org containing public templates and tutorials. templateOrg?: string | null; // The doc id of the tutorial shown during onboarding. onboardingTutorialDocId?: string; // The id of the Youtube video to show for the onboarding onboardingTutorialVideoId?: string; // Whether to show the "Delete Account" button in the account page. canCloseAccount?: boolean; experimentalPlugins?: boolean; // If backend has an email service for sending notifications. notifierEnabled?: boolean; // Set on /admin pages only, when AdminControls are available and should be enabled in UI. adminControls?: boolean; formFraming?: FormFraming; adminDefinedUrls?: string; // Maximum users to display for user presence features (e.g. active user list) userPresenceMaxUsers?: number; warnBeforeSharingPublicly?: boolean; // Whether there is a parent process that can restart Grist. runningUnderSupervisor?: boolean; } export const Features = StringUnion( "helpCenter", "billing", "templates", "createSite", "multiSite", "multiAccounts", "importFromAirtable", "sendToDrive", "tutorials", "supportGrist", "themes", ); export type IFeature = typeof Features.type; // Features that are enabled, even if not explicitly listed in GRIST_UI_FEATURES. // These should be still be disabled if listed in GRIST_HIDE_UI_ELEMENTS. export const ImplicitlyEnabledFeatures: IFeature[] = ["importFromAirtable"]; export function isFeatureEnabled(feature: IFeature): boolean { return (getGristConfig().features || []).includes(feature); } export function getPageTitleSuffix(config?: GristLoadConfig) { return config?.pageTitleSuffix ?? " - Grist"; } export interface TelemetryConfig { telemetryLevel: TelemetryLevel; } export const GristDeploymentTypes = StringUnion("saas", "core", "enterprise", "electron", "static"); export type GristDeploymentType = typeof GristDeploymentTypes.type; // Acceptable org subdomains are alphanumeric (hyphen also allowed) and of // non-zero length. const subdomainRegex = /^[-a-z0-9]+$/i; export interface OrgParts { subdomain: string | null; orgFromHost: string | null; orgFromPath: string | null; pathRemainder: string; mismatch: boolean; } /** * Returns true if code is running in client, false if running in server. */ export function isClient() { return (typeof window !== "undefined") && window && window.location?.hostname; } function getCustomizableValue( clientSideConfigKey: keyof GristLoadConfig, serverSideEnvVar: keyof NodeJS.ProcessEnv, ) { return isClient() ? (window as any).gristConfig?.[clientSideConfigKey] : process.env[serverSideEnvVar]; } /** * Returns a known org "subdomain" if Grist is configured in single-org mode * (GRIST_SINGLE_ORG= on the server) or if the page includes an org in gristConfig. */ export function getSingleOrg(): string | null { return getCustomizableValue("singleOrg", "GRIST_SINGLE_ORG") || null; } export function getHelpCenterUrl(): string { const defaultUrl = "https://support.getgrist.com"; return getCustomizableValue("helpCenterUrl", "GRIST_HELP_CENTER") || defaultUrl; } export function getOnboardingVideoId(): string { const defaultId = "56AieR9rpww"; return getCustomizableValue("onboardingTutorialVideoId", "GRIST_ONBOARDING_VIDEO_ID") || defaultId; } export function getTermsOfServiceUrl(): string | undefined { return getCustomizableValue("termsOfServiceUrl", "GRIST_TERMS_OF_SERVICE_URL") || undefined; } export function getFreeCoachingCallUrl(): string { const defaultUrl = "https://calendly.com/grist-team/grist-free-coaching-call"; return getCustomizableValue("freeCoachingCallUrl", "FREE_COACHING_CALL_URL") || defaultUrl; } export function getContactSupportUrl(): string { const defaultUrl = "https://www.getgrist.com/contact"; return getCustomizableValue("contactSupportUrl", "GRIST_CONTACT_SUPPORT_URL") || defaultUrl; } export function getWebinarsUrl(): string { const defaultUrl = "https://www.getgrist.com/webinars/grist-101-new-users-guide"; return getCustomizableValue("webinarsUrl", "GRIST_WEBINARS_URL") || defaultUrl; } export function getMaxUploadSizeAttachmentMB(): number { const value = getCustomizableValue("maxUploadSizeAttachment", "GRIST_MAX_UPLOAD_ATTACHMENT_MB"); return Number(value) || Infinity; } /** * Returns true if org must be encoded in path, not in domain. Determined from * gristConfig on the client. On the server, returns true if the host is * supplied and is 'localhost', or if GRIST_ORG_IN_PATH is set to 'true'. */ export function isOrgInPathOnly(host?: string): boolean { if (isClient()) { const gristConfig: GristLoadConfig = (window as any).gristConfig; return (gristConfig?.pathOnly) || false; } else { if (host?.match(localhostRegex)) { return true; } return (process.env.GRIST_ORG_IN_PATH === "true"); } } // Extract an organization name from the host. Returns null if an organization name // could not be recovered. Organization name may be overridden by server configuration. export function getOrgFromHost(reqHost: string): string | null { const singleOrg = getSingleOrg(); if (singleOrg) { return singleOrg; } if (isOrgInPathOnly()) { return null; } return parseSubdomain(reqHost).org || null; } /** * Get any information about an organization that is embedded in the host name or the * path. * For example, on nasa.getgrist.com, orgFromHost and subdomain will be set to "nasa". * On localhost:8000/o/nasa, orgFromPath and subdomain will be set to "nasa". * On nasa.getgrist.com/o/nasa, orgFromHost, orgFromPath, and subdomain will all be "nasa". * On spam.getgrist.com/o/nasa, orgFromHost will be "spam", orgFromPath will be "nasa", * subdomain will be null, and mismatch will be true. */ export function extractOrgParts(reqHost: string | undefined, reqPath: string): OrgParts { let orgFromHost: string | null = getSingleOrg(); if (!orgFromHost && reqHost) { orgFromHost = getOrgFromHost(reqHost); if (orgFromHost) { // Some subdomains are shared, and do not reflect the name of an organization. // See /documentation/urls.md for a list. if (/^(api|v1-.*|doc-worker-.*)$/.test(orgFromHost)) { orgFromHost = null; } } } const part = parseFirstUrlPart("o", reqPath); if (part.value) { const orgFromPath = part.value.toLowerCase(); const mismatch = Boolean(orgFromHost && orgFromPath && (orgFromHost !== orgFromPath)); const subdomain = mismatch ? null : orgFromPath; return { orgFromHost, orgFromPath, pathRemainder: part.path, mismatch, subdomain }; } return { orgFromHost, orgFromPath: null, pathRemainder: reqPath, mismatch: false, subdomain: orgFromHost }; } /** * When a prefix is extracted from the path, the remainder of the path may be empty. * This method makes sure there is at least a "/". */ export function sanitizePathTail(path: string | undefined) { path = path || "/"; return (path.startsWith("/") ? "" : "/") + path; } /* * If path starts with /{tag}/{value}{/rest}, returns value and the remaining path (/rest). * Otherwise, returns value of undefined and the path unchanged. * E.g. parseFirstUrlPart('o', '/o/foo/bar') returns {value: 'foo', path: '/bar'}. */ export function parseFirstUrlPart(tag: string, path: string): { value?: string, path: string } { const match = path.match(/^\/([^/?#]+)\/([^/?#]+)(.*)$/); if (match?.[1] === tag) { return { value: match[2], path: sanitizePathTail(match[3]) }; } else { return { path }; } } /** * The internal structure of a UrlId. There is no internal structure, * except in the following cases. The id may be for a fork, in which * case the fork has a separate id, and a user id may also be embedded * to track ownership. The id may be a share key, in which case it * has some special syntax to identify it as so. */ export interface UrlIdParts { trunkId: string; forkId?: string; forkUserId?: number; snapshotId?: string; shareKey?: string; } // Parse a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId // or trunkId[....]~v=snapshotId // or shareKey export function parseUrlId(urlId: string): UrlIdParts { let snapshotId: string | undefined; const parts = urlId.split("~"); const bareParts = parts.filter(part => !part.includes("v=")); for (const part of parts) { if (part.startsWith("v=")) { snapshotId = decodeURIComponent(part.substr(2).replace(/_/g, "%")); } } const trunkId = bareParts[0]; // IDs starting with SHARE_KEY_PREFIX are in fact shares. const shareKey = removePrefix(trunkId, SHARE_KEY_PREFIX) || undefined; return { trunkId: bareParts[0], forkId: bareParts[1], forkUserId: (bareParts[2] !== undefined) ? parseInt(bareParts[2], 10) : undefined, snapshotId, shareKey, }; } // Construct a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId // or trunkId[....]~v=snapshotId export function buildUrlId(parts: UrlIdParts): string { let token = [parts.trunkId, parts.forkId, parts.forkUserId].filter(x => x !== undefined).join("~"); if (parts.snapshotId) { // This could be an S3 VersionId, about which AWS makes few promises. // encodeURIComponent leaves untouched the following: // alphabetic; decimal; any of: - _ . ! ~ * ' ( ) // We further encode _.!~*'() to fit within existing limits on what characters // may be in a docId (leaving just the hyphen, which is permitted). The limits // could be loosened, but without much benefit. const codedSnapshotId = encodeURIComponent(parts.snapshotId) .replace(/[_.!~*'()-]/g, ch => `_${ch.charCodeAt(0).toString(16).toUpperCase()}`) .replace(/%/g, "_"); token = `${token}~v=${codedSnapshotId}`; } return token; } /** * Values that may be encoded in a hash in a document url. */ export interface HashLink { sectionId?: number; rowId?: UIRowId; colRef?: number; popup?: boolean; comments?: boolean; // Whether to show comments in the popup. rickRow?: boolean; recordCard?: boolean; linkingRowIds?: UIRowId[]; anchor?: string; } /** * Encode a HashLink as a string to include as the URL fragment (hash prooperty). For example, * in https://templates.getgrist.com/doc/lightweight-crm#a1.s1.r7.c2, the "a1.s1.r7.c2" portion is * the anchor link. The parts have the following meaning: * a = identifies the type of link (1 is normal, 2 for popup, 3 for record-card) * s = sectionId (rowId of the page-widget) * r = rowId (of the actual row in the user table) * c = colRef (rowId of the column's metadata record) */ export function makeAnchorLinkValue(hash: HashLink): string { const hashParts: string[] = []; if (hash.rowId || hash.popup || hash.recordCard) { if (hash.comments) { hashParts.push("a4"); } else if (hash.recordCard) { hashParts.push("a3"); } else if (hash.popup) { hashParts.push("a2"); } else if (hash.anchor) { hashParts.push(hash.anchor); } else { hashParts.push("a1"); } for (const key of ["sectionId", "rowId", "colRef"] as (keyof HashLink)[]) { let enhancedRowId: string | undefined; if (key === "rowId" && hash.linkingRowIds?.length) { enhancedRowId = [hash.rowId, ...hash.linkingRowIds].join("-"); } const partValue = enhancedRowId ?? hash[key]; if (partValue) { const partKey = key === "rowId" && hash.rickRow ? "rr" : key[0]; hashParts.push(`${partKey}${partValue}`); } } } return hashParts.join("."); } // Check whether a urlId is a prefix of the docId, and adequately long to be // a candidate for use in prettier urls. function shouldIncludeSlug(doc: { id: string, urlId: string | null }): boolean { if (!doc.urlId || doc.urlId.length < MIN_URLID_PREFIX_LENGTH) { return false; } return doc.id.startsWith(doc.urlId) || doc.urlId.startsWith(SHARE_KEY_PREFIX); } // Convert the name of a document into a slug. The slugify library normalizes unicode characters, // replaces those with a reasonable ascii representation. Only alphanumerics are retained, and // spaces are replaced with hyphens. function nameToSlug(name: string): string { return slugify(name, { strict: true }); } // Returns a slug for the given docId/urlId/name, or undefined if a slug should // not be used. export function getSlugIfNeeded(doc: { id: string, urlId: string | null, name: string }): string | undefined { if (!shouldIncludeSlug(doc)) { return; } return nameToSlug(doc.name); } /** * It is possible we want to remap Grist URLs in some way - specifically, * grist-static does this. We allow for a hook that is called after * encoding state as a URL, and a hook that is called before decoding * state from a URL. */ export interface UrlTweaks { /** * Tweak an encoded URL. Operates on the URL directly, in place. */ postEncode?(options: { url: URL, parts: string[], state: IGristUrlState, baseLocation: Location | URL, }): void; /** * Tweak a URL prior to decoding it. Operates on the URL directly, in place. */ preDecode?(options: { url: URL, }): void; } function withAdminDefinedUrls(defaultUrls: ICommonUrls): ICommonUrls { const adminDefinedUrlsStr = getCustomizableValue("adminDefinedUrls", "GRIST_CUSTOM_COMMON_URLS"); if (!adminDefinedUrlsStr) { return defaultUrls; } let adminDefinedUrls; try { adminDefinedUrls = JSON.parse(adminDefinedUrlsStr); } catch (e) { throw new Error("The JSON passed to GRIST_CUSTOM_COMMON_URLS is malformed"); } const merged = { ...defaultUrls, ...(adminDefinedUrls), }; ICommonUrlsChecker.strictCheck(merged); return merged; } ================================================ FILE: app/common/gutil.ts ================================================ import { BindableValue, Computed, DomElementMethod, Holder, IDisposableOwner, IKnockoutReadObservable, ISubscribable, Listener, MultiHolder, Observable, subscribeElem, UseCB, UseCBOwner, } from "grainjs"; import { Observable as KoObservable } from "knockout"; import identity from "lodash/identity"; // Some definitions have moved to be used by plugin API. export { arrayRepeat } from "app/plugin/gutil"; export const UP_TRIANGLE = "\u25B2"; export const DOWN_TRIANGLE = "\u25BC"; const EMAIL_RE = new RegExp("^\\w[\\w%+/='-]*(\\.[\\w%+/='-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z" + "0-9])?\\.)+[A-Za-z]{2,24}$", "u"); // Returns whether str starts with prefix. (Note that this implementation avoids creating a new // string, and only checks a single location.) export function startsWith(str: string, prefix: string): boolean { return str.lastIndexOf(prefix, 0) === 0; } // Returns whether str ends with suffix. export function endsWith(str: string, suffix: string): boolean { return str.includes(suffix, str.length - suffix.length); } // If str starts with prefix, removes it and returns what remains. Otherwise, returns null. export function removePrefix(str: string, prefix: string): string | null { return startsWith(str, prefix) ? str.slice(prefix.length) : null; } // If str ends with suffix, removes it and returns what remains. Otherwise, returns null. export function removeSuffix(str: string, suffix: string): string | null { return endsWith(str, suffix) ? str.slice(0, str.length - suffix.length) : null; } export function removeTrailingSlash(str: string): string { const result = removeSuffix(str, "/"); return result === null ? str : result; } // Expose .padStart. The version of node we use has it, but they typings // need the es2017 typescript target. TODO: replace once typings in place. export function padStart(str: string, targetLength: number, padString: string) { return (str as any).padStart(targetLength, padString); } // Capitalizes every word in a string. export function capitalize(str: string): string { return str.replace(/\b[a-z]/gi, c => c.toUpperCase()); } // Capitalizes the first word in a string. export function capitalizeFirstWord(str: string): string { return str.replace(/\b[a-z]/i, c => c.toUpperCase()); } // Returns whether the string n represents a valid number. // http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric export function isNumber(n: string): boolean { // This wasn't right for a long time: isFinite() is key to failing on strings like "5a". return !isNaN(parseFloat(n)) && isFinite(n as any); } /** * Returns a value clamped to the given min-max range. * @param {Number} value - some numeric value. * @param {Number} min - minimum value allowed. * @param {Number} max - maximum value allowed. Must have min <= max. * @returns {Number} - value restricted to the given range. */ export function clamp(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); } /** * Checks if ele is contained within the given bounds. * @param {Number} value * @param {Number} bound1 - does not have to be less than/equal to bound2 * @param {Number} bound2 * @returns {Boolean} - True/False */ export function between(value: number, bound1: number, bound2: number): boolean { const lower = Math.min(bound1, bound2); const upper = Math.max(bound1, bound2); return lower <= value && value <= upper; } /** * Returns the positive modulo of x by n. (Javascript default allows negatives) */ export function mod(x: number, n: number): number { return ((x % n) + n) % n; } /** * Returns a number that is n rounded down to the next nearest number divisible by m */ export function roundDownToMultiple(n: number, m: number): number { return Math.floor(n / m) * m; } /** * Returns the first argument unless it's undefined, in which case returns the second one. */ export function undefDefault(x: T | undefined, y: T): T { return (x !== void 0) ? x : y; } // for typescript 4 // type Undef = T extends [infer A, ...infer B] ? undefined extends A ? NonNullable | Undef : A : unknown; type Undef1 = T extends [infer A] ? A : unknown; type Undef2 = T extends [infer A, infer B] ? undefined extends A ? NonNullable | Undef1<[B]> : A : Undef1; type Undef3 = T extends [infer A, infer B, infer C] ? undefined extends A ? NonNullable | Undef2<[B, C]> : A : Undef2; type Undef = T extends [infer A, infer B, infer C, infer D] ? undefined extends A ? NonNullable | Undef3<[B, C, D]> : A : Undef3; /* Undef can detect correct type that will be returned as a first defined value: const t1: number = undef(1, 1 as number | undefined); const t1: number | undefined = undef(2 as number | undefined, 3 as number | undefined); const t3: number = undef(3 as number | undefined, undefined, 4); const t4: number = undef(1, ''); const t5: number = undef(1 as number | undefined, 4); const t6: string = undef('1', 2); const t7: string | number = undef(undefined, 2 as number | undefined, '3'); const t8: string = undef(undefined, undefined, '3'); const t9: string = undef(undefined, '2' as string | undefined, '3'); const ta: string | number | undefined = undef(undefined, '2' as string | undefined, 3 as number | undefined); const tb: string | number = undef(undefined, '2' as string | undefined, 3 as number | undefined, 5); */ /** * Returns the first defined value from the list or unknown. * Use with typed result, so the typescript type checker can provide correct type. */ export function undef(...list: T): Undef { for (const value of list) { if (value !== undefined) { return value; } } return undefined as any; } /** * Returns the number representation of `value`, or `defaultVal` if it cannot * be represented as a valid number. */ export function numberOrDefault(value: unknown, defaultVal: T): number | T { if (typeof value === "number") { return !Number.isNaN(value) ? value : defaultVal; } else if (typeof value === "string") { const maybeNumber = Number.parseFloat(value); return !Number.isNaN(maybeNumber) ? maybeNumber : defaultVal; } else { return defaultVal; } } /** * Parses json and returns the result, or returns defaultVal if parsing fails. */ export function safeJsonParse(json: string, defaultVal: any): any { try { return json !== "" && json !== undefined ? JSON.parse(json) : defaultVal; } catch (e) { return defaultVal; } } /** * Just like encodeURIComponent, but does not encode slashes. Slashes don't hurt to be included in * URL parameters, and look much friendlier not encoded. */ export function encodeQueryParam(str: string | number | undefined): string { return encodeURIComponent(String(str === undefined ? null : str)).replace(/%2F/g, "/"); } /** * Encode an object into a querystring ("key=value&key2=value2"). * This is similar to JQuery's $.param, but only works on shallow objects. */ export function encodeQueryParams(obj: { [key: string]: string | number | undefined }): string { return Object.keys(obj).map((k: string) => encodeQueryParam(k) + "=" + encodeQueryParam(obj[k])).join("&"); } /** * Return a list of the words in the string, using the given separator string. At most * maxNumSplits splits are done, so the result will have at most maxNumSplits + 1 elements (this * is the main difference from how JS built-in string.split() works, and similar to Python split). * @param {String} str: String to split. * @param {String} sep: Separator to split on. * @param {Number} maxNumSplits: Maximum number of splits to do. * @return {Array[String]} Array of words, of length at most maxNumSplits + 1. */ export function maxsplit(str: string, sep: string, maxNumSplits: number): string[] { const result: string[] = []; let start = 0, pos; for (let i = 0; i < maxNumSplits; i++) { pos = str.indexOf(sep, start); if (pos === -1) { break; } result.push(str.slice(start, pos)); start = pos + sep.length; } result.push(str.slice(start)); return result; } // Compare arrays of scalars for equality. export function arraysEqual(a: any[], b: any[]): boolean { if (a === b) { return true; } if (!a || !b) { return false; } if (a.length !== b.length) { return false; } for (let i = 0; i < a.length; i++) { if (a[i] !== b[i]) { return false; } } return true; } // Gives a set representing the set difference a - b. export function setDifference(a: Set, b: Set): Set { const c = new Set(); for (const ai of a) { if (!b.has(ai)) { c.add(ai); } } return c; } // Like array.indexOf, but works with array-like objects like HTMLCollection. export function indexOf(arrayLike: ArrayLike, item: T): number { return Array.prototype.indexOf.call(arrayLike, item); } /** * Removes a value from the given array. Only the first instance is removed. * Returns true on success, false if the value was not found. */ export function arrayRemove(array: T[], value: T): boolean { const index = array.indexOf(value); if (index === -1) { return false; } array.splice(index, 1); return true; } /** * Inserts value into the array before nextValue, or at the end if nextValue is not found. */ export function arrayInsertBefore(array: T[], value: T, nextValue: T): void { const index = array.indexOf(nextValue); if (index === -1) { array.push(value); } else { array.splice(index, 0, value); } } /** * Extends the first array with the second. Like native push, but adds all values in anotherArray. */ export function arrayExtend(array: T[], anotherArray: T[]): void { for (let i = 0, len = anotherArray.length; i < len; i++) { array.push(anotherArray[i]); } } /** * Copies count items from fromArray to toArray, copying in a forward direction (which matters * when the arrays are the same and source and destination indices overlap). * * See test/common/arraySplice.js for alternative implementations with timings, from which this * one is chosen as consistently among the faster ones. */ export function arrayCopyForward(toArray: T[], toStart: number, fromArray: ArrayLike, fromStart: number, count: number): void { const end = toStart + count; for (const xend = end - 7; toStart < xend; fromStart += 8, toStart += 8) { toArray[toStart] = fromArray[fromStart]; toArray[toStart + 1] = fromArray[fromStart + 1]; toArray[toStart + 2] = fromArray[fromStart + 2]; toArray[toStart + 3] = fromArray[fromStart + 3]; toArray[toStart + 4] = fromArray[fromStart + 4]; toArray[toStart + 5] = fromArray[fromStart + 5]; toArray[toStart + 6] = fromArray[fromStart + 6]; toArray[toStart + 7] = fromArray[fromStart + 7]; } for (; toStart < end; ++fromStart, ++toStart) { toArray[toStart] = fromArray[fromStart]; } } /** * Copies count items from fromArray to toArray, copying in a backward direction (which matters * when the arrays are the same and source and destination indices overlap). * * See test/common/arraySplice.js for alternative implementations with timings, from which this * one is chosen as consistently among the faster ones. */ export function arrayCopyBackward(toArray: T[], toStart: number, fromArray: ArrayLike, fromStart: number, count: number): void { let i = toStart + count - 1, j = fromStart + count - 1; for (const xStart = toStart + 7; i >= xStart; i -= 8, j -= 8) { toArray[i] = fromArray[j]; toArray[i - 1] = fromArray[j - 1]; toArray[i - 2] = fromArray[j - 2]; toArray[i - 3] = fromArray[j - 3]; toArray[i - 4] = fromArray[j - 4]; toArray[i - 5] = fromArray[j - 5]; toArray[i - 6] = fromArray[j - 6]; toArray[i - 7] = fromArray[j - 7]; } for (; i >= toStart; --i, --j) { toArray[i] = fromArray[j]; } } /** * Appends a slice of fromArray to the end of toArray. * * See test/common/arraySplice.js for alternative implementations with timings, from which this * one is chosen as consistently among the faster ones. */ export function arrayAppend(toArray: T[], fromArray: ArrayLike, fromStart: number, count: number): void { if (count === 1) { toArray.push(fromArray[fromStart]); } else { const len = toArray.length; toArray.length = len + count; arrayCopyForward(toArray, len, fromArray, fromStart, count); } } /** * Splices array arrToInsert into target starting at the given start index. * This implementation tries to be smart by avoiding allocations, appending to the array * contiguously, then filling in the gap. * * See test/common/arraySplice.js for alternative implementations with timings, from which this * one is chosen as consistently among the faster ones. */ export function arraySplice(target: T[], start: number, arrToInsert: ArrayLike): T[] { const origLen = target.length; const tailLen = origLen - start; const insLen = arrToInsert.length; target.length = origLen + insLen; if (insLen > tailLen) { arrayCopyForward(target, origLen, arrToInsert, tailLen, insLen - tailLen); arrayCopyForward(target, start + insLen, target, start, tailLen); arrayCopyForward(target, start, arrToInsert, 0, tailLen); } else { arrayCopyForward(target, origLen, target, origLen - insLen, insLen); arrayCopyBackward(target, start + insLen, target, start, tailLen - insLen); arrayCopyForward(target, start, arrToInsert, 0, insLen); } return target; } // Type for a compare func that returns a positive, negative, or zero value, as used for sorting. export type CompareFunc = (a: T, b: T) => number; /** * Returns the index at which the given element can be inserted to keep the array sorted. * This is equivalent to underscore's sortedIndex and python's bisect_left. * @param {Array} array - sorted array of elements based on the given compareFunc * @param {object} elem - object to be inserted in the given array * @param {function} compareFunc - compares 2 elements. Returns a pos value if the 1st element is * larger, 0 if they're equal, a neg value if the 2nd is larger. */ export function sortedIndex(array: ArrayLike, elem: T, compareFunc: CompareFunc): number { let lo = 0, mid; let hi = array.length; if (array.length === 0) { return 0; } while (lo < hi) { mid = Math.floor((lo + hi) / 2); if (compareFunc(array[mid], elem) < 0) { // mid < elem lo = mid + 1; } else { hi = mid; } } return lo; } /** * Returns true if an array contains duplicate values. * Values are considered equal if their toString() representations are equal. */ export function hasDuplicates(array: any[]): boolean { const prevVals = Object.create(null); for (const value of array) { if (value in prevVals) { return true; } prevVals[value] = true; } return false; } /** * Counts the number of items in array which satisfy the callback. */ export function countIf(array: readonly T[], callback: (item: T) => boolean): number { let count = 0; array.forEach((item) => { if (callback(item)) { count++; } }); return count; } /** * For two parallel arrays, calls mapFunc(a[i], b[i]) for each pair of corresponding elements, and * returns an array of the results. */ export function map2(array1: ArrayLike, array2: ArrayLike, mapFunc: (a: T, b: U) => V): V[] { const len = array1.length; const result: V[] = new Array(len); for (let i = 0; i < len; i++) { result[i] = mapFunc(array1[i], array2[i]); } return result; } /** * Takes a 2d array returns a new matrix with r rows and c columns * @param [Array] dataMatrix: a 2d array * @param [Number] r: final row length * @param [Number] c: final column length */ export function growMatrix(dataMatrix: T[][], r: number, c: number): T[][] { const colArr = dataMatrix.map(colVals => Array.from({ length: c }, (_v, k) => colVals[k % colVals.length]), ); return Array.from({ length: r }, (_v, k) => colArr[k % colArr.length]); } /** * Returns a function that compares two elements based on multiple sort keys and the * given compare functions. * Elements are compared using the sort key functions with index 0 having the greatest priority. * Subsequent sort key functions are used as tie breakers. * @param {function Array} sortKeyFuncs - a list of sort key functions. * @param {function Array} compareKeyFuncs - a list of comparison functions parallel to sortKeyFuncs * Each compare function must satisfy the comparison invariant: * If compare(a, b) > 0 then a > b, * If compare(a, b) < 0 then a < b, * If compare(a, b) == 0 then a == b, * @param {Array of 1/-1's} optAscending - Comparison on sortKeyFuncs[i] is inverted if optAscending[i] == -1 */ export function multiCompareFunc(sortKeyFuncs: readonly ((a: T) => U)[], compareFuncs: ArrayLike>, optAscending?: number[]): CompareFunc { if (sortKeyFuncs.length !== compareFuncs.length) { throw new Error("Number of sort key funcs must be the same as the number of compare funcs"); } const ascending = optAscending || sortKeyFuncs.map(() => 1); return function(a: T, b: T): number { let compareOutcome, keyA, keyB; for (let i = 0; i < compareFuncs.length; i++) { keyA = sortKeyFuncs[i](a); keyB = sortKeyFuncs[i](b); compareOutcome = compareFuncs[i](keyA, keyB); if (compareOutcome !== 0) { return ascending[i] * compareOutcome; } } return 0; }; } export function nativeCompare(a: T, b: T): number { return (a < b ? -1 : (a > b ? 1 : 0)); } /** * Creates a function that compares objects by a property value. */ export function propertyCompare(property: keyof T) { return function(a: T, b: T) { return nativeCompare(a[property], b[property]); }; } // TODO: In the future, locale should be a value associated with the document or the user. export const defaultLocale = "en-US"; export const defaultCollator = new Intl.Collator(defaultLocale); export const localeCompare = defaultCollator.compare; /** * A copy of python`s `setdefault` function. * Sets key in mapInst to value, if key is not already set. * @param {Map} mapInst: Instance of Map. * @param {Object} key: Key into the map. * @param {Object} value: Value to insert, possibly. */ export function setDefault(mapInst: Map, key: K, val: V): V { if (!mapInst.has(key)) { mapInst.set(key, val); } return mapInst.get(key)!; } /** * Similar to Python's `setdefault`: returns the key `key` from `mapInst`, or if it's not there, sets * it to the result buildValue(). */ export function getSetMapValue(mapInst: Map, key: K, buildValue: () => V): V { if (!mapInst.has(key)) { mapInst.set(key, buildValue()); } return mapInst.get(key)!; } /** * If key is in mapInst, remove it and return its value, else return `undefined`. * @param {Map} mapInst: Instance of Map. * @param {Object} key: Key into the map to remove. */ export function popFromMap(mapInst: Map, key: K): V | undefined { const value = mapInst.get(key); mapInst.delete(key); return value; } /** * For each encountered value in `values`, increment the corresponding counter in `valueCounts`. */ export function addCountsToMap(valueCounts: Map, values: Iterable, mapFunc: (v: any) => any = identity) { for (const v of values) { const mappedValue = mapFunc(v); valueCounts.set(mappedValue, (valueCounts.get(mappedValue) || 0) + 1); } } /** * Returns whether one Set is a subset of another. */ export function isSubset(smaller: Set, larger: Set): boolean { for (const value of smaller) { if (!larger.has(value)) { return false; } } return true; } /** * Merges the contents of two or more objects together into the first object, recursing into * nested objects and arrays (like jquery.extend(true, ...)). * @param {Object} target - The object to modify. Use {} to create a new merged object. * @param {Object} ... - Additional objects from which to copy properties into target. * @returns {Object} The first argument, target, modified. */ export function deepExtend(target: any, _varArgObjects: any): any { for (let i = 1; i < arguments.length; i++) { const object = arguments[i]; // Extend the base object for (const name in object) { if (!object.hasOwnProperty(name)) { continue; } let src = object[name]; if (src === target || src === undefined) { // Prevent one kind of infinite loop, as JQuery's extend does, and skip undefined values. continue; } if (src) { // Recurse if we're merging plain objects or arrays const tgt = target[name]; if (Array.isArray(src)) { src = deepExtend(tgt && Array.isArray(tgt) ? tgt : [], src); } else if (typeof src === "object") { src = deepExtend(tgt && typeof tgt === "object" ? tgt : {}, src); } } target[name] = src; } } // Return the modified object return target; } /** * Returns a human-readable string containing a number of bytes, KB, or MB. * @param {Number} bytes. Number of bytes. * @returns {String} A description such as "4.1KB". */ export function byteString(bytes: number): string { if (bytes < 1024) { return bytes + "B"; } else if (bytes < 1024 * 1024) { return (bytes / 1024).toFixed(1) + "KB"; } else { return (bytes / 1024 / 1024).toFixed(1) + "MB"; } } /** * Creates a new object mapping each key in keysArray to the value returned by callback. * @param {Array} keysArray - Array of strings to use as the properties of the returned object. * @param {Function} callback - Function that produces the value for each key. Called in the same * way as array.map() calls its callbacks. * @param {Object} optThisArg - Value to use as `this` when executing callback. * @returns {Object} - object mapping keys from `keysArray` to values returned by `callback`. */ export function mapToObject(keysArray: string[], callback: (key: string) => T, optThisArg: any): { [key: string]: T } { const values: T[] = keysArray.map(callback, optThisArg); const map: { [key: string]: T } = {}; for (let i = 0; i < keysArray.length; i++) { map[keysArray[i]] = values[i]; } return map; } /** * Remove the specified elements from the array, with the elements specified by * their index. The array arr is modified in-place. The indexes must be provided * in order, sorted lowest to highest, with no duplicates, or out-of-bound indices, * etc (this method does no error checking; it is used in place of lodash-pullAt * for performance reasons). */ export function pruneArray(arr: T[], indexes: number[]) { if (indexes.length === 0) { return; } if (indexes.length === 1) { arr.splice(indexes[0], 1); return; } const len = arr.length; let arrAt = 0; let indexesAt = 0; for (let i = 0; i < len; i++) { if (i === indexes[indexesAt]) { indexesAt++; continue; } if (i !== arrAt) { arr[arrAt] = arr[i]; } arrAt++; } arr.length = arrAt; } /** * A List of python identifiers; the result of running keywords.kwlist in Python 3.9, * plus additional illegal identifiers None, False, True * Using [] instead of new Array causes a "comprehension error" for some reason */ const _kwlist = ["False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield"]; /** * Given an arbitrary string, makes substitutions to make it a valid SQL/Python identifier. * Corresponds to sandbox/grist/gencode.sanitize_ident */ export function sanitizeIdent(ident: string, prefix?: string) { prefix = prefix || "c"; // Remove non-alphanumeric non-_ chars ident = ident.replace(/[^a-zA-Z0-9_]+/g, "_"); // Remove leading and trailing _ ident = ident.replace(/^_+|_+$/g, ""); // Place prefix at front if the beginning isn't a number ident = ident.replace(/^(?=[0-9])/g, prefix); // Append prefix until it is not python keyword while (_kwlist.includes(ident)) { ident = prefix + ident; } return ident; } /** * Clone a function, returning a function object that represents a brand new function with the * same code. If the same function is used with different argument types, it would prevent JS V8 * engine optimizations (or cause it to deoptimize it). If different clones are called with * different argument types, they can be optimized independently. * * As with all micro-optimizations, only do this when the optimization matters. */ export function cloneFunc(fn: Function): Function { /* jshint evil:true */ // suppress eval warning. return eval("(" + fn.toString() + ")"); } /** * Generates a random id using a sequence of uppercase alphanumeric characters * preceded by an optional prefix. */ export function genRandomId(len: number, optPrefix?: string): string { const chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; let ret = optPrefix || ""; for (let i = 0; i < len; i++) { ret += chars[Math.floor(Math.random() * chars.length)]; } return ret; } /** * Scans through two sorted arrays, calling a function on each item or pair of items * for every present key in order. * @param {Array} arrA - First array to scan. NOTE: Should be sorted by the key value. * @param {Array} arrB - Second array to scan. NOTE: Should be sorted by the key value. * @param {Function} callback - Called with an item from arrA as the first argument and an * item from arrB as the second. Called for every unique key in order, either with one of the * arguments null if the key is present only in one array, or both non-null if the key is * present in both arrays. NOTE: Key values should not be null. * @param {Function} optKeyFunc - Optional function to map each array value to a sort key. * Defaults to the identity function. */ export function sortedScan(arrA: ArrayLike, arrB: ArrayLike, callback: (a: T | null, B: U | null) => void, optKeyFunc?: (item: T | U) => any) { const keyFunc = optKeyFunc || identity; let i = 0, j = 0; while (i < arrA.length || j < arrB.length) { const a = arrA[i], b = arrB[j]; const keyA = i < arrA.length ? keyFunc(a) : null; const keyB = j < arrB.length ? keyFunc(b) : null; if (keyA !== null && (keyB === null || keyA < keyB)) { callback(a, null); i++; } else if (keyA === null || keyA > keyB) { callback(null, b); j++; } else { callback(a, b); i++; j++; } } } /** * Returns the time in ms to wait until attempting another connection. * @param {Number} attemptNumber - Reconnect attempt number starting at 0. * @param {Array} intervals - Array of reconnect intervals in ms. * @returns {Number} */ export function getReconnectTimeout(attemptNumber: number, intervals: ArrayLike): number { if (attemptNumber >= intervals.length) { // Add an additional wait time if already at max attempts. const timeout = intervals[intervals.length - 1]; return timeout + Math.random() * timeout; } else { return intervals[attemptNumber]; } } /** * Returns whether the given email is a valid formatted email string. * @param {String} email - Email to test. * @returns {Boolean} */ export function isEmail(email: string): boolean { return EMAIL_RE.test(email.toLowerCase()); } /* * Takes an observable and returns a promise for when the observable's value matches the given * predicate. It then unsubscribes from the observable, and returns its value. * If a predicate is not given, resolves to the observable values as soon as it's truthy. */ export function waitObs(observable: KoObservable, predicate: (value: T) => boolean = Boolean): Promise { return new Promise((resolve, _reject) => { const value = observable.peek(); if (predicate(value)) { return resolve(value); } const sub = observable.subscribe((val: T) => { if (predicate(val)) { sub.dispose(); resolve(val); } }); }); } /** * Same as waitObs but for grainjs observables. */ export async function waitGrainObs(observable: Observable): Promise>; export async function waitGrainObs(observable: Observable, predicate?: (value: T) => boolean): Promise; export async function waitGrainObs(observable: Observable, predicate: (value: T) => boolean = Boolean): Promise { let sub: Listener | undefined; const res: T = await new Promise((resolve, _reject) => { const value = observable.get(); if (predicate(value)) { return resolve(value); } sub = observable.addListener((val: T) => { if (predicate(val)) { resolve(val); } }); }); if (sub) { sub.dispose(); } return res; } // `dom.style` does not work here because custom css property (ie: `--foo`) needs to be set using // `style.setProperty` (credit: https://vanseodesign.com/css/custom-properties-and-javascript/). // TODO: consider making PR to fix `dom.style` in grainjs. export function inlineStyle(property: string, valueObs: BindableValue): DomElementMethod { return elem => subscribeElem(elem, valueObs, (val) => { elem.style.setProperty(property, String(val ?? "")); }); } /** * Class to maintain a chain of promise-returning callbacks. All scheduled callbacks will be * called in order as long as the previous one is successful. If a callback fails is rejected, * already-scheduled callbacks will be skipped, but newly-scheduled ones will be run. */ export class PromiseChain { private _last: Promise = Promise.resolve(); // Adds a callback to the chain. If the callback runs, the return value is the return value of // the callback. If it's skipped due to a failure earlier in the chain, the return value is the // rejection with the message "Skipped due to an earlier error". public add(nextCB: () => Promise): Promise { const next = this._last.catch(() => { throw new Error("Skipped due to an earlier error"); }).then(nextCB); // If any callback fails, all queued ones will be skipped. Here we reset the chain, so that // callbacks added later do get run. next.catch(() => { this._last = Promise.resolve(); }); this._last = next; return next; } } /** * Indicates if a hex color value, e.g. '#000000', is darker than the given value. * Darkness is measured from 0..255, where 0 is the darkest and 255 is the lightest. * * Taken from: https://stackoverflow.com/questions/12043187/how-to-check-if-hex-color-is-too-black */ export function isColorDark(hexColor: string, isDarkBelow: number = 220): boolean { const c = hexColor.substring(1); // strip # const rgb = parseInt(c, 16); // convert rrggbb to decimal // Extract RGB components const r = (rgb >> 16) & 0xff; const g = (rgb >> 8) & 0xff; const b = (rgb >> 0) & 0xff; const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b; // per ITU-R BT.709 return luma < isDarkBelow; } /** * Returns true if val is a valid hex color value. For instance: #aabbaa is valid, #aabba is not. Do * not accept neither short notation nor hex with transparency, ie: #aab, #aabb and #aabbaabb are * invalid. */ export function isValidHex(val: unknown): val is string { if (typeof val !== "string") { return false; } return /^#([0-9A-F]{6})$/i.test(val); } /** * Resolves to true if promise is still pending after msec milliseconds have passed. Otherwise * returns false, including when promise is rejected. */ export async function timeoutReached( msec: number, promise: Promise, options: { rethrow: boolean } = { rethrow: false }, ): Promise { // For test purposes, support negative timeout, by failing // immediately. if (msec < 0) { return true; } const timedOut = {}; // Be careful to clean up the timer after ourselves, so it doesn't remain in the event loop. let timer: NodeJS.Timeout; const delayPromise = new Promise((resolve) => { timer = setTimeout(() => resolve(timedOut), msec); }); try { const res = await Promise.race([promise, delayPromise]); return res == timedOut; } catch (err) { if (options.rethrow) { throw err; } return false; } finally { clearTimeout(timer!); } } /** * Returns a promise that resolves to true if promise takes longer than timeoutMsec to resolve. If not * or if promise throws returns false. Same as timeoutReached(), with reversed order of arguments. */ export async function isLongerThan(promise: Promise, timeoutMsec: number): Promise { return timeoutReached(timeoutMsec, promise); } /** * Returns true if the parameter, when rendered as a string, matches * 1, on, or true (case insensitively). Useful for processing query * parameters that may have been manually set. */ export function isAffirmative(parameter: any): boolean { return ["1", "on", "true", "yes"].includes(String(parameter).toLowerCase()); } /** * Returns whether a value is neither null nor undefined, with a type guard for the return type. * * This is particularly useful for filtering, e.g. if `array` includes values of type * T|null|undefined, then TypeScript can tell that `array.filter(isNonNullish)` has the type T[]. */ export function isNonNullish(value: T | null | undefined): value is T { return value !== null && value !== undefined; } /** * Ensures that a value is truthy, with a type guard for the return type. */ export function truthy(value: T | null | undefined): value is Exclude { return Boolean(value); } /** * Returns the value of both grainjs and knockout observable without creating a dependency. */ export const unwrap: UseCB = (obs: ISubscribable) => { if ("_getDepItem" in obs) { return obs.get(); } return (obs as ko.Observable).peek(); }; /** * Subscribes to BindableValue */ export function useBindable(use: UseCBOwner, obs: BindableValue): T { if (obs === null || obs === undefined) { return obs; } const smth = obs as any; // If knockout if (typeof smth === "function" && "peek" in smth) { return use(smth) as T; } // If grainjs Observable or Computed if (typeof smth === "object" && "_getDepItem" in smth) { return use(smth) as T; } // If use function ComputedCallback if (typeof smth === "function") { return smth(use) as T; } return obs as T; } /** * Useful helper for simple boolean negation. */ export const not = ( obs: Observable | IKnockoutReadObservable | boolean | undefined | null, ) => (use: UseCBOwner) => { if (typeof obs === "boolean") { return !obs; } if (obs === null || obs === undefined) { return true; } return !use(obs); }; /** * Get a set of up to `count` distinct values of `values`. */ export function getDistinctValues(values: readonly T[], count: number = Infinity): Set { const distinct = new Set(); // Add values to the set until it reaches the desired size, or until there are no more values. for (let i = 0; i < values.length && distinct.size < count; i++) { distinct.add(values[i]); } return distinct; } /** * Asserts that variable `name` has a non-nullish `value`. */ export function assertIsDefined(name: string, value: T): asserts value is NonNullable { if (value === undefined || value === null) { throw new Error(`Expected '${name}' to be defined, but received ${value}`); } } /** * Calls function `fn`, passes any thrown errors to function `recover`, and finally calls `fn` * once more if `recover` doesn't throw. */ export async function retryOnce(fn: () => Promise, recover: (e: unknown) => Promise): Promise { try { return await fn(); } catch (e) { await recover(e); return await fn(); } } /** * Checks if value is 'empty' (like null, undefined, empty string, empty array/set/map, empty object). * Values like 0, true, false are not empty. */ export function notSet(value: any) { return value === undefined || value === null || value === "" || (Array.isArray(value) && !value.length) || (typeof value === "object" && !Object.keys(value).length) || (["[object Map]", "[object Set"].includes(value.toString()) && !value.size); } /** * Checks if value is 'empty', if it is, returns the default value (which is null). */ export function ifNotSet(value: any, def: any = null) { return notSet(value) ? def : value; } /** * Creates a computed observable with a nested owner that can be used to dispose, * any disposables created inside the computed. Similar to domComputedOwned method. */ export function computedOwned( owner: IDisposableOwner, func: (owner: IDisposableOwner, use: UseCBOwner) => T, ): Computed { const holder = Holder.create(owner); return Computed.create(owner, (use) => { const computedOwner = MultiHolder.create(holder); return func(computedOwner, use); }); } export type Constructor = new (...args: any[]) => T; /** * Simple memoization function that caches the result of a function call based on its arguments. * Unlike lodash's memoize, it uses all arguments to generate the key. */ export function cached(fn: T): T { const dict = new Map(); const impl = (...args: any[]) => { const key = JSON.stringify(args); if (!dict.has(key)) { dict.set(key, (fn as any)(...args)); } return dict.get(key); }; return impl as any as T; } /** * Converts a duration string like "1d", "2h", "14m", "10s" to seconds. */ export function inSeconds(text: string): number { const match = text.match(/^(\d+)([smhd])$/); if (!match) { throw new Error(`Invalid duration: ${text}`); } const [, value, unit] = match; const seconds = parseInt(value, 10); switch (unit) { case "s": return seconds; case "m": return seconds * 60; case "h": return seconds * 60 * 60; case "d": return seconds * 60 * 60 * 24; default: throw new Error(`Invalid duration unit: ${unit}`); } } ================================================ FILE: app/common/isHiddenTable.ts ================================================ import { TableData } from "app/common/TableData"; import { UIRowId } from "app/plugin/GristAPI"; /** * Return whether a table (identified by the rowId of its metadata record) should * normally be hidden from the user (e.g. as an option in the page-widget picker). */ export function isHiddenTable(tablesData: TableData, tableRef: UIRowId): boolean { const tableId = tablesData.getValue(tableRef, "tableId") as string | undefined; // The `!tableId` check covers the case of censored tables (see isTableCensored() below). return !tableId || isSummaryTable(tablesData, tableRef) || tableId.startsWith("GristHidden_"); } /** * Return whether a table (identified by the rowId of its metadata record) is a * summary table. */ export function isSummaryTable(tablesData: TableData, tableRef: UIRowId): boolean { return tablesData.getValue(tableRef, "summarySourceTable") !== 0; } // Check if a table record (from _grist_Tables) is censored. // Metadata records get censored by clearing certain of their fields, so it's expected that a // record may exist even though various code should consider it as hidden. export function isTableCensored(tablesData: TableData, tableRef: UIRowId): boolean { const tableId = tablesData.getValue(tableRef, "tableId"); return !tableId; } /** * Returns whether this is a metadata table, as opposed to a user table. */ export function isMetadataTable(tableId: string): boolean { return tableId.startsWith("_grist"); } ================================================ FILE: app/common/loginProviders.ts ================================================ // Special provider that uses default user, which effectively means no authentication. export const MINIMAL_PROVIDER_KEY = "minimal"; // OpenID Connect provider key. export const OIDC_PROVIDER_KEY = "oidc"; // ForwardAuth provider key. export const FORWARD_AUTH_PROVIDER_KEY = "forward-auth"; // This provider is only available in grist-ee version. export const GRIST_CONNECT_PROVIDER_KEY = "grist-connect"; // getgrist.com provider key. export const GETGRIST_COM_PROVIDER_KEY = "getgrist.com"; // SAML provider key. export const SAML_PROVIDER_KEY = "saml"; // Deprecated/unmaintained providers, hidden unless already configured or active. export const DEPRECATED_PROVIDERS: string[] = [ GRIST_CONNECT_PROVIDER_KEY, ]; ================================================ FILE: app/common/marshal.ts ================================================ /** * Module for serializing data in the format of Python 'marshal' module. It's used for * communicating with the Python-based formula engine running in a Pypy sandbox. It supports * version 0 of python marshalling format, which is what the Pypy sandbox supports. * * Usage: * Marshalling: * const marshaller = new Marshaller({version: 2}); * marshaller.marshal(value); * marshaller.marshal(value); * const buf = marshaller.dump(); // Leaves the marshaller empty. * * Unmarshalling: * const unmarshaller = new Unmarshaller(); * unmarshaller.on('value', function(value) { ... }); * unmarshaller.push(buffer); * unmarshaller.push(buffer); * * In Python, and in the marshalled format, there is a distinction between strings and unicode * objects. In JS, there is a good correspondence to Uint8Array objects and strings, respectively. * Python unicode objects always become JS strings. JS Uint8Arrays always become Python strings. * * JS strings become Python unicode objects, but can be marshalled to Python strings with * 'stringToBuffer' option. Similarly, Python strings become JS Uint8Arrays, but can be * unmarshalled to JS strings if 'bufferToString' option is set. */ import { BigInt } from "app/common/BigInt"; import MemBuffer from "app/common/MemBuffer"; import { EventEmitter } from "events"; import * as util from "util"; export interface MarshalOptions { stringToBuffer?: boolean; version?: number; // True if we want keys in dicts to be buffers. // It is convenient to have some freedom here to simplify implementation // of marshaling for some SQLite wrappers. This flag was initially // introduced for a fork of Grist using better-sqlite3, and I don't // remember exactly what the issues were. keysAreBuffers?: boolean; } export interface UnmarshalOptions { bufferToString?: boolean; } function ord(str: string): number { return str.charCodeAt(0); } /** * Type codes used for python marshalling of values. * See pypy: rpython/translator/sandbox/_marshal.py. */ const marshalCodes = { NULL: ord("0"), NONE: ord("N"), FALSE: ord("F"), TRUE: ord("T"), STOPITER: ord("S"), ELLIPSIS: ord("."), INT: ord("i"), INT64: ord("I"), /* BFLOAT, for 'binary float', is an encoding of float that just encodes the bytes of the double in standard IEEE 754 float64 format. It is used by Version 2+ of Python's marshal module. Previously (in versions 0 and 1), the FLOAT encoding is used, which stores floats through their string representations. Version 0 (FLOAT) is mandatory for system calls within the sandbox, while Version 2 (BFLOAT) is recommended for Grist's communication because it is more efficient and faster to encode/decode */ BFLOAT: ord("g"), FLOAT: ord("f"), COMPLEX: ord("x"), LONG: ord("l"), STRING: ord("s"), INTERNED: ord("t"), STRINGREF: ord("R"), TUPLE: ord("("), LIST: ord("["), DICT: ord("{"), CODE: ord("c"), UNICODE: ord("u"), UNKNOWN: ord("?"), SET: ord("<"), FROZENSET: ord(">"), }; type MarshalCode = keyof typeof marshalCodes; // A little hack to test if the value is a 32-bit integer. Actually, for Python, int might be up // to 64 bits (if that's the native size), but this is simpler. // See http://stackoverflow.com/questions/3885817/how-to-check-if-a-number-is-float-or-integer. function isInteger(n: number): boolean { // Float have +0.0 and -0.0. To represent -0.0 precisely, we have to use a float, not an int // (see also https://stackoverflow.com/questions/7223359/are-0-and-0-the-same). return n === +n && n === (n | 0) && !Object.is(n, -0.0); } // ---------------------------------------------------------------------- /** * To force a value to be serialized using a particular representation (e.g. a number as INT64), * wrap it into marshal.wrap('INT64', value) and serialize that. */ export function wrap(codeStr: MarshalCode, value: unknown) { return new WrappedObj(marshalCodes[codeStr], value); } export class WrappedObj { constructor(public code: number, public value: unknown) {} public inspect() { return util.inspect(this.value); } } // ---------------------------------------------------------------------- /** * @param {Boolean} options.stringToBuffer - If set, JS strings will become Python strings rather * than unicode objects (as if each JS string is wrapped into MemBuffer.stringToArray(str)). * This flag becomes a same-named property of Marshaller, which can be set at any time. * @param {Number} options.version - If version >= 2, uses binary representation for floats. The * default version 0 formats floats as strings. * * TODO: The default should be version 2. (0 was used historically because it was needed for * communication with PyPy-based sandbox.) */ export class Marshaller { private _memBuf: MemBuffer; private readonly _floatCode: number; private readonly _stringCode: number; private readonly _keysAreBuffers: boolean; constructor(options?: MarshalOptions) { this._memBuf = new MemBuffer(undefined); this._floatCode = options?.version && options.version >= 2 ? marshalCodes.BFLOAT : marshalCodes.FLOAT; this._stringCode = options?.stringToBuffer ? marshalCodes.STRING : marshalCodes.UNICODE; this._keysAreBuffers = Boolean(options?.keysAreBuffers); } public dump(): Uint8Array { // asByteArray returns a view on the underlying data, and the constructor creates a new copy. // For some usages, we may want to avoid making the copy. const bytes = new Uint8Array(this._memBuf.asByteArray()); this._memBuf.clear(); return bytes; } public dumpAsBuffer(): Buffer { const bytes = Buffer.from(this._memBuf.asByteArray()); this._memBuf.clear(); return bytes; } public getCode(value: any) { switch (typeof value) { case "number": return isInteger(value) ? marshalCodes.INT : this._floatCode; case "string": return this._stringCode; case "boolean": return value ? marshalCodes.TRUE : marshalCodes.FALSE; case "undefined": return marshalCodes.NONE; case "object": { if (value instanceof WrappedObj) { return value.code; } else if (value === null) { return marshalCodes.NONE; } else if (value instanceof Uint8Array) { return marshalCodes.STRING; } else if (Buffer.isBuffer(value)) { return marshalCodes.STRING; } else if (Array.isArray(value)) { return marshalCodes.LIST; } return marshalCodes.DICT; } default: { throw new Error("Marshaller: Unsupported value of type " + (typeof value)); } } } public marshal(value: any): void { const code = this.getCode(value); if (value instanceof WrappedObj) { value = value.value; } this._memBuf.writeUint8(code); switch (code) { case marshalCodes.NULL: return; case marshalCodes.NONE: return; case marshalCodes.FALSE: return; case marshalCodes.TRUE: return; case marshalCodes.INT: return this._memBuf.writeInt32LE(value); case marshalCodes.INT64: return this._writeInt64(value); case marshalCodes.FLOAT: return this._writeStringFloat(value); case marshalCodes.BFLOAT: return this._memBuf.writeFloat64LE(value); case marshalCodes.STRING: return (value instanceof Uint8Array || Buffer.isBuffer(value) ? this._writeByteArray(value) : this._writeUtf8String(value)); case marshalCodes.TUPLE: return this._writeList(value); case marshalCodes.LIST: return this._writeList(value); case marshalCodes.DICT: return this._writeDict(value); case marshalCodes.UNICODE: return this._writeUtf8String(value); // None of the following are supported. case marshalCodes.STOPITER: case marshalCodes.ELLIPSIS: case marshalCodes.COMPLEX: case marshalCodes.LONG: case marshalCodes.INTERNED: case marshalCodes.STRINGREF: case marshalCodes.CODE: case marshalCodes.UNKNOWN: case marshalCodes.SET: case marshalCodes.FROZENSET: throw new Error("Marshaller: Can't serialize code " + code); default: throw new Error("Marshaller: Can't serialize code " + code); } } private _writeInt64(value: number) { if (!isInteger(value)) { // TODO We could actually support 53 bits or so. throw new Error("Marshaller: int64 still only supports 32-bit ints for now: " + value); } this._memBuf.writeInt32LE(value); this._memBuf.writeInt32LE(value >= 0 ? 0 : -1); } private _writeStringFloat(value: number) { // This could be optimized a bit, but it's only used in V0 marshalling, which is only used in // sandbox system calls, which don't really ever use floats anyway. const bytes = MemBuffer.stringToArray(value.toString()); if (bytes.byteLength >= 127) { throw new Error("Marshaller: Trying to write a float that takes " + bytes.byteLength + " bytes"); } this._memBuf.writeUint8(bytes.byteLength); this._memBuf.writeByteArray(bytes); } private _writeByteArray(value: Uint8Array | Buffer) { // This works for both Uint8Arrays and Node Buffers. this._memBuf.writeInt32LE(value.length); this._memBuf.writeByteArray(value); } private _writeUtf8String(value: string) { const offset = this._memBuf.size(); // We don't know the length until we write the value. this._memBuf.writeInt32LE(0); this._memBuf.writeString(value); const byteLength = this._memBuf.size() - offset - 4; // Overwrite the 0 length we wrote earlier with the correct byte length. this._memBuf.asDataView.setInt32(this._memBuf.startPos + offset, byteLength, true); } private _writeList(array: unknown[]) { this._memBuf.writeInt32LE(array.length); for (const item of array) { this.marshal(item); } } private _writeDict(obj: { [key: string]: any }) { const keys = Object.keys(obj); keys.sort(); for (const key of keys) { this.marshal(this._keysAreBuffers ? Buffer.from(key) : key); this.marshal(obj[key]); } this._memBuf.writeUint8(marshalCodes.NULL); } } // ---------------------------------------------------------------------- const TwoTo32 = 0x100000000; // 2**32 const TwoTo15 = 0x8000; // 2**15 /** * @param {Boolean} options.bufferToString - If set, Python strings will become JS strings rather * than Buffers (as if each decoded buffer is wrapped into `buf.toString()`). * This flag becomes a same-named property of Unmarshaller, which can be set at any time. * Note that options.version isn't needed, since this will decode both formats. * TODO: Integers (such as int64 and longs) that are too large for JS are currently represented as * decimal strings. They may need a better representation, or a configurable option. */ export class Unmarshaller extends EventEmitter { public memBuf: MemBuffer; private _consumer: any = null; private _lastCode: number | null = null; private readonly _bufferToString: boolean; private _emitter: (v: any) => boolean; private _stringTable: (string | Uint8Array)[] = []; constructor(options?: UnmarshalOptions) { super(); this.memBuf = new MemBuffer(undefined); this._bufferToString = Boolean(options?.bufferToString); this._emitter = this.emit.bind(this, "value"); } /** * Adds more data for parsing. Parsed values will be emitted as 'value' events. * @param {Uint8Array|Buffer} byteArray: Uint8Array or Node Buffer with bytes to parse. */ public push(byteArray: Uint8Array | Buffer) { this.parse(byteArray, this._emitter); } /** * Adds data to parse, and calls valueCB(value) for each value parsed. If valueCB returns the * Boolean false, stops parsing and returns. */ public parse(byteArray: Uint8Array | Buffer, valueCB: (val: any) => boolean | void) { this.memBuf.writeByteArray(byteArray); try { while (this.memBuf.size() > 0) { this._consumer = this.memBuf.makeConsumer(); // Have to reset stringTable for interned strings before each top-level parse call. this._stringTable.length = 0; const value = this._parse(); this.memBuf.consume(this._consumer); if (valueCB(value) === false) { return; } } } catch (err) { // If the error is `needMoreData`, we silently return. We'll retry by reparsing the message // from scratch after the next push(). If buffers contain complete serialized messages, the // cost should be minor. But this design might get very inefficient if we have big messages // of arrays or dictionaries. if (err.needMoreData) { if (!err.consumedData || err.consumedData > 1024) { console.log("Unmarshaller: Need more data; wasted parsing of %d bytes", err.consumedData); } } else { err.message = "Unmarshaller: " + err.message; throw err; } } } private _parse(): unknown { const code = this.memBuf.readUint8(this._consumer); this._lastCode = code; switch (code) { case marshalCodes.NULL: return null; case marshalCodes.NONE: return null; case marshalCodes.FALSE: return false; case marshalCodes.TRUE: return true; case marshalCodes.INT: return this._parseInt(); case marshalCodes.INT64: return this._parseInt64(); case marshalCodes.FLOAT: return this._parseStringFloat(); case marshalCodes.BFLOAT: return this._parseBinaryFloat(); case marshalCodes.STRING: return this._parseByteString(); case marshalCodes.TUPLE: return this._parseList(); case marshalCodes.LIST: return this._parseList(); case marshalCodes.DICT: return this._parseDict(); case marshalCodes.UNICODE: return this._parseUnicode(); case marshalCodes.INTERNED: return this._parseInterned(); case marshalCodes.STRINGREF: return this._parseStringRef(); case marshalCodes.LONG: return this._parseLong(); // None of the following are supported. // case marshalCodes.STOPITER: // case marshalCodes.ELLIPSIS: // case marshalCodes.COMPLEX: // case marshalCodes.CODE: // case marshalCodes.UNKNOWN: // case marshalCodes.SET: // case marshalCodes.FROZENSET: default: throw new Error(`Unmarshaller: unsupported code "${String.fromCharCode(code)}" (${code})`); } } private _parseInt() { return this.memBuf.readInt32LE(this._consumer); } private _parseInt64() { const low = this.memBuf.readInt32LE(this._consumer); const hi = this.memBuf.readInt32LE(this._consumer); if ((hi === 0 && low >= 0) || (hi === -1 && low < 0)) { return low; } const unsignedLow = low < 0 ? TwoTo32 + low : low; if (hi >= 0) { return new BigInt(TwoTo32, [unsignedLow, hi], 1).toNative(); } else { // This part is tricky. See unittests for check of correctness. return new BigInt(TwoTo32, [TwoTo32 - unsignedLow, -hi - 1], -1).toNative(); } } private _parseLong() { // The format is a 32-bit size whose sign is the sign of the result, followed by 16-bit digits // in base 2**15. const size = this.memBuf.readInt32LE(this._consumer); const sign = size < 0 ? -1 : 1; const numDigits = size < 0 ? -size : size; const digits = []; for (let i = 0; i < numDigits; i++) { digits.push(this.memBuf.readInt16LE(this._consumer)); } return new BigInt(TwoTo15, digits, sign).toNative(); } private _parseStringFloat() { const len = this.memBuf.readUint8(this._consumer); const buf = this.memBuf.readString(this._consumer, len); return parseFloat(buf); } private _parseBinaryFloat() { return this.memBuf.readFloat64LE(this._consumer); } private _parseByteString(): string | Uint8Array { const len = this.memBuf.readInt32LE(this._consumer); return (this._bufferToString ? this.memBuf.readString(this._consumer, len) : this.memBuf.readByteArray(this._consumer, len)); } private _parseInterned() { const s = this._parseByteString(); this._stringTable.push(s); return s; } private _parseStringRef() { const index = this._parseInt(); return this._stringTable[index]; } private _parseList() { const len = this.memBuf.readInt32LE(this._consumer); const value = []; for (let i = 0; i < len; i++) { value[i] = this._parse(); } return value; } private _parseDict() { const dict: { [key: string]: any } = {}; while (true) { let key = this._parse() as string | Uint8Array; if (key === null && this._lastCode === marshalCodes.NULL) { break; } const value = this._parse(); if (key !== null) { if (key instanceof Uint8Array) { key = MemBuffer.arrayToString(key); } dict[key as string] = value; } } return dict; } private _parseUnicode() { const len = this.memBuf.readInt32LE(this._consumer); return this.memBuf.readString(this._consumer, len); } } /** * Similar to python's marshal.loads(). Parses the given bytes and returns the parsed value. There * must not be any trailing data beyond the single marshalled value. */ export function loads(byteArray: Uint8Array | Buffer, options?: UnmarshalOptions): any { const unmarshaller = new Unmarshaller(options); let parsedValue; unmarshaller.parse(byteArray, function(value) { parsedValue = value; return false; }); if (typeof parsedValue === "undefined") { throw new Error("loads: input data truncated"); } else if (unmarshaller.memBuf.size() > 0) { throw new Error("loads: extra bytes past end of input"); } return parsedValue; } /** * Serializes arbitrary data by first marshalling then converting to a base64 string. */ export function dumpBase64(data: any, options?: MarshalOptions) { const marshaller = new Marshaller(options || { version: 2 }); marshaller.marshal(data); return marshaller.dumpAsBuffer().toString("base64"); } /** * Loads data from a base64 string, as serialized by dumpBase64(). */ export function loadBase64(data: string, options?: UnmarshalOptions) { return loads(Buffer.from(data, "base64"), options); } ================================================ FILE: app/common/normalizedDateTimeString.ts ================================================ import moment from "moment-timezone"; /** * Output an ISO8601 format datetime string, with timezone. * Any string fed in without timezone is expected to be in UTC. * * When connected to postgres, dates will be extracted as Date objects, * with timezone information. The normalization done here is not * really needed in this case. * * Timestamps in SQLite are stored as UTC, and read as strings * (without timezone information). The normalization here is * pretty important in this case. */ export function normalizedDateTimeString(dateTime: any): string { if (!dateTime) { return dateTime; } if (dateTime instanceof Date) { return moment(dateTime).toISOString(); } if (typeof dateTime === "string" || typeof dateTime === "number") { // When SQLite returns a string, it will be in UTC. // Need to make sure it actually have timezone info in it // (will not by default). return moment.utc(dateTime).toISOString(); } throw new Error(`normalizedDateTimeString cannot handle ${dateTime}`); } ================================================ FILE: app/common/orgNameUtils.ts ================================================ const BLACKLISTED_SUBDOMAINS = new Set([ // from wiki page as of 2018-12-14 "aws", "gristlogin", "issues", "metrics", "phab", "releases", "test", "vpn", "www", // A few more reserved just in case. The minimum length requirement would eliminate // some in any case, but specified here also in case that minimum changes. "w", "ww", "wwww", "wwwww", "docs", "api", "static", "ftp", "imap", "pop", "smtp", "mail", "git", "blog", "wiki", "support", "kb", "help", "admin", "store", "dev", "beta", "community", "try", "wpx", "telemetry", // a few random tech brands "google", "apple", "microsoft", "ms", "facebook", "fb", "twitter", "youtube", "yt", // updates for new special domains "current", "staging", "prod", "login", "login-dev", "login-s", // some domains that look suspicious "1ogin", "1ogin-dev", "1ogin-s", ]); /** * * Checks whether the subdomain is on the list of forbidden subdomains. * See /documentation/urls.md#organization-subdomains * * Also enforces various sanity checks. * * Throws if the subdomain is invalid. * */ export function checkSubdomainValidity(subdomain: string): void { // stick with limited alphanumeric subdomains. if (!(/^[a-z0-9][-a-z0-9]*$/.test(subdomain))) { throw new Error("Domain must include lower-case letters, numbers, and dashes only."); } // 'docs-*' is reserved for personal orgs. if (subdomain.startsWith("docs-")) { throw new Error('Domain cannot use reserved prefix "docs-".'); } // 'o-*' is reserved for automatic org domains. if (subdomain.startsWith("o-")) { throw new Error('Domain cannot use reserved prefix "o-".'); } // 'doc-worker-*' is reserved for doc workers. if (subdomain.startsWith("doc-worker-")) { throw new Error('Domain cannot use reserved prefix "doc-worker-".'); } // special subdomains like _domainkey. if (subdomain.startsWith("_")) { throw new Error('Domain cannot use reserved prefix "_".'); } // some domains are currently in use for testing v1. if (subdomain.startsWith("v1-")) { throw new Error('Domain cannot use reserved prefix "v1-".'); } // check limit of 63 characters on dns label. if (subdomain.length > 63) { throw new Error("Domain must contain less than 64 characters."); } // check the subdomain isn't too short. if (subdomain.length <= 2) { throw new Error("Domain must contain more than 2 characters."); } // a small blacklist prepared by hand. if (BLACKLISTED_SUBDOMAINS.has(subdomain)) { throw new Error("Invalid domain value."); } } ================================================ FILE: app/common/parseDate.ts ================================================ import { getDistinctValues, isNonNullish } from "app/common/gutil"; import guessFormat from "@gristlabs/moment-guess/dist/bundle.js"; import escapeRegExp from "lodash/escapeRegExp"; import last from "lodash/last"; import memoize from "lodash/memoize"; // Simply importing 'moment-guess' inconsistently imports bundle.js or bundle.esm.js depending on environment import moment from "moment-timezone"; // When using YY format, use a consistent interpretation in datepicker and in moment parsing: add // 2000 if the result is at most 10 years greater than the current year; otherwise add 1900. See // https://bootstrap-datepicker.readthedocs.io/en/latest/options.html#assumenearbyyear and // "Parsing two digit years" in https://momentjs.com/docs/#/parsing/string-format/. export const TWO_DIGIT_YEAR_THRESHOLD = 10; const MAX_TWO_DIGIT_YEAR = new Date().getFullYear() + TWO_DIGIT_YEAR_THRESHOLD - 2000; // Moment suggests that overriding this is fine, but we need to force TypeScript to allow it. (moment as any).parseTwoDigitYear = function(yearString: string): number { const year = parseInt(yearString, 10); return year + (year > MAX_TWO_DIGIT_YEAR ? 1900 : 2000); }; // Order of formats to try if the date cannot be parsed as the currently set format. // Formats are parsed in momentjs strict mode, but separator matching and the MM/DD // two digit requirement are ignored. Also, partial completion is permitted, so formats // may match even if only beginning elements are provided. // TODO: These should be affected by the user's locale/settings. // TODO: We may want to consider adding default time formats as well to support more // time formats. const PARSER_FORMATS: string[] = [ "M D YYYY", "M D YY", "M D", "M", "MMMM D YYYY", "MMMM D", "MMMM Do YYYY", "MMMM Do", "D MMMM YYYY", "D MMMM", "Do MMMM YYYY", "Do MMMM", "MMMM", "MMM D YYYY", "MMM D", "MMM Do YYYY", "MMM Do", "D MMM YYYY", "D MMM", "Do MMM YYYY", "Do MMM", "MMM", "YYYY M D", "YYYY M", "YYYY", "D M YYYY", "D M YY", "D M", "D", ]; const UNAMBIGUOUS_FORMATS = [ "YYYY M D", ...PARSER_FORMATS.filter(f => f.includes("MMM")), ]; const TIME_REGEX = /(?:^|\s+|T)(?:(\d\d?)(?::(\d\d?)(?::(\d\d?))?)?|(\d\d?)(\d\d))\s*([ap]m?)?$/i; // [^a-zA-Z] because no letters are allowed directly before the abbreviation const UTC_REGEX = /[^a-zA-Z](UTC?|GMT|Z)$/i; const NUMERIC_TZ_REGEX = /([+-]\d\d?)(?::?(\d\d))?$/i; // Not picky about separators, so replace them in the date and format strings to be spaces. const SEPARATORS = /[\W_]+/g; const tzAbbreviations = memoize((tzName: string): RegExp => { // Some abbreviations are just e.g. +05 // and escaping the + seems better than filtering const abbreviations = new Set(moment.tz.zone(tzName)!.abbrs.map(escapeRegExp)); const union = [...abbreviations].join("|"); // [^a-zA-Z] because no letters are allowed directly before the abbreviation // so for example CEST won't match even if EST does return new RegExp(`[^a-zA-Z](${union})$`, "i"); }); interface ParseOptions { time?: string; dateFormat?: string; timeFormat?: string; timezone?: string; } /** * parseDate - Attempts to parse a date string using several common formats. Returns the * timestamp of the parsed date in seconds since epoch, or returns null on failure. * @param {String} date - The date string to parse. * @param {String} options.dateFormat - The preferred momentjs format to use to parse the * date. This is attempted before the default formats. * @param {String} options.time - The time string to parse. * @param {String} options.timeFormat - The momentjs format to use to parse the time. This * must be given if options.time is given. * @param {String} options.timezone - The timezone string for the date/time, which affects * the resulting timestamp. */ export function parseDate(date: string, options: ParseOptions = {}): number | null { // If no date, return null. if (!date) { return null; } // If this looks like a timestamp (string with 9 or more digits), just return it. const timestamp = parseTimeStamp(date); if (timestamp !== null) { return timestamp; } const dateFormat = options.dateFormat || "YYYY-MM-DD"; const dateFormats = [..._buildVariations(dateFormat, date), ...PARSER_FORMATS]; const cleanDate = date.replace(SEPARATORS, " "); let datetime = cleanDate.trim(); let timeformat = ""; let time = options.time; if (time) { const parsedTimeZone = parseTimeZone(time, options.timezone!); const parsedTime = standardizeTime(parsedTimeZone.remaining); if (!parsedTime || parsedTime.remaining) { return null; } time = parsedTime.time; const { tzOffset } = parsedTimeZone; datetime += " " + time + tzOffset; timeformat = " HH:mm:ss" + (tzOffset ? "Z" : ""); } for (const format of dateFormats) { const fullFormat = format + timeformat; const m = moment.tz(datetime, fullFormat, true, options.timezone || "UTC"); if (m.isValid()) { return m.unix(); } } return null; } /** * Similar to parseDate, with these differences: * - Only for a date (no time part) * - Only falls back to UNAMBIGUOUS_FORMATS, not the full PARSER_FORMATS * - Optionally adds all dates which match some format to `results`, otherwise returns first match. * This is safer so it can be used for parsing when pasting a large number of dates * and won't silently swap around day and month. */ export function parseDateStrict( date: string, dateFormat: string | null, results?: Set, timezone: string = "UTC", ): number | undefined { if (!date) { return; } // If this looks like a timestamp (string with 9 or more digits), just return it. const timestamp = parseTimeStamp(date); if (timestamp !== null) { return timestamp; } dateFormat = dateFormat || "YYYY-MM-DD"; const dateFormats = [..._buildVariations(dateFormat, date), ...UNAMBIGUOUS_FORMATS]; const cleanDate = date.replace(SEPARATORS, " ").trim(); for (const format of dateFormats) { const m = moment.tz(cleanDate, format, true, timezone); if (m.isValid()) { const value = m.valueOf() / 1000; if (results) { results.add(value); } else { return value; } } } } export function parseDateTime(dateTime: string, options: ParseOptions): number | undefined { dateTime = dateTime.trim(); if (!dateTime) { return; } const dateFormat = options.dateFormat || "YYYY-MM-DD"; const timezone = options.timezone || "UTC"; const dateOnly = parseDateStrict(dateTime, dateFormat, undefined, timezone); if (dateOnly) { return dateOnly; } const parsedTimeZone = parseTimeZone(dateTime, timezone); let tzOffset = ""; if (parsedTimeZone) { tzOffset = parsedTimeZone.tzOffset; dateTime = parsedTimeZone.remaining; } const parsedTime = standardizeTime(dateTime); if (!parsedTime) { return; } dateTime = parsedTime.remaining; const date = parseDateStrict(dateTime, dateFormat); if (!date) { return; } // date is a timestamp of midnight in UTC, so to get a formatted representation (for parsing // together with time), take care to interpret it in UTC. const dateString = moment.unix(date).utc().format("YYYY-MM-DD"); dateTime = dateString + " " + parsedTime.time + tzOffset; const fullFormat = "YYYY-MM-DD HH:mm:ss" + (tzOffset ? "Z" : ""); return moment.tz(dateTime, fullFormat, true, timezone).valueOf() / 1000; } // Helper function to get the partial format string based on the input. Momentjs has a feature // which allows defaulting to the current year, month and/or day if not accounted for in the // parser. We remove any parts of the parser not given in the input to take advantage of this // feature. function _getPartialFormat(input: string, format: string): string { // Define a regular expression to match contiguous non-separators. const re = /Y+|M+o?|D+o?|[a-zA-Z0-9]+/ig; // Count the number of meaningful parts in the input. const numInputParts = input.match(re)?.length || 0; // Count the number of parts in the format string. let numFormatParts = format.match(re)?.length || 0; if (numFormatParts > numInputParts) { // Remove year from format first, to default to current year. if (/Y+/.test(format)) { format = format.replace(/Y+/, " ").trim(); numFormatParts -= 1; } if (numFormatParts > numInputParts) { // Remove month from format next. format = format.replace(/M+/, " ").trim(); } } return format; } // Moment non-strict mode is considered bad, as it's far too lax. But moment's strict mode is too // strict. We want to allow YY|YYYY for either year specifier, as well as M for MMM or MMMM month // specifiers. It's silly that we need to create multiple format variations to support this. function _buildVariations(dateFormat: string, date: string) { // Momentjs has an undesirable feature in strict mode where MM and DD // matches require two digit numbers. Change MM, DD to M, D. let format = dateFormat.replace(/MM+/g, m => (m === "MM" ? "M" : m)) .replace(/DD+/g, m => (m === "DD" ? "D" : m)) .replace(SEPARATORS, " ") .trim(); // Allow the input date to end with a 4-digit year even if the format doesn't mention the year if ( format.includes("M") && format.includes("D") && !format.includes("Y") ) { format += " YYYY"; } format = _getPartialFormat(date, format); // Consider some alternatives to the preferred format. const variations = new Set([format]); const otherYear = format.replace(/Y{2,4}/, m => (m === "YY" ? "YYYY" : (m === "YYYY" ? "YY" : m))); variations.add(otherYear); variations.add(format.replace(/MMM+/, "M")); if (otherYear !== format) { variations.add(otherYear.replace(/MMM+/, "M")); } return variations; } // Based on private calculateOffset in moment source code. function calculateOffset(tzMatch: string[]): string { const [, hhOffset, mmOffset] = tzMatch; const sign = hhOffset.slice(0, 1); return sign + hhOffset.slice(1).padStart(2, "0") + ":" + (mmOffset || "0").padStart(2, "0"); } function parseTimeZone(str: string, timezone: string): { remaining: string, tzOffset: string } { str = str.trim(); let tzMatch = UTC_REGEX.exec(str); let matchStart = 0; let tzOffset = ""; if (tzMatch) { tzOffset = "+00:00"; matchStart = tzMatch.index + 1; // skip [^a-zA-Z] at regex start } else { tzMatch = NUMERIC_TZ_REGEX.exec(str); if (tzMatch) { tzOffset = calculateOffset(tzMatch); matchStart = tzMatch.index; } else if (timezone) { // Abbreviations are simply stripped and ignored, so tzOffset is not set in this case tzMatch = tzAbbreviations(timezone).exec(str); if (tzMatch) { matchStart = tzMatch.index + 1; // skip [^a-zA-Z] at regex start } } } if (tzMatch) { str = str.slice(0, matchStart).trim(); } return { remaining: str, tzOffset }; } // Parses time of the form, roughly, HH[:MM[:SS]][am|pm]. Returns the time in the // standardized HH:mm:ss format. // This turns out easier than coaxing moment to parse time sensibly and flexibly. function standardizeTime(timeString: string): { remaining: string, time: string } | undefined { const match = TIME_REGEX.exec(timeString); if (!match) { return; } let hours = parseInt(match[1] || match[4], 10); const mm = (match[2] || match[5] || "0").padStart(2, "0"); const ss = (match[3] || "0").padStart(2, "0"); const ampm = (match[6] || "").toLowerCase(); if (hours < 12 && hours > 0 && ampm.startsWith("p")) { hours += 12; } else if (hours === 12 && ampm.startsWith("a")) { hours = 0; } const hh = String(hours).padStart(2, "0"); return { remaining: timeString.slice(0, match.index).trim(), time: `${hh}:${mm}:${ss}` }; } /** * Guesses a full date[time] format that best matches the given strings. * If several formats match equally well, picks the last one lexicographically to match the old date guessing. * This means formats with an early Y and/or M are favoured. * If no formats match, returns the default YYYY-MM-DD. */ export function guessDateFormat(values: (string | null)[], timezone: string = "UTC"): string { const formats = guessDateFormats(values, timezone); if (!formats) { return "YYYY-MM-DD"; } return last(formats)!; } /** * Returns all full date[time] formats that best match the given strings. * If several formats match equally well, returns them all. * May return null if there are no matching formats or choosing one is too expensive. */ export function guessDateFormats(values: (string | null)[], timezone: string = "UTC"): string[] | null { const dateStrings: string[] = values.filter(isNonNullish); const sample = getDistinctValues(dateStrings, 100); const formats: Record = {}; for (const dateString of sample) { let guessed: string | string[]; try { guessed = guessFormat(dateString); } catch { continue; } if (typeof guessed === "string") { guessed = [guessed]; } for (const guess of guessed) { formats[guess] = 0; } } const formatKeys = Object.keys(formats); if (!formatKeys.length || formatKeys.length > 10) { return null; } for (const format of formatKeys) { for (const dateString of dateStrings) { const m = moment.tz(dateString, format, true, timezone); if (m.isValid()) { formats[format] += 1; } } } const maxCount = Math.max(...Object.values(formats)); // Return all formats that tied for first place. // Sort lexicographically for consistency in tests and with the old dateguess.py. return formatKeys.filter(format => formats[format] === maxCount).sort(); } export const dateFormatOptions = [ "YYYY-MM-DD", "MM-DD-YYYY", "MM/DD/YYYY", "MM-DD-YY", "MM/DD/YY", "DD MMM YYYY", "MMMM Do, YYYY", "DD-MM-YYYY", ]; export const timeFormatOptions = [ "h:mma", "h:mma z", "HH:mm", "HH:mm z", "HH:mm:ss", "HH:mm:ss z", ]; /** * Construct widget options for a Date or DateTime column based on a single moment string * which may or may not contain both date and time parts. * If defaultTimeFormat is true, fallback to a non-empty default time format when none is found in fullFormat. */ export function dateTimeWidgetOptions(fullFormat: string, defaultTimeFormat: boolean) { const index = fullFormat.match(/[hHkaAmsSzZT]|$/)!.index!; const dateFormat = fullFormat.substr(0, index).trim(); const timeFormat = fullFormat.substr(index).trim() || (defaultTimeFormat ? timeFormatOptions[0] : ""); return { dateFormat, timeFormat, isCustomDateFormat: !dateFormatOptions.includes(dateFormat), isCustomTimeFormat: !timeFormatOptions.includes(timeFormat), }; } /** * Attempts to parse a timestamp string. Returns the timestamp in seconds * since epoch, or returns null on failure. Accepts only strings with 9 to 11 digits. * Lowest 11 digit timestamp is 2286-11-20, so we don't consider them valid. */ export function parseTimeStamp(date: string): number | null { // If this looks like a timestamp (number with 9 or more digits), just return it. // This covers most of the cases leaving some time around the unix epoch not covered. // So time before 100 000 000 (1974-04-26) is not covered. Also negative values // are also not supported, as they overlap with the YYYYYY date format. if (date && /^[1-9]\d{8,9}$/.test(date)) { const parsedDate = moment(date, "X"); if (parsedDate.isValid()) { return parsedDate.unix(); } } return null; } ================================================ FILE: app/common/plugin.ts ================================================ /** * Plugin's utilities common to server and client. */ import { BarePlugin, Implementation } from "app/plugin/PluginManifest"; export type LocalPluginKind = "installed" | "builtIn"; export interface ImplDescription { localPluginId: string; implementation: Implementation; } export interface FileParser { fileExtensions: string[]; parseOptions?: ImplDescription; fileParser: ImplDescription; } // Deprecated, use FileParser or ImportSource instead. export interface FileImporter { id: string; fileExtensions?: string[]; script?: string; scriptFullPath?: string; filePicker?: string; filePickerFullPath?: string; } /** * Manifest parsing error. */ export interface ManifestParsingError { yamlError?: any; jsonError?: any; cannotReadError?: any; missingEntryErrors?: string; } /** * Whether the importer provides a file picker. */ export function isPicker(importer: FileImporter): boolean { return importer.filePicker !== undefined; } /** * A Plugin that was found in the system, either installed or builtin. */ export interface LocalPlugin { /** * the plugin's manifest */ manifest: BarePlugin; /** * The path to the plugin's folder. */ path: string; /** * A name to uniquely identify a LocalPlugin. */ readonly id: string; } export interface DirectoryScanEntry { manifest?: BarePlugin; /** * User-friendly error messages. */ errors?: any[]; path: string; id: string; } /** * The contributions type. */ export type Contribution = "importSource" | "fileParser"; ================================================ FILE: app/common/resetOrg.ts ================================================ import { isOwner } from "app/common/roles"; import { ManagerDelta, PermissionDelta, UserAPI } from "app/common/UserAPI"; /** * A utility to reset an organization into the state it would have when first * created - no docs, one workspace called "Home", a single user. Should be * called by a user who is both an owner of the org and a billing manager. */ export async function resetOrg(api: UserAPI, org: string | number) { const session = await api.getSessionActive(); if (!isOwner(session.org)) { throw new Error("user must be an owner of the org to be reset"); } const billing = api.getBillingAPI(); // If billing api is not available, don't bother setting billing manager. const account = await billing.getBillingAccount().catch(e => null); if (account && !account.managers.some(manager => (manager.id === session.user.id))) { throw new Error("user must be a billing manager"); } const wss = await api.getOrgWorkspaces(org); for (const ws of wss) { if (!ws.isSupportWorkspace) { await api.deleteWorkspace(ws.id); } } await api.newWorkspace({ name: "Home" }, org); const permissions: PermissionDelta = { users: {} }; for (const user of (await api.getOrgAccess(org)).users) { if (user.id !== session.user.id) { permissions.users![user.email] = null; } } await api.updateOrgPermissions(org, permissions); // For non-individual accounts, update billing managers (individual accounts will // throw an error if we try to do this). if (account && !account.individual) { const managers: ManagerDelta = { users: {} }; for (const user of account.managers) { if (user.id !== session.user.id) { managers.users[user.email] = null; } } await billing.updateBillingManagers(managers); } return api; } ================================================ FILE: app/common/roles.ts ================================================ import { Organization } from "app/common/UserAPI"; export const OWNER = "owners"; export const EDITOR = "editors"; export const VIEWER = "viewers"; export const GUEST = "guests"; export const MEMBER = "members"; // Roles ordered from most to least permissive. const roleOrder: (Role | null)[] = [OWNER, EDITOR, VIEWER, MEMBER, GUEST, null]; export type BasicRole = "owners" | "editors" | "viewers"; export type NonMemberRole = BasicRole | "guests"; export type NonGuestRole = BasicRole | "members"; export type Role = NonMemberRole | "members"; // Returns the BasicRole (or null) with the same effective access as the given role. export function getEffectiveRole(role: Role | null): BasicRole | null { if (role === GUEST || role === MEMBER) { return VIEWER; } else { return role; } } export function canEditAccess(role: string | null): boolean { return role === OWNER; } // Note that while canEdit has the same return value as canDelete, the functions are // kept separate as they may diverge in the future. export function canEdit(role: string | null): boolean { return role === OWNER || role === EDITOR; } export function canDelete(role: string | null): boolean { return role === OWNER || role === EDITOR; } export function canView(role: string | null): boolean { return role !== null; } export function isOwner(resource: { access: Role | null } | null): resource is { access: Role } { return resource?.access === OWNER; } export function isOwnerOrEditor(resource: { access: Role | null } | null): resource is { access: Role } { return canEdit(resource?.access ?? null); } export function canUpgradeOrg(org: Organization | null): org is Organization { // TODO: Need to consider billing managers and support user. return isOwner(org); } // Returns true if the role string is a valid role or null. export function isValidRole(role: string | null): role is Role | null { return (roleOrder as (string | null)[]).includes(role); } // Returns true if the role string is a valid non-Guest, non-Member, non-null role. export function isBasicRole(role: string | null): role is BasicRole { return Boolean(role && role !== GUEST && role !== MEMBER && isValidRole(role)); } // Returns true if the role string is a valid non-Guest, non-null role. export function isNonGuestRole(role: string | null): role is NonGuestRole { return Boolean(role && role !== GUEST && isValidRole(role)); } /** * Returns out of any number of group role names the one that offers more permissions. The function * is overloaded so that the output type matches the specificity of the input values. */ export function getStrongestRole(...args: T[]): T { return getFirstMatchingRole(roleOrder, args); } /** * Returns out of any number of group role names the one that offers fewer permissions. The function * is overloaded so that the output type matches the specificity of the input values. */ export function getWeakestRole(...args: T[]): T { return getFirstMatchingRole(roleOrder.slice().reverse(), args); } // Returns which of the `anyOf` args comes first in `array`. Helper for getStrongestRole // and getWeakestRole. function getFirstMatchingRole(array: (Role | null)[], anyOf: T[]): T { if (anyOf.length === 0) { throw new Error(`getFirstMatchingRole: No roles given`); } for (const role of anyOf) { if (!isValidRole(role)) { throw new Error(`getFirstMatchingRole: Invalid role ${role}`); } } return array.find(item => anyOf.includes(item as T)) as T; } ================================================ FILE: app/common/schema.ts ================================================ /* eslint-disable */ /*** THIS FILE IS AUTO-GENERATED BY core/sandbox/gen_js_schema.py ***/ import { GristObjCode } from "app/plugin/GristData"; // tslint:disable:object-literal-key-quotes export const SCHEMA_VERSION = 46; export const schema = { "_grist_DocInfo": { docId : "Text", peers : "Text", basketId : "Text", schemaVersion : "Int", timezone : "Text", documentSettings : "Text", }, "_grist_Tables": { tableId : "Text", primaryViewId : "Ref:_grist_Views", summarySourceTable : "Ref:_grist_Tables", onDemand : "Bool", rawViewSectionRef : "Ref:_grist_Views_section", recordCardViewSectionRef: "Ref:_grist_Views_section", }, "_grist_Tables_column": { parentId : "Ref:_grist_Tables", parentPos : "PositionNumber", colId : "Text", type : "Text", widgetOptions : "Text", isFormula : "Bool", formula : "Text", label : "Text", description : "Text", untieColIdFromLabel : "Bool", summarySourceCol : "Ref:_grist_Tables_column", displayCol : "Ref:_grist_Tables_column", visibleCol : "Ref:_grist_Tables_column", rules : "RefList:_grist_Tables_column", reverseCol : "Ref:_grist_Tables_column", recalcWhen : "Int", recalcDeps : "RefList:_grist_Tables_column", }, "_grist_Imports": { tableRef : "Ref:_grist_Tables", origFileName : "Text", parseFormula : "Text", delimiter : "Text", doublequote : "Bool", escapechar : "Text", quotechar : "Text", skipinitialspace : "Bool", encoding : "Text", hasHeaders : "Bool", }, "_grist_External_database": { host : "Text", port : "Int", username : "Text", dialect : "Text", database : "Text", storage : "Text", }, "_grist_External_table": { tableRef : "Ref:_grist_Tables", databaseRef : "Ref:_grist_External_database", tableName : "Text", }, "_grist_TableViews": { tableRef : "Ref:_grist_Tables", viewRef : "Ref:_grist_Views", }, "_grist_TabItems": { tableRef : "Ref:_grist_Tables", viewRef : "Ref:_grist_Views", }, "_grist_TabBar": { viewRef : "Ref:_grist_Views", tabPos : "PositionNumber", }, "_grist_Pages": { viewRef : "Ref:_grist_Views", indentation : "Int", pagePos : "PositionNumber", shareRef : "Ref:_grist_Shares", options : "Text", }, "_grist_Views": { name : "Text", type : "Text", layoutSpec : "Text", }, "_grist_Views_section": { tableRef : "Ref:_grist_Tables", parentId : "Ref:_grist_Views", parentKey : "Text", title : "Text", description : "Text", defaultWidth : "Int", borderWidth : "Int", theme : "Text", options : "Text", chartType : "Text", layoutSpec : "Text", filterSpec : "Text", sortColRefs : "Text", linkSrcSectionRef : "Ref:_grist_Views_section", linkSrcColRef : "Ref:_grist_Tables_column", linkTargetColRef : "Ref:_grist_Tables_column", embedId : "Text", rules : "RefList:_grist_Tables_column", shareOptions : "Text", }, "_grist_Views_section_field": { parentId : "Ref:_grist_Views_section", parentPos : "PositionNumber", colRef : "Ref:_grist_Tables_column", width : "Int", widgetOptions : "Text", displayCol : "Ref:_grist_Tables_column", visibleCol : "Ref:_grist_Tables_column", filter : "Text", rules : "RefList:_grist_Tables_column", }, "_grist_Validations": { formula : "Text", name : "Text", tableRef : "Int", }, "_grist_REPL_Hist": { code : "Text", outputText : "Text", errorText : "Text", }, "_grist_Attachments": { fileIdent : "Text", fileName : "Text", fileType : "Text", fileSize : "Int", fileExt : "Text", imageHeight : "Int", imageWidth : "Int", timeDeleted : "DateTime", timeUploaded : "DateTime", }, "_grist_Triggers": { tableRef : "Ref:_grist_Tables", eventTypes : "ChoiceList", isReadyColRef : "Ref:_grist_Tables_column", actions : "Text", label : "Text", memo : "Text", enabled : "Bool", watchedColRefList : "RefList:_grist_Tables_column", options : "Text", condition : "Text", }, "_grist_ACLRules": { resource : "Ref:_grist_ACLResources", permissions : "Int", principals : "Text", aclFormula : "Text", aclColumn : "Ref:_grist_Tables_column", aclFormulaParsed : "Text", permissionsText : "Text", rulePos : "PositionNumber", userAttributes : "Text", memo : "Text", }, "_grist_ACLResources": { tableId : "Text", colIds : "Text", }, "_grist_ACLPrincipals": { type : "Text", userEmail : "Text", userName : "Text", groupName : "Text", instanceId : "Text", }, "_grist_ACLMemberships": { parent : "Ref:_grist_ACLPrincipals", child : "Ref:_grist_ACLPrincipals", }, "_grist_Filters": { viewSectionRef : "Ref:_grist_Views_section", colRef : "Ref:_grist_Tables_column", filter : "Text", pinned : "Bool", }, "_grist_Cells": { tableRef : "Ref:_grist_Tables", colRef : "Ref:_grist_Tables_column", rowId : "Int", root : "Bool", parentId : "Ref:_grist_Cells", type : "Int", content : "Text", userRef : "Text", timeCreated : "DateTime", timeUpdated : "DateTime", resolved : "Bool", }, "_grist_Shares": { linkId : "Text", options : "Text", label : "Text", description : "Text", }, }; export interface SchemaTypes { "_grist_DocInfo": { docId: string; peers: string; basketId: string; schemaVersion: number; timezone: string; documentSettings: string; }; "_grist_Tables": { tableId: string; primaryViewId: number; summarySourceTable: number; onDemand: boolean; rawViewSectionRef: number; recordCardViewSectionRef: number; }; "_grist_Tables_column": { parentId: number; parentPos: number; colId: string; type: string; widgetOptions: string; isFormula: boolean; formula: string; label: string; description: string; untieColIdFromLabel: boolean; summarySourceCol: number; displayCol: number; visibleCol: number; rules: [GristObjCode.List, ...number[]]|null; reverseCol: number; recalcWhen: number; recalcDeps: [GristObjCode.List, ...number[]]|null; }; "_grist_Imports": { tableRef: number; origFileName: string; parseFormula: string; delimiter: string; doublequote: boolean; escapechar: string; quotechar: string; skipinitialspace: boolean; encoding: string; hasHeaders: boolean; }; "_grist_External_database": { host: string; port: number; username: string; dialect: string; database: string; storage: string; }; "_grist_External_table": { tableRef: number; databaseRef: number; tableName: string; }; "_grist_TableViews": { tableRef: number; viewRef: number; }; "_grist_TabItems": { tableRef: number; viewRef: number; }; "_grist_TabBar": { viewRef: number; tabPos: number; }; "_grist_Pages": { viewRef: number; indentation: number; pagePos: number; shareRef: number; options: string; }; "_grist_Views": { name: string; type: string; layoutSpec: string; }; "_grist_Views_section": { tableRef: number; parentId: number; parentKey: string; title: string; description: string; defaultWidth: number; borderWidth: number; theme: string; options: string; chartType: string; layoutSpec: string; filterSpec: string; sortColRefs: string; linkSrcSectionRef: number; linkSrcColRef: number; linkTargetColRef: number; embedId: string; rules: [GristObjCode.List, ...number[]]|null; shareOptions: string; }; "_grist_Views_section_field": { parentId: number; parentPos: number; colRef: number; width: number; widgetOptions: string; displayCol: number; visibleCol: number; filter: string; rules: [GristObjCode.List, ...number[]]|null; }; "_grist_Validations": { formula: string; name: string; tableRef: number; }; "_grist_REPL_Hist": { code: string; outputText: string; errorText: string; }; "_grist_Attachments": { fileIdent: string; fileName: string; fileType: string; fileSize: number; fileExt: string; imageHeight: number; imageWidth: number; timeDeleted: number; timeUploaded: number; }; "_grist_Triggers": { tableRef: number; eventTypes: [GristObjCode.List, ...string[]]|null; isReadyColRef: number; actions: string; label: string; memo: string; enabled: boolean; watchedColRefList: [GristObjCode.List, ...number[]]|null; options: string; condition: string; }; "_grist_ACLRules": { resource: number; permissions: number; principals: string; aclFormula: string; aclColumn: number; aclFormulaParsed: string; permissionsText: string; rulePos: number; userAttributes: string; memo: string; }; "_grist_ACLResources": { tableId: string; colIds: string; }; "_grist_ACLPrincipals": { type: string; userEmail: string; userName: string; groupName: string; instanceId: string; }; "_grist_ACLMemberships": { parent: number; child: number; }; "_grist_Filters": { viewSectionRef: number; colRef: number; filter: string; pinned: boolean; }; "_grist_Cells": { tableRef: number; colRef: number; rowId: number; root: boolean; parentId: number; type: number; content: string; userRef: string; timeCreated: number; timeUpdated: number; resolved: boolean; }; "_grist_Shares": { linkId: string; options: string; label: string; description: string; }; } ================================================ FILE: app/common/tagManager.ts ================================================ /** * Returns the Google Tag Manager snippet to insert into of the page, if * `tagId` is set to a non-empty value. Otherwise returns an empty string. */ export function getTagManagerSnippet(tagId?: string) { // Note also that we only insert the snippet for the . The second recommended part (for // ) is for