Repository: raimohanska/ourboard Branch: master Commit: b0a194724d94 Files: 230 Total size: 1.0 MB Directory structure: gitextract_96m72lw6/ ├── .dockerignore ├── .editorconfig ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-report.md │ │ └── feature_request.md │ └── workflows/ │ └── build.yml ├── .gitignore ├── .huskyrc.js ├── .nvmrc ├── .prettierignore ├── .prettierrc.js ├── Dockerfile ├── LICENSE.txt ├── README.md ├── backend/ │ ├── migrations/ │ │ ├── 001_init.js │ │ ├── 002_add_first_serial.js │ │ ├── 003_refactor_access.sql │ │ ├── 004_public_boards_access.sql │ │ ├── 005_uuid_extension.sql │ │ ├── 006_unique_email.sql │ │ ├── 007_add_crdt_update.sql │ │ ├── 008_allow_crdt_only_bundle.sql │ │ ├── 009_drop_board_event_primary_key.sql │ │ └── 010_drop_history_column.sql │ ├── package.json │ ├── src/ │ │ ├── api/ │ │ │ ├── api-routes.ts │ │ │ ├── board-create.ts │ │ │ ├── board-csv-get.ts │ │ │ ├── board-get.ts │ │ │ ├── board-hierarchy-get.ts │ │ │ ├── board-history-get.ts │ │ │ ├── board-update.ts │ │ │ ├── github-webhook.ts │ │ │ ├── item-create-or-update.ts │ │ │ ├── item-create.ts │ │ │ └── utils.ts │ │ ├── board-event-handler.ts │ │ ├── board-state.test.ts │ │ ├── board-state.ts │ │ ├── board-store.ts │ │ ├── board-yjs-server.ts │ │ ├── common-event-handler.ts │ │ ├── compact-history.ts │ │ ├── config.ts │ │ ├── connection-handler.ts │ │ ├── db.ts │ │ ├── decodeOrThrow.ts │ │ ├── env.ts │ │ ├── expiring-map.ts │ │ ├── express-server.ts │ │ ├── generic-oidc-auth.ts │ │ ├── github-webhook/ │ │ │ └── example-payload.json │ │ ├── google-auth.ts │ │ ├── host-config.ts │ │ ├── http-session.ts │ │ ├── locker.ts │ │ ├── oauth.ts │ │ ├── professions.ts │ │ ├── require-auth.ts │ │ ├── s3.ts │ │ ├── server.ts │ │ ├── storage.ts │ │ ├── tools/ │ │ │ └── wait-for-db.ts │ │ ├── user-store.ts │ │ ├── uwebsockets-server.ts │ │ ├── websocket-sessions.ts │ │ ├── ws-wrapper.ts │ │ └── y-websocket-server/ │ │ ├── Docs.ts │ │ ├── Persistence.ts │ │ ├── Protocol.ts │ │ ├── WSSharedDoc.ts │ │ └── YWebSocketServer.ts │ └── tsconfig.json ├── benchmark/ │ └── benchmark.ts ├── common/ │ └── src/ │ ├── action-folding.ts │ ├── arrays.ts │ ├── assertNotNull.ts │ ├── authenticated-user.ts │ ├── board-crdt-helper.ts │ ├── board-reducer.benchmark.ts │ ├── board-reducer.ts │ ├── colors.ts │ ├── connection-utils.ts │ ├── domain.ts │ ├── geometry.ts │ ├── migration.test.ts │ ├── migration.ts │ ├── sets.ts │ ├── sleep.ts │ └── vector2.ts ├── cypress.json ├── docker-compose.yaml ├── frontend/ │ ├── .sassrc │ ├── esbuild.js │ ├── index.tmpl.html │ ├── package.json │ ├── src/ │ │ ├── app.scss │ │ ├── board/ │ │ │ ├── BoardView.tsx │ │ │ ├── BoardViewMessage.tsx │ │ │ ├── CollaborativeTextView.tsx │ │ │ ├── ConnectionsView.tsx │ │ │ ├── CursorsView.tsx │ │ │ ├── DragBorder.tsx │ │ │ ├── ImageView.tsx │ │ │ ├── ItemView.tsx │ │ │ ├── RectangularDragSelection.tsx │ │ │ ├── SaveAsTemplate.tsx │ │ │ ├── SelectionBorder.tsx │ │ │ ├── TextView.tsx │ │ │ ├── VideoView.tsx │ │ │ ├── autoFontSize.ts │ │ │ ├── board-coordinates.ts │ │ │ ├── board-drag.ts │ │ │ ├── board-focus.ts │ │ │ ├── board-permissions.ts │ │ │ ├── board-scroll-and-zoom.ts │ │ │ ├── boardContentArea.ts │ │ │ ├── contextmenu/ │ │ │ │ ├── ContextMenuView.tsx │ │ │ │ ├── alignments.tsx │ │ │ │ ├── areaTiling.tsx │ │ │ │ ├── colors.tsx │ │ │ │ ├── colorsAndShapes.tsx │ │ │ │ ├── connection-ends.tsx │ │ │ │ ├── fontSizes.tsx │ │ │ │ ├── hideContents.tsx │ │ │ │ ├── lock.tsx │ │ │ │ ├── shapes.tsx │ │ │ │ ├── textAlignments.tsx │ │ │ │ └── textFormats.tsx │ │ │ ├── contrasting-color.ts │ │ │ ├── double-click.ts │ │ │ ├── header/ │ │ │ │ ├── BoardViewHeader.tsx │ │ │ │ ├── OtherUsersView.tsx │ │ │ │ ├── SharingModalDialog.tsx │ │ │ │ ├── UserInfoModal.tsx │ │ │ │ └── UserInfoView.tsx │ │ │ ├── image-upload.ts │ │ │ ├── item-connect.ts │ │ │ ├── item-create.ts │ │ │ ├── item-cut-copy-paste.ts │ │ │ ├── item-delete.ts │ │ │ ├── item-drag.ts │ │ │ ├── item-dragmove.ts │ │ │ ├── item-duplicate.ts │ │ │ ├── item-hide-contents.ts │ │ │ ├── item-move-with-arrow-keys.ts │ │ │ ├── item-organizer.test.ts │ │ │ ├── item-organizer.ts │ │ │ ├── item-packer.ts │ │ │ ├── item-select-all.ts │ │ │ ├── item-selection.ts │ │ │ ├── item-setcontainer.ts │ │ │ ├── item-undo-redo.ts │ │ │ ├── keyboard-shortcuts.ts │ │ │ ├── local-storage-atom.ts │ │ │ ├── quillClickableLink.ts │ │ │ ├── quillPasteLinkOverText.ts │ │ │ ├── synchronize-focus-with-server.ts │ │ │ ├── tool-selection.ts │ │ │ ├── toolbars/ │ │ │ │ ├── BackToAllBoardsLink.tsx │ │ │ │ ├── BoardToolLayer.tsx │ │ │ │ ├── MainToolBar.tsx │ │ │ │ ├── MiniMapView.tsx │ │ │ │ ├── PaletteView.tsx │ │ │ │ ├── ToolSelector.tsx │ │ │ │ ├── UndoRedo.tsx │ │ │ │ └── ZoomControls.tsx │ │ │ ├── touchScreen.ts │ │ │ ├── zIndices.ts │ │ │ └── zoom-shortcuts.ts │ │ ├── board-navigation.ts │ │ ├── components/ │ │ │ ├── BoardAccessPolicyEditor.tsx │ │ │ ├── BoardCrdtModeSelector.tsx │ │ │ ├── EditableSpan.tsx │ │ │ ├── HTMLEditableSpan.tsx │ │ │ ├── Icons.tsx │ │ │ ├── ModalContainer.tsx │ │ │ ├── UIColors.ts │ │ │ ├── browser.ts │ │ │ ├── components.tsx │ │ │ ├── onClickOutside.tsx │ │ │ └── sanitizeHTML.ts │ │ ├── dashboard/ │ │ │ └── DashboardView.tsx │ │ ├── embedding.tsx │ │ ├── google-auth.ts │ │ ├── index.tsx │ │ ├── store/ │ │ │ ├── asset-store.ts │ │ │ ├── board-local-store.ts │ │ │ ├── board-store.test.ts │ │ │ ├── board-store.ts │ │ │ ├── crdt-store.ts │ │ │ ├── cursors-store.ts │ │ │ ├── recent-boards.ts │ │ │ ├── server-connection.ts │ │ │ └── user-session-store.ts │ │ └── style/ │ │ ├── board.scss │ │ ├── dashboard.scss │ │ ├── global.scss │ │ ├── header.scss │ │ ├── modal.scss │ │ ├── sharing-modal.scss │ │ ├── tool-layer.scss │ │ ├── user-info-modal.scss │ │ ├── utils.scss │ │ └── variables.scss │ └── tsconfig.json ├── integration/ │ └── src/ │ └── compact-history.test.ts ├── keycloak/ │ ├── README.md │ └── keycloak-db.dump ├── lint-staged.config.js ├── package.json ├── perf-tester/ │ ├── README.md │ ├── package.json │ ├── src/ │ │ ├── create-boards.ts │ │ └── index.ts │ └── tsconfig.json ├── playwright/ │ └── src/ │ ├── pages/ │ │ ├── BoardApi.ts │ │ ├── BoardPage.ts │ │ └── DashboardPage.ts │ └── tests/ │ ├── accessPolicy.spec.ts │ ├── api.spec.ts │ ├── board.spec.ts │ ├── collaboration.spec.ts │ ├── dashboard.spec.ts │ └── navigation.spec.ts ├── playwright.config.ts ├── scripts/ │ ├── migrate_user_email.sh │ └── run_dockerized.sh ├── state-management.md ├── tsconfig.json └── vitest.config.ts ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ backend/dist backend/localfiles Dockerfile frontend/.cache ignore latest.dump* node_modules npm-debug.log **/node_modules **/.env **/yarn-error.log **/.DS_Store ================================================ FILE: .editorconfig ================================================ root = true [*] indent_style = space indent_size = 4 charset = utf-8 trim_trailing_whitespace = false insert_final_newline = true [package.json] indent_size = 2 ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.md ================================================ --- name: Bug Report about: Create a bug report to help us out title: "" labels: bug assignees: "" --- # Feature request story or issue ## Screenshot YOUR SCREENSHOT HERE: ## Url - Url where you took the screenshot: - URL: ## A sentence description of the problem Short DESCRIPTION: That's it! Thanks. ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project title: "[Feature Request]" labels: enhancement assignees: "" --- **Is your feature request related to a problem? There is a Please describe.** A clear and concise description of what the problem is. Ex. When I want to do X, I am always missing Y to make it easier [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Take a screenshot of the area you're working in** Even if the feature request doesn't exist there yet, what area would the feature request help within? SCREENSHOT: URL (of where you're working): **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/workflows/build.yml ================================================ name: build on: [push] jobs: test: runs-on: ubuntu-latest services: postgres: image: postgres:12 env: POSTGRES_USER: r-board POSTGRES_PASSWORD: secret ports: - 13338:5432 # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 strategy: matrix: node-version: [18.x] steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - name: Install deps run: yarn - name: Run unit tests run: yarn test:unit - name: Wait for DB run: yarn wait-for-db - name: Run integration tests run: yarn test:integration - name: Build run: yarn build - name: Start server run: yarn start& env: SESSION_SIGNING_SECRET: notsosecretthing - name: Prepare Playwright run: npx playwright install chromium firefox - name: Run playwright tests run: yarn test:playwright - name: Archive results if: always() uses: actions/upload-artifact@v4 with: name: test-results path: | playwright/results lint: runs-on: ubuntu-latest strategy: matrix: node-version: [18.x] steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - name: Install dependencies run: yarn install --frozen-lockfile - name: Check code formatting & generated files run: yarn lint docker-image: needs: test runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' steps: - uses: actions/checkout@v2 - name: docker login env: DOCKER_HUB_USER: ${{secrets.DOCKER_HUB_USER}} DOCKER_HUB_PASSWORD: ${{secrets.DOCKER_HUB_PASSWORD}} run: docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD - name: docker build run: docker build . -t raimohanska/ourboard:latest -t raimohanska/ourboard:${{github.sha}} - name: docker push run: docker push raimohanska/ourboard:latest && docker push raimohanska/ourboard:${{github.sha}} ================================================ FILE: .gitignore ================================================ .cache/ node_modules/ dist/ yarn-error.log .env *.mp4 backend/localfiles/ /ignore .DS_Store latest.dump .vscode backups playwright/results ================================================ FILE: .huskyrc.js ================================================ module.exports = { hooks: { "pre-commit": "lint-staged", }, } ================================================ FILE: .nvmrc ================================================ 18 ================================================ FILE: .prettierignore ================================================ package.json .cache/ node_modules/ dist/ yarn-error.log .env *.mp4 backend/localfiles/ ignore/ ================================================ FILE: .prettierrc.js ================================================ module.exports = { printWidth: 120, semi: false, trailingComma: "all", } ================================================ FILE: Dockerfile ================================================ FROM node:18 as builder # Create app directory WORKDIR /usr/src/app COPY package.json yarn.lock ./ COPY frontend/package.json ./frontend/ COPY backend/package.json ./backend/ COPY perf-tester/package.json ./perf-tester/ RUN yarn install --frozen-lockfile --non-interactive COPY backend ./backend COPY frontend ./frontend COPY common ./common COPY perf-tester ./perf-tester COPY tsconfig.json . run yarn build FROM node:18 as runner COPY --from=builder /usr/src/app/backend/dist/index.js /usr/src/app/backend/dist/index.js COPY --from=builder /usr/src/app/backend/migrations /usr/src/app/backend/migrations COPY --from=builder /usr/src/app/frontend/public /usr/src/app/frontend/public COPY --from=builder /usr/src/app/frontend/dist /usr/src/app/frontend/dist WORKDIR /usr/src/app EXPOSE 1337 WORKDIR /usr/src/app/backend CMD [ "node", "dist/index.js" ] ================================================ FILE: LICENSE.txt ================================================ This project is licensed under the MIT license. Copyrights are respective of each contributor listed at the beginning of each definition file. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ An online whiteboard. Think of it as very poor man's Miro that's open source, free to use and which you can also host yourself. Feel free to try at https://www.ourboard.io/ In this Readme: - [User guide](#features-and-user-guide) - [API](#api) - [Development instructions](#development) - [Hosting Ourboard](#hosting) ## Features and User Guide The user guide here is bound to be incomplete and out-of-date. Feel welcome to improve it! ### Basics Setting your nickname or sign in - Click on the top right corner nickname field to change your nickname - Optionally, log in with your Google user account. This allows you to create private boards and to track your favorite boards across devices Adding items - Drag from palette - Double click to add a new note - Use keyboard shortcuts below Adding links - Paste a link to text, it'll automatically converted to a hyperlink - Select text and paste a link to convert the text to a link Adding images - Add by dragging a file from your computer or from a browser window - Add by pasting an image from the clipboard using Command-V Organizing your board - Create an Area and drag items on it to keep them together. When you move the Area, the items move with it! - Use Areas, lines, connections and images to give a visual structure to your board - Lock items in place to prevent accidental moves of static items and lines - Use "structured stickies": 1. Create an Area to be used as a template, choose a nice color 2. Add template content, e.g.. labels like "size" 3. Lock the labels in place. Now you can clone the whole area with a single click on the Clone button (or Cmd-D). Copy and paste - You can cut/copy/paste contents on the board using keyboard shortcuts - Copy-paste works across boards, so you can do a "backup" by selecting all notes and pasting on another board - You should be able to paste text and images on the board from other applications as well - You can create a full clone of your current board by clicking on the Clone button beside board name Keyboard shortcuts These are for Mac. For other Linux/Windows, replace Command with Control. ``` DEL/Backspace Delete item A Create new area N Create new note T Create new text box C Select the Connect tool Esc Select the default tool H Hide contents of an area Command-A Select all items Command-V Paste Command-C Copy Command-X Cut Command-Z Undo Command-Shift-Z Redo Command-D Duplicate Command-Minus Zoom out Command-Plus Zoom in Command-Zero Reset zoom Arrow keys Move selected items. SHIFT for big steps, ALT for fine-tuning. ``` Pro tips - You can drag the toolbar / palette around, if it gets in your way at the top-center position ### Board access controls All boards created accessible to anyone with the link by default. If you Sign In using Google authentication, you'll also be able to create boards with restricted access. It's possible to grant access to certain emails or to people with an email in a given domain. ## API Ourboard has a limited HTTP API for creating, exporting and updating boards. All POST and PUT endpoints accept application/json content. API requests against boards with restricted access require you to supply an API_TOKEN header with a valid API token. The token is returned in the response of the POST request used to create the board. ### POST /api/v1/board Creates a new board. Payload: ```js { "name": "board name as string", } ``` You can also specify board access policy, including individual users by email and user email domains: ```js { "name": "board name as string", "accessPolicy": { "allowList": [ { email: "coolgirl@reaktor.com" }, { domain: "reaktor.fi" } ] } } ``` Response: ```js { "id": "board id", "accessToken": "************" } ``` The `accessToken` returned here is required for further API calls in case you set an access policy. So, make sure to save the token. ### PUT /api/v1/board/:boardId Changes board name and, optionally, access policy. Payload is similar to the POST request above. This endpoint always requires the API_TOKEN header. ### POST /api/v1/board/:boardId/item Creates a new item on given board. If you want to add the item onto a specific area/container element on the board, you can find the id of the container by inspecting with your browser. To create a new item inside a container element: ```js { "type": "note", "text": "text on note", "container": "container element text or id", "color": "hexadecimal color code" } ``` To create a new item using coordinates: ```js { "type": "note", "text": "text on note", "color": "hexadecimal color code", "x": 100, "y": 100, "width": 100, "height": 100 } ``` Response: ```js { "id": "ITEM_ID" } ``` ### PUT /api/v1/board/:boardId/item/:itemId Creates a new item on given board or updates an existing one. If you want to add the item onto a specific area/container element on the board, you can find the id of the container by inspecting with your browser. Payload: ```js { "type": "note", "text": "text on note", "container": "container element text or id", "color": "hexadecimal color code", "replaceTextIfExists": boolean, // Override text if item with this id exists. Defaults to false. "replaceColorIfExists": boolean, // Override color if item with this id exists. Defaults to false. "replaceContainerIfExists": boolean, // Override container in item with this id exists. Defaults to true. } ``` or ```js { "x": "integer", "y": "integer", "type": "note", "text": "text on note", "color": "hexadecimal color code", "width": "integer", "height": "integer", } ``` ### GET /api/v1/board/:boardId Return board current state as JSON. ### GET /api/v1/board/:boardId/hierarchy Return board current state in a hierarchical format (items inside containers) ### GET /api/v1/board/:boardId/csv Return board current state in CSV format, where - A container containing only leaf items (note, text) creates a row and each item in that container gets its own column - Container name is a column on the left - Any wrapping containers also add a column on the left ### GET /api/v1/board/:boardId/history Returns the full history of given board as JSON. ## Github Issues Integration 1. Create a board and an Area named "new issues" (case insensitive) on the board. 2. Add a webhook to a git repo, namely 1. Use URL https://www.ourboard.io/api/v1/webhook/github/{board-id}, with board-id from the URL of you board. 2. Use content type to application/json 3. Select "Let me select individual events" and pick Issues only. 3. Create a new issue or change labels of an existing issue. 4. You should see new notes appear on your board ## Development Running locally requires `docker-compose` which is used for starting the local PostgreSQL database. The script below starts the database, but you must make sure you have a working docker setup on your machine, of course. Running locally: ``` yarn install yarn dev ``` Run end-to end Playwright tests - `yarn test:playwright` to run tests once - `yarn test:playwright --ui` to open the Playwright UI Connect to the local PostgreSQL database yarn psql ### Tech stack - TypeScript - [Harmaja](https://github.com/raimohanska/harmaja) frontend library - WebSockets (express-ws / uWebSockets.js both!) - Express server - Typera for HTTP API ### Developing with production data Do not run your local server against the production database, or you'll corrupt production. The server's in memory state will be out of sync with DB and bad things will happen. Instead, do this. 1. Capture a backup and download it: `heroku pg:backups:capture`, then `heroku pg:backups:download`. 2. Restore the backup to your local database: `pg_restore --verbose --clean --no-acl --no-owner -d postgres://r-board:secret@localhost:13338/r-board latest.dump` 3. Start you local server using `yarn dev` If you need the local state for a given board in localStorage, you can 1. extract the content in the browser devtools, when viewing production site in browser, using `localStorage["board_"]` 2. Copy that string to clipboard 3. Run the following in your localhost site console: localStorage["board_32de1a50-09a6-4453-9b9e-ed10c56afa99"]=JSON.stringify( ) Copy the result string, navigate to your localhost site and paste the same value to the same localStorage key. Refresh and enjoy. ### Building and pushing the raimohanska/ourboard docker image ``` docker login docker build . -t raimohanska/ourboard:latest docker push raimohanska/ourboard:latest ``` ## Hosting OurBoard is made to be easy to deploy and host. It consists of a single Node.js process that needs a PostgreSQL database for storing data. Using environment variables (see below) you can set the URL for the database and optionally configure OurBoard to use S3 for image assets and Google for authentication. By default it comes without authentication / authorization and stores image assets in the local file system. ### Heroku If it suits you, Heroku is likely to be the easiest way to host your own OurBoard server. You should be able to host your own OurBoard instance pretty easily in Heroku. This repository should be runnable as-is, provided you set up some environment variables, which are listed below. ### Docker OurBoard is available as a Docker image so you can deploy it with Docker or your favorite container environment of choice. To get an OurBoard docker image, you can either: 1. Use the [raimohanska/ourboard image](https://hub.docker.com/r/raimohanska/ourboard) in Docker Hub (just skip to running, it will be downloaded automatically) 2. Build it from this repository: `docker build . -t raimohanska/ourboard:latest` You can run it like this: 1. Start a posgres database. For example, running `docker-compose up` in your local clone of this directory will start one. 2. Start the Ourboard container. Run the example script `scripts/run_dockerized.sh` to try it out. With the example script, you'll have a setup which - Doesn't have authentication. See environment variables below for configuring Google authentication, which is the only supported option for now. - Stores uploaded assets (images) on the local filesystem. The example script binds the local directory `backend/localfiles` to be used for storage. In your own script, you'll probably want to point out a more suitable directory on your server machine. - Uses an absolutely insecure SESSION_SIGNING_KEY. Make sure to use a long random string instead. - Uses plain HTTP and responds at http://localhost:1337. If you want it to respond at some other URL, you'll need to set the ROOT\__\_URL variable (and all the WS_ variables unless you have the latest ourboard image) Read on! ### Environment variables The OurBoard server is configured using environment variables. Here's a most likely incomplete list of supported environment variables for the server. When developing the application locally, set these variables in `backend/.env` file. The most important first - you'll most likely need to set these. ``` DATABASE_URL Postgres database URL. In Heroku, you can just add the PostgreSQL add on and this variable will be correctly set. The free one will get you started. ROOT_URL Root URL used for redirects. Use https:///. If you don't have authentication configured or you're actually planning to access your server using the address http://localhost:1337, you can omit this one. PORT HTTP port that OurBoard should bind. Defaults to 1337. ``` HTTPS and TLS related settings: ``` HTTPS_PORT Local port to use for HTTPS sockets. Use this if you want OurBoard to terminate HTTPS. If you use a proxy like NGINX or run in a hosted environment like Heroku, you won't be needing this one. HTTPS_CERT_FILE Path to HTTPS certificate file. When running in docker, make sure to add appropriate mounts to make the file available to the dockerized process. HTTPS_KEY_FILE Path to HTTPS key file. When running in docker, make sure to add appropriate mounts to make the file available to the dockerized process. REDIRECT_URL Put your OurBoard application root URL here, if you want the server to redirect all requests that don't have the x-forwarded-proto=https DATABASE_SSL_ENABLED Use `true` to use SSL for database connection. Recommended. ``` AWS environment variables, needed for hosting board assets on AWS S3. If these are missing, all uploaded assests (images, videos) will be stored in the local filesystem (using the path "localfiles"), which is a viable solution only if you have a persistent file system with backups. ``` AWS_ACCESS_KEY_ID AWS access key ID AWS_SECRET_ACCESS_KEY Secret access key AWS_ASSETS_BUCKET_URL URL to the AWS bucket. For example https://r-board-assets.s3.eu-north-1.amazonaws.com ``` The experimental collaborative editing feature is controlled using environment variables as well: ``` COLLABORATIVE_EDITING `true` to enable for all new boards, `false` to disable for new boards, `opt-in` to allow opt-in on creation (default), `opt-in-authenticated` to allow opt-in for authenticated users only ``` And finally some more settings you're unlikely to need. ``` WS_HOST_DEFAULT Your domain name here, used for routing websocket connections. Is automatically derived from ROOT_URL in latest image. WS_HOST_LOCAL Your domain name here as well. Is automatically derived from ROOT_URL in latest image. WS_PROTOCOL `wss` for secure, `ws` for non-secure WebSockets. Is automatically derived from ROOT_URL in latest image. BOARD_ALIAS_tutorial Board identifier for the "tutorial" board that will be cloned for new users. Allows you to create a custom tutorial board. For example, the value `782d4942-a438-44c9-ad5f-3187cc1d0a63` is used in ourboard.io, and this points to a publicly readable, but privately editable board ``` ### Authentication configuration Ourboard can be configured to use Google or generic OpenID Connect (OIDC) authentication using environment variables. Even if you set an auth provider (see below), the server will default to allowing anonymous access as well - only boards that are explicitly set with access restrictions will require authentication. However, if you want to require authentication for all access, you can use the following environment variable. ``` REQUIRE_AUTH Use `true` to require authentication for all access. ``` #### Google authentication Google authentication is supported. To enable this feature, you'll need to supply the following environment variables. ``` GOOGLE_OAUTH_CLIENT_ID GOOGLE_OAUTH_CLIENT_SECRET ``` You'll of course need to set up an account on the Google side and configure a client so that you can get the client id and secret variables you'll use on OurBoard side. When configuring the Google client, you should allow the URL `/google-callback` as a valid callback URL. #### OpenID Connect configuration Generic OpenID Connect (OIDC) authentication is also supported as an experimental feature. To enable this feature, you'll need to supply the following environment variables. ``` OIDC_CONFIG_URL Your OpenID configuration endpoint. For example: https://accounts.google.com/.well-known/openid-configuration OIDC_CLIENT_ID Your OAuth2 client id OIDC_CLIENT_SECRET Your OAuth2 client secret OIDC_LOGOUT URL to redirect the user after a logout on Ourboard. This allows you to sign out from the OIDC provider. You can also use the value `true` to automatically determine this URL based on the `end_session_endpoint` field in the response from the OIDC_CONFIG_URL endpoint. If omitted and `REQUIRE_AUTH=true` is not set, OurBoard will simply allow anonymous usage after a logout. ``` You'll of course need an external auth provider and configure a client so that you can get the client id and secret variables you'll use on OurBoard side. When configuring the OIDC client, you should allow the URL `/google-callback` as a valid callback URL. OurBoard uses the OAuth "standard flow" or "authorization code flow" and expects to be able to find your OIDC configuration at the URL pointed by tge `OIDC_CONFIG_URL` environment variable. In the Id Token received from the Auth provider, OurBoard expects to find the following claims: - Either `name` or `preferred_username` representing the display name for the user - `email` representing the email address of the user. OurBoard does not expect this to be a valid email address; it just uses the email as the unique identifier for the user. - Optional `picture` for a URL for the user's profile picture Thus far, I've tested Ourboard OIDC with Google and Keycloak. #### OpenID Connect Using KeyCloak NOTICE: This is just a simple example for testing and **not a production-grade setup**. Make sure to configure Keycloak properly before using it in production. An example KeyCloak setup is bundled with the OurBoard development environment. To try it, set the following environment variables in `backend/.env`: ``` OICD_CONFIG_URL=http://127.0.0.1:8080/realms/ourboard/.well-known/openid-configuration OIDC_CLIENT_ID=ourboard OIDC_CLIENT_SECRET=S2qHjCg12IDxz89Lffo49NQ19ooWCUwF ``` When you start OurBoard in development mode using `yarn dev-with-keycloak`, you can now Sign In using the username `ourboard-test` and password `password`. ## Contribution See Issues! ## A word from our sponsor I want to thank [Reaktor](https://www.reaktor.com/) for the huge and essential support for this project! - We (Reaktorian contributors) get some monetary support from the Reaktor open-source support program - Hosting costs covered by Reaktor (Heroku, AWS) - The UI design was done by Reaktor's experts (Mira Myllylä for the tool and Mari Halla-aho for the dashboard) ================================================ FILE: backend/migrations/001_init.js ================================================ exports.up = (pgm) => { pgm.sql(` CREATE TABLE IF NOT EXISTS board (id text PRIMARY KEY, name text NOT NULL); ALTER TABLE board ADD COLUMN IF NOT EXISTS content JSONB NOT NULL; ALTER TABLE board ADD COLUMN IF NOT EXISTS history JSONB NOT NULL default '[]'; CREATE TABLE IF NOT EXISTS board_event (board_id text REFERENCES board(id), last_serial integer, events JSONB NOT NULL, PRIMARY KEY (board_id, last_serial)); ALTER TABLE board_event ALTER COLUMN last_serial SET NOT NULL; ALTER TABLE board_event ALTER COLUMN board_id SET NOT NULL; CREATE TABLE IF NOT EXISTS board_api_token (board_id text REFERENCES board(id), token TEXT NOT NULL); CREATE TABLE IF NOT EXISTS app_user (id text PRIMARY KEY, email text NOT NULL); CREATE TABLE IF NOT EXISTS user_board (user_id text REFERENCES app_user(id), board_id text REFERENCES board(id), last_opened TIMESTAMP NOT NULL, PRIMARY KEY (user_id, board_id)); ALTER TABLE board ADD COLUMN IF NOT EXISTS ws_host TEXT NULL; ALTER TABLE board ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NULL DEFAULT now(); ALTER TABLE board_event ADD COLUMN IF NOT EXISTS saved_at TIMESTAMP NULL DEFAULT now(); `) } ================================================ FILE: backend/migrations/002_add_first_serial.js ================================================ exports.up = (pgm) => { pgm.sql(` ALTER TABLE board_event ADD COLUMN IF NOT EXISTS first_serial int; UPDATE board_event SET first_serial = COALESCE(CAST(events#>'{events, 0, serial}' as int), 0); ALTER TABLE board_event ALTER COLUMN first_serial SET NOT NULL; `) } ================================================ FILE: backend/migrations/003_refactor_access.sql ================================================ alter table board add public_read boolean null; alter table board add public_write boolean null; create table board_access ( board_id text not null references board(id), domain text null, email text null, access text null ); with allow_json as ( select id, jsonb_array_elements(content -> 'accessPolicy' -> 'allowList') as e from board), allow_entry as ( select id, e ->> 'domain' as domain, e ->> 'email' as email, e ->> 'access' as access from allow_json ) insert into board_access(board_id, domain, email, access) ( select ae.id as board_id, ae.domain, ae.email, ae.access from allow_entry ae ); update board set public_read = coalesce(content -> 'accessPolicy' ->> 'publicRead', 'false') :: boolean, public_write = coalesce(content -> 'accessPolicy' ->> 'publicWrite', 'false') :: boolean where content -> 'accessPolicy' is not null ================================================ FILE: backend/migrations/004_public_boards_access.sql ================================================ update board set public_read = 't', public_write = 't' where content -> 'accessPolicy' is null ================================================ FILE: backend/migrations/005_uuid_extension.sql ================================================ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; ================================================ FILE: backend/migrations/006_unique_email.sql ================================================ ALTER TABLE app_user ADD CONSTRAINT unique_email UNIQUE (email); ================================================ FILE: backend/migrations/007_add_crdt_update.sql ================================================ alter table board_event add column crdt_update bytea null; ================================================ FILE: backend/migrations/008_allow_crdt_only_bundle.sql ================================================ alter table board_event alter column first_serial drop not null; alter table board_event add constraint first_serial_or_crdt check (crdt_update is not null OR first_serial is not null); ================================================ FILE: backend/migrations/009_drop_board_event_primary_key.sql ================================================ ALTER TABLE board_event DROP CONSTRAINT board_event_pkey; CREATE INDEX board_event_board_index ON board_event (board_id); ================================================ FILE: backend/migrations/010_drop_history_column.sql ================================================ alter table board drop column history; ================================================ FILE: backend/package.json ================================================ { "name": "rboard-backend", "version": "1.0.0", "main": "dist/index.js", "license": "MIT", "dependencies": { "@types/cookies": "^0.7.7", "@types/date-fns": "^2.6.0", "@types/express-ws": "^3.0.0", "@types/html-entities": "^1.2.16", "@types/json-diff": "^0.5.0", "@types/lodash": "^4.14.161", "@types/node": "^14.6.2", "@types/pg": "^7.14.4", "@types/ramda": "^0.27.29", "@types/tcp-port-used": "^1.0.0", "@types/ws": "^7.4.0", "aws-sdk": "^2.778.0", "cookies": "^0.8.0", "csv-writer": "^1.6.0", "date-fns": "^2.17.0", "dotenv": "^8.2.0", "express": "^4.17.1", "express-ws": "^4.0.0", "fp-ts": "^2.9.5", "google-auth-library": "^7.0.2", "googleapis": "^110.0.0", "html-entities": "^2.1.0", "io-ts": "^2.2.15", "io-ts-types": "^0.5.15", "json-diff": "^0.5.4", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.20", "lonna": "^0.12.2", "monocle-ts": "^2.3.7", "newtype-ts": "^0.3.4", "node-pg-migrate": "^6.0.0", "pg": "^8.3.3", "pg-query-stream": "^4.2.1", "tcp-port-used": "^1.0.2", "tsx": "3.13.0", "typera-express": "^2.3.0", "typescript": "^5.3", "uWebSockets.js": "uNetworking/uWebSockets.js#v18.14.0", "uuid": "^8.3.0", "uuid4": "^2.0.2", "y-protocols": "^1.0.6" }, "scripts": { "start": "node --enable-source-maps .", "dev": "npm-run-all --parallel watch", "build": "tsc && ncc build dist/backend/src/server.js -o dist", "watch": "tsc-watch --onSuccess \"node --enable-source-maps dist/backend/src/server.js\" --preserveWatchOutput", "watch-ts": "tsc-watch --preserveWatchOutput", "lint": "echo 'No linting configured'" }, "engines": { "node": ">=14" }, "devDependencies": { "@types/express": "^4.17.7", "@types/jsonwebtoken": "^8.5.0", "@types/node-fetch": "^2.5.9", "@types/uuid": "^8.3.0", "@vercel/ncc": "^0.38.1", "tsc-watch": "^4.2.9" } } ================================================ FILE: backend/src/api/api-routes.ts ================================================ import { router } from "typera-express" import { boardCreate } from "./board-create" import { boardCSVGet } from "./board-csv-get" import { boardGet } from "./board-get" import { boardHierarchyGet } from "./board-hierarchy-get" import { boardHistoryGet } from "./board-history-get" import { boardUpdate } from "./board-update" import { githubWebhook } from "./github-webhook" import { itemCreate } from "./item-create" import { itemCreateOrUpdate } from "./item-create-or-update" export default router( boardGet, boardHierarchyGet, boardCSVGet, boardCreate, boardUpdate, githubWebhook, itemCreate, itemCreateOrUpdate, boardHistoryGet, ) ================================================ FILE: backend/src/api/board-create.ts ================================================ import * as t from "io-ts" import { NonEmptyString } from "io-ts-types" import { ok } from "typera-common/response" import { body } from "typera-express/parser" import { Board, BoardAccessPolicyCodec, CrdtDisabled, CrdtEnabled, newBoard, optional, } from "../../../common/src/domain" import { addBoard } from "../board-state" import { route } from "./utils" import { getConfig } from "../config" /** * Creates a new board. * * @tags Board */ export const boardCreate = route .post("/api/v1/board") .use(body(t.type({ name: NonEmptyString, accessPolicy: BoardAccessPolicyCodec, crdt: optional(t.literal(true)) }))) .handler(async (request) => { const crdt = (getConfig().crdt === "true" || request.body.crdt) ?? false let board: Board = newBoard(request.body.name, crdt ? CrdtEnabled : CrdtDisabled, request.body.accessPolicy) const boardWithHistory = await addBoard(board, true) return ok({ id: boardWithHistory.board.id, accessToken: boardWithHistory.accessTokens[0] }) }) ================================================ FILE: backend/src/api/board-csv-get.ts ================================================ import { createArrayCsvStringifier } from "csv-writer" import _ from "lodash" import { ok } from "typera-common/response" import { Board, Container, Item, TextItem } from "../../../common/src/domain" import { apiTokenHeader, checkBoardAPIAccess, route } from "./utils" import { augmentBoardWithCRDT } from "../../../common/src/board-crdt-helper" import { yWebSocketServer } from "../board-yjs-server" /** * Gets board current contents * * @tags Board */ export const boardCSVGet = route .get("/api/v1/board/:boardId/csv") .use(apiTokenHeader) .handler((request) => checkBoardAPIAccess(request, async (boardState) => { const board = augmentBoardWithCRDT( await yWebSocketServer.docs.getYDocAndWaitForFetch(boardState.board.id), boardState.board, ) const textItemsWithParent = Object.values(board.items).filter( (i) => i.containerId !== undefined && (i.type === "text" || i.type === "note"), ) const textItemGroups = _.groupBy(textItemsWithParent, (i) => i.containerId) const rows = Object.entries(textItemGroups).map(([parentId, textItems]) => { const rowContainer = board.items[parentId] return { parents: parentChain(board)(rowContainer), rowContainer, textItems, } as Row }) if (rows.length === 0) return csv(board, []) const maxDepth = _.max(rows.map((r) => r.parents.length))! const csvData = rows.map((r) => { return [ ...r.parents.map((c) => c.text), ..._.times(maxDepth - r.parents.length, () => ""), r.rowContainer.text, ...r.textItems.map((i) => i.text), ] }) return csv(board, csvData) }), ) type Row = { parents: Container[]; rowContainer: Container; textItems: TextItem[] } const parentChain = (board: Board) => (item: Item): Container[] => { if (!item.containerId) return [] const parent = board.items[item.containerId] if (parent.type !== "container") throw Error(`Parent item ${item.containerId} is of type ${parent.type}, expecting container`) return [parent, ...parentChain(board)(parent)] } function csv(board: Board, rows: string[][]) { const result = createArrayCsvStringifier({}).stringifyRecords(rows) return ok(result, { "content-type": "text/csv", "content-disposition": `attachment; filename=${board.name}.csv` }) } ================================================ FILE: backend/src/api/board-get.ts ================================================ import { ok } from "typera-common/response" import { apiTokenHeader, checkBoardAPIAccess, route } from "./utils" import { yWebSocketServer } from "../board-yjs-server" import { augmentBoardWithCRDT } from "../../../common/src/board-crdt-helper" /** * Gets board current contents * * @tags Board */ export const boardGet = route .get("/api/v1/board/:boardId") .use(apiTokenHeader) .handler((request) => checkBoardAPIAccess(request, async (boardState) => { const board = augmentBoardWithCRDT( await yWebSocketServer.docs.getYDocAndWaitForFetch(boardState.board.id), boardState.board, ) return ok({ board }) }), ) ================================================ FILE: backend/src/api/board-hierarchy-get.ts ================================================ import { ok } from "typera-common/response" import { Board, Item } from "../../../common/src/domain" import { apiTokenHeader, checkBoardAPIAccess, route } from "./utils" import { augmentBoardWithCRDT } from "../../../common/src/board-crdt-helper" import { yWebSocketServer } from "../board-yjs-server" /** * Gets board current contents * * @tags Board */ export const boardHierarchyGet = route .get("/api/v1/board/:boardId/hierarchy") .use(apiTokenHeader) .handler((request) => checkBoardAPIAccess(request, async (boardState) => { const board = augmentBoardWithCRDT( await yWebSocketServer.docs.getYDocAndWaitForFetch(boardState.board.id), boardState.board, ) return ok({ board: getBoardHierarchy(board) }) }), ) export type ItemHierarchy = Item & { children: ItemHierarchy[] } export function getBoardHierarchy(board: Board) { const allItems = Object.values(board.items) const rootItems = allItems.filter((i) => i.containerId === undefined).map(getItemHierarchy(allItems)) return { ...board, items: rootItems } } const getItemHierarchy = (items: Item[]) => (item: Item): ItemHierarchy => { const children: ItemHierarchy[] = items.filter((i) => i.containerId === item.id).map(getItemHierarchy(items)) return { ...item, children } } ================================================ FILE: backend/src/api/board-history-get.ts ================================================ import { ok, streamingBody } from "typera-common/response" import { getFullBoardHistory } from "../board-store" import { withDBClient } from "../db" import { apiTokenHeader, checkBoardAPIAccess, route } from "./utils" /** * List the history of a board * * @tags Board */ export const boardHistoryGet = route .get("/api/v1/board/:boardId/history") .use(apiTokenHeader) .handler((request) => checkBoardAPIAccess(request, async (board) => { return ok( streamingJSONBody("history", async (callback) => { await withDBClient( async (client) => await getFullBoardHistory(board.board.id, client, (bundle) => bundle.forEach(callback)), ) }), ) }), ) function streamingJSONBody(fieldName: string, generator: (callback: (item: any) => void) => Promise) { return streamingBody(async (stream) => { // Due to memory concerns we fetch board histories from DB as chunks, so this API // response must also be chunked try { stream.write(`{"${fieldName}":[`) let chunksProcessed = 0 await generator((item) => { let prefix = chunksProcessed === 0 ? "" : "," stream.write(`${prefix}${JSON.stringify(item)}`) chunksProcessed++ }) stream.write("]}") stream.end() //console.log(`Wrote ${chunksProcessed} chunks`) } catch (e) { console.error(`Error writing a streamed body: ${e}`) stream.end() } }) } ================================================ FILE: backend/src/api/board-update.ts ================================================ import * as t from "io-ts" import { NonEmptyString } from "io-ts-types" import { ok } from "typera-common/response" import { body } from "typera-express/parser" import { BoardAccessPolicyCodec } from "../../../common/src/domain" import { apiTokenHeader, checkBoardAPIAccess, dispatchSystemAppEvent, route } from "./utils" import { renameBoardConvenienceColumnOnly, updateBoardAccessPolicy } from "../board-store" /** * Changes board name and, optionally, access policy. * * @tags Board */ export const boardUpdate = route .put("/api/v1/board/:boardId") .use(apiTokenHeader, body(t.type({ name: NonEmptyString, accessPolicy: BoardAccessPolicyCodec }))) .handler((request) => checkBoardAPIAccess(request, async (board) => { const { boardId } = request.routeParams const { name, accessPolicy } = request.body await renameBoardConvenienceColumnOnly(boardId, name) dispatchSystemAppEvent(board, { action: "board.rename", boardId, name }) if (accessPolicy) { await updateBoardAccessPolicy(boardId, accessPolicy) dispatchSystemAppEvent(board, { action: "board.setAccessPolicy", boardId, accessPolicy }) } return ok({ ok: true }) }), ) ================================================ FILE: backend/src/api/github-webhook.ts ================================================ import { encode as htmlEncode } from "html-entities" import * as t from "io-ts" import { badRequest, internalServerError, ok } from "typera-common/response" import { body } from "typera-express/parser" import { RED, YELLOW } from "../../../common/src/colors" import { Note } from "../../../common/src/domain" import { getBoard } from "../board-state" import { addItem, dispatchSystemAppEvent, InvalidRequest, route } from "./utils" // TODO: require API_TOKEN header for github too! /** * GitHub webhook * * @tags Webhooks */ export const githubWebhook = route .post("/api/v1/webhook/github/:boardId") .use( body( t.partial({ issue: t.type({ html_url: t.string, title: t.string, number: t.number, state: t.string, labels: t.array(t.type({ name: t.string })), }), }), ), ) .handler(async (request) => { try { const boardId = request.routeParams.boardId const body = request.body const board = await getBoard(boardId) if (!board) { console.warn(`Github webhook call for unknown board ${boardId}`) return ok() } if (body.issue) { const url = body.issue.html_url const title = body.issue.title const number = body.issue.number.toString() const state = body.issue.state if (state !== "open") { console.log(`Github webhook call board ${boardId}: Item in ${state} state`) } else { const linkStart = `` const linkHTML = `${linkStart}${htmlEncode(number)} ${htmlEncode(title)}` const existingItem = Object.values(board.board.items).find( (i) => i.type === "note" && i.text.includes(url), ) as Note | undefined const isBug = body.issue.labels.some((l) => l.name === "bug") const color = isBug ? RED : YELLOW if (!existingItem) { console.log(`Github webhook call board ${boardId}: New item`) addItem(board, "note", linkHTML, color, "New issues") } else { console.log(`Github webhook call board ${boardId}: Item exists`) const updatedItem: Note = { ...existingItem, color } dispatchSystemAppEvent(board, { action: "item.update", boardId, items: [updatedItem] }) } } } return ok() } catch (e) { console.error(e) if (e instanceof InvalidRequest) { return badRequest(e.message) } else { return internalServerError() } } }) ================================================ FILE: backend/src/api/item-create-or-update.ts ================================================ import * as t from "io-ts" import { NonEmptyString } from "io-ts-types" import _ from "lodash" import { ok } from "typera-common/response" import { body } from "typera-express/parser" import { Color, isNote, Note } from "../../../common/src/domain" import { ServerSideBoardState } from "../board-state" import { addItem, apiTokenHeader, checkBoardAPIAccess, dispatchSystemAppEvent, findContainer, getItemAttributesForContainer, InvalidRequest, route, } from "./utils" /** * Creates a new item on given board or updates an existing one. * If you want to add the item onto a specific area/container element on the board, you can * find the id of the container by inspecting with your browser. * * @tags Board */ export const itemCreateOrUpdate = route .put("/api/v1/board/:boardId/item/:itemId") .use( apiTokenHeader, body( t.intersection([ t.type({ type: t.literal("note"), text: NonEmptyString, color: t.string, }), t.partial({ container: t.string, x: t.number, y: t.number, width: t.number, height: t.number, replaceTextIfExists: t.boolean, replaceColorIfExists: t.boolean, replaceContainerIfExists: t.boolean, }), ]), ), ) .handler((request) => checkBoardAPIAccess(request, async (board) => { const { itemId } = request.routeParams let { type, text, color, container, replaceTextIfExists, replaceColorIfExists, replaceContainerIfExists = true, ...rest } = request.body console.log(`PUT item for board ${board.board.id} item ${itemId}: ${JSON.stringify(request.req.body)}`) const existingItem = board.board.items[itemId] if (existingItem) { updateItem( board, rest.x ?? existingItem.x, rest.y ?? existingItem.y, type, text, color, container, rest.width ?? existingItem.width, rest.height ?? existingItem.height, itemId, replaceTextIfExists, replaceColorIfExists, replaceContainerIfExists, ) } else { console.log(`Adding new item`) addItem(board, type, text, color, container, itemId, rest) } return ok({ ok: true }) }), ) function updateItem( board: ServerSideBoardState, x: number, y: number, type: "note", text: string, color: Color, container: string | undefined, width: number, height: number, itemId: string, replaceTextIfExists: boolean | undefined, replaceColorIfExists: boolean | undefined, replaceContainerIfExists: boolean | undefined, ) { const existingItem = board.board.items[itemId] if (!isNote(existingItem)) { throw new InvalidRequest("Unexpected item type") } let updatedItem: Note = { ...existingItem, x: x !== undefined ? x : existingItem.x, y: y !== undefined ? y : existingItem.y, text: replaceTextIfExists !== false ? text : existingItem.text, color: replaceColorIfExists !== false ? color || existingItem.color : existingItem.color, width: width !== undefined ? width : existingItem.width, height: height !== undefined ? height : existingItem.height, } if (container && replaceContainerIfExists !== false) { const containerItem = findContainer(container, board.board) const currentContainer = findContainer(existingItem.containerId, board.board) const containerAttrs = containerItem !== currentContainer ? getItemAttributesForContainer(container, board.board) : {} updatedItem = { ...updatedItem, ...containerAttrs, } } if (!_.isEqual(updatedItem, existingItem)) { console.log(`Updating existing item`) dispatchSystemAppEvent(board, { action: "item.update", boardId: board.board.id, items: [updatedItem] }) } else { console.log(`Not updating: item not changed`) } } ================================================ FILE: backend/src/api/item-create.ts ================================================ import * as t from "io-ts" import { ok } from "typera-common/response" import { body } from "typera-express/parser" import { addItem, apiTokenHeader, checkBoardAPIAccess, route } from "./utils" /** * Creates a new item on given board. If you want to add the item onto a * specific area/container element on the board, you can find the id of the * container by inspecting with your browser. * * @tags Board */ export const itemCreate = route .post("/api/v1/board/:boardId/item") .use( apiTokenHeader, body( t.intersection([ t.type({ type: t.literal("note"), text: t.string, color: t.string, }), t.partial({ container: t.string, x: t.number, y: t.number, width: t.number, height: t.number, }), ]), ), ) .handler((request) => checkBoardAPIAccess(request, async (board) => { const { type, text, color, container, ...rest } = request.body console.log(`POST item for board ${board.board.id}: ${JSON.stringify(request.req.body)}`) const item = addItem(board, type, text, color, container, undefined, rest) return ok(item) }), ) ================================================ FILE: backend/src/api/utils.ts ================================================ import * as bodyParser from "body-parser" import * as t from "io-ts" import { badRequest, internalServerError, notFound } from "typera-common/response" import { applyMiddleware } from "typera-express" import { wrapNative } from "typera-express/middleware" import { headers } from "typera-express/parser" import { DEFAULT_NOTE_COLOR } from "../../../common/src/colors" import { AppEvent, Board, BoardHistoryEntry, Color, Container, EventUserInfo, newISOTimeStamp, newNote, Note, PersistableBoardItemEvent, } from "../../../common/src/domain" import { getBoard, ServerSideBoardState, updateBoards } from "../board-state" import { broadcastBoardEvent } from "../websocket-sessions" export const route = applyMiddleware(wrapNative(bodyParser.json())) export const apiTokenHeader = headers(t.partial({ API_TOKEN: t.string })) export async function checkBoardAPIAccess( request: { routeParams: { boardId: string }; headers: { API_TOKEN?: string | undefined } }, fn: (board: ServerSideBoardState) => Promise, ) { const boardId = request.routeParams.boardId const apiToken = request.headers.API_TOKEN try { const board = await getBoard(boardId) if (!board) return notFound() if (board.board.accessPolicy || board.accessTokens.length) { if (!apiToken) { return badRequest("API_TOKEN header is missing") } if (!board.accessTokens.some((t) => t === apiToken)) { console.log(`API_TOKEN ${apiToken} not on list ${board.accessTokens}`) return badRequest("Invalid API_TOKEN") } } return await fn(board) } catch (e) { console.error(e) if (e instanceof InvalidRequest) { return badRequest(e.message) } else { return internalServerError() } } } export function findContainer(container: string | undefined, board: Board): Container | null { if (container !== undefined) { if (typeof container !== "string") { throw new InvalidRequest("Expecting container to be undefined, or an id or name of an Container item") } const containerItem = Object.values(board.items).find( (i) => i.type === "container" && (i.text.toLowerCase() === container.toLowerCase() || i.id === container), ) if (!containerItem) { throw new InvalidRequest(`Container "${container}" not found by id or name`) } return containerItem as Container } else { return null } } export function getItemAttributesForContainer(container: string | undefined, board: Board) { const containerItem = findContainer(container, board) if (containerItem) { return { containerId: containerItem.id, x: containerItem.x + 2, y: containerItem.y + 2, } } return {} } export function dispatchSystemAppEvent(board: ServerSideBoardState, appEvent: PersistableBoardItemEvent) { const user: EventUserInfo = { userType: "system", nickname: "Github webhook" } let historyEntry: BoardHistoryEntry = { ...appEvent, user, timestamp: newISOTimeStamp() } console.log(JSON.stringify(historyEntry)) // TODO: refactor, this is the same sequence as done in connection-handler for messages from clients const serial = updateBoards(board, historyEntry) historyEntry = { ...historyEntry, serial } broadcastBoardEvent(historyEntry) } export function addItem( board: ServerSideBoardState, type: "note", text: string, color: Color, container: string | undefined, itemId?: string, partialParams?: Partial<{ x: number; y: number; width: number; height: number }>, ) { if (type !== "note") throw new InvalidRequest("Expecting type: note") if (typeof text !== "string" || text.length === 0) throw new InvalidRequest("Expecting non zero-length text") let itemAttributes: object = getItemAttributesForContainer(container, board.board) if (itemId) itemAttributes = { ...itemAttributes, id: itemId } // Merge partial parameters with existing attributes if (partialParams) { itemAttributes = { ...itemAttributes, ...partialParams } } const item: Note = { ...newNote(text, color || DEFAULT_NOTE_COLOR), ...itemAttributes } const appEvent: AppEvent = { action: "item.add", boardId: board.board.id, items: [item], connections: [] } dispatchSystemAppEvent(board, appEvent) return item } export class InvalidRequest extends Error { constructor(message: string) { super(message) } } ================================================ FILE: backend/src/board-event-handler.ts ================================================ import { AppEvent, BoardHistoryEntry, canWrite, checkBoardAccess, Id, isBoardItemEvent, isPersistableBoardItemEvent, newISOTimeStamp, } from "../../common/src/domain" import { getBoard, maybeGetBoard, updateBoards } from "./board-state" import { getBoardInfo, renameBoardConvenienceColumnOnly, updateBoardAccessPolicy } from "./board-store" import { handleCommonEvent } from "./common-event-handler" import { MessageHandlerResult } from "./connection-handler" import { WS_HOST_DEFAULT, WS_HOST_LOCAL, WS_PROTOCOL } from "./host-config" import { obtainLock } from "./locker" import { associateUserWithBoard } from "./user-store" import { addSessionToBoard, broadcastBoardEvent, getSession } from "./websocket-sessions" import { toBuffer, WsWrapper } from "./ws-wrapper" export const handleBoardEvent = (allowedBoardId: Id | null, getSignedPutUrl: (key: string) => string) => async ( socket: WsWrapper, appEvent: AppEvent, ): Promise => { if (await handleCommonEvent(socket, appEvent)) return true const session = getSession(socket) if (!session) { console.error("Session missing for socket " + socket.id) return true } if (appEvent.action === "board.join") { //await sleep(3000) // simulate latency const boardInfo = await getBoardInfo(appEvent.boardId) if (!boardInfo) { console.warn(`Trying to join unknown board ${appEvent.boardId}`) session.sendEvent({ action: "board.join.denied", boardId: appEvent.boardId, reason: "not found", }) return true } const wsHost = boardInfo.ws_host ?? WS_HOST_DEFAULT if (!allowedBoardId || appEvent.boardId !== allowedBoardId || !WS_HOST_LOCAL.includes(wsHost)) { // Path - board id mismatch -> always redirect const wsAddress = `${WS_PROTOCOL}://${wsHost}/socket/board/${appEvent.boardId}` /* console.info( `Trying to join board ${appEvent.boardId} on socket for board ${allowedBoardId}, board host ${wsHost} local hostnames ${WS_HOST_LOCAL}`, )*/ session.sendEvent({ action: "board.join.denied", boardId: appEvent.boardId, reason: "redirect", wsAddress, }) return true } let board = (await getBoard(appEvent.boardId))! const accessPolicy = board.board.accessPolicy const accessLevel = checkBoardAccess(accessPolicy, session.userInfo) if (session.userInfo.userType === "authenticated") { if (accessLevel === "none") { console.warn("Access denied to board by user not on allowList") session.sendEvent({ action: "board.join.denied", boardId: appEvent.boardId, reason: "forbidden", }) return true } else { await associateUserWithBoard(session.userInfo.userId, appEvent.boardId) } } else { if (accessLevel === "none") { console.warn("Access denied to board by anonymous user") session.sendEvent({ action: "board.join.denied", boardId: appEvent.boardId, reason: "unauthorized", }) return true } } await addSessionToBoard(board, socket, accessLevel, appEvent.initAtSerial) return true } if (!session.boardSession) { console.warn("Trying to send event to board without session", appEvent) return true } if (isBoardItemEvent(appEvent)) { const boardId = appEvent.boardId const state = await getBoard(boardId) if (!state) { return true // Just ignoring for now, see above todo } if (!canWrite(session.boardSession.accessLevel)) { console.warn("Trying to change read-only board") return true } obtainLock(state.locks, appEvent, socket) // Allow even if was locked (offline use) if (isPersistableBoardItemEvent(appEvent)) { if (!session.isOnBoard(appEvent.boardId)) { console.warn("Trying to send event to board without valid session") } else { let historyEntry: BoardHistoryEntry = { ...appEvent, user: session.userInfo, timestamp: newISOTimeStamp(), } try { const serial = updateBoards(state, historyEntry) historyEntry = { ...historyEntry, serial } broadcastBoardEvent(historyEntry, session) if (appEvent.action === "board.rename") { // special case: keeping name up to date as it's in a separate column await renameBoardConvenienceColumnOnly(appEvent.boardId, appEvent.name) } if (appEvent.action === "board.setAccessPolicy") { if (session.boardSession.accessLevel !== "admin") { console.warn("Trying to change access policy without admin access") return true } await updateBoardAccessPolicy(appEvent.boardId, appEvent.accessPolicy) } return { boardId, serial } } catch (e) { console.warn(`Error applying event ${JSON.stringify(appEvent)}: ${e} -> forcing board refresh`) session.sendEvent({ action: "board.action.apply.failed" }) return true } } } else if (appEvent.action === "item.unlock") { return true } } else { switch (appEvent.action) { case "cursor.move": { const { boardId, position } = appEvent const { x, y } = position const state = maybeGetBoard(boardId) if (state) { const session = getSession(socket) if (session && session.isOnBoard(appEvent.boardId)) { state.cursorPositions[socket.id] = { x, y, sessionId: socket.id } state.cursorsMoved = true } } return true } case "user.bringAllToMe": { const session = getSession(socket) const state = maybeGetBoard(appEvent.boardId) if (session && state && session.isOnBoard(appEvent.boardId)) { if (session.sessionId !== appEvent.sessionId) { console.warn("Incorrect sessionId in user.bringAllToMe") } else { broadcastBoardEvent(appEvent, session) } } return true } case "asset.put.request": { const { assetId } = appEvent const signedUrl = getSignedPutUrl(assetId) socket.send(toBuffer({ action: "asset.put.response", assetId, signedUrl })) return true } } } return false } ================================================ FILE: backend/src/board-state.test.ts ================================================ import { describe, expect, it } from "vitest" describe("board state iteration", () => { it("is safe", () => { // Checking that map value iteration is safe when deleteting items on the way const boards = new Map() boards.set(1, 1) boards.set(2, 2) const results: number[] = [] for (let b of boards.values()) { results.push(b) boards.delete(b) } expect(results).toEqual([1, 2]) }) }) ================================================ FILE: backend/src/board-state.ts ================================================ import { merge } from "lodash" import { boardReducer } from "../../common/src/board-reducer" import { Board, BoardCursorPositions, BoardHistoryEntry, Id } from "../../common/src/domain" import { sleep } from "../../common/src/sleep" import { createAccessToken, createBoard, fetchBoard, storeEventHistoryBundle } from "./board-store" import { quickCompactBoardHistory } from "./compact-history" import { Locks } from "./locker" import { UserSession, broadcastItemLocks, getBoardSessionCount, getSessionCount } from "./websocket-sessions" import * as Y from "yjs" import { inTransaction } from "./db" // A mutable state object for server side state export type ServerSideBoardState = { ready: true board: Board recentEvents: BoardHistoryEntry[] recentCrdtUpdate: Uint8Array | null currentlyStoring: { events: BoardHistoryEntry[] crdtUpdate: Uint8Array | null } | null locks: ReturnType cursorsMoved: boolean cursorPositions: BoardCursorPositions accessTokens: string[] sessions: UserSession[] } export type ServerSideBoardStateInternal = | ServerSideBoardState | { ready: false fetch: Promise } let boards: Map = new Map() export async function getBoard(id: Id): Promise { let state = boards.get(id) if (!state) { console.log(`Loading board ${id} into memory`) const fetchState = async () => { const boardData = await fetchBoard(id) if (!boardData) return null const { board, accessTokens } = boardData return { ready: true, board, accessTokens, recentEvents: [], recentCrdtUpdate: null, currentlyStoring: null, locks: Locks((changedLocks) => broadcastItemLocks(id, changedLocks)), cursorsMoved: false, cursorPositions: {}, sessions: [], } as ServerSideBoardState } const fetch = fetchState() const temporaryState = { ready: false as const, fetch, } boards.set(id, temporaryState) try { const finalState = await fetch if (!finalState) { boards.delete(id) return null } else { boards.set(id, finalState) console.log(`Board loaded into memory: ${id}`) return finalState } } catch (e) { boards.delete(id) // TODO: avoid retry loop console.error(`Board load failed for board ${id}`) throw e } } else if (!state.ready) { return await state.fetch } else { return state } } export function maybeGetBoard(id: Id): ServerSideBoardState | undefined { const state = boards.get(id) if (state?.ready) return state } export function updateBoards(boardState: ServerSideBoardState, appEvent: BoardHistoryEntry) { const currentSerial = boardState.board.serial const serial = currentSerial + 1 if (appEvent.serial !== undefined) { throw Error("Event already has serial") } const eventWithSerial = { ...appEvent, serial } const updatedBoard = boardReducer(boardState.board, eventWithSerial, { inplace: true, strictOnSerials: true })[0] boardState.board = updatedBoard boardState.recentEvents.push(eventWithSerial) return serial } export function updateBoardCrdt(id: Id, crdtUpdate: Uint8Array) { const boardState = maybeGetBoard(id) if (!boardState) { console.warn("CRDT update for board not loaded into memory", id) } else { boardState.recentCrdtUpdate = combineCrdtUpdates(boardState.recentCrdtUpdate, crdtUpdate) } } export async function addBoard(board: Board, createToken?: boolean): Promise { await createBoard(board) const accessTokens = createToken ? [await createAccessToken(board)] : [] const boardState = { ready: true as const, board, serial: 0, recentEvents: [], recentCrdtUpdate: null, currentlyStoring: null, locks: Locks((changedLocks) => broadcastItemLocks(board.id, changedLocks)), cursorsMoved: false, cursorPositions: {}, accessTokens, sessions: [], } boards.set(board.id, boardState) return boardState } export function getActiveBoards() { return [...boards.values()].filter((b) => b.ready) as ServerSideBoardState[] } let savingPromise: Promise = saveBoards() async function saveBoards() { await sleep(1000) for (let state of boards.values()) { if (state.ready) await saveBoardChanges(state) } savingPromise = saveBoards() } export async function awaitSavingChanges() { await savingPromise } async function saveBoardChanges(state: ServerSideBoardState) { if (state.recentEvents.length > 0 || state.recentCrdtUpdate !== null) { if (state.currentlyStoring) { throw Error("Invariant failed: storingEvents not empty") } const events = state.recentEvents.splice(0) const crdtUpdate = state.recentCrdtUpdate state.currentlyStoring = { events, crdtUpdate, } state.recentCrdtUpdate = null console.log( `Saving board ${state.board.id} at serial ${state.board.serial} with ${ state.currentlyStoring.events.length } new events ${crdtUpdate ? "and CRDT update of size " + crdtUpdate.length : ""}`, ) const lastSerial = state.board.serial try { await inTransaction((client) => storeEventHistoryBundle(state.board.id, events, lastSerial, crdtUpdate, client), ) } catch (e) { // Push event back to the head of save list for retrying later state.recentEvents = [...state.currentlyStoring.events, ...state.recentEvents] state.recentCrdtUpdate = merge(state.currentlyStoring.crdtUpdate, state.recentCrdtUpdate) console.error("Board save failed for board", state.board.id, e) } state.currentlyStoring = null } if (state.recentEvents.length === 0 && getBoardSessionCount(state.board.id) === 0) { console.log(`Purging board ${state.board.id} from memory`) boards.delete(state.board.id) await quickCompactBoardHistory(state.board.id) } } export function combineCrdtUpdates(a: Uint8Array | null, b: Uint8Array | null) { if (!a) return b if (!b) return a return Y.mergeUpdates([a, b]) } export function getActiveBoardCount() { return boards.size } setInterval(() => { console.log("Statistics: active boards " + getActiveBoardCount() + ", sessions " + getSessionCount()) }, 60000) ================================================ FILE: backend/src/board-store.ts ================================================ import { PoolClient } from "pg" import QueryStream from "pg-query-stream" import * as uuid from "uuid" import { boardReducer } from "../../common/src/board-reducer" import { Board, BoardAccessPolicy, BoardHistoryEntry, Id, isBoardEmpty, Serial } from "../../common/src/domain" import { migrateBoard, migrateEvent, mkBootStrapEvent } from "../../common/src/migration" import { inTransaction, withDBClient } from "./db" import { assertNotNull } from "../../common/src/assertNotNull" export type BoardAndAccessTokens = { board: Board accessTokens: string[] } export type BoardInfo = { id: Id name: string ws_host: string | null } export async function getBoardInfo(id: Id): Promise { const result = await withDBClient((client) => client.query("SELECT id, name, ws_host FROM board WHERE id=$1", [id])) return result.rows.length === 1 ? (result.rows[0] as BoardInfo) : null } const selectBoardQuery = ` select id, jsonb_set (content - 'accessPolicy', '{accessPolicy}', cast(json_build_object( 'allowList', ( coalesce(( select jsonb_agg(jsonb_strip_nulls(jsonb_build_object('domain', domain, 'access', access, 'email', email))) from board_access where board_access.board_id = board.id ), '[]') ), 'publicRead', public_read, 'publicWrite', public_write ) as jsonb)) as content from board where id=$1 ` export async function fetchBoard(id: Id): Promise { return await inTransaction(async (client) => { const started = new Date().getTime() const result = await client.query(selectBoardQuery, [id]) if (result.rows.length == 0) { return null } else { const snapshot = result.rows[0].content as Board if ( snapshot.accessPolicy && snapshot.accessPolicy.allowList.length === 0 && snapshot.accessPolicy.publicRead && snapshot.accessPolicy.publicWrite ) { // Effectively no access policy delete snapshot.accessPolicy } let historyEventCount = 0 let lastSerial = 0 let board = snapshot let i = 0 let rebuildingSnapshot = false function updateBoardWithEventChunk(chunk: BoardHistoryEntry[]) { if (chunk.length === 0) { return // CRDT-only bundle } board = chunk.reduce((b, e) => { i++ if (e.action === "board.setAccessPolicy") { // Don't process access policy event when fetching board // Access policy may have been changed in the database after the event // And the board table status is considered the master return { ...b, serial: assertNotNull(e.serial) } } return boardReducer(b, e, { inplace: true, strictOnSerials: !rebuildingSnapshot })[0] }, board) historyEventCount += chunk.length lastSerial = chunk[chunk.length - 1].serial ?? snapshot.serial } await getBoardHistory(id, snapshot.serial, updateBoardWithEventChunk).catch(async (error) => { console.error(error.message) console.error( `Error applying board history for snapshot update for board ${id}. Loop index ${i}. Rebooting snapshot. This may be a lossy operation.`, ) i = 0 board = { ...snapshot, items: {}, connections: [] } rebuildingSnapshot = true try { await getFullBoardHistory(id, client, updateBoardWithEventChunk) } catch (e) { console.error(`Unable to reboot snapshot, failing at loop index ${i}. Giving up.`) // TODO: this board cannot be repaired automatically. We should block usage, or it will be // and endless loop. Local dev, board ee803db1-f41a-43c6-9e39-83057faace60. throw e } }) const serial = (historyEventCount > 0 ? lastSerial : snapshot.serial) || 0 const elapsed = new Date().getTime() - started console.log( `Loaded board ${id} at serial ${serial} from snapshot at serial ${snapshot.serial} and ${historyEventCount} events after snapshot. Took ${elapsed}ms`, ) if (historyEventCount > 1000 || rebuildingSnapshot /* time to create a new snapshot*/) { console.log( `Saving snapshot for board ${id} at serial ${serial}/${snapshot.serial} with ${historyEventCount} new events`, ) await saveBoardSnapshot(mkSnapshot(board, serial), client) } const accessTokens = ( await client.query("SELECT token FROM board_api_token WHERE board_id=$1", [id]) ).rows.map((row) => row.token) return { board: { ...board, serial }, accessTokens } } }) } export async function createBoard(board: Board): Promise { await inTransaction(async (client) => { const result = await client.query("SELECT id FROM board WHERE id=$1", [board.id]) if (result.rows.length > 0) throw Error("Board already exists: " + board.id) await client.query(`INSERT INTO board(id, name, content) VALUES ($1, $2, $3)`, [ board.id, board.name, mkSnapshot(board, 0), ]) await updateAccessPolicy(board.id, board.accessPolicy, client) if (!isBoardEmpty(board)) { console.log(`Creating non-empty board ${board.id} -> bootstrapping history`) const event = mkBootStrapEvent(board.id, board) storeEventHistoryBundle(board.id, [event], event.serial!, null, client) } }) } async function updateAccessPolicy(boardId: string, accessPolicy: BoardAccessPolicy, client: PoolClient): Promise { const publicRead = accessPolicy ? !!accessPolicy.publicRead : true const publicWrite = accessPolicy ? !!accessPolicy.publicWrite : true await client.query(`UPDATE board SET public_read=$1, public_write=$2 WHERE id=$3`, [ publicRead, publicWrite, boardId, ]) await client.query(`DELETE FROM board_access WHERE board_id=$1`, [boardId]) if (accessPolicy) { for (const entry of accessPolicy.allowList) { const domain = "domain" in entry ? entry.domain : null const email = "email" in entry ? entry.email : null await client.query(`INSERT INTO BOARD_access (board_id, domain, email, access) VALUES ($1, $2, $3, $4)`, [ boardId, domain, email, entry.access, ]) } } } // Updates the name column, which is there for admin convenience (not used by application logic) export async function renameBoardConvenienceColumnOnly(boardId: Id, name: string) { await inTransaction(async (client) => { const result = await client.query("SELECT content FROM board WHERE id=$1", [boardId]) if (result.rows.length !== 1) throw Error("Board not found: " + boardId) await client.query("UPDATE board SET name=$1 WHERE id=$2", [name, boardId]) }) } export async function updateBoardAccessPolicy(boardId: Id, accessPolicy: BoardAccessPolicy) { await inTransaction(async (client) => { const result = await client.query("SELECT content FROM board WHERE id=$1", [boardId]) if (result.rows.length !== 1) throw Error("Board not found: " + boardId) await updateAccessPolicy(boardId, accessPolicy, client) }) } export async function createAccessToken(board: Board): Promise { const token = uuid.v4() await inTransaction(async (client) => client.query("INSERT INTO board_api_token (board_id, token) VALUES ($1, $2)", [board.id, token]), ) return token } type StreamingBoardEventCallback = (chunk: BoardHistoryEntry[]) => void // Due to memory concerns we fetch board histories from DB as chunks, // which are currently implemented as sort of a poor-man's observable function streamingBoardEventsQuery(text: string, values: any[], client: PoolClient, cb: StreamingBoardEventCallback) { return new Promise((resolve, reject) => { const query = new QueryStream(text, values) const stream = client.query(query) stream.on("error", reject) stream.on("end", resolve) stream.on("data", (row) => { try { const chunk = row.events?.events as BoardHistoryEntry[] | undefined if (!chunk) { throw Error(`Unexpected DB row value ${chunk}`) } cb(chunk.map(migrateEvent)) } catch (error) { console.error(error) stream.destroy() reject(error) } }) }) } export function getFullBoardHistory(id: Id, client: PoolClient, cb: StreamingBoardEventCallback) { return streamingBoardEventsQuery( ` SELECT events FROM board_event WHERE board_id=$1 ORDER BY last_serial, first_serial `, [id], client, cb, ) } export async function getBoardHistory(id: Id, afterSerial: Serial, cb: StreamingBoardEventCallback): Promise { await withDBClient(async (client) => { let firstSerial = -1 let lastSerial = -1 let firstValidSerial = -1 await streamingBoardEventsQuery( ` SELECT events FROM board_event WHERE board_id=$1 AND last_serial >= $2 ORDER BY last_serial, first_serial `, [id, afterSerial], client, (chunk) => { if (chunk.length === 0) { return // CRDT-only bundle } if (firstSerial === -1 && typeof chunk[0]?.serial === "number") { firstSerial = chunk[0]?.serial } lastSerial = chunk[chunk.length - 1].serial ?? -1 const validEventsAfter = chunk.filter((r) => r.serial! > afterSerial) if (validEventsAfter.length === 0) { // Got chunk where no events have serial greater than the snapshot point -- discard it return } if (firstValidSerial === -1 && typeof validEventsAfter[0].serial === "number") { firstValidSerial = validEventsAfter[0].serial } cb(validEventsAfter) return }, ) // Client is up to date, ok if (lastSerial === afterSerial) { return } // Found continuous history, ok if (firstValidSerial === afterSerial + 1) { return } if (firstValidSerial === -1) { if (afterSerial === 0) { // Requesting from start, zero events found, is ok return } // Client claims to be in the future, not ok throw Error( `Cannot find history to start after the requested serial ${afterSerial} for board ${id}. Seems like the requested serial is higher than currently stored in DB`, ) } // Found noncontinuous event timeline, not ok throw Error( `Cannot find history to start after the requested serial ${afterSerial} for board ${id}. Found history for ${firstValidSerial}..${lastSerial}`, ) }) } export function verifyContinuity(boardId: Id, init: Serial, ...histories: BoardHistoryEntry[][]) { for (let history of histories) { if (history.length > 0) { if (!verifyTwoPoints(boardId, init, history[0].serial!)) { return false } init = history[history.length - 1].serial! } } return true } export function verifyEventArrayContinuity(boardId: Id, init: Serial, events: BoardHistoryEntry[]) { for (let event of events) { if (!verifyTwoPoints(boardId, init, event.serial!)) { return false } init = event.serial! } return true } function verifyTwoPoints(boardId: Id, a: Serial, b: Serial) { if (b !== a + 1) { console.error(`History discontinuity: ${a} -> ${b} for board ${boardId}`) return false } return true } function mkSnapshot(board: Board, serial: Serial) { const { accessPolicy, ...result } = migrateBoard({ ...board, serial }) return result } export async function saveBoardSnapshot(board: Board, client: PoolClient) { console.log(`Save board snapshot ${board.id} at serial ${board.serial}`) client.query( ` UPDATE board SET name=$2, content=$3 WHERE id=$1`, [board.id, board.name, board], ) } export async function storeEventHistoryBundle( boardId: Id, events: BoardHistoryEntry[], lastSerial: Serial, // Needed in case events is empty crdtUpdate: Uint8Array | null, client: PoolClient, savedAt = new Date(), ) { if (events.length === 0 && crdtUpdate === null) { throw Error("Trying to store a bundle without events or crdtUpdate") } if (events[0]?.firstSerial !== undefined) { throw Error("Assertion failed: folded events not expected on the server side.") } const firstSerial = events.length > 0 ? assertNotNull(events[0].serial) : null if (events.length > 0 && events[events.length - 1].serial != lastSerial) { throw Error("Serial mismatch between lastSerial and lastEvent.serial") } await client.query( ` INSERT INTO board_event(board_id, first_serial, last_serial, events, crdt_update, saved_at) VALUES ($1, $2, $3, $4, $5, $6) `, [boardId, firstSerial, lastSerial, { events }, crdtUpdate, savedAt], ) } export async function storeCRDTOnlyEventHistoryBundle( boardId: Id, boardSerial: Serial, crdtUpdate: Uint8Array | null, client: PoolClient, savedAt = new Date(), ) { await client.query( ` INSERT INTO board_event(board_id, first_serial, last_serial, events, crdt_update, saved_at) VALUES ($1, $2, $3, $4, $5, $6) `, [boardId, null, boardSerial, null, crdtUpdate, savedAt], ) } export type BoardHistoryBundle = { board_id: Id last_serial: Serial events: { events: BoardHistoryEntry[] } crdt_update: Uint8Array | null } export async function getBoardHistoryBundlesWithLastSerialsBetween( client: PoolClient, id: Id, lsMin: Serial, lsMax: Serial, ): Promise { return ( await client.query( ` SELECT board_id, last_serial, events, crdt_update FROM board_event WHERE board_id=$1 AND last_serial >= $2 AND last_serial <= $3 ORDER BY last_serial, first_serial `, [id, lsMin, lsMax], ) ).rows.map(migrateBundle) } export async function getBoardHistoryCrdtUpdates(client: PoolClient, id: Id): Promise { return ( await client.query( ` SELECT crdt_update FROM board_event WHERE board_id=$1 AND crdt_update IS NOT NULL ORDER BY last_serial, first_serial `, [id], ) ).rows.map((row) => row.crdt_update) } function migrateBundle(b: BoardHistoryBundle): BoardHistoryBundle { return { ...b, events: { ...b.events, events: b.events.events.map(migrateEvent) } } } export type BoardHistoryBundleMeta = { board_id: Id first_serial: Serial | null last_serial: Serial saved_at: Date } export async function getBoardHistoryBundleMetas(client: PoolClient, id: Id): Promise { return ( await client.query( ` SELECT board_id, last_serial, first_serial, saved_at FROM board_event WHERE board_id=$1 ORDER BY last_serial, first_serial `, [id], ) ).rows } export function verifyContinuityFromMetas(boardId: Id, init: Serial, bundles: BoardHistoryBundleMeta[]) { for (let bundle of bundles) { if (!bundle.first_serial) { // CRDT only bundle if (bundle.last_serial !== init) { console.error( `History discontinuity: ${init} -> ${bundle.last_serial} for CRDT-only bundle for board ${boardId}`, ) return false } } else { if (!verifyTwoPoints(boardId, init, bundle.first_serial)) { return false } init = bundle.last_serial } } return true } export async function findAllBoards(client: PoolClient): Promise { const result = await client.query("SELECT id FROM board") return result.rows.map((row) => row.id) } ================================================ FILE: backend/src/board-yjs-server.ts ================================================ import expressWs from "express-ws" import * as Y from "yjs" import { updateBoardCrdt } from "./board-state" import { getBoardHistoryCrdtUpdates } from "./board-store" import { withDBClient } from "./db" import { getSessionIdFromCookies } from "./http-session" import { getSessionById } from "./websocket-sessions" import YWebSocketServer from "./y-websocket-server/YWebSocketServer" import * as WebSocket from "ws" import { canRead, canWrite } from "../../common/src/domain" const socketsBySessionId: Record = {} export function closeYjsSocketsBySessionId(sessionId: string) { const sockets = socketsBySessionId[sessionId] if (sockets) { for (const socket of sockets) { socket.close() } delete socketsBySessionId[sessionId] console.log( `CLOSED ${sockets.length} y.js sockets by session id ${sessionId} - remaining sockets exist for ${ Object.keys(socketsBySessionId).length } other sessions`, ) } } export const yWebSocketServer = new YWebSocketServer({ persistence: { bindState: async (docName, ydoc) => { const boardId = docName const updates = await withDBClient(async (client) => getBoardHistoryCrdtUpdates(client, boardId)) if (updates.length === 0) { const initUpdate = Y.encodeStateAsUpdate(ydoc) console.log(`Storing initial CRDT state to DB for board ${boardId}`) updateBoardCrdt(boardId, initUpdate) } else { console.log(`Loaded ${updates.length} CRDT updates from DB for board ${boardId}`) for (const update of updates) { Y.applyUpdate(ydoc, update) } } ydoc.on("update", (update: Uint8Array, origin: any, doc: Y.Doc) => { updateBoardCrdt(boardId, update) }) }, writeState: async (docName, ydoc) => { // TODO: needed? }, }, }) export function BoardYJSServer(ws: expressWs.Instance, path: string) { ws.app.ws(path, async (socket, req) => { const boardId = req.params.boardId const sessionId = getSessionIdFromCookies(req) const session = sessionId ? getSessionById(sessionId) : undefined if ( !sessionId || !session || !session.boardSession || session.boardSession.boardId !== boardId || !canRead(session.boardSession.accessLevel) ) { // TODO: implement read-only YJS connections //console.warn("No session for YJS connection for board", boardId) socket.close() return } if (!socketsBySessionId[sessionId]) { socketsBySessionId[sessionId] = [] } socketsBySessionId[sessionId].push(socket) console.log( `OPENED y.js connection for session ${sessionId}. Now sockets exist for ${ Object.keys(socketsBySessionId).length } sessions`, ) socket.addEventListener("close", () => { if (socketsBySessionId[sessionId]) { socketsBySessionId[sessionId] = socketsBySessionId[sessionId].filter((s) => s !== socket) if (socketsBySessionId[sessionId].length === 0) { delete socketsBySessionId[sessionId] } console.log( `CLOSED y.js connection. Now sockets exist for ${Object.keys(socketsBySessionId).length} sessions`, ) } }) const readOnly = !canWrite(session.boardSession.accessLevel) const docName = boardId try { await yWebSocketServer.setupWSConnection(socket, docName, readOnly) } catch (e) { console.error("Error setting up YJS connection", e) socket.close() } }) } ================================================ FILE: backend/src/common-event-handler.ts ================================================ import * as Y from "yjs" import { AppEvent, Board, CrdtEnabled, checkBoardAccess, defaultBoardSize } from "../../common/src/domain" import { addBoard } from "./board-state" import { fetchBoard } from "./board-store" import { yWebSocketServer } from "./board-yjs-server" import { MessageHandlerResult } from "./connection-handler" import { getAuthenticatedUserFromJWT } from "./http-session" import { associateUserWithBoard, dissociateUserWithBoard, getUserAssociatedBoards, getUserIdForEmail, } from "./user-store" import { getSession, logoutUser, setNicknameForSession, setVerifiedUserForSession } from "./websocket-sessions" import { WsWrapper, toBuffer } from "./ws-wrapper" export async function handleCommonEvent(socket: WsWrapper, appEvent: AppEvent): Promise { switch (appEvent.action) { case "auth.login.jwt": { const user = getAuthenticatedUserFromJWT(appEvent.jwt) const session = getSession(socket) if (session && user !== null) { const userId = await getUserIdForEmail(user.email) const userInfo = await setVerifiedUserForSession(user, session) console.log(`${user.name} logged in`) session.sendEvent({ action: "auth.login.response", success: true, userId }) if (session.boardSession) { await associateUserWithBoard(userId, session.boardSession.boardId) } session.sendEvent({ action: "user.boards", email: user.email, boards: await getUserAssociatedBoards(userInfo), }) } else if (session) { session.sendEvent({ action: "auth.login.response", success: false }) } return true } case "auth.logout": { const session = getSession(socket) if (session && session.userInfo.userType === "authenticated") { logoutUser(appEvent, socket) console.log(`${session.userInfo.name} logged out`) } socket.close() return true } case "nickname.set": { setNicknameForSession(appEvent, socket) return true } case "board.associate": { // TODO: maybe access check? Not security-wise necessary const session = getSession(socket) if (session) { if (session.userInfo.userType !== "authenticated") { console.warn("Trying to associate board without authenticated user") return true } const userId = session.userInfo.userId await associateUserWithBoard(userId, appEvent.boardId, appEvent.lastOpened) } return true } case "board.dissociate": { const session = getSession(socket) if (session) { if (session.userInfo.userType !== "authenticated") { console.warn("Trying to dissociate board without authenticated user") return true } const userId = session.userInfo.userId await dissociateUserWithBoard(userId, appEvent.boardId) } return true } case "board.add": { const session = getSession(socket) if (session) { const { payload } = appEvent let template: Board | null = null if ("templateId" in payload && payload.templateId) { const aliased = process.env[`BOARD_ALIAS_${payload.templateId}`] const templateId = aliased || payload.templateId const found = await fetchBoard(templateId) if (found) { const accessLevel = checkBoardAccess(found.board.accessPolicy, session.userInfo) if (accessLevel === "none") { console.warn(`Trying to use board ${found.board.id} as template, without board permissions`) return true } template = { ...found.board, accessPolicy: undefined } } else { console.error(`Template ${payload.templateId}${aliased ? `(${templateId})` : ""} not found`) } } const board = { ...defaultBoardSize, items: {}, connections: [], ...template, ...payload, serial: 0 } if (template && template.crdt === CrdtEnabled) { const templateDoc = await yWebSocketServer.docs.getYDocAndWaitForFetch(template.id) const newDoc = await yWebSocketServer.docs.getYDocAndWaitForFetch(board.id) Y.applyUpdate(newDoc, Y.encodeStateAsUpdate(templateDoc)) } await addBoard(board) socket.send(toBuffer({ action: "board.add.ack", boardId: board.id })) } return true } case "ping": { return true } } return false } ================================================ FILE: backend/src/compact-history.ts ================================================ import { format } from "date-fns" import _ from "lodash" import { BoardHistoryEntry, Id } from "../../common/src/domain" import { BoardHistoryBundleMeta, getBoardHistoryBundleMetas, getBoardHistoryBundlesWithLastSerialsBetween, storeEventHistoryBundle, verifyContinuity, verifyContinuityFromMetas, verifyEventArrayContinuity, } from "./board-store" import * as Y from "yjs" import { inTransaction } from "./db" function chunkBy(arr: T[], shouldSplit: (a: T, b: T) => boolean) { const result = [] let currentChunk = [] for (let i = 0; i < arr.length; i++) { if (i > 0 && shouldSplit(arr[i - 1], arr[i])) { result.push(currentChunk) currentChunk = [] } currentChunk.push(arr[i]) } result.push(currentChunk) return result } function getHour(b: BoardHistoryBundleMeta) { return format(new Date(b.saved_at), "yyyy-MM-dd hh") } export async function quickCompactBoardHistory(id: Id): Promise { try { return await inTransaction(async (client) => { // Lock the board to prevent loading the board while compacting await client.query("select 1 from board where id=$1 for update", [id]) const bundleMetas = await getBoardHistoryBundleMetas(client, id) if (bundleMetas.length === 0) return 0 const consistent = verifyContinuityFromMetas(id, 0, bundleMetas) if (consistent) { // Group in one-hour bundles //console.log("Grouped by date", groupedByHour) const toCompact = chunkBy( bundleMetas, (a, b) => getHour(a) !== getHour(b) && a.last_serial !== b.last_serial, ).filter((chunk) => chunk.length > 1) let compactions = 0 for (let bs of toCompact) { const firstBundle = bs[0] const lastBundle = bs[bs.length - 1] console.log( `Compacting ${bs.length} bundles into one for board ${id}, containing serials ${firstBundle.first_serial}...${lastBundle.last_serial}`, ) const lastSerial = lastBundle.last_serial const bundlesWithData = await getBoardHistoryBundlesWithLastSerialsBetween( client, id, firstBundle.last_serial, lastSerial, ) const eventArrays = bundlesWithData.map((b) => b.events.events) const events: BoardHistoryEntry[] = eventArrays.flat() const crdtUpdates = bundlesWithData.flatMap((d) => (d.crdt_update ? [d.crdt_update] : [])) const combinedCrdtUpdate = crdtUpdates.length ? Y.mergeUpdates(crdtUpdates) : null const initSerial = firstBundle.first_serial ? firstBundle.first_serial - 1 : firstBundle.last_serial - 1 const consistent = verifyContinuity(id, initSerial, ...eventArrays) && verifyEventArrayContinuity(id, initSerial, events) if (consistent && bundlesWithData.length == bs.length) { // 1. delete existing bundles const deleteResult = await client.query( `DELETE FROM board_event where board_id=$1 and last_serial in (${bundlesWithData .map((b) => b.last_serial) .join(",")})`, [id], ) if (deleteResult.rowCount != bs.length) { throw Error( `Unexpected rowcount when deleting on compaction: ${deleteResult.rowCount} for board ${id}`, ) } // 2. store as a single bundle await storeEventHistoryBundle( id, events, lastSerial, combinedCrdtUpdate, client, lastBundle.saved_at, ) } else { throw Error("Discontinuity detected in compacted history.") } compactions++ } if (compactions > 0) { } else { console.log( `Board ${id}: Verified ${bundleMetas.length} bundles containing ${ bundleMetas[bundleMetas.length - 1].last_serial } events => no need to compact`, ) } return compactions } else { throw Error("Discontinuity detected in bundle metadata.") } }) } catch (e) { console.error(`Aborting compaction of board ${id} because of an error: ${e}`) return 0 } } ================================================ FILE: backend/src/config.ts ================================================ import path from "path" import fs from "fs" import { authProvider } from "./oauth" import * as t from "io-ts" import { optional } from "../../common/src/domain" import { decodeOrThrow } from "./decodeOrThrow" export type StorageBackend = Readonly< { type: "LOCAL"; directory: string; assetStorageURL: string } | { type: "AWS"; assetStorageURL: string } > export type Config = Readonly<{ storageBackend: StorageBackend; authSupported: boolean; crdt: CrdtConfigString }> const CrdtConfigString = t.union([ t.literal("opt-in"), t.literal("opt-in-authenticated"), t.literal("true"), t.literal("false"), ]) export type CrdtConfigString = t.TypeOf export const getConfig = (): Config => { const storageBackend: StorageBackend = process.env.AWS_ASSETS_BUCKET_URL ? { type: "AWS", assetStorageURL: process.env.AWS_ASSETS_BUCKET_URL } : { type: "LOCAL", directory: path.resolve("localfiles"), assetStorageURL: "/assets" } if (storageBackend.type === "LOCAL") { try { fs.mkdirSync(storageBackend.directory) } catch (e) {} } const crdt = decodeOrThrow(CrdtConfigString, process.env.COLLABORATIVE_EDITING ?? "opt-in") return { storageBackend, authSupported: authProvider !== null, crdt, } } ================================================ FILE: backend/src/connection-handler.ts ================================================ import { AppEvent, Id, Serial, EventWrapper } from "../../common/src/domain" import { getActiveBoards } from "./board-state" import { getConfig } from "./config" import { releaseLocksFor } from "./locker" import { broadcastCursorPositions, endSession, startSession } from "./websocket-sessions" import { WsWrapper, toBuffer } from "./ws-wrapper" export type ConnectionHandlerParams = Readonly<{ getSignedPutUrl: (key: string) => string }> export const connectionHandler = (socket: WsWrapper, handleMessage: MessageHandler) => { startSession(socket) const config = getConfig() socket.send( toBuffer({ action: "server.config", assetStorageURL: config.storageBackend.assetStorageURL, authSupported: config.authSupported, crdt: config.crdt, }), ) socket.onError(() => { socket.close() }) socket.onMessage(async (o: object) => { try { let event = o as EventWrapper let serialsToAck: Record = {} for (const e of event.events) { const serialAck = await handleMessage(socket, e) if (serialAck === true) { } else if (serialAck === false) { console.warn("Unhandled app-event message", e) } else { serialsToAck[serialAck.boardId] = serialAck.serial } } if (event.ackId) { socket.send(toBuffer({ action: "ack", ackId: event.ackId, serials: serialsToAck })) } } catch (e) { console.error("Error while handling event from client. Closing connection.", e) socket.close() } }) socket.onClose(() => { endSession(socket) getActiveBoards().forEach((state) => { delete state.cursorPositions[socket.id] state.cursorsMoved = true }) releaseLocksFor(socket) }) } setInterval(() => { getActiveBoards().forEach((bh) => { if (bh.cursorsMoved) { broadcastCursorPositions(bh.board.id, bh.cursorPositions) bh.cursorsMoved = false } }) }, 100) export type MessageHandler = (socket: WsWrapper, appEvent: AppEvent) => Promise export type MessageHandlerResult = { boardId: Id; serial: Serial } | boolean ================================================ FILE: backend/src/db.ts ================================================ import pg, { PoolClient } from "pg" import process from "process" import migrate from "node-pg-migrate" const DATABASE_URL = process.env.DATABASE_URL ?? "postgres://r-board:secret@127.0.0.1:13338/r-board" const DATABASE_SSL_ENABLED = process.env.DATABASE_SSL_ENABLED === "true" const pgConfig = { connectionString: DATABASE_URL, ssl: DATABASE_SSL_ENABLED ? { rejectUnauthorized: false, } : undefined, } const connectionPool = new pg.Pool(pgConfig) export function closeConnectionPool() { connectionPool.end() } export async function initDB(backendDir: string = ".") { console.log("Running database migrations") await inTransaction((client) => migrate({ count: 100000, databaseUrl: DATABASE_URL, migrationsTable: "pgmigrations", dir: `${backendDir}/migrations`, direction: "up", dbClient: client, }), ) console.log("Completed database migrations") return { onEvent: async (eventNames: string[], cb: (n: pg.Notification) => any) => { const client = await connectionPool.connect() eventNames.map((e) => client.query(`LISTEN ${e}`)) client.on("notification", cb) }, } } export async function withDBClient(f: (client: PoolClient) => Promise): Promise { const client = await connectionPool.connect() try { await client.query("BEGIN;SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;") return await f(client) } finally { await client.query("ROLLBACK;") client.release() } } export async function inTransaction(f: (client: PoolClient) => Promise): Promise { const client = await connectionPool.connect() try { await client.query(` BEGIN; SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE; `) const result = await f(client) await client.query("COMMIT;") return result } catch (e) { await client.query("ROLLBACK;") throw e } finally { client.release() } } ================================================ FILE: backend/src/decodeOrThrow.ts ================================================ import * as t from "io-ts" import { Left, isLeft, left } from "fp-ts/lib/Either" import { PathReporter } from "io-ts/lib/PathReporter" export function decodeOrThrow(codec: t.Type, input: any): T { const validationResult = codec.decode(input) if (isLeft(validationResult)) { throw new ValidationError(validationResult) } return validationResult.right } class ValidationError extends Error { constructor(errors: Left) { super(report_(errors.left)) } } function report_(errors: t.Errors) { return PathReporter.report(left(errors)).join("\n") } ================================================ FILE: backend/src/env.ts ================================================ import process from "process" export function getEnv(name: string): string { const value = process.env[name] if (!value) throw new Error("Missing ENV: " + name) return value } ================================================ FILE: backend/src/expiring-map.ts ================================================ export function AutoExpiringMap(ttlSeconds: number) { const timers = new Map() const data: Record = {} const listeners: ((v: Record) => any)[] = [] const proxy = new Proxy(data, { set(target, key, value) { if (typeof key === "symbol") return false target[key] = value setExpiryTimer(key) listeners.forEach((l) => l(target)) return true }, deleteProperty(target, key) { if (typeof key === "symbol") return false const didDelete = delete target[key] if (!didDelete) { return false } listeners.forEach((l) => l(target)) return true }, }) const setExpiryTimer = (key: string | number) => { if (timers.has(key)) { clearTimeout(timers.get(key)!) } timers.set( key, setTimeout(() => { timers.delete(key) delete proxy[key] }, ttlSeconds * 1000), ) } const autoExpiringMap = { get: (key: string) => proxy[key], has: (key: string) => !!proxy[key], entries: () => Object.entries(proxy), delete: (key: string) => delete proxy[key], set: (key: string, value: any) => { proxy[key] = value }, onChange: (fn: (v: Record) => any) => { listeners.push(fn) return autoExpiringMap }, } return autoExpiringMap } ================================================ FILE: backend/src/express-server.ts ================================================ import dotenv from "dotenv" import express from "express" import expressWs from "express-ws" import fs from "fs" import * as Http from "http" import * as Https from "https" import * as path from "path" import apiRoutes from "./api/api-routes" import { handleBoardEvent } from "./board-event-handler" import { BoardYJSServer } from "./board-yjs-server" import { getConfig } from "./config" import { connectionHandler } from "./connection-handler" import { getEnv } from "./env" import { authProvider, setupAuth } from "./oauth" import { possiblyRequireAuth } from "./require-auth" import { createGetSignedPutUrl } from "./storage" import { WsWrapper } from "./ws-wrapper" import Cookies from "cookies" import { removeAuthenticatedUser, setAuthenticatedUser } from "./http-session" dotenv.config() export const startExpressServer = (httpPort?: number, httpsPort?: number): (() => void) => { const config = getConfig() const app = express() if (authProvider) { setupAuth(app, authProvider) } else { app.get("/logout", async (req, res) => { removeAuthenticatedUser(req, res) res.redirect("/") }) } app.get("/test-callback", async (req, res) => { const cookies = new Cookies(req, res) const returnTo = cookies.get("returnTo") || "/" setAuthenticatedUser(req, res, { domain: null, email: "ourboardtester@test.com", name: "Ourboard tester" }) res.redirect(returnTo) }) possiblyRequireAuth(app) app.use("/", express.static("../frontend/dist")) app.use("/", express.static("../frontend/public")) if (config.storageBackend.type === "LOCAL") { const localDirectory = config.storageBackend.directory app.put("/assets/:id", (req, res) => { if (!req.params.id) { return res.sendStatus(400) } const w = fs.createWriteStream(localDirectory + "/" + req.params.id) req.pipe(w) req.on("end", () => { !res.headersSent && res.sendStatus(200) }) w.on("error", () => { res.sendStatus(500) }) }) app.use("/assets", express.static(localDirectory)) } app.get("/assets/external", (req, res) => { const src = req.query.src if (typeof src !== "string" || ["http://", "https://"].every((prefix) => !src.startsWith(prefix))) return res.send(400) const protocol = src.startsWith("https://") ? Https : Http protocol .request(src, (upstreamResponse) => { res.writeHead(upstreamResponse.statusCode!, upstreamResponse.headers) upstreamResponse .pipe(res, { end: true, }) .on("error", (err) => res.status(500).send(err.message)) }) .end() }) app.get("/b/:boardId", async (req, res) => { res.sendFile(path.resolve("../frontend/dist/index.html")) }) app.use(apiRoutes.handler()) let stop = () => {} if (httpPort) { const http = new Http.Server(app) startWs(http, app) http.listen(httpPort, () => { console.log("Listening HTTP on port " + httpPort) }) const prevStop = stop stop = () => { prevStop() http.close() } } if (httpsPort) { let https = new Https.Server( { cert: fs.readFileSync(getEnv("HTTPS_CERT_FILE")), key: fs.readFileSync(getEnv("HTTPS_KEY_FILE")), }, app, ) startWs(https, app) https.listen(httpsPort, () => { console.log("Listening HTTPS on port " + httpsPort) }) const prevStop = stop stop = () => { prevStop() https.close() } } const redirectURL = process.env.REDIRECT_URL if (redirectURL) { app.get("*", function (req, res, next) { if (req.headers["x-forwarded-proto"] !== "https") { res.redirect(redirectURL) } else { next() } }) } return stop } function startWs(http: any, app: express.Express) { const ws: expressWs.Instance = expressWs(app, http) const signedPutUrl = createGetSignedPutUrl(getConfig().storageBackend) ws.app.ws("/socket/lobby", (socket, req) => { connectionHandler(WsWrapper(socket), handleBoardEvent(null, signedPutUrl)) }) ws.app.ws("/socket/board/:boardId", (socket, req) => { const boardId = req.params.boardId connectionHandler(WsWrapper(socket), handleBoardEvent(boardId, signedPutUrl)) }) BoardYJSServer(ws, "/socket/yjs/board/:boardId/") ws.app.ws("*", (socket, req) => { console.warn(`Unexpected WS connection: ${req.url} `) socket.close() }) } ================================================ FILE: backend/src/generic-oidc-auth.ts ================================================ import { Request, Response } from "express" import * as t from "io-ts" import JWT from "jsonwebtoken" import { OAuthAuthenticatedUser } from "../../common/src/authenticated-user" import { optional } from "../../common/src/domain" import { decodeOrThrow } from "./decodeOrThrow" import { getEnv } from "./env" import { ROOT_URL } from "./host-config" import { AuthProvider } from "./oauth" import { REQUIRE_AUTH } from "./require-auth" type GenericOAuthConfig = { OIDC_CONFIG_URL: string OIDC_CLIENT_ID: string OIDC_CLIENT_SECRET: string OIDC_LOGOUT?: string } export const genericOIDCConfig: GenericOAuthConfig | null = process.env.OIDC_CONFIG_URL ? { OIDC_CONFIG_URL: getEnv("OIDC_CONFIG_URL"), OIDC_CLIENT_ID: getEnv("OIDC_CLIENT_ID"), OIDC_CLIENT_SECRET: getEnv("OIDC_CLIENT_SECRET"), OIDC_LOGOUT: process.env.OIDC_LOGOUT, } : null export function GenericOIDCAuthProvider(config: GenericOAuthConfig): AuthProvider { console.log(`Setting up generic OAuth authentication using client id ${config.OIDC_CLIENT_ID}`) const callbackUrl = `${ROOT_URL}/google-callback` const openIdConfiguration = (async () => { const response = await fetch(config.OIDC_CONFIG_URL) return decodeOrThrow(OpenIdConfiguration, await response.json()) })() async function getAccountFromCode(code: string): Promise { const response = await fetch((await openIdConfiguration).token_endpoint, { method: "POST", headers: { "content-type": "application/x-www-form-urlencoded", }, body: `grant_type=authorization_code&code=${encodeURIComponent(code)}&client_id=${encodeURIComponent( config.OIDC_CLIENT_ID, )}&client_secret=${config.OIDC_CLIENT_SECRET}&redirect_uri=${callbackUrl}`, }) const body = await response.json() const idToken = JWT.decode(body.id_token) //console.log(JSON.stringify(idToken, null, 2)) const user = decodeOrThrow(IdToken, idToken) return { email: user.email, name: "name" in user ? user.name : user.preferred_username, picture: user.picture ?? undefined, domain: user.hd ?? null, } } async function getAuthPageURL() { const scopes = "email openid profile" const state = "TODO" const redirectUri = callbackUrl return `${(await openIdConfiguration).authorization_endpoint}?scope=${encodeURIComponent( scopes, )}&response_type=code&state=${encodeURIComponent(state)}&redirect_uri=${encodeURIComponent( redirectUri, )}&client_id=${config.OIDC_CLIENT_ID}` } const shouldHandleLogout = REQUIRE_AUTH || config.OIDC_LOGOUT async function getLogoutUrl(): Promise { if (config.OIDC_LOGOUT && config.OIDC_LOGOUT !== "true") { return config.OIDC_LOGOUT } const logoutUrl = (await openIdConfiguration).end_session_endpoint if (!logoutUrl) { throw Error( `OIDC configuration at ${config.OIDC_CONFIG_URL} does not specify end_session_endpoint. Use OIDC_LOGOUT environment variable to define the logout endpoint explicitly.`, ) } return logoutUrl } const logout = shouldHandleLogout ? async (req: Request, res: Response) => { res.redirect(await getLogoutUrl()) } : undefined return { getAccountFromCode, getAuthPageURL, logout, } } const OpenIdConfiguration = t.type({ authorization_endpoint: t.string, token_endpoint: t.string, end_session_endpoint: optional(t.string), }) const IdToken = t.union([ t.type({ email: t.string, name: t.string, picture: optional(t.string), hd: optional(t.string), }), t.type({ email: t.string, preferred_username: t.string, picture: optional(t.string), hd: optional(t.string), }), ]) ================================================ FILE: backend/src/github-webhook/example-payload.json ================================================ { "action": "assigned", "issue": { "url": "https://api.github.com/repos/raimohanska/r-board/issues/129", "repository_url": "https://api.github.com/repos/raimohanska/r-board", "labels_url": "https://api.github.com/repos/raimohanska/r-board/issues/129/labels{/name}", "comments_url": "https://api.github.com/repos/raimohanska/r-board/issues/129/comments", "events_url": "https://api.github.com/repos/raimohanska/r-board/issues/129/events", "html_url": "https://github.com/raimohanska/r-board/issues/129", "id": 810436967, "node_id": "MDU6SXNzdWU4MTA0MzY5Njc=", "number": 129, "title": "Github issues integration", "user": { "login": "raimohanska", "id": 292964, "node_id": "MDQ6VXNlcjI5Mjk2NA==", "avatar_url": "https://avatars.githubusercontent.com/u/292964?v=4", "gravatar_id": "", "url": "https://api.github.com/users/raimohanska", "html_url": "https://github.com/raimohanska", "followers_url": "https://api.github.com/users/raimohanska/followers", "following_url": "https://api.github.com/users/raimohanska/following{/other_user}", "gists_url": "https://api.github.com/users/raimohanska/gists{/gist_id}", "starred_url": "https://api.github.com/users/raimohanska/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/raimohanska/subscriptions", "organizations_url": "https://api.github.com/users/raimohanska/orgs", "repos_url": "https://api.github.com/users/raimohanska/repos", "events_url": "https://api.github.com/users/raimohanska/events{/privacy}", "received_events_url": "https://api.github.com/users/raimohanska/received_events", "type": "User", "site_admin": false }, "labels": [ { "id": 2438308116, "node_id": "MDU6TGFiZWwyNDM4MzA4MTE2", "url": "https://api.github.com/repos/raimohanska/r-board/labels/enhancement", "name": "bug", "color": "a2eeef", "default": true, "description": "Bug" } ], "state": "open", "locked": false, "assignee": { "login": "raimohanska", "id": 292964, "node_id": "MDQ6VXNlcjI5Mjk2NA==", "avatar_url": "https://avatars.githubusercontent.com/u/292964?v=4", "gravatar_id": "", "url": "https://api.github.com/users/raimohanska", "html_url": "https://github.com/raimohanska", "followers_url": "https://api.github.com/users/raimohanska/followers", "following_url": "https://api.github.com/users/raimohanska/following{/other_user}", "gists_url": "https://api.github.com/users/raimohanska/gists{/gist_id}", "starred_url": "https://api.github.com/users/raimohanska/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/raimohanska/subscriptions", "organizations_url": "https://api.github.com/users/raimohanska/orgs", "repos_url": "https://api.github.com/users/raimohanska/repos", "events_url": "https://api.github.com/users/raimohanska/events{/privacy}", "received_events_url": "https://api.github.com/users/raimohanska/received_events", "type": "User", "site_admin": false }, "assignees": [ { "login": "raimohanska", "id": 292964, "node_id": "MDQ6VXNlcjI5Mjk2NA==", "avatar_url": "https://avatars.githubusercontent.com/u/292964?v=4", "gravatar_id": "", "url": "https://api.github.com/users/raimohanska", "html_url": "https://github.com/raimohanska", "followers_url": "https://api.github.com/users/raimohanska/followers", "following_url": "https://api.github.com/users/raimohanska/following{/other_user}", "gists_url": "https://api.github.com/users/raimohanska/gists{/gist_id}", "starred_url": "https://api.github.com/users/raimohanska/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/raimohanska/subscriptions", "organizations_url": "https://api.github.com/users/raimohanska/orgs", "repos_url": "https://api.github.com/users/raimohanska/repos", "events_url": "https://api.github.com/users/raimohanska/events{/privacy}", "received_events_url": "https://api.github.com/users/raimohanska/received_events", "type": "User", "site_admin": false } ], "milestone": null, "comments": 0, "created_at": "2021-02-17T18:39:11Z", "updated_at": "2021-02-17T19:08:21Z", "closed_at": null, "author_association": "OWNER", "active_lock_reason": null, "body": "", "performed_via_github_app": null }, "assignee": { "login": "raimohanska", "id": 292964, "node_id": "MDQ6VXNlcjI5Mjk2NA==", "avatar_url": "https://avatars.githubusercontent.com/u/292964?v=4", "gravatar_id": "", "url": "https://api.github.com/users/raimohanska", "html_url": "https://github.com/raimohanska", "followers_url": "https://api.github.com/users/raimohanska/followers", "following_url": "https://api.github.com/users/raimohanska/following{/other_user}", "gists_url": "https://api.github.com/users/raimohanska/gists{/gist_id}", "starred_url": "https://api.github.com/users/raimohanska/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/raimohanska/subscriptions", "organizations_url": "https://api.github.com/users/raimohanska/orgs", "repos_url": "https://api.github.com/users/raimohanska/repos", "events_url": "https://api.github.com/users/raimohanska/events{/privacy}", "received_events_url": "https://api.github.com/users/raimohanska/received_events", "type": "User", "site_admin": false }, "repository": { "id": 305431036, "node_id": "MDEwOlJlcG9zaXRvcnkzMDU0MzEwMzY=", "name": "r-board", "full_name": "raimohanska/r-board", "private": false, "owner": { "login": "raimohanska", "id": 292964, "node_id": "MDQ6VXNlcjI5Mjk2NA==", "avatar_url": "https://avatars.githubusercontent.com/u/292964?v=4", "gravatar_id": "", "url": "https://api.github.com/users/raimohanska", "html_url": "https://github.com/raimohanska", "followers_url": "https://api.github.com/users/raimohanska/followers", "following_url": "https://api.github.com/users/raimohanska/following{/other_user}", "gists_url": "https://api.github.com/users/raimohanska/gists{/gist_id}", "starred_url": "https://api.github.com/users/raimohanska/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/raimohanska/subscriptions", "organizations_url": "https://api.github.com/users/raimohanska/orgs", "repos_url": "https://api.github.com/users/raimohanska/repos", "events_url": "https://api.github.com/users/raimohanska/events{/privacy}", "received_events_url": "https://api.github.com/users/raimohanska/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/raimohanska/r-board", "description": "An online whiteboard", "fork": false, "url": "https://api.github.com/repos/raimohanska/r-board", "forks_url": "https://api.github.com/repos/raimohanska/r-board/forks", "keys_url": "https://api.github.com/repos/raimohanska/r-board/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/raimohanska/r-board/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/raimohanska/r-board/teams", "hooks_url": "https://api.github.com/repos/raimohanska/r-board/hooks", "issue_events_url": "https://api.github.com/repos/raimohanska/r-board/issues/events{/number}", "events_url": "https://api.github.com/repos/raimohanska/r-board/events", "assignees_url": "https://api.github.com/repos/raimohanska/r-board/assignees{/user}", "branches_url": "https://api.github.com/repos/raimohanska/r-board/branches{/branch}", "tags_url": "https://api.github.com/repos/raimohanska/r-board/tags", "blobs_url": "https://api.github.com/repos/raimohanska/r-board/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/raimohanska/r-board/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/raimohanska/r-board/git/refs{/sha}", "trees_url": "https://api.github.com/repos/raimohanska/r-board/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/raimohanska/r-board/statuses/{sha}", "languages_url": "https://api.github.com/repos/raimohanska/r-board/languages", "stargazers_url": "https://api.github.com/repos/raimohanska/r-board/stargazers", "contributors_url": "https://api.github.com/repos/raimohanska/r-board/contributors", "subscribers_url": "https://api.github.com/repos/raimohanska/r-board/subscribers", "subscription_url": "https://api.github.com/repos/raimohanska/r-board/subscription", "commits_url": "https://api.github.com/repos/raimohanska/r-board/commits{/sha}", "git_commits_url": "https://api.github.com/repos/raimohanska/r-board/git/commits{/sha}", "comments_url": "https://api.github.com/repos/raimohanska/r-board/comments{/number}", "issue_comment_url": "https://api.github.com/repos/raimohanska/r-board/issues/comments{/number}", "contents_url": "https://api.github.com/repos/raimohanska/r-board/contents/{+path}", "compare_url": "https://api.github.com/repos/raimohanska/r-board/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/raimohanska/r-board/merges", "archive_url": "https://api.github.com/repos/raimohanska/r-board/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/raimohanska/r-board/downloads", "issues_url": "https://api.github.com/repos/raimohanska/r-board/issues{/number}", "pulls_url": "https://api.github.com/repos/raimohanska/r-board/pulls{/number}", "milestones_url": "https://api.github.com/repos/raimohanska/r-board/milestones{/number}", "notifications_url": "https://api.github.com/repos/raimohanska/r-board/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/raimohanska/r-board/labels{/name}", "releases_url": "https://api.github.com/repos/raimohanska/r-board/releases{/id}", "deployments_url": "https://api.github.com/repos/raimohanska/r-board/deployments", "created_at": "2020-10-19T15:33:11Z", "updated_at": "2021-02-17T18:39:38Z", "pushed_at": "2021-02-17T18:31:01Z", "git_url": "git://github.com/raimohanska/r-board.git", "ssh_url": "git@github.com:raimohanska/r-board.git", "clone_url": "https://github.com/raimohanska/r-board.git", "svn_url": "https://github.com/raimohanska/r-board", "homepage": null, "size": 1137, "stargazers_count": 4, "watchers_count": 4, "language": "TypeScript", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "forks_count": 2, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 33, "license": { "key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA=" }, "forks": 2, "open_issues": 33, "watchers": 4, "default_branch": "master" }, "sender": { "login": "raimohanska", "id": 292964, "node_id": "MDQ6VXNlcjI5Mjk2NA==", "avatar_url": "https://avatars.githubusercontent.com/u/292964?v=4", "gravatar_id": "", "url": "https://api.github.com/users/raimohanska", "html_url": "https://github.com/raimohanska", "followers_url": "https://api.github.com/users/raimohanska/followers", "following_url": "https://api.github.com/users/raimohanska/following{/other_user}", "gists_url": "https://api.github.com/users/raimohanska/gists{/gist_id}", "starred_url": "https://api.github.com/users/raimohanska/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/raimohanska/subscriptions", "organizations_url": "https://api.github.com/users/raimohanska/orgs", "repos_url": "https://api.github.com/users/raimohanska/repos", "events_url": "https://api.github.com/users/raimohanska/events{/privacy}", "received_events_url": "https://api.github.com/users/raimohanska/received_events", "type": "User", "site_admin": false } } ================================================ FILE: backend/src/google-auth.ts ================================================ import { google } from "googleapis" import { OAuthAuthenticatedUser } from "../../common/src/authenticated-user" import { assertNotNull } from "../../common/src/assertNotNull" import { getEnv } from "./env" import { AuthProvider } from "./oauth" import { ROOT_URL } from "./host-config" import { decodeOrThrow } from "./decodeOrThrow" import * as T from "io-ts" import { optional } from "../../common/src/domain" import JWT from "jsonwebtoken" type GoogleConfig = { clientID: string clientSecret: string callbackURL: string } export const googleConfig: GoogleConfig | null = process.env.GOOGLE_OAUTH_CLIENT_ID ? { clientID: getEnv("GOOGLE_OAUTH_CLIENT_ID"), clientSecret: getEnv("GOOGLE_OAUTH_CLIENT_SECRET"), callbackURL: `${ROOT_URL}/google-callback`, } : null export const GoogleAuthProvider = (googleConfig: GoogleConfig): AuthProvider => { console.log(`Setting up Google authentication using client ID ${googleConfig.clientID}`) const googleScopes = ["email", "https://www.googleapis.com/auth/userinfo.profile"] function googleOAUTH2() { if (!googleConfig.clientID || !googleConfig.clientSecret) throw new Error("Missing environment variables for Google OAuth") return new google.auth.OAuth2(googleConfig.clientID, googleConfig.clientSecret, googleConfig.callbackURL) } async function getAuthPageURL() { return googleOAUTH2().generateAuthUrl({ scope: googleScopes, prompt: "select_account", }) } const IdToken = T.strict({ hd: optional(T.string), email: T.string, email_verified: T.boolean, name: T.string, picture: optional(T.string), }) async function getAccountFromCode(code: string): Promise { const auth = googleOAUTH2() const data = await auth.getToken(code) const idToken = decodeOrThrow(IdToken, JWT.decode(assertNotNull(data.tokens.id_token))) const email = idToken.email return { name: idToken.name, email, picture: idToken.picture ?? undefined, domain: idToken.hd ?? null, } } return { getAuthPageURL, getAccountFromCode, } } ================================================ FILE: backend/src/host-config.ts ================================================ export const ROOT_URL = process.env.ROOT_URL ?? "http://localhost:1337" export const ROOT_HOST = new URL(ROOT_URL).host export const ROOT_PROTOCOL = new URL(ROOT_URL).protocol export const WS_HOST_LOCAL = (process.env.WS_HOST_LOCAL ?? ROOT_HOST).split(",") export const WS_HOST_DEFAULT = process.env.WS_HOST_DEFAULT ?? ROOT_HOST export const WS_PROTOCOL = process.env.WS_PROTOCOL ?? (ROOT_PROTOCOL.startsWith("https") ? "wss" : "ws") ================================================ FILE: backend/src/http-session.ts ================================================ import Cookies from "cookies" import { IncomingMessage, ServerResponse } from "http" import JWT from "jsonwebtoken" import { getEnv } from "./env" import { OAuthAuthenticatedUser } from "../../common/src/authenticated-user" import { ISOTimeStamp, newISOTimeStamp } from "../../common/src/domain" const secret = getEnv("SESSION_SIGNING_SECRET") export type LoginInfo = OAuthAuthenticatedUser & { timestamp: ISOTimeStamp | undefined } export function getSessionIdFromCookies(req: IncomingMessage): string | null { return new Cookies(req, null as any).get("sessionId") ?? null } // Get / set authenticated user stored in cookies export function getAuthenticatedUser(req: IncomingMessage): LoginInfo | null { const userCookie = new Cookies(req, null as any).get("user") if (userCookie) { return getAuthenticatedUserFromJWT(userCookie) } return null } export function getAuthenticatedUserFromJWT(jwt: string): LoginInfo | null { try { JWT.verify(jwt, secret) const loginInfo = JWT.decode(jwt) as LoginInfo if (loginInfo.domain === undefined) { console.log("Rejecting legacy token without domain") return null } return loginInfo } catch (e) { console.warn("Token verification failed", jwt, e) } return null } export function setAuthenticatedUser(req: IncomingMessage, res: ServerResponse, userInfo: OAuthAuthenticatedUser) { const loginInfo: LoginInfo = { ...userInfo, timestamp: newISOTimeStamp() } const jwt = JWT.sign(loginInfo, secret) new Cookies(req, res).set("user", jwt, { maxAge: 365 * 24 * 3600 * 1000, httpOnly: false, }) // Max 365 days expiration } export function removeAuthenticatedUser(req: IncomingMessage, res: ServerResponse) { new Cookies(req, res).set("user", "", { maxAge: 0, httpOnly: true }) } ================================================ FILE: backend/src/locker.ts ================================================ import { Id, BoardItemEvent, isPersistableBoardItemEvent, getItemIds, ItemLocks } from "../../common/src/domain" import { getActiveBoards, ServerSideBoardState } from "./board-state" import { AutoExpiringMap } from "./expiring-map" import { WsWrapper } from "./ws-wrapper" const LOCK_TTL_SECONDS = 10 export function Locks(onChange: (locks: ItemLocks) => any) { const locks = AutoExpiringMap(LOCK_TTL_SECONDS).onChange(onChange) return { lockItem: (itemId: Id, sessionId: Id) => { if (locks.has(itemId) && locks.get(itemId) !== sessionId) { return false } locks.set(itemId, sessionId) return true }, unlockItem: (itemId: Id, sessionId: Id) => { if (locks.get(itemId) === sessionId) { return locks.delete(itemId) } return false }, delete: (itemId: string) => locks.delete(itemId), entries: () => locks.entries(), } } export function obtainLock(locks: ServerSideBoardState["locks"], e: BoardItemEvent, socket: WsWrapper) { if (isPersistableBoardItemEvent(e)) { const itemIds = getItemIds(e) // Since we are operating on multiple items at a time, locking must succeed for all of them // for the action to succeed return itemIds.every((id) => locks.lockItem(id, socket.id)) } else { const { itemId, action } = e switch (action) { case "item.lock": return locks.lockItem(itemId, socket.id) case "item.unlock": return locks.unlockItem(itemId, socket.id) } } } export function releaseLocksFor(socket: WsWrapper) { getActiveBoards().forEach((state) => { const locks = state.locks for (const [itemId, sessionId] of locks.entries()) { if (socket.id === sessionId) { locks.delete(itemId) } } }) } ================================================ FILE: backend/src/oauth.ts ================================================ import Cookies from "cookies" import { Express, Request, Response } from "express" import { OAuthAuthenticatedUser } from "../../common/src/authenticated-user" import { removeAuthenticatedUser, setAuthenticatedUser } from "./http-session" import { GoogleAuthProvider, googleConfig } from "./google-auth" import { GenericOIDCAuthProvider, genericOIDCConfig } from "./generic-oidc-auth" export interface AuthProvider { getAuthPageURL: () => Promise getAccountFromCode: (code: string) => Promise logout?: (req: Request, res: Response) => Promise } export function setupAuth(app: Express, provider: AuthProvider) { app.get("/login", async (req, res) => { new Cookies(req, res).set("returnTo", parseReturnPath(req), { maxAge: 24 * 3600 * 1000, httpOnly: true, }) // Max 24 hours const authUrl = await provider.getAuthPageURL() res.setHeader("content-type", "text/html") res.send(`Signing in...`) }) app.get("/logout", async (req, res) => { removeAuthenticatedUser(req, res) if (provider.logout) { await provider.logout(req, res) } else { res.redirect(parseReturnPath(req)) } }) app.get("/google-callback", async (req, res) => { const code = (req.query?.code as string) || "" const cookies = new Cookies(req, res) const returnTo = cookies.get("returnTo") || "/" cookies.set("returnTo", "", { maxAge: 0, httpOnly: true }) console.log("Verifying google auth", code) try { const userInfo = await provider.getAccountFromCode(code) console.log("Found", userInfo) setAuthenticatedUser(req, res, userInfo) res.redirect(returnTo) } catch (e) { console.error(e) res.status(500).send("Internal error") } }) function parseReturnPath(req: Request) { return (req.query.returnTo as string) || "/" } } export const authProvider: AuthProvider | null = googleConfig ? GoogleAuthProvider(googleConfig) : genericOIDCConfig ? GenericOIDCAuthProvider(genericOIDCConfig) : null ================================================ FILE: backend/src/professions.ts ================================================ export const professions = [ "Accountant", "Actress", "Architect", "Astronomer", "Author", "Baker", "Bricklayer", "Bus driver", "Butcher", "Carpenter", "Chef", "Cleaner", "Coder", "Consultant", "Dentist", "Designer", "DevOps specialist", "Doctor", "Dustman", "Electrician", "Engineer", "Factory worker", "Farmer", "Fire fighter", "Fisherman", "Florist", "Gardener", "Hairdresser", "Journalist", "Judge", "Lawyer", "Lecturer", "Librarian", "Lifeguard", "Mechanic", "Model", "Newsreader", "Nurse", "Optician", "Painter", "Pharmacist", "Photographer", "Physician", "Pilot", "Plumber", "Pointy-haired boss", "Politician", "Postman", "Product Owner", "Real estate agent", "Receptionist", "Scientist", "Scrum master", "Secretary", "Shop assistant", "Tailor", "Taxi driver", "Teacher", "Tester", "Translator", "Traffic warden", "Travel agent", "Vet", "UNIX guru", "Waiter/Waitress", "Window cleaner", ] export function randomProfession() { return professions[Math.floor(Math.random() * professions.length)] } ================================================ FILE: backend/src/require-auth.ts ================================================ import { Express, Request, Response, NextFunction } from "express" import { getAuthenticatedUser } from "./http-session" export const REQUIRE_AUTH = process.env.REQUIRE_AUTH === "true" export function possiblyRequireAuth(app: Express) { if (REQUIRE_AUTH) { // Require authentication for all resources except the URLs bound by setupAuth above app.use("/", (req: Request, res: Response, next: NextFunction) => { if (!getAuthenticatedUser(req)) { res.redirect("/login") } else { next() } }) } } ================================================ FILE: backend/src/s3.ts ================================================ import * as AWS from "aws-sdk" const s3Config = { region: "eu-north-1", apiVersion: "2006-03-01", signatureVersion: "v4", } let s3Instance: AWS.S3 | null = null export const s3 = () => { s3Instance = s3Instance || new AWS.S3(s3Config) return s3Instance } export function getSignedPutUrl(Key: string) { const signedUrlExpireSeconds = 60 * 5 const url = s3().getSignedUrl("putObject", { Bucket: "r-board-assets", Key, Expires: signedUrlExpireSeconds, }) return url } ================================================ FILE: backend/src/server.ts ================================================ import dotenv from "dotenv" dotenv.config() import * as Http from "http" import { exampleBoard } from "../../common/src/domain" import { awaitSavingChanges } from "./board-state" import { createBoard, fetchBoard } from "./board-store" import { initDB } from "./db" import { startExpressServer } from "./express-server" import { terminateSessions } from "./websocket-sessions" let stopServer: (() => void) | null = null async function shutdown() { console.log("Shutdown initiated. Closing sockets.") if (stopServer) stopServer() terminateSessions() console.log("Shutdown in progress. Waiting for all changes to be saved...") await awaitSavingChanges() console.log("Shutdown complete. Exiting process.") process.exit(0) } process.on("SIGTERM", () => { console.log("Received SIGTERM. Initiating shutdown.") shutdown() }) const PORT = parseInt(process.env.PORT || "1337") const HTTPS_PORT = process.env.HTTPS_PORT ? parseInt(process.env.HTTPS_PORT) : undefined const BIND_UWEBSOCKETS_TO_PORT = process.env.BIND_UWEBSOCKETS_TO_PORT === "true" if (BIND_UWEBSOCKETS_TO_PORT && process.env.UWEBSOCKETS_PORT) { throw Error("Cannot have both UWEBSOCKETS_PORT and BIND_UWEBSOCKETS_TO_PORT envs") } const HTTP_PORT = BIND_UWEBSOCKETS_TO_PORT ? null : PORT const UWEBSOCKETS_PORT = BIND_UWEBSOCKETS_TO_PORT ? PORT : process.env.UWEBSOCKETS_PORT ? parseInt(process.env.UWEBSOCKETS_PORT) : null initDB() .then(async () => { if (!(await fetchBoard("default"))) { await createBoard(exampleBoard) } }) .then(() => { if (HTTP_PORT) { stopServer = startExpressServer(HTTP_PORT, HTTPS_PORT) } if (UWEBSOCKETS_PORT) { import("./uwebsockets-server").then((uwebsockets) => { uwebsockets.startUWebSocketsServer(UWEBSOCKETS_PORT) }) } }) .catch((e) => { console.error(e) }) ================================================ FILE: backend/src/storage.ts ================================================ import { getSignedPutUrl as s3GetSignedPutUrl } from "./s3" import { StorageBackend } from "./config" function localFSGetSignedPutUrl(Key: string): string { return "/assets/" + Key } export const createGetSignedPutUrl = (storageBackend: StorageBackend): ((key: string) => string) => storageBackend.type === "AWS" ? s3GetSignedPutUrl : localFSGetSignedPutUrl ================================================ FILE: backend/src/tools/wait-for-db.ts ================================================ import TcpPortUsed from "tcp-port-used" const port = 13338 ;(async function () { console.log(`Waiting for DB to bind port ${port}...`) try { await TcpPortUsed.waitUntilUsed(port, 100, 10000) } catch { console.error("Timed out waiting for DB") process.exit(1) } })() ================================================ FILE: backend/src/user-store.ts ================================================ import { inTransaction, withDBClient } from "./db" import * as uuid from "uuid" import { EventUserInfo, Id, RecentBoard, EventUserInfoAuthenticated, ISOTimeStamp } from "../../common/src/domain" import { uniqBy } from "lodash" export function getUserIdForEmail(email: string): Promise { return inTransaction(async (client) => { let id: string | undefined = (await client.query("SELECT id FROM app_user WHERE email=$1", [email])).rows[0]?.id if (!id) { id = uuid.v4() await client.query("INSERT INTO app_user (id, email) VALUES ($1, $2);", [id, email]) } return id }) } export async function associateUserWithBoard( userId: string, boardId: Id, lastOpened: ISOTimeStamp = new Date().toISOString(), ) { try { await inTransaction(async (client) => { await client.query( `INSERT INTO user_board (user_id, board_id, last_opened) values ($1, $2, $3) ON CONFLICT (user_id, board_id) DO UPDATE SET last_opened=EXCLUDED.last_opened`, [userId, boardId, lastOpened], ) }) } catch (e) { console.error(`Failed to associate user ${userId} with board ${boardId}`) } } export async function dissociateUserWithBoard(userId: string, boardId: Id) { try { await inTransaction(async (client) => { await client.query(`DELETE FROM user_board WHERE user_id=$1 and board_id=$2`, [userId, boardId]) }) } catch (e) { console.error(`Failed to dissociate user ${userId} with board ${boardId}`) } } export async function getUserAssociatedBoards(user: EventUserInfoAuthenticated): Promise { const rows = ( await withDBClient((client) => client.query( "SELECT b.id, b.name, ub.last_opened FROM user_board ub JOIN board b on (ub.board_id = b.id) WHERE ub.user_id = $1", [user.userId], ), ) ).rows return rows.map((r) => { return { id: r.id, name: r.name, userEmail: user.email, opened: r.last_opened.toISOString(), } }) } ================================================ FILE: backend/src/uwebsockets-server.ts ================================================ import uws from "uWebSockets.js" import { EventFromServer } from "../../common/src/domain" import * as uuid from "uuid" import { connectionHandler, MessageHandler } from "./connection-handler" import { handleBoardEvent } from "./board-event-handler" import { createGetSignedPutUrl } from "./storage" import { getConfig } from "./config" import * as L from "lonna" import { handleCommonEvent } from "./common-event-handler" export const WsWrapper = (ws: uws.WebSocket) => { const errorE = L.bus() const closeE = L.bus() const msgE = L.bus() const onError = (f: () => void) => { errorE.forEach(f) } const onMessage = (f: (msg: object) => void) => { msgE.forEach(f) } const onClose = (f: () => void) => { closeE.forEach(f) } return { send: (buffer: Buffer) => { try { ws.send(buffer, false) } catch (e) { ws.close() } }, onError, onMessage, onClose, id: uuid.v4(), close: () => { ws.close() closeE.push() }, errorE, closeE, msgE, } } type WsWrapper = ReturnType type WsUserData = { handler: MessageHandler } export function startUWebSocketsServer(port: number) { const config = getConfig() const app = uws.App() const sockets = new Map() const textDecoder = new TextDecoder() const signedPutUrl = createGetSignedPutUrl(config.storageBackend) mountWs("/socket/lobby", () => handleBoardEvent(null, signedPutUrl)) mountWs("/socket/board/:boardId", (req) => handleBoardEvent(req.getParameter(0), signedPutUrl)) app.get("/", (res) => res.writeStatus("200 OK").end("Sorry, we only serve websocket clients here.")) function mountWs(path: string, f: (req: uws.HttpRequest) => MessageHandler) { app.ws(path, { upgrade: (res, req, context) => { res.upgrade( { handler: f(req), } as WsUserData, req.getHeader("sec-websocket-key"), req.getHeader("sec-websocket-protocol"), req.getHeader("sec-websocket-extensions"), // 3 headers are used to setup websocket context, ) }, open: (ws) => { const wrapper = WsWrapper(ws) const handler = (ws as WsUserData & uws.WebSocket).handler sockets.set(ws, wrapper) connectionHandler(wrapper, handler) }, message: (ws, message, isBinary) => { if (isBinary) throw Error("Binary message") const object = JSON.parse(textDecoder.decode(message)) const wrapper = sockets.get(ws) if (!wrapper) { throw Error("Wrapper not found for socket " + ws) } wrapper.msgE.push(object) }, close: (ws) => { const wrapper = sockets.get(ws) if (wrapper) { wrapper.closeE.push() } }, }) } app.listen(port, () => { console.log("uWebSockets listening on " + port) }) } ================================================ FILE: backend/src/websocket-sessions.ts ================================================ import { OAuthAuthenticatedUser } from "../../common/src/authenticated-user" import { AccessLevel, AckJoinBoard, AuthLogout, BoardHistoryEntry, CURSOR_POSITIONS_ACTION_TYPE, EventFromServer, EventUserInfoAuthenticated, Id, ItemLocks, JoinedBoard, Serial, SessionUserInfo, SetNickname, UnidentifiedUserInfo, UserCursorPosition, UserInfoUpdate, getBoardAttributes, isBoardHistoryEntry, } from "../../common/src/domain" import { ServerSideBoardState, maybeGetBoard } from "./board-state" import { getBoardHistory } from "./board-store" import { closeYjsSocketsBySessionId } from "./board-yjs-server" import { randomProfession } from "./professions" import { getUserIdForEmail } from "./user-store" import { WsWrapper, toBuffer } from "./ws-wrapper" export type UserSession = { readonly sessionId: Id boardSession: UserSessionBoardEntry | null userInfo: SessionUserInfo sendEvent: (event: EventFromServer) => void isOnBoard: (boardId: Id) => boolean close(): void } export type UserSessionBoardEntry = { boardId: Id status: "ready" | "buffering" accessLevel: AccessLevel bufferedEvents: BoardHistoryEntry[] } /* socket: WsWrapper boards: Id[] userInfo: EventUserInfo */ export type SocketId = string const sessions: Record = {} const everyoneOnTheBoard = (boardId: string) => { const boardState = maybeGetBoard(boardId) if (!boardState) { console.warn(`Trying to send to a board not in memory: ${boardId}`) return [] } return boardState.sessions } const sendTo = (recipients: UserSession[], message: EventFromServer) => { recipients.forEach((c) => c.sendEvent(message)) } const everyoneElseOnTheSameBoard = (boardId: Id, session?: UserSession) => everyoneOnTheBoard(boardId).filter((s) => s !== session) export function startSession(socket: WsWrapper) { sessions[socket.id] = userSession(socket) } function userSession(socket: WsWrapper): UserSession { const sessionId = socket.id function sendEvent(event: EventFromServer) { if (isBoardHistoryEntry(event)) { const entry = session.boardSession if (!entry) throw Error("Board " + event.boardId + " not found for session " + sessionId) if (entry.status === "buffering") { entry.bufferedEvents.push(event) return } } socket.send(toBuffer(event)) } const session: UserSession = { sessionId, userInfo: anonymousUser("Anonymous " + randomProfession()), boardSession: null, sendEvent, isOnBoard: (boardId: Id) => session.boardSession != null && session.boardSession.boardId === boardId, close: () => socket.close(), } sessions[socket.id] = session return session } function anonymousUser(nickname: string): UnidentifiedUserInfo { return { userType: "unidentified", nickname } } export function endSession(socket: WsWrapper) { const sessionId = socket.id const session = sessions[sessionId] if (!session) { console.warn(`Ending non-existing session ${sessionId}`) return } if (session.boardSession) { const boardState = maybeGetBoard(session.boardSession.boardId) if (boardState) { boardState.sessions = boardState.sessions.filter((s) => s.sessionId !== sessionId) broadcastBoardEvent({ action: "board.left", boardId: boardState.board.id, sessionId }) } else { console.warn(`Board state not found when ending session: ${session.boardSession.boardId}`) } } delete sessions[socket.id] closeYjsSocketsBySessionId(sessionId) } export function getBoardSessionCount(id: Id) { return everyoneOnTheBoard(id).length } export function getSession(socket: WsWrapper): UserSession | undefined { return getSessionById(socket.id) } export function getSessionById(sessionId: string): UserSession | undefined { return sessions[sessionId] } export function terminateSessions() { Object.values(sessions).forEach((session) => session.close()) } export async function addSessionToBoard( boardState: ServerSideBoardState, origin: WsWrapper, accessLevel: AccessLevel, initAtSerial?: Serial, ): Promise { const session = sessions[origin.id] if (!session) throw new Error("No session found for socket " + origin.id) const boardId = boardState.board.id if (!boardState.sessions.includes(session)) { boardState.sessions = [...boardState.sessions, session] } const initDiff = initAtSerial && boardState.board.serial - initAtSerial if (initDiff && initDiff > Object.keys(boardState.board.items).length) { console.log(`Sending fresh board state for board ${boardId} instead of diff (${initDiff} events to sync)`) initAsNew(session, boardId, accessLevel, boardState) } else if (initAtSerial) { const entry: UserSessionBoardEntry = { boardId, status: "buffering", accessLevel, bufferedEvents: [] } // 1. Add session to the board with "buffering" status, to collect all events that were meant to be sent during this async initialization session.boardSession = entry try { const boardAttributes = getBoardAttributes(boardState.board, session.userInfo) //console.log(`Starting session at ${initAtSerial}`) // 2. capture all board events that haven't yet been flushed to the DB const inMemoryEvents = (boardState.currentlyStoring?.events ?? []) .concat(boardState.recentEvents) .filter((e) => e.serial! > initAtSerial) // 3. Fetch events from DB as chunks // IMPORTANT NOTE: this is the only await here and must remain so, as the logic here depends on everything else being synchronous. console.log(`Loading board history for board ${boardState.board.id} session at serial ${initAtSerial}`) let first = true await getBoardHistory(boardState.board.id, initAtSerial, (chunk) => { // Send a chunk of events with done: false, so that client knows to wait for more session.sendEvent({ action: "board.init.diff", first, last: false, boardAttributes, recentEvents: chunk, initAtSerial, accessLevel, }) first = false }) console.log(`Got board history for board ${boardState.board.id} session at serial ${initAtSerial}`) // 4. Send the last chunk containing both the inMemoryEvents and the buffered events (done: true) // In memory events: not yet flushed to DB when query was made // Buffered events: events that occurred after the in memory events were captured session.sendEvent({ action: "board.init.diff", boardAttributes, first, last: true, recentEvents: [...inMemoryEvents, ...entry.bufferedEvents], initAtSerial, accessLevel, }) // 5. Set the client to "ready" status so that new events will be flushed entry.status = "ready" entry.bufferedEvents = [] } catch (e) { console.warn( `Failed to bootstrap client on board ${boardId} at serial ${initAtSerial}. Sending full state.`, ) entry.status = "ready" entry.bufferedEvents = [] session.sendEvent({ action: "board.init", board: boardState.board, accessLevel, }) } } else { initAsNew(session, boardId, accessLevel, boardState) } // TODO SECURITY: don't reveal authenticated emails to unidentified users on same board // TODO: what to include in joined events? Not just nickname, as we want to show who's identified (beside the cursor) session.sendEvent({ action: "board.join.ack", boardId: boardState.board.id, sessionId: session.sessionId, nickname: session.userInfo.nickname, } as AckJoinBoard) // Notify new user of existing users everyoneOnTheBoard(boardState.board.id).forEach((s) => { session.sendEvent({ action: "board.joined", boardId: boardState.board.id, sessionId: s.sessionId, ...s.userInfo, } as JoinedBoard) }) // Notify existing users of new user broadcastJoinEvent(boardState.board.id, session) } function initAsNew(session: UserSession, boardId: string, accessLevel: AccessLevel, boardState: ServerSideBoardState) { session.boardSession = { boardId, status: "ready", accessLevel, bufferedEvents: [] } session.sendEvent({ action: "board.init", board: boardState.board, accessLevel, }) } export function setNicknameForSession(event: SetNickname, origin: WsWrapper) { const session = getSession(origin) if (!session) { console.warn(`Session not found: ${origin.id}`) return } session.userInfo = session.userInfo.userType === "unidentified" ? anonymousUser(event.nickname) : { ...session.userInfo, nickname: event.nickname } const updateInfo: UserInfoUpdate = { action: "userinfo.set", sessionId: session.sessionId, ...session.userInfo, } if (session.boardSession) { sendTo(everyoneOnTheBoard(session.boardSession.boardId), updateInfo) } } export async function setVerifiedUserForSession( event: OAuthAuthenticatedUser, session: UserSession, ): Promise { const userId = await getUserIdForEmail(event.email) session.userInfo = { userType: "authenticated", nickname: event.name, name: event.name, email: event.email, picture: event.picture, domain: event.domain, userId, } if (session.boardSession) { // TODO SECURITY: don't reveal authenticated emails to unidentified users on same board sendTo(everyoneElseOnTheSameBoard(session.boardSession.boardId, session), { action: "user.login", email: event.email, name: event.name, picture: event.picture, }) } return session.userInfo } export function logoutUser(event: AuthLogout, origin: WsWrapper) { const session = getSession(origin) if (!session) { console.warn("Session not found for socket " + origin.id) } else { session.userInfo = { userType: "unidentified", nickname: session.userInfo.nickname } } } export function broadcastBoardEvent(event: EventFromServer & { boardId: string }, origin?: UserSession) { //console.log("Broadcast", event.action, "to", everyoneElseOnTheSameBoard(event.boardId, origin).length) sendTo(everyoneElseOnTheSameBoard(event.boardId, origin), event) } export function broadcastJoinEvent(boardId: Id, session: UserSession) { sendTo(everyoneElseOnTheSameBoard(boardId, session), { action: "board.joined", boardId, sessionId: session.sessionId, ...session.userInfo, } as JoinedBoard) } export function broadcastCursorPositions(boardId: Id, positions: Record) { sendTo(everyoneOnTheBoard(boardId), { action: CURSOR_POSITIONS_ACTION_TYPE, p: positions }) } const BROADCAST_DEBOUNCE_MS = 20 // Debounce by 20ms per board id, otherwise every item interaction (e.g. drag on 10 items, one event each) broadcasts locks export const broadcastItemLocks = (() => { let timeouts: Record = {} const hasActiveTimer = (boardId: string) => timeouts[boardId] !== undefined return function _broadcastItemLocks(boardId: string, locks: ItemLocks) { if (hasActiveTimer(boardId)) { return } timeouts[boardId] = setTimeout(() => { const boardState = maybeGetBoard(boardId) if (boardState) { sendTo(boardState.sessions, { action: "board.locks", boardId, locks }) } timeouts[boardId] = undefined }, BROADCAST_DEBOUNCE_MS) } })() export function getSessionCount() { return Object.values(sessions).length } ================================================ FILE: backend/src/ws-wrapper.ts ================================================ import * as WebSocket from "ws" import * as uuid from "uuid" import { EventFromServer } from "../../common/src/domain" export const WsWrapper = (ws: WebSocket) => { const onError = (f: () => void) => { ws.addEventListener("error", f) } const onMessage = (f: (msg: object) => void) => { ws.addEventListener("message", (msg: any) => { try { f(JSON.parse(msg.data)) } catch (e) { console.error("Error in WsWrapper/onMessage. Closing connection.", e) ws.close() } }) } const onClose = (f: () => void) => { ws.addEventListener("close", f) } return { send: (buffer: Buffer) => { try { ws.send(buffer, { binary: false }) } catch (e) { ws.close() } }, onError, onMessage, onClose, id: uuid.v4(), close: () => ws.close(), } } export type WsWrapper = ReturnType type CachedBuffer = { msg: EventFromServer; buffer: Buffer } let cachedBuffer: CachedBuffer | null = null export function toBuffer(msg: EventFromServer) { // We cache the latest buffer to avoid creating a new buffer for every message. if (cachedBuffer && cachedBuffer.msg === msg) { return cachedBuffer.buffer } cachedBuffer = { msg, buffer: Buffer.from(JSON.stringify(msg)) } return cachedBuffer.buffer } ================================================ FILE: backend/src/y-websocket-server/Docs.ts ================================================ import { WSSharedDoc } from "./WSSharedDoc" import { Persistence } from "./Persistence" export interface DocsOptions { persistence?: Persistence gc?: boolean } interface DocState { doc: WSSharedDoc fetchPromise: Promise } export class Docs { readonly docs = new Map() readonly persistence: Persistence | null readonly gc: boolean constructor(options: DocsOptions = {}) { this.persistence = options.persistence || null this.gc = options.gc ?? true } /** * Gets a Y.Doc by name, whether in memory or on disk */ getYDoc(docname: string): WSSharedDoc { return this.getDocState(docname).doc } async getYDocAndWaitForFetch(docname: string): Promise { const state = this.getDocState(docname) await state.fetchPromise return state.doc } private getDocState(docname: string): DocState { let state = this.docs.get(docname) if (!state) { const doc = new WSSharedDoc(this, docname) console.log(`Loading document ${doc.name} into memory`) doc.gc = this.gc if (this.persistence !== null) { void this.persistence.bindState(docname, doc) } const fetchPromise = this.persistence !== null ? this.persistence.bindState(docname, doc) : Promise.resolve() state = { doc, fetchPromise } this.docs.set(docname, state) } return state } deleteYDoc(doc: WSSharedDoc) { console.log(`Purging document ${doc.name} from memory`) this.docs.delete(doc.name) } } ================================================ FILE: backend/src/y-websocket-server/Persistence.ts ================================================ import * as Y from "yjs" export interface Persistence { bindState: (docName: string, ydoc: Y.Doc) => Promise writeState: (docName: string, ydoc: Y.Doc) => Promise } export function createLevelDbPersistence(persistenceDir: string): Persistence { console.info('Persisting documents to "' + persistenceDir + '"') // @ts-ignore const LeveldbPersistence = require("y-leveldb").LeveldbPersistence const ldb = new LeveldbPersistence(persistenceDir) return { bindState: async (docName, ydoc) => { const persistedYdoc = await ldb.getYDoc(docName) const newUpdates = Y.encodeStateAsUpdate(ydoc) ldb.storeUpdate(docName, newUpdates) Y.applyUpdate(ydoc, Y.encodeStateAsUpdate(persistedYdoc)) ydoc.on("update", (update) => { ldb.storeUpdate(docName, update) }) }, writeState: async (docName, ydoc) => {}, } } ================================================ FILE: backend/src/y-websocket-server/Protocol.ts ================================================ export const messageSync = 0 export const messageAwareness = 1 ================================================ FILE: backend/src/y-websocket-server/WSSharedDoc.ts ================================================ import * as Y from "yjs" import * as awarenessProtocol from "y-protocols/awareness" import * as syncProtocol from "y-protocols/sync" import * as encoding from "lib0/encoding" import * as WebSocket from "ws" import { Docs } from "./Docs" import { messageAwareness, messageSync } from "./Protocol" export const wsReadyStateConnecting = 0 export const wsReadyStateOpen = 1 export const wsReadyStateClosing = 2 export const wsReadyStateClosed = 3 export class WSSharedDoc extends Y.Doc { private docs: Docs readonly name: string private conns: Map> = new Map>() readonly awareness = new awarenessProtocol.Awareness(this) constructor(docs: Docs, name: string) { super({ gc: docs.gc }) this.docs = docs this.name = name this.awareness.setLocalState(null) const awarenessChangeHandler = ( { added, updated, removed }: { added: number[]; updated: number[]; removed: number[] }, conn: WebSocket, ) => { const changedClients = added.concat(updated, removed) if (conn !== null) { const connControlledIDs = this.conns.get(conn) if (connControlledIDs !== undefined) { added.forEach((clientID) => { connControlledIDs.add(clientID) }) removed.forEach((clientID) => { connControlledIDs.delete(clientID) }) } } const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageAwareness) encoding.writeVarUint8Array( encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients), ) const buff = encoding.toUint8Array(encoder) this.conns.forEach((_, c) => { this.send(c, buff) }) } this.awareness.on("update", awarenessChangeHandler) const updateHandler = (update: Uint8Array, origin: any, doc: WSSharedDoc) => { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) syncProtocol.writeUpdate(encoder, update) const message = encoding.toUint8Array(encoder) doc.conns.forEach((_, conn) => this.send(conn, message)) } this.on("update", updateHandler) } send(conn: WebSocket, m: Uint8Array) { if (conn.readyState !== wsReadyStateConnecting && conn.readyState !== wsReadyStateOpen) { this.closeConn(conn) } try { conn.send(m, (err: any) => { err != null && this.closeConn(conn) }) } catch (e) { console.error("Failed to send message to client. Closing connection.", e) this.closeConn(conn) } } closeConn(conn: WebSocket) { if (this.conns.has(conn)) { const controlledIds = this.conns.get(conn)! this.conns.delete(conn) awarenessProtocol.removeAwarenessStates(this.awareness, Array.from(controlledIds), null) if (this.conns.size === 0 && this.docs.persistence !== null) { // if persisted, we store state and destroy ydocument this.docs.persistence.writeState(this.name, this).then(() => { this.destroy() }) this.docs.deleteYDoc(this) } } conn.close() } addConnection(conn: WebSocket) { this.conns.set(conn, new Set()) } hasConnection(conn: WebSocket) { return this.conns.has(conn) } } ================================================ FILE: backend/src/y-websocket-server/YWebSocketServer.ts ================================================ import * as awarenessProtocol from "y-protocols/awareness" import * as syncProtocol from "y-protocols/sync" import * as decoding from "lib0/decoding" import * as encoding from "lib0/encoding" import * as WebSocket from "ws" import { Docs, DocsOptions } from "./Docs" import { messageAwareness, messageSync } from "./Protocol" import { WSSharedDoc } from "./WSSharedDoc" const pingTimeout = 30000 export default class YWebSocketServer { docs: Docs constructor(options?: DocsOptions) { this.docs = new Docs(options) } async setupWSConnection(conn: WebSocket, docName: string, readOnly: boolean) { conn.binaryType = "arraybuffer" // get doc, initialize if it does not exist yet const doc = this.docs.getYDoc(docName) console.log(`YJS connection established for ${docName}`) doc.addConnection(conn) // listen and reply to events conn.on("message", (message: ArrayBuffer) => messageListener(conn, doc, readOnly, new Uint8Array(message))) // Check if connection is still alive let pongReceived = true const pingInterval = setInterval(() => { if (!pongReceived) { if (doc.hasConnection(conn)) { doc.closeConn(conn) } clearInterval(pingInterval) } else if (doc.hasConnection(conn)) { pongReceived = false try { conn.ping() } catch (e) { doc.closeConn(conn) clearInterval(pingInterval) } } }, pingTimeout) conn.on("close", () => { doc.closeConn(conn) clearInterval(pingInterval) }) conn.on("pong", () => { pongReceived = true }) // put the following in a variables in a block so the interval handlers don't keep in in // scope { // send sync step 1 const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) syncProtocol.writeSyncStep1(encoder, doc) doc.send(conn, encoding.toUint8Array(encoder)) const awarenessStates = doc.awareness.getStates() if (awarenessStates.size > 0) { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageAwareness) encoding.writeVarUint8Array( encoder, awarenessProtocol.encodeAwarenessUpdate(doc.awareness, Array.from(awarenessStates.keys())), ) doc.send(conn, encoding.toUint8Array(encoder)) } } } } // Read-only implementation found at https://discuss.yjs.dev/t/read-only-or-one-way-only-sync/135/3 const readSyncMessage = ( decoder: decoding.Decoder, encoder: encoding.Encoder, doc: WSSharedDoc, readOnly = false, transactionOrigin: any, ) => { const messageType = decoding.readVarUint(decoder) switch (messageType) { case syncProtocol.messageYjsSyncStep1: syncProtocol.readSyncStep1(decoder, encoder, doc) break case syncProtocol.messageYjsSyncStep2: if (!readOnly) syncProtocol.readSyncStep2(decoder, doc, transactionOrigin) break case syncProtocol.messageYjsUpdate: if (!readOnly) syncProtocol.readUpdate(decoder, doc, transactionOrigin) break default: throw new Error("Unknown message type") } return messageType } const messageListener = (conn: WebSocket, doc: WSSharedDoc, readOnly: boolean, message: Uint8Array) => { try { const encoder = encoding.createEncoder() const decoder = decoding.createDecoder(message) const messageType = decoding.readVarUint(decoder) switch (messageType) { case messageSync: encoding.writeVarUint(encoder, messageSync) readSyncMessage(decoder, encoder, doc, readOnly, conn) // If the `encoder` only contains the type of reply message and no // message, there is no need to send the message. When `encoder` only // contains the type of reply, its length is 1. if (encoding.length(encoder) > 1) { doc.send(conn, encoding.toUint8Array(encoder)) } break case messageAwareness: { awarenessProtocol.applyAwarenessUpdate(doc.awareness, decoding.readVarUint8Array(decoder), conn) break } default: { console.warn("Unexpected message type" + messageType) } } } catch (err) { console.error(err) doc.emit("error", [err]) } } ================================================ FILE: backend/tsconfig.json ================================================ { "extends": "../tsconfig", "compilerOptions": { "module": "commonjs", "outDir": "./dist", "rootDir": "..", "sourceMap": true } } ================================================ FILE: benchmark/benchmark.ts ================================================ import { uniqueId } from "lodash" import { arrayToRecordById } from "../common/src/arrays" import { boardReducer } from "../common/src/board-reducer" import { Board, Item, Note, newNote } from "../common/src/domain" type Foo = Board function createRandomItems(count: number): Item[] { const items: Item[] = [] for (let i = 0; i < count; i++) { items.push({ id: uniqueId(), type: "note", color: "yellow", height: 100, width: 100, x: Math.random() * 10000, y: Math.random() * 10000, z: 0, shape: "square", text: "Hello world", }) } return items } function createTestBoard(size = 1000): Board { return { id: uniqueId(), height: 10000, width: 10000, serial: 0, name: "Bigass board", items: arrayToRecordById(createRandomItems(size)), connections: [], } } let testBoards = Array.from({ length: 100 }, () => { const board = createTestBoard(100) const items = Object.values(board.items) as Note[] return { board, items, } }) console.time("10000 item.updates on a randomly picked 100 item board") for (let i = 0; i < 10000; i++) { let randomBoard = testBoards[Math.floor(Math.random() * testBoards.length)] // const itemsArray = Object.values(randomBoard.items) as Note[] const item = randomBoard.items[i % randomBoard.items.length] ;[randomBoard.board] = boardReducer(randomBoard.board, { boardId: randomBoard.board.id, action: "item.update", items: [ { ...item, text: "Hello world", }, ], }) } console.timeEnd("10000 item.updates on a randomly picked 100 item board") testBoards = Array.from({ length: 100 }, () => { const board = createTestBoard(100) const items = Object.values(board.items) as Note[] return { board, items, } }) console.time("10000 item.adds on a randomly picked 100 item board") for (let i = 0; i < 10000; i++) { const randomBoard = testBoards[Math.floor(Math.random() * testBoards.length)] const item = newNote("foo") ;[randomBoard.board] = boardReducer(randomBoard.board, { boardId: randomBoard.board.id, action: "item.add", items: [item], connections: [], }) } console.timeEnd("10000 item.adds on a randomly picked 100 item board") testBoards = Array.from({ length: 100 }, () => { const board = createTestBoard(100) const items = Object.values(board.items) as Note[] return { board, items, } }) console.time("10000 item.moves on a randomly picked 100 item board") for (let i = 0; i < 10000; i++) { const randomBoard = testBoards[Math.floor(Math.random() * testBoards.length)] const item = randomBoard.items[i % randomBoard.items.length] ;[randomBoard.board] = boardReducer(randomBoard.board, { boardId: randomBoard.board.id, action: "item.move", items: [ { ...item, x: Math.random() * 10000, y: Math.random() * 10000, }, ], connections: [], }) } console.timeEnd("10000 item.moves on a randomly picked 100 item board") console.time("Reference: creating 10000 input objects and doing nothing") const noop = (item: any) => { item.text } for (let i = 0; i < 10000; i++) { const item = newNote("foo") noop({ ...item, text: "Hello world", }) } console.timeEnd("Reference: creating 10000 input objects and doing nothing") ================================================ FILE: common/src/action-folding.ts ================================================ import { arrayEquals, arrayIdAndKeysMatch, arrayIdMatch, idsOf } from "./arrays" import { AppEvent, BoardHistoryEntry, CURSOR_POSITIONS_ACTION_TYPE, MoveItem, isBoardHistoryEntry, isSameUser, } from "./domain" type FoldOptions = { cursorsOnly?: boolean } const defaultOptions = { foldAddUpdate: true, cursorsOnly: false, } export const CURSORS_ONLY: FoldOptions = { cursorsOnly: true } export function foldActions(a: AppEvent, b: AppEvent, options: FoldOptions = defaultOptions): AppEvent | null { if (isBoardHistoryEntry(a) && isBoardHistoryEntry(b)) { if (options.cursorsOnly) return null if (!isSameUser(a.user, b.user)) return null const folded = foldActions_(a, b, options) if (!folded) return null const firstSerial = a.firstSerial ? a.firstSerial : a.serial const serial = b.serial return { ...(folded as BoardHistoryEntry), serial, firstSerial } as BoardHistoryEntry } else { return foldActions_(a, b, options) } } /* Folding can be done if in any given state S, applying actions A and B consecutively can be replaced with a single action C. This function should return that composite action or null if folding is not possible. */ export function foldActions_(a: AppEvent, b: AppEvent, options: FoldOptions = defaultOptions): AppEvent | null { if (a.action === CURSOR_POSITIONS_ACTION_TYPE && b.action === CURSOR_POSITIONS_ACTION_TYPE) { return b } if (a.action === "cursor.move" && b.action === "cursor.move" && b.boardId === a.boardId) { return b // This is a local cursor move } if (options.cursorsOnly) return null if (isBoardHistoryEntry(a) && isBoardHistoryEntry(b)) { if (!isSameUser(a.user, b.user)) return null } if (a.action === "item.front") { if (b.action === "item.front" && b.boardId === a.boardId && arrayEquals(b.itemIds, a.itemIds)) return b } else if (a.action === "item.move") { if (b.action === "item.move" && b.boardId === a.boardId && everyMovedItemMatches(b, a)) return b } else if (a.action === "item.update") { if ( b.action === "item.update" && b.boardId === a.boardId && arrayIdAndKeysMatch(b.items, a.items) && arrayIdAndKeysMatch(b.connections ?? [], a.connections ?? []) ) { return b } } else if (a.action === "item.lock" || a.action === "item.unlock") { if (b.action === a.action && b.boardId === a.boardId && b.itemId === a.itemId) return b } else if (a.action === "connection.modify" && b.action === "connection.modify") { if (arrayIdMatch(a.connections, b.connections)) return b } else if (a.action === "connection.modify" && b.action === "connection.delete") { if (arrayEquals(b.connectionIds, idsOf(a.connections))) return b } return null } function everyMovedItemMatches(evt: MoveItem, evt2: MoveItem) { return arrayIdMatch(evt.items, evt2.items) && arrayIdMatch(evt.connections, evt2.connections) } export function addOrReplaceEvent(event: E, q: E[], options: FoldOptions = defaultOptions): E[] { for (let i = 0; i < q.length; i++) { let eventInQueue = q[i] const folded = foldActions(eventInQueue, event, options) if (folded) { return [...q.slice(0, i), folded, ...q.slice(i + 1)] as E[] } } return q.concat(event) } ================================================ FILE: common/src/arrays.ts ================================================ import { isArray, isEqual } from "lodash" export function toArray(x: T | T[]) { if (isArray(x)) return x return [x] } export function arrayIdMatch(a: T[] | T, b: T[] | T) { return arrayEquals(idsOf(a), idsOf(b)) } export function arrayObjectKeysMatch(a: T[] | T, b: T[] | T) { return arrayEquals(keysOf(a), keysOf(b)) } export function arrayIdAndKeysMatch(a: T[] | T, b: T[] | T) { return arrayIdMatch(a, b) && arrayObjectKeysMatch(a, b) } export function idsOf(a: T[] | T): string[] { return toArray(a).map((x) => x.id) } export function keysOf(a: T[] | T): string[][] { return toArray(a).map((x) => Object.keys(x)) } export function arrayEquals(a: T[] | T, b: T[] | T) { return isEqual(toArray(a), toArray(b)) } export function arrayToRecordById(arr: T[], init: Record = {}): Record { return arr.reduce((acc: Record, elem: T) => { acc[elem.id] = elem return acc }, init) } ================================================ FILE: common/src/assertNotNull.ts ================================================ export function assertNotNull(x: T | null | undefined): T { if (x === null || x === undefined) throw Error("Assertion failed: " + x) return x } ================================================ FILE: common/src/authenticated-user.ts ================================================ export type OAuthAuthenticatedUser = { name: string email: string picture?: string domain: string | null } ================================================ FILE: common/src/board-crdt-helper.ts ================================================ import * as Y from "yjs" import { Board, Id, Item, QuillDelta, isTextItem } from "./domain" export function getCRDTField(doc: Y.Doc, itemId: Id, fieldName: string) { return doc.getText(`items.${itemId}.${fieldName}`) } export function augmentBoardWithCRDT(doc: Y.Doc, board: Board): Board { const items = augmentItemsWithCRDT(doc, Object.values(board.items)) return { ...board, items: Object.fromEntries(items.map((i) => [i.id, i])), } } export function augmentItemsWithCRDT(doc: Y.Doc, items: Item[]): Item[] { return items.map((item) => { if (isTextItem(item) && item.crdt) { const field = getCRDTField(doc, item.id, "text") const textAsDelta = field.toDelta() as QuillDelta const text = field.toString() return { ...item, textAsDelta, text } } return item }) } export function importItemsIntoCRDT(doc: Y.Doc, items: Item[], options?: { fallbackToText: boolean }) { for (const item of items) { if (isTextItem(item) && item.crdt) { if (item.textAsDelta) { getCRDTField(doc, item.id, "text").applyDelta(item.textAsDelta) } else if (options?.fallbackToText) { getCRDTField(doc, item.id, "text").insert(0, item.text) } else { throw Error("textAsDelta is missing ") } } } } ================================================ FILE: common/src/board-reducer.benchmark.ts ================================================ import _ from "lodash" import { NOTE_COLORS } from "./colors" import { Board } from "./domain" import * as uuid from "uuid" import { boardReducer } from "./board-reducer" import { assertNotNull } from "./assertNotNull" function createBoard(): Board { const itemCount = 10000 const board: Board = { id: "0f5b9d6c-02c2-4b81-beb7-3a3b9035e8a2", name: "Perf3", width: 800, height: 600, serial: 320577, connections: [], items: {}, } for (let i = 0; i < itemCount; i++) { const id = uuid.v4() board.items[id] = { id, type: "text", x: Math.random() * 800, y: Math.random() * 600, z: 3, width: 5, height: 5, text: "Hello world", fontSize: 12, locked: false, color: "#FBFC86", } } return board } const board = createBoard() const boardId = board.id const eventCount = 1000 const items = Object.values(board.items) const started = new Date().getTime() for (let i = 0; i < eventCount; i++) { const target = assertNotNull(_.sample(items)) const updated = { ...target, text: "EDIT " + i, color: _.sample(NOTE_COLORS)?.color! } boardReducer( board, { action: "item.update", boardId, items: [updated], }, { inplace: true }, ) } const elapsed = new Date().getTime() - started console.log(`Processed ${eventCount} events in ${elapsed}ms. (${eventCount / elapsed} events/ms)`) ================================================ FILE: common/src/board-reducer.ts ================================================ import { partition } from "lodash" import { maybeChangeContainerForItem } from "../../frontend/src/board/item-setcontainer" import { arrayToRecordById } from "./arrays" import { rerouteConnection, resolveEndpoint } from "./connection-utils" import { Board, Connection, ConnectionEndPoint, ConnectionUpdate, Container, findItem, findItemIdsRecursively, getConnection, getEndPointItemId, getItem, Id, Image, isBoardHistoryEntry, isContainedBy, isContainer, isItem, isItemEndPoint, isTextItem, Item, ItemUpdate, MoveItem, Note, PersistableBoardItemEvent, Point, Text, TextItem, Update, Video, } from "./domain" import { equalRect, Rect } from "./geometry" import { BoardPermission, canChangeFont, canChangeShapeAndColor, canChangeText, canChangeTextAlign, canDelete, canMove, nullablePermission, } from "../../frontend/src/board/board-permissions" type BoardReducerOptions = { inplace?: boolean strictOnSerials?: boolean } export function boardReducer( board: Board, event: PersistableBoardItemEvent, options: BoardReducerOptions = {}, ): [Board, (() => PersistableBoardItemEvent) | null] { const inplace = options.inplace ?? false if (isBoardHistoryEntry(event) && event.serial) { const firstSerial = event.firstSerial ? event.firstSerial : event.serial if (firstSerial !== board.serial + 1) { const message = `Serial skip on ${event.action}, ${board.serial} -> ${firstSerial} (firstSerial ${event.firstSerial} serial ${event.serial})` if (options.strictOnSerials) { throw Error(message) } else { console.warn(message) } } board = { ...board, serial: event.serial } } //console.log(event.action, inplace) switch (event.action) { case "connection.add": { const newConnections = event.connections for (let connection of newConnections) { validateConnection(board, connection) if (board.connections.some((c) => c.id === connection.id)) { throw Error(`Connection ${connection.id} already exists on board ${board.id}`) } } return [ { ...board, connections: applyListModification( board.connections, (cs) => { cs.push(...newConnections) }, inplace, ), }, () => ({ action: "connection.delete", boardId: event.boardId, connectionIds: newConnections.map((c) => c.id), }), ] } case "connection.modify": { const connections = event.connections.filter(canMove) const existingConnections = connections.map((r) => { validateConnection(board, r) const existingConnection = board.connections.find((c) => c.id === r.id) if (!existingConnection) { throw Error(`Trying to modify nonexisting connection ${r.id} on board ${board.id}`) } return existingConnection }) return [ { ...board, connections: applyListModification( board.connections, (cs) => replaceById(cs, connections), inplace, ), }, () => ({ action: "connection.modify", boardId: event.boardId, connections: existingConnections }), ] } case "connection.delete": { const ids = new Set(event.connectionIds) const existingConnections = board.connections.filter((c) => ids.has(c.id) || !canDelete(c)) return [ { ...board, connections: board.connections.filter((c) => !ids.has(c.id)) }, () => ({ action: "connection.add", boardId: event.boardId, connections: existingConnections }), ] } case "board.rename": return [{ ...board, name: event.name }, null] case "board.setAccessPolicy": return [{ ...board, accessPolicy: event.accessPolicy }, null] case "item.bootstrap": //if (board.items.length > 0) throw Error("Trying to bootstrap non-empty board") return [{ ...board, items: event.items, connections: event.connections }, null] case "item.add": if (event.items.some((a) => board.items[a.id])) { throw new Error("Adding duplicate item " + JSON.stringify(event.items)) } const updatedItems = applyModification( board.items, (items) => { event.items.forEach((item) => { if ( item.containerId && !findItem(board)(item.containerId) && !findItem(arrayToRecordById(event.items))(item.containerId) ) { // Add item but don't try to assign to a non-existing container items[item.id] = { ...item, containerId: undefined } } else { items[item.id] = item } }, {}) }, inplace, ) const boardWithAddedItems = { ...board, items: updatedItems } const connectionsToAdd = event.connections || [] connectionsToAdd.forEach((connection) => { validateConnection(boardWithAddedItems, connection) if (board.connections.some((c) => c.id === connection.id)) { throw Error(`Connection ${connection.id} already exists on board ${board.id}`) } }) return [ { ...boardWithAddedItems, connections: applyListModification( board.connections, (cs) => { cs.push(...connectionsToAdd) }, inplace, ), }, () => ({ action: "item.delete", boardId: board.id, itemIds: event.items.map((i) => i.id), connectionIds: event.connections.map((c) => c.id), }), ] case "item.font.increase": return [ { ...board, items: applyFontSize( board.items, 1.1, filterItemIdsByPermissions(event.itemIds, board, canChangeFont), inplace, ), }, () => ({ ...event, action: "item.font.decrease", }), ] case "item.font.decrease": return [ { ...board, items: applyFontSize( board.items, 1 / 1.1, filterItemIdsByPermissions(event.itemIds, board, canChangeFont), inplace, ), }, () => ({ ...event, action: "item.font.increase", }), ] case "item.update": { const updatedConnections = updateConnections(board, event.connections || []) const updatedItems = updateItems(board, event.items, updatedConnections, inplace) return [ { ...board, items: updatedItems, connections: updatedConnections, }, () => ({ action: "item.update", boardId: board.id, items: event.items.map((update) => copyMatchingKeysFromOriginal(update, getItem(board)(update.id))), connections: (event.connections || []).map((update) => copyMatchingKeysFromOriginal(update, getConnection(board)(update.id)), ), }), ] } case "item.move": return [ moveItems(board, event, inplace), () => ({ action: "item.move", boardId: board.id, items: event.items.map((i) => { const item = getItem(board)(i.id) return { id: i.id, x: item.x, y: item.y, containerId: item.containerId } }), connections: event.connections.map((c) => { const conn = getConnection(board)(c.id) const startPoint = resolveEndpoint(conn.from, board) return { id: c.id, x: startPoint.x, y: startPoint.y } }), }), ] case "item.delete": { const itemIds = filterItemIdsByPermissions(event.itemIds, board, canDelete) const connectionIds = filterConnectionIdsByPermissions(event.connectionIds, board, canDelete) const itemIdsToDelete = findItemIdsRecursively(itemIds, board) const connectionIdsToDelete = new Set(connectionIds) const updatedItems = inplace ? board.items : { ...board.items } itemIdsToDelete.forEach((id) => { delete updatedItems[id] }) const [connectionsToKeep, connectionsDeleted] = partition( board.connections, (c) => !connectionIdsToDelete.has(c.id) && !(c.containerId && itemIdsToDelete.has(c.containerId)) && (!isItemEndPoint(c.from) || !itemIdsToDelete.has(getEndPointItemId(c.from))) && (!isItemEndPoint(c.to) || !itemIdsToDelete.has(getEndPointItemId(c.to))), ) return [ { ...board, connections: connectionsToKeep, items: updatedItems, }, () => ({ action: "item.add", boardId: board.id, items: Array.from(itemIdsToDelete).map(getItem(board)), connections: connectionsDeleted, }), ] } case "item.front": let maxZ = 0 let maxZCount = 0 const itemsList = Object.values(board.items) for (let i of itemsList) { if (i.z > maxZ) { maxZCount = 1 maxZ = i.z } else if (i.z === maxZ) { maxZCount++ } } const isFine = (item: Item) => { return !event.itemIds.includes(item.id) || item.z === maxZ } if (maxZCount === event.itemIds.length && itemsList.every(isFine)) { // Requested items already on front return [board, null] } const updated = event.itemIds.reduce( (acc: Record, id) => { const item = board.items[id] if (!item) { console.warn(`Warning: trying to "item.front" nonexisting item ${id} on board ${board.id}`) return acc } const u = item.type !== "container" ? { ...item, z: maxZ + 1 } : item acc[u.id] = u return acc }, inplace ? board.items : {}, ) return [ { ...board, items: inplace ? board.items : { ...board.items, ...updated, }, }, null, ] // TODO: return item.back default: console.warn("Unknown event", event) return [board, null] } } function copyMatchingKeysFromOriginal(update: Update, original: T): Update { const keysAndValues = Object.keys(update).map((key) => [key, original[key as keyof T]]) const result = Object.fromEntries(keysAndValues) return result } function validateConnection(board: Board, connection: Connection) { validateEndPoint(board, connection, "from") validateEndPoint(board, connection, "to") } function validateEndPoint(board: Board, connection: Connection, key: "to" | "from") { const endPoint = connection[key] if (isItemEndPoint(endPoint)) { const toItem = board.items[getEndPointItemId(endPoint)] if (!toItem) { throw Error(`Connection ${connection.id} refers to nonexisting item ${endPoint}`) } } } function updateConnections(board: Board, updates: ConnectionUpdate[]): Connection[] { if (updates.length === 0) return board.connections updates = filterConnectionUpdatesByPermissions(updates, board) const updatedConnections = updates.map((update) => { const existing = board.connections.find((c) => c.id === update.id) if (!existing) { throw Error(`Trying to modify nonexisting connection ${update.id} on board ${board.id}`) } const updated = { ...existing, ...update } validateConnection(board, updated) return updated }) return board.connections.map((c) => { const replacement = updatedConnections.find((r) => r.id === c.id) return replacement ? replacement : c }) } function updateItems( board: Board, updateList: ItemUpdate[], updatedConnections: Connection[], inplace: boolean, ): Record { updateList = filterItemUpdatesByPermissions(updateList, board) const updatedItems: Item[] = updateList.map((update) => ({ ...board.items[update.id], ...update } as Item)) const resultItems = applyModification( board.items, (items) => { arrayToRecordById(updatedItems, items) }, inplace, ) updatedItems.filter(isContainer).forEach((container) => { const previous = board.items[container.id] if (previous && !equalRect(previous, container)) { // Container shape changed -> check items Object.values(board.items) .filter( (i) => i.containerId === container.id || // Check all previously contained items containedBy(i, container), // Check all items inside the new bounds ) .forEach((item) => { const newContainer = maybeChangeContainerForItem(item, resultItems) if (newContainer?.id !== item.containerId) { resultItems[item.id] = { ...item, containerId: newContainer ? newContainer.id : undefined } } }) } }) function setVisibilityRecursively(parent: Item, hidden: boolean) { const children = Object.values(resultItems).filter((i) => i.containerId === parent.id) children.forEach((child) => { const resultItem = { ...child, hidden } if (!hidden && isContainer(resultItem) && resultItem.contentsHidden) { resultItem.contentsHidden = false } resultItems[child.id] = resultItem setVisibilityRecursively(child, hidden) }) } function adjustConnectionVisibility() { updatedConnections.forEach((c, i) => { let shouldHide: boolean if (c.containerId) { // When a connection has containerId, it should be hidden if the container is hidden (or has contentsHidden) // A connection practically has containerId in case its endpoints are not attached to an item and it's contained by a container const container = resultItems[c.containerId] shouldHide = (container?.hidden || (isContainer(container) && container.contentsHidden)) ?? false } else { // In other cases, hide connections in case either end is connected to a hidden item const from = resolveEndpoint(c.from, resultItems) const to = resolveEndpoint(c.to, resultItems) shouldHide = ((isItem(from) && from.hidden) || (isItem(to) && to.hidden)) ?? false } if (shouldHide !== c.hidden ?? false) { updatedConnections[i] = { ...c, hidden: shouldHide } } }) } updateList.forEach((update) => { if ("contentsHidden" in update) { const container = board.items[update.id] if (container) { setVisibilityRecursively(container, update.contentsHidden ?? false) adjustConnectionVisibility() } } }) return resultItems } function applyModification( items: Record, modification: (items: Record) => void, inplace: boolean, ): Record { const updated = inplace ? items : { ...items } modification(updated) return updated } function applyListModification(list: T[], modification: (list: T[]) => void, inplace: boolean) { const newList = inplace ? list : [...list] modification(newList) return newList } function replaceById(list: T[], replacements: T[]) { replacements.forEach((replacement) => { const index = list.findIndex((item) => item.id === replacement.id) if (index === -1) { throw Error(`Trying to replace nonexisting item ${replacement.id}`) } list[index] = replacement }) } function applyFontSize(items: Record, factor: number, itemIds: Id[], inplace: boolean) { return applyModification( items, (items) => { itemIds.forEach((id) => { const u = items[id] && isTextItem(items[id]) ? (items[id] as TextItem) : null if (u) { items[u.id] = { ...u, fontSize: ((u as TextItem).fontSize || 1) * factor, } } }) }, inplace, ) } function filterItemIdsByPermissions(itemIds: Id[], board: Board, permission: BoardPermission) { return itemIds.filter((id) => nullablePermission(permission)(findItem(board)(id))) } function filterConnectionIdsByPermissions(connectionIds: Id[], board: Board, permission: BoardPermission) { return connectionIds.filter((id) => permission(getConnection(board)(id))) } function filterMoveByPermissions(event: MoveItem, board: Board) { return { ...event, items: event.items.filter((i) => canMove(getItem(board)(i.id))), connections: event.connections.filter((c) => canMove(getConnection(board)(c.id))), } } function filterItemUpdatesByPermissions(updates: ItemUpdate[], board: Board): ItemUpdate[] { type AnyItemKey = keyof Note | keyof Text | keyof Container | keyof Image | keyof Video const propertyToPermissionMapping: Partial> = { align: canChangeTextAlign, color: canChangeShapeAndColor, fontSize: canChangeFont, x: canMove, y: canMove, width: canMove, height: canMove, text: canChangeText, } return updates.filter((update) => { const item = findItem(board)(update.id) if (!item) return false const keys = Object.keys(update) as AnyItemKey[] const permissionFns = keys.map((key) => propertyToPermissionMapping[key]) for (let fn of permissionFns) { if (fn && !fn(item)) { console.log("Deny update", keys) return false } } return true }) } function filterConnectionUpdatesByPermissions(updates: ConnectionUpdate[], board: Board): ConnectionUpdate[] { const propertyToPermissionMapping: Partial> = { from: canMove, to: canMove, fromStyle: canChangeShapeAndColor, toStyle: canChangeShapeAndColor, controlPoints: canMove, } return updates.filter((update) => { const connection = getConnection(board)(update.id) const keys = Object.keys(update) as (keyof Connection)[] const permissionFns = keys.map((key) => propertyToPermissionMapping[key]) for (let fn of permissionFns) { if (fn && !fn(connection)) { console.log("Deny update", keys) return false } } return true }) } function moveItems(board: Board, event: MoveItem, inplace: boolean) { event = filterMoveByPermissions(event, board) const itemMoves: Record = {} const itemsOnBoard = board.items const connectionMovesInEvent = event.connections || [] for (let mainItemMove of event.items) { const { id, x, y, containerId } = mainItemMove const mainItem = itemsOnBoard[id] if (mainItem === undefined) { console.warn("Moving unknown item", id) continue } const xDiff = x - mainItem.x const yDiff = y - mainItem.y for (let movedItem of Object.values(itemsOnBoard)) { const movedId = movedItem.id if (movedId === id || isContainedBy(itemsOnBoard, mainItem)(movedItem)) { const move = { xDiff, yDiff, containerChanged: movedId === id, containerId } itemMoves[movedId] = move } } } const connectionMoves: Record = {} for (let connection of board.connections) { const move = findConnectionMove(connection, itemMoves, itemsOnBoard) if (move) { connectionMoves[connection.id] = move } else { const m = connectionMovesInEvent.find((m) => m.id === connection.id) if (m) { const startPoint = resolveEndpoint(connection.from, board) connectionMoves[connection.id] = { ends: "both", xDiff: m.x - startPoint.x, yDiff: m.y - startPoint.y, } } } } let updatedConnections: Connection[] = board.connections.flatMap((connection) => { const move = connectionMoves[connection.id] if (!move) return connection if (move.ends === "both") { return { ...connection, from: moveEndPoint(connection.from, move), to: moveEndPoint(connection.to, move), controlPoints: connection.controlPoints.map((cp) => moveEndPoint(cp, move)), } as Connection } return rerouteConnection(connection, board) }) const updatedItems = Object.entries(itemMoves).reduce( (items, [id, move]) => { const item = items[id] const updated = { ...item, x: item.x + move.xDiff, y: item.y + move.yDiff } if (move.containerChanged) { updated.containerId = move.containerId if (move.containerId) { const newContainer = board.items[move.containerId] if ( newContainer && (newContainer.hidden || (isContainer(newContainer) && newContainer.contentsHidden)) ) { updated.hidden = true } } } items[id] = updated return items }, inplace ? board.items : { ...board.items }, ) return { ...board, items: updatedItems, connections: updatedConnections, } } type Move = { xDiff: number; yDiff: number } type ItemMove = Move & { containerChanged: boolean; containerId: Id | undefined } type ConnectionMove = (Move & { ends: "both" }) | { ends: "one" } function findConnectionMove( connection: Connection, moves: Record, items: Record, ): ConnectionMove | null { const endPoints = [connection.to, connection.from] let move: Move | null = null let partial = false let hasItemEndPoints = false for (let endPoint of endPoints) { if (isItemEndPoint(endPoint)) { hasItemEndPoints = true const itemId = getEndPointItemId(endPoint) if (moves[itemId]) { move = moves[itemId] } else { // linked to item not being moved -> maybe a partial move partial = true } } } if (!move && !hasItemEndPoints && connection.containerId) { move = moves[connection.containerId] } if (!move) return null if (partial) return { ends: "one" } return { ends: "both", ...move } } function moveEndPoint(endPoint: ConnectionEndPoint, move: Move) { if (isItemEndPoint(endPoint)) { return endPoint // points to an item } const x = endPoint.x + move.xDiff const y = endPoint.y + move.yDiff return { ...endPoint, x, y } } function containedBy(a: Point, b: Rect) { return a.x > b.x && a.y > b.y && a.x < b.x + b.width && a.y < b.y + b.height } ================================================ FILE: common/src/colors.ts ================================================ export const LIGHT_BLUE = "#9FECFC" export const LIGHT_GREEN = "#C8FC87" export const YELLOW = "#FBFC86" export const ORANGE = "#FDDF90" export const PINK = "#FDC4E7" export const LIGHT_PURPLE = "#E0BDFA" export const RED = "#F62A5C" export const BLACK = "#000000" export const LIGHT_GRAY = "#f4f4f6" export const WHITE = "#ffffff" export const TRANSPARENT = "#ffffff00" export const DEFAULT_NOTE_COLOR = YELLOW export const NOTE_COLORS = [ { name: "light-blue", color: LIGHT_BLUE }, { name: "light-green", color: LIGHT_GREEN }, { name: "yellow", color: YELLOW }, { name: "orange", color: ORANGE }, { name: "pink", color: PINK }, { name: "light-purple", color: LIGHT_PURPLE }, { name: "red", color: RED }, { name: "black", color: BLACK }, { name: "light-gray", color: LIGHT_GRAY }, { name: "white", color: WHITE }, { name: "transparent", color: TRANSPARENT }, ] ================================================ FILE: common/src/connection-utils.ts ================================================ import * as _ from "lodash" import { maybeChangeContainerForConnection } from "../../frontend/src/board/item-setcontainer" import { AttachmentLocation, AttachmentSide, Board, Connection, ConnectionEndPoint, ConnectionEndPointToItem, getEndPointItemId, getItem, isItem, isItemEndPoint, Item, ItemAttachmentLocation, Point, } from "./domain" import { centerPoint, containedBy, Rect, subtract } from "./geometry" import { getAngleDeg, Vector2 } from "./vector2" export function resolveEndpoint(e: Point | Item | ConnectionEndPoint, b: Board | Record): Point | Item { if (isItemEndPoint(e)) { return resolveItemEndpoint(e, b) } return e } export function resolveItemEndpoint(e: ConnectionEndPointToItem, b: Board | Record): Item { return getItem(b)(getEndPointItemId(e)) } export function findNearestAttachmentLocationForConnectionNode( i: Point | Item, reference: Point | Item, ): AttachmentLocation { if (!isItem(i)) return { side: "none", point: i } const options: ItemAttachmentLocation[] = findItemAttachmentLocations(i) const from = centerPoint(reference) return withStraightestAngle(options, from)! } function angleDiff(option: ItemAttachmentLocation, from: Point) { const directionFromEndPoint: Vector2 = subtract(from, option.point) const endpointDirection = getEndPointDirection(option.side) const diff = Math.abs(getAngleDeg(directionFromEndPoint) - getAngleDeg(endpointDirection)) if (diff > 180) return 360 - diff return diff } function withStraightestAngle(options: ItemAttachmentLocation[], to: Point) { return _.minBy(options, (p) => angleDiff(p, to)) } function getEndPointDirection(side: AttachmentSide): Vector2 { switch (side) { case "top": return Vector2(0, -1) case "right": return Vector2(1, 0) case "bottom": return Vector2(0, 1) case "left": return Vector2(-1, 0) } } const sides: AttachmentSide[] = ["top", "left", "bottom", "right"] function findItemAttachmentLocations(i: Item): ItemAttachmentLocation[] { return sides.map((side) => findAttachmentLocation(i, side)) } function p(x: number, y: number) { return { x, y } } export function findAttachmentLocation(i: Item, side: AttachmentSide): ItemAttachmentLocation { const margin = 0.1 switch (side) { case "top": return { item: i, side, point: p(i.x + i.width / 2, i.y - margin) } case "left": return { item: i, side, point: p(i.x - margin, i.y + i.height / 2) } case "right": return { item: i, side, point: p(i.x + i.width + margin, i.y + i.height / 2) } case "bottom": return { item: i, side, point: p(i.x + i.width / 2, i.y + i.height + margin) } } } function findMidpoint(fromCoords: AttachmentLocation, toCoords: AttachmentLocation) { const midpoint = { x: mid(fromCoords.point.x, toCoords.point.x), y: mid(fromCoords.point.y, toCoords.point.y), } if (toCoords.side === "left" || toCoords.side === "right") { return { x: midpoint.x, y: mid(midpoint.y, toCoords.point.y), } } if (toCoords.side === "top" || toCoords.side === "bottom") { return { x: mid(midpoint.x, toCoords.point.x), y: midpoint.y, } } if (fromCoords.side === "left" || fromCoords.side === "right") { return { x: midpoint.x, y: mid(midpoint.y, fromCoords.point.y), } } if (fromCoords.side === "top" || fromCoords.side === "bottom") { return { x: mid(midpoint.x, fromCoords.point.x), y: midpoint.y, } } return midpoint } function attachmentLocation2EndPoint(l: AttachmentLocation): ConnectionEndPoint { if (l.side === "none") { return l.point } return { side: l.side, id: l.item.id } } export function rerouteConnection(c: Connection, b: Board): Connection { const resolvedFrom = resolveEndpoint(c.from, b) const resolvedTo = resolveEndpoint(c.to, b) let to = findNearestAttachmentLocationForConnectionNode(resolvedTo, resolvedFrom) const from = findNearestAttachmentLocationForConnectionNode(resolvedFrom, to.point) to = findNearestAttachmentLocationForConnectionNode(resolvedTo, from.point) const rerouted: Connection = { ...c, from: attachmentLocation2EndPoint(from), to: attachmentLocation2EndPoint(to), controlPoints: c.controlPoints.length ? [findMidpoint(from, to)] : [], } const container = maybeChangeContainerForConnection(rerouted, b.items) return { ...rerouted, containerId: container ? container.id : undefined } } function rerouteEndPoint(e: ConnectionEndPoint, from: ConnectionEndPoint, b: Board) { return attachmentLocation2EndPoint( findNearestAttachmentLocationForConnectionNode(resolveEndpoint(e, b), resolveEndpoint(from, b)), ) } export function rerouteByNewControlPoints(c: Connection, controlPoints: Point[], b: Board): Connection { const first = controlPoints[0] return { ...c, from: rerouteEndPoint(c.from, controlPoints[0] || c.to, b), to: rerouteEndPoint(c.to, controlPoints[controlPoints.length - 1] || c.from, b), controlPoints, } } function mid(x: number, y: number) { return (x + y) * 0.5 } export const connectionRect = (b: Board | Record) => (c: Connection): Rect => { const start = resolveEndpoint(c.from, b) const end = resolveEndpoint(c.to, b) const allPoints = [start, ...c.controlPoints, end] const minX = _.min(allPoints.map((p) => p.x))! const maxX = _.max(allPoints.map((p) => p.x))! const minY = _.min(allPoints.map((p) => p.y))! const maxY = _.max(allPoints.map((p) => p.y))! const x = minX if (isNaN(x)) { throw Error("Assertion fail") } const y = minY const width = maxX - minX const height = maxY - minY return { x, y, width, height } } export function isFullyContainedConnection(connection: Connection, item: Item, context: Record | Board) { const start = resolveEndpoint(connection.from, context) const end = resolveEndpoint(connection.to, context) return !isItem(start) && !isItem(end) && containedBy(start, item) && containedBy(end, item) } ================================================ FILE: common/src/domain.ts ================================================ import * as t from "io-ts" import * as uuid from "uuid" import { LocalStorageBoard } from "../../frontend/src/store/board-local-store" import { arrayToRecordById } from "./arrays" import { DEFAULT_NOTE_COLOR, LIGHT_BLUE, PINK, RED } from "./colors" import { Rect } from "./geometry" export type Id = string export type ISOTimeStamp = string export function newISOTimeStamp(): ISOTimeStamp { return new Date().toISOString() } export function optional>(c: T) { return t.union([c, t.undefined, t.null]) } export const CrdtDisabled = undefined export const CrdtEnabled = 1 as const export type CrdtMode = typeof CrdtDisabled | typeof CrdtEnabled export type BoardAttributes = { id: Id name: string width: number height: number accessPolicy?: BoardAccessPolicy crdt?: CrdtMode } export type BoardContents = { items: Record connections: Connection[] } export type Board = BoardAttributes & BoardContents & { serial: Serial } export type BoardStub = Pick & { templateId?: Id } export const AccessLevelCodec = t.union([ t.literal("admin"), t.literal("read-write"), t.literal("read-only"), t.literal("none"), ]) export type AccessLevel = t.TypeOf export const AccessListEntryCodec = t.union([ t.type({ email: t.string, access: optional(AccessLevelCodec), }), t.type({ domain: t.string, access: optional(AccessLevelCodec), }), ]) export type AccessListEntry = t.TypeOf export const BoardAccessPolicyDefinedCodec = t.type({ allowList: t.array(AccessListEntryCodec), publicRead: optional(t.boolean), publicWrite: optional(t.boolean), }) export type BoardAccessPolicyDefined = t.TypeOf export const BoardAccessPolicyCodec = t.union([t.undefined, BoardAccessPolicyDefinedCodec]) export type BoardAccessPolicy = t.TypeOf export type EventUserInfo = UnidentifiedUserInfo | SystemUserInfo | EventUserInfoAuthenticated export type UnidentifiedUserInfo = { nickname: string; userType: "unidentified" } export type SystemUserInfo = { nickname: string; userType: "system" } export type EventUserInfoAuthenticated = { nickname: string userType: "authenticated" name: string email: string userId: string } export type SessionUserInfo = UnidentifiedUserInfo | SystemUserInfo | SessionUserInfoAuthenticated export type SessionUserInfoAuthenticated = { nickname: string userType: "authenticated" name: string email: string picture: string | undefined userId: string domain: string | null } export type UserSessionInfo = SessionUserInfo & { sessionId: Id } export type BoardHistoryEntry = { user: EventUserInfo timestamp: ISOTimeStamp serial?: Serial firstSerial?: Serial } & PersistableBoardItemEvent export type BoardWithHistory = { board: Board; history: BoardHistoryEntry[] } export type CompactBoardHistory = { boardAttributes: BoardAttributes; history: BoardHistoryEntry[] } export const defaultBoardSize = { width: 800, height: 600 } export interface CursorPosition { x: number y: number } export type UserCursorPosition = CursorPosition & { sessionId: Id } export type BoardCursorPositions = Record export type Color = string export type ItemBounds = { x: number; y: number; width: number; height: number; z: number } export type LockState = false | "locked" | "read-only" export type ItemProperties = { id: string; containerId?: string; locked: LockState; hidden?: boolean } & ItemBounds export const ITEM_TYPES = { NOTE: "note", TEXT: "text", IMAGE: "image", VIDEO: "video", CONTAINER: "container", } as const export type ItemType = typeof ITEM_TYPES[keyof typeof ITEM_TYPES] export type QuillDelta = any // TODO: define this properly export type TextItemProperties = ItemProperties & { text: string fontSize?: number align?: Align crdt?: CrdtMode textAsDelta?: QuillDelta } export type NoteShape = "round" | "square" | "rect" | "diamond" export type Note = TextItemProperties & { type: typeof ITEM_TYPES.NOTE color: Color shape: NoteShape | undefined } export type Text = TextItemProperties & { type: typeof ITEM_TYPES.TEXT; color: Color } export type Image = ItemProperties & { type: typeof ITEM_TYPES.IMAGE; assetId: string; src?: string } export type Video = ItemProperties & { type: typeof ITEM_TYPES.VIDEO; assetId: string; src?: string } export type Container = TextItemProperties & { type: typeof ITEM_TYPES.CONTAINER color: Color contentsHidden?: boolean } export type Point = { x: number; y: number } export function Point(x: number, y: number) { return { x, y } } export const isPoint = (u: unknown): u is Point => typeof u === "object" && !!u && "x" in u && "y" in u export type ConnectionEndStyle = "none" | "arrow" | "black-dot" export type Connection = { id: Id from: ConnectionEndPoint controlPoints: Point[] to: ConnectionEndPoint containerId?: string locked: LockState fromStyle: ConnectionEndStyle toStyle: ConnectionEndStyle pointStyle: "none" | "black-dot" action: "connect" | "line" hidden?: boolean } export type ConnectionEndPoint = Point | ConnectionEndPointToItem export type ConnectionEndPointToItem = Id | ConectionEndPointDirectedToItem export type ConectionEndPointDirectedToItem = { id: Id; side: AttachmentSide } export function getEndPointItemId(e: ConnectionEndPointToItem) { if (typeof e === "string") return e return e.id } export function isItemEndPoint(e: ConnectionEndPoint): e is ConnectionEndPointToItem { if (typeof e === "string") return true if ("side" in e) return true return false } export function isDirectedItemEndPoint(e: ConnectionEndPoint): e is ConectionEndPointDirectedToItem { return isItemEndPoint(e) && typeof e === "object" } export type AttachmentSide = "left" | "right" | "top" | "bottom" export type AttachmentLocation = { side: "none"; point: Point } | ItemAttachmentLocation export type ItemAttachmentLocation = { side: AttachmentSide; point: Point; item: Item } export type RenderableConnection = Omit & { from: AttachmentLocation to: AttachmentLocation } export type TextItem = Note | Text | Container export type ColoredItem = Item & { color: Color } export type ShapedItem = Note export type Item = TextItem | Image | Video export type ItemLocks = Record export type RecentBoardAttributes = { id: Id; name: string } export type RecentBoard = RecentBoardAttributes & { opened: ISOTimeStamp; userEmail: string | null } export type BoardEvent = { boardId: Id } export type UIEvent = BoardItemEvent | ClientToServerRequest | LocalUIEvent export type LocalUIEvent = Undo | Redo | SetLocalBoard | GoOnline | BoardLoggedOut | GoOffline | TextFormat export type EventFromServer = BoardHistoryEntry | BoardStateSyncEvent | LoginResponse | AckAddBoard | ServerConfig export type ServerConfig = { action: "server.config" authSupported: boolean assetStorageURL: string crdt: "true" | "false" | "opt-in" | "opt-in-authenticated" } export type Serial = number export type AppEvent = | BoardItemEvent | BoardStateSyncEvent | LocalUIEvent | ClientToServerRequest | LoginResponse | AckAddBoard | ServerConfig export type EventWrapper = { events: AppEvent[] ackId?: string } export type PersistableBoardItemEvent = | AddItem | UpdateItem | MoveItem | DeleteItem | AddConnection | ModifyConnection | DeleteConnection | IncreaseItemFont | DecreaseItemFont | BringItemToFront | BootstrapBoard | RenameBoard | SetBoardAccessPolicy export type BoardInit = InitBoardNew | InitBoardDiff export type TransientBoardItemEvent = LockItem | UnlockItem export type BoardItemEvent = PersistableBoardItemEvent | TransientBoardItemEvent export type BoardStateSyncEvent = | BoardInit | RecentBoardsFromServer | GotBoardLocks | CursorPositions | JoinedBoard | LeftBoard | UserLoggedIn | AckJoinBoard | DeniedJoinBoard | UserInfoUpdate | ActionApplyFailed | AssetPutUrlResponse | Ack | BringAllToMe export type ClientToServerRequest = | CursorMove | AddBoard | LockItem | UnlockItem | JoinBoard | AssociateBoard | DissociateBoard | SetNickname | AssetPutUrlRequest | AuthJWTLogin | UserLoggedIn | AuthLogout | Ping | BringAllToMe export type LoginResponse = | { action: "auth.login.response"; success: false } | { action: "auth.login.response"; success: true; userId: string } export type AddConnection = { action: "connection.add"; boardId: Id; connections: Connection[] } export type ModifyConnection = { action: "connection.modify"; boardId: Id; connections: Connection[] } export type DeleteConnection = { action: "connection.delete"; boardId: Id; connectionIds: Id[] } export type UserLoggedIn = { action: "user.login" name: string email: string picture: string | undefined } export type AuthJWTLogin = { action: "auth.login.jwt" jwt: string } export type AuthLogout = { action: "auth.logout" } export type Ping = { action: "ping" } export type AddItem = { action: "item.add"; boardId: Id; items: Item[]; connections: Connection[] } export type UpdateItem = { action: "item.update"; boardId: Id; items: ItemUpdate[]; connections?: ConnectionUpdate[] } export type Update = Partial & { id: Id } export type ItemUpdate = Update export type ConnectionUpdate = Update export type MoveItem = { action: "item.move" boardId: Id items: { id: Id; x: number; y: number; containerId?: Id | undefined }[] connections: { id: Id; x: number; y: number }[] // Coordinates are for connection start point. } export type IncreaseItemFont = { action: "item.font.increase"; boardId: Id; itemIds: Id[] } export type DecreaseItemFont = { action: "item.font.decrease"; boardId: Id; itemIds: Id[] } export type BringItemToFront = { action: "item.front"; boardId: Id; itemIds: Id[] } export type DeleteItem = { action: "item.delete"; boardId: Id; itemIds: Id[]; connectionIds: Id[] } export type BootstrapBoard = { action: "item.bootstrap"; boardId: Id } & BoardContents export type LockItem = { action: "item.lock"; boardId: Id; itemId: Id } export type UnlockItem = { action: "item.unlock"; boardId: Id; itemId: Id } export type GotBoardLocks = { action: "board.locks"; boardId: Id; locks: ItemLocks } export type AddBoard = { action: "board.add"; payload: Board | BoardStub } export type AckAddBoard = { action: "board.add.ack"; boardId: Id } export type JoinBoard = { action: "board.join"; boardId: Id; initAtSerial?: Serial } export type BringAllToMe = { action: "user.bringAllToMe"; boardId: Id; sessionId: Id; viewRect: Rect; nickname: string } export type AssociateBoard = { action: "board.associate"; boardId: Id; lastOpened: ISOTimeStamp } export type DissociateBoard = { action: "board.dissociate"; boardId: Id } export type SetBoardAccessPolicy = { action: "board.setAccessPolicy" boardId: Id accessPolicy: BoardAccessPolicyDefined } export type AckJoinBoard = { action: "board.join.ack"; boardId: Id } & UserSessionInfo export type DeniedJoinBoard = | { action: "board.join.denied" boardId: Id reason: "unauthorized" | "forbidden" | "not found" } | { action: "board.join.denied" boardId: Id reason: "redirect" wsAddress: string } export type RecentBoardsFromServer = { action: "user.boards"; email: string; boards: RecentBoard[] } export type Ack = { action: "ack"; ackId: string; serials: Record } export type ActionApplyFailed = { action: "board.action.apply.failed" } export type JoinedBoard = { action: "board.joined"; boardId: Id } & UserSessionInfo export type LeftBoard = { action: "board.left"; boardId: Id; sessionId: Id } export type UserInfoUpdate = { action: "userinfo.set" } & UserSessionInfo export type InitBoardNew = { action: "board.init"; board: Board; accessLevel: AccessLevel } export type InitBoardDiff = { action: "board.init.diff" initAtSerial: Serial first: boolean last: boolean recentEvents: BoardHistoryEntry[] boardAttributes: BoardAttributes accessLevel: AccessLevel } export type RenameBoard = { action: "board.rename"; boardId: Id; name: string } export type CursorMove = { action: "cursor.move"; position: CursorPosition; boardId: Id } export type SetNickname = { action: "nickname.set"; nickname: string } export type AssetPutUrlRequest = { action: "asset.put.request"; assetId: string } export type AssetPutUrlResponse = { action: "asset.put.response"; assetId: string; signedUrl: string } export type Undo = { action: "ui.undo" } export type Redo = { action: "ui.redo" } export type TextFormat = { action: "ui.text.format"; itemIds: Id[]; format: "bold" | "italic" | "underline" } export type SetLocalBoard = { action: "ui.board.setLocal" boardId: Id | undefined storedInitialState: LocalStorageBoard | undefined } export type BoardLoggedOut = { action: "ui.board.logged.out"; boardId: Id } export type GoOffline = { action: "ui.offline" } export type GoOnline = { action: "ui.online" } export const CURSOR_POSITIONS_ACTION_TYPE = "c" as const export type CursorPositions = { action: typeof CURSOR_POSITIONS_ACTION_TYPE; p: Record } export const exampleBoard: Board = { id: "default", name: "Test Board", items: arrayToRecordById([ newNote("Hello", PINK, 10, 5), newNote("World", LIGHT_BLUE, 20, 10), newNote("Welcome", RED, 5, 14), ]), connections: [], ...defaultBoardSize, serial: 0, } export function newBoard(name: string, crdt?: CrdtMode, accessPolicy?: BoardAccessPolicy): Board { return { id: uuid.v4(), name, items: {}, accessPolicy, connections: [], ...defaultBoardSize, serial: 0, crdt } } export function newNote( text: string, color: Color = DEFAULT_NOTE_COLOR, x: number = 20, y: number = 20, width: number = 5, height: number = 5, shape: NoteShape = "square", z: number = 0, ): Note { return { id: uuid.v4(), type: "note", text, color, x, y, width, height, z, shape, locked: false } } export function newSimilarNote(note: Note) { return newNote("HELLO", note.color, 20, 20, note.width, note.height, note.shape) } export function newText( crdt: CrdtMode, text: string = "HELLO", x: number = 20, y: number = 20, width: number = 5, height: number = 2, z: number = 0, ): Text { return { id: uuid.v4(), type: "text", text, x, y, width, height, z, color: "none", locked: false, crdt, } } export function newContainer( crdt: CrdtMode, x: number = 20, y: number = 20, width: number = 30, height: number = 20, z: number = 0, ): Container { return { id: uuid.v4(), type: "container", text: "Unnamed area", x, y, width, height, z, color: "white", locked: false, crdt, } } export function newImage( assetId: string, x: number = 20, y: number = 20, width: number = 5, height: number = 5, z: number = 0, ): Image { return { id: uuid.v4(), type: "image", assetId, x, y, width, height, z, locked: false } } export function newVideo( assetId: string, x: number = 20, y: number = 20, width: number = 5, height: number = 5, z: number = 0, ): Video { return { id: uuid.v4(), type: "video", assetId, x, y, width, height, z, locked: false } } export const isBoardItemEvent = (a: AppEvent): a is BoardItemEvent => a.action.startsWith("item.") || a.action.startsWith("connection.") || a.action === "board.rename" || a.action === "board.setAccessPolicy" export const isPersistableBoardItemEvent = (e: any): e is PersistableBoardItemEvent => isBoardItemEvent(e) && !["item.lock", "item.unlock"].includes(e.action) export const isBoardHistoryEntry = (e: AppEvent): e is BoardHistoryEntry => isPersistableBoardItemEvent(e) && !!(e as BoardHistoryEntry).user && !!(e as BoardHistoryEntry).timestamp export const isLocalUIEvent = (e: AppEvent): e is LocalUIEvent => e.action.startsWith("ui.") export const isCursorMove = (e: AppEvent): e is CursorMove => e.action === "cursor.move" export function isSameUser(a: EventUserInfo, b: EventUserInfo) { return a.userType == b.userType && a.nickname == b.nickname } export function isColoredItem(i: Item): i is ColoredItem { return i.type === "note" || i.type === "container" || i.type === "text" } export function isShapedItem(i: Item): i is ShapedItem { return i.type === "note" } export function isTextItem(i: Item): i is TextItem { return i.type === "note" || i.type === "text" || i.type === "container" } export function isNote(i: Item): i is Note { return i.type === "note" } export function isContainer(i: Item): i is Container { return i.type === "container" } export function isText(i: Item): i is Text { return i.type === "text" } export function isItem(i: Item | Point | Connection): i is Item { return "type" in i } export function getItemText(i: Item) { if (isTextItem(i)) return i.text return "" } export function getItemBackground(i: Item) { if (isColoredItem(i)) { return i.color || "white" // Default for legacy containers } return "none" } export function getItemShape(i: Item) { return i.type === "note" && i.shape ? i.shape : "rect" } type NamespacedEvent = T extends { action: `${Namespace}.${string}` } ? T : never export function actionNamespaceIs( ns: Namespace, a: AppEvent, ): a is NamespacedEvent { return a.action.startsWith(ns + ".") } export function getItemIds(e: BoardHistoryEntry | PersistableBoardItemEvent): Id[] { switch (e.action) { case "item.front": case "item.delete": case "item.font.decrease": case "item.font.increase": return e.itemIds case "item.move": return e.items.map((i) => i.id) case "item.update": return e.items.map((i) => i.id) case "item.add": return e.items.map((i) => i.id) case "item.bootstrap": return Object.keys(e.items) case "board.rename": case "board.setAccessPolicy": case "connection.add": case "connection.modify": case "connection.delete": return [] } } export const getItem = (boardOrItems: Board | Record) => (id: Id) => { const item = findItem(boardOrItems)(id) if (!item) throw Error("Item not found: " + id) return item } export const getConnection = (b: Board) => (id: Id) => { const conn = b.connections.find((c) => c.id === id) if (!conn) throw Error("Connection not found: " + id) return conn } export const findItem = (boardOrItems: Board | Record) => (id: Id): Item | null => { const items = getItems(boardOrItems) const item = items[id] return item || null } export const findConnection = (board: Board) => (id: Id) => { const conn = board.connections.find((c) => c.id === id) return conn || null } export function findItemIdsRecursively(ids: Id[], board: Board): Set { const recursiveIds = new Set() const addIdRecursively = (id: Id) => { recursiveIds.add(id) Object.values(board.items).forEach((i) => i.containerId === id && addIdRecursively(i.id)) } ids.forEach(addIdRecursively) return recursiveIds } export function findItemsRecursively(ids: Id[], board: Board): Item[] { const recursiveIds = findItemIdsRecursively(ids, board) return [...recursiveIds].map(getItem(board)) } export const isContainedBy = (boardOrItems: Board | Record, parentCandidate: Item) => ( i: Item, ): boolean => { if (!i.containerId) return false if (i.containerId === parentCandidate!.id) return true const itemsOnBoard = getItems(boardOrItems) const parent = findItem(itemsOnBoard)(i.containerId) if (i.containerId === i.id) throw Error("Self-contained") if (parent == i) throw Error("self parent") if (!parent) return false // Don't fail here, because when folding create+move, the action is run in an incomplete board context return isContainedBy(boardOrItems, parentCandidate)(parent) } const isBoard = (u: unknown): u is Board => typeof u === "object" && !!u && "items" in u const getItems = (boardOrItems: Board | Record) => isBoard(boardOrItems) ? boardOrItems.items : boardOrItems export function isBoardEmpty(board: Board) { return board.connections.length === 0 && Object.values(board.items).length === 0 } export function getBoardAttributes(board: Board, userInfo?: EventUserInfo): BoardAttributes { const accessPolicy = board.accessPolicy ? userInfo && userInfo.userType === "authenticated" ? board.accessPolicy : { ...board.accessPolicy, allowList: [] } // Anonymize access policy for anonymous users : undefined return { id: board.id, name: board.name, width: board.width, height: board.height, accessPolicy, } } export const BOARD_ITEM_BORDER_MARGIN = 0.5 export function checkBoardAccess(accessPolicy: BoardAccessPolicy | undefined, userInfo: SessionUserInfo): AccessLevel { if (!accessPolicy) return "read-write" let accessLevel: AccessLevel = accessPolicy.publicWrite ? "read-write" : accessPolicy.publicRead ? "read-only" : "none" if (userInfo.userType === "unidentified" || userInfo.userType === "system") { return accessLevel } const email = userInfo.email const domain = userInfo.domain const defaultAccess = "read-write" for (let entry of accessPolicy.allowList) { const nextLevel = "email" in entry && entry.email === email ? entry.access || defaultAccess : "domain" in entry && domain === entry.domain ? entry.access || defaultAccess : "none" accessLevel = combineAccessLevels(accessLevel, nextLevel) } return accessLevel } function combineAccessLevels(a: AccessLevel, b: AccessLevel): AccessLevel { if (a === "admin" || b === "admin") return "admin" if (a === "read-write" || b === "read-write") return "read-write" if (a === "read-only" || b === "read-only") return "read-only" return "none" } export function canRead(a: AccessLevel) { return a !== "none" } export function canWrite(a: AccessLevel) { return a === "read-write" || a === "admin" } export type Align = "TL" | "TC" | "TR" | "ML" | "MC" | "MR" | "BL" | "BC" | "BR" export function getAlign(item: TextItem) { return item.align ?? (isNote(item) ? "MC" : "TL") } export type HorizontalAlign = "left" | "center" | "right" export function getHorizontalAlign(align: Align): HorizontalAlign { switch (align) { case "TL": case "ML": case "BL": return "left" case "TC": case "MC": case "BC": return "center" case "TR": case "MR": case "BR": return "right" } console.log("Unknown align", align) return "center" } export type VerticalAlign = "top" | "middle" | "bottom" export function getVerticalAlign(align: Align): VerticalAlign { switch (align) { case "TL": case "TC": case "TR": return "top" case "ML": case "MC": case "MR": return "middle" case "BL": case "BC": case "BR": return "bottom" } console.log("Unknown align", align) return "middle" } export function setHorizontalAlign(item: I, a: HorizontalAlign): ItemUpdate { const letter = a === "left" ? "L" : a === "center" ? "C" : "R" const align = `${getAlign(item)[0]}${letter}` return { id: item.id, align } as ItemUpdate } export function setVerticalAlign(item: I, a: VerticalAlign): ItemUpdate { const letter = a === "top" ? "T" : a === "middle" ? "M" : "B" const align = `${letter}${getAlign(item)[1]}` return { id: item.id, align } as ItemUpdate } ================================================ FILE: common/src/geometry.ts ================================================ import { Point } from "./domain" export const origin = { x: 0, y: 0 } export type Coordinates = { x: number; y: number } export type Dimensions = { width: number; height: number } export type Rect = { x: number; y: number; width: number; height: number } export const ZERO_RECT = { x: 0, y: 0, height: 0, width: 0 } export function add(a: Coordinates, b: Coordinates) { return { x: a.x + b.x, y: a.y + b.y } } export function subtract(a: Coordinates, b: Coordinates) { return { x: a.x - b.x, y: a.y - b.y } } export function negate(a: Coordinates) { return { x: -a.x, y: -a.y } } export function multiply(a: Coordinates, factor: number) { return { x: a.x * factor, y: a.y * factor } } export function overlaps(a: Rect, b: Rect) { if (b.x >= a.x + a.width) return false if (b.x + b.width <= a.x) return false if (b.y >= a.y + a.height) return false if (b.y + b.height <= a.y) return false return true } export function equalRect(a: Rect, b: Rect) { return a.x == b.x && a.y == b.y && a.width == b.width && a.height == b.height } export function distance(a: Coordinates, b: Coordinates) { return Math.sqrt(Math.pow(a.x - b.x, 2) + Math.pow(a.y - b.y, 2)) } export function containedBy(a: Point, b: Rect): boolean export function containedBy(a: Rect, b: Rect): boolean export function containedBy(a: Rect | Point, b: Rect) { if ("width" in a) { return a.x > b.x && a.y > b.y && a.x + a.width < b.x + b.width && a.y + a.height < b.y + b.height } else { return a.x > b.x && a.y > b.y && a.x < b.x + b.width && a.y < b.y + b.height } } export function rectFromPoints(a: Coordinates, b: Coordinates) { const x = Math.min(a.x, b.x) const y = Math.min(a.y, b.y) const width = Math.abs(a.x - b.x) const height = Math.abs(a.y - b.y) return { x, y, width, height } } export function isRect(i: Point | Rect): i is Rect { return "width" in i } export function centerPoint(i: Point | Rect) { if (isRect(i)) { return { x: i.x + i.width / 2, y: i.y + i.height / 2, } } return i } ================================================ FILE: common/src/migration.test.ts ================================================ import { describe, expect, it } from "vitest" import { arrayToRecordById } from "./arrays" import { AddConnection, AddItem, Board, Connection, ConnectionEndPoint, DeleteConnection, DeleteItem, ModifyConnection, MoveItem, defaultBoardSize, exampleBoard, newISOTimeStamp, } from "./domain" import { migrateBoard, migrateEvent } from "./migration" describe("Migration", () => { describe("Migrate board", () => { it("Migrates boards correctly", () => { const containedNoteWithNoType = { id: "a", x: 1, y: 1, width: 1, height: 1, text: "note a", color: "yellow", } const containedNote2 = { type: "note", id: "b", x: 2, y: 2, width: 1, height: 1, text: "note b", color: "yellow", } const unContainedNoteWithNoDimensions = { type: "note", id: "c", x: 3, y: 3, text: "note c", color: "yellow", } const oldFormContainerWithItemsAndNoText = { type: "container", id: "d", x: 0, y: 0, width: 5, height: 5, items: ["a", "b"], } const legacyBoard: any = { id: "foo", name: "board with no size, where containers have items property and items do not have containerId property", items: [ containedNoteWithNoType, containedNote2, unContainedNoteWithNoDimensions, oldFormContainerWithItemsAndNoText, ], } const board = migrateBoard(legacyBoard) expect(board).toEqual({ ...legacyBoard, ...defaultBoardSize, connections: [], items: arrayToRecordById([ { ...containedNoteWithNoType, type: "note", containerId: "d", z: 0, locked: false }, { ...containedNote2, containerId: "d", z: 0, locked: false }, { ...unContainedNoteWithNoDimensions, width: 5, height: 5, z: 0, locked: false }, { type: "container", id: "d", x: 0, y: 0, width: 5, height: 5, z: 0, text: "", locked: false }, ]), }) }) it("Removes broken connections", () => { const borkenEndpoint: ConnectionEndPoint = { id: "asdf", side: "bottom" } const b: Board = { ...exampleBoard, connections: [ { from: borkenEndpoint, to: borkenEndpoint, id: "asfdoi", controlPoints: [], fromStyle: "none", toStyle: "none", pointStyle: "none", action: "connect", locked: false, }, ], } expect(migrateBoard(b)).toEqual(exampleBoard) }) it("Sets connection end styles", () => { const b: Board = { ...exampleBoard, connections: [{ from: { x: 0, y: 0 }, to: { x: 0, y: 0 }, id: "asfdoi", controlPoints: [] } as any], } expect(migrateBoard(b)).toEqual({ ...exampleBoard, connections: [ { from: { x: 0, y: 0 }, to: { x: 0, y: 0 }, id: "asfdoi", controlPoints: [], fromStyle: "black-dot", toStyle: "arrow", pointStyle: "black-dot", action: "connect", locked: false, } as Connection, ], }) }) }) describe("Migrate event", () => { const headers = { user: { userType: "unidentified", nickname: "asdf" }, timestamp: newISOTimeStamp(), boardId: "", } it("connection.add", () => { const connection = { from: "a", to: "b", controlPoints: [], id: "c" } expect( (migrateEvent({ ...headers, action: "connection.add", connection, } as any) as AddConnection).connections, ).toEqual([connection]) expect( (migrateEvent({ ...headers, action: "connection.add", connection: [connection], } as any) as AddConnection).connections, ).toEqual([connection]) }) it("connection.modify", () => { const connection = { from: "a", to: "b", controlPoints: [], id: "c" } expect( (migrateEvent({ ...headers, action: "connection.modify", connection, } as any) as ModifyConnection).connections, ).toEqual([connection]) expect( (migrateEvent({ ...headers, action: "connection.modify", connection: [connection], } as any) as ModifyConnection).connections, ).toEqual([connection]) }) it("connection.delete", () => { const connectionId = "c" expect( (migrateEvent({ ...headers, action: "connection.delete", connectionId, } as any) as DeleteConnection).connectionIds, ).toEqual([connectionId]) expect( (migrateEvent({ ...headers, action: "connection.delete", connectionId: [connectionId], } as any) as DeleteConnection).connectionIds, ).toEqual([connectionId]) }) it("item.move", () => { expect( (migrateEvent({ ...headers, action: "item.move", items: [], } as any) as MoveItem).connections, ).toEqual([]) }) it("item.delete", () => { expect( (migrateEvent({ ...headers, action: "item.delete", itemIds: [], } as any) as DeleteItem).connectionIds, ).toEqual([]) }) it("item.add", () => { expect( (migrateEvent({ ...headers, action: "item.add", items: [], } as any) as AddItem).connections, ).toEqual([]) }) }) }) ================================================ FILE: common/src/migration.ts ================================================ import { isArray } from "lodash" import { arrayToRecordById, toArray } from "./arrays" import { resolveEndpoint } from "./connection-utils" import { Board, BoardHistoryEntry, Container, Connection, defaultBoardSize, Id, Item, Serial } from "./domain" export function mkBootStrapEvent(boardId: Id, snapshot: Board, serial: Serial = 1) { return { action: "item.bootstrap", boardId, items: snapshot.items, connections: snapshot.connections, timestamp: new Date().toISOString(), user: { nickname: "admin", userType: "system" }, serial, } as BoardHistoryEntry } export function migrateBoard(origBoard: Board) { const board = { ...origBoard } const items: Item[] = [] const width = Math.max(board.width || 0, defaultBoardSize.width) const height = Math.max(board.height || 0, defaultBoardSize.height) for (const item of Object.values(board.items)) { if (items.find((i) => i.id === item.id)) { console.warn("Duplicate item", item, "found on table", board.name) } else { items.push(migrateItem(item, items, board.items)) } } if (board.accessPolicy) { if (!board.accessPolicy.allowList.some((e) => e.access === "admin")) { console.log(`No board admin for board ${board.id} -> mapping all read-write users as admins`) board.accessPolicy.allowList = board.accessPolicy.allowList.map((e) => ({ ...e, access: e.access === "read-write" ? "admin" : e.access, })) } } const connections = (board.connections ?? []) .filter((c) => { try { resolveEndpoint(c.from, board) resolveEndpoint(c.to, board) } catch (e) { console.error(`Error resolving connection ${JSON.stringify(c)}`) return false } return true }) .map(migrateConnection) return { ...board, connections, width, height, items: arrayToRecordById(items) } } function migrateConnection(c: Connection): Connection { c = c.fromStyle && c.fromStyle !== ("white-dot" as any) && c.toStyle && c.pointStyle ? c : { ...c, fromStyle: "black-dot", toStyle: "arrow", pointStyle: "black-dot" } c = c.action !== undefined ? c : { ...c, action: "connect" } c = c.locked !== undefined ? c : { ...c, locked: false } return c } function migrateItem(item: Item, migratedItems: Item[], boardItems: Record): Item { const { width, height, z, type, locked, ...rest } = item // Force type, width and height for all items let fixedItem = { type: type || "note", width: width || 5, height: height || 5, z: z || 0, locked: locked || false, ...rest, } as Item if (fixedItem.type === "text") { fixedItem.color ??= "none" } if (fixedItem.type === "container") { let container = fixedItem as Container & { items?: string[] } // Force container to have text container.text = container.text || "" // If container had items property, migrate each corresponding item to have containerId of that container instead if (container.items) { const ids = container.items delete container.items ids.forEach((i) => { const containedItem = migratedItems.find((mi) => mi.id === i) || boardItems[i] containedItem && (containedItem.containerId = container.id) }) } } return fixedItem } export function migrateEvent(event: BoardHistoryEntry): BoardHistoryEntry { if (event.action === "connection.add") { if (!isArray(event.connections)) { return { ...event, connections: toArray((event as any).connection) } } } else if (event.action === "connection.modify") { if (!isArray(event.connections)) { return { ...event, connections: toArray((event as any).connection) } } } else if (event.action === "connection.delete") { if (!isArray(event.connectionIds)) { return { ...event, connectionIds: toArray((event as any).connectionId) } } } else if (event.action === "item.move") { if (!event.connections) { return { ...event, connections: [] } } } else if (event.action === "item.delete") { if (!event.connectionIds) { return { ...event, connectionIds: [] } } } else if (event.action === "item.add") { if (!event.connections) { return { ...event, connections: [] } } } return event } ================================================ FILE: common/src/sets.ts ================================================ export function toggleInSet(item: T, set: Set) { if (set.has(item)) { return new Set([...set].filter((i) => i !== item)) } return new Set([...set].concat(item)) } export function difference(setA: Set, setB: Set) { let _difference = new Set(setA) for (let elem of setB) { _difference.delete(elem) } return _difference } export const emptySet = () => new Set() ================================================ FILE: common/src/sleep.ts ================================================ export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(() => resolve(undefined), ms)) } ================================================ FILE: common/src/vector2.ts ================================================ export type Vector2 = { x: number; y: number } export function Vector2(x: number, y: number) { return { x, y } } export function getAngleRad(v: Vector2) { const unit = withLength(v, 1) return Math.atan2(unit.y, unit.x) } export function getAngleDeg(v: Vector2) { return radToDeg(getAngleRad(v)) } export function getLength(v: Vector2) { return Math.sqrt(v.x * v.x + v.y * v.y) } export function withLength(v: Vector2, newLength: number) { return multiply(v, newLength / getLength(v)) } export function multiply(v: Vector2, multiplier: number) { return Vector2(v.x * multiplier, v.y * multiplier) } export function add(v: Vector2, other: Vector2) { return Vector2(v.x + other.x, v.y + other.y) } export function rotateRad(v: Vector2, radians: number) { var length = getLength(v) var currentRadians = getAngleRad(v) var resultRadians = radians + currentRadians var rotatedUnit = { x: Math.cos(resultRadians), y: Math.sin(resultRadians) } return withLength(rotatedUnit, length) } export function rotateDeg(v: Vector2, degrees: number) { return rotateRad(v, degToRad(degrees)) } export function degToRad(degrees: number) { return (degrees * 2 * Math.PI) / 360 } export function radToDeg(rad: number) { return (rad * 360) / 2 / Math.PI } ================================================ FILE: cypress.json ================================================ { "pluginsFile": false, "supportFile": false, "fixturesFolder": false, "retries": 2 } ================================================ FILE: docker-compose.yaml ================================================ version: "3.1" services: db: image: postgres:12 restart: always ports: - 13338:5432 environment: POSTGRES_USER: r-board POSTGRES_PASSWORD: secret keycloak-db: image: postgres:12 restart: always ports: - 13339:5432 environment: POSTGRES_USER: keycloak POSTGRES_PASSWORD: secret volumes: - ./keycloak/keycloak-db.dump:/docker-entrypoint-initdb.d/keycloak-db.dump.sql keycloak: depends_on: - keycloak-db image: quay.io/keycloak/keycloak:22.0.5 command: start-dev --db postgres --db-url jdbc:postgresql://keycloak-db/keycloak --db-username keycloak --db-password secret ports: - 8080:8080 environment: - KEYCLOAK_ADMIN=admin - KEYCLOAK_ADMIN_PASSWORD=admin - DATABASE_URL=postgres://keycloak:secreto@keycloak-db:5432/keycloak ================================================ FILE: frontend/.sassrc ================================================ { "includePaths": ["node_modules"] } ================================================ FILE: frontend/esbuild.js ================================================ require("dotenv").config() const sass = require("sass") const path = require("path") const fs = require("fs") const esbuild = require("esbuild") const rimraf = require("rimraf") const chokidar = require("chokidar") const mode = process.argv[2] if (!mode) { throw Error("Specify 'build' or 'watch' as argument") } const stubImportsPlugin = (paths) => { return { name: "stub-imports", setup(build) { const regex = new RegExp(`^(${paths.join("|")})\$`) build.onResolve({ filter: regex }, (args) => ({ path: args.path, namespace: "stub-imports-namespace", })) build.onLoad({ filter: /.*/, namespace: "stub-imports-namespace" }, () => ({ contents: "{}", loader: "json", })) }, } } const sassPlugin = { name: "sass", setup(build) { build.onResolve({ filter: /(\.svg|\.png)$/ }, (args) => { return { path: path.resolve(CWD, "src", args.path), } }) build.onResolve({ filter: /\.scss$/ }, (args) => { return { path: path.resolve(args.resolveDir, args.path), namespace: "sass", } }) build.onLoad({ filter: /.*/, namespace: "sass" }, (args) => { let compiled = sass.renderSync({ file: args.path }) return { contents: compiled.css.toString(), loader: "css", } }) }, } const CWD = process.cwd() const DIST_FOLDER = path.resolve(CWD, "dist") const envFallback = (envVar, fb = null) => (envVar ? `"${envVar}"` : `${fb}`) async function build() { if (fs.existsSync(DIST_FOLDER)) rimraf.sync(DIST_FOLDER) const randomString = Math.random().toString(36).slice(2) const outfile = path.resolve(CWD, `dist/bundle.${randomString}.js`) const metafile = path.resolve(CWD, `dist/bundle.${randomString}.meta.json`) const now = Date.now() await esbuild.build({ entryPoints: [path.resolve(CWD, "src", "index.tsx")], bundle: true, minify: mode !== "watch", outfile, metafile, sourcemap: true, platform: "browser", plugins: [sassPlugin, stubImportsPlugin(["path"])], loader: { ".png": "file", ".svg": "file" }, define: { "process.env.NODE_ENV": envFallback(process.env.NODE_ENV, `"development"`), }, }) fs.writeFileSync( path.resolve(CWD, "dist/index.html"), fs .readFileSync(path.resolve(CWD, "index.tmpl.html"), "utf8") .replace("JAVASCRIPT_BUNDLE", `/bundle.${randomString}.js`) .replace("CSS_BUNDLE", `/bundle.${randomString}.css`), ) console.log(`Frontend build done, took ${Date.now() - now} ms`) } if (mode === "build") { build().catch((e) => !console.error(e) && process.exit(1)) } else if (mode === "watch") { build() .catch((e) => console.error(e)) .then(() => { chokidar .watch([path.resolve(CWD, "src"), path.resolve(CWD, "../common/src")], { ignoreInitial: true }) .on("all", (...arg) => { build().catch((e) => console.error(e)) }) }) } else { throw Error("Unknown mode: " + mode) } ================================================ FILE: frontend/index.tmpl.html ================================================ OurBoard
================================================ FILE: frontend/package.json ================================================ { "name": "rboard-frontend", "version": "1.0.0", "main": "index.js", "license": "MIT", "dependencies": { "@types/cookie": "^0.4.0", "@types/email-validator": "^1.0.6", "@types/js-cookie": "^3.0.6", "@types/lodash": "^4.14.161", "@types/md5": "^2.2.1", "@types/path-to-regexp": "^1.7.0", "@types/pretty-ms": "^5.0.1", "@types/quill": "^2.0.14", "@types/ramda": "^0.27.40", "@types/sanitize-html": "^1.27.1", "bezier-js": "^4.0.3", "binpackingjs": "^3.0.2", "cookie": "^0.4.1", "email-validator": "^2.0.4", "esbuild": "^0.8.57", "fp-ts": "^2.9.5", "harmaja": "^0.24", "harmaja-router": "^0.3.3", "io-ts": "^2.2.15", "js-cookie": "^3.0.5", "jsonwebtoken": "^8.5.1", "jwt-decode": "^3.1.2", "localforage": "^1.9.0", "lodash": "^4.17.20", "lonna": "^0.12.2", "md5": "^2.3.0", "path-to-regexp": "^6.2.0", "pretty-ms": "^7.0.1", "quill": "^1.3.7", "quill-cursors": "^4.0.2", "ramda": "^0.27.1", "rimraf": "^3.0.2", "sanitize-html": "^2.3.2", "sass": "^1.32.8", "uuid": "^8.3.0", "y-indexeddb": "^9.0.12", "y-quill": "^0.1.5", "y-websocket": "^1.5.3", "yjs": "^13.6.12" }, "scripts": { "build": "node esbuild.js build", "watch": "node esbuild.js watch", "tsc:watch": "tsc --watch --noEmit" }, "devDependencies": { "@types/jsonwebtoken": "^8.5.0", "@types/uuid": "^8.3.0", "chokidar": "^3.5.1", "core-js": "^3.8.3", "cssnano": "^4.1.10", "dotenv": "^8.2.0", "nodemon": "^2.0.4", "npm-run-all": "^4.1.5", "typescript": "^5.3" }, "browserslist": [ "since 2017-06" ] } ================================================ FILE: frontend/src/app.scss ================================================ @import "style/variables.scss"; @import "style/global.scss"; @import "style/dashboard.scss"; @import "style/utils.scss"; @import "style/board.scss"; @import "style/tool-layer.scss"; @import "style/header.scss"; @import "style/modal.scss"; @import "style/sharing-modal.scss"; @import "style/user-info-modal.scss"; ================================================ FILE: frontend/src/board/BoardView.tsx ================================================ import * as H from "harmaja" import { componentScope, h, ListView } from "harmaja" import * as L from "lonna" import { Board, canWrite, findConnection, findItem, getConnection, Id, Image, Item, newNote, Note, Video, } from "../../../common/src/domain" import { isFirefox } from "../components/browser" import { ModalContainer } from "../components/ModalContainer" import { onClickOutside } from "../components/onClickOutside" import { isEmbedded } from "../embedding" import { AssetStore } from "../store/asset-store" import { BoardState, BoardStore, Dispatch } from "../store/board-store" import { CursorsStore } from "../store/cursors-store" import { UserSessionState } from "../store/user-session-store" import { boardCoordinateHelper } from "./board-coordinates" import { boardDragHandler } from "./board-drag" import { BoardFocus, getSelectedItemIds, getSelectedItem, getSelectedItems, noFocus, getSelectedConnectionIds, } from "./board-focus" import { boardScrollAndZoomHandler } from "./board-scroll-and-zoom" import { BoardToolLayer } from "./toolbars/BoardToolLayer" import { ConnectionsView } from "./ConnectionsView" import { ContextMenuView } from "./contextmenu/ContextMenuView" import { CursorsView } from "./CursorsView" import * as G from "../../../common/src/geometry" import { imageUploadHandler, imageDropHandler } from "./image-upload" import { ImageView } from "./ImageView" import { itemCreateHandler } from "./item-create" import { cutCopyPasteHandler } from "./item-cut-copy-paste" import { itemDeleteHandler } from "./item-delete" import { itemDuplicateHandler } from "./item-duplicate" import { itemMoveWithArrowKeysHandler } from "./item-move-with-arrow-keys" import { itemSelectAllHandler } from "./item-select-all" import { withCurrentContainer } from "./item-setcontainer" import { itemUndoHandler } from "./item-undo-redo" import { ItemView } from "./ItemView" import { installKeyboardShortcut, plainKey } from "./keyboard-shortcuts" import { RectangularDragSelection } from "./RectangularDragSelection" import { SelectionBorder } from "./SelectionBorder" import { synchronizeFocusWithServer } from "./synchronize-focus-with-server" import { ToolController } from "./tool-selection" import { BoardViewHeader } from "./header/BoardViewHeader" import { VideoView } from "./VideoView" import { startConnecting } from "./item-connect" import { emptySet } from "../../../common/src/sets" import { installZoomKeyboardShortcuts } from "./zoom-shortcuts" import { itemHideContentsHandler } from "./item-hide-contents" const emptyNote = newNote("") export const BoardView = ({ boardId, cursors, boardStore, sessionState, assets, dispatch, }: { boardId: string cursors: CursorsStore boardStore: BoardStore sessionState: L.Property assets: AssetStore dispatch: Dispatch }) => { const boardState = boardStore.state const board = boardState.pipe( L.map((s: BoardState) => s.board!), L.filter((b: Board) => !!b, componentScope()), ) const accessLevel = L.view(boardState, "accessLevel") const locks = L.view(boardState, (s) => s.locks) const sessionId = L.view(sessionState, (s) => s.sessionId) const sessions = L.view(boardState, (s) => s.users) const zoom = L.atom({ zoom: 1, quickZoom: 1 }) const containerElement = L.atom(null) const scrollElement = L.atom(null) const boardElement = L.atom(null) const latestNoteId = L.atom(null) const latestNote = L.view(latestNoteId, board, (id, b) => { const note = id ? findItem(b)(id) : null return (note as Note) || emptyNote }) const latestConnectionId = L.atom(null) const latestConnection = L.view(latestConnectionId, board, (id, b) => { return id ? findConnection(b)(id) : null }) const focus = synchronizeFocusWithServer(board, locks, sessionId, dispatch) const coordinateHelper = boardCoordinateHelper(containerElement, scrollElement, boardElement, zoom) const toolController = ToolController() const tool = toolController.tool let previousFocus: BoardFocus | null = null focus.forEach((f) => { const previousIDs = previousFocus && getSelectedItemIds(previousFocus) const itemIds = [...getSelectedItemIds(f)].filter((id) => !previousIDs || !previousIDs.has(id)) previousFocus = f if (itemIds.length > 0) { dispatch({ action: "item.front", boardId: board.get().id, itemIds }) const item = getSelectedItem(board.get())(f) if (item && item.type === "note") { latestNoteId.set(item.id) } } const connectionId = [...getSelectedConnectionIds(f)][0] if (connectionId && getConnection(board.get())(connectionId)?.action === "connect") { latestConnectionId.set(connectionId) } }) tool.pipe(L.changes).forEach((tool) => { if (tool !== "note" && tool !== "container" && tool !== "text" && focus.get().status === "adding") { focus.set(noFocus) } }) const doOnUnmount: Function[] = [] const itemsList = L.view(L.view(board, "items"), Object.values) function onURL(assetId: string, url: string) { itemsList.get().forEach((i) => { if ((i.type === "image" || i.type === "video") && i.assetId === assetId && i.src != url) { dispatch({ action: "item.update", boardId, items: [{ id: i.id, src: url }] }) } }) } const uploadImageFile = imageUploadHandler(assets, coordinateHelper, onAdd, onURL) doOnUnmount.push( cutCopyPasteHandler(board, boardStore.crdtStore, focus, coordinateHelper, dispatch, uploadImageFile), ) const zoomControls = boardScrollAndZoomHandler( board, boardElement, scrollElement, zoom, coordinateHelper, toolController, ) const { viewRect } = zoomControls boardStore.eventsFromServer.forEach((e) => { if (e.action === "user.bringAllToMe") { console.log(`Following user ${e.nickname}`, e) viewRect.set(e.viewRect) } }) imageDropHandler(boardElement, assets, focus, uploadImageFile) itemCreateHandler(board, focus, latestNote, boardElement, onAdd) itemDeleteHandler(boardId, dispatch, focus) itemDuplicateHandler(board, boardStore.crdtStore, dispatch, focus) itemHideContentsHandler(board, focus, dispatch) itemMoveWithArrowKeysHandler(board, dispatch, focus) itemUndoHandler(dispatch) itemSelectAllHandler(board, focus) installZoomKeyboardShortcuts(zoomControls) installKeyboardShortcut( (e) => e.key === "Escape", () => { toolController.useDefaultTool() focus.set(noFocus) }, ) installKeyboardShortcut(plainKey("c"), () => toolController.tool.set("connect")) installKeyboardShortcut(plainKey("l"), () => toolController.tool.set("line")) L.fromEvent(window, "click") .pipe(L.applyScope(componentScope())) .forEach((event) => { if (!boardElement.get()!.contains(event.target as Node)) { // Click outside => reset selection focus.set(noFocus) } }) onClickOutside(boardElement, () => { focus.set(noFocus) }) function onClick(e: JSX.UIEvent) { const f = focus.get() if (f.status === "connection-adding") { toolController.useDefaultTool() } else if (f.status === "adding") { onAdd(f.item) } else { if (e.target === boardElement.get()) { if (tool.get() === "connect") { startConnecting( board, coordinateHelper, latestConnection, dispatch, toolController, focus, coordinateHelper.currentBoardCoordinates.get(), ) } else { focus.set(noFocus) } } } } function onAdd(item: Item) { toolController.useDefaultTool() const point = coordinateHelper.currentBoardCoordinates.get() const { x, y } = item.type !== "container" ? G.add(point, { x: -item.width / 2, y: -item.height / 2 }) : point item = withCurrentContainer({ ...item, x, y }, board.get()) dispatch({ action: "item.add", boardId, items: [item], connections: [] }) if (item.type === "note" || item.type === "text") { focus.set({ status: "editing", itemId: item.id }) } else { focus.set({ status: "selected", itemIds: new Set([item.id]), connectionIds: emptySet() }) } } coordinateHelper.currentBoardCoordinates.pipe(L.throttle(30)).forEach((position) => { dispatch({ action: "cursor.move", position, boardId }) }) const { selectionRect } = boardDragHandler({ ...{ board, boardElem: boardElement, coordinateHelper, latestConnection, focus, toolController, dispatch, }, }) H.onUnmount(() => { doOnUnmount.forEach((fn) => fn()) }) const boardAccessStatus = L.view(boardState, (s) => s.status) const quickZoom = L.view(zoom, "quickZoom") const mainZoom = L.view(zoom, "zoom") const borderContainerStyle = L.combineTemplate({ width: L.view(board, quickZoom, (b) => b.width + "em"), height: L.view(board, quickZoom, (b) => b.height + "em"), fontSize: L.view(mainZoom, (z) => z + "em"), transform: L.view(quickZoom, (z) => { const percentTranslate = ((z - 1) / 2) * 100 return `translate(${percentTranslate}%, ${percentTranslate}%) scale(${z})` }), "will-change": "transform, fontSize", }) const className = L.view( boardAccessStatus, (status) => `board-container ${isEmbedded() ? "embedded" : ""} ${status}`, ) const items = L.view(L.view(board, "items"), Object.values, (items) => items.filter((i) => !i.hidden)) const selectedItems = L.view(board, focus, (b, f) => getSelectedItems(b)(f)) const modalContent = L.atom(null) L.interval(100, null, componentScope()).forEach(() => { if (window.scrollY !== 0 || window.scrollX !== 0) { if (focus.get().status !== "editing") { // Reset scroll position when not editing. At least iOS seems to need this. It's vital that our scroll pos stays at origin. window.scrollTo(0, 0) } } }) return (
s.status === "online"), crdtStore: boardStore.crdtStore, }} />
"board " + t)} draggable={isFirefox ? L.view(focus, (f) => f.status !== "editing") : true} ref={boardElement.set} onClick={onClick} onTouchEnd={onClick} > i.id} /> {L.view(tool, (t) => t === "connect" ? null : ( i.id} /> ), )}
) function renderSelectionBorder(id: string, item: L.Property) { return } function renderItem(id: string, item: L.Property) { const isLocked = L.combineTemplate({ locks, sessionId }).pipe( L.map(({ locks, sessionId }) => !!locks[id] && locks[id] !== sessionId), ) return L.view(L.view(item, "type"), (t) => { switch (t) { case "container": case "text": case "note": return ( , isLocked, focus, coordinateHelper, latestConnection, dispatch, toolController, accessLevel, boardStore, }} /> ) case "image": return ( , assets, board, isLocked, focus, toolController, coordinateHelper, latestConnection, dispatch, }} /> ) case "video": return ( , assets, board, isLocked, focus, toolController, coordinateHelper, latestConnection, dispatch, }} /> ) default: throw Error("Unsupported item: " + t) } }) } } ================================================ FILE: frontend/src/board/BoardViewMessage.tsx ================================================ import { h } from "harmaja" import * as L from "lonna" import { Board } from "../../../common/src/domain" import { signIn } from "../google-auth" import { BoardAccessStatus } from "../store/board-store" import { UserSessionState } from "../store/user-session-store" export const BoardViewMessage = ({ boardAccessStatus, sessionState, board, }: { boardAccessStatus: L.Property sessionState: L.Property board: L.Property }) => { // TODO: login may be disabled due to Incognito mode or other reasons return L.combine(boardAccessStatus, L.view(sessionState, "status"), (s: BoardAccessStatus, sessionStatus) => { if (s === "not-found") { return (

Board not found. A typo, maybe?

) } if (s === "denied-permanently") { return (

Sorry, access denied. Click here to sign in with another account.

) } if (s === "login-required") { if (sessionStatus === "login-failed") { return (
Something went wrong with logging in. Click here to try again.
) } return (
This board is for authorized users only. Click here to sign in.
) } return null }) } ================================================ FILE: frontend/src/board/CollaborativeTextView.tsx ================================================ import { componentScope, h } from "harmaja" import * as L from "lonna" import Quill from "quill" import QuillCursors from "quill-cursors" import { QuillBinding } from "y-quill" import { AccessLevel, Board, LocalUIEvent, TextItem, canWrite, getAlign, getHorizontalAlign, getItemBackground, } from "../../../common/src/domain" import { CRDTStore } from "../store/crdt-store" import { getAlignItems } from "./ItemView" import { BoardFocus } from "./board-focus" import { contrastingColor } from "./contrasting-color" import { preventDoubleClick } from "./double-click" import PasteLinkOverText from "./quillPasteLinkOverText" import ClickableLink from "./quillClickableLink" Quill.register("modules/cursors", QuillCursors) Quill.register("modules/pasteLinkOverText", PasteLinkOverText) Quill.register(ClickableLink) interface CollaborativeTextViewProps { item: L.Property board: L.Property id: string accessLevel: L.Property focus: L.Atom itemFocus: L.Property<"none" | "selected" | "dragging" | "editing"> crdtStore: CRDTStore isLocked: L.Property uiEvents: L.EventStream } export function CollaborativeTextView({ id, item, board, accessLevel, focus, itemFocus, isLocked, crdtStore, uiEvents, }: CollaborativeTextViewProps) { const fontSize = L.view(item, (i) => `${i.fontSize ? i.fontSize : 1}em`) const color = L.view(item, getItemBackground, contrastingColor) const quillEditor = L.atom(null) accessLevel.applyScope(componentScope()).forEach((al) => { quillEditor.get()?.enable(canWrite(al)) }) function initQuill(el: HTMLElement) { const quill = new Quill(el, { modules: { cursors: true, toolbar: false, pasteLinkOverText: true, history: { userOnly: true, // Local undo shouldn't undo changes from remote users }, }, theme: "snow", readOnly: !canWrite(accessLevel.get()), }) const crdt = crdtStore.getBoardCrdt(board.get().id) const ytext = crdt.getField(id, "text") new QuillBinding(ytext, quill, crdt.awareness) quillEditor.set(quill) } const editingThis = L.view(itemFocus, (f) => f === "editing") editingThis.forEach((e) => { const q = quillEditor.get() if (q) { if (e) { const multipleLines = q .getText() .split("\n") .filter((x) => x).length > 1 if (!multipleLines) { // For one-liners, select the whole text on double click q.setSelection(0, 1000000) } } else { // Clear text selecting when not editing q.setSelection(null as any) } } }) function handleClick() { if (itemFocus.get() === "selected") { focus.set({ status: "editing", itemId: id }) } } const pointerEvents = L.view(itemFocus, isLocked, (f, l) => f === "editing" || f === "selected" || l ? "auto" : "none", ) const hAlign = L.view(item, getAlign, getHorizontalAlign).applyScope(componentScope()) hAlign.onChange((align) => { quillEditor.get()?.formatText(0, 10000000, "align", align === "left" ? "" : align) }) uiEvents.applyScope(componentScope()).forEach((e) => { if (e.action === "ui.text.format" && e.itemIds.includes(id)) { const quill = quillEditor.get() const selection = quill?.getSelection() const format = selection && quill?.getFormat(selection) const newValue = !(format && format[e.format]) quill?.format(e.format, newValue) } }) let touchMoves = 0 return (
{ e.stopPropagation() if (e.key === "Escape") { focus.set({ status: "selected", itemIds: new Set([id]), connectionIds: new Set() }) } }} onKeyDown={(e) => { e.stopPropagation() }} onKeyPress={(e) => { e.stopPropagation() }} onDoubleClick={(e) => { e.stopPropagation() quillEditor.get()?.focus() }} onTouchStart={(e) => { preventDoubleClick(e) touchMoves = 0 }} onTouchMove={() => touchMoves++} onTouchEnd={() => { if (touchMoves === 0) { // This is a way to detect a tap (vs swipe) quillEditor.get()?.focus() } }} onClick={handleClick} style={L.combineTemplate({ alignItems: L.view(item, getAlignItems) })} >
) } ================================================ FILE: frontend/src/board/ConnectionsView.tsx ================================================ import { Fragment, h, ListView } from "harmaja" import * as L from "lonna" import { findAttachmentLocation, resolveItemEndpoint } from "../../../common/src/connection-utils" import { AttachmentLocation, Board, ConnectionEndPoint, ConnectionEndStyle, isDirectedItemEndPoint, Item, Point, } from "../../../common/src/domain" import { Dispatch } from "../store/board-store" import { BoardCoordinateHelper } from "./board-coordinates" import { BoardFocus, getSelectedConnectionIds } from "./board-focus" import { existingConnectionHandler } from "./item-connect" import { Z_CONNECTIONS } from "./zIndices" export const ConnectionsView = ({ board, dispatch, zoom, coordinateHelper, focus, }: { board: L.Property dispatch: Dispatch zoom: L.Property coordinateHelper: BoardCoordinateHelper focus: L.Atom }) => { // Item position might change but connection doesn't -- need to rerender connections anyway // Connection objects normally only hold the ID to the "from" and "to" items // This populates the actual object in place of the ID function determineAttachmenLocation( e: ConnectionEndPoint, control: Point, is: Record, ): AttachmentLocation { if (isDirectedItemEndPoint(e)) { return findAttachmentLocation(resolveItemEndpoint(e, is), e.side) } const fromItem: Point = resolveEndpoint(e, is) // Support legacy routing (side not fixed) return findNearestAttachmentLocationForConnectionNode(fromItem, control) } const connectionsWithItemsPopulated = L.view( L.view(board, (b) => ({ is: b.items, cs: b.connections })), focus, L.view(zoom, "zoom"), ({ is, cs }, f, z) => { return cs .filter((c) => !c.hidden) .map((c) => { const fromItem: Point = resolveEndpoint(c.from, is) const toItemOrPoint = resolveEndpoint(c.to, is) const firstControlPoint = c.controlPoints[0] || fromItem const lastControlPoint = c.controlPoints[c.controlPoints.length - 1] || toItemOrPoint return { ...c, from: determineAttachmenLocation(c.from, firstControlPoint, is), to: determineAttachmenLocation(c.to, lastControlPoint, is), selected: getSelectedConnectionIds(f).has(c.id), } }) }, ) // We want to render round draggable nodes at the end of edges (paths), // But SVG elements are not draggable by default, so get a flat list of // nodes and render them as regular HTML elements const connectionNodes = L.view(connectionsWithItemsPopulated, (cs) => cs.flatMap((c) => [ { id: c.id, type: "from" as const, node: c.from, selected: c.selected, style: c.fromStyle }, { id: c.id, type: "to" as const, node: c.to, selected: c.selected, style: c.toStyle }, ...c.controlPoints.map((cp) => ({ id: c.id, type: "control" as const, node: { point: cp, side: "none" as const }, selected: c.selected, style: c.pointStyle, })), ]), ) const svgElementStyle = L.combineTemplate({ width: L.view(board, (b) => b.width + "em"), height: L.view(board, (b) => b.height + "em"), position: "absolute", top: 0, left: 0, pointerEvents: "none", zIndex: Z_CONNECTIONS, }) return ( <> observable={connectionNodes} renderObservable={ConnectionNode} getKey={(c) => c.id + c.type} /> { const curve = L.combine( L.view(conn, "from"), L.view(conn, "to"), L.view(conn, "controlPoints"), (from, to, cps) => { return quadraticCurveSVGPath( { x: coordinateHelper.emToBoardPx(from.point.x), y: coordinateHelper.emToBoardPx(from.point.y), }, { x: coordinateHelper.emToBoardPx(to.point.x), y: coordinateHelper.emToBoardPx(to.point.y), }, cps.map((cp) => ({ x: coordinateHelper.emToBoardPx(cp.x), y: coordinateHelper.emToBoardPx(cp.y), })), ) }, ) return ( (c.selected ? "connection selected" : "connection"))} d={curve} > ) }} getKey={(c) => c.id} /> ) type ConnectionNodeProps = { id: string node: AttachmentLocation type: "to" | "from" | "control" style: ConnectionEndStyle selected: boolean } function ConnectionNode(key: string, cNode: L.Property) { function onRef(el: HTMLDivElement) { const { id, type } = cNode.get() existingConnectionHandler(el, id, type, coordinateHelper, board, dispatch) } const id = L.view(cNode, (cn) => `connection-${cn.id}-${cn.type}`) const angle = L.view(cNode, (cn) => { if (cn.style !== "arrow" || cn.type === "control") return null const conn = connectionsWithItemsPopulated.get().find((c) => c.id === cn.id) if (!conn) { return null } const [thisEnd, otherEnd] = cn.type === "from" ? [conn.from, conn.to] : [conn.to, conn.from] const bez = bezierCurveFromPoints( otherEnd.point, getControlPoint(otherEnd.point, thisEnd.point, conn.controlPoints), thisEnd.point, ) const derivative = bez.derivative(1) // tangent vector at the very end of the curve const angleInDegrees = ((Math.atan2(derivative.y, derivative.x) - Math.atan2(0, Math.abs(derivative.x))) * 180) / Math.PI return Math.round(angleInDegrees) }) const wrapperStyle = L.view(cNode, (cn) => ({ top: `${cn.node.point.y}em`, left: `${cn.node.point.x}em`, zIndex: Z_CONNECTIONS + 1, })) const nodeStyle = L.view(angle, (ang) => ({ transform: ang !== null ? `rotate(${ang}deg)` : undefined, })) const selectThisConnection = (e: JSX.MouseEvent) => { const id = cNode.get().id const f = focus.get() if (e.shiftKey && f.status === "selected") { focus.set({ ...f, connectionIds: toggleInSet(id, f.connectionIds) }) } else { focus.set({ status: "selected", connectionIds: new Set([id]), itemIds: emptySet() }) } } return (
{ let cls = `connection-node ${cn.type}-node ${cn.style}-style ` if (cn.selected) cls += "highlight " cls += cn.node.side === "none" ? "unattached" : "attached" return cls })} style={nodeStyle} >
) } } // @ts-ignore import { Bezier } from "bezier-js" import { findNearestAttachmentLocationForConnectionNode, resolveEndpoint } from "../../../common/src/connection-utils" import { emptySet, toggleInSet } from "../../../common/src/sets" import { BoardZoom } from "./board-scroll-and-zoom" function quadraticCurveSVGPath(from: Point, to: Point, controlPoints: Point[]) { if (!controlPoints.length) { // fallback if no control points: straight line const midPoint = getControlPoint(from, to, controlPoints) return "M" + from.x + " " + from.y + " Q " + midPoint.x + " " + midPoint.y + " " + to.x + " " + to.y } else { const peakPointOfCurve = controlPoints[0] const bez = bezierCurveFromPoints(from, peakPointOfCurve, to) return bez .getLUT() .reduce( (acc: string, p: Point, i: number) => i === 0 ? (acc += `M ${p.x} ${p.y}`) : (acc += `L ${p.x} ${p.y}`), "", ) } } function getControlPoint(from: Point, to: Point, controlPoints: Point[]) { if (controlPoints.length > 0) return controlPoints[0] // fallback if no control points: midpoint return { x: (to.x + from.x) * 0.5, y: (to.y + from.y) * 0.5 } } function bezierCurveFromPoints(from: Point, middle: Point, to: Point): any { return Bezier.quadraticFromPoints(from, middle, to) } ================================================ FILE: frontend/src/board/CursorsView.tsx ================================================ import { componentScope, h, ListView } from "harmaja" import * as L from "lonna" import { UserCursorPosition, UserSessionInfo } from "../../../common/src/domain" import { CursorsStore } from "../store/cursors-store" import { BoardZoom } from "./board-scroll-and-zoom" import { Rect } from "../../../common/src/geometry" import _ from "lodash" export const CursorsView = ({ sessions, cursors, viewRect, }: { cursors: CursorsStore sessions: L.Property viewRect: L.Property }) => { const transitionFromCursorDelay = cursors.cursorDelay.pipe( L.changes, L.throttle(2000, componentScope()), L.map((d) => { const t = (Math.min(d, 1000) / 1000).toFixed(1) return `all ${t}s, top ${t}s` }), ) const transitionFromZoom = viewRect.pipe( L.changes, L.map(() => "none"), ) const transition = L.merge(transitionFromCursorDelay, transitionFromZoom).pipe( L.toProperty("none", componentScope()), ) const scope = componentScope() return ( observable={cursors.cursors} renderObservable={(sessionId: string, pos_: L.Property) => { const pos = pos_.pipe(L.skipDuplicates(_.isEqual), L.applyScope(scope)) const changes = pos.pipe(L.changes) const stale = L.merge( changes.pipe( L.debounce(1000), L.map(() => true), ), changes.pipe(L.map(() => false)), ).pipe(L.toProperty(false, scope)) const className = L.view(stale, (s) => (s ? "cursor stale" : "cursor")) const style = L.view(pos, transition, viewRect, (p, t, vr) => { const x = _.clamp(p.x, vr.x, vr.x + vr.width - 1) const y = _.clamp(p.y, vr.y, vr.y + vr.height - 1) return { transition: t, left: x + "em", top: y + "em", } }) const userInfo = L.view(sessions, (sessions) => { const session = sessions.find((s) => s.sessionId === sessionId) return { name: session ? session.nickname : null, picture: session && session.userType === "authenticated" ? : null, } }) return ( {L.view(userInfo, "picture")} {L.view(userInfo, "name")} ) }} getKey={(c: UserCursorPosition) => c.sessionId} /> ) } ================================================ FILE: frontend/src/board/DragBorder.tsx ================================================ import { h, Fragment } from "harmaja" import * as L from "lonna" import { Board, Connection } from "../../../common/src/domain" import { BoardCoordinateHelper } from "./board-coordinates" import { BoardFocus } from "./board-focus" import { Dispatch } from "../store/board-store" import { itemDragToMove } from "./item-dragmove" import { Tool, ToolController } from "./tool-selection" type Position = "left" | "right" | "top" | "bottom" export const DragBorder = ({ id, board, coordinateHelper, latestConnection, focus, toolController, dispatch, }: { id: string coordinateHelper: BoardCoordinateHelper latestConnection: L.Property focus: L.Atom board: L.Property toolController: ToolController dispatch: Dispatch }) => { return ( <> ) function DragHandle({ position }: { position: Position }) { const ref = (e: HTMLElement) => itemDragToMove(id, board, focus, toolController, coordinateHelper, latestConnection, dispatch, false)(e) return } } ================================================ FILE: frontend/src/board/ImageView.tsx ================================================ import { h } from "harmaja" import * as L from "lonna" import { Board, Connection, Image } from "../../../common/src/domain" import { AssetStore } from "../store/asset-store" import { Dispatch } from "../store/board-store" import { BoardCoordinateHelper } from "./board-coordinates" import { BoardFocus } from "./board-focus" import { itemDragToMove } from "./item-dragmove" import { itemSelectionHandler } from "./item-selection" import { ToolController } from "./tool-selection" import { itemZIndex } from "./zIndices" export const ImageView = ({ id, image, assets, board, isLocked, focus, toolController, coordinateHelper, latestConnection, dispatch, }: { board: L.Property id: string image: L.Property isLocked: L.Property focus: L.Atom toolController: ToolController coordinateHelper: BoardCoordinateHelper latestConnection: L.Property dispatch: Dispatch assets: AssetStore }) => { const { selected, onClick, onTouchStart } = itemSelectionHandler( id, "image", focus, toolController, board, coordinateHelper, latestConnection, dispatch, ) const tool = toolController.tool return ( ({ top: 0, left: 0, transform: `translate(${p.x}em, ${p.y}em)`, height: p.height + "em", width: p.width + "em", zIndex: itemZIndex(p), position: "absolute", } as any), )} > assets.getAsset(i.assetId, i.src))} /> {L.view(isLocked, (l) => l && 🔒)} ) } ================================================ FILE: frontend/src/board/ItemView.tsx ================================================ import { componentScope, h } from "harmaja" import * as L from "lonna" import { AccessLevel, Board, Connection, getAlign, getHorizontalAlign, getItemBackground, getItemShape, getVerticalAlign, isContainer, isLocalUIEvent, isTextItem, Item, ItemType, TextItem, } from "../../../common/src/domain" import { BoardStore, Dispatch } from "../store/board-store" import { BoardCoordinateHelper } from "./board-coordinates" import { BoardFocus } from "./board-focus" import { CollaborativeTextView } from "./CollaborativeTextView" import { DragBorder } from "./DragBorder" import { itemDragToMove } from "./item-dragmove" import { itemSelectionHandler } from "./item-selection" import { TextView } from "./TextView" import { ToolController } from "./tool-selection" import { itemZIndex } from "./zIndices" import { VisibilityOffIcon } from "../components/Icons" export const ItemView = ({ board, accessLevel, id, type, item, isLocked, focus, coordinateHelper, latestConnection, dispatch, toolController, boardStore, }: { board: L.Property accessLevel: L.Property id: string type: ItemType item: L.Property isLocked: L.Property focus: L.Atom coordinateHelper: BoardCoordinateHelper latestConnection: L.Property dispatch: Dispatch toolController: ToolController boardStore: BoardStore }) => { const element = L.atom(null) const ref = (el: HTMLElement) => { itemDragToMove( id, board, focus, toolController, coordinateHelper, latestConnection, dispatch, type === "container", )(el) element.set(el) } const { itemFocus, selected, onClick, onTouchStart } = itemSelectionHandler( id, type, focus, toolController, board, coordinateHelper, latestConnection, dispatch, ) function itemPadding(i: Item) { if (i.type != "note") return undefined const shape = getItemShape(i) return shape == "diamond" ? `${i.width / 4}em` : shape == "round" ? `${i.width / 8}em` : shape == "square" || shape == "rect" ? `${(i.fontSize || 1) / 3}em` : undefined } const shape = L.view(item, getItemShape) const itemNow = item.get() return ( (l ? "Item is selected by another user" : ""))} ref={ref} data-itemid={id} draggable={L.view(itemFocus, (f) => f !== "editing")} onClick={onClick} onTouchStart={onTouchStart} className={L.view( selected, L.view(item, getItemBackground), isLocked, (s, b, l) => `${type} ${"color-" + b.replace("#", "").toLowerCase()} ${s ? "selected" : ""} ${ l ? "locked" : "" }`, )} style={L.view(item, (i) => { return { top: 0, left: 0, height: i.height + "em", width: i.width + "em", transform: `translate(${i.x}em, ${i.y}em)`, zIndex: itemZIndex(i), position: "absolute", padding: itemPadding(i), justifyContent: getJustifyContent(i), alignItems: getAlignItems(i), textAlign: getTextAlign(i), } })} > "shape " + s)} style={L.view(item, (i) => { return { background: getItemBackground(i), } })} /> {isTextItem(itemNow) && itemNow.crdt ? ( } board={board} id={id} accessLevel={accessLevel} focus={focus} itemFocus={itemFocus} crdtStore={boardStore.crdtStore} isLocked={isLocked} uiEvents={boardStore.events.pipe(L.filter(isLocalUIEvent), L.applyScope(componentScope()))} /> ) : ( } dispatch={dispatch} board={board} toolController={toolController} accessLevel={accessLevel} focus={focus} itemFocus={itemFocus} coordinateHelper={coordinateHelper} element={element} /> )} {L.view( item, (i) => isContainer(i) && i.contentsHidden, (hidden) => (hidden && (
)) ?? null, )} {type === "container" && ( )} ) } export function getJustifyContent(item: Item) { if (isTextItem(item)) { switch (getHorizontalAlign(getAlign(item))) { case "left": return "flex-start" case "center": return "center" case "right": return "flex-end" } } return null } export function getAlignItems(item: Item) { if (isTextItem(item)) { switch (getVerticalAlign(getAlign(item))) { case "top": return "flex-start" case "middle": return "center" case "bottom": return "flex-end" } } return null } function getTextAlign(item: Item) { if (isTextItem(item)) { return getHorizontalAlign(getAlign(item)) } return null } ================================================ FILE: frontend/src/board/RectangularDragSelection.tsx ================================================ import { h } from "harmaja" import * as L from "lonna" import { Rect } from "../../../common/src/geometry" export const RectangularDragSelection = ({ rect }: { rect: L.Property }) => { return L.view( rect, (r) => r && ( ), ) } ================================================ FILE: frontend/src/board/SaveAsTemplate.tsx ================================================ import { h } from "harmaja" import * as L from "lonna" import { Board } from "../../../common/src/domain" export const SaveAsTemplate = ({ board }: { board: L.Property }) => { const currSavedBoard = L.atom(null) function handleLocalTemplateSave() { const b = board.get() if (!b) return const saved = localStorage.getItem("rboard_templates") const templates = saved ? (JSON.parse(saved) as Record) : {} templates[b.name] = b localStorage.setItem("rboard_templates", JSON.stringify(templates)) currSavedBoard.set(b) } const changed = L.combineTemplate({ curr: currSavedBoard, next: board, }).pipe(L.map((c) => c.curr !== c.next)) return (
  • (c ? "" : "disabled"))} data-test="palette-save-as-template" onClick={() => handleLocalTemplateSave()} > Save as template
    • ) } ================================================ FILE: frontend/src/board/SelectionBorder.tsx ================================================ import { h } from "harmaja" import * as L from "lonna" import { BoardCoordinateHelper } from "./board-coordinates" import { Board, Container } from "../../../common/src/domain" import { BoardFocus } from "./board-focus" import { onBoardItemDrag } from "./item-drag" import { Dispatch } from "../store/board-store" import { canMove } from "./board-permissions" type Horizontal = "left" | "right" type Vertical = "top" | "bottom" const borderOffset = 0.25 export const SelectionBorder = ({ id, board, coordinateHelper, focus, dispatch, }: { id: string coordinateHelper: BoardCoordinateHelper focus: L.Atom board: L.Property dispatch: Dispatch }) => { const item = L.view(board, (b) => b.items[id]) const style = L.view(item, (i) => { return { top: -borderOffset + "rem", left: -borderOffset + "rem", height: `calc(${i.height}em + 2 * ${borderOffset}rem)`, width: `calc(${i.width}em + 2 * ${borderOffset}rem)`, transform: `translate(${i.x}em, ${i.y}em)`, } }) return L.view( item, (i) => !i.hidden && canMove(i), (m) => m ? ( ) : null, ) function DragCorner({ vertical, horizontal }: { vertical: Vertical; horizontal: Horizontal }) { const ref = (e: HTMLElement) => onBoardItemDrag( e, id, board, focus, coordinateHelper, false, (b, startPos, items, connections, xDiff, yDiff) => { const updatedItems = items.map(({ current, dragStartPosition }) => { const maintainAspectRatio = current.type === "image" || (current.type === "note" && current.shape !== "rect") if (maintainAspectRatio) { let minDiff = Math.min(Math.abs(xDiff), Math.abs(yDiff)) if (minDiff < 0.1) { xDiff = 0 yDiff = 0 } else { const aspectRatio = dragStartPosition.width / dragStartPosition.height const invert = (horizontal == "left" && vertical == "bottom") || (horizontal == "right" && vertical == "top") const factor = invert ? -1 : 1 if (Math.abs(xDiff) == minDiff) { // x is the smaller adjustment, use that as basis yDiff = (minDiff / aspectRatio) * factor * sign(xDiff) } else { xDiff = minDiff * aspectRatio * factor * sign(yDiff) } } } const x = horizontal === "left" ? dragStartPosition.x + xDiff : dragStartPosition.x const y = vertical === "top" ? dragStartPosition.y + yDiff : dragStartPosition.y const width = Math.max( 0.5, horizontal === "left" ? dragStartPosition.width - xDiff : dragStartPosition.width + xDiff, ) const height = Math.max( 0.5, vertical === "top" ? dragStartPosition.height - yDiff : dragStartPosition.height + yDiff, ) const updatedItem = { id: current.id, x, y, width, height, } return updatedItem }) dispatch({ action: "item.update", boardId: b.id, items: updatedItems }) function sign(x: number) { return x / Math.abs(x) } }, ) return } } ================================================ FILE: frontend/src/board/TextView.tsx ================================================ import { h } from "harmaja" import * as L from "lonna" import { AccessLevel, Board, canWrite, getItemBackground, TextItem } from "../../../common/src/domain" import { emptySet } from "../../../common/src/sets" import { HTMLEditableSpan } from "../components/HTMLEditableSpan" import { Dispatch } from "../store/board-store" import { autoFontSize } from "./autoFontSize" import { BoardCoordinateHelper } from "./board-coordinates" import { BoardFocus, getSelectedItemIds } from "./board-focus" import { contrastingColor } from "./contrasting-color" import { ToolController } from "./tool-selection" interface TextViewProps { id: string item: L.Property dispatch: Dispatch board: L.Property toolController: ToolController accessLevel: L.Property focus: L.Atom itemFocus: L.Property<"none" | "selected" | "dragging" | "editing"> coordinateHelper: BoardCoordinateHelper element: L.Property } export function TextView({ id, item, dispatch, board, toolController, focus, coordinateHelper, itemFocus, accessLevel, element, }: TextViewProps) { const textAtom = L.atom(L.view(item, "text"), (text) => dispatch({ action: "item.update", boardId: board.get().id, items: [{ id, text }] }), ) const showCoords = false const focused = L.view(focus, (f) => getSelectedItemIds(f).has(id)) const setEditing = (e: boolean) => { if (toolController.tool.get() === "connect") return // Don't switch to editing in middle of connecting dispatch({ action: "item.front", boardId: board.get().id, itemIds: [id] }) focus.set( e ? { status: "editing", itemId: id } : { status: "selected", itemIds: new Set([id]), connectionIds: emptySet() }, ) } const color = L.view(item, getItemBackground, contrastingColor) const fontSize = autoFontSize( item, L.view(item, (i) => (i.fontSize ? i.fontSize : 1)), L.view(item, "text"), focused, coordinateHelper, element, ) return ( e.stopPropagation()} style={L.combineTemplate({ fontSize, color })} > f === "editing"), setEditing, ), editable: L.view(accessLevel, canWrite), }} /> {showCoords && {L.view(item, (p) => Math.floor(p.x) + ", " + Math.floor(p.y))}} ) } ================================================ FILE: frontend/src/board/VideoView.tsx ================================================ import { h } from "harmaja" import * as L from "lonna" import { BoardCoordinateHelper } from "./board-coordinates" import { Board, Connection, Image, Video } from "../../../common/src/domain" import { BoardFocus } from "./board-focus" import { AssetStore } from "../store/asset-store" import { itemDragToMove } from "./item-dragmove" import { itemSelectionHandler } from "./item-selection" import { Dispatch } from "../store/board-store" import { Tool, ToolController } from "./tool-selection" import { DragBorder } from "./DragBorder" import { itemZIndex } from "./zIndices" export const VideoView = ({ id, video, assets, board, isLocked, focus, toolController, coordinateHelper, latestConnection, dispatch, }: { board: L.Property id: string video: L.Property