Repository: deepstreamIO/deepstream.io Branch: master Commit: e789afa7497b Files: 254 Total size: 1.0 MB Directory structure: gitextract_5br62wbj/ ├── .dockerignore ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ └── bug_report.md │ └── workflows/ │ ├── lint-test.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── .npmignore ├── CHANGELOG.md ├── Dockerfile ├── Dockerfile.alpine ├── LICENSE ├── README.md ├── SECURITY.md ├── ascii-logo.txt ├── bin/ │ ├── deepstream-cluster.ts │ ├── deepstream-daemon.ts │ ├── deepstream-hash.ts │ ├── deepstream-info.ts │ ├── deepstream-nginx.ts │ ├── deepstream-service.ts │ ├── deepstream-start.ts │ └── deepstream.ts ├── conf/ │ ├── config.yml │ ├── permissions.yml │ └── users.yml ├── package.json ├── scripts/ │ ├── connector/ │ │ ├── package-connector.sh │ │ └── test-connector.sh │ ├── details.js │ ├── executable-test.js │ ├── license-aggregator.js │ ├── linux-package.sh │ ├── linux-test.sh │ ├── node-test.js │ ├── package.sh │ ├── pkg.js │ ├── release.sh │ ├── resources/ │ │ ├── PackageInfo │ │ ├── daemon/ │ │ │ ├── after-install │ │ │ ├── after-upgrade │ │ │ ├── before-remove │ │ │ └── before-upgrade │ │ ├── missing-licenses.txt │ │ └── node.rc │ ├── sanity-test.sh │ ├── setup.sh │ ├── trigger-build.sh │ └── tsc.sh ├── src/ │ ├── config/ │ │ ├── config-initialiser.spec.ts │ │ ├── config-initialiser.ts │ │ ├── config-validator.ts │ │ ├── ds-info.ts │ │ ├── file-utils.spec.ts │ │ ├── file-utils.ts │ │ ├── js-yaml-loader.spec.ts │ │ └── js-yaml-loader.ts │ ├── connection-endpoint/ │ │ ├── base/ │ │ │ ├── connection-endpoint.spec.ts │ │ │ ├── connection-endpoint.ts │ │ │ └── socket-wrapper.ts │ │ ├── http/ │ │ │ ├── connection-endpoint.spec.ts │ │ │ ├── connection-endpoint.ts │ │ │ └── socket-wrapper.ts │ │ ├── mqtt/ │ │ │ ├── connection-endpoint.ts │ │ │ ├── message-parser.ts │ │ │ └── socket-wrapper-factory.ts │ │ └── websocket/ │ │ ├── binary/ │ │ │ ├── connection-endpoint.ts │ │ │ └── socket-wrapper-factory.ts │ │ ├── json/ │ │ │ ├── connection-endpoint.ts │ │ │ └── socket-wrapper-factory.ts │ │ └── text/ │ │ ├── connection-endpoint.ts │ │ ├── socket-wrapper-factory.ts │ │ └── text-protocol/ │ │ ├── constants.ts │ │ ├── message-builder.ts │ │ ├── message-parser.ts │ │ └── utils.ts │ ├── constants.ts │ ├── deepstream.io.spec.ts │ ├── deepstream.io.ts │ ├── default-options.ts │ ├── handlers/ │ │ ├── event/ │ │ │ ├── event-handler.spec.ts │ │ │ └── event-handler.ts │ │ ├── monitoring/ │ │ │ └── monitoring.ts │ │ ├── presence/ │ │ │ ├── presence-handler.spec.ts │ │ │ └── presence-handler.ts │ │ ├── record/ │ │ │ ├── record-deletion.spec.ts │ │ │ ├── record-deletion.ts │ │ │ ├── record-handler-permission.spec.ts │ │ │ ├── record-handler.spec.ts │ │ │ ├── record-handler.ts │ │ │ ├── record-request.spec.ts │ │ │ ├── record-request.ts │ │ │ ├── record-transition.spec.ts │ │ │ ├── record-transition.ts │ │ │ ├── record-write-acknowledgement.spec.ts │ │ │ └── test-messages.ts │ │ └── rpc/ │ │ ├── rpc-handler.spec.ts │ │ ├── rpc-handler.ts │ │ ├── rpc-proxy.spec.ts │ │ ├── rpc-proxy.ts │ │ └── rpc.ts │ ├── jif/ │ │ ├── jif-handler.spec.ts │ │ ├── jif-handler.ts │ │ └── jif-schema.ts │ ├── listen/ │ │ ├── listener-registry.spec.ts │ │ ├── listener-registry.ts │ │ └── listener-test-utils.ts │ ├── plugins/ │ │ └── heap-snapshot/ │ │ └── heap-snapshot.ts │ ├── service/ │ │ ├── daemon.ts │ │ ├── service.ts │ │ └── template/ │ │ ├── initd.ts │ │ └── systemd.ts │ ├── services/ │ │ ├── authentication/ │ │ │ ├── combine/ │ │ │ │ └── combine-authentication.ts │ │ │ ├── file/ │ │ │ │ ├── file-based-authentication.spec.ts │ │ │ │ └── file-based-authentication.ts │ │ │ ├── http/ │ │ │ │ ├── http-authentication.spec.ts │ │ │ │ └── http-authentication.ts │ │ │ ├── open/ │ │ │ │ ├── open-authentication.spec.ts │ │ │ │ └── open-authentication.ts │ │ │ └── storage/ │ │ │ └── storage-based-authentication.ts │ │ ├── cache/ │ │ │ ├── local-cache.spec.ts │ │ │ └── local-cache.ts │ │ ├── cluster-node/ │ │ │ ├── single-cluster-node.ts │ │ │ └── vertical-cluster-node.ts │ │ ├── cluster-registry/ │ │ │ ├── distributed-cluster-registry.ts │ │ │ └── distributed-state-registry-factory.ts │ │ ├── cluster-state/ │ │ │ ├── distributed-state-registry-factory.ts │ │ │ ├── distributed-state-registry.ts │ │ │ └── single-state-registry.ts │ │ ├── http/ │ │ │ ├── node/ │ │ │ │ └── node-http.ts │ │ │ └── uws/ │ │ │ └── uws-http.ts │ │ ├── lock/ │ │ │ └── distributed-lock-registry.ts │ │ ├── logger/ │ │ │ ├── pino/ │ │ │ │ └── pino-logger.ts │ │ │ └── std/ │ │ │ ├── std-out-logger.spec.ts │ │ │ └── std-out-logger.ts │ │ ├── monitoring/ │ │ │ ├── combine-monitoring.ts │ │ │ ├── http/ │ │ │ │ ├── monitoring-http.spec.ts │ │ │ │ └── monitoring-http.ts │ │ │ ├── log/ │ │ │ │ ├── monitoring-log.spec.ts │ │ │ │ └── monitoring-log.ts │ │ │ ├── monitoring-base.ts │ │ │ └── noop-monitoring.ts │ │ ├── permission/ │ │ │ ├── open/ │ │ │ │ ├── open-permission.spec.ts │ │ │ │ └── open-permission.ts │ │ │ └── valve/ │ │ │ ├── config-compiler.spec.ts │ │ │ ├── config-compiler.ts │ │ │ ├── config-permission-basic.spec.ts │ │ │ ├── config-permission-create.spec.ts │ │ │ ├── config-permission-cross-reference.spec.ts │ │ │ ├── config-permission-load.spec.ts │ │ │ ├── config-permission-nested-cross-reference.spec.ts │ │ │ ├── config-permission-other.spec.ts │ │ │ ├── config-permission-record-patch.spec.ts │ │ │ ├── config-permission.ts │ │ │ ├── config-schema.ts │ │ │ ├── config-validator.spec.ts │ │ │ ├── config-validator.ts │ │ │ ├── path-parser.spec.ts │ │ │ ├── path-parser.ts │ │ │ ├── rule-application.ts │ │ │ ├── rule-cache.spec.ts │ │ │ ├── rule-cache.ts │ │ │ ├── rule-parser.spec.ts │ │ │ ├── rule-parser.ts │ │ │ ├── rules-map.spec.ts │ │ │ └── rules-map.ts │ │ ├── storage/ │ │ │ ├── noop-storage.spec.ts │ │ │ └── noop-storage.ts │ │ ├── subscription-registry/ │ │ │ ├── default-subscription-registry-factory.ts │ │ │ ├── default-subscription-registry.spec.ts │ │ │ └── default-subscription-registry.ts │ │ └── telemetry/ │ │ └── deepstreamio-telemetry.ts │ ├── test/ │ │ ├── common.ts │ │ ├── config/ │ │ │ ├── basic-permission-config.json │ │ │ ├── basic-valid-json.json │ │ │ ├── blank-config.json │ │ │ ├── config-broken.js │ │ │ ├── config-broken.yml │ │ │ ├── config.js │ │ │ ├── config.yml │ │ │ ├── empty-map-config.json │ │ │ ├── exists-test/ │ │ │ │ ├── a-file.js │ │ │ │ ├── a-file.yml │ │ │ │ └── a-json-file.json │ │ │ ├── invalid-permission-conf.json │ │ │ ├── invalid-user-config.json │ │ │ ├── json-with-env-variables.json │ │ │ ├── no-private-events-permission-config.json │ │ │ ├── sslKey.pem │ │ │ ├── users-unhashed.json │ │ │ └── users.json │ │ ├── helper/ │ │ │ ├── start-test-server.ts │ │ │ ├── test-helper.ts │ │ │ ├── test-http-server.ts │ │ │ └── test-mocks.ts │ │ └── mock/ │ │ ├── authentication-handler-mock.ts │ │ ├── http-mock.ts │ │ ├── logger-mock.ts │ │ ├── message-connector-mock.ts │ │ ├── permission-handler-mock.ts │ │ ├── plugin-mock.ts │ │ ├── socket-mock.ts │ │ ├── socket-wrapper-factory-mock.ts │ │ └── storage-mock.ts │ └── utils/ │ ├── dependency-initialiser.spec.ts │ ├── dependency-initialiser.ts │ ├── json-path.spec.ts │ ├── json-path.ts │ ├── message-distributor.spec.ts │ ├── message-distributor.ts │ ├── message-processor.spec.ts │ ├── message-processor.ts │ ├── utils.spec.ts │ └── utils.ts ├── telemetry-server/ │ ├── package.json │ ├── telemetry-server.config.js │ └── telemetry-server.ts ├── test-e2e/ │ ├── config/ │ │ ├── permissions-complex.json │ │ └── permissions-open.json │ ├── framework/ │ │ ├── client-handler.ts │ │ ├── client.ts │ │ ├── event.ts │ │ ├── listening.ts │ │ ├── presence.ts │ │ ├── record.ts │ │ ├── rpc.ts │ │ ├── utils.ts │ │ └── world.ts │ ├── framework-v3/ │ │ ├── client-handler.ts │ │ ├── client.ts │ │ ├── event.ts │ │ ├── listening.ts │ │ ├── presence.ts │ │ ├── record.ts │ │ ├── rpc.ts │ │ ├── utils.ts │ │ └── world.ts │ ├── steps/ │ │ ├── client/ │ │ │ ├── client-definition-step.ts │ │ │ ├── connection-steps.ts │ │ │ ├── event-steps.ts │ │ │ ├── listening-steps.ts │ │ │ ├── presence-steps.ts │ │ │ ├── record-steps.ts │ │ │ └── rpc-steps.ts │ │ ├── http/ │ │ │ └── http-steps.ts │ │ └── server/ │ │ └── step-definition-server.ts │ └── tools/ │ ├── e2e-authentication.ts │ ├── e2e-cluster-node.ts │ ├── e2e-harness.ts │ ├── e2e-logger.ts │ └── e2e-server-config.ts ├── tsconfig.json ├── tslint.json └── types/ ├── global.d.ts └── uws.d.ts ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ **/.git/ **/.github/ **/node_modules/ **/Dockerfile **/Dockerfile.* **/.env.*.local **/.env.local ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: '' labels: '' assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. **Server version** **Protocol (binary or text)** **Websocket server (node-http or uws)** **Server OS** **Node version** **Server config** Please share the config sections that might be relevant for this issue **Client (Js, Java, other) and client version** **Client config** Please share the config sections that might be relevant for this issue **Additional context** Add any other context about the problem here. ================================================ FILE: .github/workflows/lint-test.yml ================================================ name: lint-and-test on: [push, pull_request] jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v3 with: node-version: '22.x' - run: npm install - run: npm run lint - run: npm run test:all:coverage - run: npm run e2e:uws - name: Coveralls if: startsWith(matrix.node-version, '22.') uses: coverallsapp/github-action@master with: github-token: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/release.yml ================================================ name: release on: push: # Sequence of patterns matched against refs/tags tags: - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 jobs: create_release: name: Create release runs-on: ubuntu-latest steps: - name: Create release id: create_release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ github.ref }} release_name: Release ${{ github.ref }} npm_publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 # Setup .npmrc file to publish to npm - uses: actions/setup-node@v3 with: node-version: '22.x' registry-url: 'https://registry.npmjs.org' - run: npm install - run: npm run lint - run: npm run test - run: npm run tsc - run: npm publish --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} linux: runs-on: ubuntu-latest steps: - name: Checkout reposistory uses: actions/checkout@v3 - name: Checkout submodules run: git submodule update --init --recursive - name: Use Node.js env: DEFAULT_DELAY: 50 uses: actions/setup-node@v3 with: node-version: '22.x' - run: npm install - run: npm run lint - run: npm run test - run: bash ./scripts/package.sh true true - name: Upload Release Asset uses: alexellis/upload-assets@0.2.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: asset_paths: '["build/*/*.tar.gz"]' windows: runs-on: windows-latest steps: - name: Checkout reposistory uses: actions/checkout@v3 - name: Checkout submodules run: git submodule update --init --recursive - name: Use Node.js env: DEFAULT_DELAY: 50 uses: actions/setup-node@v3 with: node-version: '22.x' - run: npm install - run: npm run lint - run: npm run test - run: bash ./scripts/package.sh true true - name: Upload Release Asset uses: alexellis/upload-assets@0.2.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: asset_paths: '["build/*/*.zip"]' macos: runs-on: macos-latest steps: - name: Checkout reposistory uses: actions/checkout@v3 - name: Checkout submodules run: git submodule update --init --recursive - name: Use Node.js env: DEFAULT_DELAY: 50 uses: actions/setup-node@v3 with: node-version: '22.x' - run: npm install - run: npm run lint - run: npm run test - run: bash ./scripts/package.sh true true - name: Upload Release Asset uses: alexellis/upload-assets@0.2.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: asset_paths: '["build/*/*.pkg"]' docker-amd: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Log in to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 with: images: deepstreamio/deepstream.io flavor: | latest=true tags: | type=semver,pattern={{version}} - name: Build and push Docker image uses: docker/build-push-action@v4 with: context: . file: Dockerfile push: true platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} docker-arm: runs-on: ubuntu-latest steps: - uses: docker/setup-qemu-action@v1 - uses: docker/setup-buildx-action@v1 - uses: actions/checkout@v3 - name: Log in to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 with: images: deepstreamio/deepstream.io flavor: | latest=true tags: | type=semver,pattern={{version}} - name: Build and push Docker image uses: docker/build-push-action@v4 with: context: . file: Dockerfile push: true platforms: linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} docker-alpine: runs-on: ubuntu-latest steps: - uses: docker/setup-qemu-action@v1 - uses: docker/setup-buildx-action@v1 - uses: actions/checkout@v3 - name: Log in to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 with: images: deepstreamio/deepstream.io flavor: | latest=true suffix=-alpine,onlatest=true tags: | type=semver,pattern={{version}} - name: Build and push Docker image uses: docker/build-push-action@v4 with: context: . file: Dockerfile.alpine push: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} ================================================ FILE: .gitignore ================================================ heap-snapshots .nyc_combined_coverage dist .DS_Store .vscode/settings.json .nyc_output temp-e2e-test local-storage # Logs logs *.log # Runtime data pids *.pid *.seed # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) .grunt # pkg build meta.json # Compiled binary addons (http://nodejs.org/api/addons.html) build/Release # Dependency directory # Commenting this out is preferred by some people, see # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- node_modules npm-shrinkwrap.json npm-debug.* # Users Environment Variables .lock-wscript # IDE configuration .idea licenses.json # Typescript dist ================================================ FILE: .gitmodules ================================================ [submodule "test-e2e/features"] path = test-e2e/features url = https://github.com/deepstreamIO/deepstream.io-e2e.git [submodule "connectors/cache/redis"] path = connectors/cache/redis url = https://github.com/deepstreamIO/deepstream.io-cache-redis.git [submodule "connectors/cache/memcached"] path = connectors/cache/memcached url = https://github.com/deepstreamIO/deepstream.io-cache-memcached.git [submodule "connectors/cache/hazelcast"] path = connectors/cache/hazelcast url = https://github.com/deepstreamIO/deepstream.io-cache-hazelcast.git [submodule "connectors/storage/postgres"] path = connectors/storage/postgres url = https://github.com/deepstreamIO/deepstream.io-storage-postgres.git [submodule "connectors/storage/rethinkdb"] path = connectors/storage/rethinkdb url = https://github.com/deepstreamIO/deepstream.io-storage-rethinkdb.git [submodule "connectors/storage/mongodb"] path = connectors/storage/mongodb url = https://github.com/deepstreamIO/deepstream.io-storage-mongodb.git [submodule "connectors/storage/elasticsearch"] path = connectors/storage/elasticsearch url = https://github.com/deepstreamIO/deepstream.io-storage-elasticsearch.git [submodule "client"] path = client url = https://github.com/deepstreamIO/deepstream.io-client-js.git [submodule "connectors/clusterNode/redis"] path = connectors/clusterNode/redis url = https://github.com/deepstreamIO/deepstream.io-clusternode-redis.git [submodule "connectors/logger/winston"] path = connectors/logger/winston url = https://github.com/deepstreamIO/deepstream.io-logger-winston.git [submodule "plugins/aws"] path = plugins/aws url = https://github.com/deepstreamIO/deepstream.io-plugin-aws.git ================================================ FILE: .npmignore ================================================ # Typescript files /scripts /bin .tsconfig.json .tslint.json # Logs logs *.log # Runtime data pids *.pid *.seed # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) .grunt # pkg build meta.json # Users Environment Variables .lock-wscript # IDE configuration .idea # CI .travis.yml appveyor.yml # testing test test-e2e benchmarks ================================================ FILE: CHANGELOG.md ================================================ ## [10.0.0] - 2025.08.02 ### Fix - BREAKING CHANGE The storage authentication service, when creating new users automatically, was encoding the password hashes in 'ascii' format which can create characters that are not allowed in a database text field. This has been fixed by encoding the password hashes in 'base64'. Since this is a crippling bug and no one noticed before, I'm going to assume the feature is not being used in production and therefore a new non-backwards compatible version will be released. ### Chore ## [9.1.3] - 2025.07.04 ### Fix Handle Erase messages as Delete messages in valve permissions ## [9.1.2] - 2025.06.30 ### Fix Log metadata ## [9.1.1] - 2025.06.18 ### Fix - lint ## [9.1.0] - 2025.06.18 ### Fix - breaking change when using uws http server response.writeStatus must be called before any other method, otherwise all response status are 200. I decided not to make a major release because if somebody else was using it a bug report would have been made. And thanks to a LLM's for the quick refactoring! ## [9.0.1] - 2025.06.17 ### Feature Enable pino logger options thus making it possible to use transports ## [9.0.0] - 2025.06.13 ### Fix - Breaking change - The log level namig logic was not being implemented properly. logLevels as String where not converted to numbers, therefore those levels where not being properly enforced. The simplest path is just to use logLevels as numbers everywhere. This is a breaking change because config files with log levels as string will error at the config validation stage. ### Chore - Update deps ## [8.0.0] - 2025.04.26 ### Chore - Update nodejs to version 22.x - Update uWebsockets - Update deps ## [7.0.10] - 2024.03.06 ### Task - Separate docker ubuntu amd and arm releases due to intermitent build failures in arm. See https://github.com/nodejs/docker-node/issues/1335 ## [7.0.9] - 2024.03.04 ### Fix - http auth issue #1138 ## [7.0.8] - 2023.10.23 ### Task - multi arch docker image builds. Thanks @daanh432 ### Chore - update deps. Thanks @hugojosefson ## [7.0.7] - 2023.07.20 ### Fix - Cork uWebsocketjs http responses ## [7.0.6] - 2023.07.19 ### Task - update uWebsocketjs ## [7.0.5] - 2023.03.24 ### Fix - use cli/config file options for cluster ## [7.0.4] - 2023.03.08 ### Feature - deepstream cluster CLI enabled in order to run a cluster of deepstream servers on each available processor core - combine monitoring: now deepstream accepts an array of monitoring plugins in order to have separate plugins when it comes to monitoring messages, server activity and other custom functionality that might be required. For example you can run the included log monitoring, and an audit plugin that could save to another system/server which users are writing to which records by listening to incoming messages. This allows to create inmediate database replication strategies and so forth. ## [7.0.3] - 2023.03.02 ### Fix - set pkg entrypoint for daemon ## [7.0.2] - 2023.02.27 ### Fix - log monitoring - daemon issues due to pkg bug ## [7.0.1] - 2023.02.03 ### Fix - pkg issues ## [7.0.0] - 2023.02.03 ### Chore - Update nodejs to version 18.x - Update uWebsockets - Update deps ## [6.2.2] - 2022.12.22 ### Fix - Enforce http origins options - log a warning when a websocket message is not sent - uws server add maxBackpressure option, defaults to 1024*1024 ## [6.2.1] - 2022.06.20 ### Task - update deps ## [6.2.0] - 2022.06.05 ### Task - eliminate externalUrl from config and cluster messages since it was not used - set `provideRequestorName` and `provideRequestorData` as false by default to avoid overhead since from now on that data can actually be sent on rpc messages and handled on @deepstream/client > 6.0.2. ### Misc - update submodules and deps ## [6.1.2] - 2022.04.22 ### Fix - Revert to lockfileVersion 1 ## [6.1.1] - 2022.04.22 ### Fix - Check maxMessageSize limit on POST requests when using uws http server ### Chore - update @deepstream/client dev dependency and update e2e tests ## [6.1.0] - 2022.03.17 ### Fix - Send write ack error messages when cache, storage or permission fails. For this to work it requires client version >= 6 - Do not remove cluster nodes if cluster size is 1 ### Feature - Enable vertical cluster node option: Now we can use all available cores in a vertical cluster! By setting the cluster option to `clusterNode: { name: 'vertical' }` you can run as many deepstream instances you want in each of the server cores and it will be one syncronized cluster. ## [6.0.1] - 2022.01.02 ### Task - Check that executable works in pre-push hook - Use pkg instead of nexe ## [6.0.0] - 2021.12.12 ### Chore - Update nodejs to version 14.x - Update uWebsockets ## [5.2.6] - 2021.11.27 ### Misc - Update dependencies ## [5.2.5] - 2021.11.27 ### Misc - Update dependencies ## [5.2.4] - 2021.04.05 ### Misc - Disable telemetry ## [5.2.3] - 2021.03.09 ### Feature - enable combine authentication. Now when the auth config has more than one authentication strategy the server will query them in order untill one passes or all fail. ## [5.2.2] - 2021.03.01 ### Feature - enable querying for specific user presence on http endpoint ## [5.2.1] - 2021.02.24 ### Fixes - uws idleTimeout is in seconds! That's why it didn't closed the connection on time. - buffer ack messages - remove noDelay from default-options and use it on subscription registry as a param to enable/disable buffering ## [5.2.0] - 2021.02.17 ### Task - Github Actions CI/CD ## [5.1.8] - 2020.11.25 ### Misc - Updating uws to support apple silicon ### Fix - Increasing CI heartbeat timeouts ## [5.1.7] - 2020.11.25 ### Fix - Calling destroy on socketWrapper instead of close, since close is a reaction and destroy an action ## [5.1.6] - 2020.11.24 ### Fix - Fix #1091 heartbeat not working with node-http/ws This fix now adds a timestamps to every message frame recieved and sets up an interval per socket to check heartbeat exists. Didn't do any performance tests but I'm assuming having a single interval is cheaper than setting a timeout and canceling on each message. This fix also exposes that we don't serialize STATE_TOPIC_REGISTRY/NOT_SUBSCRIBED message correctly which throws an error and crashes the server. That fix will require more elbow grease and hasn't been reported so will just keep an eye on issues. ## [5.1.5] - 2020.10.29 ### Fix - Fix #1089 install service gives error ### Misc - Updating dependencies ## [5.1.4] - 2020.10.16 ### Fix - Fixed bug causing the server to crash with older sdks. ## [5.1.3] - 2020.08.08 ### Fix - Use upgrade property for uWebSockets http server. ### Misc - Do not run e2e:v3 test on the pre-push hook ## [5.1.2] - 2020.07.05 ### Misc - Updating all dependencies ## [5.1.1] - 2020.05.11 ### Fix Disabling telemetry for tests Fixing critical bug with client sdk version tracking ## [5.1.0] - 2020.05.11 ### Feat Adding telemetry. The server code is also in the rep (under telemetry-server). This uses a random uuid as your deploymentId, which is pretty much the only way I can avoid having thousands of records in the database from one machine restarting / ci process. If possible please use a different ID for production environments! ```yaml # This disables specific feature in DS, which is a more performant way # than disabling via permissions and is also how telemetry figures out # what features are enabled enabledFeatures: record: true event: true rpc: true presence: true telemetry: type: deepstreamIO options: # Disable telemetry entirely enabled: true # Prints whatever will be sent to the telemetry endpoint, # without actually sending it debug: false # An anonymous uuid that allows us to know its one unique # deployment. Please don't generate these randomly, it really # skews up analytics. This is in the config and user generated # because we don't want to deploymentId: ``` ### Fix Fixes by the awesome @jaime-ez around heartbeats and ping messages! ## [5.0.16] - 2020.04.30 ### Feat - Add two new plugins: - heap-snapshot This allows deepstream to save its heap space for analysis by v8 tools ```yaml plugins: heap-snapshot: name: 'heap-snapshot' options: interval: 60000 outputDir: file(../heap-snapshots) ``` - aws This is a general plugin for all AWS services, currently allows us to sync the heap-snapshot directory to S3 which is useful when running via docker. The functionality however is generic, so you could have a plugin that outputs other useful data. You can also very simply add more services (or more of the same ones). This just makes it easier for us to maintain the plugin. ```yaml plugins: aws: name: aws options: accessKeyId: ${AWS_ACCESS_KEY} secretAccessKey: ${AWS_SECRET_ACCESS_KEY} services: - type: s3-sync options: syncInterval: 60000 syncDir: file(../heap-snapshots) bucketName: ${SYNC_BUCKET_NAME} bucketRegion: ${AWS_DEFAULT_REGION} ``` - SocketData is passed to monitoring service in order to allow for fined grain monitoring. ### Fix - uws service accepts options requests and enforces CORS params ## [5.0.15] - 2020.04.24 ### Fix - Allowing valve file to be passed in via nodeJS config (@jaime-ez) ### Misc - Updating uws dependency ## [5.0.14] - 2020.04.19 ### Misc - Adding a log line for MQTT incoming connections for clarity ## [5.0.13] - 2020.04.16 ### Fix - Fixing issue where record updates via clusters didn't always get sent correctly to subscribers ### Misc - Updating dependencies ## [5.0.12] - 2020.03.06 ### Fix - Fixing issue where sending messages between multiple protocols can break. Verbose logging will be removed in the next release. ## [5.0.11] - 2020.03.05 ### Fix - adding debug logs for listening, and allowing to subscribers to listen to self ## [5.0.10] - 2020.03.05 ### Fix - Fixing log level output as debug logs are being ignored ## [5.0.9] - 2020.02.22 ### Misc - Attempt to fix npm publishing issue due to travis bug ## [5.0.8] - 2020.02.16 ### Fixes - Call onClientDisconnect with userId instead of socket ## [5.0.7] - 2020.02.08 ### Fixes - Fixes in storage auth handler by @abird ## [5.0.6] - 2020.02.08 ### Improvement - Updating all dependencies ### Fixes - Call onClientDisconnect from combined auth handler ## [5.0.5] - 2019.11.24 ### Feat - Adding winston to docker and package image ### Fix - Fixed critical issue where invalid websocket frames kill server ## [5.0.4] - 2019.11.05 ### Fix - Provide all HTTP headers to auth endpoint when using ws ## [5.0.3] - 2019.11.05 ### Feat - Adding a log monitoring plugin, useful for kibana log forwarding - Adding header authentication for http monitoring ## [5.0.2] - 2019.11.04 ### Fix - Production only dependency did not include ds-types - Close down MQTT server on stop ## [5.0.1] - 2019.11.02 ### Features - Replaces ENV variables in config loaded using fileLoader [#1022](https://github.com/deepstreamIO/deepstream.io/issues/1022) ### Fixes - Fixes odd types scripting issue breaking plugin interfaces when using `deepstream.getServices()` - Fixes hash generation via CLI [#1025](https://github.com/deepstreamIO/deepstream.io/issues/1025) ## [5.0.0] - 2019.10.27 ### Features: - New License - Singular HTTP Service - SSL Support reintroduced - Better Config file validation - JSON Logger - NGINX Helper - Combined authentication handler - Embedded dependencies - Builtin HTTP Monitoring - Storage authentication endpoint - Guess whats back, official clustering support! ### Backwards compatibility - Custom authentication plugins now have to use the async/await API - Custom permission handlers need to slightly tweak the function arguments - Deployment configuration has to be (simplified) to respect the single HTTP/Websocket port ### Upgrade guide You can see the upgrade guide for backwards compatibility [here](https://deepstream.io/tutorials/upgrade-guides/v5/server/) ## [4.2.5] - 2019.10.01 ### Fix Removing dom lib from typescript, which exposed variables that throw exceptions #1008 ## [4.2.4] - 2019.10.01 ### Fix Cannot read property 'N' of undefined #920 ## [4.2.3] - 2019.10.01 ### Improvement Hardening the config validator ### Fix Allow empty password for mqtt endpoint when authentication is not enabled (@Aapkostka) #1003 ResolvePluginClass should look for lowercase plugins #1002 RPC not routed to different deepstream node depending on startup order #1001 ### Misc Updating dependencies ## [4.2.2] - 2019.09.17 ### Fix Adding health-checks for all ws based endpoints. ## [4.2.1] - 2019.09.17 ### Fix Remove conflicting port for those starting using node with an empty config object. ### Improvement Limit the dead socket log to reduce insane spam. ## [4.2.0] - 2019.09.09 ### Feat Two new connection endpoints have been added. They are currently experimental and will be properly announced with associated documentation. One endpoint is mqtt! This allows us to support mqtt auth (using username and password), retain using records and QoS 1 using write acks. The only issue is since mqtt only supports one sort of concept (with flags distinguishing them) we bridge both events and records together. That means if you subscribe to 'temperature/london', you'll get the update from both a client doing `event.emit('temperature/london')` and `record.setData('temperature/london')`. The second endpoint is `ws-json` which allows users to interact with deepstream by just passing through json serialized text blobs instead of protobuf. This is mainly to help a few people trying to write SDKs without the hassle of a protobuf layer. Value also injects a `name` variable which allows you to reference the name your currently in. Useful for cross referencing. ### Fix Subscription registry seemed to have a massive leak when it came to dead sockets! This has now been fixed. The sockets seemed to have gotten the remove event deleted earlier in their lifecycle which prohibited it from doing a proper clean up later. ## [4.1.0] - 2019.08.30 ### Feat Backwards compatibility with V3 clients / text protocol using a ws-text connection endpoint This has a couple of small differences, like `has` is no longer supported and `snapshot` errors are exposed using the global `error` callback instead of via the response. Otherwise all the e2e tests work, and best of all you can run both at the same time if you want to run JS 4.0 and Java 3.0 simultaneously! It is worth keeping in mind there is a small CPU overhead between switching from V3 custom deepstream encoding to JSON (V4), so it is advised to monitor your CPU when possible! ``` - type: ws-text options: # port for the websocket server port: 6021 # host for the websocket server host: 0.0.0.0 ``` ## [4.0.6] - 2019.08.19 ### Feat Allow SUBSCRIBE and READ without CREATE actions, for clients that are in read only mode ### Improvement Adding declaration types (thank you @Vortex375!) ## [4.0.5] - 2019.08.09 ### Improvement Adding meta objects to logs and monitoring for easier tagging to monitoring solutions ## [4.0.4] - 2019.08.05 ### Fix - Don't buffer error messages in relation to connections, otherwise the client will get the close event first - Ignore ping messages during the connecting and authenticating stages ## [4.0.3] - 2019.08.04 ### Fix - Notify monitoring plugin of all messages sent out individually ## [4.0.2] - 2019.08.03 ### Features - Alpine docker image ### Fix - Override the http port and host correctly ## [4.0.1] - 2019.07.31 ### Improvements - Exit immediately if HTTP server port is occupied - When using debug, log exact error to why a plugin could not be loaded ## [4.0.0] - 2019.07.30 ### Features: - New protobuf protocol support (under the hood) - Bulk actions instead of individual subscribes (under the hood) - Official Plugin Support - Monitoring Support - Clustering Support (with small caveats) - Listening Discovery Simplification - V2 storage API - V2 cache API - Notify API ### Improvements - Lazy data parsing - Improved deepstream lifecycle - Upgraded development tools - New deepstream.io website ### Backwards compatibility - All V3 SDKs no longer compatible due to protobuf binary protocol ### Upgrade guide You can see the upgrade guide for backwards compatibility [here](https://deepstream.io/tutorials/upgrade-guides/v4/server/) ### TLDR; You can see the in depth side explanation of the changes [here](https://deepstream.io/releases/server/v4-0-0/) ## [3.1.0] - 2017.09.25 ### Features - a new standardised logging API with `debug`, `info`, `warn` and `error` methods - the presence feature can now be used on a per user basis. The online status of individual users can be queried for as well as subscribed to. Check out the tutorial on our website [here](https://deepstreamhub.com/tutorials/guides/presence/) ### Improvements - `perMessageDeflate` option can now be passed to uws, courtesy of [@daviderenger](@daviderenger) [#786](https://github.com/deepstreamIO/deepstream.io/pull/786) - various fixes and performance improvements to the subscription registry [#780](https://github.com/deepstreamIO/deepstream.io/pull/780), courtesy of [@ronag](@ronag). ### Fixes - allow updating and writing to Lists via the HTTP API [#788](https://github.com/deepstreamIO/deepstream.io/pull/788) courtesy of [@rbarroetavena](@rbarroetavena) - no data when sending HTTP requests is now considered undefined, rather than null [#798](https://github.com/deepstreamIO/deepstream.io/pull/798). ### Miscellaneous - internal refactor to pull e2e client operations into framework and abstract from Cucumber steps. ## [3.0.1] - 2017.08.14 ### Features - Added a `restart` option to deepstream service CLI - deepstream will now write to a pid file at `/var/run/deepstream/deepstream.pid` while running as a service - Authentication and permission plugins can now be configured via config and will be resolved as normal plugins. Either a path or name will need to be provided at the top level, and any options specified will also be passed in. ## [3.0.0] - 2017.07.26 ### Features #### [HTTP API](https://deepstreamhub.com/docs/http/v1/) Enabling clients to create, read, update and delete records, emit events, request RPCS and read presence using a JSON bulk request/response format via HTTP. - The HTTP API is enabled by default on PORT 8080 and can be configured in the connectionEndpoints -> http section of deepstream's `config.yml` - To disable the HTTP API set the above config to null #### [PHP Client Support](https://deepstreamhub.com/docs/client-php/DeepstreamClient/) The above HTTP API makes deepstream.io compatible with the deepstream PHP client #### Multi Endpoint Architecture The deepstream 3.0 release lays the groundwork for multiple combinable endpoints/protocols, e.g. GraphQL or Binary to be used together. It also introduces a new endpoint type enabling developers to write their own. Please note - at the moment it is not possible to run multiple subscription based endpoints (e.g. websocket) simultaneously. #### Message Connector Discontinuation To address the scalability issues associated with the message connector interface's coarse topics deepstream will move to a build-in, high performance p2p/small world network based clustering approach, available as an enterprise plugin. The current message connector support is discontinued. ### Miscellaneous - Moved end-to-end tests into this repository from `deepstream.io-client-js`. - Replaced `javascript-state-machine` dependency with custom state machine. ### Fixes - Improved handling of invalid record names. ## [2.4.0] - 2017.07.01 ## Features - Added new CLI command, including: + deepstream daemon This command forks deepstream and monitors it for crashes, allowing it to restart automatically to avoid downtime + deepstream service add This command allows you to create an init.d or systemd script automatically and add it to your system. ```bash sudo deepstream service --help Usage: service [options] [add|remove|start|stop|status] Add, remove, start or stop deepstream as a service to your operating system ``` - Added brew cask support You can now install easily install deepstream on your mac using `brew cask install deepstream` driven by config files within `/user/local/etc/deepstream/conf` ## Fixes - Fix issue where certain invalid paths would return 'Invalid Type' on the server. - Fix issue in request/response where selecting a remote server as not done uniformly. ## [2.3.7] - 2017.06.20 ## Fixes - Fix issue where using both `.0.` and `[0]` within a json path resulted in inserting into an array. However, when using other SDKs such as Java they would be treated as an Object key or array index. - Fix issue where nested array access/manipulation didn't work via json paths. ## Compatability Issue Due to the nature of this fix, it may result in compatability issues with applications that used json paths incorrectly ( using `.0.` intead of `[0]` ). Please ensure you change those before upgrading. ## [2.3.6] - 2017.06.12 ## Fixes - Fix for issue [#703](https://github.com/deepstreamIO/deepstream.io/issues/703) where record deletions were not being propogated correctly within a cluster. - Fixes config-loading issue present in the binary release of 2.3.5. ## [2.3.5] - 2017.06.12 ## Fixes - Hardcode v3.0.0-rc1 dependency on javascript-state-machine, as v3.0.1 causes deepstream.io startup to fail ## [2.3.4] - 2017.06.02 ## Fixes - Hot path needs to store values in the correct format ## [2.3.3] - 2017.06.02 ### Fixes - Binary config files have the correct latest structure - Fix an issue where heavy concurrent writes on the same record fail ## [2.3.2] - 2017.05.31 ### Fixes - Fixing a connection data regression where it wasn't formatted the same as pre 2.3.0 ## [2.3.1] - 2017.05.30 ### Fixes - Correctly merging config options from `config.yml` file with the default options ## [2.3.0] - 2017.05.29 ### Features - Adds "storageHotPathPatterns" config option. - Adds support for `setData()`: upsert-style record updates without requiring that a client is subscribed to the record. This uses a new 'CU' (Create and Update) message. The `setData()` API is up to 10x faster than subscribing, setting, then discarding a record. - Support for connection endpoint plugins. ### Enhancements - Significant performance improvements stemming from message batching. ### Miscellaneous - Moved uws into a connection endpoint plugin. - Explicit state-machine that initializes and closes dependencies in a well-defined order. ## [2.2.2] - 2017.05.03 ### Enhancements - Adds support for custom authentication and permissioning plugins. - Adds support for generic plugins. ### Fixes - Added check to ensure subscriptions are not removed from distributed state registry prematurely. ## [2.2.1] - 2017.04.24 ### Enhancements - Unsolicited RPCs now get a `INVALID_RPC_CORRELATION_ID` message ### Fixes - RPC lifecycles have been improved and don't throw exceptions on response after a timeout by [ronag](ronag) - Correct options now being passed into the `RuleCache`, courtesy of [ralphtheninja](ralphtheninja) ## [2.2.0] - 2017.04.08 ### Enhancements - Records now can be set with a version -1, which ignores version conflicts by [datasage](datasage) - Delete events are now propagated in the correct order by [datasage](datasage) - You can now request the HEAD of a record to retrieve just its version number by [datasage](datasage) - Providers for listeners are now by default selected randomly instead of in order of subscription - Ensure record updates are not scalar values before trying to save them in cache by [datasage](datasage) - Long lived RPC requests now use dynamic lookups for providers rather than building the Set upfront by [ronag]{ronag} - Huge optimization to subscription registry, where the time for registering a subscriber has been reduced from n^2 to O(n log n) ### Miscellaneous - Deleting grunt since everything is script based ## [2.1.6] - 2017.03.29 ### Miscellaneous - Due to uws releases being pulled from NPM, we're now using uws from a git repo - Created a separate repo [uws-dependency](https://github.com/deepstreamIO/uws-dependency) with binaries. ## [2.1.4 - 2.1.5] - Due to problems with build resulting from uws unpublishing, these two npm packages have been unpublished (noop) ## [2.1.3] - 2017.02.25 ### Bug Fixes - Unsolicited message in Listening when all clients unsubscribe [#531] - Handle Non text based websocket frame [#538] - Aligning binary config with node [#488] - Event subscription data mishandled in Valve [#510] - Logging after logger is destroyed [#527] - Deepstream crash on empty users file [#512] - Logging error object instead of name in connection error [#420] ### Enhancements - maxRuleIterations must be 1 or higher [#498] - Ignore sender in subscriptionRegistry if messagebus [#473] - Removing dead config options [#599] - getAlternativeProvider in RPC Handler deals with more edge cases [#566] - Update UWS build version to 0.12 - Packages built against node 6.10 ## [2.1.2] - 2016.12.28 ### Bug fixes - Fixing write error where only initial value is written to storage [#517](https://github.com/deepstreamIO/deepstream.io/issues/517) ## [2.1.1] - 2016.12.28 ### Bug fixes - Valve cross referencing in both a create and read results in a ack timeout [#514](https://github.com/deepstreamIO/deepstream.io/issues/514) ## [2.1.0] - 2016.12.20 ### Features - Record write acknowledgement. Records are now able to be set with an optional callback which will be called with any errors from storing the record in cache/storage [#472](https://github.com/deepstreamIO/deepstream.io/pull/472) ### Enhancements - Applying an ESLint rule set to the repo [#482](https://github.com/deepstreamIO/deepstream.io/pull/482) - Stricter valve permissioning language checks [#486](https://github.com/deepstreamIO/deepstream.io/pull/486) by [@Iiridayn](https://github.com/Iiridayn) - Update uWS version to [v0.12.0](https://github.com/uWebSockets/uWebSockets/releases/tag/v0.12.0) ### Bug fixes - Better handling/parsing of authentication messages [#463](https://github.com/deepstreamIO/deepstream.io/issues/463) - Properly returning handshake data (headers) from SocketWrapper [#450](https://github.com/deepstreamIO/deepstream.io/issues/450) - Fix case where CLIENT_DISCONNECTED is not sent from SocketWrapper [#470](https://github.com/deepstreamIO/deepstream.io/issues/470) - Fixed issue where listen does not recover from server restart [#476](https://github.com/deepstreamIO/deepstream.io/issues/476) - Handling presence events properly. Now when a user logs in, subscribed clients are only notified the first time the user logs in, and the last time they log out [#499](https://github.com/deepstreamIO/deepstream.io/pull/499) ## [2.0.1] - 2016.11.21 ### Bug Fixes - Fixed issue where connectionData was not available in auth requests [#450](https://github.com/deepstreamIO/deepstream.io/issues/450) - Changelog of 2.0.0 mistakenly said that heartbeats were on port 80 instead of 6020 ## [2.0.0] - 2016.11.18 ### Features - User presence has been added, enabling querying and subscription to who is online within a cluster - Introduces the configuration option `broadcastTimeout` to `config.yml` to allow coalescing of broadcasts. This option can be used to improve broadcast message latency such as events, data-sync and presence For example, the performance of broadcasting 100 events to 1000 subscribers was improved by a factor of 20 - Adds client heartbeats, along with configuration option`heartbeatInterval` in `config.yml`. If a connected client fails to send a heartbeat within this timeout, it will be considered to have disconnected [#419](https://github.com/deepstreamIO/deepstream.io/issues/419) - Adds healthchecks – deepstream now responds to http GET requests to path `/health-check` on port 6020 with code 200. This path can be configured with the `healthCheckPath` option in `config.yml` ### Enhancements - E2E tests refactored - uWS is now compiled into the deepstream binary, eliminating reliability issues caused by dynamic linking ### Breaking Changes - Clients prior to v2.0.0 are no longer compatible - Changed format of RPC request ACK messages to be more consistent with the rest of the specs [#408](https://github.com/deepstreamIO/deepstream.io/issues/408) - We removed support for TCP and engine.io, providing huge performance gains by integrating tightly with native uWS - Support for webRTC has been removed - You can no longer set custom data transforms directly on deepstream ## [1.1.2] - 2016-10-17 ### Bug Fixes - Sending an invalid connection message is not caught by server [#401](https://github.com/deepstreamIO/deepstream.io/issues/401) ## [1.1.1] - 2016-09-30 ### Bug Fixes - Storage connector now logs errors with the correct namepspace [@Iiridayn](@Iiridayn) ### Enhancements - RPC now uses distributed state and no longer depends on custom rpc discovery logic - Deepstream now uses connection challenges by default rather than automatically replying with an ack - Upgraded to uWS 0.9.0 ## [1.1.0] - 2016-09-08 ### Bug Fixes - Fix wrong validation of Valve Permissions when `data` is used as a property [#346](https://github.com/deepstreamIO/deepstream.io/pull/346) ### Enhancements - Outgoing connections now have throttle options that allow you to configure maximum package sizes to find your personal sweet spot between latency and speed ```yaml # the time (in milliseconds) to wait for a buffer to fill before sending it out timeBetweenSendingQueuedPackages: 1 # the amount of messages that should fit into a buffer before sending between the time to fill maxMessagesPerPacket: 1000 ``` ### Features - Listening: Listeners have been drastically improved [https://github.com/deepstreamIO/deepstream.io/issues/211], and now: - works correctly across a cluster - can inform the user whenever the last subscription has been removed even if the listener itself is subscribed - only allows a single listener to provide a record - has a concept of provided, allowing records on the client side to be aware if the data is being actively updated by a backend component As part of this story, we now have multiple significant improvements in the server itself, such as: - a `distributed state registry` which allows all clusters to keep their state in sync - a `unique state provider` allowing cluster wide locks - a `cluster-registry` that provides shares server presence and state across the cluster Because of these we can now start working on some really cool features such as advanced failover, user presence and others! ## [1.0.4] - 2016-08-16 ### Bug Fixes - Auth: File authentication sends server data to client on cleartext passwords [#322](https://github.com/deepstreamIO/deepstream.io/issues/322) - Auth: HTTP authentication missing logger during when attempting to log any errors occured on http server [#320](https://github.com/deepstreamIO/deepstream.io/issues/320) ## [1.0.3] - 2016-07-28 ### Bug Fixes - CLI: installer for connectors sometimes fail to download (and extract) the archive [#305](https://github.com/deepstreamIO/deepstream.io/issues/305) - Auth: File authentication doesn't contain `serverData` and `clientData` [#304](https://github.com/deepstreamIO/deepstream.io/issues/304) ###### Read data using `FileAuthentication` using clientData and serverData rather than data ```yaml userA: password: tsA+yfWGoEk9uEU/GX1JokkzteayLj6YFTwmraQrO7k=75KQ2Mzm serverData: role: admin clientData: nickname: Dave ``` ### Features ###### Make connection timeout Users can now provide a `unauthenticatedClientTimeout` config option that forces connections to close if they don't authenticate in time. This helps reduce load on server by terminating idle connections. - `null`: Disable timeout - `number`: Time in milliseconds before connection is terminated ## [1.0.2] - 2016-07-19 ### Bug Fixes - Fixed issue regarding last subscription to a deleted record not being cleared up ## [1.0.1] - 2016-07-18 ### Bug Fixes - Fix issue when try to pass options to the default logger [#288](https://github.com/deepstreamIO/deepstream.io/pull/288) ([update docs](https://github.com/deepstreamIO/deepstream.io-website/pull/35/commits/838617d93cf00e66176cdf06d161fd8f86574aa1) as well) - Fix issue deleting a record does not unsubscribe it and all other connections, not allowing resubscriptions to occur #293 #### Enhancements ###### Throw better error if dependency doesn't implement Emitter or isReady ## [1.0.0] - 2016-07-09 ### Features ###### CLI You can start deepstream via a command line interface. You find it in the _bin_ directory. It provides these subcommands: - `start` - `stop` - `status` - `install` - `info` - `hash` append a `--help` to see the usage. ###### File based configuration You can now use a file based configuration instead of setting options via `ds.set(key, value)`. deepstream is shipped with a _conf_ directory which contains three files: - __config.yml__ this is the main config file, you can specify most of the deepstream options in that file - __permissions.yml__ this file can be consumed by the PermissionHandler. It's not used by default, but you can enable it in the _config.yml_ - __users.yml__ this file can be consumed by the AuthenticationHandler. It's not used by default, but you can enable it in the _config.yml_ For all config types support these file types: __.yml__, __.json__ and __.js__ ###### Constructor API There are different options what you can pass: - not passing any arguments ( consistent with 0.x ) - passing `null` will result in loading the default configuration file in the directory _conf/config.yml_ - passing a string which is a path to a configuration file, supported formats: __.yml__, __.json__ and __.js__ - passing an object which defines several options, all other options will be merged from deepstream's default values ###### Valve Permissions rules You can write your permission into a structured file. This file supports a special syntax, which allows you to do advanced permission checks. This syntax is called __Valve__. #### Enhancements ###### uws deepstream now uses [uws](https://github.com/uWebSockets/uWebSockets), a native C++ websocket server ###### no process.exit on plugin initialization error or timeout deepstream will not longer stops your process via `process.exit()`. This happened before when a connector failed to initialize correctly [#243](https://github.com/deepstreamIO/deepstream.io/issues/243) instead it will throw an error now. Currently the API provides no event or callback to handle this error other than subscribing to the global `uncaughtException` event. ```javascript process.once('uncaughtException', err => { // err.code will equal to of these constant values: // C.EVENT.PLUGIN_INITIALIZATION_TIMEOUT // or C.EVENT.PLUGIN_INITIALIZATION_ERROR }) ``` Keep in mind that deepstream will be in an unpredictable state and you should consider to create a new instance. ### Breaking Changes ###### Permission Handler In 0.x you can set a `permissionHandler` which needs to implement two functions: - `isValidUser(connectionData, authData, callback)` - `canPerformAction(username, message, callback)` In deepstream 1.0 the `isValidUser` and `onClientDisconnect` methods are no longer part of the `permissionHandler` and are instead within the new `authenticationHandler`. You can reuse the same 0.x permission handler except you will have to set it on both explicitly. ```javascript const permissionHandler = new CustomPermissionHandler() ds.set( 'permissionHandler', permissionHandler ) ds.set( 'authenticationHandler', permissionHandler ) ``` ###### Plugin API All connectors including, the `permissionHandler`, `authenticationHandler` and `logger` all need to implement the plugin interface which means exporting an object that: - has a constructor - has an `isReady` property which is true once the connector has been initialized. For example in the case a database connector this would only be `true` once the connection has been established. If the connector is synchronous you can set this to true within the constructor. - extends the EventEmitter, and emits a `ready` event once initialized and `error` on error. ###### Logger and colors options The color flag can't be set in the root level of the configuration anymore. The default logger will print logs to the StdOut/StdErr in colors. You can use the [deepstream.io-logger-winston](https://www.npmjs.com/package/deepstream.io-logger-winston) which can be configured in the config.yml file with several options. ###### Connection redirects deepstream clients now have a handshake protocol which allows them to be redirected to the most efficient node and expect an initial connection ack before logging in. As such In order to connect a client to deepstream server you need also to have a client with version 1.0 or higher. More details in the [client changelog](https://github.com/deepstreamIO/deepstream.io-client-js/blob/master/CHANGELOG.md). ================================================ FILE: Dockerfile ================================================ FROM node:22 as builder WORKDIR /app COPY package*.json ./ RUN npm ci RUN npm install --omit=dev \ @deepstream/cache-redis \ # @deepstream/cache-memcached \ # @deepstream/cache-hazelcast \ @deepstream/clusternode-redis \ @deepstream/storage-mongodb \ @deepstream/storage-rethinkdb \ @deepstream/storage-elasticsearch \ @deepstream/storage-postgres \ @deepstream/logger-winston \ @deepstream/plugin-aws COPY . . RUN npm run tsc FROM node:22 WORKDIR /usr/local/deepstream COPY --from=builder /app/node_modules/ ./node_modules COPY --from=builder /app/dist/ . EXPOSE 6020 EXPOSE 8080 EXPOSE 9229 CMD ["node", "./bin/deepstream.js", "start", "--inspect=0.0.0.0:9229"] ================================================ FILE: Dockerfile.alpine ================================================ FROM node:22-alpine as builder WORKDIR /app # RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" # RUN unzip awscliv2.zip # RUN ./aws/install COPY package*.json ./ RUN npm ci RUN npm install --omit=dev \ @deepstream/cache-redis \ # @deepstream/cache-memcached \ # @deepstream/cache-hazelcast \ @deepstream/clusternode-redis \ @deepstream/storage-mongodb \ @deepstream/storage-rethinkdb \ @deepstream/storage-elasticsearch \ @deepstream/storage-postgres \ @deepstream/logger-winston \ @deepstream/plugin-aws RUN npm uninstall --save uWebSockets.js COPY . . RUN npm run tsc FROM node:22-alpine WORKDIR /usr/local/deepstream COPY --from=builder /app/node_modules/ ./node_modules COPY --from=builder /app/dist/ . EXPOSE 6020 EXPOSE 8080 EXPOSE 9229 CMD ["node", "./bin/deepstream.js", "start", "--inspect=0.0.0.0:9229"] ================================================ FILE: LICENSE ================================================ Copyright 2019 deepstreamHub GmbH Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # deepstream - the open realtime server deepstream deepstream is an open source server inspired by concepts behind financial trading technology. It allows clients and backend services to sync data, send messages and make rpcs at very high speed and scale. [![npm version](https://badge.fury.io/js/%40deepstream%2Fserver.svg)](https://badge.fury.io/js/%40deepstream%2Fserver)[![Docker Stars](https://img.shields.io/docker/pulls/deepstreamio/deepstream.io.svg)](https://hub.docker.com/r/deepstreamio/deepstream.io/) deepstream has three core concepts for enabling realtime application development - **records** ([realtime document sync](https://deepstreamio.github.io/docs/tutorials/core/datasync/records)) records are schema-less, persistent documents that can be manipulated and observed. Any change is synchronized with all connected clients and backend processes in milliseconds. Records can reference each other and be arranged in lists to allow modelling of relational data - **events** ([publish subscribe messaging](https://deepstreamio.github.io/docs/tutorials/core/pubsub/events)) events allow for high performance, many-to-many messaging. deepstream provides topic based routing from sender to subscriber, data serialisation and subscription listening. - **rpcs** ([request response workflows](https://deepstreamio.github.io/docs/tutorials/core/request-response/rpc)) remote procedure calls allow for secure and highly available request response communication. deepstream handles load-balancing, failover, data-transport and message routing. - **security** ([Authentication](https://deepstreamio.github.io/docs/tutorials/core/auth/auth-introduction) and [Permissions](https://deepstreamio.github.io/docs/tutorials/core/permission/valve-introduction)) deepstream offers a combination of different authentication mechanisms with a powerful permission-language called Valve that allows you to specify which user can perform which action with which data. ### Getting Started: 1. [Tutorials - What is deepstream](https://deepstreamio.github.io/docs/tutorials/concepts/what-is-deepstream) 2. [Installing deepstream](https://deepstreamio.github.io/docs/tutorials/install/linux) 3. [Quickstart](https://deepstreamio.github.io/docs/tutorials/getting-started/javascript) 4. [Documentation](https://deepstreamio.github.io/docs/docs) ### Community Links 1. [Stack Overflow](https://stackoverflow.com/questions/tagged/deepstream.io) 2. [Github Discussions](https://github.com/deepstreamIO/deepstreamIO.github.io/discussions) ### Contributing deepstream development is a great way to get into building performant Node.js applications, and contributions are always welcome with lots of ❤. Contributing to deepstream is as simple as having Node.js (10+) and TypeScript (3+) installed, cloning the repo and making some changes. ``` ~ » git clone git@github.com:deepstreamIO/deepstream.io.git ~ » cd deepstream.io ~/deepstream.io » git submodule update --init ~/deepstream.io » npm i ~/deepstream.io » npm start _ _ __| | ___ ___ _ __ ___| |_ _ __ ___ __ _ _ __ ____ / _` |/ _ \/ _ \ '_ \/ __| __| '__/ _ \/ _` | '_ ` _ \ | (_| | __/ __/ |_) \__ \ |_| | | __/ (_| | | | | | | \__,_|\___|\___| .__/|___/\__|_| \___|\__,_|_| |_| |_| |_| ===================== starting ===================== ``` From here you can make your changes, and check the unit tests pass: ``` ~/deepstream.io » npm t ``` If your changes are substantial you can also run our extensive end-to-end testing framework: ``` ~/deepstream.io » npm run e2e ``` For power users who want to make sure the binary works, you can run `sh scripts/package.sh true`. You'll need to download the usual [node-gyp](https://github.com/nodejs/node-gyp) build environment for this to work and we only support the latest LTS version to compile. This step is usually not needed though unless your modifying resource files or changing dependencies. ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Reporting a Vulnerability Report the vulnerability in the security [advisory section](https://github.com/deepstreamIO/deepstream.io/security) of the repository. If your submission is valid, a GitHub advisory will be published and if present in NPM or other released packages, a CVE will be requested. ================================================ FILE: ascii-logo.txt ================================================ _ _ __| | ___ ___ _ __ ___| |_ _ __ ___ __ _ _ __ ____ / _` |/ _ \/ _ \ '_ \/ __| __| '__/ _ \/ _` | '_ ` _ \ | (_| | __/ __/ |_) \__ \ |_| | | __/ (_| | | | | | | \__,_|\___|\___| .__/|___/\__|_| \___|\__,_|_| |_| |_| |_| ================================================ FILE: bin/deepstream-cluster.ts ================================================ import { Command } from 'commander' import * as cluster from 'cluster' const numCPUs = require('os').cpus().length import { EVENT } from '@deepstream/types' export const verticalCluster = (program: Command) => { program .command('cluster') .description('start a vertical cluster of deepstream servers') .option('-c, --config [file]', 'configuration file, parent directory will be used as prefix for other config files') .option('-l, --lib-dir [directory]', 'path where to lookup for plugins like connectors and logger') .option('--cluster-size ', 'the amount of nodes to run in the cluster. Defaults to all available cores') .option('--host ', 'host for the http service') .option('--port ', 'port for the http service') .option('--disable-auth', 'Force deepstream to use "none" auth type') .option('--disable-permissions', 'Force deepstream to use "none" permissions') .option('--log-level ', 'Log messages with this level and above') .action(action) } function action () { // @ts-ignore global.deepstreamCLI = this const workers = new Set() if (!global.deepstreamCLI.clusterSize) { global.deepstreamCLI.clusterSize = numCPUs } if (global.deepstreamCLI.clusterSize && global.deepstreamCLI.clusterSize > numCPUs) { console.warn('Setting more nodes than available cores can decrease performance') } const setupWorkerProcesses = () => { console.log('Master cluster setting up ' + global.deepstreamCLI.clusterSize + ' deepstream nodes') for (let i = 0; i < global.deepstreamCLI.clusterSize; i++) { workers.add(cluster.fork()) } // process is clustered on a core and process id is assigned cluster.on('online', (worker) => { console.log(`Deepstream ${worker.process.pid} is listening`) }) // if any of the worker process dies then start a new one by simply forking another one cluster.on('exit', (worker, code, signal) => { console.log(`Deepstream ${worker.process.pid} died with code: ${code}, and signal: ${signal}`) console.log('Starting a new worker') workers.delete(worker) workers.add(cluster.fork()) }) } // if it is a master process then call setting up worker process // @ts-ignore if (cluster.isPrimary) { setupWorkerProcesses() } else { const { Deepstream } = require('../src/deepstream.io') try { const ds = new Deepstream(null) ds.on(EVENT.FATAL_EXCEPTION, () => process.exit(1)) ds.start() process .removeAllListeners('SIGINT').on('SIGINT', () => { ds.on('stopped', () => process.exit(0)) ds.stop() }) } catch (err: any) { console.error(err.toString()) process.exit(1) } } } ================================================ FILE: bin/deepstream-daemon.ts ================================================ // @ts-ignore import * as dsDaemon from '../src/service/daemon' import { Command } from 'commander' export const daemon = (program: Command) => { program .command('daemon') .description('start a daemon for deepstream server') .option('-c, --config [file]', 'configuration file, parent directory will be used as prefix for other config files') .option('-l, --lib-dir [directory]', 'path where to lookup for plugins like connectors and logger') .option('--host ', 'host for the http service') .option('--port ', 'port for the http service') .option('--disable-auth', 'Force deepstream to use "none" auth type') .option('--disable-permissions', 'Force deepstream to use "none" permissions') .option('--log-level ', 'Log messages with this level and above') .action(action) } function action () { dsDaemon.start({ processExec: process.argv[1] }) } ================================================ FILE: bin/deepstream-hash.ts ================================================ import * as jsYamlLoader from '../src/config/js-yaml-loader' import { Command } from 'commander' import { createHash } from '../src/utils/utils' export const hash = (program: Command) => { program .command('hash [password]') .description('Generate a hash from a plaintext password using file auth configuration settings') .option('-c, --config [file]', 'configuration file containing file auth and hash settings') .action(action) } async function action (this: any, password: string) { // @ts-ignore global.deepstreamCLI = this const config = (await jsYamlLoader.loadConfigWithoutInitialization()).config const fileAuthHandlerConfig = config.auth.find((auth) => auth.type === 'file') if (fileAuthHandlerConfig === undefined) { console.error('Error: Can only use hash with file authentication as auth type') return process.exit(1) } if (!fileAuthHandlerConfig.options.hash) { console.error('Error: Can only use hash with file authentication') return process.exit(1) } fileAuthHandlerConfig.options.path = '' if (!password) { console.error('Error: Must provide password to hash') return process.exit(1) } const { iterations, keyLength, hash: algorithm } = fileAuthHandlerConfig.options try { const { hash: generatedHash, salt } = await createHash(password, { iterations, keyLength, algorithm }) console.log(`Password hash: ${generatedHash.toString('base64')}${salt}`) } catch (e) { console.error('Hash could not be created', e) process.exit(1) } } ================================================ FILE: bin/deepstream-info.ts ================================================ import * as jsYamlLoader from '../src/config/js-yaml-loader' import { Command } from 'commander' import { getDSInfo } from '../src/config/ds-info' export const info = (program: Command) => { program .command('info') .description('print meta information about build and runtime') .option('-c, --config [file]', 'configuration file containing lib directory') .option('-l, --lib-dir [directory]', 'directory of libraries') .action(printMeta) } async function printMeta (this: any) { if (!this.libDir) { try { // @ts-ignore global.deepstreamCLI = this await jsYamlLoader.loadConfigWithoutInitialization() // @ts-ignore this.libDir = global.deepstreamLibDir } catch (e) { console.log(e) console.error('Please provide a libDir or a configFile to provide the relevant install information') process.exit(1) } } const dsInfo = await getDSInfo(this.libDir) console.log(JSON.stringify(dsInfo, null, 2)) } ================================================ FILE: bin/deepstream-nginx.ts ================================================ // @ts-ignore import * as dsService from '../src/service/service' import { Command } from 'commander' import { writeFileSync } from 'fs' import * as jsYamlLoader from '../src/config/js-yaml-loader' import * as fileUtil from '../src/config/file-utils' export const nginx = (program: Command) => { program .command('nginx') .description('Generate an nginx config file for deepstream') .option('-c, --config [file]', 'The deepstream config file') .option('-p, --port', 'The nginx port, defaults to 8080') .option('-h, --host', 'The nginx host, defaults to localhost') .option('--ssl', 'If ssl encryption should be added') .option('--ssl-cert', 'The SSL Certificate') .option('--ssl-key', 'The SSL Key') .option('-o, --output [file]', 'The file to save the configuration to') .action(execute) } async function execute (this: any, action: string) { // @ts-ignore global.deepstreamCLI = this const { config: dsConfig } = await jsYamlLoader.loadConfigWithoutInitialization() if (this.ssl && (!this.sslCert || !this.sslKey)) { console.error('Missing --ssl-cert or/key --ssl-key options') process.exit(1) } const sslConfig = ` ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers HIGH:!aNULL:!MD5; ssl_certificate ${this.sslCert}; ssl_certificate_key ${this.sslKey}; ` const websocketConfig = dsConfig.connectionEndpoints.reduce((result: string, endpoint: any) => { if (endpoint.options.urlPath) { return result + ` location ${endpoint.options.urlPath} { proxy_pass http://deepstream; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; } ` } return result }, '') let httpConfig = '' const http = dsConfig.connectionEndpoints.find((endpoint: any) => endpoint.type === 'http') if (http) { const paths = new Set([http.options.getPath, http.options.postPath, http.options.authPath]) httpConfig = [...paths].reduce((result, path) => { return result + ` location ${path} { proxy_pass http://deepstream; proxy_http_version 1.1; } ` }, '') } const config = ` worker_processes 1; events { worker_connections 1024; } http { map $http_upgrade $connection_upgrade { default upgrade; '' close; } upstream deepstream { server ${dsConfig.httpServer.options.host}:${dsConfig.httpServer.options.port}; # Insert more deepstream hosts / ports here for clustering to magically work } server { listen ${this.port || 8080}${this.ssl ? ' ssl' : ''}; server_name ${this.serverName || 'localhost'}; ${this.ssl ? sslConfig : ''} ${websocketConfig} ${httpConfig} } }` if (this.output) { writeFileSync(fileUtil.lookupConfRequirePath(this.output), config, 'utf8') console.log(`Configuration written to ${fileUtil.lookupConfRequirePath(this.output)}`) } else { console.log(config) } } ================================================ FILE: bin/deepstream-service.ts ================================================ // @ts-ignore import * as dsService from '../src/service/service' import { Command } from 'commander' export const service = (program: Command) => { program .command('service [add|remove|start|stop|restart|status]') .description('Add, remove, start or stop deepstream as a service to your operating system') .option('-c, --config [file]', 'configuration file, parent directory will be used as prefix for other config files') .option('-n, --service-name ', 'the name to register the service') .option('-l, --log-dir ', 'the directory for output logs') .option('-p, --pid-directory ', 'the directory for the pid file') .option('--dry-run', 'outputs the service file to screen') .action(execute) } function response (error: Error | string | null, result: string) { if (error) { console.log(error) } else { console.log(result) } } function execute (this: any, action: string) { const name = this.serviceName || 'deepstream' if (action === 'add') { if (!this.logDir || !this.config) { console.error('Please provide the config and log directory when adding a service') process.exit(1) } const options = { exec: process.argv[1], programArgs: [] as string[], pidFile: this.pidFile || `/var/run/deepstream/${name}.pid`, logDir: this.logDir, dryRun: this.dryRun } if (this.config) { options.programArgs.push('-c') options.programArgs.push(this.config) } dsService.add (name, options, response) } else if (action === 'remove') { dsService.remove (name, response) } else if (action === 'start' ) { dsService.start (name, response) } else if (action === 'stop' ) { dsService.stop (name, response) } else if (action === 'status') { dsService.status(name, response) } else if (action === 'restart') { dsService.restart(name, response) } else { console.log('Unknown action for service, please "add", "remove", "start", "stop", "restart" or "status"') } } ================================================ FILE: bin/deepstream-start.ts ================================================ import { Command } from 'commander' import { EVENT } from '@deepstream/types' export const start = (program: Command) => { program .command('start', { isDefault: true }) .description('start a deepstream server') .option('-c, --config [file]', 'configuration file, parent directory will be used as prefix for other config files') .option('-l, --lib-dir [directory]', 'path where to lookup for plugins like connectors and logger') .option('--host ', 'host for the http service') .option('--port ', 'port for the http service', (value: string) => parseInteger('--port', value)) .option('--disable-auth', 'Force deepstream to use "none" auth type') .option('--disable-permissions', 'Force deepstream to use "none" permissions') .option('--log-level ', 'Log messages with this level and above', parseLogLevel) .option('--colors [true|false]', 'Enable or disable logging with colors', (value: string) => parseBoolean('--colors', value)) .option('--inspect ', 'Enable node inspector') .action(action) } function action () { // @ts-ignore global.deepstreamCLI = this // @ts-ignore const inspectUrl = global.deepstreamCLI.inspect if (inspectUrl) { const inspector = require('inspector') // @ts-ignore const [host, port] = global.deepstreamCLI.inspect.split(':') if (!host || !port) { throw new Error('Invalid inspect url, please provide host:port') } inspector.open(port, host) } const { Deepstream } = require('../src/deepstream.io') try { const ds = new Deepstream(null) ds.on(EVENT.FATAL_EXCEPTION, () => process.exit(1)) ds.start() process .removeAllListeners('SIGINT').on('SIGINT', () => { ds.on('stopped', () => process.exit(0)) ds.stop() }) } catch (err) { console.error(err?.toString()) process.exit(1) } } /** * Used by commander to parse the log level and fails if invalid * value is passed in */ function parseLogLevel (logLevel: string) { if (!/debug|info|warn|error|off/i.test(logLevel)) { console.error('Log level must be one of the following (debug|info|warn|error|off)') process.exit(1) } return logLevel.toUpperCase() } /** * Used by commander to parse numbers and fails if invalid * value is passed in */ function parseInteger (name: string, port: string) { const portNumber = Number(port) if (!portNumber) { console.error(`Provided ${name} must be an integer`) process.exit(1) } return portNumber } /** * Used by commander to parse boolean and fails if invalid * value is passed in */ function parseBoolean (name: string, enabled: string) { let isEnabled if (typeof enabled === 'undefined' || enabled === 'true') { isEnabled = true } else if (typeof enabled !== 'undefined' && enabled === 'false') { isEnabled = false } else { console.error(`Invalid argument for ${name}, please provide true or false`) process.exit(1) } return isEnabled } ================================================ FILE: bin/deepstream.ts ================================================ #!/usr/bin/env node import * as pgk from '../package.json' import { Command } from 'commander' import { start } from './deepstream-start' import { info } from './deepstream-info' import { hash } from './deepstream-hash' import { service } from './deepstream-service' import { daemon } from './deepstream-daemon' import { verticalCluster } from './deepstream-cluster' import { nginx } from './deepstream-nginx' const program = new Command('deepstream') program .usage('[command]') .version(pgk.version.toString()) start(program) info(program) hash(program) service(program) daemon(program) verticalCluster(program) nginx(program) program.parse(process.argv) ================================================ FILE: conf/config.yml ================================================ # General # Each server within a cluster needs a unique name. Set to UUID to have deepstream autogenerate a unique id serverName: UUID # Show the deepstream logo on startup showLogo: true # Plugin startup timeout – deepstream init will fail if any plugins fail to emit a 'done' event within this timeout dependencyInitializationTimeout: 5000 # Directory where all plugins reside #libDir: ../lib # Exit the process a fatal error occurs, like losing a cache connection exitOnFatalError: false # Log messages with this level and above. Valid levels are 0 (DEBUG), 1 (INFO), 2 (WARN), 3 (ERROR), 4 (FATAL), 100(OFF) logLevel: 0 # This disables specific feature in DS, which is a more performant way # than disabling via permissions and is also how telemetry figures out # what features are enabled enabledFeatures: record: true event: true rpc: true presence: true telemetry: type: deepstreamIO options: # Disable telemetry entirely enabled: false # Prints whatever will be sent to the telemetry endpoint, # without actually sending it debug: false # An anonymous uuid that allows us to know its one unique # deployment. Please don't generate these randomly if using # node, it really skews up analytics. # deploymentId: rpc: # Timeout for client RPC acknowledgement ackTimeout: 1000 # Timeout for actual RPC provider response responseTimeout: 10000 # Don't send requestorName by default. provideRequestorName: false # Don't send requestorData by default. provideRequestorData: false record: # Maximum time permitted to fetch from cache cacheRetrievalTimeout: 30000 # Maximum time permitted to fetch from storage storageRetrievalTimeout: 30000 # A list of prefixes that, when a record starts with one of the prefixes the # records data won't be stored in the db # storageExclusionPrefixes: # - no-storage/ # - temporary-data/ # A list of prefixes that, when a record is updated via setData and it matches one of the prefixes # it will be permissioned and written directly to the cache and storage layers # storageHotPathPrefixes: # - analytics/ # - metrics/ # Invalid configuration: data should NOT have additional properties listen: # Try finding a provider randomly rather than by the order they subscribed to. shuffleProviders: true # The amount of time to wait for a provider to acknowledge or reject a listen request responseTimeout: 500 # The amount of time before trying to reattempt finding matches for subscriptions. This # is not a cheap operation so it's recommended to raise keep this at minutes rather then # second intervals if you are experiencing heavy loads rematchInterval: 60000 # The amount of time a server will refuse to retry finding a subscriber after a previously # failed attempt. This is used to avoid servers constantly trying to find a match without a # cooldown period matchCooldown: 10000 httpServer: type: default options: # url path for http health-checks, GET requests to this path will return 200 if deepstream is alive healthCheckPath: /health-check # -- CORS -- # if disabled, only requests with an 'Origin' header matching one specified under 'origins' # below will be permitted and the 'Access-Control-Allow-Credentials' response header will be # enabled allowAllOrigins: true # maximum allowed size of a POST request body, in bytes, defaults to 1 MB maxMessageSize: 1048576 # a list of allowed origins origins: - 'https://example.com' # Options required to create an ssl app # ssl: # key: fileLoad(ssl/key.pem) # cert: fileLoad(ssl/cert.pem) # ca: ... # type: uws # options: # # url path for http health-checks, GET requests to this path will return 200 if deepstream is alive # healthCheckPath: /health-check # # -- CORS -- # # if disabled, only requests with an 'Origin' header matching one specified under 'origins' # # below will be permitted and the 'Access-Control-Allow-Credentials' response header will be # # enabled # allowAllOrigins: true # # a list of allowed origins # origins: # - 'https://example.com' # # maximum allowed size of a POST request body, in bytes, defaults to 1 MB # maxMessageSize: 1048576 # # Headers to copy over from websocket # headers: # - user-agent # # Options required to create an ssl app # ssl: # key: file(ssl/key.pem) # cert: file(ssl/cert.pem) # ## dhParams: ... # ## passphrase: ... # Connection Endpoint Configuration # to disable, replace configuration with null eg. `http: null` connectionEndpoints: - type: ws-binary options: # url path websocket connections connect to urlPath: /deepstream # the amount of milliseconds between each ping/heartbeat message heartbeatInterval: 30000 # the amount of milliseconds that writes to sockets are buffered outgoingBufferTimeout: 0 # the maximum amount of bytes to buffer before flushing, stops the client from large enough packages # to block its responsiveness maxBufferByteSize: 100000 # Security # should the server log invalid auth data, defaults to false logInvalidAuthData: false # amount of time a connection can remain open while not being logged in unauthenticatedClientTimeout: 180000 # invalid login attempts before the connection is cut maxAuthAttempts: 3 # maximum allowed size of an individual message in bytes maxMessageSize: 1048576 # - type: ws-text # options: # # url path websocket connections connect to # urlPath: /deepstream-v3 # # the amount of milliseconds between each ping/heartbeat message # heartbeatInterval: 30000 # # the amount of milliseconds that writes to sockets are buffered # outgoingBufferTimeout: 0 # # the maximum amount of bytes to buffer before flushing, stops the client from large enough packages # # to block its responsiveness # maxBufferByteSize: 100000 # # Security # # should the server log invalid auth data, defaults to false # logInvalidAuthData: false # # amount of time a connection can remain open while not being logged in # unauthenticatedClientTimeout: 180000 # # invalid login attempts before the connection is cut # maxAuthAttempts: 3 # # maximum allowed size of an individual message in bytes # maxMessageSize: 1048576 # - type: ws-json # options: # # url path websocket connections connect to # urlPath: /deepstream-json # # the amount of milliseconds between each ping/heartbeat message # heartbeatInterval: 30000 # # the amount of milliseconds that writes to sockets are buffered # outgoingBufferTimeout: 0 # # the maximum amount of bytes to buffer before flushing, stops the client from large enough packages # # to block its responsiveness # maxBufferByteSize: 100000 # # Security # # should the server log invalid auth data, defaults to false # logInvalidAuthData: false # # amount of time a connection can remain open while not being logged in # unauthenticatedClientTimeout: 180000 # # invalid login attempts before the connection is cut # maxAuthAttempts: 3 # # maximum allowed size of an individual message in bytes # maxMessageSize: 1048576 - type: http options: # allow 'authData' parameter in POST requests, if disabled only token and OPEN auth is # possible allowAuthData: true # path for POST requests postPath: /api # path for GET requests getPath: /api # should the server log invalid auth data, defaults to false logInvalidAuthData: false # http request timeout in milliseconds, defaults to 20000 requestTimeout: 20000 # - type: mqtt # options: # # port for the mqtt server # port: 1883 # # host for the mqtt server # host: 0.0.0.0 # # timeout for idle devices # idleTimeout: 60000 # Logger Configuration logger: # use the default logger, this does not currently support meta objects type: default options: colors: true # log using json, this supports meta objects # name: pino # options: # # this value will always overwrite value of logLevel (line 4) # logLevel: 0 # name: winston # options: # transports: # # specify a list of transports (console, file, time) # - # type: console # options: # level: verbose # colorize: true # - # type: file # level: debug # options: # filename: 'logs.json' # - # type: time # level: warn # options: # filename: time-rotated-logfile # datePattern: .yyyy-MM-dd-HH-mm # cache: # name: redis # options: # host: ${REDIS_HOST} # port: ${REDIS_PORT} # storage: # name: mongodb # options: # connectionString: ${MONGO_CONNECTION_STRING} # db: default # Authentication auth: - type: none # # reading users and passwords from the storage layer # - type: storage # options: # # the table users are stored in storage # table: Users # # the split character used for tables (defaults to /) # tableSplitChar: string # # automatically create users if they don't exist in the database # createUser: true # # the name of a HMAC digest algorithm # hash: 'md5' # # the number of times the algorithm should be applied # iterations: 100 # # the length of the resulting key # # keyLength: 32 # - type: file # options: # # Path to the user file. Can be json, js or yml # users: fileLoad(users.yml) # # the name of a HMAC digest algorithm # hash: 'md5' # # the number of times the algorithm should be applied # iterations: 100 # # the length of the resulting key # keyLength: 32 # # getting permissions from a http webhook # - type: http # options: # # a post request will be send to this url on every incoming connection # endpointUrl: https://someurl.com/validateLogin # # any of these will be treated as access granted # permittedStatusCodes: [ 200 ] # # if the webhook didn't respond after this amount of milliseconds, the connection will be rejected # requestTimeout: 2000 # # promote the following items from the login auth data into headers # promoteToHeader: # - token # # the codes which the auth handler should retry. This is useful for when the API you depend on is # # flaky or going through a not so blue/green deployment # retryStatusCodes: [ 404, 504 ] # # the maximum amount of retries before returning a false login # retryAttempts: 3 # # the time in milliseconds between retries # retryInterval: 5000 # Permissioning permission: type: config options: # Permissions file permissions: fileLoad(permissions.yml) # Amount of times nested cross-references will be loaded. Avoids endless loops maxRuleIterations: 3 # PermissionResults are cached to increase performance. Lower number means more loading cacheEvacuationInterval: 60000 monitoring: - type: none # # Allows monitoring stats to be requested via HTTP, useful for polling agents # # such as LogStash # - type: http # options: # url: /monitoring # allowOpenPermissions: false # headerKey: deepstream-password2 # headerValue: deepstream-secret # # Logs monitoring stats, useful for kibana where you can visualize meta data # - type: log # options: # logInterval: 30000 # monitoringKey: DEEPSTREAM_MONITORING # clusterNode: # type: default # options: # host: localhost # port: 6379 # Custom Plugins # plugins: # custom: # path: '...' # heap-snapshot: # name: 'heap-snapshot' # options: # interval: 60000 # outputDir: file(../heap-snapshots) # aws: # name: aws # options: # accessKeyId: ${AWS_ACCESS_KEY} # secretAccessKey: ${AWS_SECRET_ACCESS_KEY} # services: # - type: s3-sync # options: # syncInterval: 60000 # syncDir: file(../heap-snapshots) # bucketName: ${SYNC_BUCKET_NAME} # bucketRegion: ${AWS_DEFAULT_REGION} ================================================ FILE: conf/permissions.yml ================================================ presence: "*": allow: true record: "*": create: true write: true read: true delete: true listen: true notify: true event: "*": publish: true subscribe: true listen: true rpc: "*": provide: true request: true ================================================ FILE: conf/users.yml ================================================ # Username userA: # Password hash ( hash options passed within config.yml ) # This hash represents clear text "password" with default options password: "rCOSZxJrgze2AZdVQh12c6ErDMOG0M+Rx5Yu7S5d91c=GS4SbTQYmoaGwjm2shEobg==" # Server data is used within your permission handler serverData: someOptional: "auth-data" # Client data is sent to the client on login clientData: favouriteColor: "red" ================================================ FILE: package.json ================================================ { "name": "@deepstream/server", "version": "10.0.0", "description": "a scalable server for realtime webapps", "main": "./dist/src/deepstream.io.js", "bin": { "deepstream": "./dist/bin/deepstream" }, "engines": { "node": ">=22.0.0" }, "directories": { "test": "test" }, "pkg": { "scripts": "./dist/src/config/*.js", "assets": "./dist/ascii-logo.txt" }, "mocha": { "reporter": "dot", "require": [ "ts-node/register/transpile-only", "./src/test/common.ts" ], "exit": true }, "scripts": { "start:inspect": "npm run tsc && node --inspect dist/bin/deepstream", "start": "ts-node --transpile-only --project tsconfig.json --files ./bin/deepstream.ts start", "tsc": "sh scripts/tsc.sh", "license": "mkdir -p build && node scripts/license-aggregator > build/LICENSE && cat scripts/resources/missing-licenses.txt >> build/LICENSE", "lint": "tslint --project .", "lint:fix": "npm run lint -- --fix", "test": "mocha 'src/**/*.spec.ts'", "test:coverage": "nyc mocha 'src/**/*.spec.ts' && npm run test:coverage:combine", "test:http-server": "node test/test-helper/start-test-server.js", "e2e": "ts-node --transpile-only --project tsconfig.json --files ./node_modules/.bin/cucumber-js test-e2e --require './test-e2e/steps/**/*.ts' --exit", "e2e:v3": "V3=true npm run e2e -- --tags \"not @V4\"", "e2e:uws": "uws=true npm run e2e", "e2e:uws:v3": "uws=true V3=true npm run e2e -- --tags \"not @V4\"", "e2e:rpc": "npm run e2e -- --tags \"@rpcs\"", "e2e:event": "npm run e2e -- --tags \"@events\"", "e2e:record": "npm run e2e -- --tags \"@records\"", "e2e:login": "npm run e2e -- --tags \"@login\"", "e2e:presence": "npm run e2e -- --tags \"@presence\"", "e2e:http": "npm run e2e -- --tags \"@http\"", "e2e:coverage": "nyc cucumber-js test-e2e --require './test-e2e/steps/**/*.ts' --exit && npm run test:coverage:combine", "test:all:coverage": "rm -rf .nyc_combined_coverage .nyc_output && npm run test:coverage && npm run e2e:coverage && nyc report --reporter=lcov -t .nyc_combined_coverage", "test:coverage:combine": "rm -rf .nyc_output/processinfo && mkdir -p .nyc_combined_coverage && mv -f .nyc_output/* .nyc_combined_coverage/" }, "repository": { "type": "git", "url": "https://github.com/deepstreamIO/deepstream.io.git" }, "dependencies": { "@deepstream/protobuf": "^1.0.8", "@deepstream/types": "^2.3.2", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "better-ajv-errors": "^1.2.0", "body-parser": "^2.2.0", "chalk": "^4.1.2", "commander": "^11.1.0", "content-type": "^1.0.5", "glob": "^8.1.0", "http-shutdown": "^1.2.2", "http-status": "^1.7.0", "js-yaml": "^4.1.0", "mqtt-connection": "^4.1.0", "needle": "^3.2.0", "pino": "^9.6.0", "source-map-support": "^0.5.21", "uuid": "^8.3.2", "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.51.0", "ws": "^7.5.9" }, "devDependencies": { "@deepstream/client": "^7.0.4", "@types/body-parser": "^1.19.3", "@types/content-type": "^1.1.6", "@types/cucumber": "^6.0.1", "@types/glob": "^8.1.0", "@types/js-yaml": "^4.0.7", "@types/mkdirp": "^1.0.1", "@types/mocha": "^8.0.4", "@types/needle": "^3.2.1", "@types/node": "^14.18.63", "@types/sinon": "^10.0.19", "@types/sinon-chai": "^3.2.10", "@types/uuid": "^8.3.4", "@types/ws": "^7.4.7", "@yao-pkg/pkg": "^6.4.0", "async": "^3.2.4", "chai": "^4.3.10", "coveralls": "^3.1.1", "cucumber": "^6.0.7", "deepstream.io-client-js": "^2.3.4", "husky": "^4.3.8", "istanbul": "^0.4.5", "mocha": "^10.2.0", "n0p3": "^1.0.2", "nyc": "^15.1.0", "proxyquire": "^2.1.3", "sinon": "^16.1.0", "sinon-chai": "^3.7.0", "ts-essentials": "^10.0.4", "ts-node": "^10.9.1", "tslint": "^6.1.3", "typescript": "^5.8.3" }, "author": "deepstreamHub GmbH", "license": "MIT", "bugs": { "url": "https://github.com/deepstreamIO/deepstream.io/issues" }, "homepage": "https://deepstreamio.github.io/", "husky": { "hooks": { "pre-commit": "npm run lint && npm run tsc", "pre-push": "npm run tsc && npm t && npm run e2e -- --fail-fast && npm run e2e:uws -- --fail-fast && bash scripts/package.sh true true && node scripts/node-test.js && node scripts/executable-test.js", "pre-publish": "npm run tsc" } }, "nyc": { "include": [ "src/**/*.ts" ], "exclude": [ "src/**/*.spec.ts", "src/connection-endpoint/json/*", "src/connection-endpoint/mqtt/*", "src/connection-endpoint/text/*" ], "extension": [ ".ts" ], "require": [ "ts-node/register/transpile-only" ], "reporter": [ "lcov" ], "sourceMap": true, "instrument": true } } ================================================ FILE: scripts/connector/package-connector.sh ================================================ #!/bin/bash PACKAGED_NODE_VERSION="v10" OS=$( node -e "console.log(require('os').platform())" ) NODE_VERSION=$( node --version ) COMMIT=$( git log --pretty=format:%h -n 1 ) PACKAGE_VERSION=$( cat package.json | grep version | awk '{ print $2 }' | sed s/\"//g | sed s/,//g ) PACKAGE_NAME=$( cat package.json | grep name | awk '{ print $2 }' | sed s/\"//g | sed s/,//g ) PACKAGE_NAME=$( node -e "console.log(process.argv[1].replace('@deepstream/', 'deepstream.io-'))" $PACKAGE_NAME ) # These must happen before any exits otherwise deployment would fail # Clean the build directory rm -rf build mkdir -p build/$PACKAGE_VERSION if ! [[ $NODE_VERSION == $PACKAGED_NODE_VERSION* ]]; then echo "Packaging only done on $PACKAGED_NODE_VERSION.x" exit fi if [ $OS == "darwin" ]; then PLATFORM="mac" elif [ $OS == "linux" ]; then PLATFORM="linux" elif [ $OS == "win32" ]; then PLATFORM="windows" else echo "Operating system $OS not supported for packaging" exit fi FILE_NAME=$PACKAGE_NAME-$PLATFORM-$PACKAGE_VERSION-$COMMIT # Do a git archive and a production install # to have cleanest output git archive --format=zip $COMMIT -o ./build/$PACKAGE_VERSION/temp.zip cd ./build/$PACKAGE_VERSION unzip temp.zip -d $PACKAGE_NAME cd $PACKAGE_NAME npm install npm run tsc # Generate dist rm -rf node_modules npm install --omit=dev echo 'Installed NPM Dependencies' if [ $PLATFORM == 'mac' ]; then FILE_NAME="$FILE_NAME.zip" CLEAN_FILE_NAME="$PACKAGE_NAME-$PLATFORM.zip" zip -r ../$FILE_NAME . elif [ $PLATFORM == 'windows' ]; then FILE_NAME="$FILE_NAME.zip" CLEAN_FILE_NAME="$PACKAGE_NAME-$PLATFORM.zip" 7z a ../$FILE_NAME . else FILE_NAME="$FILE_NAME.tar.gz" CLEAN_FILE_NAME="$PACKAGE_NAME-$PLATFORM.tar.gz" tar czf ../$FILE_NAME . fi cd .. rm -rf $PACKAGE_NAME temp.zip cp $FILE_NAME ./$CLEAN_FILE_NAME echo 'Done' ================================================ FILE: scripts/connector/test-connector.sh ================================================ #!/bin/bash set -e curl -o deepstream_package.json https://raw.githubusercontent.com/deepstreamIO/deepstream.io/master/package.json DEEPSTREAM_VERSION="$( cat deepstream_package.json | grep version | awk '{ print $2 }' | sed s/\"//g | sed s/,//g )" NODE_VERSION=$( node --version ) OS=$( node -e "console.log(require('os').platform())" ) PACKAGE_VERSION=$( cat package.json | grep version | awk '{ print $2 }' | sed s/\"//g | sed s/,//g ) PACKAGE_NAME=$( cat package.json | grep name | awk '{ print $2 }' | sed s/\"//g | sed s/,//g ) TYPE=$( node -e "console.log(process.argv[1].match('@deepstream/(.*)-(.*)')[1])" $PACKAGE_NAME ) CONNECTOR=$( node -e "console.log(process.argv[1].match('@deepstream/(.*)-(.*)')[2])" $PACKAGE_NAME ) rm -rf build mkdir build cd build if [ -z $1 ]; then if [[ -z ${TRAVIS_TAG} ]] && [[ -z ${APPVEYOR_REPO_TAG} ]]; then echo "Only runs on tags" exit elif [[ ${APPVEYOR_REPO_TAG} = false ]]; then echo "On appveyor, not a tag" exit else echo "Running on tag ${TRAVIS_TAG} ${APPVEYOR_REPO_TAG}" fi else echo "Build forced although not tag" fi echo "Starting deepstream.io $TYPE $CONNECTOR $PACKAGE_VERSION test for $DEEPSTREAM_VERSION on $OS" echo "Downloading deepstream $DEEPSTREAM_VERSION" if [ $OS = "win32" ]; then DEEPSTREAM=deepstream.io-windows-${DEEPSTREAM_VERSION} curl -o ${DEEPSTREAM}.zip -L https://github.com/deepstreamIO/deepstream.io/releases/download/v${DEEPSTREAM_VERSION}/${DEEPSTREAM}.zip 7z x ${DEEPSTREAM}.zip -o${DEEPSTREAM} elif [ $OS == 'darwin' ]; then DEEPSTREAM=deepstream.io-mac-${DEEPSTREAM_VERSION} curl -o ${DEEPSTREAM}.zip -L https://github.com/deepstreamIO/deepstream.io/releases/download/v${DEEPSTREAM_VERSION}/${DEEPSTREAM}.zip unzip ${DEEPSTREAM} -d ${DEEPSTREAM} else DEEPSTREAM=deepstream.io-linux-${DEEPSTREAM_VERSION} mkdir -p ${DEEPSTREAM} curl -o ${DEEPSTREAM}.tar.gz -L https://github.com/deepstreamIO/deepstream.io/releases/download/v${DEEPSTREAM_VERSION}/${DEEPSTREAM}.tar.gz tar -xzf ${DEEPSTREAM}.tar.gz -C ${DEEPSTREAM} fi cd ${DEEPSTREAM} chmod 555 deepstream echo "./deepstream --version" ./deepstream --version echo "./deepstream install $TYPE $CONNECTOR:$PACKAGE_VERSION" ./deepstream install $TYPE $CONNECTOR:$PACKAGE_VERSION --verbose ./deepstream start -c ../../example-config.yml -l ./lib & PROC_ID=$! sleep 10 if ! [ kill -0 "$PROC_ID" > /dev/null 2>&1 ]; then echo "Deepstream is not running after the first ten seconds" exit 1 fi # Rest comes on beta.5 ================================================ FILE: scripts/details.js ================================================ var exec = require( 'child_process' ).execSync; var fs = require( 'fs' ); var path = require( 'path' ); var pkg = require( '../package' ); if( process.argv[2] === 'VERSION' ) { console.log( pkg.version ); } else if( process.argv[2] === 'UWS_VERSION' ) { console.log( pkg.dependencies["uWebSockets.js"].replace('^','') ); } else if( process.argv[2] === 'NAME' ) { console.log( pkg.name ); } else if( process.argv[2] === 'OS' ) { console.log( require( 'os' ).platform() ); } else if( process.argv[2] === 'COMMIT' ) { console.log( exec( 'git log --pretty=format:%h -n 1' ).toString() ); } else if( process.argv[2] === 'META' ) { writeMetaFile(); } else { console.log( 'ERROR: Pass in VERSION or NAME as env variable' ); } function writeMetaFile() { var meta = { deepstreamVersion: pkg.version, gitRef: exec( 'git rev-parse HEAD' ).toString().trim(), buildTime: new Date().toString() }; fs.writeFileSync( path.join( __dirname, '..', 'dist', 'meta.json' ), JSON.stringify( meta, null, 2 ), 'utf8' ); } ================================================ FILE: scripts/executable-test.js ================================================ const { exec } = require('child_process') exec('./build/deepstream info', (error, stdout, stderr) => { if (error) { console.log(`error: ${error.message}`) process.exit(1) } if (stderr) { console.log(`stderr: ${stderr}`) process.exit(1) } process.exit(0) }) ================================================ FILE: scripts/license-aggregator.js ================================================ #!/usr/bin/env node const path = require('path') const fs = require('fs') const child_process = require('child_process') const async = require('async') const PRE_HEADER = fs.readFileSync('LICENSE', 'utf8') const HEADER = ` This license applies to all parts of deepstream.io that are not externally maintained libraries. The externally maintained libraries used by deepstream.io are: ` const emptyState = "see MISSING LICENSES at the bottom of this file" if (path.basename(process.cwd()) === 'scripts') { console.error('Run this script from the project root!') process.exit(0) } child_process.execSync('npm list --omit=dev --json > licenses.json') const mainModule = require('../licenses.json') const moduleNames = [] traverseDependencies(mainModule) function traverseDependencies(module) { for (let dependency in module.dependencies) { moduleNames.push(dependency) traverseDependencies(module.dependencies[dependency]) } } // This source code is taken from the 'license-spelunker' npm module, it was patched /* The MIT License (MIT) Copyright (c) 2013 Mike Brevoort Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ var projPath = path.resolve(process.argv[2] || '.') console.error('Project Path', projPath) var topPkg = require(path.join(projPath, 'package.json')) var modules = [] var count = 0 doLevel(projPath) function doLevel(nodePath) { var pkg = require(path.join(nodePath, 'package.json')) if (topPkg.name !== pkg.name && moduleNames.indexOf(pkg.name) === -1) { return } var nodeModulesPath = path.join(nodePath, 'node_modules') count ++ //console.error('package.json license', pkg.license) fs.exists(nodeModulesPath, function (dirExists) { if (dirExists) { fs.readdir(nodeModulesPath, function (err, files) { if (err) throw err files = files.map(function (f) { return path.join(nodeModulesPath, f) }) async.filter(files, isModuleDirectory, (err, directories) => { directories.forEach(doLevel) }) }) } }) licenseText(nodePath, function (license) { var licenceProperty = pkg.license || pkg.licenses var licenceUrl = (pkg.license || {}).url if ((licenceProperty || {}).type) { licenceProperty = licenceProperty.type licenceUrl = licenceProperty[0].url } if (((licenceProperty || {})[0] || []).type) { licenceProperty = licenceProperty[0].type licenceUrl = licenceProperty[0].url } if (pkg.name !== topPkg.name) { modules.push({ name: pkg.name, version: pkg.version, url: 'http://npmjs.org/package/' + pkg.name, localPath: path.relative(projPath,nodePath), pkgLicense: licenceProperty, licenceUrl: licenceUrl, license: license }) } count-- if (count == 0) { var noLicenseFile = modules.filter(function (m) { return m.license === emptyState }) var andNoPkgJsonLicense = noLicenseFile.filter(function (m) { return !m.pkgLicense }) // Status report // Write to StdErr console.error('LICENSE FILE REPORT FOR ', topPkg.name) console.error(modules.length + ' nested dependencies') console.error(noLicenseFile.length + ' without identifiable license text') console.error(andNoPkgJsonLicense.length + ' without even a package.json license declaration', '\n\n') // Write to StdOut console.log(PRE_HEADER) console.log('') console.log(HEADER) modules.forEach(function(m) { console.log((modules.indexOf(m)+1) + ' ----------------------------------------------------------------------------') console.log(m.name + '@' + m.version) console.log(m.url) console.log(m.localPath) if (m.pkgLicense) console.log('From package.json license property:', JSON.stringify(m.pkgLicense)) if (m.licenceUrl) console.log('From package.json url property:', JSON.stringify(m.licenceUrl)) console.log('') console.log(m.license) console.log('') }) } }) } function licenseText (nodePath, cb) { var possibleLicensePaths = [ path.join(nodePath, 'LICENSE'), path.join(nodePath, 'LICENCE'), path.join(nodePath, 'LICENSE.md'), path.join(nodePath, 'LICENSE.txt'), path.join(nodePath, 'LICENSE-MIT'), path.join(nodePath, 'LICENSE-BSD'), path.join(nodePath, 'LICENSE.BSD'), path.join(nodePath, 'MIT-LICENSE.txt'), path.join(nodePath, 'Readme.md'), path.join(nodePath, 'README.md'), path.join(nodePath, 'README.markdown') ] async.reduceRight(possibleLicensePaths, emptyState, function (state, licensePath, reduceCb) { var isAReadme = (licensePath.toLowerCase().indexOf('/readme') > 0) // if we already found a licnese, don't bother looking at READMEs if (state !== emptyState && isAReadme) return reduceCb (null, state) fs.exists(licensePath, function (exists) { if (!exists) return reduceCb(null, state) fs.readFile(licensePath, { encoding: 'utf8' }, function (err, text) { if (err) return logError(err, reduceCb)(err, state) if (isAReadme) { var match = text.match(/\n[# ]*license[ \t]*\n/i) if (match) { //console.log(match.input.substring(match.index)) return reduceCb (null, 'FROM README:\n' + match.input.substring(match.index)) } else { return reduceCb(null, state) } } else { return reduceCb (null, text) } return reduceCb (null, text) }) }) }, function (err, license) { if (err) return cb('ERROR FINDING LICENSE FILE ' + err ) cb (license) }) } function isModuleDirectory (dirPath, cb) { var pkgPath = path.join(dirPath, 'package.json') fs.stat(dirPath, function (err, stat) { if (err) return logError(err, cb)(false) var isdir = stat.isDirectory() if (isdir) { fs.access(pkgPath, (err) => { cb(null, !err) }) } else { cb(false) } }) } function logError(err, cb) { console.error('ERROR', err) return cb } ================================================ FILE: scripts/linux-package.sh ================================================ #!/bin/bash set -e if [[ -z $1 ]]; then echo "First param is distro ( centos | debian | ubuntu | ... )"; exit 1; fi if [[ -z $2 ]]; then echo "Second param is version ( bionic | 7 | ... )"; exit 1; fi if [[ -z $3 ]]; then echo "No distribution version provided, so using the version from package.json" curl -O https://raw.githubusercontent.com/deepstreamIO/deepstream.io/master/package.json VERSION="$( cat package.json | grep version | awk '{ print $2 }' | sed s/\"//g | sed s/,//g )" else VERSION="$3" fi DISTRO=$1 DISTRO_NAME=$2 GIT_TAG_NAME=v${VERSION} # RPM does not support dashes in versions RPM_PACKAGE_VERSION=$( sed "s/-/_/" <<< ${VERSION} ) if [[ ${DISTRO} = "ubuntu" ]] || [[ ${DISTRO} = "debian" ]]; then ENV="deb" elif [[ ${DISTRO} = "centos" ]] || [[ ${DISTRO} = "fedora" ]]; then ENV="rpm" else echo "Unsupported distro: $DISTRO" exit 1; fi mkdir -p build cd build if [[ ${DISTRO} = 'centos' ]]; then cat >Dockerfile <Dockerfile <>Dockerfile <>Dockerfile <>Dockerfile <>Dockerfile <>Dockerfile <>Dockerfile </dev/null rm -f "${CIDFILE}" echo "Build successful" ================================================ FILE: scripts/linux-test.sh ================================================ #!/bin/bash set -e if [ -z $1 ]; then echo "First param is distro ( centos | debian | ubuntu | ... )"; exit 1; fi if [ -z $2 ]; then echo "Second param is version ( wheezy | 7 | ... )"; exit 1; fi if [ -z $3 ]; then echo "No distribution version provided, so using the version from package.json" curl -o package.json https://raw.githubusercontent.com/deepstreamIO/deepstream.io/master/package.json VERSION="$( cat package.json | grep version | awk '{ print $2 }' | sed s/\"//g | sed s/,//g )" else VERSION="$3" fi DISTRO=$1 DISTRO_NAME=$2 GIT_TAG_NAME=v$VERSION if [ $DISTRO = "ubuntu" ] || [ $DISTRO = "debian" ]; then ENV="deb" elif [ $DISTRO = "centos" ]; then ENV="rpm" else echo "Unsupported distro: $DISTRO" exit 1; fi cat >Dockerfile <>Dockerfile <>Dockerfile <>Dockerfile < version RUN cat version RUN grep -q ^$VERSION version RUN deepstream install connector publishingtest EOF echo "Using Dockerfile:" sed -e 's@^@ @g' Dockerfile TAG="$DEBIAN_NAME_$GIT_TAG_NAME" echo "Building Docker image ${TAG}" docker build --tag=${TAG} . echo "Removing Dockerfile" rm -f Dockerfile CIDFILE="cidfile" ARGS="--cidfile=${CIDFILE}" rm -f ${CIDFILE} # Cannot exist echo "Running build" docker run ${ARGS} ${TAG} echo "Removing container" docker rm "$(cat ${CIDFILE})" >/dev/null rm -f "${CIDFILE}" echo "Build successful" ================================================ FILE: scripts/node-test.js ================================================ const { Deepstream } = require('../dist/src/deepstream.io') const server = new Deepstream({}) server.start() server.on('stopped', () => process.exit(0)) setTimeout(() => server.stop(), 2000) ================================================ FILE: scripts/package.sh ================================================ #!/bin/bash set -e LTS_VERSION="22" NODE_VERSION=$( node --version ) NODE_VERSION_WITHOUT_V=$( echo ${NODE_VERSION} | cut -c2-10 ) COMMIT=$( node scripts/details.js COMMIT ) PACKAGE_VERSION=$( node scripts/details.js VERSION ) PACKAGE_NAME=$( node scripts/details.js NAME ) OS=$( node scripts/details.js OS ) UWS_VERSION=$( node scripts/details.js UWS_VERSION ) PACKAGE_DIR=build/${PACKAGE_VERSION} DEEPSTREAM_PACKAGE=${PACKAGE_DIR}/deepstream.io GIT_BRANCH=$( git rev-parse --abbrev-ref HEAD ) CREATE_DISTROS=false EXECUTABLE_NAME="build/deepstream" # Needed even for void builds for travis deploy to pass mkdir -p build if ! [[ ${NODE_VERSION_WITHOUT_V} == ${LTS_VERSION}* ]]; then echo "Packaging only done on $LTS_VERSION.x" exit fi if [[ -z $1 ]]; then if ! [[ ${GIT_BRANCH} = 'master' ]]; then echo "Running on branch ${GIT_BRANCH}" else echo "Running on master" fi fi if [[ $2 ]]; then echo 'Ignoring distros' elif [[ ${OS} = "linux" ]]; then CREATE_DISTROS=true echo "Checking if FPM is installed" fpm --version fi function compile { echo "Starting deepstream.io packaging with Node.js $NODE_VERSION_WITHOUT_V" rm -rf build mkdir build echo "Installing missing npm packages, just in case something changes" npm i echo "Transpiling" npm run tsc echo "Generate License File using unmodified npm packages" ./scripts/license-aggregator.js > build/DEPENDENCIES.LICENSE echo "Generating meta.json" node scripts/details.js META # Creating package structure rm -rf build/${PACKAGE_VERSION} mkdir -p ${DEEPSTREAM_PACKAGE} mkdir ${DEEPSTREAM_PACKAGE}/var mkdir ${DEEPSTREAM_PACKAGE}/lib mkdir ${DEEPSTREAM_PACKAGE}/doc cd ${DEEPSTREAM_PACKAGE}/lib echo '{ "name": "TEMP" }' > package.json echo "Adding uWebSockets.js to libs" npm install --omit=dev --install-strategy=shallow uWebSockets.js@${UWS_VERSION} echo "Adding cache plugins" npm install --omit=dev --install-strategy=shallow \ @deepstream/cache-redis \ @deepstream/cache-memcached \ # @deepstream/cache-hazelcast echo "Adding cluster plugins" npm install --omit=dev --install-strategy=shallow \ @deepstream/clusternode-redis echo "Adding storage plugins" npm install --omit=dev --install-strategy=shallow \ @deepstream/storage-mongodb \ @deepstream/storage-rethinkdb \ @deepstream/storage-elasticsearch \ @deepstream/storage-postgres echo "Adding logger plugins" npm install --omit=dev --install-strategy=shallow \ @deepstream/logger-winston mv node_modules/* . rm -rf node_modules package.json cd - echo "Creating '$EXECUTABLE_NAME', this will take a while..." LTS=${LTS_VERSION} OS=${OS} EXECUTABLE_NAME=${EXECUTABLE_NAME} node scripts/pkg.js PROC_ID=$! SECONDS=0; while kill -0 "$PROC_ID" >/dev/null 2>&1; do echo -ne "\rCompiling deepstream... ($SECONDS)" sleep 10 done echo "" if wait ${pid}; then echo -e "\tPkg Build Succeeded" else echo -e "\tPkg Build Failed" exit 1 fi echo "Adding docs" echo -e "\tAdding Readme" echo "Documentation is available at https://deepstreamhub.com/open-source " > ${DEEPSTREAM_PACKAGE}/doc/README echo -e "\tAdding Changelog" cp CHANGELOG.md ${DEEPSTREAM_PACKAGE}/doc/CHANGELOG.md echo -e "\tAdding Licenses" curl -L https://raw.githubusercontent.com/nodejs/node/v22.x/LICENSE -o ${DEEPSTREAM_PACKAGE}/doc/NODE.LICENSE mv build/DEPENDENCIES.LICENSE ${DEEPSTREAM_PACKAGE}/doc/LICENSE echo "Moving deepstream into package structure at $DEEPSTREAM_PACKAGE" cp -r conf ${DEEPSTREAM_PACKAGE}/ cp build/deepstream ${DEEPSTREAM_PACKAGE}/ echo "Patching config file for zip lib and var directories" cp -f ./conf/config.yml ${DEEPSTREAM_PACKAGE}/conf/config.yml if [[ ${OS} = "darwin" ]]; then sed -i '' 's@#libDir: ../lib@libDir: ../lib@' ${DEEPSTREAM_PACKAGE}/conf/config.yml else sed -i 's@#libDir: ../lib@libDir: ../lib@' ${DEEPSTREAM_PACKAGE}/conf/config.yml fi } function windows { COMMIT_NAME="deepstream.io-windows-$PACKAGE_VERSION-$COMMIT.zip " CLEAN_NAME="deepstream.io-windows-$PACKAGE_VERSION.zip" echo "OS is windows" echo -e "\tCreating zip deepstream.io-windows-$PACKAGE_VERSION.zip" cd ${DEEPSTREAM_PACKAGE} 7z a ../${COMMIT_NAME} . > /dev/null cp ../${COMMIT_NAME} ../${CLEAN_NAME} cd - } function mac { COMMIT_NAME="deepstream.io-mac-${PACKAGE_VERSION}-${COMMIT}" CLEAN_NAME="deepstream.io-mac-${PACKAGE_VERSION}" echo "OS is mac" echo -e "\tCreating ${CLEAN_NAME}" cd ${DEEPSTREAM_PACKAGE} zip -r ../${COMMIT_NAME}.zip . cp ../${COMMIT_NAME}.zip ../${CLEAN_NAME}.zip cd - rm -rf build/osxpkg mkdir -p build/osxpkg/bin mkdir -p build/osxpkg/etc/deepstream mkdir -p build/osxpkg/lib/deepstream mkdir -p build/osxpkg/share/doc/deepstream mkdir -p build/osxpkg/var/log/deepstream cp -r ${DEEPSTREAM_PACKAGE}/deepstream build/osxpkg/bin/deepstream cp -r ${DEEPSTREAM_PACKAGE}/conf/* build/osxpkg/etc/deepstream cp -r ${DEEPSTREAM_PACKAGE}/lib/* build/osxpkg/lib/deepstream cp -r ${DEEPSTREAM_PACKAGE}/doc/* build/osxpkg/share/doc/deepstream echo "Patching config file for lib and var directories" sed -i '' 's@ ../lib@ /usr/local/lib/deepstream@' build/osxpkg/etc/deepstream/config.yml sed -i '' 's@ ../var@ /usr/local/var/log/deepstream@' build/osxpkg/etc/deepstream/config.yml cp build/osxpkg/etc/deepstream/config.yml build/osxpkg/etc/deepstream/config.defaults chmod -R 777 build/osxpkg/bin chmod -R 777 build/osxpkg/share chmod -R 777 build/osxpkg/var chmod -R 777 build/osxpkg/lib chmod -R 777 build/osxpkg/etc echo "\tCreating *.pkg" pkgbuild \ --root build/osxpkg \ --identifier deepstream.io \ --version ${PACKAGE_VERSION} \ --info scripts/resources/PackageInfo \ --install-location /usr/local \ ${DEEPSTREAM_PACKAGE}/../${COMMIT_NAME}.pkg cp \ ${DEEPSTREAM_PACKAGE}/../${COMMIT_NAME}.pkg \ ${DEEPSTREAM_PACKAGE}/../${CLEAN_NAME}.pkg rm -rf build/osxpkg } function linux { echo "OS is linux" echo -e "\tCreating tar.gz" COMMIT_NAME="deepstream.io-linux-${PACKAGE_VERSION}-${COMMIT}.tar.gz" CLEAN_NAME="deepstream.io-linux-${PACKAGE_VERSION}.tar.gz" cd ${DEEPSTREAM_PACKAGE} tar czf ../${COMMIT_NAME} . cp ../${COMMIT_NAME} ../${CLEAN_NAME} cd - } function distros { echo -e "\tPatching config file for linux distros..." if [[ ${OS} = "darwin" ]]; then sed -i '' 's@ ../lib@ /var/lib/deepstream@' ${DEEPSTREAM_PACKAGE}/conf/config.yml sed -i '' 's@ ../var@ /var/log/deepstream@' ${DEEPSTREAM_PACKAGE}/conf/config.yml else sed -i 's@ ../lib@ /var/lib/deepstream@' ${DEEPSTREAM_PACKAGE}/conf/config.yml sed -i 's@ ../var@ /var/log/deepstream@' ${DEEPSTREAM_PACKAGE}/conf/config.yml fi echo -e "\t\tCreating rpm" fpm \ -s dir \ -t rpm \ --package ./build/ \ --package-name-suffix ${COMMIT} \ -n deepstream.io \ -v ${PACKAGE_VERSION} \ --license "MIT" \ --vendor "deepstreamHub GmbH" \ --description "deepstream.io rpm package" \ --url https://deepstream.io/ \ -m "" \ --after-install ./scripts/resources/daemon/after-install \ --before-remove ./scripts/resources/daemon/before-remove \ --before-upgrade ./scripts/resources/daemon/before-upgrade \ --after-upgrade ./scripts/resources/daemon/after-upgrade \ -f \ ${DEEPSTREAM_PACKAGE}/doc/=/usr/share/doc/deepstream/ \ ${DEEPSTREAM_PACKAGE}/conf/=/etc/deepstream/conf.d/ \ ${DEEPSTREAM_PACKAGE}/lib/=/var/lib/deepstream/ \ ./build/deepstream=/usr/bin/deepstream echo -e "\t\tCreating deb" fpm \ -s dir \ -t deb \ --package ./build \ --package-name-suffix ${COMMIT} \ -n deepstream.io \ -v ${PACKAGE_VERSION} \ --license "MIT" \ --vendor "deepstreamHub GmbH" \ --description "deepstream.io deb package" \ --url https://deepstream.io/ \ -m "" \ --after-install ./scripts/resources/daemon/after-install \ --before-remove ./scripts/resources/daemon/before-remove \ --before-upgrade ./scripts/resources/daemon/before-upgrade \ --after-upgrade ./scripts/resources/daemon/after-upgrade \ -f \ --deb-no-default-config-files \ ${DEEPSTREAM_PACKAGE}/doc/=/usr/share/doc/deepstream/ \ ${DEEPSTREAM_PACKAGE}/conf/=/etc/deepstream/conf.d/ \ ${DEEPSTREAM_PACKAGE}/lib/=/var/lib/deepstream/ \ ./build/deepstream=/usr/bin/deepstream } function clean { rm -rf ${DEEPSTREAM_PACKAGE} } compile if [[ $OS = "win32" ]]; then windows elif [[ ${OS} = "darwin" ]]; then mac elif [[ ${OS} = "linux" ]]; then linux if [[ ${CREATE_DISTROS} = true ]]; then distros fi fi clean echo "Files in build directory are $( ls build/ )" echo "Done" ================================================ FILE: scripts/pkg.js ================================================ const { exec } = require('@yao-pkg/pkg') const LTS = process.env.LTS let OS = process.env.OS if (OS === 'win32') { OS = 'win' } if (OS === 'darwin') { OS = 'macos' } const target = 'node'+ LTS + '-' + OS exec( [ 'package.json', '--targets', target, '--output', process.env.EXECUTABLE_NAME, '--options', '--max-old-space-size=8192', '--compress', 'GZip' ]).then(() => { console.log('success') }) ================================================ FILE: scripts/release.sh ================================================ if [ -z $1 ]; then echo "Please provide a release version: patch, minor or major" exit fi if [ $( npm whoami ) != "deepstreamio" ]; then echo "Please verify you can log into npm as deepstreamio before trying to release" exit fi echo 'Starting release' npm version $1 echo "Version now: $( node scripts/details.js VERSION )" echo 'Pushing to github' git push --follow-tags echo "Now we wait for the CI to build and upload artifacts to release" ================================================ FILE: scripts/resources/PackageInfo ================================================ ================================================ FILE: scripts/resources/daemon/after-install ================================================ cp /etc/deepstream/conf.d/* /etc/deepstream/ mkdir -p /var/run/deepstream mkdir -p /var/log/deepstream mkdir -p /var/lib/deepstream chmod -R 777 /var/run/deepstream chmod -R 777 /var/log/deepstream chmod -R 777 /var/lib/deepstream chmod -R 777 /etc/deepstream ================================================ FILE: scripts/resources/daemon/after-upgrade ================================================ ================================================ FILE: scripts/resources/daemon/before-remove ================================================ deepstream stop ================================================ FILE: scripts/resources/daemon/before-upgrade ================================================ cd /var/lib/deepstream rm -rf deepstream.io-logger-winston* rm -rf /etc/deepstream/conf.d ================================================ FILE: scripts/resources/missing-licenses.txt ================================================ ---------------------------------------------------------------------------- MISSING LICENSES ---------------------------------------------------------------------------- adm-zip@0.4.7 http://npmjs.org/package/adm-zip node_modules/adm-zip From package.json license property: "MIT" Copyright (c) 2012 Another-D-Mention Software and other contributors, http://www.another-d-mention.ro/ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---------------------------------------------------------------------------- ================================================ FILE: scripts/resources/node.rc ================================================ #include "winresrc.h" #include "node_version.h" // Application icon 1 ICON deepstream.ico // Version resource VS_VERSION_INFO VERSIONINFO FILEVERSION NODE_MAJOR_VERSION,NODE_MINOR_VERSION,NODE_PATCH_VERSION,0 PRODUCTVERSION NODE_MAJOR_VERSION,NODE_MINOR_VERSION,NODE_PATCH_VERSION,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS VS_FF_DEBUG #else # ifdef NODE_VERSION_IS_RELEASE FILEFLAGS 0x0L # else FILEFLAGS VS_FF_PRERELEASE # endif #endif FILEOS VOS_NT_WINDOWS32 FILETYPE VFT_APP FILESUBTYPE 0x0L BEGIN BLOCK "StringFileInfo" BEGIN BLOCK "040904b0" BEGIN VALUE "CompanyName", "deepstreamHub GmbH" VALUE "ProductName", "deepstream.io" VALUE "FileDescription", "A Scalable Server for Realtime Applications" VALUE "FileVersion", DEEPSTREAM_VERSION VALUE "ProductVersion", DEEPSTREAM_VERSION VALUE "OriginalFilename", "deepstream.exe" VALUE "InternalName", "deepstream" VALUE "LegalCopyright", "Apache License 2.0" END END BLOCK "VarFileInfo" BEGIN VALUE "Translation", 0x409, 1200 END END ================================================ FILE: scripts/sanity-test.sh ================================================ #!/bin/bash set -e if [[ $1 == "deb" ]]; then source /etc/lsb-release && echo "deb http://dl.bintray.com/deepstreamio/deb ${DISTRIB_CODENAME} main" | sudo tee -a /etc/apt/sources.list sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 sudo apt-get update sudo apt-get install -y deepstream.io elif [[ $1 == "rpm" ]]; then sudo yum install -y wget sudo wget https://bintray.com/deepstreamio/rpm/rpm -O /etc/yum.repos.d/bintray-deepstreamio-rpm.repo sudo yum install -y deepstream.io elif [[ $1 == "tar" ]]; then if [[ -z $2 ]]; then echo 'Missing version number when testing tar release' exit 1 fi curl -OL https://github.com/deepstreamIO/deepstream.io/releases/download/v$2/deepstream.io-linux-$2.tar.gz tar xf deepstream.io-linux-$2.tar.gz elif [[ $1 == "installed" ]]; then echo "Assuming deepstream installed" else echo "use deb/rpm/tar/installed" exit 1 fi sudo deepstream service add sudo deepstream service start sudo deepstream service status sleep 2 curl localhost:6020/health-check if [[ $? == 1 ]]; then echo 'Deepstream service not running'; exit 1; fi sudo deepstream service stop sudo deepstream service remove ================================================ FILE: scripts/setup.sh ================================================ git clone https://github.com/deepstreamIO/deepstream.io.git cd deepstream.io sed -i 's/git@github.com:/https:\/\/github.com\//' .gitmodules git submodule update --init --recursive npm i npm run e2e:v3 ================================================ FILE: scripts/trigger-build.sh ================================================ if [ -z $1 ]; then echo "Missing branch name as first arguments" exit 1 fi body="{ \"request\": { \"branch\":\"$1\", \"message\": \"Tag ${TRAVIS_TAG}\" }}" echo $body curl -s -X POST \ -H "Content-Type: application/json" \ -H "Accept: application/json" \ -H "Travis-API-Version: 3" \ -H "Authorization: token ${TRAVIS_TOKEN}" \ -d "$body" \ https://api.travis-ci.org/repo/deepstreamIO%2Fdeepstream.io/requests ================================================ FILE: scripts/tsc.sh ================================================ #!/usr/bin/env bash rm -rf dist ./node_modules/.bin/tsc cp ./ascii-logo.txt ./dist/ascii-logo.txt cp ./package.json ./dist/package.json cp ./package-lock.json ./dist/package-lock.json cp -r conf ./dist/conf cp ./dist/bin/deepstream.js ./dist/bin/deepstream cp Dockerfile ./dist/Dockerfile chmod +x ./dist/bin/deepstream ================================================ FILE: src/config/config-initialiser.spec.ts ================================================ import { expect } from 'chai' import * as path from 'path' import * as defaultConfig from '../default-options' import * as configInitialiser from './config-initialiser' import { EventEmitter } from 'events' import { LOG_LEVEL } from '@deepstream/types' describe('config-initializer', () => { before(() => { global.deepstreamConfDir = null global.deepstreamLibDir = null global.deepstreamCLI = null }) describe('plugins are initialized as per configuration', () => { it('loads plugins from a relative path', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.plugins = { custom: { path: './src/test/mock/plugin-mock', options: { some: 'options' } } } as any const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.plugins.custom.description).to.equal('mock-plugin') }) it('loads plugins via module names', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.plugins = { cache: { path: 'n0p3', options: {} } } as any const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.toString()).to.equal('[object Object]') }) it('loads plugins from a relative path and lib dir', () => { global.deepstreamLibDir = './src/test/mock' const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.plugins = { mock: { path: './plugin-mock', options: { some: 'options' } } } as any const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.plugins.mock.description).to.equal('mock-plugin') }) }) describe('translates shortcodes into paths', () => { it('translates cache', () => { global.deepstreamLibDir = '/foobar' const config = defaultConfig.get() config.logLevel = LOG_LEVEL.ERROR let errored = false config.plugins = { cache: { name: 'blablub' } } as any try { configInitialiser.initialize(new EventEmitter(), config) } catch (e) { errored = true } expect(errored).to.equal(true) }) }) describe('creates the right authentication handler', () => { before(() => { global.deepstreamLibDir = './src/test/plugins' }) it('works for authtype: none', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.auth = [{ type: 'none', options: {} }] const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.authentication.description).to.equal('Open Authentication') }) it('works for authtype: user', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.auth = [{ type: 'file', options: { users: {} } }] const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.authentication.description).to.contain('File Authentication') }) it('works for authtype: http', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.auth = [{ type: 'http', options: { endpointUrl: 'http://some-url.com', permittedStatusCodes: [200], requestTimeout: 2000, retryAttempts: 2, retryInterval: 50, retryStatusCodes: [ 404 ] } }] const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.authentication.description).to.equal('http webhook to http://some-url.com') }) it('fails for missing auth sections', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF delete config.auth expect(() => { configInitialiser.initialize(new EventEmitter(), config) }).to.throw('No authentication type specified') }) it('allows passing a custom authentication handler', async () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.auth = [{ path: '../mock/authentication-handler-mock', options: { hello: 'there' } }] const result = configInitialiser.initialize(new EventEmitter(), config) await result.services.authentication.whenReady() }) it('tries to find a custom authentication handler from name', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.auth = [{ name: 'my-custom-auth-handler', options: {} }] expect(() => { configInitialiser.initialize(new EventEmitter(), config) }).to.throw() }) it('overrides with type "none" when disableAuth is set', () => { global.deepstreamCLI = { disableAuth: true } const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.auth = [{ type: 'http', options: {} }] const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.authentication.description).to.equal('Open Authentication') delete global.deepstreamCLI }) }) describe('creates the permission service', () => { it('creates the config permission service', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.permission = { type: 'config', options: { path: './test-e2e/config/permissions-complex.json' } } const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.permission.description).to.contain('Valve Permissions') }) it('allows passing a custom permission handler', async () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.permission = { path: '../mock/permission-handler-mock', options: { hello: 'there' } } const result = configInitialiser.initialize(new EventEmitter(), config) await result.services.permission.whenReady() }) it('tries to find a custom authentication handler from name', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.auth = [{ name: 'my-custom-perm-handler', options: {} }] expect(() => { configInitialiser.initialize(new EventEmitter(), config) }).to.throw() }) it('fails for missing permission configs', () => { const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF delete config.permission expect(() => { configInitialiser.initialize(new EventEmitter(), config) }).to.throw('No permission type specified') }) it('overrides with type "none" when disablePermissions is set', () => { global.deepstreamCLI = { disablePermissions: true } const config = defaultConfig.get() config.logLevel = LOG_LEVEL.OFF config.permission = { type: 'config', options: {} } const result = configInitialiser.initialize(new EventEmitter(), config) expect(result.services.permission.description).to.equal('none') delete global.deepstreamCLI }) }) }) ================================================ FILE: src/config/config-initialiser.ts ================================================ import { readFileSync } from 'fs' import { join } from 'path' import { EOL } from 'os' import * as utils from '../utils/utils' import * as fileUtils from './file-utils' import { DeepstreamConfig, DeepstreamServices, DeepstreamConnectionEndpoint, PluginConfig, DeepstreamLogger, DeepstreamAuthentication, DeepstreamPermission, LOG_LEVEL, EVENT, DeepstreamMonitoring, DeepstreamAuthenticationCombiner, DeepstreamHTTPService, DeepstreamClusterNode } from '@deepstream/types' import { DistributedClusterRegistry } from '../services/cluster-registry/distributed-cluster-registry' import { SingleClusterNode } from '../services/cluster-node/single-cluster-node' import { VerticalClusterNode } from '../services/cluster-node/vertical-cluster-node' import { DefaultSubscriptionRegistryFactory } from '../services/subscription-registry/default-subscription-registry-factory' import { HTTPConnectionEndpoint } from '../connection-endpoint/http/connection-endpoint' import { CombineAuthentication } from '../services/authentication/combine/combine-authentication' import { OpenAuthentication } from '../services/authentication/open/open-authentication' import { ConfigPermission } from '../services/permission/valve/config-permission' import { OpenPermission } from '../services/permission/open/open-permission' import { WSBinaryConnectionEndpoint } from '../connection-endpoint/websocket/binary/connection-endpoint' import { WSTextConnectionEndpoint } from '../connection-endpoint/websocket/text/connection-endpoint' import { WSJSONConnectionEndpoint } from '../connection-endpoint/websocket/json/connection-endpoint' import { MQTTConnectionEndpoint } from '../connection-endpoint/mqtt/connection-endpoint' import { FileBasedAuthentication } from '../services/authentication/file/file-based-authentication' import { StorageBasedAuthentication } from '../services/authentication/storage/storage-based-authentication' import { HttpAuthentication } from '../services/authentication/http/http-authentication' import { NoopStorage } from '../services/storage/noop-storage' import { LocalCache } from '../services/cache/local-cache' import { StdOutLogger } from '../services/logger/std/std-out-logger' import { PinoLogger } from '../services/logger/pino/pino-logger' import { DeepstreamIOTelemetry } from '../services/telemetry/deepstreamio-telemetry' import { NoopMonitoring } from '../services/monitoring/noop-monitoring' import { CombineMonitoring } from '../services/monitoring/combine-monitoring' import { DistributedLockRegistry } from '../services/lock/distributed-lock-registry' import { DistributedStateRegistryFactory } from '../services/cluster-state/distributed-state-registry-factory' import { get as getDefaultOptions } from '../default-options' import Deepstream from '../deepstream.io' import { NodeHTTP } from '../services/http/node/node-http' import HTTPMonitoring from '../services/monitoring/http/monitoring-http' import LogMonitoring from '../services/monitoring/log/monitoring-log' import { InitialLogs } from './js-yaml-loader' import * as configValidator from './config-validator' import HeapSnapshot from '../plugins/heap-snapshot/heap-snapshot' let commandLineArguments: any const customPlugins = new Map() const defaultPlugins = new Map([ ['cache', LocalCache], ['storage', NoopStorage], ['logger', StdOutLogger], ['locks', DistributedLockRegistry], ['subscriptions', DefaultSubscriptionRegistryFactory], ['clusterRegistry', DistributedClusterRegistry], ['clusterStates', DistributedStateRegistryFactory], ['clusterNode', SingleClusterNode], ['httpService', NodeHTTP], ]) export const mergeConnectionOptions = function (config: any) { if (config && config.connectionEndpoints) { const defaultConfig = getDefaultOptions() for (const connectionEndpoint of config.connectionEndpoints) { const defaultPlugin = defaultConfig.connectionEndpoints.find((defaultEndpoint) => defaultEndpoint.type === connectionEndpoint.type) if (defaultPlugin) { connectionEndpoint.options = utils.merge(defaultPlugin.options, connectionEndpoint.options) } } } } /** * Registers plugins by name. Useful when wanting to include * custom plugins in a binary */ export const registerPlugin = function (name: string, construct: Function) { customPlugins.set(name, construct) } /** * Takes a configuration object and instantiates functional properties. * CLI arguments will be considered. */ export const initialize = function (deepstream: Deepstream, config: DeepstreamConfig, initialLogs: InitialLogs = []): { config: DeepstreamConfig, services: DeepstreamServices } { configValidator.validate(config) if (config.showLogo === true) { const logo = readFileSync(join(__dirname, '..', '..', '/ascii-logo.txt'), 'utf8') process.stdout.write(logo) process.stdout.write(`${EOL}===================== starting =====================${EOL}`) } // @ts-ignore commandLineArguments = global.deepstreamCLI || {} handleUUIDProperty(config) mergeConnectionOptions(config) const services = {} as DeepstreamServices services.notifyFatalException = () => { if (config.exitOnFatalError) { process.exit(1) } else { deepstream.emit(EVENT.FATAL_EXCEPTION) } } services.logger = handleLogger(config, services) services.monitoring = handleMonitoringPlugins(config, services) initialLogs.forEach((log) => { switch (log.level) { case LOG_LEVEL.DEBUG: services.logger.debug(log.event, log.message, log.meta) break case LOG_LEVEL.ERROR: services.logger.error(log.event, log.message, log.meta) break case LOG_LEVEL.INFO: services.logger.info(log.event, log.message, log.meta) break case LOG_LEVEL.WARN: services.logger.warn(log.event, log.message, log.meta) break case LOG_LEVEL.FATAL: services.logger.fatal(log.event, log.message, log.meta) break } }) services.subscriptions = new (resolvePluginClass(config.subscriptions, 'subscriptions', services.logger))(config.subscriptions.options, services, config) services.storage = new (resolvePluginClass(config.storage, 'storage', services.logger))(config.storage.options, services, config) services.cache = new (resolvePluginClass(config.cache, 'cache', services.logger))(config.cache.options, services, config) services.authentication = handleAuthStrategies(config, services) services.permission = handlePermissionStrategies(config, services) services.connectionEndpoints = handleConnectionEndpoints(config, services) services.locks = new (resolvePluginClass(config.locks, 'locks', services.logger))(config.locks.options, services, config) services.clusterNode = handleClusterNode(config, services) services.clusterRegistry = new (resolvePluginClass(config.clusterRegistry, 'clusterRegistry', services.logger))(config.clusterRegistry.options, services, config) services.clusterStates = new (resolvePluginClass(config.clusterStates, 'clusterStates', services.logger))(config.clusterStates.options, services, config) services.httpService = handleHTTPServer(config, services) services.telemetry = handleTelemetry(config, services) handleCustomPlugins(config, services) return { config, services } } /** * Transform the UUID string config to a UUID in the config object. */ function handleUUIDProperty (config: DeepstreamConfig): void { if (config.serverName === 'UUID') { config.serverName = utils.getUid() } } function handleClusterNode (config: DeepstreamConfig, services: any): DeepstreamClusterNode { let ClusterNodeClass = defaultPlugins.get('clusterNode') if (commandLineArguments.clusterSize) { config.clusterNode.name = 'vertical' } if (config.clusterNode.name === 'vertical') { ClusterNodeClass = VerticalClusterNode } else if (config.clusterNode.name || config.clusterNode.path) { ClusterNodeClass = resolvePluginClass(config.clusterNode, 'clusterNode', services.logger) if (!ClusterNodeClass) { throw new Error(`unable to resolve plugin ${config.clusterNode.name || config.clusterNode.path}`) } } return new ClusterNodeClass(config.clusterNode.options, services, config) } /** * Initialize the logger and overwrite the root logLevel if it's set * CLI arguments will be considered. */ function handleLogger (config: DeepstreamConfig, services: DeepstreamServices): DeepstreamLogger { const configOptions = (config.logger || {}).options if (commandLineArguments.colors !== undefined) { configOptions.colors = commandLineArguments.colors } let LoggerClass = defaultPlugins.get('logger') if (config.logger.name === 'pino') { LoggerClass = PinoLogger } else if (config.logger.name || config.logger.path) { LoggerClass = resolvePluginClass(config.logger, 'logger', services.logger) if (!LoggerClass) { throw new Error(`unable to resolve plugin ${config.logger.name || config.logger.path}`) } } const logger = new LoggerClass(configOptions, services, config) if (logger.log) { logger.debug = logger.debug || logger.log.bind(logger, LOG_LEVEL.DEBUG) logger.info = logger.info || logger.log.bind(logger, LOG_LEVEL.INFO) logger.warn = logger.warn || logger.log.bind(logger, LOG_LEVEL.WARN) logger.error = logger.error || logger.log.bind(logger, LOG_LEVEL.ERROR) logger.fatal = logger.fatal || logger.log.bind(logger, LOG_LEVEL.FATAL) } if (commandLineArguments.logLevel !== undefined) { configOptions.logLevel = commandLineArguments.logLevel } if (LOG_LEVEL[configOptions.logLevel] !== undefined) { if (typeof configOptions.logLevel === 'string') { logger.setLogLevel(LOG_LEVEL[configOptions.logLevel]) } else { logger.setLogLevel(configOptions.logLevel) } } else if (configOptions.logLevel) { throw new Error (`Unknown logLevel ${LOG_LEVEL[configOptions.logLevel]}`) } return logger } /** * Handle the plugins property in the config object the connectors. * Plugins can be passed either as a __path__ property or as a __name__ property with * a naming convention: *{cache: {name: 'redis'}}* will be resolved to the * npm module *@deepstream/cache-redis* * Options to the constructor of the plugin can be passed as *options* object. * * CLI arguments will be considered. */ function handleCustomPlugins (config: DeepstreamConfig, services: any): void { services.plugins = {} if (config.plugins == null) { return } const plugins = { ...config.plugins } for (const key in plugins) { const plugin = plugins[key] if (plugin.name === 'heap-snapshot') { services.plugins[key] = new HeapSnapshot(plugin.options || {}, services) } else { const PluginConstructor = resolvePluginClass(plugin, 'plugin', services.logger) services.plugins[key] = new PluginConstructor(plugin.options || {}, services, config) } } } /** * Handle connection endpoint plugin config. * The type is typically the protocol e.g. ws * Plugins can be passed either as a __path__ property or as a __name__ property with * a naming convetion: *{amqp: {name: 'my-plugin'}}* will be resolved to the * npm module *deepstream.io/connection-my-plugin* * Exception: the name *uws* will be resolved to deepstream.io's internal uWebSockets plugin * Options to the constructor of the plugin can be passed as *options* object. * * CLI arguments will be considered. */ function handleConnectionEndpoints (config: DeepstreamConfig, services: any): DeepstreamConnectionEndpoint[] { // delete any endpoints that have been set to `null` for (const type in config.connectionEndpoints) { if (config.connectionEndpoints[type] === null) { delete config.connectionEndpoints[type] } } if (!config.connectionEndpoints || Object.keys(config.connectionEndpoints).length === 0) { throw new Error('No connection endpoints configured') } const connectionEndpoints: DeepstreamConnectionEndpoint[] = [] for (const plugin of config.connectionEndpoints) { plugin.options = plugin.options || {} let PluginConstructor if (plugin.type === 'ws-binary') { PluginConstructor = WSBinaryConnectionEndpoint } else if (plugin.type === 'ws-text') { PluginConstructor = WSTextConnectionEndpoint } else if (plugin.type === 'ws-json') { PluginConstructor = WSJSONConnectionEndpoint } else if (plugin.type === 'mqtt') { PluginConstructor = MQTTConnectionEndpoint } else if (plugin.type === 'http') { PluginConstructor = HTTPConnectionEndpoint } else { PluginConstructor = resolvePluginClass(plugin, 'connection', services.logger) } connectionEndpoints.push(new PluginConstructor(plugin.options, services, config)) } return connectionEndpoints } /** * Instantiate the given plugin, which either needs a path property or a name * property which fits to the npm module name convention. Options will be passed * to the constructor. * * CLI arguments will be considered. */ function resolvePluginClass (plugin: PluginConfig, type: string, logger: DeepstreamLogger): any { if (customPlugins.has(plugin.name)) { return customPlugins.get(plugin.name) } // Required for bundling via nexe const req = require let requirePath let pluginConstructor let es6Adaptor if (plugin.path != null) { try { requirePath = fileUtils.lookupLibRequirePath(plugin.path) es6Adaptor = req(requirePath) pluginConstructor = es6Adaptor.default ? es6Adaptor.default : es6Adaptor } catch (error) { logger.fatal(EVENT.CONFIG_ERROR, `Error loading ${type} plugin via path ${requirePath}: ${error}`) // Throw error due to how tests are written throw new Error() } } else if (plugin.name != null && type) { try { requirePath = fileUtils.lookupLibRequirePath(`@deepstream/${type.toLowerCase()}-${plugin.name.toLowerCase()}`) es6Adaptor = req(requirePath) } catch (firstError) { const firstPath = requirePath try { requirePath = fileUtils.lookupLibRequirePath(`deepstream.io-${type.toLowerCase()}-${plugin.name.toLowerCase()}`) es6Adaptor = req(requirePath) } catch (secondError) { logger.debug(EVENT.CONFIG_ERROR, `Error loading module ${firstPath}: ${firstError}`) logger.debug(EVENT.CONFIG_ERROR, `Error loading module ${requirePath}: ${secondError}`) logger.fatal(EVENT.CONFIG_ERROR, 'Error loading module, exiting') // Throw error due to how tests are written throw new Error() } } pluginConstructor = es6Adaptor.default ? es6Adaptor.default : es6Adaptor } else if (plugin.name != null) { try { requirePath = fileUtils.lookupLibRequirePath(plugin.name) es6Adaptor = req(requirePath) pluginConstructor = es6Adaptor.default ? es6Adaptor.default : es6Adaptor } catch (error) { logger.fatal(EVENT.CONFIG_ERROR, `Error loading ${type} plugin via name ${plugin.name}`) // Throw error due to how tests are written throw new Error() } } else if (plugin.type === 'default' && defaultPlugins.has(type as any)) { pluginConstructor = defaultPlugins.get(type as any) } else { // This error is used to bubble the event due to how tests are written throw new Error(`Neither name nor path property found for ${type} plugin type: ${plugin.type}`) } return pluginConstructor } /** * Instantiates the authentication handlers registered for *config.auth.type* * * CLI arguments will be considered. */ function handleAuthStrategies (config: DeepstreamConfig, services: DeepstreamServices): DeepstreamAuthenticationCombiner { if (commandLineArguments.disableAuth) { config.auth = [{ type: 'none', options: {} }] } if (!config.auth) { throw new Error('No authentication type specified') } return new CombineAuthentication(config.auth.map((auth) => handleAuthStrategy(auth, config, services))) } /** * Instantiates the authentication handler registered for *config.auth.type* * * CLI arguments will be considered. */ function handleAuthStrategy (auth: PluginConfig, config: DeepstreamConfig, services: DeepstreamServices): DeepstreamAuthentication { let AuthenticationHandlerClass const authStrategies = { none: OpenAuthentication, file: FileBasedAuthentication, http: HttpAuthentication, storage: StorageBasedAuthentication } if (auth.name || auth.path) { AuthenticationHandlerClass = resolvePluginClass(auth, 'authentication', services.logger) if (!AuthenticationHandlerClass) { throw new Error(`unable to resolve authentication handler ${auth.name || auth.path}`) } } else if (auth.type && (authStrategies as any)[auth.type]) { if (auth.options && auth.options.path) { const req = require auth.options.users = req(fileUtils.lookupConfRequirePath(auth.options.path)) } AuthenticationHandlerClass = (authStrategies as any)[auth.type] } else { throw new Error(`Unknown authentication type ${auth.type}`) } if (config.auth.length > 1) { if (!auth.options.reportInvalidParameters) auth.options.reportInvalidParameters = false } return new AuthenticationHandlerClass(auth.options, services, config) } /** * Instantiates the permission handler registered for *config.permission.type* * * CLI arguments will be considered. */ function handlePermissionStrategies (config: DeepstreamConfig, services: DeepstreamServices): DeepstreamPermission { const permission = config.permission if (!config.permission) { throw new Error('No permission type specified') } if (commandLineArguments.disablePermissions) { config.permission.type = 'none' config.permission.options = {} } let PermissionHandlerClass const permissionStrategies = { config: ConfigPermission, none: OpenPermission } if (permission.name || permission.path) { PermissionHandlerClass = resolvePluginClass(permission, 'permission', services.logger) if (!PermissionHandlerClass) { throw new Error(`unable to resolve plugin ${permission.name || permission.path}`) } } else if (permission.type && (permissionStrategies as any)[permission.type]) { if (config.permission.options && config.permission.options.path) { const req = require config.permission.options.permissions = req(fileUtils.lookupConfRequirePath(config.permission.options.path)) } PermissionHandlerClass = (permissionStrategies as any)[permission.type] } else { throw new Error(`Unknown permission type ${permission.type}`) } return new PermissionHandlerClass(permission.options, services, config) } /** * Instantiates the monitoring handlers registered for *config.monitoring.type* * */ function handleMonitoringPlugins (config: DeepstreamConfig, services: DeepstreamServices): DeepstreamMonitoring { if (!config.monitoring) { config.monitoring = [{ type: 'none', options: {} }] } else { // this is in order to make it backwards compatible if (!Array.isArray(config.monitoring)) { config.monitoring = [config.monitoring] } } return new CombineMonitoring(config.monitoring.map((monitoringConfig: PluginConfig) => handleMonitoring(monitoringConfig, config, services))) } function handleMonitoring (monitoringConfig: PluginConfig, config: DeepstreamConfig, services: DeepstreamServices): DeepstreamMonitoring { let MonitoringClass const monitoringPlugins = { default: NoopMonitoring, none: NoopMonitoring, http: HTTPMonitoring, log: LogMonitoring } if (monitoringConfig.name || monitoringConfig.path) { return new (resolvePluginClass(monitoringConfig, 'monitoring', services.logger))(monitoringConfig.options, services, config) } else if (monitoringConfig.type && (monitoringPlugins as any)[monitoringConfig.type]) { MonitoringClass = (monitoringPlugins as any)[monitoringConfig.type] } else { throw new Error(`Unknown monitoring type ${monitoringConfig.type}`) } return new MonitoringClass(monitoringConfig.options, services, config) } function handleHTTPServer (config: DeepstreamConfig, services: DeepstreamServices): DeepstreamHTTPService { let HttpServerClass const httpPlugins = { default: NodeHTTP } if (commandLineArguments.host) { config.httpServer.options.host = commandLineArguments.host } if (commandLineArguments.port) { config.httpServer.options.port = commandLineArguments.port } if (config.httpServer.name || config.httpServer.path) { return new (resolvePluginClass(config.httpServer, 'httpServer', services.logger))(config.httpServer.options, services, config) } else if (config.httpServer.type && (httpPlugins as any)[config.httpServer.type]) { HttpServerClass = (httpPlugins as any)[config.httpServer.type] } else if (config.httpServer.type === 'uws') { try { const { UWSHTTP } = require('../services/http/uws/uws-http') HttpServerClass = UWSHTTP } catch (e) { throw new Error('Error loading uws http service, this is most likely due to uWebsocket.js not being supported on this platform') } } else { throw new Error(`Unknown httpServer type ${config.httpServer.type}`) } return new HttpServerClass(config.httpServer.options, services, config) } function handleTelemetry (config: DeepstreamConfig, services: DeepstreamServices): DeepstreamMonitoring { let TelemetryPlugin const telemetryPlugins = { deepstreamIO: DeepstreamIOTelemetry } if (config.telemetry.name || config.telemetry.path) { return new (resolvePluginClass(config.telemetry, 'telemetry', services.logger))(config.telemetry.options, services, config) } else if (config.telemetry.type && (telemetryPlugins as any)[config.telemetry.type]) { TelemetryPlugin = (telemetryPlugins as any)[config.telemetry.type] } else { throw new Error(`Unknown telemetry type ${config.telemetry.type}`) } return new TelemetryPlugin(config.telemetry.options, services, config) } ================================================ FILE: src/config/config-validator.ts ================================================ import {Ajv} from 'ajv' import addFormat from 'ajv-formats' import betterAjvErrors from 'better-ajv-errors' import { LOG_LEVEL } from '@deepstream/types' const LogLevelValidation = { type: ['integer'], enum: [ LOG_LEVEL.DEBUG, LOG_LEVEL.INFO, LOG_LEVEL.WARN, LOG_LEVEL.ERROR, LOG_LEVEL.OFF ] } function getPluginOptions (name: string, types: string[], properties: any) { return { [name]: { type: 'object', properties: { type: { type: 'string', enum: types }, name: { type: 'string', minLength: 1 }, path: { type: 'string', minLength: 1 }, options: { type: 'object', properties } }, oneRequired: ['type', 'name', 'path'] } } } const generalOptions = { libDir: { type: ['string', 'null'] }, serverName: { type: 'string', minLength: 1 }, showLogo: { type: 'boolean' }, exitOnFatalError: { type: 'boolean' }, dependencyInitializationTimeout: { type: 'number', minimum: 1000 }, logLevel: LogLevelValidation } const enabledFeatures = { enabledFeatures: { record: { type: 'boolean' }, event: { type: 'boolean' }, rpc: { type: 'boolean' }, presence: { type: 'boolean' }, monitoring: { type: 'boolean' }, } } const rpcOptions = { rpc: { type: ['object'], ackTimeout: { type: 'integer', minimum: 1 }, responseTimeout: { type: 'integer', minimum: 1 } } } const recordOptions = { record: { type: ['object'], cacheRetrievalTimeout: { type: 'integer', minimum: 50 }, storageRetrievalTimeout: { type: 'integer', minimum: 50 }, storageExclusionPrefixes: { type: ['null', 'array'], items: { type: 'string' } }, storageHotPathPrefixes: { type: ['null', 'array'], items: { type: 'string' } } } } const listenOptions = { listen: { type: ['object'], shuffleProviders: { type: 'boolean' }, responseTimeout: { type: 'integer', minimum: 50 }, rematchInterval: { type: 'integer', minimum: 50 }, matchCooldown: { type: 'integer', minimum: 50 }, } } const httpServer = getPluginOptions( 'httpServer', ['default', 'uws'], { host: { type: 'string', minLength: 1 }, port: { type: 'integer', minimum: 1 }, allowAllOrigins: { type: 'boolean' }, origins: { type: 'array', items: { type: 'string', format: 'uri' } }, } ) const cacheOptions = getPluginOptions( 'cache', ['default'], { } ) const storageOptions = getPluginOptions( 'storage', ['default'], { } ) const telemetryOptions = getPluginOptions( 'telemetry', ['deepstreamIO'], { enabled: { type: 'boolean' } } ) const authenticationOptions = { auth: { type: 'array', items: { properties: { type: { type: 'string', enum: ['none', 'file', 'http', 'storage'] }, name: { type: 'string', minLength: 1 }, path: { type: 'string', minLength: 1 }, options: { type: 'object', properties: { hash: { type: 'string', minLength: 1 }, iterations: { type: 'integer', minimum: 1 }, keyLength: { type: 'integer', minimum: 1 }, createUser: { type: 'boolean' }, table: { type: 'string', minLength: 1 }, endpointUrl: { type: 'string', format: 'uri'}, permittedStatusCodes: { type: 'array', items: { type: 'integer' } }, requestTimeout: { type: 'integer', minimum: 1 }, } } } } } } const permissionOptions = getPluginOptions( 'permission', ['config', 'none'], { path: { type: 'string', minLength: 1 }, maxRuleIterations: { type: 'integer', minimum: 1 }, cacheEvacuationInterval: { type: 'integer', minimum: 1 } } ) const connEndpointsOptions = { connectionEndpoints: { type: 'array', items: { properties: { type: { type: 'string', enum: ['ws-text', 'ws-json', 'ws-binary', 'http', 'mqtt'] }, name: { type: 'string', minLength: 1 }, path: { type: 'string', minLength: 1 }, options: { type: 'object', properties: { port: { type: 'integer', minimum: 1 }, host: { type: 'string', minLength: 1 }, healthCheckPath: { type: 'string', minLength: 1 }, maxMessageSize: { type: 'integer', minimum: 0 }, // WEBSOCKET urlPath: { type: 'string', minLength: 1 }, heartbeatInterval: { type: 'integer', minimum: 1 }, outgoingBufferTimeout: { type: 'integer', minimum: 0 }, unauthenticatedClientTimeout: { type: ['integer', 'boolean'], minimum: 1 }, maxAuthAttempts: { type: 'integer', minimum: 1 }, // HTTP allowAuthData: { type: 'boolean' }, enableAuthEndpoint: { type: 'boolean' }, authPath: { type: 'string', minLength: 1 }, postPath: { type: 'string', minLength: 1 }, getPath: { type: 'string', minLength: 1 }, } } } } } } const loggerOptions = getPluginOptions( 'logger', ['default', 'json'], { options: { colors: { type: 'boolean' }, logLevel: LogLevelValidation } } ) const subscriptionsOptions = getPluginOptions( 'subscriptions', ['default'], { subscriptionsSanityTimer: { type: 'integer', minimum: 50 }, } ) const monitoringOptions = { monitoring: { type: ['array', 'object'], items: { properties: { type: { type: 'string', enum: ['none', 'log', 'http'] }, name: { type: 'string', minLength: 1 }, path: { type: 'string', minLength: 1 }, }, options: { type: 'object'} } } } const locksOptions = getPluginOptions( 'locks', ['default'], { holdTimeout: { type: 'integer', minimum: 50 }, requestTimeout: { type: 'integer', minimum: 50 }, } ) const clusterNodeOptions = getPluginOptions( 'clusterNode', ['default', 'vertical'], { } ) const clusterRegistryOptions = getPluginOptions( 'clusterRegistry', ['default'], { keepAliveInterval: { type: 'integer', minimum: 1 }, activeCheckInterval: { type: 'integer', minimum: 1 }, nodeInactiveTimeout: { type: 'integer', minimum: 1 }, } ) const clusterStatesOptions = getPluginOptions( 'clusterStates', ['default'], { reconciliationTimeout: { type: 'integer', minimum: 1 }, } ) const customPluginsOptions = { plugins: { type: ['null', 'object'], properties: { } } } const schema = { additionalProperties: false, properties: { ...generalOptions, ...enabledFeatures, ...rpcOptions, ...recordOptions, ...listenOptions, ...httpServer, ...connEndpointsOptions, ...loggerOptions, ...cacheOptions, ...storageOptions, ...authenticationOptions, ...permissionOptions, ...subscriptionsOptions, ...monitoringOptions, ...telemetryOptions, ...locksOptions, ...clusterNodeOptions, ...clusterRegistryOptions, ...clusterStatesOptions, ...customPluginsOptions } } export const validate = function (config: Object): void { const ajv = new Ajv({ allErrors: true, strict: false }) addFormat(ajv) const validator = ajv.compile(schema) const valid = validator(config) if (!valid) { const output = betterAjvErrors(schema, config, validator.errors ?? [], { format: 'js' }) console.error('There was an error validating your configuration:') output.forEach((e, i) => console.error(`${i + 1})${e.error}${e.suggestion ? `. ${e.suggestion}` : ''}`)) process.exit(1) } } ================================================ FILE: src/config/ds-info.ts ================================================ import * as fs from 'fs' import * as path from 'path' import * as os from 'os' import * as glob from 'glob' export const getDSInfo = (libDir?: string) => { let meta let pkg try { meta = require('../../meta.json') } catch (err) { // if deepstream is not installed as binary (source or npm) pkg = require('../../package.json') meta = { deepstreamVersion: pkg.version, ref: pkg.gitHead || pkg._resolved || 'N/A', buildTime: 'N/A' } } meta.platform = os.platform() meta.arch = os.arch() meta.nodeVersion = process.version if (libDir) { fetchLibs(libDir, meta) } return meta } const fetchLibs = (libDir: string, meta: any) => { const directory = libDir || 'lib' const files = glob.sync(path.join(directory, '*', 'package.json')) meta.libs = files.map((filePath: any) => { const pkg = fs.readFileSync(filePath, 'utf8') const object = JSON.parse(pkg) return `${object.name}:${object.version}` }) } ================================================ FILE: src/config/file-utils.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as fileUtils from './file-utils' const path = require('path') describe('fileUtils tests', () => { it('check cases with no or a relative prefix', () => { // node style path (no dot at the start and not absolute path) expect(fileUtils.lookupRequirePath('foo-bar')).to.deep.equal('foo-bar') expect(fileUtils.lookupRequirePath('dir/foo-bar')).to.deep.equal('dir/foo-bar') expect(fileUtils.lookupRequirePath('foo-bar', 'pre')).to.deep.equal(path.resolve('pre', 'foo-bar')) expect(fileUtils.lookupRequirePath('dir/foo-bar', 'pre')).to.deep.equal(path.resolve('pre', 'dir', 'foo-bar')) // use an absolute path for the fileUtilsname expect(fileUtils.lookupRequirePath('/usr/foo-bar')).to.deep.equal('/usr/foo-bar') expect(fileUtils.lookupRequirePath('/usr/dir/foo-bar')).to.deep.equal('/usr/dir/foo-bar') expect(fileUtils.lookupRequirePath('/usr/foo-bar', 'pre')).to.deep.equal('/usr/foo-bar') expect(fileUtils.lookupRequirePath('/usr/dir/foo-bar', 'pre')).to.deep.equal('/usr/dir/foo-bar') // use a relative path for the fileUtilsname expect(fileUtils.lookupRequirePath('./foo-bar')).to.deep.equal(path.resolve('foo-bar')) expect(fileUtils.lookupRequirePath('./dir/foo-bar')).to.deep.equal(path.resolve('dir', 'foo-bar')) expect(fileUtils.lookupRequirePath('./foo-bar', 'pre')).to.deep.equal(path.resolve('pre', 'foo-bar')) expect(fileUtils.lookupRequirePath('./dir/foo-bar', 'pre')).to.deep.equal(path.resolve('pre', 'dir', 'foo-bar')) }) it('check cases with an absolute prefix', () => { // node style path (no dot at the start and not absolute path) expect(fileUtils.lookupRequirePath('foo-bar', '/pre')).to.deep.equal(path.resolve('/pre', 'foo-bar')) expect(fileUtils.lookupRequirePath('dir/foo-bar', '/pre')).to.deep.equal(path.resolve('/pre', 'dir', 'foo-bar')) // use an absolute path for the fileUtilsname expect(fileUtils.lookupRequirePath('/usr/foo-bar', '/pre')).to.deep.equal('/usr/foo-bar') expect(fileUtils.lookupRequirePath('/usr/dir/foo-bar', '/pre')).to.deep.equal('/usr/dir/foo-bar') // use a relative path for the fileUtilsname expect(fileUtils.lookupRequirePath('./foo-bar', '/pre')).to.deep.equal(path.resolve('/pre', 'foo-bar')) expect(fileUtils.lookupRequirePath('./dir/foo-bar', '/pre')).to.deep.equal(path.resolve('/pre', 'dir', 'foo-bar')) }) }) ================================================ FILE: src/config/file-utils.ts ================================================ import * as fs from 'fs' import * as path from 'path' /** * Append the global library directory as the prefix to any path * used here */ export const lookupLibRequirePath = function (filePath: string): string { // @ts-ignore return exports.lookupRequirePath(filePath, global.deepstreamLibDir) } /** * Append the global configuration directory as the prefix to any path * used here */ export const lookupConfRequirePath = function (filePath: string): string { // @ts-ignore return exports.lookupRequirePath(filePath, global.deepstreamConfDir) } /** * Resolve a path which will be passed to *require*. * * If a prefix is not set the filePath will be returned * Otherwise it will either replace return a new path prepended with the prefix. * If the prefix is not an absolute path it will also prepend the CWD. * * file || relative (starts with .) | absolute | else (npm module path) * ----------------------------------------------------------------------------- * *prefix || *CWD + prefix + file | file | *CWD + prefix + file * *no prefix || CWD + file | file | file (resolved by nodes require) * * *CWD = ignore CWD if prefix is absolute */ export const lookupRequirePath = function (filePath: string, prefix?: string): string { // filePath is absolute if (path.parse(filePath).root !== '') { return filePath } // filePath is not relative (and not absolute) if (filePath[0] !== '.') { if (prefix == null) { return filePath } return resolvePrefixAndFile(filePath, prefix) } // filePath is relative, starts with . if (prefix == null) { return path.resolve(process.cwd(), filePath) } return resolvePrefixAndFile(filePath, prefix) } /** * Returns true if a file exists for a given path */ export const fileExistsSync = function (filePath: string): boolean { try { fs.lstatSync(filePath) return true } catch (e) { return false } } /** * Append the prefix to the current working directory, * or use it as an absolute path */ function resolvePrefixAndFile (nonAbsoluteFilePath: string, prefix: string): string { // prefix is not absolute if (path.parse(prefix).root === '') { return path.resolve(process.cwd(), prefix, nonAbsoluteFilePath) } // prefix is absolute return path.resolve(prefix, nonAbsoluteFilePath) } ================================================ FILE: src/config/js-yaml-loader.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import { spy } from 'sinon' import * as path from 'path' const proxyquire = require('proxyquire').noPreserveCache() const utils = require('../utils/utils') const jsYamlLoader = require('./js-yaml-loader') function setUpStub (fileExists?, fileContent?) { const fileMock: any = {} if (typeof fileExists !== 'undefined') { fileMock.fileExistsSync = function () { return !!fileExists } } const fsMock: any = {} if (typeof fileContent !== 'undefined') { fsMock.readFileSync = function () { return fileContent } } const configLoader = proxyquire('./js-yaml-loader', { './file-utils': fileMock, 'fs': fsMock }) spy(fileMock, 'fileExistsSync') spy(fsMock, 'readFileSync') return { configLoader, fileMock } } describe.skip('js-yaml-loader', () => { afterEach(() => { global.deepstreamConfDir = null global.deepstreamLibDir = null global.deepstreamCLI = null }) describe('js-yaml-loader loads and parses json files', () => { const jsonLoader = { load: jsYamlLoader.readAndParseFile } it('initialises the loader', () => { expect(typeof jsonLoader.load).to.equal('function') }) it('errors if invoked with an invalid path', (done) => { jsonLoader.load(null, (err, result) => { expect(err.toString()).to.contain('path') expect(result).to.equal(undefined) done() }) }) it('successfully loads and parses a valid JSON file', (done) => { jsonLoader.load('./src/test/config/basic-valid-json.json', (err, result) => { expect(err).to.equal(null) expect(result).to.deep.equal({ pet: 'pug' }) done() }) }) it('errors when trying to load non existant file', (done) => { jsonLoader.load('./src/test/config/does-not-exist.json', (err, result) => { expect(err.toString()).to.contain('no such file or directory') expect(result).to.equal(undefined) done() }) }) }) describe('js-yaml-loader', () => { it('loads the default yml file', () => { const loader = jsYamlLoader const result = loader.loadConfig() let defaultYamlConfig = result.config expect(result.file).to.deep.equal(path.join('conf', 'config.yml')) // TODO // expect(defaultYamlConfig.serverName).to.have.type('string') defaultYamlConfig = utils.merge(defaultYamlConfig, { permission: { type: 'none', options: null }, authentication: null, plugins: null, serverName: null, logger: null }) expect(defaultYamlConfig).not.to.equal(null) }) it('tries to load yaml, js and json file and then default', () => { const stub = setUpStub(false) expect(() => { stub.configLoader.loadConfig() }).to.throw() expect(stub.fileMock.fileExistsSync).to.have.callCount(28) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith(path.join('conf', 'config.js')) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith(path.join('conf', 'config.json')) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith(path.join('conf', 'config.yml')) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('/etc/deepstream/config.js') expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('/etc/deepstream/config.json') expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('/etc/deepstream/config.yml') }) it('load a custom yml file path', () => { const stub = setUpStub() const config = stub.configLoader.loadConfig('./src/test/config/config.yml').config expect(stub.fileMock.fileExistsSync).to.have.callCount(1) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('./src/test/config/config.yml') expect(config.serverName).not.to.equal(undefined) expect(config.serverName).not.to.deep.equal('') expect(config.serverName).not.to.deep.equal('UUID') expect(config.port).to.deep.equal(1337) expect(config.host).to.deep.equal('1.2.3.4') expect(config.colors).to.deep.equal(false) expect(config.showLogo).to.deep.equal(false) // TODO // expect(config.logLevel).to.deep.equal(C.LOG_LEVEL.ERROR) }) it('loads a missing custom yml file path', () => { const stub = setUpStub() expect(() => { stub.configLoader.loadConfig(null, { config: './src/test/config/does-not-exist.yml' }) }).to.throw('Configuration file not found at: ./src/test/config/does-not-exist.yml') }) it('load a custom json file path', () => { const stub = setUpStub(true, JSON.stringify({ port: 1001 })) const config = stub.configLoader.loadConfig(null, { config: './foo.json' }).config expect(stub.fileMock.fileExistsSync).to.have.callCount(1) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('./foo.json') expect(config.port).to.deep.equal(1001) }) it('load a custom js file path', () => { const stub = setUpStub() let config = stub.configLoader.loadConfig(null, { config: './src/test/config/config.js' }).config expect(stub.fileMock.fileExistsSync).to.have.callCount(1) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('./src/test/config/config.js') expect(config.port).to.deep.equal(1002) config = stub.configLoader.loadConfig(null, { config: path.join(process.cwd(), 'src/test/config/config.js') }).config expect(stub.fileMock.fileExistsSync).to.have.callCount(2) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith(path.join(process.cwd(), 'test/test/config/config.js')) expect(config.port).to.deep.equal(1002) }) it('fails if the custom file format is not supported', () => { const stub = setUpStub(true, 'content doesnt matter here') expect(() => { // tslint:disable-next-line:no-unused-expression stub.configLoader.loadConfig(null, { config: './config.foo' }).config }).to.throw('.foo is not supported as configuration file') }) it('fails if the custom file was not found', () => { const stub = setUpStub(false) expect(() => { // tslint:disable-next-line:no-unused-expression stub.configLoader.loadConfig(null, { config: './not-existing-config' }).config }).to.throw('Configuration file not found at: ./not-existing-config') expect(stub.fileMock.fileExistsSync).to.have.callCount(1) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('./not-existing-config') }) it('fails if the yaml file is invalid', () => { const stub = setUpStub() expect(() => { // tslint:disable-next-line:no-unused-expression stub.configLoader.loadConfig(null, { config: './src/test/config/config-broken.yml' }).config }).to.throw(/asdsad: ooops/) expect(stub.fileMock.fileExistsSync).to.have.callCount(1) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('./src/test/config/config-broken.yml') }) it('fails if the js file is invalid', () => { const stub = setUpStub() expect(() => { // tslint:disable-next-line:no-unused-expression stub.configLoader.loadConfig(null, { config: './src/test/config/config-broken.js' }).config }).to.throw(/foobarBreaksIt is not defined/) expect(stub.fileMock.fileExistsSync).to.have.callCount(1) expect(stub.fileMock.fileExistsSync).to.have.been.calledWith('./src/test/config/config-broken.js') }) }) describe('supports environment variable substitution', () => { let configLoader beforeEach(() => { process.env.ENVIRONMENT_VARIABLE_TEST_1 = 'an_environment_variable_value' process.env.ENVIRONMENT_VARIABLE_TEST_2 = 'another_environment_variable_value' process.env.EXAMPLE_HOST = 'host' process.env.EXAMPLE_PORT = '1234' configLoader = jsYamlLoader }) it('does environment variable substitution for yaml', () => { const config = configLoader.loadConfig(null, { config: './src/test/config/config.yml' }).config expect(config.environmentvariable).to.equal('an_environment_variable_value') expect(config.another.environmentvariable).to.equal('another_environment_variable_value') // expect(config.thisenvironmentdoesntexist).to.equal('DOESNT_EXIST') expect(config.multipleenvs).to.equal('host:1234') }) it('does environment variable substitution for json', () => { const config = configLoader.loadConfig(null, { config: './src/test/config/json-with-env-variables.json' }).config expect(config.environmentvariable).to.equal('an_environment_variable_value') expect(config.another.environmentvariable).to.equal('another_environment_variable_value') // expect(config.thisenvironmentdoesntexist).to.equal('DOESNT_EXIST') expect(config.multipleenvs).to.equal('host:1234') }) }) describe('merges in deepstreamCLI options', () => { let configLoader beforeEach(() => { global.deepstreamCLI = { port: 5555 } configLoader = jsYamlLoader }) afterEach(() => { delete process.env.deepstreamCLI }) it('does cli substitution', () => { const config = configLoader.loadConfig().config expect(config.connectionEndpoints.websocket.options.port).to.deep.equal(5555) }) }) describe('load plugins by relative path property', () => { let services beforeEach(() => { const fileMock = { fileExistsSync () { return true } } const fsMock = { readFileSync (filePath) { if (filePath === './config.json') { return `{ "plugins": { "logger": { "path": "./logger" }, "cache": { "path": "./cache", "options": { "foo": 3, "bar": 4 } } } }` } throw new Error(`should not require any other file: ${filePath}`) } } const loggerModule = function (options) { return options } loggerModule['@noCallThru'] = true loggerModule['@global'] = true class CacheModule { public options: any constructor (options) { this.options = options } } CacheModule['@noCallThru'] = true CacheModule['@global'] = true const configLoader = proxyquire('./js-yaml-loader', { 'fs': fsMock, './file-utils': fileMock, [path.resolve('./logger')]: loggerModule, [path.resolve('./cache')]: CacheModule }) services = configLoader.loadConfig(null, { config: './config.json' }).services }) it('load plugins', () => { expect(services.cache.options).to.deep.equal({ foo: 3, bar: 4 }) }) }) describe.skip('load plugins by path property (npm module style)', () => { let services beforeEach(() => { const fileMock = { fileExistsSync () { return true } } const fsMock = { readFileSync (filePath) { if (filePath === './config.json') { return `{ "plugins": { "cache": { "path": "foo-bar-qox", "options": { "foo": 3, "bar": 4 } } } }` } throw new Error(`should not require any other file: ${filePath}`) } } // tslint:disable-next-line:max-classes-per-file class FooBar { public options: any constructor (options) { this.options = options } } FooBar['@noCallThru'] = true FooBar['@global'] = true const configLoader = proxyquire('./js-yaml-loader', { 'fs': fsMock, './file-utils': fileMock, 'foo-bar-qox': FooBar }) services = configLoader.loadConfig(null, { config: './config.json' }).services }) it('load plugins', () => { expect(services.cache.options).to.deep.equal({ foo: 3, bar: 4 }) }) }) describe('load plugins by name with a name convention', () => { let services beforeEach(() => { const fileMock = { fileExistsSync () { return true } } const fsMock = { readFileSync (filePath) { if (filePath === './config.json') { return `{ "plugins": { "cache": { "name": "super-cache", "options": { "foo": 5, "bar": 6 } }, "storage": { "name": "super-storage", "options": { "foo": 7, "bar": 8 } } } }` } throw new Error(`should not require any other file: ${filePath}`) } } // tslint:disable-next-line:max-classes-per-file class SuperCache { public options: any constructor (options) { this.options = options } } SuperCache['@noCallThru'] = true SuperCache['@global'] = true // tslint:disable-next-line:max-classes-per-file class SuperStorage { public options: any constructor (options) { this.options = options } } SuperStorage['@noCallThru'] = true SuperStorage['@global'] = true const configLoader = proxyquire('./js-yaml-loader', { 'fs': fsMock, './file-utils': fileMock, 'deepstream.io-cache-super-cache': SuperCache, 'deepstream.io-storage-super-storage': SuperStorage }) services = configLoader.loadConfig(null, { config: './config.json' }).services }) it('load plugins', () => { expect(services.cache.options).to.deep.equal({ foo: 5, bar: 6 }) expect(services.storage.options).to.deep.equal({ foo: 7, bar: 8 }) }) }) describe('load plugins by name with a name convention with lib prefix', () => { let services beforeEach(() => { const fileMock = { fileExistsSync () { return true } } const fsMock = { readFileSync (filePath) { if (filePath === './config.json') { return `{ "plugins": { "cache": { "name": "super-cache", "options": { "foo": -1, "bar": -2 } }, "storage": { "name": "super-storage", "options": { "foo": -3, "bar": -4 } } } }` } throw new Error(`should not require any other file: ${filePath}`) } } // tslint:disable-next-line:max-classes-per-file class SuperCache { public options: any constructor (options) { this.options = options } } SuperCache['@noCallThru'] = true SuperCache['@global'] = true // tslint:disable-next-line:max-classes-per-file class SuperStorage { public options: any constructor (options) { this.options = options } } SuperStorage['@noCallThru'] = true SuperStorage['@global'] = true // tslint:disable-next-line:max-classes-per-file class HTTPMock { public options: any constructor (options) { this.options = options } } HTTPMock['@noCallThru'] = true HTTPMock['@global'] = true const configLoader = proxyquire('./js-yaml-loader', { 'fs': fsMock, './file-utils': fileMock, [path.resolve(process.cwd(), 'foobar', 'deepstream.io-cache-super-cache')]: SuperCache, [path.resolve(process.cwd(), 'foobar', 'deepstream.io-storage-super-storage')]: SuperStorage, [path.resolve(process.cwd(), 'foobar', 'deepstream.io-connection-http')]: HTTPMock }) services = configLoader.loadConfig(null, { config: './config.json', libDir: 'foobar' }).services }) it('load plugins', () => { expect(services.cache.options).to.deep.equal({ foo: -1, bar: -2 }) expect(services.storage.options).to.deep.equal({ foo: -3, bar: -4 }) }) }) describe('load plugins by name with a name convention with an absolute lib prefix', () => { let services beforeEach(() => { const fileMock = { fileExistsSync () { return true } } const fsMock = { readFileSync (filePath) { if (filePath === './config.json') { return `{ "plugins": { "cache": { "name": "super-cache", "options": { "foo": -1, "bar": -2 } }, "storage": { "name": "super-storage", "options": { "foo": -3, "bar": -4 } } } }` } throw new Error(`should not require any other file: ${filePath}`) } } // tslint:disable-next-line:max-classes-per-file class SuperCache { public options: any constructor (options) { this.options = options } } SuperCache['@noCallThru'] = true SuperCache['@global'] = true // tslint:disable-next-line:max-classes-per-file class SuperStorage { public options: any constructor (options) { this.options = options } } SuperStorage['@noCallThru'] = true SuperStorage['@global'] = true // tslint:disable-next-line:max-classes-per-file class HTTPMock { public options: any constructor (options) { this.options = options } } HTTPMock['@noCallThru'] = true HTTPMock['@global'] = true const configLoader = proxyquire('./js-yaml-loader', { 'fs': fsMock, './file-utils': fileMock, [path.resolve('/foobar', 'deepstream.io-cache-super-cache')]: SuperCache, [path.resolve('/foobar', 'deepstream.io-storage-super-storage')]: SuperStorage, [path.resolve('/foobar', 'deepstream.io-connection-http')]: HTTPMock }) services = configLoader.loadConfig(null, { config: './config.json', libDir: '/foobar' }).services }) it('load plugins', () => { expect(services.cache.options).to.deep.equal({ foo: -1, bar: -2 }) expect(services.storage.options).to.deep.equal({ foo: -3, bar: -4 }) }) }) }) ================================================ FILE: src/config/js-yaml-loader.ts ================================================ import * as fs from 'fs' import * as yaml from 'js-yaml' import * as path from 'path' import { get as getDefaultOptions } from '../default-options' import { merge } from '../utils/utils' import { DeepstreamConfig, LOG_LEVEL, EVENT } from '@deepstream/types' import Deepstream from '../deepstream.io' import * as configInitializer from './config-initialiser' import * as fileUtils from './file-utils' export type InitialLogs = Array<{ level: LOG_LEVEL message: string event: any, meta: any }> const SUPPORTED_EXTENSIONS = ['.yml', '.yaml', '.json', '.js'] const DEFAULT_CONFIG_DIRS = [ '/etc/deepstream/conf', path.join('.', 'conf', 'config'), path.join('..', 'conf', 'config') ] DEFAULT_CONFIG_DIRS.push(path.join(process.argv[1], '..', 'conf', 'config')) DEFAULT_CONFIG_DIRS.push(path.join(process.argv[1], '..', '..', 'conf', 'config')) /** * Reads and parse a general configuration file content. */ export const readAndParseFile = function (filePath: string, callback: Function): void { try { fs.readFile(filePath, 'utf8', (error, fileContent) => { if (error) { return callback(error) } try { const config = parseFile(filePath, fileContent) return callback(null, config) } catch (parseError) { return callback(parseError) } }) } catch (error) { callback(error) } } /** * Loads a config file without having to initialize it. Useful for one * off operations such as generating a hash via cli */ export const loadConfigWithoutInitialization = async function (filePath: string | null = null, initialLogs: InitialLogs = [], args?: object): Promise<{ config: DeepstreamConfig, configPath: string }> { // @ts-ignore const argv = args || global.deepstreamCLI || {} const configPath = setGlobalConfigDirectory(argv, filePath) let configString = fs.readFileSync(configPath, { encoding: 'utf8' }) configString = configString.replace(/(^#)*#.*$/gm, '$1') configString = configString.replace(/^\s*\n/gm, '') configString = lookupConfigPaths(configString) configString = await loadFiles(configString, initialLogs) const rawConfig = parseFile(configPath, configString) const config = extendConfig(rawConfig, argv) setGlobalLibDirectory(argv, config) return { config, configPath, } } /** * Loads a file as deepstream config. CLI args have highest priority after the * configuration file. If some properties are not set they will be defaulted * to default values defined in the defaultOptions.js file. * Configuraiton file will be transformed to a deepstream object by evaluating * some properties like the plugins (logger and connectors). */ export const loadConfig = async function (deepstream: Deepstream, filePath: string | null, args?: object) { const logs: InitialLogs = [] const config = await loadConfigWithoutInitialization(filePath, logs, args) const result = configInitializer.initialize(deepstream, config.config, logs) return { config: result.config, services: result.services, file: config.configPath, } } /** * Parse a general configuration file * These file extension ans formats are allowed: * .yml, .js, .json * * If no fileContent is passed the file is read synchronously */ function parseFile (filePath: string, fileContent: string): ConfigType { const extension = path.extname(filePath) if (extension === '.yml' || extension === '.yaml') { return yaml.load(replaceEnvironmentVariables(fileContent)) as unknown as ConfigType } else if (extension === '.js') { return require(path.resolve(filePath)) } else if (extension === '.json') { return JSON.parse(replaceEnvironmentVariables(fileContent)) } else { throw new Error(`${extension} is not supported as configuration file`) } } /** * Set the globalConfig prefix that will be used as the directory for ssl, permissions and auth * relative files within the config file */ function setGlobalConfigDirectory (argv: any, filePath?: string | null): string { const customConfigPath = argv.c || argv.config || filePath || process.env.DEEPSTREAM_CONFIG_DIRECTORY const configPath = customConfigPath ? verifyCustomConfigPath(customConfigPath) : getDefaultConfigPath() // @ts-ignore global.deepstreamConfDir = path.dirname(configPath) return configPath } /** * Set the globalLib prefix that will be used as the directory for the logger * and plugins within the config file */ function setGlobalLibDirectory (argv: any, config: DeepstreamConfig): void { // @ts-ignore const libDir = argv.l || argv.libDir || (config.libDir && fileUtils.lookupConfRequirePath(config.libDir)) || process.env.DEEPSTREAM_LIBRARY_DIRECTORY // @ts-ignore global.deepstreamLibDir = libDir } /** * Augments the basic configuration with command line parameters * and normalizes paths within it */ function extendConfig (config: any, argv: any): DeepstreamConfig { const cliArgs = {} let key for (key in getDefaultOptions()) { (cliArgs as any)[key] = argv[key] } return merge({ plugins: {} }, getDefaultOptions(), config, cliArgs) as DeepstreamConfig } /** * Checks if a config file is present at a given path */ function verifyCustomConfigPath (configPath: string): string { if (fileUtils.fileExistsSync(configPath)) { return configPath } throw new Error(`Configuration file not found at: ${configPath}`) } /** * Fallback if no config path is specified. Will attempt to load the file from the default directory */ function getDefaultConfigPath (): string { let filePath let i let k for (k = 0; k < DEFAULT_CONFIG_DIRS.length; k++) { for (i = 0; i < SUPPORTED_EXTENSIONS.length; i++) { filePath = DEFAULT_CONFIG_DIRS[k] + SUPPORTED_EXTENSIONS[i] if (fileUtils.fileExistsSync(filePath)) { return filePath } } } throw new Error('No config file found') } /** * Handle the introduction of global environment variables within * the yml file, allowing value substitution. * * For example: * ``` * host: $HOST_NAME * port: $HOST_PORT * ``` */ function replaceEnvironmentVariables (fileContent: string): string { const environmentVariable = new RegExp(/\${([^}]+)}/g) return fileContent.replace(environmentVariable, (a, b) => process.env[b] || '') } function lookupConfigPaths (fileContent: string): string { const matches = fileContent.match(/file\((.*)\)/g) if (matches) { matches.forEach((match) => { const [, filename] = match.match(/file\((.*)\)/) as any fileContent = fileContent.replace(match, fileUtils.lookupConfRequirePath(filename)) }) } return fileContent } async function loadFiles (fileContent: string, initialLogs: InitialLogs): Promise { const matches = fileContent.match(/fileLoad\((.*)\)/g) if (matches) { const promises = matches.map(async (match) => { const [, filename] = match.match(/fileLoad\((.*)\)/) as any try { let content: string = await new Promise((resolve, reject) => fs.readFile(fileUtils.lookupConfRequirePath(filename), { encoding: 'utf8' }, (err, data) => { err ? reject(err) : resolve(data) }) ) content = replaceEnvironmentVariables(content) try { if (['.yml', '.yaml', '.js', '.json'].includes(path.extname(filename))) { content = parseFile(filename, content) } initialLogs.push({ level: LOG_LEVEL.INFO, message: `Loaded content from ${fileUtils.lookupConfRequirePath(filename)} for ${match}`, event: EVENT.CONFIG_TRANSFORM, meta: undefined }) } catch (e) { initialLogs.push({ level: LOG_LEVEL.FATAL, event: EVENT.CONFIG_ERROR, message: `Error loading config file, invalid format in file ${fileUtils.lookupConfRequirePath(filename)} for ${match}`, meta: undefined }) } fileContent = fileContent.replace(match, JSON.stringify(content)) } catch (e) { initialLogs.push({ level: LOG_LEVEL.FATAL, event: EVENT.CONFIG_ERROR, message: `Error loading config file, missing file ${fileUtils.lookupConfRequirePath(filename)} for ${match}`, meta: undefined }) } }) await Promise.all(promises) } return fileContent } ================================================ FILE: src/connection-endpoint/base/connection-endpoint.spec.ts ================================================ // import * as C from '../../src/constants' // const proxyquire = require('proxyquire').noPreserveCache() // import uwsMock from '../test-mocks/uws-mock' // import HttpMock from '../test-mocks/http-mock' // import LoggerMock from '../test-mocks/logger-mock' // import DependencyInitialiser from '../../src/utils/dependency-initialiser' // import PermissionHandlerMock from '../test-mocks/permission-handler-mock' // import AuthenticationHandlerMock from '../test-mocks/authentication-handler-mock' // import SocketMock from '../test-mocks/socket-mock' // // import { getTestMocks } from '../test-helper/test-mocks' // // const httpMock = new HttpMock() // const httpsMock = new HttpMock() // // since proxyquire.callThru is enabled, manually capture members from prototypes // httpMock.createServer = httpMock.createServer // httpsMock.createServer = httpsMock.createServer // // let client // let handshakeData // // const ConnectionEndpoint = proxyquire('../../src/message/uws/connection-endpoint', { // 'uws': uwsMock, // 'http': httpMock, // 'https': httpsMock, // './socket-wrapper-factory': { // createSocketWrapper: (options, data) => { // handshakeData = data // client = getTestMocks().getSocketWrapper('client') // return client.socketWrapper // } // } // }).default // // let lastAuthenticatedMessage = null // let connectionEndpoint // // let authenticationHandlerMock // let config // let services // // describe.skip('connection endpoint', () => { // beforeEach(done => { // authenticationHandlerMock = new AuthenticationHandlerMock() // // config = { // unauthenticatedClientTimeout: null, // maxAuthAttempts: 3, // logInvalidAuthData: true, // heartbeatInterval: 4000 // } // // services = { // authenticationHandler: authenticationHandlerMock, // logger: new LoggerMock(), // permission: new PermissionHandlerMock() // } // // connectionEndpoint = new ConnectionEndpoint(config, services) // const depInit = new DependencyInitialiser({ config, services }, config, services, connectionEndpoint, 'connectionEndpoint') // depInit.on('ready', () => { // connectionEndpoint.unauthenticatedClientTimeout = 100 // connectionEndpoint.onMessages() // connectionEndpoint.onMessages = function (socket, parsedMessages) { // lastAuthenticatedMessage = parsedMessages[parsedMessages.length - 1] // } // connectionEndpoint.server._simulateUpgrade(new SocketMock()) // expect(uwsMock.lastUserData).not.to.equal(null) // done() // }) // }) // // afterEach(done => { // connectionEndpoint.once('close', done) // connectionEndpoint.close() // client.socketWrapperMock.verify() // }) // // it.skip('sets autopings on the websocket server', () => { // expect(uwsMock.heartbeatInterval).to.equal(config.heartbeatInterval) // expect(uwsMock.pingMessage).to.equal({ // topic: C.TOPIC.CONNECTION, // action: CONNECTION_ACTION.PING // }) // }) // // describe('the connection endpoint handles invalid connection messages', () => { // it('handles invalid connection topic', () => { // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.CONNECTION, // action: CONNECTION_ACTION.INVALID_MESSAGE, // originalTopic: C.TOPIC.AUTH, // originalAction: AUTH_ACTION.AUTH_UNSUCCESSFUL, // data: 'gibbeerish' // }) // // client.socketWrapperMock // .expects('destroy') // .never() // const message: C.Message = { // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.AUTH_UNSUCCESSFUL, // raw: 'gibbeerish' // } // uwsMock.messageHandler([message], client.socketWrapper) // }) // }) // // it('the connection endpoint handles parser errors', () => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.PARSER, // action: C.PARSER_ACTION.UNKNOWN_ACTION, // data: Buffer.from('gibbeerish'), // originalTopic: 5, // originalAction: 177 // }) // // client.socketWrapperMock // .expects('destroy') // .withExactArgs() // // const message: C.ParseError = { // parseError: true, // action: C.PARSER_ACTION.UNKNOWN_ACTION, // parsedMessage: { // topic: 5, // action: 177 // }, // description: 'unknown RECORD action 177', // raw: Buffer.from('gibbeerish') // } // uwsMock.messageHandler([message], client.socketWrapper) // }) // // it('the connection endpoint handles invalid auth messages', () => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.INVALID_MESSAGE, // originalTopic: C.TOPIC.EVENT, // originalAction: C.EVENT_ACTION.EMIT, // data: 'gibbeerish' // }) // // client.socketWrapperMock // .expects('destroy') // .never() // // const message: C.Message = { // topic: C.TOPIC.EVENT, // action: C.EVENT_ACTION.EMIT, // raw: 'gibbeerish' // } // uwsMock.messageHandler([message], client.socketWrapper) // }) // // it('the connection endpoint handles auth null data', () => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.INVALID_MESSAGE_DATA, // originalAction: C.RPC_ACTION.REQUEST, // }) // // client.socketWrapperMock // .expects('destroy') // .once() // .withExactArgs() // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: 'null' }], client.socketWrapper) // }) // // it('the connection endpoint handles invalid auth json', () => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.INVALID_MESSAGE_DATA, // originalAction: C.RPC_ACTION.REQUEST, // }) // // client.socketWrapperMock // .expects('destroy') // .once() // .withExactArgs() // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{ invalid }' }], client.socketWrapper) // }) // // it('the connection endpoint does not route invalid auth messages to the permission', () => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // parsedData: 'Invalid User', // action: AUTH_ACTION.AUTH_UNSUCCESSFUL, // }) // // expect(authenticationHandlerMock.lastUserValidationQueryArgs).to.equal(null) // authenticationHandlerMock.nextUserValidationResult = false // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"wolfram"}' }], client.socketWrapper) // // expect(authenticationHandlerMock.lastUserValidationQueryArgs.length).to.equal(3) // expect(authenticationHandlerMock.lastUserValidationQueryArgs[1].user).to.equal('wolfram') // expect(services.logger.lastLogMessage.indexOf('wolfram')).not.to.equal(-1) // }) // // describe('the connection endpoint emits a client events for user with name', () => { // beforeEach(() => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // }) // // it('client has the correct connection data', () => { // expect(handshakeData.remoteAddress).to.equal('127.0.0.1') // expect(handshakeData.headers).to.not.equal(undefined) // }) // // it('emits connected event for user with name', done => { // connectionEndpoint.once('client-connected', socketWrapper => { // expect(socketWrapper.user).to.equal('test-user') // done() // }) // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // }) // // it('emits disconnected event for user with name', done => { // connectionEndpoint.once('client-disconnected', socketWrapper => { // expect(socketWrapper.user).to.equal('test-user') // done() // }) // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // client.socketWrapper.close() // }) // }) // // describe('the connection endpoint doesn\'t emit client events for user without a name', () => { // beforeEach(() => { // authenticationHandlerMock.nextUserIsAnonymous = true // authenticationHandlerMock.nextUserValidationResult = true // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // }) // // it('does not emit connected event', () => { // const spy = spy() // connectionEndpoint.once('client-connected', spy) // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // // expect(spy).to.have.callCount(0) // }) // // it('does not emit disconnected event', () => { // authenticationHandlerMock.nextUserIsAnonymous = true // const spy = spy() // // connectionEndpoint.once('client-disconnected', spy) // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // client.socketWrapper.close() // // expect(spy).to.have.callCount(0) // }) // }) // // it('disconnects if the number of invalid authentication attempts is exceeded', () => { // authenticationHandlerMock.nextUserValidationResult = false // config.maxAuthAttempts = 3 // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // // client.socketWrapperMock // .expects('sendMessage') // .thrice() // .withExactArgs({ // topic: C.TOPIC.AUTH, // parsedData: 'Invalid User', // action: AUTH_ACTION.AUTH_UNSUCCESSFUL, // }) // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.TOO_MANY_AUTH_ATTEMPTS, // }) // // client.socketWrapperMock // .expects('destroy') // .once() // .withExactArgs() // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // }) // // it('disconnects client if authentication timeout is exceeded', done => { // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.CONNECTION, // action: CONNECTION_ACTION.AUTHENTICATION_TIMEOUT, // }) // // client.socketWrapperMock // .expects('destroy') // .once() // .withExactArgs() // // setTimeout(done, 150) // }) // // it.skip('authenticates valid sockets', () => { // authenticationHandlerMock.nextUserValidationResult = true // // client.socketWrapperMock // .expects('destroy') // .never() // // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.AUTH_SUCCESSFUL, // // parsedData: undefined // }) // // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // }) // // it('notifies the permission when a client disconnects', () => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // // client.socketWrapper.close() // // expect(authenticationHandlerMock.onClientDisconnectCalledWith).to.equal('test-user') // }) // // it('routes valid auth messages to the permission', () => { // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // uwsMock.messageHandler([{ topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, data: 'test' }], client.socketWrapper) // // const result = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, data: 'test' } // expect(lastAuthenticatedMessage).to.deep.equal(result as any) // }) // // it('forwards additional data for positive authentications', () => { // authenticationHandlerMock.nextUserValidationResult = true // authenticationHandlerMock.sendNextValidAuthWithData = true // // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.AUTH_SUCCESSFUL, // parsedData: 'test-data' // }) // // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // }) // // it('connection endpoint doesn\'t log credentials if logInvalidAuthData is set to false', () => { // config.logInvalidAuthData = false // authenticationHandlerMock.nextUserValidationResult = false // // uwsMock.messageHandler([{ topic: C.TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE, data: '' }], client.socketWrapper) // uwsMock.messageHandler([{ topic: C.TOPIC.AUTH, action: C.RPC_ACTION.REQUEST, data: '{"user":"test-user"}' }], client.socketWrapper) // // expect(services.logger.lastLogMessage.indexOf('wolfram')).to.equal(-1) // }) // }) // const proxyquire = require('proxyquire').noPreserveCache() // // import * as uwsMock from '../test-mocks/uws-mock' // import HttpMock from '../test-mocks/http-mock' // import LoggerMock from '../test-mocks/logger-mock' // import PermissionHandlerMock from '../test-mocks/permission-handler-mock' // // const httpMock: any = new HttpMock() // const httpsMock: any = new HttpMock() // // since proxyquire.callThru is enabled, manually capture members from prototypes // httpMock.createServer = httpMock.createServer // httpsMock.createServer = httpsMock.createServer // const ConnectionEndpoint = proxyquire('../../src/message/uws/connection-endpoint', { // uws: uwsMock, // http: httpMock, // https: httpsMock // }).default // // const config = { // maxAuthAttempts: 3, // logInvalidAuthData: true // } // // const services = { // permission: PermissionHandlerMock, // logger: new LoggerMock() // } // // const mockDs = { config, services } // // let connectionEndpoint // // const connectionEndpointInit = (endpointOptions, onReady) => { // connectionEndpoint = new ConnectionEndpoint(endpointOptions) // connectionEndpoint.setDeepstream(mockDs) // connectionEndpoint.init() // connectionEndpoint.on('ready', onReady) // } // // describe('validates HTTPS server conditions', () => { // let error // let sslOptions // connectionEndpoint = null // // before(() => { // spyOn(httpMock, 'createServer').and.callThrough() // spyOn(httpsMock, 'createServer').and.callThrough() // }) // // beforeEach(() => { // sslOptions = { // permission: PermissionHandlerMock, // logger: { log () {} } // } // error = { message: null } // }) // // afterEach(done => { // if (!connectionEndpoint || !connectionEndpoint.isReady) { // done() // } else { // connectionEndpoint.once('close', done) // connectionEndpoint.close() // } // // httpMock.createServer.resetHistory() // httpsMock.createServer.resetHistory() // }) // // it('creates a http connection when sslKey and sslCert are not provided', done => { // connectionEndpointInit(sslOptions, () => { // expect(httpMock.createServer).to.have.been.calledWith() // expect(httpsMock.createServer).to.have.callCount(0) // done() // }) // }) // // it('creates a https connection when sslKey and sslCert are provided', done => { // sslOptions.sslKey = 'sslPrivateKey' // sslOptions.sslCert = 'sslCertificate' // connectionEndpointInit(sslOptions, () => { // expect(httpMock.createServer).to.have.callCount(0) // expect(httpsMock.createServer).to.have.been.calledWith({ key: 'sslPrivateKey', cert: 'sslCertificate', ca: undefined }) // done() // }) // }) // // it('creates a https connection when sslKey, sslCert and sslCa are provided', done => { // sslOptions.sslKey = 'sslPrivateKey' // sslOptions.sslCert = 'sslCertificate' // sslOptions.sslCa = 'sslCertificateAuthority' // connectionEndpointInit(sslOptions, () => { // expect(httpMock.createServer).to.have.callCount(0) // expect(httpsMock.createServer).to.have.been.calledWith({ key: 'sslPrivateKey', cert: 'sslCertificate', ca: 'sslCertificateAuthority' }) // done() // }) // }) // // it('throws an exception when only sslCert is provided', () => { // try { // sslOptions.sslCert = 'sslCertificate' // connectionEndpointInit(sslOptions, () => {}) // } catch (e) { // error = e // } finally { // expect(error.message).to.equal('Must also include sslKey in order to use SSL') // } // }) // // it('throws an exception when only sslKey is provided', () => { // try { // sslOptions.sslKey = 'sslPrivateKey' // connectionEndpointInit(sslOptions, () => {}) // } catch (e) { // error = e // } finally { // expect(error.message).to.equal('Must also include sslCertFile in order to use SSL') // } // }) // // it('throws an exception when sslCert and sslCa is provided', () => { // try { // sslOptions.sslCert = 'sslCertificate' // sslOptions.sslCa = 'sslCertificateAuthority' // connectionEndpointInit(sslOptions, () => {}) // } catch (e) { // error = e // } finally { // expect(error.message).to.equal('Must also include sslKey in order to use SSL') // } // }) // // it('throws an exception when sslKey and sslCa is provided', () => { // try { // sslOptions.sslKey = 'sslPrivateKey' // sslOptions.sslCa = 'sslCertificateAuthority' // connectionEndpointInit(sslOptions, () => {}) // } catch (e) { // error = e // } finally { // expect(error.message).to.equal('Must also include sslCertFile in order to use SSL') // } // }) // }) // import * as C from '../../src/constants' // const proxyquire = require('proxyquire').noPreserveCache() // import HttpMock from '../test-mocks/http-mock' // import LoggerMock from '../test-mocks/logger-mock' // // const httpMock = new HttpMock() // const httpsMock = new HttpMock() // // since proxyquire.callThru is enabled, manually capture members from prototypes // httpMock.createServer = httpMock.createServer // httpsMock.createServer = httpsMock.createServer // // import { getTestMocks } from '../test-helper/test-mocks' // // let client // // const ConnectionEndpoint = proxyquire('../../src/message/uws/connection-endpoint', { // './socket-wrapper-factory': { // createSocketWrapper: () => { // client = getTestMocks().getSocketWrapper('client') // return client.socketWrapper // } // } // }).default // import DependencyInitialiser from '../../src/utils/dependency-initialiser' // import SocketMock from '../test-mocks/socket-mock' // // const permission = { // isValidUser (connectionData, authData, callback) { // callback(true, { // username: 'someUser', // clientData: { firstname: 'Wolfram' }, // serverData: { role: 'admin' } // }) // }, // canPerformAction (username, message, callback) { // callback(null, true) // }, // onClientDisconnect () {} // } // // const config = { // maxAuthAttempts: 3, // logInvalidAuthData: true, // unauthenticatedClientTimeout: 100 // } // // const services = { // permission, // authenticationHandler: permission, // logger: new LoggerMock() // } // // describe('permission passes additional user meta data', () => { // let connectionEndpoint // // beforeEach(done => { // connectionEndpoint = new ConnectionEndpoint(config) // const depInit = new DependencyInitialiser({ config, services }, config as any, services as any, connectionEndpoint, 'connectionEndpoint') // depInit.on('ready', () => { // connectionEndpoint.onMessages = function () {} // connectionEndpoint.server._simulateUpgrade(new SocketMock()) // // uwsMock.messageHandler([{ // topic: C.TOPIC.CONNECTION, // action: CONNECTION_ACTION.CHALLENGE, // data: 'localhost:6021' // }], client.socketWrapper) // // done() // }) // }) // // it('sends an authentication message', () => { // spyOn(permission, 'isValidUser').and.callThrough() // // client.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs({ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.AUTH_SUCCESSFUL, // parsedData: { firstname: 'Wolfram' } // }) // // uwsMock.messageHandler([{ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.REQUEST, // data: '{ "token": 1234 }' // }], client.socketWrapper) // // expect(permission.isValidUser).to.have.callCount(1) // expect((permission.isValidUser as any).calls.mostRecent().args[1]).to.deep.equal({ token: 1234 }) // // client.socketWrapperMock.verify() // }) // // it('sends a record read message', () => { // spyOn(connectionEndpoint, 'onMessages') // // uwsMock.messageHandler([{ // topic: C.TOPIC.AUTH, // action: AUTH_ACTION.REQUEST, // data: '{ "token": 1234 }' // }], client.socketWrapper) // // uwsMock.messageHandler([{ // topic: C.TOPIC.RECORD, // action: C.RECORD_ACTION.READ, // name: 'recordA' // }], client.socketWrapper) // // expect(connectionEndpoint.onMessages).to.have.callCount(1) // expect(connectionEndpoint.onMessages.calls.mostRecent().args[0].authData).to.deep.equal({ role: 'admin' }) // }) // }) ================================================ FILE: src/connection-endpoint/base/connection-endpoint.ts ================================================ import { Message, ParseResult, PARSER_ACTION, TOPIC, CONNECTION_ACTION, ALL_ACTIONS, JSONObject, AUTH_ACTION } from '../../constants' import { DeepstreamPlugin, SocketConnectionEndpoint, SocketWrapper, ConnectionListener, DeepstreamServices, DeepstreamConfig, EVENT, UnauthenticatedSocketWrapper } from '@deepstream/types' const OPEN = 'OPEN' export interface WebSocketServerConfig { outgoingBufferTimeout: number, maxBufferByteSize: number, headers: string[], healthCheckPath: string, urlPath: string, [index: string]: any, } /** * This is the frontmost class of deepstream's message pipeline. It receives * connections and authentication requests, authenticates sockets and * forwards messages it receives from authenticated sockets. */ export default class BaseWebsocketConnectionEndpoint extends DeepstreamPlugin implements SocketConnectionEndpoint { public description: string = 'WebSocket Connection Endpoint' private initialized: boolean = false private flushTimeout: NodeJS.Timeout | null = null private authenticatedSocketWrappers: Set = new Set() private scheduledSocketWrapperWrites: Set = new Set() private logInvalidAuthData: boolean = false private maxAuthAttempts: number = 3 private unauthenticatedClientTimeout: number | boolean = false private connectionListener!: ConnectionListener private clientVersions: { [index: string]: Set } = {} constructor (private options: WebSocketServerConfig, protected services: DeepstreamServices, protected dsOptions: DeepstreamConfig) { super() this.flushSockets = this.flushSockets.bind(this) } public async whenReady (): Promise { await this.services.httpService.whenReady() } public createWebsocketServer () { } public closeWebsocketServer () { } public onSocketWrapperClosed (socketWrapper: UnauthenticatedSocketWrapper) { socketWrapper.close() } public setConnectionListener (connectionListener: ConnectionListener) { this.connectionListener = connectionListener } public getClientVersions () { return this.clientVersions } /** * Called for every message that's received * from an authenticated socket * * This method will be overridden by an external class and is used instead * of an event emitter to improve the performance of the messaging pipeline */ public onMessages (socketWrapper: SocketWrapper, messages: Message[]) { } /** * initialize and setup the http and WebSocket servers. */ public init (): void { if (this.initialized) { throw new Error('init() must only be called once') } this.initialized = true this.maxAuthAttempts = this.options.maxAuthAttempts this.logInvalidAuthData = this.options.logInvalidAuthData this.unauthenticatedClientTimeout = this.options.unauthenticatedClientTimeout this.createWebsocketServer() } /** * Called from a socketWrapper. This method tells the connection endpoint * to flush the socket after a certain amount of time, used to low priority * messages */ public scheduleFlush (socketWrapper: SocketWrapper) { this.scheduledSocketWrapperWrites.add(socketWrapper) if (!this.flushTimeout) { this.flushTimeout = setTimeout(this.flushSockets, this.options.outgoingBufferTimeout) } } /** * Called when the flushTimeout occurs in order to send all pending socket acks */ private flushSockets () { for (const socketWrapper of this.scheduledSocketWrapperWrites) { socketWrapper.flush() } this.scheduledSocketWrapperWrites.clear() this.flushTimeout = null } protected getOption (option: string) { return this.options[option] } public handleParseErrors (socketWrapper: SocketWrapper, parseResults: ParseResult[]): Message[] { const messages: Message[] = [] for (const parseResult of parseResults) { if (parseResult.parseError) { this.services.logger!.warn(PARSER_ACTION[PARSER_ACTION.MESSAGE_PARSE_ERROR], 'error parsing connection message') socketWrapper.sendMessage({ topic: TOPIC.PARSER, action: parseResult.action, data: parseResult.raw, originalTopic: parseResult.parsedMessage.topic, originalAction: parseResult.parsedMessage.action }, false) socketWrapper.destroy() continue } const message = parseResult as Message if ( message.topic === TOPIC.CONNECTION && message.action === CONNECTION_ACTION.PONG ) { continue } messages.push(message) } return messages } /** * Receives a connected socket, wraps it in a SocketWrapper, sends a connection ack to the user * and subscribes to authentication messages. */ public onConnection (socketWrapper: UnauthenticatedSocketWrapper) { const handshakeData = socketWrapper.getHandshakeData() this.services.logger!.info( EVENT.INCOMING_CONNECTION, `from ${handshakeData.referer} (${handshakeData.remoteAddress})` ) let disconnectTimer if (this.unauthenticatedClientTimeout !== null && this.unauthenticatedClientTimeout !== false) { const timeout = this.unauthenticatedClientTimeout as any disconnectTimer = setTimeout(this.processConnectionTimeout.bind(this, socketWrapper), timeout) socketWrapper.onClose(clearTimeout.bind(null, disconnectTimer)) } socketWrapper.authCallback = this.authenticateConnection.bind( this, socketWrapper, disconnectTimer ) socketWrapper.onMessage = this.processConnectionMessage.bind(this, socketWrapper) } /** * Always challenges the client that connects. This will be opened up later to allow users * to put in their own challenge authentication. */ public processConnectionMessage (socketWrapper: UnauthenticatedSocketWrapper, parsedMessages: Message[]) { const msg = parsedMessages[0] if (msg.topic !== TOPIC.CONNECTION) { this.services.logger!.warn(CONNECTION_ACTION[CONNECTION_ACTION.INVALID_MESSAGE], 'invalid connection message') socketWrapper.sendMessage({ topic: TOPIC.CONNECTION, action: CONNECTION_ACTION.INVALID_MESSAGE, originalTopic: msg.topic, originalAction: msg.action }, false) return } if (msg.action === CONNECTION_ACTION.PING) { return } if (msg.action === CONNECTION_ACTION.CHALLENGE) { if (msg.sdkType && msg.sdkVersion) { if (!this.clientVersions[msg.sdkType]) { this.clientVersions[msg.sdkType] = new Set() } this.clientVersions[msg.sdkType].add(msg.sdkVersion) } socketWrapper.onMessage = socketWrapper.authCallback! socketWrapper.sendMessage({ topic: TOPIC.CONNECTION, action: CONNECTION_ACTION.ACCEPT }, false) return } this.services.logger!.error(PARSER_ACTION[PARSER_ACTION.UNKNOWN_ACTION], '', { message: msg }) } /** * Callback for the first message that's received from the socket. * This is expected to be an auth-message. This method makes sure that's * the case and - if so - forwards it to the permission handler for authentication */ private authenticateConnection (socketWrapper: UnauthenticatedSocketWrapper, disconnectTimeout: NodeJS.Timeout | undefined, parsedMessages: Message[]): void { const msg = parsedMessages[0] let errorMsg if (msg.topic === TOPIC.CONNECTION && msg.action === CONNECTION_ACTION.PING) { return } if (msg.topic !== TOPIC.AUTH) { this.services.logger!.warn(AUTH_ACTION[AUTH_ACTION.INVALID_MESSAGE], `invalid auth message: ${JSON.stringify(msg)}`) socketWrapper.sendMessage({ topic: TOPIC.AUTH, action: AUTH_ACTION.INVALID_MESSAGE, originalTopic: msg.topic, originalAction: msg.action }, false) return } /** * Log the authentication attempt */ const logMsg = socketWrapper.getHandshakeData().remoteAddress this.services.logger!.debug(AUTH_ACTION[AUTH_ACTION.REQUEST], logMsg) /** * Ensure the message is a valid authentication message */ if (msg.action !== AUTH_ACTION.REQUEST) { errorMsg = this.logInvalidAuthData === true ? JSON.stringify(msg.parsedData) : '' this.sendInvalidAuthMsg(socketWrapper, errorMsg, msg.action) return } /** * Ensure the authentication data is valid JSON */ const result = socketWrapper.parseData(msg) if (result instanceof Error || !msg.parsedData || typeof msg.parsedData !== 'object') { errorMsg = 'Error parsing auth message' if (this.logInvalidAuthData === true) { errorMsg += ` "${msg.data}": ${result.toString()}` } this.sendInvalidAuthMsg(socketWrapper, errorMsg, msg.action) return } /** * Forward for authentication */ this.services.authentication.isValidUser( socketWrapper.getHandshakeData(), msg.parsedData, this.processAuthResult.bind(this, msg.parsedData, socketWrapper, disconnectTimeout) ) } /** * Will be called for syntactically incorrect auth messages. Logs * the message, sends an error to the client and closes the socket */ private sendInvalidAuthMsg (socketWrapper: UnauthenticatedSocketWrapper, msg: string, originalAction: ALL_ACTIONS): void { this.services.logger!.warn(AUTH_ACTION[AUTH_ACTION.INVALID_MESSAGE_DATA], this.logInvalidAuthData ? msg : '') socketWrapper.sendMessage({ topic: TOPIC.AUTH, action: AUTH_ACTION.INVALID_MESSAGE_DATA, originalAction }, false) socketWrapper.destroy() } /** * Callback for succesfully validated sockets. Removes * all authentication specific logic and registeres the * socket with the authenticated sockets */ private registerAuthenticatedSocket (unauthenticatedSocketWrapper: UnauthenticatedSocketWrapper, userData: any): void { const socketWrapper = this.appendDataToSocketWrapper(unauthenticatedSocketWrapper, userData) unauthenticatedSocketWrapper.authCallback = null unauthenticatedSocketWrapper.onMessage = (parsedMessages: Message[]) => { this.onMessages(socketWrapper, parsedMessages) } this.authenticatedSocketWrappers.add(socketWrapper) socketWrapper.sendMessage({ topic: TOPIC.AUTH, action: AUTH_ACTION.AUTH_SUCCESSFUL, parsedData: userData.clientData }) this.connectionListener.onClientConnected(socketWrapper) this.services.logger!.info(AUTH_ACTION[AUTH_ACTION.AUTH_SUCCESSFUL], socketWrapper.userId!) } /** * Append connection data to the socket wrapper */ private appendDataToSocketWrapper (socketWrapper: UnauthenticatedSocketWrapper, userData: any): SocketWrapper { const authenticatedSocketWrapper = socketWrapper as SocketWrapper authenticatedSocketWrapper.userId = userData.id || OPEN authenticatedSocketWrapper.serverData = userData.serverData || null authenticatedSocketWrapper.clientData = userData.clientData || null return authenticatedSocketWrapper } /** * Callback for invalid credentials. Will notify the client * of the invalid auth attempt. If the number of invalid attempts * exceed the threshold specified in options.maxAuthAttempts * the client will be notified and the socket destroyed. */ private processInvalidAuth (clientData: JSONObject, authData: JSONObject, socketWrapper: UnauthenticatedSocketWrapper): void { let logMsg = 'invalid authentication data' if (this.logInvalidAuthData === true) { logMsg += `: ${JSON.stringify(authData)}` } this.services.logger!.info(AUTH_ACTION[AUTH_ACTION.AUTH_UNSUCCESSFUL], logMsg) socketWrapper.sendMessage({ topic: TOPIC.AUTH, action: AUTH_ACTION.AUTH_UNSUCCESSFUL, parsedData: clientData }, false) socketWrapper.authAttempts++ if (socketWrapper.authAttempts >= this.maxAuthAttempts) { this.services.logger!.info(AUTH_ACTION[AUTH_ACTION.TOO_MANY_AUTH_ATTEMPTS], 'too many authentication attempts') socketWrapper.sendMessage({ topic: TOPIC.AUTH, action: AUTH_ACTION.TOO_MANY_AUTH_ATTEMPTS }, false) setTimeout(() => socketWrapper.destroy(), 10) } } /** * Callback for connections that have not authenticated succesfully within * the expected timeframe */ private processConnectionTimeout (socketWrapper: UnauthenticatedSocketWrapper): void { const log = 'connection has not authenticated successfully in the expected time' this.services.logger!.info(CONNECTION_ACTION[CONNECTION_ACTION.AUTHENTICATION_TIMEOUT], log) socketWrapper.sendMessage({ topic: TOPIC.CONNECTION, action: CONNECTION_ACTION.AUTHENTICATION_TIMEOUT }, false) socketWrapper.destroy() } /** * Callback for the results returned by the permission service */ private processAuthResult (authData: any, socketWrapper: UnauthenticatedSocketWrapper, disconnectTimeout: NodeJS.Timeout | undefined, isAllowed: boolean, userData: any): void { this.services.monitoring.onLogin(isAllowed, 'websocket') userData = userData || {} if (disconnectTimeout) { clearTimeout(disconnectTimeout) } if (isAllowed === true) { this.registerAuthenticatedSocket(socketWrapper, userData) } else { this.processInvalidAuth(userData.clientData, authData, socketWrapper) } } /** * Notifies the (optional) onClientDisconnect method of the permission * that the specified client has disconnected */ public onSocketClose (socketWrapper: UnauthenticatedSocketWrapper): void { this.scheduledSocketWrapperWrites.delete(socketWrapper) this.onSocketWrapperClosed(socketWrapper) if (this.authenticatedSocketWrappers.delete(socketWrapper as SocketWrapper)) { const authenticatedSocketWrapper = socketWrapper as SocketWrapper if (this.services.authentication.onClientDisconnect) { this.services.authentication.onClientDisconnect(authenticatedSocketWrapper.userId) } this.connectionListener.onClientDisconnected(authenticatedSocketWrapper) } } /** * Closes the ws server connection. The ConnectionEndpoint * will emit a close event once succesfully shut down */ public async close () { await this.closeWebsocketServer() } } ================================================ FILE: src/connection-endpoint/base/socket-wrapper.ts ================================================ import { TOPIC, CONNECTION_ACTION, ParseResult, Message } from '../../constants' import { WebSocketServerConfig } from './connection-endpoint' import { SocketConnectionEndpoint, StatefulSocketWrapper, DeepstreamServices, UnauthenticatedSocketWrapper, SocketWrapper, EVENT } from '@deepstream/types' export abstract class WSSocketWrapper implements UnauthenticatedSocketWrapper { public abstract socketType: string public isRemote: false = false public isClosed: boolean = false public uuid: number = Math.random() public authCallback: Function | null = null public authAttempts: number = 0 public lastMessageRecievedAt: number = 0 private bufferedWrites: SerializedType[] = [] private closeCallbacks: Set = new Set() public userId: string | null = null public serverData: object | null = null public clientData: object | null = null private bufferedWritesTotalByteSize: number = 0 constructor ( private socket: any, private handshakeData: any, private services: DeepstreamServices, private config: WebSocketServerConfig, private connectionEndpoint: SocketConnectionEndpoint, private isBinary: boolean ) { } get isOpen () { return this.isClosed !== true } protected invalidTypeReceived () { this.services.logger.error(EVENT.ERROR, `Received an invalid message type on ${this.uuid}`) this.destroy() } /** * Called by the connection endpoint to flush all buffered writes. * A buffered write is a write that is not a high priority, such as an ack * and can wait to be bundled into another message if necessary */ public flush () { if (this.bufferedWritesTotalByteSize !== 0) { this.bufferedWrites.forEach((bw) => this.writeMessage(this.socket, bw)) this.bufferedWritesTotalByteSize = 0 this.bufferedWrites = [] } } /** * Sends a message based on the provided action and topic */ public sendMessage (message: { topic: TOPIC, action: CONNECTION_ACTION } | Message, allowBuffering: boolean = true): void { this.services.monitoring.onMessageSend(message) this.sendBuiltMessage(this.getMessage(message), allowBuffering) } /** * Sends a message based on the provided action and topic */ public sendAckMessage (message: Message, allowBuffering: boolean = true): void { this.services.monitoring.onMessageSend(message) this.sendBuiltMessage(this.getAckMessage(message), allowBuffering) } public abstract getMessage (message: Message): SerializedType public abstract getAckMessage (message: Message): SerializedType public abstract parseMessage (message: SerializedType): ParseResult[] public abstract parseData (message: Message): true | Error public onMessage (messages: Message[]): void { } /** * Destroys the socket. Removes all deepstream specific * logic and closes the connection */ public destroy (): void { try { this.socket.close() } catch (e) { this.socket.end() } } public close (): void { this.isClosed = true this.authCallback = null this.closeCallbacks.forEach((cb) => cb(this)) this.services.logger.info(EVENT.CLIENT_DISCONNECTED, this.userId!) } /** * Returns a map of parameters that were collected * during the initial http request that established the * connection */ public getHandshakeData (): any { return this.handshakeData } public onClose (callback: (socketWrapper: StatefulSocketWrapper) => void): void { this.closeCallbacks.add(callback) } public removeOnClose (callback: (socketWrapper: StatefulSocketWrapper) => void): void { this.closeCallbacks.delete(callback) } public sendBuiltMessage (message: SerializedType, buffer?: boolean): void { if (this.isOpen) { if (this.config.outgoingBufferTimeout === 0) { this.writeMessage(this.socket, message) } else if (!buffer) { this.flush() this.writeMessage(this.socket, message) } else { this.bufferedWritesTotalByteSize += message.length this.bufferedWrites.push(message) if (this.bufferedWritesTotalByteSize > this.config.maxBufferByteSize) { this.flush() } else { this.connectionEndpoint.scheduleFlush(this as SocketWrapper) } } } } protected writeMessage (socket: any, message: SerializedType) { this.services.httpService.sendWebsocketMessage(socket, message, this.isBinary) } } ================================================ FILE: src/connection-endpoint/http/connection-endpoint.spec.ts ================================================ import { expect } from 'chai' import * as needle from 'needle' import LoggerMock from '../../test/mock/logger-mock' import { DeepstreamServices, DeepstreamConfig } from '@deepstream/types'; import { OpenAuthentication } from '../../services/authentication/open/open-authentication'; import { OpenPermission } from '../../services/permission/open/open-permission'; import { NodeHTTP } from '../../services/http/node/node-http' import { HTTPConnectionEndpoint } from './connection-endpoint'; const conf = { authPath: '/api/v1/auth', postPath: '/api/v1', getPath: '/api/v1', enableAuthEndpoint: true, requestTimeout: 30, allowAuthData: true } const services: any = { logger: new LoggerMock(), authentication: new OpenAuthentication(), permission: new OpenPermission(), messageDistributor: { distribute () {} } } services.httpService = new NodeHTTP({ port: 9898, host: '127.0.0.1', allowAllOrigins: true, healthCheckPath: '/health-check', maxMessageSize: 100000, hostUrl: '', headers: [] }, services as DeepstreamServices, {} as DeepstreamConfig) describe.skip('http plugin', () => { let httpConnectionEndpoint const postUrl = 'http://127.0.0.1:9898/api/v1/' before(async () => { httpConnectionEndpoint = new HTTPConnectionEndpoint(conf, services as never as DeepstreamServices, {} as never as DeepstreamConfig) httpConnectionEndpoint.init() await httpConnectionEndpoint.whenReady() }) after(async () => { await services.httpService.close() }) const message = Object.freeze({ token: 'fiwueeb-3942jjh3jh23i4h23i4h2', body: [ { topic: 'record', action: 'write', recordName: 'car/bmw', data: { tyres : 2 } }, { topic: 'record', action: 'write', recordName: 'car/bmw', path: 'tyres', data: 3 }, { topic: 'rpc', action: 'make', rpcName: 'add-two', data: { numA: 6, numB: 3 } }, { topic: 'event', action: 'emit', eventName: 'time', data: 1494343585338 } ] }) describe('POST endpoint', () => { it('should reject a request with an empty path', (done) => { needle.post('127.0.0.1:9898', message, { json: true }, (err, response) => { expect(err).to.equal(null) expect(response.statusCode).to.be.within(400, 499) expect(response.headers['content-type']).to.match(/^text\/plain/) expect(response.body).to.match(/not found/i) done() }) }) it('should reject a request with a url-encoded payload', (done) => { needle.post(postUrl, message, { json: false }, (err, response) => { expect(err).to.equal(null) expect(response.statusCode).to.be.within(400, 499) expect(response.headers['content-type']).to.match(/^text\/plain/) expect(response.body).to.match(/media type/i) done() }) }) it('should reject a request with a non-object payload', () => Promise.all([ '123', ['a', '2', '3.5'], 'foo', null, '' ].map((payload) => needle('post', postUrl, payload, { json: true }) .then((response) => { expect(response.statusCode).to.be.within(400, 499) expect(response.headers['content-type']).to.match(/^text\/plain/) expect(response.body).to.match(/(fail|invalid)/i, JSON.stringify(payload)) }) ))) it('should accept a request without an auth token', (done) => { const noToken = Object.assign({}, message, { token: undefined }) needle.post(postUrl, noToken, { json: true }, (err, response) => { expect(err).to.equal(null) expect(response.statusCode).to.equal(200) done() }) }) it('should return an unsuccessful result for an empty list of messages', (done) => { needle.post(postUrl, { token: 'foo', body: [] }, { json: true }, (err, response) => { expect(err).to.equal(null) expect(response.statusCode).to.be.within(400, 499) expect(response.body).to.match(/body.*must be a non-empty array/) done() }) }) it('should not return an error for a list of valid messages', (done) => { needle.post(postUrl, message, { json: true }, (err) => { expect(err).to.equal(null) done() }) }) it('should reject a request that has a mix of valid and invalid messages', (done) => { const someValid = message.body.slice(0, 2) const req = { token: 'foo', body: someValid.concat([{ pas: 'valide' } as any]) } needle.post(postUrl, req, { json: true }, (err, response) => { expect(err).to.equal(null) expect(response.statusCode).to.be.within(400, 499) expect(response.body).to.match(/failed to parse .* index 2/i) done() }) }) describe.skip('authentication', () => { it('should reject a request that times out', async () => { const response = await needle('post', postUrl, message, { json: true }) const resp = response.body expect(resp.result).to.equal('FAILURE') expect(resp.body[0].success).to.equal(false) expect(resp.body[0].errorTopic).to.equal('connection') expect(resp.body[0].errorEvent).to.equal('TIME') }) }) }) }) ================================================ FILE: src/connection-endpoint/http/connection-endpoint.ts ================================================ import JIFHandler from '../../jif/jif-handler' import HTTPSocketWrapper from './socket-wrapper' import * as HTTPStatus from 'http-status' import { PARSER_ACTION, AUTH_ACTION, EVENT_ACTION, RECORD_ACTION, Message, ALL_ACTIONS, JSONObject } from '../../constants' import { DeepstreamConnectionEndpoint, DeepstreamServices, SimpleSocketWrapper, SocketWrapper, JifResult, UnauthenticatedSocketWrapper, DeepstreamPlugin, DeepstreamConfig, EVENT, DeepstreamHTTPResponse, DeepstreamHTTPMeta, DeepstreamAuthenticationResult } from '@deepstream/types' export interface HTTPEvents { onAuthMessage: Function onPostMessage: Function onGetMessage: Function } interface HTTPConnectionEndpointOptionsInterface { enableAuthEndpoint: boolean, authPath: string, postPath: string, getPath: string, allowAuthData: boolean, logInvalidAuthData: boolean, requestTimeout: number } function checkConfigOption (config: any, option: string, expectedType?: string): void { if ((expectedType && typeof config[option] !== expectedType) || config[option] === undefined) { throw new Error(`The HTTP plugin requires that the "${option}" config option is set`) } } export class HTTPConnectionEndpoint extends DeepstreamPlugin implements DeepstreamConnectionEndpoint { public description: string = 'HTTP connection endpoint' private initialized: boolean = false private jifHandler!: JIFHandler private onSocketMessageBound: Function private onSocketErrorBound: Function private logInvalidAuthData: boolean = false private requestTimeout!: number constructor (private pluginOptions: HTTPConnectionEndpointOptionsInterface, private services: DeepstreamServices, public dsOptions: DeepstreamConfig) { super() checkConfigOption(pluginOptions, 'enableAuthEndpoint', 'boolean') checkConfigOption(pluginOptions, 'authPath', 'string') checkConfigOption(pluginOptions, 'postPath', 'string') checkConfigOption(pluginOptions, 'getPath', 'string') this.onSocketMessageBound = this.onSocketMessage.bind(this) this.onSocketErrorBound = this.onSocketError.bind(this) this.onPermissionResponse = this.onPermissionResponse.bind(this) this.jifHandler = new JIFHandler(this.services) } public async whenReady (): Promise { await this.services.httpService.whenReady() } public async close () { } public getClientVersions () { return {} } /** * Initialize the http server. */ public init (): void { if (this.initialized) { throw new Error('init() must only be called once') } this.initialized = true if (this.pluginOptions.enableAuthEndpoint) { this.services.httpService.registerPostPathPrefix(this.pluginOptions.authPath, this.onAuthMessage.bind(this)) } this.services.httpService.registerPostPathPrefix(this.pluginOptions.postPath, this.onPostMessage.bind(this)) this.services.httpService.registerGetPathPrefix(this.pluginOptions.getPath, this.onGetMessage.bind(this)) this.logInvalidAuthData = this.pluginOptions.logInvalidAuthData this.requestTimeout = this.pluginOptions.requestTimeout if (this.requestTimeout === undefined) { this.requestTimeout = 20000 } } /** * Called for every message that's received * from an authenticated socket * * This method will be overridden by an external class and is used instead * of an event emitter to improve the performance of the messaging pipeline */ public onMessages (socketWrapper: SimpleSocketWrapper, messages: Message[]): void { } private onGetMessage (meta: DeepstreamHTTPMeta, responseCallback: any) { const message = 'Reading records via HTTP GET is not yet implemented, please use a post request instead.' this.services.logger.warn(RECORD_ACTION[RECORD_ACTION.READ], message) responseCallback({ statusCode: 400, message }) // TODO: implement a GET endpoint that reads the current state of a record } /** * Handle a message to the authentication endpoint (for token generation). * * Passes the entire message to the configured authentication handler. */ private onAuthMessage (authData: JSONObject, metadata: DeepstreamHTTPMeta, responseCallback: DeepstreamHTTPResponse): void { this.services.authentication.isValidUser( metadata, authData, (isAllowed, data) => { this.services.monitoring.onLogin(isAllowed, 'http') if (isAllowed === true) { responseCallback(null, { token: data!.token, clientData: data!.clientData }) return } let error = typeof data === 'string' ? data : 'Invalid authentication data.' responseCallback({ statusCode: HTTPStatus.UNAUTHORIZED, message: error }) if (this.logInvalidAuthData === true) { error += `: ${JSON.stringify(authData)}` } this.services.logger.debug(AUTH_ACTION[AUTH_ACTION.AUTH_UNSUCCESSFUL], error) } ) } /** * Handle a message to the POST endpoint * * Authenticates the message using authData, a token, or OPEN auth if enabled/provided. */ private onPostMessage ( messageData: { token?: string, authData?: object, body: object[] }, metadata: DeepstreamHTTPMeta, responseCallback: DeepstreamHTTPResponse ): void { if (!Array.isArray(messageData.body) || messageData.body.length < 1) { const message = `Invalid message: the "body" parameter must ${ messageData.body ? 'be a non-empty array of Objects.' : 'exist.' }` responseCallback({ statusCode: HTTPStatus.BAD_REQUEST, message }) this.services.logger.warn( PARSER_ACTION[PARSER_ACTION.INVALID_MESSAGE], JSON.stringify(messageData.body) ) return } let authData = {} if (messageData.authData !== undefined) { if (this.pluginOptions.allowAuthData !== true) { const message = 'Authentication using authData is disabled. Try using a token instead.' responseCallback({ statusCode: HTTPStatus.BAD_REQUEST, message }) this.services.logger.debug( AUTH_ACTION[AUTH_ACTION.INVALID_MESSAGE_DATA], 'Auth rejected because allowAuthData was disabled' ) return } if (messageData.authData === null || typeof messageData.authData !== 'object') { const message = 'Invalid message: the "authData" parameter must be an object' responseCallback({ statusCode: HTTPStatus.BAD_REQUEST, message }) this.services.logger.debug( AUTH_ACTION[AUTH_ACTION.INVALID_MESSAGE_DATA], `authData was not an object: ${ this.logInvalidAuthData === true ? JSON.stringify(messageData.authData) : '-' }` ) return } authData = messageData.authData } else if (messageData.token !== undefined) { if (typeof messageData.token !== 'string' || messageData.token.length === 0) { const message = 'Invalid message: the "token" parameter must be a non-empty string' responseCallback({ statusCode: HTTPStatus.BAD_REQUEST, message }) this.services.logger.debug( AUTH_ACTION[AUTH_ACTION.INVALID_MESSAGE_DATA], `auth token was not a string: ${ this.logInvalidAuthData === true ? messageData.token : '-' }` ) return } authData = { ...authData, token: messageData.token } } this.services.authentication.isValidUser( metadata, authData, this.onMessageAuthResponse.bind(this, responseCallback, messageData) ) } /** * Create and initialize a new SocketWrapper */ private createSocketWrapper ( authResponseData: DeepstreamAuthenticationResult, messageIndex: number, messageResults: any, responseCallback: Function, requestTimeoutId: NodeJS.Timeout ): UnauthenticatedSocketWrapper { const socketWrapper = new HTTPSocketWrapper( this.services, this.onSocketMessageBound, this.onSocketErrorBound ) socketWrapper.init( authResponseData, messageIndex, messageResults, responseCallback, requestTimeoutId ) return socketWrapper } /** * Handle response from authentication handler relating to a POST request. * * Parses, permissions and distributes the individual messages */ private onMessageAuthResponse ( responseCallback: Function, messageData: { body: object[] }, success: boolean, authResponseData?: DeepstreamAuthenticationResult ): void { if (success !== true) { const error = typeof authResponseData === 'string' ? authResponseData : 'Unsuccessful authentication attempt.' responseCallback({ statusCode: HTTPStatus.UNAUTHORIZED, message: error }) return } const messageCount = messageData.body.length const messageResults = new Array(messageCount).fill(null) const parseResults = new Array(messageCount) for (let messageIndex = 0; messageIndex < messageCount; messageIndex++) { const parseResult = this.jifHandler.fromJIF(messageData.body[messageIndex]) parseResults[messageIndex] = parseResult if (!parseResult.success) { const message = `Failed to parse JIF object at index ${messageIndex}.` responseCallback({ statusCode: HTTPStatus.BAD_REQUEST, message: parseResult.error ? `${message} Reason: ${parseResult.error}` : message }) this.services.logger.debug(PARSER_ACTION[PARSER_ACTION.MESSAGE_PARSE_ERROR], parseResult.error) return } } const requestTimeoutId = setTimeout( () => this.onRequestTimeout(responseCallback, messageResults), this.requestTimeout ) // @ts-ignore const dummySocketWrapper = this.createSocketWrapper(authResponseData, null, null, null, null) as SocketWrapper for (let messageIndex = 0; messageIndex < messageCount; messageIndex++) { const parseResult = parseResults[messageIndex] if (parseResult.done) { // Messages such as event emits do not need to wait for a response. However, we need to // check that the message was successfully permissioned, so bypass the message-processor. this.permissionEventEmit( dummySocketWrapper, parseResult.message, messageResults, messageIndex ) // check if a response can be sent immediately if (messageIndex === messageCount - 1) { HTTPConnectionEndpoint.checkComplete(messageResults, responseCallback, requestTimeoutId) } } else { const socketWrapper = this.createSocketWrapper( authResponseData!, messageIndex, messageResults, responseCallback, requestTimeoutId ) /* * TODO: work out a way to safely enable socket wrapper pooling * if (this.socketWrapperPool.length === 0) { * socketWrapper = new HTTPSocketWrapper( * this.onSocketMessageBound, * this.onSocketErrorBound * ) * } else { * socketWrapper = this.socketWrapperPool.pop() * } */ // emit the message this.onMessages(socketWrapper, [parseResult.message]) } } } /** * Handle messages from deepstream socketWrappers and inserts message responses into the HTTP * response where possible. */ private onSocketMessage ( messageResults: JifResult[], index: number, message: Message, responseCallback: Function, requestTimeoutId: NodeJS.Timer ): void { const parseResult = this.jifHandler.toJIF(message) if (!parseResult) { const errorMessage = `${message.topic} ${message.action} ${JSON.stringify(message.data)}` this.services.logger.error(PARSER_ACTION[PARSER_ACTION.MESSAGE_PARSE_ERROR], errorMessage) return } if (parseResult.done !== true) { return } if (messageResults[index] === null) { messageResults[index] = parseResult.message HTTPConnectionEndpoint.checkComplete(messageResults, responseCallback, requestTimeoutId) } } /** * Handle errors from deepstream socketWrappers and inserts message rejections into the HTTP * response where necessary. */ private onSocketError ( messageResults: JifResult[], index: number, message: Message, event: string, errorMessage: string, responseCallback: Function, requestTimeoutId: NodeJS.Timer ): void { const parseResult = this.jifHandler.errorToJIF(message, event) if (parseResult.done && messageResults[index] === null) { messageResults[index] = parseResult.message HTTPConnectionEndpoint.checkComplete(messageResults, responseCallback, requestTimeoutId) } } /** * Check whether any more responses are outstanding and finalize http response if not. */ private static checkComplete (messageResults: JifResult[], responseCallback: Function, requestTimeoutId: NodeJS.Timer): void { const messageResult = HTTPConnectionEndpoint.calculateMessageResult(messageResults) if (messageResult === null) { // insufficient responses received return } clearTimeout(requestTimeoutId) responseCallback(null, { result: messageResult, body: messageResults }) } /** * Handle request timeout, sending any responses that have already resolved. */ private onRequestTimeout (responseCallback: Function, messageResults: JifResult[]): void { let numTimeouts = 0 for (let i = 0; i < messageResults.length; i++) { if (messageResults[i] === null) { messageResults[i] = { success: false, error: 'Request exceeded timeout before a response was received.', errorTopic: 'connection', errorEvent: EVENT.HTTP_REQUEST_TIMEOUT } numTimeouts++ } } if (numTimeouts === 0) { return } this.services.logger.warn(EVENT.HTTP_REQUEST_TIMEOUT, 'HTTP Request timeout') const result = HTTPConnectionEndpoint.calculateMessageResult(messageResults) responseCallback(null, { result, body: messageResults }) } /** * Calculate the 'result' field in a response depending on how many responses resolved * successfully. Can be one of 'SUCCESS', 'FAILURE' or 'PARTIAL SUCCSS' */ private static calculateMessageResult (messageResults: JifResult[]): string | null { let numSucceeded = 0 for (let i = 0; i < messageResults.length; i++) { if (!messageResults[i]) { // todo: when does this happen return null } if (messageResults[i].success) { numSucceeded++ } } if (numSucceeded === messageResults.length) { return 'SUCCESS' } if (numSucceeded === 0) { return 'FAILURE' } return 'PARTIAL_SUCCESS' } /** * Permission an event emit and capture the response directly */ private permissionEventEmit ( socketWrapper: SocketWrapper, parsedMessage: Message, messageResults: JifResult[], messageIndex: number ): void { this.services.permission.canPerformAction( socketWrapper, parsedMessage, this.onPermissionResponse, { messageResults, messageIndex } ) } /** * Handle an event emit permission response */ private onPermissionResponse ( socketWrapper: SocketWrapper, message: Message, { messageResults, messageIndex }: { messageResults: JifResult[], messageIndex: number }, error: string | Error | ALL_ACTIONS | null, permissioned: boolean ): void { if (error !== null) { this.services.logger.warn(EVENT_ACTION[EVENT_ACTION.MESSAGE_PERMISSION_ERROR], error.toString()) } if (permissioned !== true) { messageResults[messageIndex] = { success: false, error: 'Message denied. Action \'emit\' is not permitted.', errorEvent: EVENT_ACTION[EVENT_ACTION.MESSAGE_DENIED], errorAction: 'emit', errorTopic: 'event' } return } messageResults[messageIndex] = { success: true } this.services.messageDistributor.distribute(socketWrapper, message) } } ================================================ FILE: src/connection-endpoint/http/socket-wrapper.ts ================================================ import { parseData } from '@deepstream/protobuf/dist/src/message-parser' import { EventEmitter } from 'events' import { DeepstreamServices, UnauthenticatedSocketWrapper, EVENT, DeepstreamAuthenticationResult } from '@deepstream/types' import { Message, ParseResult } from '../../constants' export default class HTTPSocketWrapper extends EventEmitter implements UnauthenticatedSocketWrapper { public socketType = 'http' public userId: string | null = null public serverData: object | null = null public clientData: object | null = null public uuid: number = Math.random() private correlationIndex: number = -1 private messageResults: any[] = [] private responseCallback: Function | null = null private requestTimeout: NodeJS.Timeout | null = null public authCallback: Function | null = null public isRemote: boolean = false public isClosed: boolean = false // TODO: This isn't used here but is part of a stateful socketWrapper public authAttempts = 0 constructor (private services: DeepstreamServices, private onMessageCallback: Function, private onErrorCallback: Function) { super() } public init ( authResponseData: DeepstreamAuthenticationResult, messageIndex: number, messageResults: any[], responseCallback: Function, requestTimeoutId: NodeJS.Timeout ) { this.userId = authResponseData.id || 'OPEN' this.clientData = authResponseData.clientData || null this.serverData = authResponseData.serverData || null this.correlationIndex = messageIndex this.messageResults = messageResults this.responseCallback = responseCallback this.requestTimeout = requestTimeoutId } public close () { this.isClosed = true } public flush () { } public onMessage () { } public getMessage () { } /** * Returns a map of parameters that were collected * during the initial http request that established the * connection */ public getHandshakeData () { return {} } /** * Sends an error on the specified topic. The * action will automatically be set to C.ACTION.ERROR */ public sendError (message: Message, event: EVENT, errorMessage: string) { if (this.isClosed === false) { parseData(message) this.onErrorCallback( this.messageResults, this.correlationIndex, message, event, errorMessage, this.responseCallback, this.requestTimeout ) } } /** * Sends a message based on the provided action and topic */ public sendMessage (message: Message) { if (message.action >= 100) { message.isError = true } if (this.isClosed === false) { this.services.monitoring.onMessageSend(message) parseData(message) this.onMessageCallback( this.messageResults, this.correlationIndex, message, this.responseCallback, this.requestTimeout ) } } public sendAckMessage (message: Message) { message.isAck = true this.sendMessage(message) } public parseData (message: Message) { return parseData(message) } public parseMessage (serializedMessage: any): ParseResult[] { throw new Error('Method not implemented.') } /** * Destroys the socket. Removes all deepstream specific * logic and closes the connection * * @public * @returns {void} */ public destroy () { } public onClose () { } public removeOnClose () { } } ================================================ FILE: src/connection-endpoint/mqtt/connection-endpoint.ts ================================================ import { createMQTTSocketWrapper} from './socket-wrapper-factory' import { DeepstreamServices, SocketWrapper, DeepstreamConfig, UnauthenticatedSocketWrapper, EVENT } from '@deepstream/types' import ConnectionEndpoint, { WebSocketServerConfig } from '../base/connection-endpoint' import { createServer as createTCPServer, Server as TCPServer } from 'net' import { createServer as createTLSServer, Server as TLSServer } from 'tls' // @ts-ignore import * as mqttCon from 'mqtt-connection' import { TOPIC, CONNECTION_ACTION, AUTH_ACTION } from '../../constants' import { Message } from '@deepstream/client/dist/src/constants' import { EventEmitter } from 'events' export interface MQTTConnectionEndpointConfig extends WebSocketServerConfig { port: number, host: string, idleTimeout: number, ssl?: { key: string, cert: string } } type MQTTPacket = any type MQTTConnection = any /** * This is the frontmost class of deepstream's message pipeline. It receives * connections and authentication requests, authenticates sockets and * forwards messages it receives from authenticated sockets. */ export class MQTTConnectionEndpoint extends ConnectionEndpoint { private server!: TCPServer | TLSServer private connections = new Map() private logger = this.services.logger.getNameSpace('MQTT') private isReady: boolean = false private emitter = new EventEmitter() constructor (private mqttOptions: MQTTConnectionEndpointConfig, services: DeepstreamServices, config: DeepstreamConfig) { super(mqttOptions, services, config) this.description = 'MQTT Protocol Connection Endpoint' this.onMessages = this.onMessages.bind(this) } public async whenReady (): Promise { if (!this.isReady) { return new Promise((resolve) => this.emitter.once('ready', resolve)) } } public async close (): Promise { return new Promise((resolve) => this.server.close(() => resolve())) } /** * Initialize the ws endpoint, setup callbacks etc. */ public createWebsocketServer () { if (this.mqttOptions.ssl) { this.server = createTLSServer({ key: this.mqttOptions.ssl.key, cert: this.mqttOptions.ssl.cert }) } else { this.server = createTCPServer() } this.server.on(this.mqttOptions.ssl ? 'secureConnection' : 'connection', (stream) => { const client: MQTTConnection = mqttCon(stream) const socketWrapper = createMQTTSocketWrapper(client, {}, this.services, this.logger) this.connections.set(client, socketWrapper) this.onConnection(socketWrapper) socketWrapper.onMessage([{ topic: TOPIC.CONNECTION, action: CONNECTION_ACTION.CHALLENGE }]) const logger = this.services.logger // client connected client.on('connect', function (packet: MQTTPacket) { logger.debug(EVENT.INCOMING_CONNECTION, `MQTT Connection with username ${packet.username}`, { username: packet.username }) socketWrapper.onMessage([{ topic: TOPIC.AUTH, action: AUTH_ACTION.REQUEST, parsedData: { username: packet.username, password: packet.password && packet.password.toString() } }]) }) const closeClient = () => { if (!this.connections.has(client)) { return } this.onSocketClose(socketWrapper) this.connections.delete(client) socketWrapper.destroy() } // client disconnect client.on('disconnect', closeClient) // connection error handling client.on('close', closeClient) client.on('error', (e: any) => { this.logger.error('CLIENT ERROR', e.toString()) closeClient() }) // timeout idle streams after 5 minutes stream.setTimeout(this.mqttOptions.idleTimeout) // stream timeout stream.on('timeout', function () { client.destroy() }) // client published client.on('publish', (packet: MQTTPacket) => { this.onMessages(socketWrapper as any, socketWrapper.parseMessage(packet) as Message[]) }) // // client pinged client.on('pingreq', function () { client.pingresp() }) // client subscribed client.on('subscribe', (packet: MQTTPacket) => { this.onMessages(socketWrapper as any, socketWrapper.parseMessage(packet) as Message[]) }) // client subscribed client.on('unsubscribe', (packet: MQTTPacket) => { this.onMessages(socketWrapper as any, socketWrapper.parseMessage(packet) as Message[]) }) }) this.server.listen(this.mqttOptions.port, this.mqttOptions.host, () => { this.services.logger.info(EVENT.INFO, `Listening for MQTT ${this.mqttOptions.ssl ? 'TLS' : 'TCP' } connections on ${this.mqttOptions.host}:${this.mqttOptions.port}`) this.isReady = true this.emitter.emit('ready') }) return this.server } public async closeWebsocketServer () { this.connections.forEach((conn) => { if (!conn.isClosed) { conn.destroy() } }) this.connections.clear() return new Promise((resolve) => this.server.close(resolve)) } public onSocketWrapperClosed (socketWrapper: SocketWrapper) { socketWrapper.close() } } ================================================ FILE: src/connection-endpoint/mqtt/message-parser.ts ================================================ import { TOPIC, EVENT_ACTION, Message, PARSER_ACTION, RECORD_ACTION } from '../../constants' export const parseMQTT = (msg: any): Message[] => { let topic = TOPIC.EVENT if (msg.retain) { topic = TOPIC.RECORD } if (msg.cmd === 'subscribe') { const names = msg.subscriptions.map((mqttMsg: any) => mqttMsg.topic) return [{ topic: TOPIC.EVENT, action: EVENT_ACTION.SUBSCRIBE, names, correlationId: msg.messageId }, { topic: TOPIC.RECORD, action: RECORD_ACTION.SUBSCRIBE, names, correlationId: msg.messageId }] } if (msg.cmd === 'unsubscribe') { const names = msg.subscriptions.map((mqttMsg: any) => mqttMsg.topic) return [{ topic: TOPIC.EVENT, action: EVENT_ACTION.UNSUBSCRIBE, names, correlationId: msg.messageId }, { topic: TOPIC.RECORD, action: RECORD_ACTION.UNSUBSCRIBE, names, correlationId: msg.messageId }] } if (msg.cmd === 'publish') { if (topic === TOPIC.EVENT) { return [{ topic, action: EVENT_ACTION.EMIT, name: msg.topic, parsedData: msg.payload.toString() }] } else if (topic === TOPIC.RECORD) { return [{ topic, action: RECORD_ACTION.CREATEANDUPDATE, name: msg.topic, parsedData: JSON.parse(msg.payload.toString()), isWriteAck: msg.qos > 0, version: -1, correlationId: msg.messageId }, { topic: TOPIC.EVENT, action: EVENT_ACTION.EMIT, name: msg.topic, parsedData: msg.payload.toString() }] } } return [{ topic: TOPIC.PARSER, action: PARSER_ACTION.INVALID_MESSAGE }] } ================================================ FILE: src/connection-endpoint/mqtt/socket-wrapper-factory.ts ================================================ import { StatefulSocketWrapper, DeepstreamServices, UnauthenticatedSocketWrapper, EVENT, NamespacedLogger } from '@deepstream/types' import { TOPIC, CONNECTION_ACTION, Message, EVENT_ACTION, AUTH_ACTION, RECORD_ACTION, ParseResult } from '../../constants' import { ACTIONS_BYTE_TO_KEY } from '../websocket/text/text-protocol/constants' import { parseMQTT } from './message-parser' /** * This class wraps around a websocket * and provides higher level methods that are integrated * with deepstream's message structure */ export class MQTTSocketWrapper implements UnauthenticatedSocketWrapper { public socketType = 'mqtt' public userId: string | null = null public serverData: object | null = null public clientData: object | null = null public isRemote: false = false public isClosed: boolean = false public uuid: number = Math.random() public authCallback: Function | null = null public authAttempts: number = 0 private closeCallbacks: Set = new Set() constructor ( private socket: any, private handshakeData: any, private services: DeepstreamServices, private logger: NamespacedLogger ) { } get isOpen () { return this.isClosed !== true } public flush () { } /** * Sends a message based on the provided action and topic */ public sendMessage (message: { topic: TOPIC, action: CONNECTION_ACTION } | Message, allowBuffering: boolean = true): void { this.services.monitoring.onMessageSend(message) this.sendBuiltMessage(message) } /** * Sends a message based on the provided action and topic */ public sendAckMessage (message: Message, allowBuffering: boolean = true): void { this.services.monitoring.onMessageSend(message) if (message.topic === TOPIC.EVENT) { if (message.action === EVENT_ACTION.SUBSCRIBE) { this.socket.suback({ granted: [0], messageId: Number(message.correlationId) }) return } } if (message.topic === TOPIC.RECORD) { if (message.action === RECORD_ACTION.SUBSCRIBE) { this.socket.suback({ granted: [1], messageId: Number(message.correlationId) }) return } } this.logger.warn(EVENT.UNKNOWN_ACTION, `Unhandled ack message for ${TOPIC[message.topic]}:${ACTIONS_BYTE_TO_KEY[message.topic][message.action]}`) } public getMessage (message: Message): Message { return message } public parseData (message: Message): true | Error { return true } public onMessage (messages: Message[]): void { } /** * Destroys the socket. Removes all deepstream specific * logic and closes the connection */ public destroy (): void { this.socket.destroy() } public close (): void { this.isClosed = true this.authCallback = null this.closeCallbacks.forEach((cb) => cb(this)) this.services.logger.info(EVENT.CLIENT_DISCONNECTED, this.userId!) } public parseMessage (serializedMessage: any): ParseResult[] { return parseMQTT(serializedMessage) } /** * Returns a map of parameters that were collected * during the initial http request that established the * connection */ public getHandshakeData (): any { return this.handshakeData } public onClose (callback: (socketWrapper: StatefulSocketWrapper) => void): void { this.closeCallbacks.add(callback) } public removeOnClose (callback: (socketWrapper: StatefulSocketWrapper) => void): void { this.closeCallbacks.delete(callback) } public sendBuiltMessage (message: Message, buffer?: boolean): void { if (this.isOpen) { if (message.topic === TOPIC.CONNECTION) { if (message.action === CONNECTION_ACTION.ACCEPT) { return } } if (message.topic === TOPIC.AUTH) { if (message.action === AUTH_ACTION.AUTH_SUCCESSFUL) { this.socket.connack({ returnCode: 0 }) return } if (message.action === AUTH_ACTION.AUTH_UNSUCCESSFUL) { this.socket.connack({ returnCode: 5, reason: message.reason }) return } } if (message.topic === TOPIC.EVENT) { if (message.action === EVENT_ACTION.EMIT) { let payload = message.data if (!payload && message.parsedData) { payload = Buffer.from(JSON.stringify(message.parsedData)) } this.socket.publish({ cmd: 'publish', topic: message.name, payload, length: payload && payload.length }) return } } if (message.topic === TOPIC.RECORD) { if (message.action === RECORD_ACTION.WRITE_ACKNOWLEDGEMENT) { this.socket.puback({ messageId: message.correlationId }) return } if (message.action === RECORD_ACTION.UPDATE) { const payload = Buffer.from(JSON.stringify(message.parsedData)) this.socket.publish({ cmd: 'publish', topic: message.name, payload, length: payload.length }) return } if (message.action === RECORD_ACTION.PATCH) { this.logger.warn(EVENT.UNSUPPORTED_ACTION, 'Patches are not currently supported via the MQTT API') return } } this.logger.warn(EVENT.UNKNOWN_ACTION, `Unhandled message for ${TOPIC[message.topic]}:${ACTIONS_BYTE_TO_KEY[message.topic][message.action]}`) } } } export const createMQTTSocketWrapper = function ( socket: any, handshakeData: any, services: DeepstreamServices, logger: NamespacedLogger ) { return new MQTTSocketWrapper(socket, handshakeData, services, logger) } ================================================ FILE: src/connection-endpoint/websocket/binary/connection-endpoint.ts ================================================ import BaseWebsocketConnectionEndpoint, { WebSocketServerConfig } from '../../base/connection-endpoint' import { createWSSocketWrapper } from './socket-wrapper-factory' import { DeepstreamServices, DeepstreamConfig, WebSocketConnectionEndpoint } from '@deepstream/types' export class WSBinaryConnectionEndpoint extends BaseWebsocketConnectionEndpoint implements WebSocketConnectionEndpoint { public description = 'Binary WebSocket Connection Endpoint' constructor (public wsOptions: WebSocketServerConfig, services: DeepstreamServices, config: DeepstreamConfig) { super(wsOptions, services, config) } public async init () { super.init() this.services.httpService.registerWebsocketEndpoint(this.wsOptions.urlPath, createWSSocketWrapper, this) } } ================================================ FILE: src/connection-endpoint/websocket/binary/socket-wrapper-factory.ts ================================================ import * as binaryMessageBuilder from '@deepstream/protobuf/dist/src/message-builder' import * as binaryMessageParser from '@deepstream/protobuf/dist/src/message-parser' import { ParseResult, Message } from '../../../constants' import { WebSocketServerConfig } from '../../base/connection-endpoint' import { SocketConnectionEndpoint, DeepstreamServices } from '@deepstream/types' import { WSSocketWrapper } from '../../base/socket-wrapper' export class WSBinarySocketWrapper extends WSSocketWrapper { public socketType = 'wsBinary' public getAckMessage (message: Message): Uint8Array { return binaryMessageBuilder.getMessage(message, true) } public getMessage (message: Message): Uint8Array { return binaryMessageBuilder.getMessage(message, false) } public parseMessage (message: ArrayBuffer): ParseResult[] { if (typeof message === 'string') { this.invalidTypeReceived() return [] } /* we copy the underlying buffer (since a shallow reference won't be safe * outside of the callback) * the copy could be avoided if we make sure not to store references to the * raw buffer within the message */ return binaryMessageParser.parse(Buffer.from(Buffer.from(message))) } public parseData (message: Message): true | Error { return binaryMessageParser.parseData(message) } } export const createWSSocketWrapper = function ( socket: any, handshakeData: any, services: DeepstreamServices, config: WebSocketServerConfig, connectionEndpoint: SocketConnectionEndpoint ) { return new WSBinarySocketWrapper(socket, handshakeData, services, config, connectionEndpoint, true) } ================================================ FILE: src/connection-endpoint/websocket/json/connection-endpoint.ts ================================================ import WebsocketConnectionEndpoint, { WebSocketServerConfig } from '../../base/connection-endpoint' import {createWSSocketWrapper} from './socket-wrapper-factory' import { DeepstreamServices, DeepstreamConfig } from '@deepstream/types' export class WSJSONConnectionEndpoint extends WebsocketConnectionEndpoint { public description = 'WS Text Connection Endpoint' constructor (public wsOptions: WebSocketServerConfig, services: DeepstreamServices, config: DeepstreamConfig) { super(wsOptions, services, config) } public init () { super.init() this.services.httpService.registerWebsocketEndpoint(this.wsOptions.urlPath, createWSSocketWrapper, this) } } ================================================ FILE: src/connection-endpoint/websocket/json/socket-wrapper-factory.ts ================================================ import { ParseResult, Message } from '../../../constants' import { WebSocketServerConfig } from '../../base/connection-endpoint' import { SocketConnectionEndpoint, DeepstreamServices } from '@deepstream/types' import { WSSocketWrapper } from '../../base/socket-wrapper' export class JSONSocketWrapper extends WSSocketWrapper { public socketType = 'wsJSON' public getMessage (message: Message): string { return JSON.stringify(message) } public getAckMessage (message: Message): string { return this.getMessage(message) } public parseMessage (message: string): ParseResult[] { if (typeof message !== 'string') { this.invalidTypeReceived() return [] } try { return [JSON.parse(message)] } catch (e) { this.invalidTypeReceived() return [] } } public parseData (message: Message): true | Error { try { if (message.data) { message.parsedData = JSON.parse(message.data as string) } return true } catch (e) { if (e instanceof Error) { return e } return new Error(`Unknown error: ${e}`) } } } export const createWSSocketWrapper = function ( socket: any, handshakeData: any, services: DeepstreamServices, config: WebSocketServerConfig, connectionEndpoint: SocketConnectionEndpoint ) { return new JSONSocketWrapper(socket, handshakeData, services, config, connectionEndpoint, false) } ================================================ FILE: src/connection-endpoint/websocket/text/connection-endpoint.ts ================================================ import { WebSocketServerConfig } from '../../base/connection-endpoint' import BaseWebsocketConnectionEndpoint from '../../base/connection-endpoint' import {createWSSocketWrapper} from './socket-wrapper-factory' import { DeepstreamServices, DeepstreamConfig, UnauthenticatedSocketWrapper, WebSocketConnectionEndpoint } from '@deepstream/types' import * as textMessageBuilder from './text-protocol/message-builder' import { TOPIC, CONNECTION_ACTION } from '../../../constants' export class WSTextConnectionEndpoint extends BaseWebsocketConnectionEndpoint implements WebSocketConnectionEndpoint { public description = 'WS Text Protocol Connection Endpoint' private pingMessage: string constructor (public wsOptions: WebSocketServerConfig, services: DeepstreamServices, config: DeepstreamConfig) { super(wsOptions, services, config) this.pingMessage = textMessageBuilder.getMessage({ topic: TOPIC.CONNECTION, action: CONNECTION_ACTION.PING }) } public async init () { super.init() this.services.httpService.registerWebsocketEndpoint(this.wsOptions.urlPath, createWSSocketWrapper, this) } public onConnection (socketWrapper: UnauthenticatedSocketWrapper) { super.onConnection(socketWrapper) socketWrapper.onMessage = socketWrapper.authCallback! socketWrapper.sendMessage({ topic: TOPIC.CONNECTION, action: CONNECTION_ACTION.ACCEPT }, false) this.sendPing(socketWrapper) } private sendPing (socketWrapper: UnauthenticatedSocketWrapper) { if (!socketWrapper.isClosed) { socketWrapper.sendBuiltMessage!(this.pingMessage) setTimeout(this.sendPing.bind(this, socketWrapper), this.wsOptions.heartbeatInterval) } } } ================================================ FILE: src/connection-endpoint/websocket/text/socket-wrapper-factory.ts ================================================ import { ParseResult, Message } from '../../../constants' import * as textMessageBuilder from './text-protocol/message-builder' import * as textMessageParse from './text-protocol/message-parser' import { SocketConnectionEndpoint, DeepstreamServices } from '@deepstream/types' import { WebSocketServerConfig } from '../../base/connection-endpoint' import { WSSocketWrapper } from '../../base/socket-wrapper' export class TextWSSocketWrapper extends WSSocketWrapper { public socketType = 'wsText' public getMessage (message: Message): string { return textMessageBuilder.getMessage(message, false) } public getAckMessage (message: Message): string { return textMessageBuilder.getMessage(message, true) } public parseMessage (message: string): ParseResult[] { if (typeof message !== 'string') { this.invalidTypeReceived() return [] } return textMessageParse.parse(message) } public parseData (message: Message): true | Error { return textMessageParse.parseData(message) } } export const createWSSocketWrapper = function ( socket: any, handshakeData: any, services: DeepstreamServices, config: WebSocketServerConfig, connectionEndpoint: SocketConnectionEndpoint, ) { return new TextWSSocketWrapper(socket, handshakeData, services, config, connectionEndpoint, false) } ================================================ FILE: src/connection-endpoint/websocket/text/text-protocol/constants.ts ================================================ import { AUTH_ACTION as AA, CONNECTION_ACTION as CA, EVENT_ACTION as EA, PARSER_ACTION as XA, PRESENCE_ACTION as UA, RECORD_ACTION as RA, RPC_ACTION as PA, TOPIC as T, } from '../../../../constants' export const MESSAGE_SEPERATOR = String.fromCharCode(30) // ASCII Record Seperator 1E export const MESSAGE_PART_SEPERATOR = String.fromCharCode(31) // ASCII Unit Separator 1F export const PAYLOAD_ENCODING = { JSON: 0x00, DEEPSTREAM: 0x01, } export const TOPIC = { PARSER: { TEXT: 'X', BYTE: T.PARSER }, CONNECTION: { TEXT: 'C', BYTE: T.CONNECTION }, AUTH: { TEXT: 'A', BYTE: T.AUTH }, ERROR: { TEXT: 'X', BYTE: T.ERROR }, EVENT: { TEXT: 'E', BYTE: T.EVENT }, RECORD: { TEXT: 'R', BYTE: T.RECORD }, RPC: { TEXT: 'P', BYTE: T.RPC }, PRESENCE: { TEXT: 'U', BYTE: T.PRESENCE }, } export const PARSER_ACTIONS = { UNKNOWN_TOPIC: { BYTE: XA.UNKNOWN_TOPIC }, UNKNOWN_ACTION: { BYTE: XA.UNKNOWN_ACTION }, INVALID_MESSAGE: { BYTE: XA.INVALID_MESSAGE }, INVALID_META_PARAMS: { BYTE: XA.INVALID_META_PARAMS }, MESSAGE_PARSE_ERROR: { BYTE: XA.MESSAGE_PARSE_ERROR }, MAXIMUM_MESSAGE_SIZE_EXCEEDED: { BYTE: XA.MAXIMUM_MESSAGE_SIZE_EXCEEDED }, ERROR: { BYTE: XA.ERROR }, } export const CONNECTION_ACTIONS = { ERROR: { TEXT: 'E', BYTE: CA.ERROR }, PING: { TEXT: 'PI', BYTE: CA.PING }, PONG: { TEXT: 'PO', BYTE: CA.PONG }, ACCEPT: { TEXT: 'A', BYTE: CA.ACCEPT }, CHALLENGE: { TEXT: 'CH', BYTE: CA.CHALLENGE }, REJECTION: { TEXT: 'REJ', BYTE: CA.REJECT }, REDIRECT: { TEXT: 'RED', BYTE: CA.REDIRECT }, CLOSED: { BYTE: CA.CLOSED }, CLOSING: { BYTE: CA.CLOSING }, CONNECTION_AUTHENTICATION_TIMEOUT: { BYTE: CA.AUTHENTICATION_TIMEOUT }, INVALID_MESSAGE: { BYTE: CA.INVALID_MESSAGE }, } export const AUTH_ACTIONS = { ERROR: { TEXT: 'E', BYTE: AA.ERROR }, REQUEST: { TEXT: 'REQ', BYTE: AA.REQUEST }, AUTH_SUCCESSFUL: { BYTE: AA.AUTH_SUCCESSFUL, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, AUTH_UNSUCCESSFUL: { BYTE: AA.AUTH_UNSUCCESSFUL, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, TOO_MANY_AUTH_ATTEMPTS: { BYTE: AA.TOO_MANY_AUTH_ATTEMPTS }, // MESSAGE_PERMISSION_ERROR: { BYTE: AA.MESSAGE_PERMISSION_ERROR }, // MESSAGE_DENIED: { BYTE: AA.MESSAGE_DENIED }, INVALID_MESSAGE_DATA: { BYTE: AA.INVALID_MESSAGE_DATA }, INVALID_MESSAGE: { BYTE: AA.INVALID_MESSAGE }, } export const EVENT_ACTIONS = { ERROR: { TEXT: 'E', BYTE: EA.ERROR }, EMIT: { TEXT: 'EVT', BYTE: EA.EMIT, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, SUBSCRIBE: { TEXT: 'S', BYTE: EA.SUBSCRIBE }, UNSUBSCRIBE: { TEXT: 'US', BYTE: EA.UNSUBSCRIBE }, LISTEN: { TEXT: 'L', BYTE: EA.LISTEN }, UNLISTEN: { TEXT: 'UL', BYTE: EA.UNLISTEN }, LISTEN_ACCEPT: { TEXT: 'LA', BYTE: EA.LISTEN_ACCEPT }, LISTEN_REJECT: { TEXT: 'LR', BYTE: EA.LISTEN_REJECT }, SUBSCRIPTION_FOR_PATTERN_FOUND: { TEXT: 'SP', BYTE: EA.SUBSCRIPTION_FOR_PATTERN_FOUND }, SUBSCRIPTION_FOR_PATTERN_REMOVED: { TEXT: 'SR', BYTE: EA.SUBSCRIPTION_FOR_PATTERN_REMOVED }, MESSAGE_PERMISSION_ERROR: { BYTE: EA.MESSAGE_PERMISSION_ERROR }, MESSAGE_DENIED: { BYTE: EA.MESSAGE_DENIED }, INVALID_MESSAGE_DATA: { BYTE: EA.INVALID_MESSAGE_DATA }, MULTIPLE_SUBSCRIPTIONS: { BYTE: EA.MULTIPLE_SUBSCRIPTIONS }, NOT_SUBSCRIBED: { BYTE: EA.NOT_SUBSCRIBED }, } export const RECORD_ACTIONS = { ERROR: { TEXT: 'E', BYTE: RA.ERROR }, CREATE: { TEXT: 'CR', BYTE: RA.CREATE }, READ: { TEXT: 'R', BYTE: RA.READ }, READ_RESPONSE: { BYTE: RA.READ_RESPONSE, PAYLOAD_ENCODING: PAYLOAD_ENCODING.JSON }, HEAD: { TEXT: 'HD', BYTE: RA.HEAD }, HEAD_RESPONSE: { BYTE: RA.HEAD_RESPONSE }, CREATEANDUPDATE: { TEXT: 'CU', BYTE: RA.CREATEANDUPDATE }, CREATEANDPATCH: { BYTE: RA.CREATEANDPATCH, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, UPDATE: { TEXT: 'U', BYTE: RA.UPDATE, PAYLOAD_ENCODING: PAYLOAD_ENCODING.JSON }, PATCH: { TEXT: 'P', BYTE: RA.PATCH, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, ERASE: { BYTE: RA.ERASE, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, WRITE_ACKNOWLEDGEMENT: { TEXT: 'WA', BYTE: RA.WRITE_ACKNOWLEDGEMENT }, DELETE: { TEXT: 'D', BYTE: RA.DELETE }, DELETE_SUCCESS: { BYTE: RA.DELETE_SUCCESS }, DELETED: { BYTE: RA.DELETED }, LISTEN_RESPONSE_TIMEOUT: { BYTE: RA.LISTEN_RESPONSE_TIMEOUT }, SUBSCRIBEANDHEAD: { BYTE: RA.SUBSCRIBEANDHEAD }, // SUBSCRIBEANDHEAD_RESPONSE: { BYTE: RA.SUBSCRIBEANDHEAD_RESPONSE }, SUBSCRIBEANDREAD: { BYTE: RA.SUBSCRIBEANDREAD }, // SUBSCRIBEANDREAD_RESPONSE: { BYTE: RA.SUBSCRIBEANDREAD_RESPONSE }, SUBSCRIBECREATEANDREAD: { TEXT: 'CR', BYTE: RA.SUBSCRIBECREATEANDREAD }, // SUBSCRIBECREATEANDREAD_RESPONSE: { BYTE: RA.SUBSCRIBECREATEANDREAD_RESPONSE }, SUBSCRIBECREATEANDUPDATE: { BYTE: RA.SUBSCRIBECREATEANDUPDATE }, // SUBSCRIBECREATEANDUPDATE_RESPONSE: { BYTE: RA.SUBSCRIBECREATEANDUPDATE_RESPONSE }, SUBSCRIBE: { TEXT: 'S', BYTE: RA.SUBSCRIBE }, UNSUBSCRIBE: { TEXT: 'US', BYTE: RA.UNSUBSCRIBE }, LISTEN: { TEXT: 'L', BYTE: RA.LISTEN }, UNLISTEN: { TEXT: 'UL', BYTE: RA.UNLISTEN }, LISTEN_ACCEPT: { TEXT: 'LA', BYTE: RA.LISTEN_ACCEPT }, LISTEN_REJECT: { TEXT: 'LR', BYTE: RA.LISTEN_REJECT }, SUBSCRIPTION_HAS_PROVIDER: { TEXT: 'SH', BYTE: RA.SUBSCRIPTION_HAS_PROVIDER }, SUBSCRIPTION_HAS_NO_PROVIDER: { BYTE: RA.SUBSCRIPTION_HAS_NO_PROVIDER }, SUBSCRIPTION_FOR_PATTERN_FOUND: { TEXT: 'SP', BYTE: RA.SUBSCRIPTION_FOR_PATTERN_FOUND }, SUBSCRIPTION_FOR_PATTERN_REMOVED: { TEXT: 'SR', BYTE: RA.SUBSCRIPTION_FOR_PATTERN_REMOVED }, CACHE_RETRIEVAL_TIMEOUT: { BYTE: RA.CACHE_RETRIEVAL_TIMEOUT }, STORAGE_RETRIEVAL_TIMEOUT: { BYTE: RA.STORAGE_RETRIEVAL_TIMEOUT }, VERSION_EXISTS: { BYTE: RA.VERSION_EXISTS }, // HAS: { TEXT: 'H', BYTE: RA.HAS }, // HAS_RESPONSE: { BYTE: RA.HAS_RESPONSE }, SNAPSHOT: { TEXT: 'SN', BYTE: RA.READ }, RECORD_LOAD_ERROR: { BYTE: RA.RECORD_LOAD_ERROR }, RECORD_CREATE_ERROR: { BYTE: RA.RECORD_CREATE_ERROR }, RECORD_UPDATE_ERROR: { BYTE: RA.RECORD_UPDATE_ERROR }, RECORD_DELETE_ERROR: { BYTE: RA.RECORD_DELETE_ERROR }, // RECORD_READ_ERROR: { BYTE: RA.RECORD_READ_ERROR }, RECORD_NOT_FOUND: { BYTE: RA.RECORD_NOT_FOUND }, INVALID_VERSION: { BYTE: RA.INVALID_VERSION }, INVALID_PATCH_ON_HOTPATH: { BYTE: RA.INVALID_PATCH_ON_HOTPATH }, MESSAGE_PERMISSION_ERROR: { BYTE: RA.MESSAGE_PERMISSION_ERROR }, MESSAGE_DENIED: { BYTE: RA.MESSAGE_DENIED }, INVALID_MESSAGE_DATA: { BYTE: RA.INVALID_MESSAGE_DATA }, MULTIPLE_SUBSCRIPTIONS: { BYTE: RA.MULTIPLE_SUBSCRIPTIONS }, NOT_SUBSCRIBED: { BYTE: RA.NOT_SUBSCRIBED }, } export const RPC_ACTIONS = { ERROR: { BYTE: PA.ERROR }, REQUEST: { TEXT: 'REQ', BYTE: PA.REQUEST, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, ACCEPT: { BYTE: PA.ACCEPT }, RESPONSE: { TEXT: 'RES', BYTE: PA.RESPONSE, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, REJECT: { TEXT: 'REJ', BYTE: PA.REJECT }, REQUEST_ERROR: { TEXT: 'E', BYTE: PA.REQUEST_ERROR, PAYLOAD_ENCODING: PAYLOAD_ENCODING.DEEPSTREAM }, PROVIDE: { TEXT: 'S', BYTE: PA.PROVIDE }, UNPROVIDE: { TEXT: 'US', BYTE: PA.UNPROVIDE }, NO_RPC_PROVIDER: { BYTE: PA.NO_RPC_PROVIDER }, RESPONSE_TIMEOUT: { BYTE: PA.RESPONSE_TIMEOUT }, ACCEPT_TIMEOUT: { BYTE: PA.ACCEPT_TIMEOUT }, MULTIPLE_ACCEPT: { BYTE: PA.MULTIPLE_ACCEPT }, MULTIPLE_RESPONSE: { BYTE: PA.MULTIPLE_RESPONSE }, INVALID_RPC_CORRELATION_ID: { BYTE: PA.INVALID_RPC_CORRELATION_ID }, MESSAGE_PERMISSION_ERROR: { BYTE: PA.MESSAGE_PERMISSION_ERROR }, MESSAGE_DENIED: { BYTE: PA.MESSAGE_DENIED }, INVALID_MESSAGE_DATA: { BYTE: PA.INVALID_MESSAGE_DATA }, MULTIPLE_PROVIDERS: { BYTE: PA.MULTIPLE_PROVIDERS }, NOT_PROVIDED: { BYTE: PA.NOT_PROVIDED }, } export const PRESENCE_ACTIONS = { ERROR: { TEXT: 'E', BYTE: UA.ERROR }, QUERY_ALL: { BYTE: UA.QUERY_ALL }, QUERY_ALL_RESPONSE: { BYTE: UA.QUERY_ALL_RESPONSE, PAYLOAD_ENCODING: PAYLOAD_ENCODING.JSON }, QUERY: { TEXT: 'Q', BYTE: UA.QUERY }, QUERY_RESPONSE: { BYTE: UA.QUERY_RESPONSE, PAYLOAD_ENCODING: PAYLOAD_ENCODING.JSON }, PRESENCE_JOIN: { TEXT: 'PNJ', BYTE: UA.PRESENCE_JOIN }, PRESENCE_JOIN_ALL: { TEXT: 'PNJ', BYTE: UA.PRESENCE_JOIN_ALL }, PRESENCE_LEAVE: { TEXT: 'PNL', BYTE: UA.PRESENCE_LEAVE }, PRESENCE_LEAVE_ALL: { TEXT: 'PNL', BYTE: UA.PRESENCE_LEAVE_ALL }, SUBSCRIBE: { TEXT: 'S', BYTE: UA.SUBSCRIBE }, UNSUBSCRIBE: { TEXT: 'US', BYTE: UA.UNSUBSCRIBE }, SUBSCRIBE_ALL: { BYTE: UA.SUBSCRIBE_ALL }, UNSUBSCRIBE_ALL: { BYTE: UA.UNSUBSCRIBE_ALL }, INVALID_PRESENCE_USERS: { BYTE: UA.INVALID_PRESENCE_USERS }, MESSAGE_PERMISSION_ERROR: { BYTE: UA.MESSAGE_PERMISSION_ERROR }, MESSAGE_DENIED: { BYTE: UA.MESSAGE_DENIED }, // INVALID_MESSAGE_DATA: { BYTE: UA.INVALID_MESSAGE_DATA }, MULTIPLE_SUBSCRIPTIONS: { BYTE: UA.MULTIPLE_SUBSCRIPTIONS }, NOT_SUBSCRIBED: { BYTE: UA.NOT_SUBSCRIBED }, } export const DEEPSTREAM_TYPES = { STRING: 'S', OBJECT: 'O', NUMBER: 'N', NULL: 'L', TRUE: 'T', FALSE: 'F', UNDEFINED: 'U', } export const TOPIC_BYTE_TO_TEXT = convertMap(TOPIC, 'BYTE', 'TEXT') export const TOPIC_TEXT_TO_BYTE = convertMap(TOPIC, 'TEXT', 'BYTE') export const TOPIC_TEXT_TO_KEY = reverseMap(specifyMap(TOPIC, 'TEXT')) export const TOPIC_BYTE_TO_KEY = reverseMap(specifyMap(TOPIC, 'BYTE')) export const TOPIC_BYTES = specifyMap(TOPIC, 'BYTE') export const ACTIONS_BYTE_TO_PAYLOAD: any = {} export const ACTIONS_BYTE_TO_TEXT: any = {} export const ACTIONS_TEXT_TO_BYTE: any = {} export const ACTIONS_BYTES: any = {} export const ACTIONS_TEXT_TO_KEY: any = {} export const ACTIONS_BYTE_TO_KEY: any = {} export const ACTIONS = { [TOPIC.PARSER.BYTE]: PARSER_ACTIONS, [TOPIC.CONNECTION.BYTE]: CONNECTION_ACTIONS, [TOPIC.AUTH.BYTE]: AUTH_ACTIONS, [TOPIC.EVENT.BYTE]: EVENT_ACTIONS, [TOPIC.RECORD.BYTE]: RECORD_ACTIONS, [TOPIC.RPC.BYTE]: RPC_ACTIONS, [TOPIC.PRESENCE.BYTE]: PRESENCE_ACTIONS, } for (const key in ACTIONS) { ACTIONS_BYTE_TO_PAYLOAD[key] = convertMap(ACTIONS[key], 'BYTE', 'PAYLOAD_ENCODING') ACTIONS_BYTE_TO_TEXT[key] = convertMap(ACTIONS[key], 'BYTE', 'TEXT') ACTIONS_TEXT_TO_BYTE[key] = convertMap(ACTIONS[key], 'TEXT', 'BYTE') ACTIONS_BYTES[key] = specifyMap(ACTIONS[key], 'BYTE') ACTIONS_TEXT_TO_KEY[key] = reverseMap(specifyMap(ACTIONS[key], 'TEXT')) ACTIONS_BYTE_TO_KEY[key] = reverseMap(specifyMap(ACTIONS[key], 'BYTE')) } /** * convertMap({ a: { x: 1 }, b: { x: 2 }, c: { x : 3 } }, 'x', 'y') * === * { a: { y: 1 }, b: { y: 2 }, c: { y : 3 } } */ function convertMap (map: any, from: any, to: any) { const result: any = {} for (const key in map) { result[map[key][from]] = map[key][to] } return result } /** * specifyMap({ a: { x: 1 }, b: { x: 2 }, c: { x : 3 } }, 'x') * === * { a: 1, b: 2, c: 3 } */ function specifyMap (map: any, innerKey: any) { const result: any = {} for (const key in map) { result[key] = map[key][innerKey] } return result } /** * Takes a key-value map and returns * a map with { value: key } of the old map */ function reverseMap (map: any) { const reversedMap: any = {} for (const key in map) { reversedMap[map[key]] = key } return reversedMap } ================================================ FILE: src/connection-endpoint/websocket/text/text-protocol/message-builder.ts ================================================ import { ACTIONS_BYTE_TO_PAYLOAD as ABP, ACTIONS_BYTE_TO_TEXT as ABT, AUTH_ACTIONS as AA, CONNECTION_ACTIONS as CA, DEEPSTREAM_TYPES as TYPES, EVENT_ACTIONS as EA, MESSAGE_PART_SEPERATOR as y, MESSAGE_SEPERATOR as x, PRESENCE_ACTIONS as UA, RECORD_ACTIONS as RA, RPC_ACTIONS as PA, TOPIC, TOPIC_BYTE_TO_TEXT as TBT, PAYLOAD_ENCODING, } from './constants' import { Message } from '../../../../constants' import { correlationIdToVersion, bulkNameToCorrelationId } from './message-parser' const WA = y + JSON.stringify({ writeSuccess: true }) const NWA = y + '{}' const A = 'A' + y const genericError = (msg: Message) => `${TBT[msg.topic]}${y}E${y}${msg.correlationId}${y}${msg.parsedData}${x}` const invalidMessageData = (msg: Message) => `${TBT[msg.topic]}${y}E${y}INVALID_MESSAGE_DATA${y}${msg.data}${x}` const messagePermissionError = (msg: Message) => `${TBT[msg.topic]}${y}E${y}MESSAGE_PERMISSION_ERROR${y}${msg.name}${ABT[msg.topic][msg.action] ? y + ABT[msg.topic][msg.action] : '' }${msg.correlationId ? y + msg.correlationId : '' }${x}` const messageDenied = (msg: Message) => { let version if (msg.topic === TOPIC.RECORD.BYTE && msg.correlationId) { version = correlationIdToVersion.get(msg.correlationId!) correlationIdToVersion.delete(msg.correlationId!) delete msg.correlationId } return `${TBT[msg.topic]}${y}E${y}MESSAGE_DENIED${y}${msg.name}${ABT[msg.topic][msg.action] ? y + ABT[msg.topic][msg.action] : '' }${msg.originalAction ? y + ABT[msg.topic][msg.originalAction] : '' }${msg.correlationId ? y + msg.correlationId : '' }${version !== undefined ? y + version : '' }${x}` } const notSubscribed = (msg: Message) => `${TBT[msg.topic]}${y}E${y}NOT_SUBSCRIBED${y}${msg.name}${x}` const invalidAuth = (msg: Message) => `A${y}E${y}INVALID_AUTH_DATA${y}${msg.data ? msg.data : 'U' }${x}` const recordUpdate = (msg: Message) => `R${y}U${y}${msg.name}${y}${msg.version}${y}${msg.data}${msg.isWriteAck ? WA : '' }${x}` const recordPatch = (msg: Message) => `R${y}P${y}${msg.name}${y}${msg.version}${y}${msg.path}${y}${msg.data}${msg.isWriteAck ? WA : '' }${x}` const subscriptionForPatternFound = (msg: Message) => `${TBT[msg.topic]}${y}SP${y}${msg.name}${y}${msg.subscription}${x}` const subscriptionForPatternRemoved = (msg: Message) => `${TBT[msg.topic]}${y}SR${y}${msg.name}${y}${msg.subscription}${x}` const listen = (msg: Message, isAck: boolean) => `${TBT[msg.topic]}${y}${isAck ? A : '' }L${y}${msg.name}${x}` const unlisten = (msg: Message, isAck: boolean) => `${TBT[msg.topic]}${y}${isAck ? A : '' }UL${y}${msg.name}${x}` const listenAccept = (msg: Message) => `${TBT[msg.topic]}${y}LA${y}${msg.name}${y}${msg.subscription}${x}` const listenReject = (msg: Message) => `${TBT[msg.topic]}${y}LR${y}${msg.name}${y}${msg.subscription}${x}` const multipleSubscriptions = (msg: Message) => `${TBT[msg.topic]}${y}E${y}MULTIPLE_SUBSCRIPTIONS${y}${msg.name}${x}` const BUILDERS = { [TOPIC.CONNECTION.BYTE]: { [CA.ERROR.BYTE]: genericError, [CA.CHALLENGE.BYTE]: (msg: Message) => `C${y}CH${x}`, [CA.ACCEPT.BYTE]: (msg: Message) => `C${y}A${x}`, [CA.REJECTION.BYTE]: (msg: Message) => `C${y}REJ${y}${msg.data}${x}`, [CA.REDIRECT.BYTE]: (msg: Message) => `C${y}RED${y}${msg.data}${x}`, [CA.PING.BYTE]: (msg: Message) => `C${y}PI${x}`, [CA.PONG.BYTE]: (msg: Message) => `C${y}PO${x}`, [CA.CONNECTION_AUTHENTICATION_TIMEOUT.BYTE]: (msg: Message) => `C${y}E${y}CONNECTION_AUTHENTICATION_TIMEOUT${x}`, }, [TOPIC.AUTH.BYTE]: { [AA.ERROR.BYTE]: genericError, [AA.REQUEST.BYTE]: (msg: Message) => `A${y}REQ${y}${msg.data}${x}`, [AA.AUTH_SUCCESSFUL.BYTE]: (msg: Message) => `A${y}A${msg.data ? y + msg.data : ''}${x}`, [AA.AUTH_UNSUCCESSFUL.BYTE]: invalidAuth, [AA.INVALID_MESSAGE_DATA.BYTE]: invalidAuth, [AA.TOO_MANY_AUTH_ATTEMPTS.BYTE]: (msg: Message) => `A${y}E${y}TOO_MANY_AUTH_ATTEMPTS${x}`, }, [TOPIC.EVENT.BYTE]: { [EA.ERROR.BYTE]: genericError, [EA.SUBSCRIBE.BYTE]: (msg: Message, isAck: boolean) => { let name = msg.name if (isAck) { name = bulkNameToCorrelationId.get(msg.correlationId!) bulkNameToCorrelationId.delete(msg.correlationId!) } return `E${y}${isAck ? A : '' }S${y}${name}${x}` }, [EA.UNSUBSCRIBE.BYTE]: (msg: Message, isAck: boolean) => { let name = msg.name if (isAck) { name = bulkNameToCorrelationId.get(msg.correlationId!) bulkNameToCorrelationId.delete(msg.correlationId!) } return `E${y}${isAck ? A : '' }US${y}${name}${x}` }, [EA.EMIT.BYTE]: (msg: Message) => `E${y}EVT${y}${msg.name}${y}${msg.data ? msg.data : 'U'}${x}`, [EA.LISTEN.BYTE]: listen, [EA.UNLISTEN.BYTE]: unlisten, [EA.LISTEN_ACCEPT.BYTE]: listenAccept, [EA.LISTEN_REJECT.BYTE]: listenReject, [EA.SUBSCRIPTION_FOR_PATTERN_FOUND.BYTE]: subscriptionForPatternFound, [EA.SUBSCRIPTION_FOR_PATTERN_REMOVED.BYTE]: subscriptionForPatternRemoved, [EA.INVALID_MESSAGE_DATA.BYTE]: invalidMessageData, [EA.MESSAGE_DENIED.BYTE]: messageDenied, [EA.MESSAGE_PERMISSION_ERROR.BYTE]: messagePermissionError, [EA.NOT_SUBSCRIBED.BYTE]: notSubscribed, [EA.MULTIPLE_SUBSCRIPTIONS.BYTE]: multipleSubscriptions, }, [TOPIC.RECORD.BYTE]: { [RA.ERROR.BYTE]: genericError, [RA.HEAD.BYTE]: (msg: Message) => `R${y}HD${y}${msg.name}${x}`, [RA.HEAD_RESPONSE.BYTE]: (msg: Message) => `R${y}HD${y}${msg.name}${y}${msg.version}${y}null${x}`, [RA.READ.BYTE]: (msg: Message) => `R${y}R${y}${msg.name}${x}`, [RA.READ_RESPONSE.BYTE]: (msg: Message) => `R${y}R${y}${msg.name}${y}${msg.version}${y}${msg.data}${x}`, [RA.UPDATE.BYTE]: recordUpdate, [RA.PATCH.BYTE]: recordPatch, [RA.ERASE.BYTE]: (msg: Message) => `R${y}P${y}${msg.name}${y}${msg.version}${y}${msg.path}${y}U${msg.isWriteAck ? WA : '' }${x}`, [RA.CREATEANDUPDATE.BYTE]: (msg: Message) => `R${y}CU${y}${msg.name}${y}${msg.version}${y}${msg.data}${msg.isWriteAck ? WA : NWA }${x}`, [RA.CREATEANDPATCH.BYTE]: (msg: Message) => `R${y}CU${y}${msg.name}${y}${msg.version}${y}${msg.path}${y}${msg.data}${msg.isWriteAck ? WA : NWA }${x}`, [RA.DELETE.BYTE]: (msg: Message, isAck: boolean) => `R${y}${isAck ? A : '' }D${y}${msg.name}${x}`, [RA.DELETED.BYTE]: (msg: Message) => `R${y}A${y}D${y}${msg.name}${x}`, [RA.DELETE_SUCCESS.BYTE]: (msg: Message) => `R${y}A${y}D${y}${msg.name}${x}`, [RA.SUBSCRIBECREATEANDREAD.BYTE]: (msg: Message, isAck: boolean) => { if (isAck) { return `R${y}A${y}S${y}${msg.name}${x}` } return `R${y}CR${y}${msg.name}${x}` }, [RA.UNSUBSCRIBE.BYTE]: (msg: Message, isAck: boolean) => { let name = msg.name if (isAck) { name = bulkNameToCorrelationId.get(msg.correlationId!) bulkNameToCorrelationId.delete(msg.correlationId!) } return `R${y}${isAck ? A : '' }US${y}${name}${x}` }, [RA.WRITE_ACKNOWLEDGEMENT.BYTE]: (msg: Message) => { return `R${y}WA${y}${msg.name}${y}[${correlationIdToVersion.get(msg.correlationId!)}]${y}${TYPES.NULL}${x}` }, [RA.LISTEN.BYTE]: listen, [RA.LISTEN_RESPONSE_TIMEOUT.BYTE]: (msg: Message) => `C${y}PO${x}`, [RA.UNLISTEN.BYTE]: unlisten, [RA.LISTEN_ACCEPT.BYTE]: listenAccept, [RA.LISTEN_REJECT.BYTE]: listenReject, [RA.SUBSCRIPTION_FOR_PATTERN_FOUND.BYTE]: subscriptionForPatternFound, [RA.SUBSCRIPTION_FOR_PATTERN_REMOVED.BYTE]: subscriptionForPatternRemoved, [RA.SUBSCRIPTION_HAS_PROVIDER.BYTE]: (msg: Message) => `R${y}SH${y}${msg.name}${y}T${x}`, [RA.SUBSCRIPTION_HAS_NO_PROVIDER.BYTE]: (msg: Message) => `R${y}SH${y}${msg.name}${y}F${x}`, [RA.STORAGE_RETRIEVAL_TIMEOUT.BYTE]: (msg: Message) => `R${y}E${y}STORAGE_RETRIEVAL_TIMEOUT${y}${msg.name}${x}`, [RA.CACHE_RETRIEVAL_TIMEOUT.BYTE]: (msg: Message) => `R${y}E${y}CACHE_RETRIEVAL_TIMEOUT${y}${msg.name}${x}`, [RA.VERSION_EXISTS.BYTE]: (msg: Message) => `R${y}E${y}VERSION_EXISTS${y}${msg.name}${y}${msg.version}${y}${msg.data}${msg.isWriteAck ? WA : ''}${x}`, [RA.RECORD_NOT_FOUND.BYTE]: (msg: Message) => `R${y}E${y}RECORD_NOT_FOUND${y}${msg.name}${x}`, [RA.INVALID_MESSAGE_DATA.BYTE]: invalidMessageData, [RA.MESSAGE_DENIED.BYTE]: messageDenied, [RA.MESSAGE_PERMISSION_ERROR.BYTE]: messagePermissionError, [RA.NOT_SUBSCRIBED.BYTE]: notSubscribed, [RA.MULTIPLE_SUBSCRIPTIONS.BYTE]: multipleSubscriptions, }, [TOPIC.RPC.BYTE]: { [PA.ERROR.BYTE]: genericError, [PA.PROVIDE.BYTE]: (msg: Message, isAck: boolean) => { let name = msg.name if (isAck) { name = bulkNameToCorrelationId.get(msg.correlationId!) bulkNameToCorrelationId.delete(msg.correlationId!) } return `P${y}${isAck ? A : '' }S${y}${name}${x}` }, [PA.UNPROVIDE.BYTE]: (msg: Message, isAck: boolean) => { let name = msg.name if (isAck) { name = bulkNameToCorrelationId.get(msg.correlationId!) bulkNameToCorrelationId.delete(msg.correlationId!) } return `P${y}${isAck ? A : '' }US${y}${name}${x}` }, [PA.REQUEST.BYTE]: (msg: Message) => `P${y}REQ${y}${msg.name}${y}${msg.correlationId}${y}${msg.data}${x}`, [PA.RESPONSE.BYTE]: (msg: Message) => `P${y}RES${y}${msg.name}${y}${msg.correlationId}${y}${msg.data}${x}`, [PA.REQUEST_ERROR.BYTE]: (msg: Message) => `P${y}E${y}${msg.data}${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.REJECT.BYTE]: (msg: Message) => `P${y}REJ${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.ACCEPT.BYTE]: (msg: Message) => `P${y}A${y}REQ${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.NO_RPC_PROVIDER.BYTE]: (msg: Message) => `P${y}E${y}NO_RPC_PROVIDER${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.INVALID_RPC_CORRELATION_ID.BYTE]: (msg: Message) => `P${y}E${y}INVALID_RPC_CORRELATION_ID${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.RESPONSE_TIMEOUT.BYTE]: (msg: Message) => `P${y}E${y}RESPONSE_TIMEOUT${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.MULTIPLE_RESPONSE.BYTE]: (msg: Message) => `P${y}E${y}MULTIPLE_RESPONSE${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.MULTIPLE_ACCEPT.BYTE]: (msg: Message) => `P${y}E${y}MULTIPLE_ACCEPT${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.ACCEPT_TIMEOUT.BYTE]: (msg: Message) => `P${y}E${y}ACCEPT_TIMEOUT${y}${msg.name}${y}${msg.correlationId}${x}`, [PA.INVALID_MESSAGE_DATA.BYTE]: invalidMessageData, [PA.MESSAGE_DENIED.BYTE]: messageDenied, [PA.MESSAGE_PERMISSION_ERROR.BYTE]: messagePermissionError, [PA.NOT_PROVIDED.BYTE]: notSubscribed, [PA.MULTIPLE_PROVIDERS.BYTE]: multipleSubscriptions, }, [TOPIC.PRESENCE.BYTE]: { [UA.ERROR.BYTE]: genericError, [UA.SUBSCRIBE.BYTE]: (msg: Message, isAck: boolean) => `U${y}${isAck ? A : '' }S${y}${msg.correlationId ? msg.correlationId + y : '' }${msg.name ? msg.name : msg.data}${x}`, [UA.SUBSCRIBE_ALL.BYTE]: (msg: Message, isAck: boolean) => `U${y}${isAck ? A : '' }S${y}S${x}`, [UA.UNSUBSCRIBE.BYTE]: (msg: Message, isAck: boolean) => `U${y}${isAck ? A : '' }US${y}${msg.correlationId ? msg.correlationId + y : '' }${msg.name ? msg.name : msg.data}${x}`, [UA.UNSUBSCRIBE_ALL.BYTE]: (msg: Message, isAck: boolean) => `U${y}${isAck ? A : '' }US${y}US${x}`, [UA.QUERY.BYTE]: (msg: Message) => `U${y}Q${y}${msg.correlationId}${y}${msg.data}${x}`, [UA.QUERY_RESPONSE.BYTE]: (msg: Message) => `U${y}Q${y}${msg.correlationId}${y}${msg.data}${x}`, [UA.QUERY_ALL.BYTE]: (msg: Message) => `U${y}Q${y}Q${x}`, [UA.QUERY_ALL_RESPONSE.BYTE]: (msg: Message) => `U${y}Q${(msg.names as string[]).length > 0 ? y + (msg.names as string[]).join(y) : '' }${x}`, [UA.PRESENCE_JOIN.BYTE]: (msg: Message) => `U${y}PNJ${y}${msg.name}${x}`, [UA.PRESENCE_JOIN_ALL.BYTE]: (msg: Message) => `U${y}PNJ${y}${msg.name}${x}`, [UA.PRESENCE_LEAVE.BYTE]: (msg: Message) => `U${y}PNL${y}${msg.name}${x}`, [UA.PRESENCE_LEAVE_ALL.BYTE]: (msg: Message) => `U${y}PNL${y}${msg.name}${x}`, [UA.INVALID_PRESENCE_USERS.BYTE]: (msg: Message) => `U${y}E${y}INVALID_PRESENCE_USERS${y}${msg.data}${x}`, [UA.MESSAGE_DENIED.BYTE]: messageDenied, [UA.MESSAGE_PERMISSION_ERROR.BYTE]: messagePermissionError, [UA.NOT_SUBSCRIBED.BYTE]: notSubscribed, [UA.MULTIPLE_SUBSCRIPTIONS.BYTE]: multipleSubscriptions, }, } /** * Creates a deepstream message string, based on the * provided parameters */ export const getMessage = (message: Message, isAck: boolean = false): string => { if (!BUILDERS[message.topic] || !BUILDERS[message.topic][message.action]) { console.trace('missing builder for', message, isAck) return '' } const builder = BUILDERS[message.topic][message.action] if ( !message.parsedData && !message.data && ( (message.topic === TOPIC.RPC.BYTE && (message.action === PA.RESPONSE.BYTE || message.action === PA.REQUEST.BYTE)) || (message.topic === TOPIC.RECORD.BYTE && (message.action === RA.PATCH.BYTE || message.action === RA.ERASE.BYTE)) ) ) { message.data = 'U' } else if (message.parsedData) { if (ABP[message.topic][message.action] === PAYLOAD_ENCODING.DEEPSTREAM) { message.data = typed(message.parsedData) } else { message.data = JSON.stringify(message.parsedData) } } else if (message.data && ABP[message.topic][message.action] === PAYLOAD_ENCODING.DEEPSTREAM) { message.data = typed(JSON.parse(message.data.toString())) } return builder(message, isAck) } /** * Converts a serializable value into its string-representation and adds * a flag that provides instructions on how to deserialize it. * * Please see messageParser.convertTyped for the counterpart of this method */ export const typed = function (value: any): string { const type = typeof value if (type === 'string') { return TYPES.STRING + value } if (value === null) { return TYPES.NULL } if (type === 'object') { return TYPES.OBJECT + JSON.stringify(value) } if (type === 'number') { return TYPES.NUMBER + value.toString() } if (value === true) { return TYPES.TRUE } if (value === false) { return TYPES.FALSE } if (value === undefined) { return TYPES.UNDEFINED } throw new Error(`Can't serialize type ${value}`) } ================================================ FILE: src/connection-endpoint/websocket/text/text-protocol/message-parser.ts ================================================ import { ACTIONS_BYTE_TO_PAYLOAD as ABP, ACTIONS_TEXT_TO_BYTE, AUTH_ACTIONS as AA, CONNECTION_ACTIONS as CA, DEEPSTREAM_TYPES as TYPES, EVENT_ACTIONS as EA, MESSAGE_PART_SEPERATOR, MESSAGE_SEPERATOR, PRESENCE_ACTIONS as UA, RECORD_ACTIONS as RA, RPC_ACTIONS as PA, TOPIC, TOPIC_TEXT_TO_BYTE, PRESENCE_ACTIONS, PAYLOAD_ENCODING, } from './constants' import { Message } from '../../../../constants' import { getUid } from '../../../../utils/utils' export const correlationIdToVersion = new Map() export const bulkNameToCorrelationId = new Map() /** * This method tries to parse a value, and returns * an object containing the value or error. * * This is an optimization to avoid doing try/catch * inline since it incurs a massive performance hit * in most versions of node. */ function parseJSON (text: string, reviver?: any): any { try { return { value: JSON.parse(text, reviver), } } catch (err) { return { error: err, } } } const writeConfig = JSON.stringify({ writeSuccess: true }) export const parse = (rawMessage: string) => { const parsedMessages: any[] = [] const rawMessages = rawMessage.split(MESSAGE_SEPERATOR) for (let i = 0; i < rawMessages.length; i++) { if (rawMessages[i].length < 3) { continue } const parts = rawMessages[i].split(MESSAGE_PART_SEPERATOR) const topic = TOPIC_TEXT_TO_BYTE[parts[0]] if (topic === undefined) { console.log('unknown topic', rawMessages[i]) // invalid topic continue } let index: number = 1 let name let names let data let version let path let isWriteAck let subscription let correlationId let isAck = false let isErr = false if (parts[index] === 'A') { isAck = true index++ } if (parts[index] === 'E') { isErr = true index++ } const A = ACTIONS_TEXT_TO_BYTE[topic] const rawAction = parts[index++] let action = A[rawAction] if (action === undefined) { if ( (isErr && topic === TOPIC.RPC.BYTE) || (topic === TOPIC.CONNECTION.BYTE && isAck) || (topic === TOPIC.AUTH.BYTE && (isErr || isAck)) || (isErr && topic === TOPIC.RECORD.BYTE) ) { // ignore } else { console.log('unknown action', parts[index - 1], rawMessages[i]) continue } } if (topic === TOPIC.RECORD.BYTE) { /************************ *** RECORD *************************/ name = parts[index++] names = [name] if (isErr) { isErr = false if (rawAction === 'VERSION_EXISTS') { action = RA.VERSION_EXISTS.BYTE version = (parts[index++] as unknown as number) * 1 data = parts[index++] isWriteAck = parts.length - index > 1 } else if (rawAction === 'CACHE_RETRIEVAL_TIMEOUT') { action = RA.CACHE_RETRIEVAL_TIMEOUT.BYTE } else if (rawAction === 'STORAGE_RETRIEVAL_TIMEOUT') { action = RA.STORAGE_RETRIEVAL_TIMEOUT.BYTE } } else if ( action === RA.CREATEANDUPDATE.BYTE || action === RA.UPDATE.BYTE || action === RA.PATCH.BYTE ) { isWriteAck = (parts[parts.length - 1] === writeConfig) version = (parts[index++] as unknown as number) * 1 if (action === RA.CREATEANDUPDATE.BYTE && parts.length === 7) { action = RA.CREATEANDPATCH.BYTE } if (action === RA.CREATEANDPATCH.BYTE || action === RA.PATCH.BYTE) { path = parts[index++] } if (parts.length - index === 2) { data = parts[parts.length - 2] } else { data = parts[index++] } } else if ( action === RA.LISTEN_ACCEPT.BYTE || action === RA.LISTEN_REJECT.BYTE || action === RA.SUBSCRIPTION_FOR_PATTERN_FOUND.BYTE || action === RA.SUBSCRIPTION_FOR_PATTERN_REMOVED.BYTE ) { subscription = parts[index++] } else if (action === RA.SUBSCRIPTION_HAS_PROVIDER.BYTE) { if (parts[index++] === 'F') { action = RA.SUBSCRIPTION_HAS_NO_PROVIDER.BYTE } } } else if (topic === TOPIC.EVENT.BYTE) { /************************ *** EVENT *************************/ name = parts[index++] names = [name] if ( action === EA.LISTEN_ACCEPT.BYTE || action === EA.LISTEN_REJECT.BYTE || action === EA.SUBSCRIPTION_FOR_PATTERN_FOUND.BYTE || action === EA.SUBSCRIPTION_FOR_PATTERN_REMOVED.BYTE ) { subscription = parts[index++] } else if (action === EA.EMIT.BYTE) { data = parts[index++] } } else if (topic === TOPIC.RPC.BYTE) { /************************ *** RPC *************************/ name = parts[index++] names = [name] if (isAck && action === PA.REQUEST.BYTE) { isAck = false action = PA.ACCEPT.BYTE } if (isErr) { isErr = false action = PA.REQUEST_ERROR.BYTE data = JSON.stringify(rawAction) } if (action !== PA.PROVIDE.BYTE && action !== PA.UNPROVIDE.BYTE) { correlationId = parts[index++] } if (action === PA.RESPONSE.BYTE || action === PA.REQUEST.BYTE) { data = parts[index++] } } else if (topic === TOPIC.PRESENCE.BYTE) { /************************ *** Presence *************************/ if (action === UA.QUERY.BYTE) { if (parts.length === 3) { action = UA.QUERY_ALL.BYTE } else { correlationId = parts[index++] names = JSON.parse(parts[index++]) } } else if (action === UA.SUBSCRIBE.BYTE || action === UA.UNSUBSCRIBE.BYTE) { if (parts.length === 4 && !isAck) { correlationId = parts[index++] } data = parts[index++] if (action === UA.SUBSCRIBE.BYTE && (data === 'S' || data === 'U')) { action = PRESENCE_ACTIONS.SUBSCRIBE_ALL.BYTE } else if (action === UA.UNSUBSCRIBE.BYTE && data === 'US') { action = PRESENCE_ACTIONS.UNSUBSCRIBE_ALL.BYTE } else { names = JSON.parse(data) } } } else if (topic === TOPIC.CONNECTION.BYTE) { /************************ *** Connection *************************/ if (action === CA.PONG.BYTE) { continue } if (isAck) { action = CA.ACCEPT.BYTE isAck = false } else if (action === CA.REDIRECT.BYTE || action === CA.REJECTION.BYTE) { data = parts[index++] } } else if (topic === TOPIC.AUTH.BYTE) { /************************ *** Authentication *************************/ if (isAck) { action = AA.AUTH_SUCCESSFUL.BYTE } else if (isErr) { if (rawAction === 'INVALID_AUTH_DATA') { isErr = false action = AA.AUTH_UNSUCCESSFUL.BYTE } else if (rawAction === 'TOO_MANY_AUTH_ATTEMPTS') { isErr = false action = AA.TOO_MANY_AUTH_ATTEMPTS.BYTE } } if (action === AA.AUTH_SUCCESSFUL.BYTE) { isAck = false data = rawAction } else if (action === AA.REQUEST.BYTE || action === AA.AUTH_UNSUCCESSFUL.BYTE) { data = parts[index++] } } if (names && !correlationId && version !== undefined) { correlationId = getUid() correlationIdToVersion.set(correlationId, version) } else if (names && !correlationId) { correlationId = getUid() bulkNameToCorrelationId.set(correlationId, names[0]) } const message = JSON.parse(JSON.stringify({ isAck, isErr, topic, action, name, names, data, // rpc / presence query correlationId, // subscription by listening subscription, // record path, version, // parsedData: null, isWriteAck, })) parseData(message) parsedMessages.push(message) } return parsedMessages } export const parseData = (message: Message) => { if (message.parsedData || !message.data) { return true } if ( message.topic === TOPIC.PRESENCE.BYTE && ( message.action === PRESENCE_ACTIONS.SUBSCRIBE_ALL.BYTE || message.action === PRESENCE_ACTIONS.UNSUBSCRIBE_ALL.BYTE ) ) { // @ts-ignore message.parsedData = message.data message.data = JSON.stringify(message.data) } if (ABP[message.topic][message.action] === PAYLOAD_ENCODING.DEEPSTREAM) { const parsedData = convertTyped(message.data as string) if (parsedData instanceof Error) { return parsedData } message.parsedData = parsedData message.data = JSON.stringify(message.parsedData) return true } else { const res = parseJSON((message.data as string)!) if (res.error) { return res.error } message.parsedData = res.value return true } } /** * Deserializes values created by MessageBuilder.typed to * their original format */ export const convertTyped = (value: string): any => { if (value === 'null') { return null } if (value === undefined) { return undefined } const type = value.charAt(0) if (type === TYPES.STRING) { return value.substr(1) } if (type === TYPES.OBJECT) { const result = parseJSON(value.substr(1)) if (result.value) { return result.value } return result.error } if (type === TYPES.NUMBER) { return parseFloat(value.substr(1)) } if (type === TYPES.NULL || type === 'null') { return null } if (type === TYPES.TRUE) { return true } if (type === TYPES.FALSE) { return false } if (type === TYPES.UNDEFINED) { return undefined } return new Error('Unknown type') } export const isError = (message: any) => false ================================================ FILE: src/connection-endpoint/websocket/text/text-protocol/utils.ts ================================================ export const isWriteAck = (action: any) => false export const WRITE_ACK_TO_ACTION = {} ================================================ FILE: src/constants.ts ================================================ export * from '@deepstream/protobuf/dist/types/all' export * from '@deepstream/protobuf/dist/types/messages' export enum STATES { CONFIG_LOADED = 'CONFIG_LOADED', LOGGER_INIT = 'LOGGER_INIT', SERVICE_INIT = 'SERVICE_INIT', HANDLER_INIT = 'HANDLER_INIT', CONNECTION_ENDPOINT_INIT = 'CONNECTION_ENDPOINT_INIT', PLUGIN_INIT = 'PLUGIN_INIT', RUNNING = 'RUNNING', PLUGIN_SHUTDOWN = 'PLUGIN_SHUTDOWN', CONNECTION_ENDPOINT_SHUTDOWN = 'CONNECTION_ENDPOINT_SHUTDOWN', HANDLER_SHUTDOWN = 'HANDLER_SHUTDOWN', SERVICE_SHUTDOWN = 'SERVICE_SHUTDOWN', LOGGER_SHUTDOWN = 'LOGGER_SHUTDOWN', STOPPED = 'STOPPED' } ================================================ FILE: src/deepstream.io.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import { Deepstream } from './deepstream.io' import { PromiseDelay } from './utils/utils'; describe('deepstream.io', () => { describe('the main server class', () => { it('sets a supported option', () => { const server = new Deepstream() expect(() => { server.set('serverName', 'my lovely horse') }).not.to.throw() }) it.skip('sets an unsupported option', async () => { const server = new Deepstream() await PromiseDelay(50) expect(() => { server.set('gibberish', 4444) }).to.throw() }) }) }) ================================================ FILE: src/deepstream.io.ts ================================================ require('source-map-support').install() import { EventEmitter } from 'events' import * as pkg from '../package.json' import { merge } from './utils/utils' import { STATES, TOPIC } from './constants' import MessageProcessor from './utils/message-processor' import MessageDistributor from './utils/message-distributor' import EventHandler from './handlers/event/event-handler' import RpcHandler from './handlers/rpc/rpc-handler' import PresenceHandler from './handlers/presence/presence-handler' import MonitoringHandler from './handlers/monitoring/monitoring' import { get as getDefaultOptions } from './default-options' import * as configInitializer from './config/config-initialiser' import * as jsYamlLoader from './config/js-yaml-loader' import { DependencyInitialiser } from './utils/dependency-initialiser' import { DeepstreamConfig, DeepstreamServices, DeepstreamPlugin, PartialDeepstreamConfig, EVENT, SocketWrapper, ConnectionListener } from '@deepstream/types' import RecordHandler from './handlers/record/record-handler' import { getValue, setValue } from './utils/json-path' import { CombineAuthentication } from './services/authentication/combine/combine-authentication' /** * Sets the name of the process */ process.title = 'deepstream server' export class Deepstream extends EventEmitter { public constants: any private configFile!: string private config!: DeepstreamConfig private services!: DeepstreamServices private messageProcessor: any private messageDistributor: any private eventHandler!: EventHandler private rpcHandler!: RpcHandler private recordHandler!: RecordHandler private presenceHandler!: PresenceHandler private monitoringHandler!: MonitoringHandler private connectionListeners = new Set() private stateMachine: any private currentState: STATES private overrideSettings: Array<{key: string, value: any}> = [] private startWhenLoaded: boolean = false /** * Deepstream is a realtime data server that supports data-sync, * publish-subscribe, request-response, listening, permissions * and a host of other features! */ constructor (config: PartialDeepstreamConfig | string | null = null) { super() this.stateMachine = { init: STATES.STOPPED, transitions: [ { name: 'loading-config', from: STATES.STOPPED, to: STATES.CONFIG_LOADED, handler: this.configLoaded }, { name: 'start', from: STATES.CONFIG_LOADED, to: STATES.LOGGER_INIT, handler: this.loggerInit }, { name: 'logger-started', from: STATES.LOGGER_INIT, to: STATES.SERVICE_INIT, handler: this.serviceInit }, { name: 'services-started', from: STATES.SERVICE_INIT, to: STATES.HANDLER_INIT, handler: this.handlerInit }, { name: 'handlers-started', from: STATES.HANDLER_INIT, to: STATES.PLUGIN_INIT, handler: this.pluginsInit }, { name: 'plugins-started', from: STATES.PLUGIN_INIT, to: STATES.CONNECTION_ENDPOINT_INIT, handler: this.connectionEndpointInit }, { name: 'connection-endpoints-started', from: STATES.CONNECTION_ENDPOINT_INIT, to: STATES.RUNNING, handler: this.run }, { name: 'stop', from: STATES.LOGGER_INIT, to: STATES.LOGGER_SHUTDOWN, handler: this.loggerShutdown }, { name: 'stop', from: STATES.SERVICE_INIT, to: STATES.SERVICE_SHUTDOWN, handler: this.serviceShutdown }, { name: 'stop', from: STATES.CONNECTION_ENDPOINT_INIT, to: STATES.CONNECTION_ENDPOINT_SHUTDOWN, handler: this.connectionEndpointShutdown }, { name: 'stop', from: STATES.PLUGIN_INIT, to: STATES.PLUGIN_SHUTDOWN, handler: this.pluginsShutdown }, { name: 'stop', from: STATES.RUNNING, to: STATES.CONNECTION_ENDPOINT_SHUTDOWN, handler: this.connectionEndpointShutdown }, { name: 'connection-endpoints-closed', from: STATES.CONNECTION_ENDPOINT_SHUTDOWN, to: STATES.PLUGIN_SHUTDOWN, handler: this.pluginsShutdown }, { name: 'plugins-closed', from: STATES.PLUGIN_SHUTDOWN, to: STATES.HANDLER_SHUTDOWN, handler: this.handlerShutdown }, { name: 'handlers-closed', from: STATES.HANDLER_SHUTDOWN, to: STATES.SERVICE_SHUTDOWN, handler: this.serviceShutdown }, { name: 'services-closed', from: STATES.SERVICE_SHUTDOWN, to: STATES.LOGGER_SHUTDOWN, handler: this.loggerShutdown }, { name: 'logger-closed', from: STATES.LOGGER_SHUTDOWN, to: STATES.STOPPED, handler: this.stopped }, ] } this.currentState = this.stateMachine.init this.loadConfig(config) this.messageProcessor = null this.messageDistributor = null } /** * Set a deepstream option. For a list of all available options * please see default-options. */ public set (key: string, value: any): any { if (this.currentState === STATES.STOPPED) { this.overrideSettings.push({ key, value }) return } if (key === 'storageExclusion') { throw new Error('storageExclusion has been replace with record.storageExclusionPrefixes instead, which is an array of prefixes') } if (key === 'auth') { throw new Error('auth has been replaced with authentication') } if (key === 'authentication') { this.services.authentication = new CombineAuthentication(value instanceof Array ? value : [value]) return } if ((this.services as any)[key] !== undefined) { (this.services as any)[key] = value } else if (getValue(this.config, key) !== undefined) { setValue(this.config, key, value) } else { throw new Error(`Unknown option or service "${key}"`) } return this } /** * Returns true if the deepstream server is running, otherwise false */ public isRunning (): boolean { return this.currentState === STATES.RUNNING } /** * Starts up deepstream. The startup process has three steps: * * - First of all initialize the logger and wait for it (ready event) * - Then initialize all other dependencies (cache connector, message connector, storage connector) * - Instantiate the messaging pipeline and record-, rpc- and event-handler * - Start WS server */ public start (): void { if (this.currentState !== STATES.CONFIG_LOADED) { this.startWhenLoaded = true return } this.transition('start') } /** * Stops the server and closes all connections. Will emit a 'stopped' event once done */ public stop (): void { if (this.currentState === STATES.STOPPED) { throw new Error('The server is already stopped.') } if ([STATES.CONNECTION_ENDPOINT_SHUTDOWN, STATES.SERVICE_SHUTDOWN, STATES.PLUGIN_SHUTDOWN, STATES.LOGGER_SHUTDOWN].indexOf(this.currentState) !== -1) { this.services.logger.info(EVENT.INFO, `Server is currently shutting down, currently in state ${STATES[this.currentState]}`) return } this.transition('stop') } public getServices (): Readonly { return this.services } public getConfig (): Readonly { return this.config } /* ======================================================================= * * ========================== State Transitions ========================== * * ======================================================================= */ /** * Try to perform a state change */ private transition (transitionName: string): void { let transition for (let i = 0; i < this.stateMachine.transitions.length; i++) { transition = this.stateMachine.transitions[i] if (transitionName === transition.name && this.currentState === transition.from) { // found transition this.onTransition(transition) this.currentState = transition.to transition.handler.call(this) this.emit(EVENT.DEEPSTREAM_STATE_CHANGED, this.currentState) return } } const details = JSON.stringify({ transition: transitionName, state: this.currentState }) throw new Error(`Invalid state transition: ${details}`) } /** * Log state transitions for debugging. */ private onTransition (transition: { from: STATES, to: STATES, name: string }): void { const logger = this.services.logger if (logger && STATES[transition.to] !== STATES.CONFIG_LOADED) { logger.debug( EVENT.INFO, `State transition (${transition.name}): ${STATES[transition.from]} -> ${STATES[transition.to]}` ) } } private configLoaded (): void { if (this.startWhenLoaded) { this.overrideSettings.forEach((setting) => this.set(setting.key, setting.value)) this.start() } } /** * First stage in the Deepstream initialization sequence. Initialises the logger. */ private async loggerInit (): Promise { const logger = this.services.logger const loggerInitialiser = new DependencyInitialiser(this.config, this.services, logger, 'logger') await loggerInitialiser.whenReady() const infoLogger = (message: string) => this.services.logger.info(EVENT.INFO, message) infoLogger(`server name: ${this.config.serverName}`) infoLogger(`deepstream version: ${pkg.version}`) // otherwise (no configFile) deepstream was invoked by API if (this.configFile != null) { infoLogger(`configuration file loaded from ${this.configFile}`) } // @ts-ignore if (global.deepstreamLibDir) { // @ts-ignore infoLogger(`library directory set to: ${global.deepstreamLibDir}`) } this.transition('logger-started') } /** * Invoked once the logger is initialised. Initialises all deepstream services. */ private async serviceInit () { const readyPromises = Object.keys(this.services).reduce((promises, serviceName) => { if (['connectionEndpoints', 'plugins', 'notifyFatalException', 'logger'].includes(serviceName)) { return promises } const service = (this.services as any)[serviceName] as DeepstreamPlugin const initialiser = new DependencyInitialiser(this.config, this.services, service, serviceName) promises.push(initialiser.whenReady()) return promises }, [] as Array>) await Promise.all(readyPromises) this.messageProcessor = new MessageProcessor(this.config, this.services) this.messageDistributor = new MessageDistributor(this.config, this.services) this.services.messageDistributor = this.messageDistributor this.transition('services-started') } /** * Invoked once all plugins are initialised. Instantiates the messaging pipeline and * the various handlers. */ private async handlerInit () { if (this.config.enabledFeatures.event) { this.eventHandler = new EventHandler(this.config, this.services) this.messageDistributor.registerForTopic( TOPIC.EVENT, this.eventHandler.handle.bind(this.eventHandler) ) } if (this.config.enabledFeatures.rpc) { this.rpcHandler = new RpcHandler(this.config, this.services) this.messageDistributor.registerForTopic( TOPIC.RPC, this.rpcHandler.handle.bind(this.rpcHandler) ) } if (this.config.enabledFeatures.record) { this.recordHandler = new RecordHandler(this.config, this.services) this.messageDistributor.registerForTopic( TOPIC.RECORD, this.recordHandler.handle.bind(this.recordHandler) ) } if (this.config.enabledFeatures.presence) { this.presenceHandler = new PresenceHandler(this.config, this.services) this.messageDistributor.registerForTopic( TOPIC.PRESENCE, this.presenceHandler.handle.bind(this.presenceHandler) ) this.connectionListeners.add(this.presenceHandler as ConnectionListener) } if (this.config.enabledFeatures.monitoring) { this.monitoringHandler = new MonitoringHandler(this.config, this.services) this.messageDistributor.registerForTopic( TOPIC.MONITORING, this.monitoringHandler.handle.bind(this.monitoringHandler) ) } this.messageProcessor.onAuthenticatedMessage = this.messageDistributor.distribute.bind(this.messageDistributor) if (this.services.permission.setRecordHandler) { this.services.permission.setRecordHandler(this.recordHandler) } this.transition('handlers-started') } private async pluginsInit () { const readyPromises = Object.keys(this.services.plugins).reduce((promises, pluginName) => { const plugin = this.services.plugins[pluginName] if (isConnectionListener(plugin)) { this.connectionListeners.add(plugin) } const initialiser = new DependencyInitialiser(this.config, this.services, plugin, pluginName) promises.push(initialiser.whenReady()) return promises }, [] as Array>) await Promise.all(readyPromises) this.transition('plugins-started') } /** * Invoked once all dependencies and services are initialised. * The startup sequence will be complete once the connection endpoint is started and listening. */ private async connectionEndpointInit (): Promise { const endpoints = this.services.connectionEndpoints const readyPromises: Array> = [] for (let i = 0; i < endpoints.length; i++) { const connectionEndpoint = endpoints[i] const dependencyInitialiser = new DependencyInitialiser( this.config, this.services, connectionEndpoint, 'connectionEndpoint' ) connectionEndpoint.onMessages = this.messageProcessor.process.bind(this.messageProcessor) if (connectionEndpoint.setConnectionListener) { connectionEndpoint.setConnectionListener({ onClientConnected: this.onClientConnected.bind(this), onClientDisconnected: this.onClientDisconnected.bind(this) }) } readyPromises.push(dependencyInitialiser.whenReady()) } await Promise.all(readyPromises) this.transition('connection-endpoints-started') } /** * Initialization complete - Deepstream is up and running. */ private run (): void { this.services.logger.info(EVENT.INFO, 'Deepstream started') this.emit('started') } /** * Close any (perhaps partially initialised) plugins. */ private async pluginsShutdown () { const shutdownPromises = Object.keys(this.services.plugins).reduce((promises, pluginName) => { const plugin = this.services.plugins[pluginName] if (plugin.close) { promises.push(plugin.close()) } return promises }, [] as Array> ) await Promise.all(shutdownPromises) this.transition('plugins-closed') } /** * Begin deepstream shutdown. * Closes the (perhaps partially initialised) connectionEndpoints. */ private async connectionEndpointShutdown (): Promise { const closeCallbacks = this.services.connectionEndpoints.map((endpoint) => endpoint.close()) await Promise.all(closeCallbacks) this.transition('connection-endpoints-closed') } private async handlerShutdown () { if (this.config.enabledFeatures.event) { await this.eventHandler.close() } if (this.config.enabledFeatures.rpc) { await this.rpcHandler.close() } if (this.config.enabledFeatures.record) { await this.recordHandler.close() } if (this.config.enabledFeatures.presence) { await this.presenceHandler.close() } if (this.config.enabledFeatures.monitoring) { await this.monitoringHandler.close() } this.transition('handlers-closed') } /** * Shutdown the services. */ private async serviceShutdown (): Promise { const shutdownPromises = Object.keys(this.services).reduce((promises, serviceName) => { const service = (this.services as any)[serviceName] if (service.close) { promises.push(service.close()) } return promises }, [] as Array> ) await Promise.all(shutdownPromises) this.transition('services-closed') } /** * Close the (perhaps partially initialised) logger. */ private async loggerShutdown () { const logger = this.services.logger as any await logger.close() this.transition('logger-closed') } /** * Final stop state. * Deepstream can now be started again. */ private stopped (): void { this.emit('stopped') } /** * Synchronously loads a configuration file * Initialization of plugins and logger will be triggered by the * configInitialiser, but it should not block. Instead the ready events of * those plugins are handled through the DependencyInitialiser in this instance. */ private async loadConfig (config: PartialDeepstreamConfig | string | null): Promise { let result if (config === null || typeof config === 'string') { result = await jsYamlLoader.loadConfig(this, config) this.configFile = result.file } else { configInitializer.mergeConnectionOptions(config) const rawConfig = merge(getDefaultOptions(), config) as DeepstreamConfig result = configInitializer.initialize(this, rawConfig) } this.config = result.config this.services = result.services this.transition('loading-config') } private onClientConnected (socketWrapper: SocketWrapper): void { this.connectionListeners.forEach((connectionListener) => connectionListener.onClientConnected(socketWrapper)) } private onClientDisconnected (socketWrapper: SocketWrapper): void { this.connectionListeners.forEach((connectionListener) => connectionListener.onClientDisconnected(socketWrapper)) } } function isConnectionListener (object: any): object is ConnectionListener { return 'onClientConnected' in object && 'onClientDisconnected' in object } export default Deepstream ================================================ FILE: src/default-options.ts ================================================ import { getUid } from './utils/utils' import { DeepstreamConfig, LOG_LEVEL } from '@deepstream/types' const WebSocketDefaultOptions = { urlPath: '/deepstream', heartbeatInterval: 30000, outgoingBufferTimeout: 0, maxBufferByteSize: 100000, headers: [], /* * Security */ unauthenticatedClientTimeout: 180000, maxAuthAttempts: 3, logInvalidAuthData: false, maxMessageSize: 1048576 } export function get (): DeepstreamConfig { return { /* * General */ libDir: null, serverName: getUid(), showLogo: false, logLevel: LOG_LEVEL.INFO, dependencyInitializationTimeout: 2000, // defaults to false as the event is captured via commander when run via binary or standalone exitOnFatalError: false, /* * Connection Endpoints */ connectionEndpoints: [ { type: 'ws-binary', options: { ...WebSocketDefaultOptions, urlPath: '/deepstream' } }, { type: 'ws-text', options: { ...WebSocketDefaultOptions, urlPath: '/deepstream-v3' } }, { type: 'ws-json', options: { ...WebSocketDefaultOptions, urlPath: '/deepstream-json' } }, { type: 'http', options: { allowAuthData: true, enableAuthEndpoint: true, authPath: '/api/auth', postPath: '/api', getPath: '/api' } }, { type: 'mqtt', options: { port: 1883, host: '0.0.0.0', idleTimeout: 180000, /* * Security */ unauthenticatedClientTimeout: 180000, } } ], logger: { type: 'default', options: {} }, httpServer: { type: 'default', options: { host: '0.0.0.0', port: 6020, healthCheckPath: '/health-check', allowAllOrigins: true, origins: [], headers: [], maxMessageSize: 1048576 } }, subscriptions: { type: 'default', options: { subscriptionsSanityTimer: 10000 } }, auth: [{ type: 'none', options: {} }], permission: { type: 'none', options: { maxRuleIterations: 3, cacheEvacuationInterval: 60000 } }, cache: { type: 'default', options: {} }, storage: { type: 'default', options: {} }, monitoring: { type: 'none', options: {} }, telemetry: { type: 'deepstreamIO', options: { enabled: false, } }, locks: { type: 'default', options: { holdTimeout: 1000, requestTimeout: 1000 } }, clusterNode: { type: 'default', options: { } }, clusterRegistry: { type: 'default', options: { keepAliveInterval: 5000, activeCheckInterval: 1000, nodeInactiveTimeout: 6000 } }, clusterStates: { type: 'default', options: { reconciliationTimeout: 500 } }, plugins: { }, rpc: { /** * Don't send requestorName by default. */ provideRequestorName: false, /** * Don't send requestorData by default. */ provideRequestorData: false, ackTimeout: 1000, responseTimeout: 10000, }, record: { storageHotPathPrefixes: [], storageExclusionPrefixes: [], cacheRetrievalTimeout: 1000, storageRetrievalTimeout: 2000, }, listen: { shuffleProviders: true, responseTimeout: 500, rematchInterval: 30000, matchCooldown: 10000 }, enabledFeatures: { record: true, event: true, rpc: true, presence: true, monitoring: false }, } } ================================================ FILE: src/handlers/event/event-handler.spec.ts ================================================ import 'mocha' import * as C from '../../constants' import EventHandler from './event-handler' import * as testHelper from '../../test/helper/test-helper' import { getTestMocks } from '../../test/helper/test-mocks' const options = testHelper.getDeepstreamOptions() const config = options.config const services = options.services describe('the eventHandler routes events correctly', () => { let testMocks let eventHandler let socketWrapper beforeEach(() => { testMocks = getTestMocks() eventHandler = new EventHandler( config, services, testMocks.subscriptionRegistry, testMocks.listenerRegistry ) socketWrapper = testMocks.getSocketWrapper().socketWrapper }) afterEach(() => { testMocks.subscriptionRegistryMock.verify() testMocks.listenerRegistryMock.verify() }) it('subscribes to events', () => { const subscriptionMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.SUBSCRIBE, names: ['someEvent'] } testMocks.subscriptionRegistryMock .expects('subscribeBulk') .once() .withExactArgs(subscriptionMessage, socketWrapper) eventHandler.handle(socketWrapper, subscriptionMessage) }) it('unsubscribes to events', () => { const unSubscriptionMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.UNSUBSCRIBE, names: ['someEvent'] } testMocks.subscriptionRegistryMock .expects('unsubscribeBulk') .once() .withExactArgs(unSubscriptionMessage, socketWrapper) eventHandler.handle(socketWrapper, unSubscriptionMessage) }) it('triggers event without data', () => { const eventMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'someEvent' } testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs('someEvent', eventMessage, false, socketWrapper) eventHandler.handle(socketWrapper, eventMessage) }) it('triggers event with data', () => { const eventMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'someEvent', data: JSON.stringify({ data: 'payload' }) } testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs('someEvent', eventMessage, false, socketWrapper) eventHandler.handle(socketWrapper, eventMessage) }) it('registers a listener', () => { const listenMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.LISTEN, name: 'event/.*' } testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(socketWrapper, listenMessage) eventHandler.handle(socketWrapper, listenMessage) }) it('removes listeners', () => { const unlistenMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.UNLISTEN, name: 'event/.*' } testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(socketWrapper, unlistenMessage) eventHandler.handle(socketWrapper, unlistenMessage) }) it('processes listen accepts', () => { const listenAcceptMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.LISTEN_ACCEPT, name: 'event/.*', subscription: 'event/A' } testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(socketWrapper, listenAcceptMessage) eventHandler.handle(socketWrapper, listenAcceptMessage) }) it('processes listen rejects', () => { const listenRejectMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.LISTEN_REJECT, name: 'event/.*', subscription: 'event/A' } testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(socketWrapper, listenRejectMessage) eventHandler.handle(socketWrapper, listenRejectMessage) }) }) ================================================ FILE: src/handlers/event/event-handler.ts ================================================ import { EVENT_ACTION, TOPIC, EventMessage, ListenMessage, STATE_REGISTRY_TOPIC, BulkSubscriptionMessage } from '../../constants' import { ListenerRegistry } from '../../listen/listener-registry' import { DeepstreamConfig, DeepstreamServices, SocketWrapper, Handler, SubscriptionRegistry, EVENT } from '@deepstream/types' export default class EventHandler implements Handler { private subscriptionRegistry: SubscriptionRegistry private listenerRegistry: ListenerRegistry /** * Handles incoming and outgoing messages for the EVENT topic. */ constructor (config: DeepstreamConfig, private services: DeepstreamServices, subscriptionRegistry?: SubscriptionRegistry, listenerRegistry?: ListenerRegistry) { this.subscriptionRegistry = subscriptionRegistry || services.subscriptions.getSubscriptionRegistry(TOPIC.EVENT, STATE_REGISTRY_TOPIC.EVENT_SUBSCRIPTIONS) this.listenerRegistry = listenerRegistry || new ListenerRegistry(TOPIC.EVENT, config, services, this.subscriptionRegistry, null) this.subscriptionRegistry.setSubscriptionListener(this.listenerRegistry) } public async close () { this.listenerRegistry.close() } /** * The main distribution method. Routes messages to functions * based on the provided action parameter of the message */ public handle (socketWrapper: SocketWrapper | null, message: EventMessage) { if (message.action === EVENT_ACTION.EMIT) { this.triggerEvent(socketWrapper, message) return } if (socketWrapper === null) { this.services.logger.error(EVENT.ERROR, 'missing socket wrapper') return } if (message.action === EVENT_ACTION.SUBSCRIBE) { this.subscriptionRegistry.subscribeBulk(message as BulkSubscriptionMessage, socketWrapper) return } if (message.action === EVENT_ACTION.UNSUBSCRIBE) { this.subscriptionRegistry.unsubscribeBulk(message as BulkSubscriptionMessage, socketWrapper) return } if (message.action === EVENT_ACTION.LISTEN || message.action === EVENT_ACTION.UNLISTEN || message.action === EVENT_ACTION.LISTEN_ACCEPT || message.action === EVENT_ACTION.LISTEN_REJECT) { this.listenerRegistry.handle(socketWrapper, message as ListenMessage) return } console.log('unknown action', message) } /** * Notifies subscribers of events. This method is invoked for the EVENT action. It can * be triggered by messages coming in from both clients and the message connector. */ public triggerEvent (socket: SocketWrapper | null, message: EventMessage) { this.services.logger.debug(EVENT_ACTION[EVENT_ACTION.EMIT], `event: ${message.name} with data: ${message.data || message.parsedData}`) this.subscriptionRegistry.sendToSubscribers(message.name, message, false, socket) } } ================================================ FILE: src/handlers/monitoring/monitoring.ts ================================================ import { DeepstreamConfig, DeepstreamServices, SubscriptionRegistry, SocketWrapper, Handler } from '@deepstream/types' import { MonitoringMessage } from '../../constants' export default class MonitoringHandler extends Handler { // private subscriptionRegistry: SubscriptionRegistry /** * Handles incoming and outgoing messages for the EVENT topic. */ constructor (config: DeepstreamConfig, services: DeepstreamServices, subscriptionRegistry?: SubscriptionRegistry) { super() // this.subscriptionRegistry = // subscriptionRegistry || services.subscriptions.getSubscriptionRegistry(TOPIC.MONITORING, TOPIC.MONITORING_SUBSCRIPTIONS) } /** * The main distribution method. Routes messages to functions * based on the provided action parameter of the message */ public handle (socket: SocketWrapper, message: MonitoringMessage) { console.log('unknown action', message) } } ================================================ FILE: src/handlers/presence/presence-handler.spec.ts ================================================ import 'mocha' import PresenceHandler from './presence-handler' const EVERYONE = '%_EVERYONE_%' import * as C from '../../constants' import * as testHelper from '../../test/helper/test-helper' import { getTestMocks } from '../../test/helper/test-mocks' import { PresenceMessage } from '../../../../client/dist/constants' const { config, services } = testHelper.getDeepstreamOptions() describe('presence handler', () => { let testMocks let presenceHandler: PresenceHandler let userOne beforeEach(() => { testMocks = getTestMocks() presenceHandler = new PresenceHandler( config, services, testMocks.subscriptionRegistry, testMocks.stateRegistry ) userOne = testMocks.getSocketWrapper('Marge') }) afterEach(() => { testMocks.subscriptionRegistryMock.verify() testMocks.listenerRegistryMock.verify() userOne.socketWrapperMock.verify() }) it('subscribes to client logins and logouts', () => { const subscriptionMessage = { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.SUBSCRIBE_ALL, } as PresenceMessage testMocks.subscriptionRegistryMock .expects('subscribe') .once() .withExactArgs(EVERYONE, { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.SUBSCRIBE_ALL, name: EVERYONE }, userOne.socketWrapper, true) presenceHandler.handle(userOne.socketWrapper, subscriptionMessage) }) it('unsubscribes to client logins and logouts', () => { const unsubscriptionMessage = { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.UNSUBSCRIBE_ALL, } as PresenceMessage testMocks.subscriptionRegistryMock .expects('unsubscribe') .once() .withExactArgs(EVERYONE, { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.UNSUBSCRIBE_ALL, name: EVERYONE }, userOne.socketWrapper, true) presenceHandler.handle(userOne.socketWrapper, unsubscriptionMessage) }) it('does not return own name when queried and only user', () => { const queryMessage = { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.QUERY_ALL } as PresenceMessage testMocks.stateRegistryMock .expects('getAll') .once() .withExactArgs() .returns(['Marge']) userOne.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.QUERY_ALL_RESPONSE, names: [] }) presenceHandler.handle(userOne.socketWrapper, queryMessage) }) it('client joining gets added to state registry', () => { testMocks.stateRegistryMock .expects('add') .once() .withExactArgs(userOne.socketWrapper.userId) presenceHandler.onClientConnected(userOne.socketWrapper) }) it('client joining multiple times gets added once state registry', () => { testMocks.stateRegistryMock .expects('add') .once() .withExactArgs(userOne.socketWrapper.userId) presenceHandler.onClientConnected(userOne.socketWrapper) presenceHandler.onClientConnected(userOne.socketWrapper) }) it('a duplicate client logs out does not remove from state', () => { testMocks.stateRegistryMock .expects('add') .once() .withExactArgs(userOne.socketWrapper.userId) testMocks.stateRegistryMock .expects('remove') .never() presenceHandler.onClientConnected(userOne.socketWrapper) presenceHandler.onClientConnected(userOne.socketWrapper) presenceHandler.onClientDisconnected(userOne.socketWrapper) }) it('a client logging out removes from state', () => { testMocks.stateRegistryMock .expects('add') .once() .withExactArgs(userOne.socketWrapper.userId) testMocks.stateRegistryMock .expects('remove') .once() .withExactArgs(userOne.socketWrapper.userId) presenceHandler.onClientConnected(userOne.socketWrapper) presenceHandler.onClientDisconnected(userOne.socketWrapper) }) it('returns one user when queried', () => { const queryMessage = { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.QUERY_ALL, } as PresenceMessage testMocks.stateRegistryMock .expects('getAll') .once() .withExactArgs() .returns(['Bart']) userOne.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.QUERY_ALL_RESPONSE, names: ['Bart'] }) presenceHandler.handle(userOne.socketWrapper, queryMessage) }) it('returns mutiple user when queried', () => { const queryMessage = { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.QUERY_ALL } as PresenceMessage testMocks.stateRegistryMock .expects('getAll') .once() .withExactArgs() .returns(['Bart', 'Homer', 'Maggie']) userOne.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.QUERY_ALL_RESPONSE, names: ['Bart', 'Homer', 'Maggie'] }) presenceHandler.handle(userOne.socketWrapper, queryMessage) }) it.skip('notifies subscribed users when user added to state', () => { testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(EVERYONE, { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.PRESENCE_JOIN_ALL, name: 'Bart' }, false, null, false) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs('Bart', { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.PRESENCE_JOIN, name: 'Bart' }, false, null, false) // This needs extra work testMocks.stateRegistry.add('Bart') }) it.skip('notifies subscribed users when user removed from state', () => { testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(EVERYONE, { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.PRESENCE_LEAVE_ALL, name: 'Bart' }, false, null, false) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs('Bart', { topic: C.TOPIC.PRESENCE, action: C.PRESENCE_ACTION.PRESENCE_LEAVE, name: 'Bart' }, false, null, false) testMocks.stateRegistry.emit('remove', 'Bart') }) }) ================================================ FILE: src/handlers/presence/presence-handler.ts ================================================ import { PARSER_ACTION, PRESENCE_ACTION, TOPIC, PresenceMessage, Message, BulkSubscriptionMessage, STATE_REGISTRY_TOPIC } from '../../constants' import { DeepstreamConfig, DeepstreamServices, SocketWrapper, StateRegistry, Handler, SubscriptionRegistry, ConnectionListener } from '@deepstream/types' import { Dictionary } from 'ts-essentials' const EVERYONE = '%_EVERYONE_%' /** * This class handles incoming and outgoing messages in relation * to deepstream presence. It provides a way to inform clients * who else is logged into deepstream */ export default class PresenceHandler extends Handler implements ConnectionListener { private localClients: Map = new Map() private subscriptionRegistry: SubscriptionRegistry private connectedClients: StateRegistry constructor (config: DeepstreamConfig, private services: DeepstreamServices, subscriptionRegistry?: SubscriptionRegistry, stateRegistry?: StateRegistry, private metaData?: any) { super() this.subscriptionRegistry = subscriptionRegistry || services.subscriptions.getSubscriptionRegistry(TOPIC.PRESENCE, STATE_REGISTRY_TOPIC.PRESENCE_SUBSCRIPTIONS) this.connectedClients = stateRegistry || this.services.clusterStates.getStateRegistry(STATE_REGISTRY_TOPIC.ONLINE_USERS) this.connectedClients.onAdd(this.onClientAdded.bind(this)) this.connectedClients.onRemove(this.onClientRemoved.bind(this)) } /** * The main entry point to the presence handler class. * * Handles subscriptions, unsubscriptions and queries */ public handle (socketWrapper: SocketWrapper, message: PresenceMessage): void { if (message.action === PRESENCE_ACTION.QUERY_ALL) { this.handleQueryAll(message.correlationId, socketWrapper) return } if (message.action === PRESENCE_ACTION.SUBSCRIBE_ALL) { this.subscriptionRegistry.subscribe(EVERYONE, { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.SUBSCRIBE_ALL, name: EVERYONE }, socketWrapper, true) socketWrapper.sendAckMessage({ topic: message.topic, action: message.action }) return } if (message.action === PRESENCE_ACTION.UNSUBSCRIBE_ALL) { this.subscriptionRegistry.unsubscribe(EVERYONE, { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.UNSUBSCRIBE_ALL, name: EVERYONE }, socketWrapper, true) socketWrapper.sendAckMessage({ topic: message.topic, action: message.action }) return } const users = message.names if (!users) { this.services.logger.error( PARSER_ACTION[PARSER_ACTION.INVALID_MESSAGE], `invalid presence names parameter ${PRESENCE_ACTION[message.action]}` ) return } if (message.action === PRESENCE_ACTION.SUBSCRIBE) { this.subscriptionRegistry.subscribeBulk(message as BulkSubscriptionMessage, socketWrapper) return } if (message.action === PRESENCE_ACTION.UNSUBSCRIBE) { this.subscriptionRegistry.unsubscribeBulk(message as BulkSubscriptionMessage, socketWrapper) return } if (message.action === PRESENCE_ACTION.QUERY) { this.handleQuery(users, message.correlationId, socketWrapper) return } this.services.logger.warn(PARSER_ACTION[PARSER_ACTION.UNKNOWN_ACTION], PRESENCE_ACTION[message.action], this.metaData) } /** * Called whenever a client has succesfully logged in with a username */ public onClientConnected (socketWrapper: SocketWrapper): void { if (socketWrapper.userId === 'OPEN') { return } const currentCount = this.localClients.get(socketWrapper.userId) if (currentCount === undefined) { this.localClients.set(socketWrapper.userId, 1) this.connectedClients.add(socketWrapper.userId) } else { this.localClients.set(socketWrapper.userId, currentCount + 1) } } /** * Called whenever a client has disconnected */ public onClientDisconnected (socketWrapper: SocketWrapper): void { if (socketWrapper.userId === 'OPEN') { return } const currentCount = this.localClients.get(socketWrapper.userId) if (!currentCount) { // TODO: Log Error } else if (currentCount === 1) { this.localClients.delete(socketWrapper.userId) this.connectedClients.remove(socketWrapper.userId) } else { this.localClients.set(socketWrapper.userId, currentCount - 1) } } private handleQueryAll (correlationId: string, socketWrapper: SocketWrapper): void { const clients = this.connectedClients.getAll() const index = clients.indexOf(socketWrapper.userId) if (index !== -1) { clients.splice(index, 1) } socketWrapper.sendMessage({ topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.QUERY_ALL_RESPONSE, names: clients }) } /** * Handles finding clients who are connected and splicing out the client * querying for users */ private handleQuery (users: string[], correlationId: string, socketWrapper: SocketWrapper): void { const result: Dictionary = {} const clients = this.connectedClients.getAll() for (let i = 0; i < users.length; i++) { result[users[i]] = clients.includes(users[i]) } socketWrapper.sendMessage({ topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.QUERY_RESPONSE, correlationId, parsedData: result, }) } /** * Alerts all clients who are subscribed to * PRESENCE_JOIN that a new client has been added. */ private onClientAdded (username: string): void { const individualMessage: Message = { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.PRESENCE_JOIN, name : username, } const allMessage: Message = { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.PRESENCE_JOIN_ALL, name: username } this.subscriptionRegistry.sendToSubscribers(EVERYONE, allMessage, false, null, true) this.subscriptionRegistry.sendToSubscribers(username, individualMessage, false, null, true) } /** * Alerts all clients who are subscribed to * PRESENCE_LEAVE that the client has left. */ private onClientRemoved (username: string): void { const individualMessage: Message = { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.PRESENCE_LEAVE, name : username, } const allMessage: Message = { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.PRESENCE_LEAVE_ALL, name: username } this.subscriptionRegistry.sendToSubscribers(EVERYONE, allMessage, false, null) this.subscriptionRegistry.sendToSubscribers(username, individualMessage, false, null) } } ================================================ FILE: src/handlers/record/record-deletion.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import {spy} from 'sinon' import RecordDeletion from './record-deletion' import * as M from './test-messages' import * as C from '../../../src/constants' import * as testHelper from '../../test/helper/test-helper' import { getTestMocks } from '../../test/helper/test-mocks' describe('record deletion', () => { let testMocks let recordDeletion let client let config let services let callback beforeEach(() => { testMocks = getTestMocks() client = testMocks.getSocketWrapper() const options = testHelper.getDeepstreamOptions() config = options.config services = options.services callback = spy() }) afterEach(() => { client.socketWrapperMock.verify() }) it('deletes records - happy path', () => { client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(M.deletionSuccessMsg) recordDeletion = new RecordDeletion( config, services, client.socketWrapper, M.deletionMsg, callback ) expect(services.cache.completedDeleteOperations).to.equal(1) expect(services.storage.completedDeleteOperations).to.equal(1) expect(recordDeletion.isDestroyed).to.equal(true) expect(callback).to.have.callCount(1) }) it('encounters an error during record deletion', (done) => { services.cache.nextOperationWillBeSuccessful = false services.cache.nextOperationWillBeSynchronous = false client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.RECORD_DELETE_ERROR, name: 'someRecord' }) recordDeletion = new RecordDeletion( config, services, client.socketWrapper, M.deletionMsg, callback ) setTimeout(() => { expect(recordDeletion.isDestroyed).to.equal(true) expect(callback).to.have.callCount(0) expect(services.logger.logSpy.firstCall.args).to.deep.equal([3, C.RECORD_ACTION[C.RECORD_ACTION.RECORD_DELETE_ERROR], 'storageError']) done() }, 20) }) it('encounters an ack delete timeout', (done) => { config.record.cacheRetrievalTimeout = 10 services.cache.nextOperationWillBeSuccessful = false services.cache.nextOperationWillBeSynchronous = false client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.RECORD_DELETE_ERROR, name: 'someRecord' }) recordDeletion = new RecordDeletion( config, services, client.socketWrapper, M.deletionMsg, callback ) setTimeout(() => { expect(recordDeletion.isDestroyed).to.equal(true) expect(callback).to.have.callCount(0) expect(services.logger.logSpy.firstCall.args).to.deep.equal([3, C.RECORD_ACTION[C.RECORD_ACTION.RECORD_DELETE_ERROR], 'cache timeout']) done() }, 100) }) it('doesn\'t delete excluded messages from storage', () => { config.record.storageExclusionPrefixes = ['no-storage/'] client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(M.anotherDeletionSuccessMsg) recordDeletion = new RecordDeletion( config, services, client.socketWrapper, M.anotherDeletionMsg, callback ) expect(services.cache.completedDeleteOperations).to.equal(1) expect(services.storage.completedDeleteOperations).to.equal(0) expect(recordDeletion.isDestroyed).to.equal(true) expect(callback).to.have.callCount(1) }) }) ================================================ FILE: src/handlers/record/record-deletion.ts ================================================ import { DeepstreamConfig, DeepstreamServices, SocketWrapper } from '@deepstream/types' import { Message, RecordMessage, RECORD_ACTION, TOPIC } from '../../constants' import { isExcluded } from '../../utils/utils' export default class RecordDeletion { private metaData: any private config: DeepstreamConfig private services: DeepstreamServices private socketWrapper: SocketWrapper private message: Message private successCallback: Function private recordName: string private completed: 0 private isDestroyed: boolean private cacheTimeout: any private storageTimeout: any /** * This class represents the deletion of a single record. It handles it's removal * from cache and storage and handles errors and timeouts */ constructor (config: DeepstreamConfig, services: DeepstreamServices, socketWrapper: SocketWrapper, message: RecordMessage, successCallback: Function, metaData: any = {}) { this.metaData = metaData this.config = config this.services = services this.socketWrapper = socketWrapper this.message = message this.successCallback = successCallback this.recordName = message.name this.completed = 0 this.isDestroyed = false this.onCacheDelete = this.onCacheDelete.bind(this) this.onStorageDelete = this.onStorageDelete.bind(this) this.cacheTimeout = setTimeout( this.handleError.bind(this, 'cache timeout'), this.config.record.cacheRetrievalTimeout, ) this.services.cache.delete( this.recordName, this.onCacheDelete.bind(this), metaData, ) if (!isExcluded(this.config.record.storageExclusionPrefixes, this.recordName)) { this.storageTimeout = setTimeout( this.handleError.bind(this, 'storage timeout'), this.config.record.storageRetrievalTimeout, ) this.services.storage.delete( this.recordName, this.onStorageDelete, metaData, ) } else { this.onStorageDelete(null) } } /** * Callback for completed cache and storage interactions. Will invoke * _done() once both are completed */ private onCacheDelete (error: string | null): void { clearTimeout(this.cacheTimeout) this.stageComplete(error) } private onStorageDelete (error: string | null) { clearTimeout(this.storageTimeout) this.stageComplete(error) } private stageComplete (error: string | null) { this.completed++ if (this.isDestroyed) { return } if (error) { this.handleError(error.toString()) return } if (this.completed === 2) { this.done() } } /** * Callback for successful deletions. Notifies the original sender and calls * the callback to allow the recordHandler to broadcast the deletion */ private done (): void { this.services.logger.info(RECORD_ACTION[RECORD_ACTION.DELETE], this.recordName, this.metaData) this.socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.DELETE_SUCCESS, name: this.message.name }) this.message = Object.assign({}, this.message, { action: RECORD_ACTION.DELETED }) this.successCallback(this.recordName, this.message, this.socketWrapper) this.destroy() } /** * Destroyes the class and null down its dependencies */ private destroy (): void { clearTimeout(this.cacheTimeout) clearTimeout(this.storageTimeout) this.isDestroyed = true // this.options = null // this.socketWrapper = null // this.message = null } /** * Handle errors that occured during deleting the record */ private handleError (errorMsg: string) { this.socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.RECORD_DELETE_ERROR, name: this.recordName }) this.services.logger.error(RECORD_ACTION[RECORD_ACTION.RECORD_DELETE_ERROR], errorMsg, this.metaData) this.destroy() } } ================================================ FILE: src/handlers/record/record-handler-permission.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as C from '../../../src/constants' import RecordHandler from './record-handler' import * as M from './test-messages' import * as testHelper from '../../test/helper/test-helper' import { getTestMocks } from '../../test/helper/test-mocks' describe('record handler handles messages', () => { let testMocks let recordHandler let client let config let services beforeEach(() => { ({config, services} = testHelper.getDeepstreamOptions()) recordHandler = new RecordHandler(config, services) testMocks = getTestMocks() client = testMocks.getSocketWrapper() }) afterEach(() => { client.socketWrapperMock.verify() }) it('triggers create and read actions if record doesnt exist', () => { client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(M.readResponseMessage) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) expect(services.permission.lastArgs.length).to.equal(2) expect(services.permission.lastArgs[0][1].action).to.equal(C.RECORD_ACTION.CREATE) expect(services.permission.lastArgs[1][1].action).to.equal(C.RECORD_ACTION.READ) }) it('triggers only read action if record does exist', () => { services.cache.set('some-record', 0, {}, () => {}) client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(M.readResponseMessage) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) expect(services.permission.lastArgs.length).to.equal(1) expect(services.permission.lastArgs[0][1].action).to.equal(C.RECORD_ACTION.READ) }) it('rejects a create', () => { services.permission.nextResult = false const { names, ...msg } = M.subscribeCreateAndReadDeniedMessage client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ ...msg, name: 'some-record', isError: true }) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) expect(services.permission.lastArgs.length).to.equal(1) expect(services.permission.lastArgs[0][1].action).to.equal(C.RECORD_ACTION.CREATE) }) it('rejects a read', () => { services.cache.set('some-record', 0, {}, () => {}) services.permission.nextResult = false const { names, ...msg } = M.subscribeCreateAndReadDeniedMessage client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ ...msg, name: 'some-record', isError: true }) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) expect(services.permission.lastArgs.length).to.equal(1) expect(services.permission.lastArgs[0][1].action).to.equal(C.RECORD_ACTION.READ) }) it('handles a permission error', () => { services.permission.nextError = 'XXX' services.permission.nextResult = false const { names, ...msg } = M.subscribeCreateAndReadPermissionErrorMessage client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ ...msg, name: 'some-record', isError: true }) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) expect(services.permission.lastArgs.length).to.equal(1) expect(services.permission.lastArgs[0][1].action).to.equal(C.RECORD_ACTION.CREATE) }) }) ================================================ FILE: src/handlers/record/record-handler.spec.ts ================================================ import 'mocha' import * as C from '../../../src/constants' import { expect } from 'chai' import RecordHandler from './record-handler' import * as M from './test-messages' import * as testHelper from '../../test/helper/test-helper' import { getTestMocks } from '../../test/helper/test-mocks' describe('record handler handles messages', () => { let testMocks let recordHandler let client let config let services beforeEach(() => { testMocks = getTestMocks() client = testMocks.getSocketWrapper('someUser') const options = testHelper.getDeepstreamOptions() config = options.config services = options.services recordHandler = new RecordHandler( config, services, testMocks.subscriptionRegistry, testMocks.listenerRegistry ) }) afterEach(() => { client.socketWrapperMock.verify() testMocks.subscriptionRegistryMock.verify() }) it('creates a non existing record', () => { client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(M.readResponseMessage) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) expect(services.cache.lastSetKey).to.equal('some-record') expect(services.cache.lastSetVersion).to.equal(0) expect(services.cache.lastSetValue).to.deep.equal({}) expect(services.storage.lastSetKey).to.equal('some-record') expect(services.storage.lastSetVersion).to.equal(0) expect(services.storage.lastSetValue).to.deep.equal({}) }) it('tries to create a non existing record, but receives an error from the cache', () => { services.cache.failNextSet = true client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.RECORD_CREATE_ERROR, originalAction: C.RECORD_ACTION.SUBSCRIBECREATEANDREAD, name: M.subscribeCreateAndReadMessage.names[0] }) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) // expect(options.logger.lastLogMessage).to.equal('storage:storageError') }) it('does not store new record when excluded', () => { config.record.storageExclusionPrefixes = ['some-record'] recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) expect(services.storage.lastSetKey).to.equal(null) expect(services.storage.lastSetVersion).to.equal(null) expect(services.storage.lastSetValue).to.equal(null) }) it('returns an existing record', () => { services.cache.set('some-record', M.recordVersion, M.recordData, () => {}) client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ_RESPONSE, name: 'some-record', version: M.recordVersion, parsedData: M.recordData }) recordHandler.handle(client.socketWrapper, M.subscribeCreateAndReadMessage) }) it('returns a snapshot of the data that exists with version number and data', () => { services.cache.set('some-record', M.recordVersion, M.recordData, () => {}) client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ_RESPONSE, name: 'some-record', parsedData: M.recordData, version: M.recordVersion }) recordHandler.handle(client.socketWrapper, M.recordSnapshotMessage) }) it('returns an error for a snapshot of data that doesn\'t exists', () => { client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.RECORD_NOT_FOUND, originalAction: M.recordSnapshotMessage.action, name: M.recordSnapshotMessage.name, isError: true }) recordHandler.handle(client.socketWrapper, M.recordSnapshotMessage) }) it('returns an error for a snapshot if message error occurs with record retrieval', () => { services.cache.nextOperationWillBeSuccessful = false client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.RECORD_LOAD_ERROR, originalAction: M.recordSnapshotMessage.action, name: M.recordSnapshotMessage.name, isError: true }) recordHandler.handle(client.socketWrapper, M.recordSnapshotMessage) }) it('returns a version of the data that exists with version number', () => { ['record/1', 'record/2', 'record/3'].forEach((name) => { const version = Math.floor(Math.random() * 100) const data = { firstname: 'Wolfram' } services.cache.set(name, version, data, () => {}) client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(Object.assign({}, M.recordHeadResponseMessage, { name, version })) recordHandler.handle(client.socketWrapper, Object.assign({}, M.recordHeadMessage, { name })) }) }) it('returns an version of -1 for head request of data that doesn\'t exist', () => { ['record/1', 'record/2', 'record/3'].forEach((name) => { client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(Object.assign({}, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.HEAD_RESPONSE, name: M.recordHeadMessage.name, version: -1 }, { name })) recordHandler.handle(client.socketWrapper, Object.assign({}, M.recordHeadMessage, { name })) }) }) it('returns an error for a version if message error occurs with record retrieval', () => { services.cache.nextOperationWillBeSuccessful = false client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.RECORD_LOAD_ERROR, originalAction: M.recordHeadMessage.action, name: M.recordHeadMessage.name, isError: true }) recordHandler.handle(client.socketWrapper, M.recordHeadMessage) }) it('patches a record', () => { const recordPatch = Object.assign({}, M.recordPatch) services.cache.set('some-record', M.recordVersion, Object.assign({}, M.recordData), () => {}) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.recordPatch.name, recordPatch, false, client.socketWrapper) recordHandler.handle(client.socketWrapper, recordPatch) services.cache.get('some-record', (error, version, record) => { expect(version).to.equal(version) expect(record).to.deep.equal({ name: 'Kowalski', lastname: 'Egon' }) }) }) it('updates a record', () => { services.cache.set('some-record', M.recordVersion, M.recordData, () => {}) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.recordUpdate.name, M.recordUpdate, false, client.socketWrapper) recordHandler.handle(client.socketWrapper, M.recordUpdate) services.cache.get('some-record', (error, version, result) => { expect(version).to.equal(6) expect(result).to.deep.equal({ name: 'Kowalski' }) }) }) it('rejects updates for existing versions', () => { services.cache.set(M.recordUpdate.name, M.recordVersion, M.recordData, () => {}) const ExistingVersion = Object.assign({}, M.recordUpdate, { version: M.recordVersion }) client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.VERSION_EXISTS, originalAction: ExistingVersion.action, name: ExistingVersion.name, version: ExistingVersion.version, parsedData: M.recordData, isWriteAck: false, correlationId: undefined }) recordHandler.handle(client.socketWrapper, ExistingVersion) expect(services.logger.lastLogMessage).to.equal('someUser tried to update record some-record to version 5 but it already was 5') }) describe('notifies when db/cache remotely changed', () => { beforeEach(() => { services.storage.nextGetWillBeSynchronous = true services.cache.nextGetWillBeSynchronous = true }) it ('notifies users when record changes', () => { M.notify.names.forEach(name => { services.storage.set(name, 123, { name }, () => {}) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(name, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name, parsedData: { name }, version: 123 }, true, null) }) recordHandler.handle(client.socketWrapper, M.notify) }) it('notifies users when records deleted', () => { M.notify.names.forEach(name => { testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(name, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETED, name }, true, null) }) recordHandler.handle(client.socketWrapper, M.notify) }) it('notifies users when records updated and deleted combined', () => { services.storage.set(M.notify.names[0], 1, { name: M.notify.names[0] }, () => {}) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.notify.names[0], { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: M.notify.names[0], parsedData: { name: M.notify.names[0] }, version: 1 }, true, null) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.notify.names[1], { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETED, name: M.notify.names[1] }, true, null) recordHandler.handle(client.socketWrapper, M.notify) }) }) describe('subscription registry', () => { it('handles unsubscribe messages', () => { testMocks.subscriptionRegistryMock .expects('unsubscribeBulk') .once() .withExactArgs(M.unsubscribeMessage, client.socketWrapper) recordHandler.handle(client.socketWrapper, M.unsubscribeMessage) }) }) it('updates a record via same client to the same version', (done) => { config.record.cacheRetrievalTimeout = 50 services.cache.nextGetWillBeSynchronous = false services.cache.set(M.recordUpdate.name, M.recordVersion, M.recordData, () => {}) client.socketWrapperMock .expects('sendMessage') .twice() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.VERSION_EXISTS, originalAction: C.RECORD_ACTION.UPDATE, version: M.recordVersion, parsedData: M.recordData, name: M.recordUpdate.name, isWriteAck: false, correlationId: undefined }) recordHandler.handle(client.socketWrapper, M.recordUpdate) recordHandler.handle(client.socketWrapper, M.recordUpdate) recordHandler.handle(client.socketWrapper, M.recordUpdate) setTimeout(() => { /** * Important to note this is a race condition since version exists errors are sent as soon as record is retrieved, * which means it hasn't yet been written to cache. */ done() }, 50) }) it('handles deletion messages', () => { services.cache.nextGetWillBeSynchronous = false services.cache.set(M.recordDelete.name, 1, {}, () => {}) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.recordDelete.name, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETED, name: M.recordDelete.name }, true, client.socketWrapper) testMocks.subscriptionRegistryMock .expects('getLocalSubscribers') .once() .returns(new Set()) recordHandler.handle(client.socketWrapper, M.recordDelete) services.cache.get(M.recordDelete.name, (error, version, data) => { expect(version).to.deep.equal(-1) expect(data).to.equal(null) }) }) it('updates a record with a -1 version number', () => { services.cache.set(M.recordUpdate.name, 5, Object.assign({}, M.recordData), () => {}) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.recordUpdate.name, Object.assign({}, M.recordUpdate, { version: 6 }), false, client.socketWrapper) recordHandler.handle(client.socketWrapper, Object.assign({}, M.recordUpdate, { version: -1 })) services.cache.get(M.recordUpdate.name, (error, version, data) => { expect(data).to.deep.equal(M.recordUpdate.parsedData) expect(version).to.equal(6) }) }) it('updates multiple updates with an -1 version number', () => { const data = Object.assign({}, M.recordData) services.cache.set(M.recordUpdate.name, 5, data, () => {}) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.recordUpdate.name, Object.assign({}, M.recordUpdate, { version: 6 }), false, client.socketWrapper) testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(M.recordUpdate.name, Object.assign({}, M.recordUpdate, { version: 7 }), false, client.socketWrapper) recordHandler.handle(client.socketWrapper, Object.assign({}, M.recordUpdate, { version: -1 })) recordHandler.handle(client.socketWrapper, Object.assign({}, M.recordUpdate, { version: -1 })) services.cache.get(M.recordUpdate.name, (error, version, result) => { expect(result).to.deep.equal(M.recordUpdate.parsedData) }) }) it.skip('creates records when using CREATEANDUPDATE', () => { testMocks.subscriptionRegistryMock .expects('sendToSubscribers') .once() .withExactArgs( M.createAndUpdate.name, Object.assign({}, M.createAndUpdate, { action: C.RECORD_ACTION.UPDATE, version: 1 }), false, client.socketWrapper ) recordHandler.handle(client.socketWrapper, M.createAndUpdate) services.cache.get(M.createAndUpdate.name, (error, version, data) => { expect(version).to.deep.equal(1) expect(data).to.deep.equal(M.recordData) }) }) it('registers a listener', () => { testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(client.socketWrapper, M.listenMessage) recordHandler.handle(client.socketWrapper, M.listenMessage) }) it('removes listeners', () => { testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(client.socketWrapper, M.unlistenMessage) recordHandler.handle(client.socketWrapper, M.unlistenMessage) }) it('processes listen accepts', () => { testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(client.socketWrapper, M.listenAcceptMessage) recordHandler.handle(client.socketWrapper, M.listenAcceptMessage) }) it('processes listen rejects', () => { testMocks.listenerRegistryMock .expects('handle') .once() .withExactArgs(client.socketWrapper, M.listenRejectMessage) recordHandler.handle(client.socketWrapper, M.listenRejectMessage) }) }) ================================================ FILE: src/handlers/record/record-handler.ts ================================================ import RecordDeletion from './record-deletion' import { recordRequestBinding } from './record-request' import { RecordTransition } from './record-transition' import { SubscriptionRegistry, Handler, DeepstreamConfig, DeepstreamServices, SocketWrapper, EVENT } from '@deepstream/types' import { ListenerRegistry } from '../../listen/listener-registry' import { isExcluded } from '../../utils/utils' import { STATE_REGISTRY_TOPIC, RecordMessage, TOPIC, RecordWriteMessage, BulkSubscriptionMessage, ListenMessage, PARSER_ACTION, RECORD_ACTION as RA, JSONObject, Message, RECORD_ACTION, ALL_ACTIONS } from '../../constants' export default class RecordHandler extends Handler { private subscriptionRegistry: SubscriptionRegistry private listenerRegistry: ListenerRegistry private transitions = new Map() private recordRequestsInProgress = new Map() private recordRequest: Function /** * The entry point for record related operations */ constructor (private readonly config: DeepstreamConfig, private readonly services: DeepstreamServices, subscriptionRegistry?: SubscriptionRegistry, listenerRegistry?: ListenerRegistry, private readonly metaData?: any) { super() this.subscriptionRegistry = subscriptionRegistry || services.subscriptions.getSubscriptionRegistry(TOPIC.RECORD, STATE_REGISTRY_TOPIC.RECORD_SUBSCRIPTIONS) this.listenerRegistry = listenerRegistry || new ListenerRegistry(TOPIC.RECORD, config, services, this.subscriptionRegistry, null) this.subscriptionRegistry.setSubscriptionListener(this.listenerRegistry) this.recordRequest = recordRequestBinding(config, services, this, metaData) this.onDeleted = this.onDeleted.bind(this) this.create = this.create.bind(this) this.onPermissionResponse = this.onPermissionResponse.bind(this) } public async close () { this.listenerRegistry.close() } /** * Handles incoming record requests. * * Please note that neither CREATE nor READ is supported as a * client send action. Instead the client sends CREATEORREAD * and deepstream works which one it will be */ public handle (socketWrapper: SocketWrapper | null, message: RecordMessage): void { const action = message.action if (socketWrapper === null) { this.handleClusterUpdate(message) return } if (action === RA.SUBSCRIBE) { this.subscriptionRegistry.subscribeBulk(message as BulkSubscriptionMessage, socketWrapper) return } if (action === RA.SUBSCRIBECREATEANDREAD || action === RA.SUBSCRIBEANDREAD) { const onSuccess = action === RA.SUBSCRIBECREATEANDREAD ? this.onSubscribeCreateAndRead : this.onSubscribeAndRead const l = message.names!.length for (let i = 0; i < l; i++) { this.recordRequest( message.names![i], socketWrapper, onSuccess, onRequestError, { ...message, name: message.names![i] } ) } socketWrapper.sendAckMessage(message) return } if ( action === RA.CREATEANDUPDATE || action === RA.CREATEANDPATCH ) { /* * Allows updates to the record without being subscribed, creates * the record if it doesn't exist */ this.createAndUpdate(socketWrapper!, message as RecordWriteMessage) return } if (action === RA.READ) { /* * Return the current state of the record in cache or db */ this.recordRequest(message.name, socketWrapper, onSnapshotComplete, onRequestError, message) return } if (action === RA.HEAD) { /* * Return the current version of the record or -1 if not found */ this.head(socketWrapper!, message) return } if (action === RA.SUBSCRIBEANDHEAD) { /* * Return the current version of the record or -1 if not found, subscribing either way */ this.subscribeAndHeadBulk(socketWrapper!, message) return } if (action === RA.UPDATE || action === RA.PATCH || action === RA.ERASE) { /* * Handle complete (UPDATE) or partial (PATCH/ERASE) updates */ this.update(socketWrapper, message as RecordWriteMessage, message.isWriteAck || false) return } if (action === RA.DELETE) { /* * Deletes the record */ this.delete(socketWrapper!, message) return } if (action === RA.UNSUBSCRIBE) { /* * Unsubscribes (discards) a record that was previously subscribed to * using read() */ this.subscriptionRegistry.unsubscribeBulk(message as BulkSubscriptionMessage, socketWrapper!) return } if (action === RA.LISTEN || action === RA.UNLISTEN || action === RA.LISTEN_ACCEPT || action === RA.LISTEN_REJECT) { /* * Listen to requests for a particular record or records * whose names match a pattern */ this.listenerRegistry.handle(socketWrapper!, message as ListenMessage) return } if (message.action === RA.NOTIFY) { this.recordUpdatedWithoutDeepstream(message, socketWrapper) return } this.services.logger.error(PARSER_ACTION[PARSER_ACTION.UNKNOWN_ACTION], RA[action], this.metaData) } private handleClusterUpdate (message: RecordMessage) { if (message.action === RA.DELETED) { this.remoteDelete(message) return } if (message.action === RA.NOTIFY) { this.recordUpdatedWithoutDeepstream(message) return } this.broadcastUpdate(message.name, { topic: message.topic, action: message.action, name: message.name, path: message.path, version: message.version, data: message.data, parsedData: message.parsedData }, false, null) } private async recordUpdatedWithoutDeepstream (message: RecordMessage, socketWrapper: SocketWrapper | null = null) { if (socketWrapper) { if (this.services.cache.deleteBulk === undefined) { const errorMessage = 'Cache needs to implement deleteBulk in order for it to work correctly' this.services.logger.error(EVENT.PLUGIN_ERROR, errorMessage) socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.RECORD_NOTIFY_ERROR, isError: true, parsedData: errorMessage, correlationId: message.correlationId }) return } try { await new Promise((resolve, reject) => this.services.cache.deleteBulk(message.names!, (error) => { error ? reject(error) : resolve() })) } catch (error) { const errorMessage = 'Error deleting messages in bulk when attempting to notify of remote changes' this.services.logger.error(EVENT.ERROR, `${errorMessage}: ${error?.toString()}`, { message }) socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.RECORD_NOTIFY_ERROR, isError: true, parsedData: errorMessage, correlationId: message.correlationId }) return } } let completed = 0 message.names!.forEach((recordName, index, names) => { if (this.subscriptionRegistry.hasLocalSubscribers(recordName)) { this.recordRequest(recordName, socketWrapper, (name: string, version: number, data: JSONObject) => { if (version === -1) { this.remoteDelete({ topic: TOPIC.RECORD, action: RECORD_ACTION.DELETED, name }) } else { this.subscriptionRegistry.sendToSubscribers(name, { topic: TOPIC.RECORD, action: RECORD_ACTION.UPDATE, name, version, parsedData: data }, true, null) } completed++ if (completed === names.length && socketWrapper) { socketWrapper.sendAckMessage(message) this.services.clusterNode.send(message) } }, (event: RA, errorMessage: string, name: string, socket: SocketWrapper, msg: Message) => { completed++ if (socket) { onRequestError(event, errorMessage, recordName, socket, msg) } if (completed === names.length && socket) { socket.sendAckMessage(message) this.services.clusterNode.send(message) } }, message) } else { completed++ if (completed === names.length && socketWrapper) { socketWrapper.sendAckMessage(message) this.services.clusterNode.send(message) } } }) } /** * Returns just the current version number of a record * Results in a HEAD_RESPONSE * If the record is not found, the version number will be -1 */ private head (socketWrapper: SocketWrapper, message: RecordMessage, name: string = message.name): void { this.recordRequest(name, socketWrapper, onHeadComplete, onRequestError, message) } private subscribeAndHeadBulk (socketWrapper: SocketWrapper, message: RecordMessage): void { this.services.cache.headBulk(message.names!, (error, versions, missing) => { if (error) { this.services.logger.error(EVENT.ERROR, `Error subscribing and head bulk for ${message.correlationId}`) return } if (Object.keys(versions!).length > -1) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RA.HEAD_RESPONSE_BULK, versions }) } this.subscriptionRegistry.subscribeBulk(message as BulkSubscriptionMessage, socketWrapper) const l = missing!.length for (let i = 0; i < l; i++) { if (versions![missing![i]] === undefined) { this.head(socketWrapper, message, missing![i]) } } }) } private onSubscribeCreateAndRead (recordName: string, version: number, data: JSONObject | null, socketWrapper: SocketWrapper, message: RecordMessage) { if (data) { this.readAndSubscribe(message, version, data, socketWrapper) } else { this.permissionAction( RA.CREATE, message, message.action, socketWrapper, this.create, ) } } private onSubscribeAndRead (recordName: string, version: number, data: JSONObject | null, socketWrapper: SocketWrapper, message: RecordMessage) { if (data) { this.readAndSubscribe(message, version, data, socketWrapper) } else { this.permissionAction(RA.READ, message, message.action, socketWrapper, () => { this.subscriptionRegistry.subscribe(message.name, { ...message, action: RA.SUBSCRIBE }, socketWrapper, message.names !== undefined) socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.READ_RESPONSE, name: message.name, version: -1, parsedData: {}, }) }) } } /** * An upsert operation where the record will be created and written to * with the data in the message. Important to note that each operation, * the create and the write are permissioned separately. * * This method also takes note of the storageHotPathPatterns option, when a record * with a name that matches one of the storageHotPathPatterns is written to with * the CREATEANDUPDATE action, it will be permissioned for both CREATE and UPDATE, then * inserted into the cache and storage. */ private createAndUpdate (socketWrapper: SocketWrapper, message: RecordWriteMessage): void { const recordName = message.name const isPatch = message.path !== undefined const originalAction = message.action message = { ...message, action: isPatch ? RA.PATCH : RA.UPDATE } // allow writes on the hot path to bypass the record transition // and be written directly to cache and storage for (let i = 0; i < this.config.record.storageHotPathPrefixes.length; i++) { const pattern = this.config.record.storageHotPathPrefixes[i] if (recordName.indexOf(pattern) === 0) { if (isPatch) { const errorMessage = { topic: TOPIC.RECORD, action: RA.INVALID_PATCH_ON_HOTPATH, originalAction, name: recordName } as RecordMessage if (message.correlationId) { errorMessage.correlationId = message.correlationId } socketWrapper.sendMessage(errorMessage) return } this.permissionAction(RA.CREATE, message, originalAction, socketWrapper, () => { this.permissionAction(RA.UPDATE, message, originalAction, socketWrapper, () => { this.forceWrite(recordName, message, socketWrapper) }) }) return } } const transition = this.transitions.get(recordName) if (transition) { this.permissionAction(message.action, message, originalAction, socketWrapper, () => { transition.add(socketWrapper, message) }) return } this.permissionAction(RA.CREATE, message, originalAction, socketWrapper, () => { this.permissionAction(RA.UPDATE, message, originalAction, socketWrapper, () => { this.update(socketWrapper, message, true) }) }) } /** * Forcibly writes to the cache and storage layers without going via * the RecordTransition. Usually updates and patches will go via the * transition which handles write acknowledgements, however in the * case of a hot path write acknowledgement we need to handle that * case here. */ private forceWrite (recordName: string, message: RecordWriteMessage, socketWrapper: SocketWrapper): void { socketWrapper.parseData(message) const writeAck = message.isWriteAck let cacheResponse = false let storageResponse = false let writeError: string | null this.services.storage.set(recordName, 0, message.parsedData, (error) => { if (writeAck) { storageResponse = true writeError = writeError || error || null this.handleForceWriteAcknowledgement( socketWrapper, message, cacheResponse, storageResponse, writeError, ) } }, this.metaData) this.services.cache.set(recordName, 0, message.parsedData, (error) => { if (!error) { this.broadcastUpdate(recordName, message, false, socketWrapper) } if (writeAck) { cacheResponse = true writeError = writeError || error || null this.handleForceWriteAcknowledgement( socketWrapper, message, cacheResponse, storageResponse, writeError, ) } }, this.metaData) } /** * Handles write acknowledgements during a force write. Usually * this case is handled via the record transition. */ public handleForceWriteAcknowledgement ( socketWrapper: SocketWrapper, message: RecordWriteMessage, cacheResponse: boolean, storageResponse: boolean, error: Error | string | null, ): void { if (storageResponse && cacheResponse) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RA.WRITE_ACKNOWLEDGEMENT, name: message.name, correlationId: message.correlationId }, true) } } /** * Creates a new, empty record and triggers a read operation once done */ private create (socketWrapper: SocketWrapper, message: RecordMessage, originalAction: RECORD_ACTION): void { const recordName = message.name // store the records data in the cache and wait for the result this.services.cache.set(recordName, 0, {}, (error) => { if (error) { this.services.logger.error(RA[RA.RECORD_CREATE_ERROR], recordName, this.metaData) socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RA.RECORD_CREATE_ERROR, originalAction, name: message.name }) return } // this isn't really needed, can subscribe and send empty data immediately this.readAndSubscribe({ ...message, action: originalAction }, 0, {}, socketWrapper) }, this.metaData) if (!isExcluded(this.config.record.storageExclusionPrefixes, message.name)) { // store the record data in the persistant storage independently and don't wait for the result this.services.storage.set(recordName, 0, {}, (error) => { if (error) { this.services.logger.error(RA[RA.RECORD_CREATE_ERROR], `storage:${error}`, this.metaData) } }, this.metaData) } } /** * Subscribes to updates for a record and sends its current data once done */ private readAndSubscribe (message: RecordMessage, version: number, data: any, socketWrapper: SocketWrapper): void { this.permissionAction(RA.READ, message, message.action, socketWrapper, () => { this.subscriptionRegistry.subscribe(message.name, { ...message, action: RA.SUBSCRIBE }, socketWrapper, message.names !== undefined) this.recordRequest(message.name, socketWrapper, (_: string, newVersion: number, latestData: any) => { if (latestData) { if (newVersion !== version) { this.services.logger.info( EVENT.INFO, `BUG CAUGHT! ${message.name} was version ${version} for readAndSubscribe, ` + `but updated during permission to ${message.version}` ) } sendRecord(message.name, version, latestData, socketWrapper) } else { this.services.logger.error( EVENT.ERROR, `BUG? ${message.name} was version ${version} for readAndSubscribe, ` + 'but was removed during permission check', { message } ) onRequestError( message.action, `"${message.name}" was removed during permission check`, message.name, socketWrapper, message ) } }, onRequestError, message) }) } /** * Applies both full and partial updates. Creates a new record transition that will live as * long as updates are in flight and new updates come in */ private update (socketWrapper: SocketWrapper | null, message: RecordWriteMessage, upsert: boolean): void { const recordName = message.name const version = message.version /* * If the update message is received from the message bus, rather than from a client, * assume that the original deepstream node has already updated the record in cache and * storage and only broadcast the message to subscribers */ if (socketWrapper === null) { this.broadcastUpdate(recordName, message, false, socketWrapper) return } const isPatch = message.path !== undefined message = { ...message, action: isPatch ? RA.PATCH : RA.UPDATE } let transition = this.transitions.get(recordName) if (transition && transition.hasVersion(version)) { transition.sendVersionExists({ message, sender: socketWrapper }) return } if (!transition) { transition = new RecordTransition(recordName, this.config, this.services, this, this.metaData) this.transitions.set(recordName, transition) } transition.add(socketWrapper, message, upsert) } /** * Invoked by RecordTransition. Notifies local subscribers and other deepstream * instances of record updates */ public broadcastUpdate (name: string, message: RecordMessage, noDelay: boolean, originalSender: SocketWrapper | null): void { this.subscriptionRegistry.sendToSubscribers(name, message, noDelay, originalSender) } /** * Called by a RecordTransition, either if it is complete or if an error occured. Removes * the transition from the registry */ public transitionComplete (recordName: string): void { this.transitions.delete(recordName) } /** * Executes or schedules a callback function once all transitions are complete * * This is called from the PermissionHandler destroy method, which * could occur in cases where 'runWhenRecordStable' is never called, * such as when no cross referencing or data loading is used. */ public removeRecordRequest (recordName: string): void { const recordRequests = this.recordRequestsInProgress.get(recordName) if (!recordRequests) { return } if (recordRequests.length === 0) { this.recordRequestsInProgress.delete(recordName) return } const callback = recordRequests.splice(0, 1)[0] callback(recordName) } /** * Executes or schedules a callback function once all record requests are removed. * This is critical to block reads until writes have occured for a record, which is * only from permissions when a rule is required to be run and the cache has not * verified it has the latest version */ public runWhenRecordStable (recordName: string, callback: Function): void { const recordRequests = this.recordRequestsInProgress.get(recordName) if (!recordRequests || recordRequests.length === 0) { this.recordRequestsInProgress.set(recordName, []) callback(recordName) } else { recordRequests.push(callback) } } /** * Deletes a record. If a transition is in progress it will be stopped. Once the deletion is * complete, an ACK is returned to the sender and broadcast to the message bus. */ private delete (socketWrapper: SocketWrapper, message: RecordMessage) { const recordName = message.name const transition = this.transitions.get(recordName) if (transition) { transition.destroy() this.transitions.delete(recordName) } // tslint:disable-next-line new RecordDeletion(this.config, this.services, socketWrapper, message, this.onDeleted, this.metaData) } /** * Handle a remote record deletion from the message bus. We assume that the original deepstream node * has already deleted the record from cache and storage and we only need to broadcast the message * to subscribers. * * If a transition is in progress it will be stopped. */ private remoteDelete (message: RecordMessage) { const recordName = message.name const transition = this.transitions.get(recordName) if (transition) { transition.destroy() this.transitions.delete(recordName) } this.onDeleted(recordName, message, null) } /* * Callback for completed deletions. Notifies subscribers of the delete and unsubscribes them */ private onDeleted (name: string, message: RecordMessage, originalSender: SocketWrapper | null) { this.broadcastUpdate(name, message, true, originalSender) for (const subscriber of this.subscriptionRegistry.getLocalSubscribers(name)) { this.subscriptionRegistry.unsubscribe(name, message, subscriber, true) } } /** * A secondary permissioning step that is performed once we know if the record exists (READ) * or if it should be created (CREATE) */ private permissionAction (actionToPermission: RA, message: Message, originalAction: RA, socketWrapper: SocketWrapper, successCallback: Function) { const copyWithAction = {...message, action: actionToPermission } this.services.permission.canPerformAction( socketWrapper, copyWithAction, this.onPermissionResponse, { originalAction, successCallback } ) } /* * Callback for complete permissions. Important to note that only compound operations like * CREATE_AND_UPDATE will end up here. */ private onPermissionResponse ( socketWrapper: SocketWrapper, message: Message, { originalAction, successCallback }: { originalAction: RA, successCallback: Function }, error: string | Error | ALL_ACTIONS | null, canPerformAction: boolean, ): void { if (error || !canPerformAction) { let action if (error) { this.services.logger.error(RA[RA.MESSAGE_PERMISSION_ERROR], error.toString()) action = RA.MESSAGE_PERMISSION_ERROR } else { action = RA.MESSAGE_DENIED } const msg = { topic: TOPIC.RECORD, action, originalAction, name: message.name, isError: true } as RecordMessage if (message.correlationId) { msg.correlationId = message.correlationId } if (message.isWriteAck) { msg.isWriteAck = true } socketWrapper.sendMessage(msg) } else { successCallback(socketWrapper, message, originalAction) } } } function onRequestError (event: RA, errorMessage: string, recordName: string, socket: SocketWrapper, message: Message) { const msg = { topic: TOPIC.RECORD, action: event, originalAction: message.action, name: recordName, isError: true, } as Message if (message.isWriteAck) { msg.isWriteAck = true } socket.sendMessage(msg) } function onSnapshotComplete (recordName: string, version: number, data: JSONObject, socket: SocketWrapper, message: Message) { if (data) { sendRecord(recordName, version, data, socket) } else { socket.sendMessage({ topic: TOPIC.RECORD, action: RA.RECORD_NOT_FOUND, originalAction: message.action, name: message.name, isError: true }) } } function onHeadComplete (name: string, version: number, data: never, socketWrapper: SocketWrapper) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RA.HEAD_RESPONSE, name, version }) } /** * Sends the records data current data once done */ function sendRecord (recordName: string, version: number, data: any, socketWrapper: SocketWrapper) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RA.READ_RESPONSE, name: recordName, version, parsedData: data, }) } ================================================ FILE: src/handlers/record/record-request.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import {spy} from 'sinon' import * as C from '../../constants' import { recordRequest } from './record-request' import { getTestMocks } from '../../test/helper/test-mocks' import { RECORD_ACTION } from '../../constants' import * as testHelper from '../../test/helper/test-helper' import { PromiseDelay } from '../../utils/utils'; describe('record request', () => { const completeCallback = spy() const errorCallback = spy() let testMocks let client let config let services const cacheData = { cache: true } const storageData = { storage: true } beforeEach(() => { const options = testHelper.getDeepstreamOptions() services = options.services config = Object.assign({}, options.config, { record: { cacheRetrievalTimeout: 100, storageRetrievalTimeout: 100, storageExclusionPrefixes: ['dont-save'] } }) services.cache.set('existingRecord', 1, cacheData, () => {}) services.storage.set('onlyExistsInStorage', 1, storageData, () => {}) testMocks = getTestMocks() client = testMocks.getSocketWrapper('someUser') completeCallback.resetHistory() errorCallback.resetHistory() }) describe('records are requested from cache and storage sequentially', () => { it('requests a record that exists in a synchronous cache', () => { services.cache.nextOperationWillBeSynchronous = true recordRequest( 'existingRecord', config, services, client.socketWrapper, completeCallback, errorCallback, null ) expect(services.cache.lastRequestedKey).to.equal('existingRecord') expect(services.storage.lastRequestedKey).to.equal(null) expect(completeCallback).to.have.been.calledWith( 'existingRecord', 1, cacheData, client.socketWrapper ) expect(errorCallback).to.have.callCount(0) }) it('requests a record that exists in an asynchronous cache', async () => { services.cache.nextGetWillBeSynchronous = false recordRequest( 'existingRecord', config, services, client.socketWrapper, completeCallback, errorCallback, null ) await PromiseDelay(30) expect(completeCallback).to.have.been.calledWith( 'existingRecord', 1, cacheData, client.socketWrapper ) expect(errorCallback).to.have.callCount(0) expect(services.cache.lastRequestedKey).to.equal('existingRecord') expect(services.storage.lastRequestedKey).to.equal(null) }) it('requests a record that doesn\'t exists in a synchronous cache, but in storage', () => { services.cache.nextGetWillBeSynchronous = true recordRequest( 'onlyExistsInStorage', config, services, client.socketWrapper, completeCallback, errorCallback, null ) expect(services.cache.lastRequestedKey).to.equal('onlyExistsInStorage') expect(services.storage.lastRequestedKey).to.equal('onlyExistsInStorage') expect(completeCallback).to.have.been.calledWith('onlyExistsInStorage', 1, storageData, client.socketWrapper) expect(errorCallback).to.have.callCount(0) }) it('requests a record that doesn\'t exists in an asynchronous cache, but in asynchronous storage', async () => { services.cache.nextGetWillBeSynchronous = false services.storage.nextGetWillBeSynchronous = false recordRequest( 'onlyExistsInStorage', config, services, client.socketWrapper, completeCallback, errorCallback, null ) await PromiseDelay(75) expect(services.cache.lastRequestedKey).to.equal('onlyExistsInStorage') expect(services.storage.lastRequestedKey).to.equal('onlyExistsInStorage') expect(errorCallback).to.have.callCount(0) expect(completeCallback).to.have.been.calledWith( 'onlyExistsInStorage', 1, storageData, client.socketWrapper ) }) it('returns null for non existent records', () => { services.cache.nextGetWillBeSynchronous = true recordRequest( 'doesNotExist', config, services, client.socketWrapper, completeCallback, errorCallback, null ) expect(completeCallback).to.have.been.calledWith('doesNotExist', -1, null, client.socketWrapper) expect(errorCallback).to.have.callCount(0) expect(services.cache.lastRequestedKey).to.equal('doesNotExist') expect(services.storage.lastRequestedKey).to.equal('doesNotExist') }) it('handles cache errors', () => { services.cache.nextGetWillBeSynchronous = true services.cache.nextOperationWillBeSuccessful = false recordRequest( 'cacheError', config, services, client.socketWrapper, completeCallback, errorCallback, null ) expect(errorCallback).to.have.been.calledWith( RECORD_ACTION.RECORD_LOAD_ERROR, 'error while loading cacheError from cache:storageError', 'cacheError', client.socketWrapper ) expect(completeCallback).to.have.callCount(0) expect(services.logger.logSpy).to.have.been.calledWith( 3, RECORD_ACTION[RECORD_ACTION.RECORD_LOAD_ERROR], 'error while loading cacheError from cache:storageError' ) // expect(client.socketWrapper.socket.lastSendMessage).to.equal( // msg('R|E|RECORD_LOAD_ERROR|error while loading cacheError from cache:storageError+' // )) }) it('handles storage errors', () => { services.cache.nextGetWillBeSynchronous = true services.cache.nextOperationWillBeSuccessful = true services.storage.nextGetWillBeSynchronous = true services.storage.nextOperationWillBeSuccessful = false recordRequest( 'storageError', config, services, client.socketWrapper, completeCallback, errorCallback, null ) expect(errorCallback).to.have.been.calledWith( RECORD_ACTION.RECORD_LOAD_ERROR, 'error while loading storageError from storage:storageError', 'storageError', client.socketWrapper ) expect(completeCallback).to.have.callCount(0) expect(services.logger.logSpy).to.have.been.calledWith(3, RECORD_ACTION[RECORD_ACTION.RECORD_LOAD_ERROR], 'error while loading storageError from storage:storageError') // expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|E|RECORD_LOAD_ERROR|error while loading storageError from storage:storageError+')) }) describe('handles cache timeouts', () => { beforeEach(() => { config.record.cacheRetrievalTimeout = 1 services.cache.nextGetWillBeSynchronous = false services.cache.nextOperationWillBeSuccessful = true }) afterEach(() => { config.cacheRetrievalTimeout = 10 }) it('sends a CACHE_RETRIEVAL_TIMEOUT message when cache times out', (done) => { recordRequest( 'willTimeoutCache', config, services, client.socketWrapper, completeCallback, errorCallback, null ) setTimeout(() => { expect(errorCallback).to.have.been.calledWith( C.RECORD_ACTION.CACHE_RETRIEVAL_TIMEOUT, 'willTimeoutCache', 'willTimeoutCache', client.socketWrapper ) expect(completeCallback).to.have.callCount(0) // ignores update from cache that may occur afterwards services.cache.triggerLastGetCallback(null, '{ data: "value" }') expect(completeCallback).to.have.callCount(0) done() }, 1) }) }) describe('handles storage timeouts', () => { beforeEach(() => { config.record.storageRetrievalTimeout = 1 services.cache.nextGetWillBeSynchronous = true services.cache.nextOperationWillBeSuccessful = true services.storage.nextGetWillBeSynchronous = false services.storage.nextOperationWillBeSuccessful = true }) it('sends a STORAGE_RETRIEVAL_TIMEOUT message when storage times out', async () => { recordRequest( 'willTimeoutStorage', config, services, client.socketWrapper, completeCallback, errorCallback, null ) await PromiseDelay(1) expect(errorCallback).to.have.been.calledWith( C.RECORD_ACTION.STORAGE_RETRIEVAL_TIMEOUT, 'willTimeoutStorage', 'willTimeoutStorage', client.socketWrapper ) expect(completeCallback).to.have.callCount(0) // ignores update from storage that may occur afterwards services.storage.triggerLastGetCallback(null, '{ data: "value" }') expect(completeCallback).to.have.callCount(0) }) }) }) describe('excluded records are not put into storage', () => { beforeEach(() => { services.cache.nextGetWillBeSynchronous = true services.storage.nextGetWillBeSynchronous = true services.storage.delete = spy() services.storage.set('dont-save/1', 1, {}, () => {}) }) it('returns null when requesting a record that doesn\'t exists in a synchronous cache, and is excluded from storage', () => { recordRequest( 'dont-save/1', config, services, client.socketWrapper, completeCallback, errorCallback, null ) expect(completeCallback).to.have.been.calledWith( 'dont-save/1', -1, null, client.socketWrapper ) expect(errorCallback).to.have.callCount(0) expect(services.storage.lastRequestedKey).to.equal(null) }) }) describe('promoting to cache can be disabled', () => { beforeEach(() => { services.cache.nextGetWillBeSynchronous = true services.storage.nextGetWillBeSynchronous = true services.cache.set = spy() services.storage.set('dont-save/1', 1, {}, () => {}) }) it('doesnt call set on cache if promoteToCache is disabled', () => { recordRequest( 'onlyExistsInStorage', config, services, client.socketWrapper, completeCallback, errorCallback, this, null, null, false ) expect(completeCallback).to.have.been.calledWith( 'onlyExistsInStorage', 1, { storage: true }, client.socketWrapper ) expect(services.cache.set).to.have.callCount(0) expect(errorCallback).to.have.callCount(0) expect(services.cache.lastRequestedKey).to.equal('onlyExistsInStorage') expect(services.storage.lastRequestedKey).to.equal('onlyExistsInStorage') }) }) }) ================================================ FILE: src/handlers/record/record-request.ts ================================================ import { SocketWrapper, DeepstreamServices, DeepstreamConfig } from '@deepstream/types' import { Message, RECORD_ACTION } from '../../constants' import { isExcluded } from '../../utils/utils' type onCompleteCallback = (recordName: string, version: number, data: any, socket: SocketWrapper | null, message?: Message) => void type onErrorCallback = (event: any, errorMessage: string, recordName: string, socket: SocketWrapper | null, message?: Message) => void /** * Sends an error to the socketWrapper that requested the * record */ function sendError ( event: RECORD_ACTION, errorMessage: string, recordName: string, socketWrapper: SocketWrapper | null, onError: onErrorCallback, services: DeepstreamServices, context: any, metaData?: any, message?: Message, ): void { services.logger.error(RECORD_ACTION[event], errorMessage, metaData) if (message) { onError.call(context, event, errorMessage, recordName, socketWrapper, message) } else { onError.call(context, event, errorMessage, recordName, socketWrapper) } } /** * Callback for responses returned by the storage connector. The request will complete or error * here, if the record couldn't be found in storage no further attempts to retrieve it will be made */ function onStorageResponse ( error: string | null, recordName: string, version: number, data: any, socketWrapper: SocketWrapper | null, onComplete: onCompleteCallback, onError: onErrorCallback, services: DeepstreamServices, context: any, metaData: any, promoteToCache: boolean, message?: Message ): void { if (error) { sendError( RECORD_ACTION.RECORD_LOAD_ERROR, `error while loading ${recordName} from storage:${error}`, recordName, socketWrapper, onError, services, context, metaData, message ) } else { if (message) { onComplete.call(context, recordName, version, data || null, socketWrapper, message) } else { onComplete.call(context, recordName, version, data || null, socketWrapper) } // Promote to cache is disabled when coming from the record transition // since that might override the last set if (data && promoteToCache) { services.cache.set(recordName, version, data, () => {}, metaData) } } } /** * Callback for responses returned by the cache connector */ function onCacheResponse ( error: string | null, recordName: string, version: number, data: any, socketWrapper: SocketWrapper | null, onComplete: onCompleteCallback, onError: onErrorCallback, config: DeepstreamConfig, services: DeepstreamServices, context: any, metaData: any, promoteToCache: boolean, message?: Message ): void { if (error) { sendError( RECORD_ACTION.RECORD_LOAD_ERROR, `error while loading ${recordName} from cache:${error}`, recordName, socketWrapper, onError, services, context, metaData, message ) } else if (data) { if (message) { onComplete.call(context, recordName, version, data, socketWrapper, message) } else { onComplete.call(context, recordName, version, data, socketWrapper) } } else if (!isExcluded(config.record.storageExclusionPrefixes, recordName)) { let storageTimedOut = false const storageTimeout = setTimeout(() => { storageTimedOut = true sendError( RECORD_ACTION.STORAGE_RETRIEVAL_TIMEOUT, recordName, recordName, socketWrapper, onError, services, context, metaData, message ) }, config.record.storageRetrievalTimeout) // tslint:disable-next-line:no-shadowed-variable services.storage.get(recordName, (storageError, version, result) => { if (!storageTimedOut) { clearTimeout(storageTimeout) onStorageResponse( storageError, recordName, version!, result, socketWrapper, onComplete, onError, services, context, metaData, promoteToCache, message ) } }, metaData) } else { if (message) { onComplete.call(context, recordName, version, data, socketWrapper, message) } else { onComplete.call(context, recordName, version, data, socketWrapper) } } } /** * This function retrieves a single record from the cache or - if it isn't in the * cache - from storage. If it isn't there either it will notify its initiator * by passing null to onComplete (but not call onError). * * It also handles all the timeout and destruction steps around this operation */ export function recordRequest ( recordName: string, config: DeepstreamConfig, services: DeepstreamServices, socketWrapper: SocketWrapper | null, onComplete: onCompleteCallback, onError: onErrorCallback, context: any, metaData?: any, message?: Message, promoteToCache: boolean = true ): void { let cacheTimedOut = false const cacheTimeout = setTimeout(() => { cacheTimedOut = true sendError( RECORD_ACTION.CACHE_RETRIEVAL_TIMEOUT, recordName, recordName, socketWrapper, onError, services, context, metaData, message ) }, config.record.cacheRetrievalTimeout) services.cache.get(recordName, (error, version, data) => { if (!cacheTimedOut) { clearTimeout(cacheTimeout) onCacheResponse( error, recordName, version!, data!, socketWrapper, onComplete, onError, config, services, context, metaData, promoteToCache, message ) } }, metaData) } export function recordRequestBinding (config: DeepstreamConfig, services: DeepstreamServices, context: any, metaData: any) { return function (recordName: string, socketWrapper: SocketWrapper, onComplete: onCompleteCallback, onError: onErrorCallback, message?: Message) { recordRequest (recordName, config, services, socketWrapper, onComplete, onError, context, metaData, message) } } ================================================ FILE: src/handlers/record/record-transition.spec.ts ================================================ import 'mocha' import * as M from './test-messages' import * as C from '../../constants' import { getTestMocks } from '../../test/helper/test-mocks' import * as testHelper from '../../test/helper/test-helper' import { RecordTransition } from './record-transition' import { PromiseDelay } from '../../utils/utils'; describe('RecordTransition', () => { let services let config // let socketWrapper let recordTransition let testMocks let client beforeEach(() => { testMocks = getTestMocks() client = testMocks.getSocketWrapper() const options = testHelper.getDeepstreamOptions() services = options.services config = options.config recordTransition = new RecordTransition(M.recordUpdate.name, config, services, testMocks.recordHandler) }) afterEach(() => { client.socketWrapperMock.verify() testMocks.recordHandlerMock.verify() }) it('sends write acknowledgement with sync cache and async storage', async () => { const message: C.RecordWriteMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'random-name', correlationId: '30', data: 'somedata', isWriteAck: true, version: -1, parsedData: { name: 'somedata' } } services.storage.nextOperationWillBeSuccessful = true services.storage.nextOperationWillBeSynchronous = false services.cache.nextOperationWillBeSuccessful = true services.cache.nextOperationWillBeSynchronous = true client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.WRITE_ACKNOWLEDGEMENT, name: message.name, correlationId: message.correlationId, isWriteAck: true }) recordTransition.add(client.socketWrapper, message, true) // Wait for the async callback to fire await PromiseDelay(60) }) }) /* describe.skip('record transitions', () => { let services let config let socketWrapper let recordTransition let testMocks let client beforeEach(() => { testMocks = getTestMocks() client = testMocks.getSocketWrapper() const options = testHelper.getDeepstreamOptions() services = options.services config = options.config recordTransition = new RecordTransition(M.recordUpdate.name, config, services, testMocks.recordHandler) services.cache.set('some-record', M.recordData, () => {}) }) afterEach(() => { client.socketWrapperMock.verify() testMocks.recordHandlerMock.verify() }) it('retrieves the empty record', () => { testMocks.recordHandlerMock .expects('broadcastUpdate') .once() .withExactArgs(M.recordUpdate.name, M.recordUpdate, false, client.socketWrapper) testMocks.recordHandlerMock .expects('transitionComplete') .once() .withExactArgs(M.recordUpdate.name) recordTransition.add(client.socketWrapper, Object.assign({}, M.recordUpdate)) }) it('adds an update to the queue', () => { services.cache.nextGetWillBeSynchronous = false expect(recordTransition.steps.length).to.equal(0) recordTransition.add(client.socketWrapper, M.recordUpdate) expect(recordTransition.steps.length).to.equal(1) }) it('adds a message with invalid data to the queue', () => { const invalidMessage = { topic: C.TOPIC.RECORD, action: C.ACTIONS.UPDATE, name: 'bob', version: 1, data: '{ b ]' } client.socketWrapperMock .expects('sendError') .once() .withExactArgs(invalidMessage, C.EVENT.INVALID_MESSAGE_DATA) recordTransition.add(client.socketWrapper, invalidMessage) }) it('adds a message with null data to the queue', () => { const invalidMessage = { topic: C.TOPIC.RECORD, action: C.ACTIONS.UPDATE, name: 'bob', version: 1, data: 'null' } client.socketWrapperMock .expects('sendError') .once() .withExactArgs(invalidMessage, C.EVENT.INVALID_MESSAGE_DATA) recordTransition.add(client.socketWrapper, invalidMessage) }) it('adds a message with string data to the queue', () => { const invalidMessage = { topic: C.TOPIC.RECORD, action: C.ACTIONS.UPDATE, name: 'bob', version: 1, data: 'This is a string' } client.socketWrapperMock .expects('sendError') .once() .withExactArgs(invalidMessage, C.EVENT.INVALID_MESSAGE_DATA) recordTransition.add(client.socketWrapper, invalidMessage) }) it('adds a message with numeric data to the queue', () => { const invalidMessage = { topic: C.TOPIC.RECORD, action: C.ACTIONS.UPDATE, name: 'bob', version: 1, data: '1234' } client.socketWrapperMock .expects('sendError') .once() .withExactArgs(invalidMessage, C.EVENT.INVALID_MESSAGE_DATA) recordTransition.add(client.socketWrapper, invalidMessage) }) it.skip('retrieves the empty record', (done) => { recordRequestMockCallback({ _v: 0, _d: { firstname: 'Egon' } }) expect(recordTransition._record).to.deep.equal({ _v: 1, _d: { firstname: 'Egon' } }) expect(services.cache.completedSetOperations).to.equal(0) const check = setInterval(() => { if (services.cache.completedSetOperations === 1) { expect(recordHandlerMock._$broadcastUpdate).to.have.been.calledWith('recordName', patchMessage, false, socketWrapper) expect(recordHandlerMock._$transitionComplete).to.have.callCount(0) expect(recordTransition._record).to.deep.equal({ _v: 2, _d: { lastname: 'Peterson' } }) clearInterval(check) done() } }, 1) }) it.skip('receives a patch message whilst the transition is in progress', () => { expect(recordHandlerMock._$transitionComplete).to.have.callCount(0) recordTransition.add(socketWrapper, 3, patchMessage2) }) it('returns hasVersion for 1,2 and 3', () => { services.cache.nextOperationWillBeSynchronous = false recordTransition.add(client.socketWrapper, M.recordUpdate) expect(recordTransition.hasVersion(0)).to.equal(true) expect(recordTransition.hasVersion(1)).to.equal(true) expect(recordTransition.hasVersion(2)).to.equal(true) expect(recordTransition.hasVersion(3)).to.equal(true) expect(recordTransition.hasVersion(4)).to.equal(true) expect(recordTransition.hasVersion(5)).to.equal(true) expect(recordTransition.hasVersion(6)).to.equal(true) expect(recordTransition.hasVersion(7)).to.equal(false) expect(recordTransition.hasVersion(8)).to.equal(false) }) it.skip('processes a queue', (done) => { testMocks.recordHandlerMock .expects('broadcastUpdate') .once() .withExactArgs(M.recordPatch.name, M.recordPatch, false, client.socketWrapper) testMocks.recordHandlerMock .expects('broadcastUpdate') .once() .withExactArgs(M.recordUpdate.name, M.recordUpdate, false, client.socketWrapper) testMocks.recordHandlerMock .expects('transitionComplete') .once() .withExactArgs(M.recordPatch.name) client.socketWrapperMock .expects('sendError') .never() recordTransition.add(client.socketWrapper, M.recordPatch) recordTransition.add(client.socketWrapper, Object.assign({}, M.recordUpdate, { version: M.recordUpdate.version + 1 })) recordTransition.add(client.socketWrapper, Object.assign({}, M.recordUpdate, { version: M.recordUpdate.version + 2 })) }) describe('does not store excluded data', () => { it('retrieves the empty record', () => { expect(recordHandlerMock._$broadcastUpdate).to.have.callCount(0) expect(recordHandlerMock._$transitionComplete).to.have.callCount(0) recordRequestMockCallback() expect(recordHandlerMock._$broadcastUpdate).to.have.been.calledWith('no-storage/1', patchMessage, false, socketWrapper) expect(recordHandlerMock._$transitionComplete).to.have.been.calledWith('no-storage/1') }) it('does not store transition in storage', (done) => { const check = setInterval(() => { if (services.storage.completedSetOperations === 0) { clearInterval(check) done() } }, 1) }) }) describe('destroys a transition between steps', () => { const secondPatchMessage = { topic: 'RECORD', action: 'P', data: ['recordName', 2, 'firstname', 'SEgon'] } before(() => { createRecordTransition() services.cache.nextOperationWillBeSynchronous = false }) it('adds a patch to the queue', () => { expect(() => { recordTransition.add(socketWrapper, 2, secondPatchMessage) expect(recordTransition.hasVersion(2)).to.equal(true) }).not.to.throw() }) }) describe('tries to set a record, but both cache and storage fail', () => { before(() => { createRecordTransition() services.cache.nextOperationWillBeSynchronous = true services.cache.nextOperationWillBeSuccessful = false services.storage.nextOperationWillBeSuccessful = false recordRequestMockCallback() }) it('logged an error', () => { expect(services.logger.logSpy).to.have.been.calledWith(3, 'RECORD_UPDATE_ERROR', 'storageError') }) }) describe('destroys the transition', () => { before(() => { createRecordTransition() services.cache.nextOperationWillBeSynchronous = false }) it('destroys the transition', (done) => { recordTransition.destroy() expect(recordTransition.isDestroyed).to.equal(true) expect(recordTransition.steps).to.equal(null) setTimeout(() => { // just leave this here to make sure no error is thrown when the // record request returns after 30ms done() }, 50) }) it('calls destroy a second time without causing problems', () => { recordTransition.destroy() expect(recordTransition.isDestroyed).to.equal(true) expect(recordTransition.steps).to.equal(null) }) }) describe('recordRequest returns an error', () => { before(() => { createRecordTransition() services.cache.nextOperationWillBeSynchronous = false }) it('receives an error', () => { expect(socketWrapper.socket.lastSendMessage).to.equal(null) recordRequestMockCallback('errorMsg', true) expect(services.logger.logSpy).to.have.been.calledWith(3, 'RECORD_UPDATE_ERROR', 'errorMsg') expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|E|RECORD_UPDATE_ERROR|1+')) }) }) describe('recordRequest returns null', () => { before(() => { createRecordTransition() services.cache.nextOperationWillBeSynchronous = false }) it('receives a non existant error', () => { expect(socketWrapper.socket.lastSendMessage).to.equal(null) recordRequestMockCallback(null) expect(services.logger.logSpy).to.have.been.calledWith(3, 'RECORD_UPDATE_ERROR', 'Received update for non-existant record recordName') expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|E|RECORD_UPDATE_ERROR|1+')) }) }) describe('handles invalid message data', () => { const invalidPatchMessage = { topic: 'RECORD', action: 'P', data: ['recordName', 2, 'somepath', 'O{"invalid":"json'] } before(() => { createRecordTransition('recordName', invalidPatchMessage) services.cache.nextOperationWillBeSynchronous = false }) it('receives an error', () => { expect(socketWrapper.socket.lastSendMessage).to.contain(msg('R|E|INVALID_MESSAGE_DATA|')) }) }) describe.skip('transition version conflicts', () => { const socketMock1 = new SocketMock() const socketMock2 = new SocketMock() const socketMock3 = new SocketMock() const socketWrapper1 = new SocketWrapper(socketMock1) const socketWrapper2 = new SocketWrapper(socketMock2) const socketWrapper3 = new SocketWrapper(socketMock3) const patchMessage2 = { topic: 'RECORD', action: 'P', data: ['recordName', 2, 'firstname', 'SEgon'] } before(() => { createRecordTransition('recordName') services.cache.nextOperationWillBeSynchronous = false }) it('gets a version exist error on two seperate updates but does not send error', () => { recordTransition.add(socketWrapper1, 2, patchMessage2) recordTransition.sendVersionExists({ sender: socketWrapper1, version: 1, message: patchMessage }) recordTransition.sendVersionExists({ sender: socketWrapper2, version: 1, message: patchMessage2 }) expect(socketMock1.lastSendMessage).to.equal(null) expect(socketMock2.lastSendMessage).to.equal(null) expect(socketMock3.lastSendMessage).to.equal(null) }) it('sends version exists error once record request is completed is retrieved', () => { recordRequestMockCallback({ _v: 1, _d: { lastname: 'Kowalski' } }) expect(socketMock1.lastSendMessage).to.equal(msg('R|E|VERSION_EXISTS|recordName|1|{"lastname":"Kowalski"}+')) expect(socketMock2.lastSendMessage).to.equal(msg('R|E|VERSION_EXISTS|recordName|1|{"lastname":"Kowalski"}+')) expect(socketMock3.lastSendMessage).to.equal(null) }) it('immediately sends version exists when record is already loaded', () => { socketMock1.lastSendMessage = null socketMock2.lastSendMessage = null socketMock3.lastSendMessage = null recordTransition.sendVersionExists({ sender: socketWrapper3, version: 1, message: patchMessage }) expect(socketMock1.lastSendMessage).to.equal(null) expect(socketMock2.lastSendMessage).to.equal(null) expect(socketMock3.lastSendMessage).to.equal(msg('R|E|VERSION_EXISTS|recordName|2|{"lastname":"Kowalski","firstname":"Egon"}+')) }) it('destroys the transition', (done) => { recordTransition.destroy() expect(recordTransition.isDestroyed).to.equal(true) expect(recordTransition.steps).to.equal(null) setTimeout(() => { // just leave this here to make sure no error is thrown when the // record request returns after 30ms done() }, 50) }) }) }) */ ================================================ FILE: src/handlers/record/record-transition.ts ================================================ import { setValue as setPathValue } from '../../utils/json-path' import RecordHandler from './record-handler' import { recordRequest } from './record-request' import { RecordWriteMessage, TOPIC, RECORD_ACTION, Message } from '../../constants' import { SocketWrapper, DeepstreamConfig, DeepstreamServices, MetaData, EVENT } from '@deepstream/types' import { isOfType, isExcluded } from '../../utils/utils' interface Step { message: RecordWriteMessage sender: SocketWrapper } export class RecordTransition { /** * This class manages one or more simultanious updates to the data of a record. * But: Why does that need to be so complicated and why does this class even exist? * * In short: Cross-network concurrency. If your record is written to by a single datasource * and consumed by many clients, this class is admittably overkill, but if deepstream is used to * build an app that allows many users to collaboratively edit the same dataset, sooner or later * two of them will do so at the same time and clash. * * Every deepstream record therefor has a number that's incremented with every change. * Every client sends this version number along with the changed data. If no other update has * been received for the same version in the meantime, the update is accepted and not much more * happens. * * If, however, another clients was able to send its updated version before this update was * processed, the second (later) update for the same version number is rejected and the issuing * client is notified of the change. * * The client is then expected to merge its changes on top of the new version and re-issue the * update message. * * Please note: For performance reasons, succesful updates are not explicitly acknowledged. * * It's this class' responsibility to manage this. It will be created when an update arrives and * only exist as long as it takes to apply it and make sure that no subsequent updates for the * same version are requested. * * Once the update is applied it will notify the record-handler to broadcast the * update and delete the instance of this class. */ public isDestroyed: boolean = false private steps: Step[] = [] private version: number = -1 private data: any = null private currentStep: Step | null = null private recordRequestMade: boolean = false private existingVersions: Step[] = [] private lastVersion: number | null = null private readonly writeAckSockets = new Map() private pendingStorageWrites: number = 0 private pendingCacheWrites: number = 0 constructor (private name: string, private config: DeepstreamConfig, private services: DeepstreamServices, private recordHandler: RecordHandler, private readonly metaData: MetaData) { this.onCacheSetResponse = this.onCacheSetResponse.bind(this) this.onStorageSetResponse = this.onStorageSetResponse.bind(this) this.onRecord = this.onRecord.bind(this) this.onFatalError = this.onFatalError.bind(this) } /** * Checks if a specific version number is already processed or * queued for processing */ public hasVersion (version: number): boolean { if (this.lastVersion === null) { return false } return version !== -1 && version <= this.lastVersion } /** * Send version exists error if the record has been already loaded, else * store the version exists error to send to the sockerWrapper once the * record is loaded */ public sendVersionExists (step: Step): void { const socketWrapper = step.sender if (this.data) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.VERSION_EXISTS, originalAction: step.message.action, name: this.name, version: this.version, parsedData: this.data, isWriteAck: step.message.isWriteAck, correlationId: step.message.correlationId }) this.services.logger.warn( RECORD_ACTION[RECORD_ACTION.VERSION_EXISTS], `${socketWrapper.userId} tried to update record ${this.name} to version ${step.message.version} but it already was ${this.version}`, this.metaData, ) } else { this.existingVersions.push({ sender: socketWrapper, message: step.message, }) } } /** * Adds a new step (either an update or a patch) to the record. The step * will be queued or executed immediatly if the queue is empty * * This method will also retrieve the current record's data when called * for the first time */ public add (socketWrapper: SocketWrapper, message: RecordWriteMessage, upsert: boolean = false): void { const version = message.version const update = { message, sender: socketWrapper, } const result = socketWrapper.parseData(message) if (result instanceof Error) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.INVALID_MESSAGE_DATA, data: message.data }) return } if (message.action === RECORD_ACTION.UPDATE) { if (!isOfType(message.parsedData, 'object') && !isOfType(message.parsedData, 'array')) { socketWrapper.sendMessage({ ...message, action: RECORD_ACTION.INVALID_MESSAGE_DATA, originalAction: message.action, }) return } } if (this.lastVersion !== null && version > this.lastVersion + 1) { socketWrapper.sendMessage({ ...message, action: RECORD_ACTION.INVALID_VERSION, originalAction: message.action, }) return } if (this.lastVersion !== null && this.lastVersion !== version - 1) { this.sendVersionExists(update) return } if (version !== -1) { this.lastVersion = version } this.steps.push(update) if (this.recordRequestMade === false) { this.recordRequestMade = true recordRequest( this.name, this.config, this.services, socketWrapper, (r: string, v: number, d: any) => this.onRecord(v, d, upsert), this.onCacheRequestError, this, this.metaData, undefined, false ) } else if (this.steps.length === 1) { this.next() } } /** * Destroys the instance */ public destroy (error?: string | null): void { if (this.isDestroyed) { return } if (error) { this.sendWriteAcknowledgementErrors(error.toString()) // send message in order to alert current message sender that the operation failed if (this.currentStep && this.currentStep.sender && !this.currentStep.sender.isRemote) { this.currentStep.sender.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.RECORD_UPDATE_ERROR, name: this.currentStep.message.name, isError: true }) } } this.recordHandler.transitionComplete(this.name) this.isDestroyed = true } /** * Callback for successfully retrieved records */ private onRecord (version: number, data: any, upsert: boolean) { if (data === null) { if (!upsert) { this.onFatalError(`Received update for non-existant record ${this.name}`) return } this.data = {} this.version = 0 } else { this.version = version this.data = data } this.flushVersionExists() this.next() } /** * Once the record is loaded this method is called recoursively * for every step in the queue of pending updates. * * It will apply every patch or update and - once done - either * call itself to process the next one or destroy the RecordTransition * of the queue has been drained */ private next (): void { if (this.isDestroyed === true) { return } if (this.data === null) { return } const currentStep = this.steps.shift() if (!currentStep) { this.destroy(null) return } this.currentStep = currentStep let message = currentStep.message if (message.version === -1) { message = Object.assign({}, message, { version: this.version + 1 }) currentStep.message = message } if (message.version > this.version + 1) { currentStep.sender.sendMessage({ ...message, action: RECORD_ACTION.INVALID_VERSION, originalAction: currentStep.message.action, version: this.version }) return } if (this.version !== message.version - 1) { this.sendVersionExists(currentStep) this.next() return } this.version = message.version if (message.path) { setPathValue(this.data, message.path, message.parsedData) } else { this.data = message.parsedData } /* * Please note: saving to storage is called first to allow for synchronous cache * responses to destroy the transition, it is however not on the critical path * and the transition will continue straight away, rather than wait for the storage response * to be returned. * * If the storage response is asynchronous and write acknowledgement is enabled, the transition * will not be destroyed until writing to storage is finished */ if (!isExcluded(this.config.record.storageExclusionPrefixes, this.name)) { this.pendingStorageWrites++ if (message.isWriteAck) { this.setUpWriteAcknowledgement(message, this.currentStep.sender) this.services.storage.set(this.name, this.version, this.data, (error) => this.onStorageSetResponse(error, this.currentStep!.sender, message), this.metaData) } else { this.services.storage.set(this.name, this.version, this.data, this.onStorageSetResponse, this.metaData) } } this.pendingCacheWrites++ if (message.isWriteAck) { this.setUpWriteAcknowledgement(message, this.currentStep.sender) this.services.cache.set(this.name, this.version, this.data, (error) => this.onCacheSetResponse(error, this.currentStep!.sender, message), this.metaData) } else { this.services.cache.set(this.name, this.version, this.data, this.onCacheSetResponse, this.metaData) } } private setUpWriteAcknowledgement (message: Message, socketWrapper: SocketWrapper) { const correlationId = message.correlationId as string const response = this.writeAckSockets.get(socketWrapper) if (!response) { this.writeAckSockets.set(socketWrapper, { [correlationId]: 1 }) return } response[correlationId] = response[correlationId] ? response[correlationId] + 1 : 1 this.writeAckSockets.set(socketWrapper, response) } /** * Send all the stored version exists errors once the record has been loaded. */ private flushVersionExists (): void { for (let i = 0; i < this.existingVersions.length; i++) { this.sendVersionExists(this.existingVersions[i]) } this.existingVersions = [] } private handleWriteAcknowledgement (error: string | null, socketWrapper: SocketWrapper, originalMessage: Message) { const correlationId = originalMessage.correlationId as string const response = this.writeAckSockets.get(socketWrapper) if (!response) { return } response[correlationId]-- if (response[correlationId] === 0) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.WRITE_ACKNOWLEDGEMENT, name: originalMessage.name, correlationId, isWriteAck: true }) delete response[correlationId] } if (Object.keys(response).length === 0) { this.writeAckSockets.delete(socketWrapper) } } private onCacheRequestError (error: string) { const errorMessage = `Cache retrieval error, nuking record transition for ${this.name}, ${error}` this.services.logger.error(EVENT.ERROR, errorMessage) this.destroy(errorMessage) } /** * Callback for responses returned by cache.set(). If an error * is returned the queue will be destroyed, otherwise * the update will be broadcast to other subscribers and the * next step invoked */ private onCacheSetResponse (error: string | null, socketWrapper?: SocketWrapper, message?: Message): void { if (this.currentStep === null) { const errorMessage = `Cache results received without a valid step in record transition for ${this.name}` this.services.logger.error(EVENT.ERROR, errorMessage) this.destroy(errorMessage) return } if (message && socketWrapper) { this.handleWriteAcknowledgement(error, socketWrapper, message) } if (error) { this.onFatalError(error) } else if (this.isDestroyed === false) { const { isWriteAck, correlationId } = this.currentStep.message // // Delete values that should not be broadcast this.currentStep.message.isWriteAck = false delete this.currentStep.message.correlationId // TODO: Optimise this.recordHandler.broadcastUpdate( this.name, this.currentStep.message, false, this.currentStep.sender, ) // Restore the message for other callers to use this.currentStep.message.isWriteAck = isWriteAck this.currentStep.message.correlationId = correlationId this.next() } else if (this.steps.length === 0 && this.pendingCacheWrites === 0 && this.pendingStorageWrites === 0) { this.destroy(null) } } /** * Callback for responses returned by storage.set() */ private onStorageSetResponse (error: string | null, socketWrapper?: SocketWrapper, message?: Message): void { if (message && socketWrapper) { this.handleWriteAcknowledgement(error, socketWrapper, message) } if (error) { this.onFatalError(error) } else if ( this.steps.length === 0 && this.pendingCacheWrites === 0 && this.pendingStorageWrites === 0 ) { this.destroy(null) } } /** * Sends all write acknowledgement messages at the end of a transition */ private sendWriteAcknowledgementErrors (errorMessage: string) { for (const [socketWrapper, pendingWrites] of this.writeAckSockets) { for (const correlationId in pendingWrites) { socketWrapper.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.RECORD_UPDATE_ERROR, correlationId, isWriteAck: true, isError: true }) } } this.writeAckSockets.clear() } /** * Generic error callback. Will destroy the queue and notify the senders of all pending * transitions */ private onFatalError (error: string): void { if (this.isDestroyed === true) { return } this.services.logger.error(RECORD_ACTION[RECORD_ACTION.RECORD_UPDATE_ERROR], error.toString(), this.metaData) for (let i = 0; i < this.steps.length; i++) { if (!this.steps[i].sender.isRemote) { this.steps[i].sender.sendMessage({ topic: TOPIC.RECORD, action: RECORD_ACTION.RECORD_UPDATE_ERROR, name: this.steps[i].message.name, isError: true, isWriteAck: this.steps[i].message.isWriteAck, correlationId: this.steps[i].message.correlationId }) } } this.destroy(error) } } ================================================ FILE: src/handlers/record/record-write-acknowledgement.spec.ts ================================================ /* 'use strict' const M = require('./messages') import * as C from '../../src/constants' import { getTestMocks } from '../test-helper/test-mocks' const testHelper = require('../test-helper/test-helper') const RecordTransition = require('../../src/record/record-transition').default const sinon = require('sinon') describe('record write acknowledgement', () => { let config let services let socketWrapper let recordTransition let testMocks let client beforeEach(() => { testMocks = getTestMocks() client = testMocks.getSocketWrapper() const options = testHelper.getDeepstreamOptions() config = options.config services = options.services recordTransition = new RecordTransition(M.recordUpdate.name, config, services, testMocks.recordHandler) }) afterEach(() => { client.socketWrapperMock.verify() }) it('sends write success to socket', () => { client.socketWrapperMock .expects('sendError') .never() client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(M.writeAck, true) recordTransition.add(client.socketWrapper, M.recordUpdateWithAck, true) }) it('sends write failure to socket', () => { services.storage.nextOperationWillBeSuccessful = false client.socketWrapperMock .expects('sendError') .once() .withExactArgs(M.recordUpdateWithAck, C.EVENT.RECORD_UPDATE_ERROR) client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RECORD, action: C.ACTIONS.WRITE_ACKNOWLEDGEMENT, name: M.recordUpdateWithAck.name, data: [[-1], C.EVENT.RECORD_LOAD_ERROR] }, true) recordTransition.add(client.socketWrapper, M.recordUpdateWithAck, true) }) it.skip('multiple write acknowledgements', () => { // processes the next step in the queue const check = setInterval(() => { if (services.storage.completedSetOperations === 2) { expect(recordHandlerMock._$broadcastUpdate).to.have.been.calledWith('recordName', patchMessage2, false, socketWrapper2) expect(recordHandlerMock._$transitionComplete).to.have.callCount(0) expect(recordTransition._record).to.deep.equal({ _v: 3, _d: { firstname: 'Lana', lastname: 'Kowalski' } }) clearInterval(check) done() } }, 1) // processes the final step in the queue if (services.storage.completedSetOperations === 3) { expect(recordHandlerMock._$broadcastUpdate).to.have.been.calledWith('recordName', patchMessage3, false, socketWrapper) expect(recordHandlerMock._$transitionComplete).to.have.callCount(1) } // stored each transition in storage // services.storage.completedSetOperations === 3 // sent write acknowledgement to each client expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|WA|recordName|[1,3]|L+')) expect(socketWrapper2.socket.lastSendMessage).to.equal(msg('R|WA|recordName|[2]|L+')) }) it.skip('transition version conflicts gets a version exist error on record retrieval', () => { // services.storage.nextOperationWillBeSynchronous = false // recordTransition.add(socketWrapper, 2, updateMessage) expect(socketWrapper.socket.lastSendMessage).to.equal(null) recordRequestMockCallback({ _v: 1, _d: { lastname: 'Kowalski' } }) expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|E|VERSION_EXISTS|recordName|1|{"lastname":"Kowalski"}|{"writeSuccess":true}+')) }) }) */ ================================================ FILE: src/handlers/record/test-messages.ts ================================================ import * as C from '../../constants' export const deletionMsg = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETE, name: 'someRecord' } export const deletionSuccessMsg = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETE_SUCCESS, name: 'someRecord' } export const anotherDeletionMsg = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETE, name: 'no-storage/1' } export const anotherDeletionSuccessMsg = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETE_SUCCESS, name: 'no-storage/1' } export const subscribeCreateAndReadMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.SUBSCRIBECREATEANDREAD, names: ['some-record'] } export const readResponseMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ_RESPONSE, name: 'some-record', version: 0, parsedData: {} } export const subscribeCreateAndReadPermissionErrorMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.MESSAGE_PERMISSION_ERROR, originalAction: C.RECORD_ACTION.SUBSCRIBECREATEANDREAD, names: ['some-record'] } export const subscribeCreateAndReadDeniedMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.MESSAGE_DENIED, originalAction: C.RECORD_ACTION.SUBSCRIBECREATEANDREAD, names: ['some-record'] } export const subscribeMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.SUBSCRIBE, names: ['some-record'] } export const unsubscribeMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UNSUBSCRIBE, names: ['some-record'] } export const recordSnapshotMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'some-record' } export const recordHeadMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.HEAD, name: 'some-record' } export const recordHeadResponseMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.HEAD_RESPONSE, name: 'some-record' } export const recordData = { name: 'Kowalski' } export const recordVersion = 5 export const recordUpdate = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'some-record', version: recordVersion + 1, parsedData: recordData, isWriteAck: false } export const recordUpdateWithAck = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'some-record', version: -1, parsedData: recordData, isWriteAck: true } export const recordPatch = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH, name: 'some-record', version: recordVersion + 1, path: 'lastname', parsedData: 'Egon', isWriteAck: false } export const recordPatchWithAck = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH, name: 'some-record', version: 4, path: 'lastname', parsedData: 'Egon', isWriteAck: true } export const recordDelete = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.DELETE, name: 'some-record' } export const createAndUpdate = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.CREATEANDUPDATE, name: 'some-record', version: -1, parsedData: recordData, isWriteAck: false } export const listenAcceptMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.LISTEN_ACCEPT, name: 'record/.*', subscription: 'record/A' } export const listenRejectMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.LISTEN_REJECT, name: 'record/.*', subscription: 'record/A' } export const unlistenMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UNLISTEN, name: 'record/.*' } export const listenMessage = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.LISTEN, name: 'record/.*' } export const writeAck = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.WRITE_ACKNOWLEDGEMENT, name: 'some-record', data: [[-1], null] } export const notify = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.NOTIFY, names: ['record1', 'record2'] } ================================================ FILE: src/handlers/rpc/rpc-handler.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as C from '../../constants' import RpcHandler from './rpc-handler' import * as testHelper from '../../test/helper/test-helper' import { getTestMocks } from '../../test/helper/test-mocks' import { RpcProxy } from './rpc-proxy' const options = testHelper.getDeepstreamOptions() const config = options.config const services = options.services describe('the rpcHandler routes events correctly', () => { let testMocks let rpcHandler let requestor let provider beforeEach(() => { testMocks = getTestMocks() rpcHandler = new RpcHandler(config, services, testMocks.subscriptionRegistry) requestor = testMocks.getSocketWrapper('requestor', {}, { color: 'blue' }) provider = testMocks.getSocketWrapper('provider') }) afterEach(() => { testMocks.subscriptionRegistryMock.verify() requestor.socketWrapperMock.verify() provider.socketWrapperMock.verify() }) it('routes subscription messages', () => { const subscriptionMessage = { topic: C.TOPIC.RPC, action: C.RPC_ACTION.PROVIDE, names: ['someRPC'], correlationId: '123' } testMocks.subscriptionRegistryMock .expects('subscribeBulk') .once() .withExactArgs(subscriptionMessage, provider.socketWrapper) rpcHandler.handle(provider.socketWrapper, subscriptionMessage) }) describe('when receiving a request', () => { const requestMessage = { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST, name: 'addTwo', correlationId: 1234, data: '{"numA":5, "numB":7}' } const acceptMessage = { topic: C.TOPIC.RPC, action: C.RPC_ACTION.ACCEPT, name: 'addTwo', correlationId: 1234 } const responseMessage = { topic: C.TOPIC.RPC, action: C.RPC_ACTION.RESPONSE, name: 'addTwo', correlationId: 1234, data: '12' } const errorMessage = { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST_ERROR, isError: true, name: 'addTwo', correlationId: 1234, data: 'ErrorOccured' } beforeEach(() => { testMocks.subscriptionRegistryMock .expects('getLocalSubscribers') .once() .withExactArgs('addTwo') .returns([provider.socketWrapper]) }) it('forwards it to a provider', () => { provider.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(Object.assign({ requestorData: { color: 'blue' }, requestorName: 'requestor' }, requestMessage)) rpcHandler.handle(requestor.socketWrapper, requestMessage) }) it('accepts first accept', () => { requestor.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(acceptMessage) rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, acceptMessage) }) it('errors when recieving more than one ack', () => { provider.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.MULTIPLE_ACCEPT, name: requestMessage.name, correlationId: requestMessage.correlationId }) provider.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(Object.assign({ requestorData: { color: 'blue' }, requestorName: 'requestor' }, requestMessage)) rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, acceptMessage) rpcHandler.handle(provider.socketWrapper, acceptMessage) }) it('gets a response', () => { requestor.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(responseMessage) rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, responseMessage) }) it('replies with an error to additonal responses', () => { rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, responseMessage) provider.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.INVALID_RPC_CORRELATION_ID, originalAction: responseMessage.action, name: responseMessage.name, correlationId: responseMessage.correlationId, isError: true }) rpcHandler.handle(provider.socketWrapper, responseMessage) }) it('gets an error', () => { requestor.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(errorMessage) rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, errorMessage) }) it('replies with an error after the first message', () => { rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, errorMessage) provider.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.INVALID_RPC_CORRELATION_ID, originalAction: errorMessage.action, name: errorMessage.name, correlationId: errorMessage.correlationId, isError: true }) rpcHandler.handle(provider.socketWrapper, errorMessage) }) it('supports multiple RPCs in quick succession', () => { testMocks.subscriptionRegistryMock .expects('getLocalSubscribers') .exactly(49) .withExactArgs('addTwo') .returns([provider.socketWrapper]) expect(() => { for (let i = 0; i < 50; i++) { rpcHandler.handle(requestor.socketWrapper, requestMessage) } }).not.to.throw() }) it('times out if no ack is received', (done) => { rpcHandler.handle(requestor.socketWrapper, requestMessage) requestor.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.ACCEPT_TIMEOUT, name: requestMessage.name, correlationId: requestMessage.correlationId }) setTimeout(done, config.rpc.ackTimeout * 2) }) it('times out if response is not received in time', (done) => { rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, acceptMessage) requestor.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.RESPONSE_TIMEOUT, name: requestMessage.name, correlationId: requestMessage.correlationId }) setTimeout(done, config.rpc.responseTimeout * 2) }) // Should an Ack for a non existant rpc should error? it.skip('ignores ack message if it arrives after response', (done) => { provider.socketWrapperMock .expects('sendMessage') .twice() rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, responseMessage) setTimeout(() => { rpcHandler.handle(provider.socketWrapper, acceptMessage) done() }, 30) }) it('doesn\'t throw error on response after timeout', (done) => { rpcHandler.handle(requestor.socketWrapper, requestMessage) rpcHandler.handle(provider.socketWrapper, acceptMessage) requestor.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.RESPONSE_TIMEOUT, name: requestMessage.name, correlationId: requestMessage.correlationId }) setTimeout(() => { provider.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.INVALID_RPC_CORRELATION_ID, originalAction: responseMessage.action, name: responseMessage.name, correlationId: responseMessage.correlationId, isError: true }) rpcHandler.handle(provider.socketWrapper, responseMessage) done() }, 30) }) }) describe.skip('rpc handler returns alternative providers for the same rpc', () => { let providerForA1 let providerForA2 let providerForA3 let usedProviders before(() => { testMocks = getTestMocks() providerForA1 = testMocks.getSocketWrapper('a1') providerForA2 = testMocks.getSocketWrapper('a2') providerForA3 = testMocks.getSocketWrapper('a3') rpcHandler = new RpcHandler(config, services, testMocks.subscriptionRegistry) testMocks.subscriptionRegistryMock .expects('getLocalSubscribers') .once() .withExactArgs('rpcA') .returns([ providerForA1.socketWrapper ]) rpcHandler.handle(providerForA1.socketWrapper, { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST, name: 'rpcA', correlationId: '1234', data: 'U' }) usedProviders = [providerForA1.socketWrapper] testMocks.subscriptionRegistryMock .expects('getLocalSubscribers') .exactly(5) .withExactArgs('rpcA') .returns([ providerForA2.socketWrapper, providerForA3.socketWrapper ]) testMocks.subscriptionRegistryMock .expects('getAllRemoteServers') .thrice() .withExactArgs('rpcA') .returns(['random-server-1', 'random-server-2']) }) after(() => { testMocks.subscriptionRegistryMock.verify() }) it('gets alternative rpc providers', () => { let alternativeProvider // first proxy alternativeProvider = rpcHandler.getAlternativeProvider('rpcA', '1234') expect(alternativeProvider).not.to.equal(null) expect(alternativeProvider instanceof RpcProxy).to.equal(false) expect(usedProviders.indexOf(alternativeProvider)).to.equal(-1) // second provider alternativeProvider = rpcHandler.getAlternativeProvider('rpcA', '1234') expect(alternativeProvider).not.to.equal(null) expect(alternativeProvider instanceof RpcProxy).to.equal(false) expect(usedProviders.indexOf(alternativeProvider)).to.equal(-1) // remote provider alternativeProvider = rpcHandler.getAlternativeProvider('rpcA', '1234') expect(alternativeProvider).not.to.equal(null) expect(alternativeProvider instanceof RpcProxy).to.equal(true) expect(usedProviders.indexOf(alternativeProvider)).to.equal(-1) // remote alternative provider alternativeProvider = rpcHandler.getAlternativeProvider('rpcA', '1234') expect(alternativeProvider).not.to.equal(null) expect(alternativeProvider instanceof RpcProxy).to.equal(true) expect(usedProviders.indexOf(alternativeProvider)).to.equal(-1) // null alternativeProvider = rpcHandler.getAlternativeProvider('rpcA', '1234') expect(alternativeProvider).to.equal(null) }) }) describe('the rpcHandler uses requestor fields correctly', () => { beforeEach(() => { testMocks = getTestMocks() requestor = testMocks.getSocketWrapper('requestor', {}, { bestLanguage: 'not BF' }) provider = testMocks.getSocketWrapper('provider') testMocks.subscriptionRegistryMock .expects('getLocalSubscribers') .once() .withExactArgs('addTwo') .returns([provider.socketWrapper]) }) afterEach(() => { testMocks.subscriptionRegistryMock.verify() requestor.socketWrapperMock.verify() provider.socketWrapperMock.verify() }) const requestMessage = { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST, name: 'addTwo', correlationId: 1234, data: '{"numA":5, "numB":7}' } for (const nameAvailable of [true, false]) { for (const dataAvailable of [true, false]) { const name = `name=${nameAvailable} data=${dataAvailable}` it(name, () => { config.rpc.provideRequestorName = nameAvailable config.rpc.provideRequestorData = dataAvailable const expectedMessage = Object.assign({}, requestMessage) if (nameAvailable) { Object.assign(expectedMessage, { requestorName: 'requestor' }) } if (dataAvailable) { Object.assign(expectedMessage, { requestorData: { bestLanguage: 'not BF' } }) } provider.socketWrapperMock .expects('sendMessage') .once() .withExactArgs(expectedMessage) rpcHandler = new RpcHandler(config, services, testMocks.subscriptionRegistry) rpcHandler.handle(requestor.socketWrapper, requestMessage) }) } } // it ('overwrites fake requestorName and fake requestorData', () => { // config.provideRPCRequestorDetails = true // config.RPCRequestorNameTerm = null // config.RPCRequestorDataTerm = null // // provider.socketWrapperMock // .expects('sendMessage') // .once() // .withExactArgs(Object.assign({ // requestorName: 'requestor', // requestorData: { bestLanguage: 'not BF' } // }, requestMessage)) // // const fakeRequestMessage = Object.assign({ // requestorName: 'evil-requestor', // requestorData: { bestLanguage: 'malbolge' } // }, requestMessage) // rpcHandler = new RpcHandler(config, services, testMocks.subscriptionRegistry) // rpcHandler.handle(requestor.socketWrapper, fakeRequestMessage) // }) }) }) ================================================ FILE: src/handlers/rpc/rpc-handler.ts ================================================ import { PARSER_ACTION, RPC_ACTION, TOPIC, RPCMessage, BulkSubscriptionMessage, STATE_REGISTRY_TOPIC } from '../../constants' import { getRandomIntInRange } from '../../utils/utils' import { Rpc } from './rpc' import { RpcProxy } from './rpc-proxy' import { SimpleSocketWrapper, DeepstreamConfig, DeepstreamServices, SocketWrapper, SubscriptionRegistry, Handler } from '@deepstream/types' interface RpcData { providers: Set, servers: Set | null, rpc: Rpc } export default class RpcHandler extends Handler { private subscriptionRegistry: SubscriptionRegistry private rpcs: Map = new Map() /** * Handles incoming messages for the RPC Topic. */ constructor (private config: DeepstreamConfig, private services: DeepstreamServices, subscriptionRegistry?: SubscriptionRegistry, private metaData?: any) { super() this.subscriptionRegistry = subscriptionRegistry || services.subscriptions.getSubscriptionRegistry(TOPIC.RPC, STATE_REGISTRY_TOPIC.RPC_SUBSCRIPTIONS) this.subscriptionRegistry.setAction('NOT_SUBSCRIBED', RPC_ACTION.NOT_PROVIDED) this.subscriptionRegistry.setAction('MULTIPLE_SUBSCRIPTIONS', RPC_ACTION.MULTIPLE_PROVIDERS) this.subscriptionRegistry.setAction('SUBSCRIBE', RPC_ACTION.PROVIDE) this.subscriptionRegistry.setAction('UNSUBSCRIBE', RPC_ACTION.UNPROVIDE) } /** * Main interface. Handles incoming messages * from the message distributor */ public handle (socketWrapper: SocketWrapper, message: RPCMessage, originServerName: string): void { if (socketWrapper === null) { this.onRemoteRPCMessage(message, originServerName) return } if (message.action === RPC_ACTION.REQUEST) { this.makeRpc(socketWrapper, message, false) return } if (message.action === RPC_ACTION.PROVIDE) { this.subscriptionRegistry.subscribeBulk(message as BulkSubscriptionMessage, socketWrapper) return } if (message.action === RPC_ACTION.UNPROVIDE) { this.subscriptionRegistry.unsubscribeBulk(message as BulkSubscriptionMessage, socketWrapper) return } if ( message.action === RPC_ACTION.RESPONSE || message.action === RPC_ACTION.REJECT || message.action === RPC_ACTION.ACCEPT || message.action === RPC_ACTION.REQUEST_ERROR ) { const rpcData = this.rpcs.get(message.correlationId) if (rpcData) { this.services.logger.debug( RPC_ACTION[message.action], `name: ${message.name} with correlation id: ${message.correlationId} from ${socketWrapper.userId}`, this.metaData ) rpcData.rpc.handle(message) return } this.services.logger.warn( RPC_ACTION[RPC_ACTION.INVALID_RPC_CORRELATION_ID], `name: ${message.name} with correlation id: ${message.correlationId}`, this.metaData ) socketWrapper.sendMessage({ topic: TOPIC.RPC, action: RPC_ACTION.INVALID_RPC_CORRELATION_ID, originalAction: message.action, name: message.name, correlationId: message.correlationId, isError: true }) return } /* * RESPONSE-, ERROR-, REJECT- and ACK messages from the provider are processed * by the Rpc class directly */ this.services.logger.warn(PARSER_ACTION[PARSER_ACTION.UNKNOWN_ACTION], message.action.toString(), this.metaData) } /** * This method is called by Rpc to reroute its request * * If a provider is temporarily unable to service a request, it can reject it. Deepstream * will then try to reroute it to an alternative provider. Finding an alternative provider * happens in this method. * * Initially, deepstream will look for a local provider that hasn't been used by the RPC yet. * If non can be found, it will go through the currently avaiblable remote providers and try * find one that hasn't been used yet. * * If a remote provider couldn't be found or all remote-providers have been tried already * this method will return null - which in turn will prompt the RPC to send a NO_RPC_PROVIDER * error to the client */ public getAlternativeProvider (rpcName: string, correlationId: string): SimpleSocketWrapper | null { const rpcData = this.rpcs.get(correlationId) if (!rpcData) { // log error return null } const subscribers = Array.from(this.subscriptionRegistry.getLocalSubscribers(rpcName)) let index = getRandomIntInRange(0, subscribers.length) for (let n = 0; n < subscribers.length; ++n) { if (!rpcData.providers.has(subscribers[index])) { rpcData.providers.add(subscribers[index]) return subscribers[index] } index = (index + 1) % subscribers.length } if (!rpcData.servers) { return null } const servers = this.subscriptionRegistry.getAllRemoteServers(rpcName) index = getRandomIntInRange(0, servers.length) for (let n = 0; n < servers.length; ++n) { if (!rpcData.servers.has(servers[index])) { rpcData.servers.add(servers[index]) return new RpcProxy(this.config, this.services, servers[index], this.metaData) } index = (index + 1) % servers.length } return null } /** * Executes a RPC. If there are clients connected to * this deepstream instance that can provide the rpc, it * will be routed to a random one of them, otherwise it will be routed * to the message connector */ private makeRpc (socketWrapper: SimpleSocketWrapper, message: RPCMessage, isRemote: boolean): void { const rpcName = message.name const correlationId = message.correlationId this.services.logger.debug( RPC_ACTION[RPC_ACTION.REQUEST], `name: ${rpcName} with correlation id: ${correlationId} from ${socketWrapper.userId}`, this.metaData ) const subscribers = Array.from(this.subscriptionRegistry.getLocalSubscribers(rpcName)) const provider = subscribers[getRandomIntInRange(0, subscribers.length)] if (provider) { this.rpcs.set(correlationId, { providers: new Set([provider]), servers: isRemote ? null : new Set(), rpc: new Rpc(this, socketWrapper, provider, this.config, message), }) return } if (isRemote) { socketWrapper.sendMessage({ topic: TOPIC.RPC, action: RPC_ACTION.NO_RPC_PROVIDER, name: rpcName, correlationId }) return } this.makeRemoteRpc(socketWrapper, message) } /** * Callback to remoteProviderRegistry.getProviderProxy() * * If a remote provider is available this method will route the rpc to it. * * If no remote provider could be found this class will return a * NO_RPC_PROVIDER error to the requestor. The RPC won't continue from * thereon */ public makeRemoteRpc (requestor: SimpleSocketWrapper, message: RPCMessage): void { const rpcName = message.name const correlationId = message.correlationId const servers = this.subscriptionRegistry.getAllRemoteServers(rpcName) const server = servers[getRandomIntInRange(0, servers.length)] if (server) { const rpcProxy = new RpcProxy(this.config, this.services, server, this.metaData) this.rpcs.set(correlationId, { providers: new Set(), servers: new Set(), rpc: new Rpc(this, requestor, rpcProxy, this.config, message), }) return } this.rpcs.delete(correlationId) this.services.logger.warn( RPC_ACTION[RPC_ACTION.NO_RPC_PROVIDER], `name: ${message.name} with correlation id: ${message.correlationId}`, this.metaData ) if (!requestor.isRemote) { requestor.sendMessage({ topic: TOPIC.RPC, action: RPC_ACTION.NO_RPC_PROVIDER, name: rpcName, correlationId }) } } /** * Callback for messages that are send directly to * this deepstream instance. * * Please note: Private messages are generic, so the RPC * specific ones need to be filtered out. */ private onRemoteRPCMessage (msg: RPCMessage, originServerName: string): void { if (msg.action === RPC_ACTION.REQUEST) { const proxy = new RpcProxy(this.config, this.services, originServerName, this.metaData) this.makeRpc(proxy, msg, true) return } const rpcData = this.rpcs.get(msg.correlationId) if (!rpcData) { this.services.logger.warn( RPC_ACTION[RPC_ACTION.INVALID_RPC_CORRELATION_ID], `Message bus response for RPC that may have been destroyed: ${JSON.stringify(msg)}`, this.metaData, ) return } this.services.logger.debug( RPC_ACTION[msg.action], `name: ${msg.name} with correlation id: ${msg.correlationId} from remote server ${originServerName}`, this.metaData ) rpcData.rpc.handle(msg) } /** * Called by the RPC with correlationId to destroy itself * when lifecycle is over. */ public onRPCDestroyed (correlationId: string): void { this.rpcs.delete(correlationId) } } ================================================ FILE: src/handlers/rpc/rpc-proxy.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as C from '../../constants' import * as testHelper from '../../test/helper/test-helper' import { RpcProxy } from './rpc-proxy' const options = testHelper.getDeepstreamOptions() const config = options.config const services = options.services describe.skip('rpcProxy proxies calls from and to the remote receiver', () => { let rpcProxy beforeEach(() => { rpcProxy = new RpcProxy(config, services, 'serverNameA') }) it('manipulates the message before sending', () => { rpcProxy.send({ topic: C.TOPIC.RPC, action: C.RPC_ACTION.ACCEPT, name: 'a', correlationId: 1234 }) expect(options.message.lastDirectSentMessage).to.deep.equal({ serverName: 'serverNameA', topic: 'PRIVATE/P', message: { topic: C.TOPIC.RPC, action: C.RPC_ACTION.ACCEPT, name: 'a', correlationId: 1234 } }) }) }) ================================================ FILE: src/handlers/rpc/rpc-proxy.ts ================================================ import { RPC_ACTION, RPCMessage, ParseResult } from '../../constants' import { SimpleSocketWrapper, DeepstreamConfig, DeepstreamServices } from '@deepstream/types' /** * This class exposes an interface that mimicks the behaviour * of a SocketWrapper, connected to a local rpc provider, but * infact relays calls from and to the message connector - sneaky. */ export class RpcProxy implements SimpleSocketWrapper { public socketType = 'RpcProxy' public userId: string = 'remote server ' + this.remoteServer public clientData = null public serverData = null public isRemote = true constructor (config: DeepstreamConfig, private services: DeepstreamServices, private remoteServer: string, private metaData: any) { } public sendAckMessage (message: RPCMessage): void { } /** * Mimicks the SocketWrapper's send method, but expects a message object, * instead of a string. * * Adds additional information to the message that enables the counterparty * to identify the sender */ public sendMessage (msg: RPCMessage): void { this.services.clusterNode.sendDirect(this.remoteServer, msg, this.metaData) } /** * Mimicks the SocketWrapper's sendError method. * Sends an error on the specified topic. The * action will automatically be set to ACTION.ERROR */ public sendError (msg: RPCMessage, type: RPC_ACTION, errorMessage: string): void { if (type === RPC_ACTION.RESPONSE_TIMEOUT) { // by the time an RPC has timed out on this server, it has already timed out on the remote // (and has been cleaned up) so no point sending return } this.services.clusterNode.sendDirect(this.remoteServer, msg, this.metaData) } public parseMessage (serializedMessage: any): ParseResult[] { throw new Error('Method not implemented.') } } ================================================ FILE: src/handlers/rpc/rpc.ts ================================================ import { RPC_ACTION, TOPIC, RPCMessage, Message } from '../../constants' import RpcHandler from './rpc-handler' import { SimpleSocketWrapper, DeepstreamConfig } from '@deepstream/types' /** * Relays a remote procedure call from a requestor to a provider and routes * the providers response to the requestor. Provider might either be a locally * connected SocketWrapper or a RpcProviderProxy that forwards messages * from a remote provider within the network */ export class Rpc { private message: Message private correlationId: string private rpcName: string private isAccepted: boolean = false private acceptTimeout: any private responseTimeout: any /** */ constructor (private rpcHandler: RpcHandler, private requestor: SimpleSocketWrapper, private provider: SimpleSocketWrapper, private config: DeepstreamConfig, message: RPCMessage) { this.rpcName = message.name this.correlationId = message.correlationId this.message = { ...message, ...this.getRequestor(requestor) } this.setProvider(provider) } private getRequestor (requestor: SimpleSocketWrapper): any { const provideAll = this.config.rpc.provideRequestorName && this.config.rpc.provideRequestorData switch (true) { case provideAll: return { requestorName: requestor.userId, requestorData: requestor.clientData } case this.config.rpc.provideRequestorName: return { requestorName: requestor.userId } case this.config.rpc.provideRequestorData: return { requestorData: requestor.clientData } default: return {} } } /** * Processor for incoming messages from the RPC provider. The * RPC provider is expected to send two messages, * * RPC|A|REQ|| * * and * * RPC|RES||] * * Both of these messages will just be forwarded directly * to the requestor */ public handle (message: RPCMessage): void { if (message.correlationId !== this.correlationId) { return } if (message.action === RPC_ACTION.ACCEPT) { this.handleAccept(message) return } if (message.action === RPC_ACTION.REJECT || message.action === RPC_ACTION.NO_RPC_PROVIDER) { this.reroute() return } if (message.action === RPC_ACTION.RESPONSE || message.action === RPC_ACTION.REQUEST_ERROR) { this.requestor.sendMessage(message) this.destroy() } } /** * Destroys this Rpc, either because its completed or because a timeout has occured */ public destroy (): void { clearTimeout(this.acceptTimeout) clearTimeout(this.responseTimeout) this.rpcHandler.onRPCDestroyed(this.correlationId) } /** * By default, a RPC is the communication between one requestor * and one provider. If the original provider however rejects * the request, deepstream will try to re-route it to another provider. * * This happens in the reroute method. This method will query * the rpc-handler for an alternative provider and - if it has * found one - call this method to replace the provider and re-do * the second leg of the rpc */ private setProvider (provider: SimpleSocketWrapper): void { clearTimeout(this.acceptTimeout) clearTimeout(this.responseTimeout) this.provider = provider this.acceptTimeout = setTimeout(this.onAcceptTimeout.bind(this), this.config.rpc.ackTimeout) this.responseTimeout = setTimeout(this.onResponseTimeout.bind(this), this.config.rpc.responseTimeout) this.provider.sendMessage(this.message) } /** * Handles rpc acknowledgement messages from the provider. * If more than one Ack is received an error will be returned * to the provider */ private handleAccept (message: RPCMessage) { if (this.isAccepted === true) { this.provider.sendMessage({ topic: TOPIC.RPC, action: RPC_ACTION.MULTIPLE_ACCEPT, name: this.message.name, correlationId: this.message.correlationId }) return } clearTimeout(this.acceptTimeout) this.isAccepted = true this.requestor.sendMessage(message) } /** * This method handles rejection messages from the current provider. If * a provider is temporarily unable to serve a request, it can reject it * and deepstream will try to reroute to an alternative provider * * If no alternative provider could be found, this method will send a NO_RPC_PROVIDER * error to the client and destroy itself */ public reroute (): void { const alternativeProvider = this.rpcHandler.getAlternativeProvider(this.rpcName, this.correlationId) if (alternativeProvider) { this.setProvider(alternativeProvider) return } this.requestor.sendMessage({ topic: TOPIC.RPC, action: RPC_ACTION.NO_RPC_PROVIDER, name: this.message.name, correlationId: this.message.correlationId }) this.destroy() } /** * Callback if the accept message hasn't been returned * in time by the provider */ private onAcceptTimeout (): void { this.requestor.sendMessage({ topic: TOPIC.RPC, action: RPC_ACTION.ACCEPT_TIMEOUT, name: this.message.name, correlationId: this.message.correlationId }) this.destroy() } /** * Callback if the response message hasn't been returned * in time by the provider */ public onResponseTimeout (): void { this.requestor.sendMessage({ topic: TOPIC.RPC, action: RPC_ACTION.RESPONSE_TIMEOUT, name: this.message.name, correlationId: this.message.correlationId }) this.destroy() } } ================================================ FILE: src/jif/jif-handler.spec.ts ================================================ import { expect } from 'chai' const JIFHandler = require('./jif-handler').default import LoggerMock from '../test/mock/logger-mock' import { RECORD_ACTION, TOPIC, EVENT_ACTION, RPC_ACTION, PRESENCE_ACTION } from '../constants'; describe('JIF Handler', () => { let jifHandler const logger = new LoggerMock() before(() => { jifHandler = new JIFHandler({ logger }) }) describe('fromJIF', () => { it('should reject an empty message', () => { const jif = {} const result = jifHandler.fromJIF(jif) expect(result.success).to.equal(false) expect(result.error).to.be.a('string') }) it('should reject a message that is not an object', () => { const jifs = [ [{ topic: 'event', eventName: 'time/berlin', action: 'emit', data: { a: ['b', 2] } }], '{"topic":"event","eventName":"time/berlin","action":"emit","data":{"a":["b",2]}}', 23, null, undefined ] const results = jifs.map((jif) => jifHandler.fromJIF(jif)) results.forEach((result, i) => { expect(result.success).to.equal(false, i.toString()) expect(result.error).to.match(/must be object/, i.toString()) }) }) describe('events', () => { it('should create an event message for a well formed jit event', () => { const jif = { topic: 'event', eventName: 'time/berlin', action: 'emit', data: { a: ['b', 2] } } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.done).to.equal(true) expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.EVENT) expect(message.action).to.equal(EVENT_ACTION.EMIT) expect(message.name).to.equal('time/berlin') expect(message.parsedData).to.deep.equal({ a: ['b', 2] }) }) it('should support events without payloads', () => { const jif = { topic: 'event', eventName: 'time/berlin', action: 'emit' } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.EVENT) expect(message.action).to.equal(EVENT_ACTION.EMIT) expect(message.name).to.equal('time/berlin') expect(message.parsedData).to.equal(undefined) }) it('should reject malformed topics', () => { const topics = [ null, 23, ['event'], { event: 'event' }, 'evnt', 'event ', 'Event', 'EVENT', ] const results = topics.map((topic) => jifHandler.fromJIF( { topic, action: 'emit', eventName: 'time/berlin' } )) results.forEach((result, i) => expect(result.success).to.equal(false, i.toString())) }) it('should reject malformed actions', () => { const actions = [ null, 23, 'emi', 'emit ', 'Emit', 'EMIT', ] const results = actions.map((action) => jifHandler.fromJIF( { topic: 'event', action, eventName: 'time/berlin' } )) results.forEach((result, i) => expect(result.success).to.equal(false, i.toString())) }) it('should not support an event without a name', () => { const jif = { topic: 'event', action: 'emit', data: '' } const result = jifHandler.fromJIF(jif) expect(result.success).to.equal(false) expect(result.error).to.be.a('string') expect(result.error).to.match(/eventName/) }) it('should reject malformed names', () => { const names = [ null, 23, ['foo'], { name: 'john' }, '' ] const results = names.map((eventName) => jifHandler.fromJIF( { topic: 'event', action: 'emit', eventName } )) results.forEach((result, i) => expect(result.success).to.equal(false, i.toString())) }) }) describe('rpcs', () => { it('should handle a valid rpc message', () => { const jif = { topic: 'rpc', action: 'make', rpcName: 'add-two', data: { numA: 6, numB: 3 } } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.done).to.equal(false) expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RPC) expect(message.action).to.equal(RPC_ACTION.REQUEST) expect(message.name).to.equal('add-two') expect(message.correlationId).to.be.a('string') expect(message.correlationId).to.have.length.above(12) expect(message.parsedData).to.deep.equal({ numA: 6, numB: 3 }) }) it('should handle an rpc without data', () => { const jif = { topic: 'rpc', action: 'make', rpcName: 'add-two', } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RPC) expect(message.action).to.equal(RPC_ACTION.REQUEST) expect(message.name).to.equal('add-two') expect(message.correlationId).to.be.a('string') expect(message.correlationId).to.have.length.above(12) expect(message.parsedData).to.equal(undefined) }) }) describe('records', () => { it('should handle a record write (object type) without path', () => { const jif = { topic: 'record', action: 'write', recordName: 'car/bmw', data: { tyres: 2, wheels: 4 } } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RECORD) expect(message.action).to.equal(RECORD_ACTION.CREATEANDUPDATE) expect(message.name).to.equal('car/bmw') expect(message.version).to.equal(-1) expect(message.parsedData).to.deep.equal({ tyres: 2, wheels: 4 }) expect(message.isWriteAck).to.equal(true) }) it('should handle a record write (array type) without path', () => { const jif = { topic: 'record', action: 'write', recordName: 'car/bmw', data: [{ model: 'M6', hp: 560 }, { model: 'X6', hp: 306 }] } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RECORD) expect(message.action).to.equal(RECORD_ACTION.CREATEANDUPDATE) expect(message.name).to.equal('car/bmw') expect(message.version).to.equal(-1) expect(message.parsedData).to.deep.equal([{ model: 'M6', hp: 560 }, { model: 'X6', hp: 306 }]) expect(message.isWriteAck).to.equal(true) }) it('should handle a record write with path', () => { const jif = { topic: 'record', action: 'write', recordName: 'car/bmw', path: 'tyres', data: 3 } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RECORD) expect(message.action).to.equal(RECORD_ACTION.CREATEANDPATCH) expect(message.name).to.equal('car/bmw') expect(message.version).to.equal(-1) expect(message.path).to.equal('tyres') expect(message.parsedData).to.deep.equal(3) expect(message.isWriteAck).to.equal(true) }) it('should handle a record read', () => { const jif = { topic: 'record', action: 'read', recordName: 'car/bmw', } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RECORD) expect(message.action).to.equal(RECORD_ACTION.READ) expect(message.name).to.equal('car/bmw') }) it('should handle a record delete', () => { const jif = { topic: 'record', action: 'delete', recordName: 'car/bmw', } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RECORD) expect(message.action).to.equal(RECORD_ACTION.DELETE) expect(message.name).to.equal('car/bmw') }) it('should handle a record notify', () => { const jif = { topic: 'record', action: 'notify', recordNames: ['car/bmw', 'car/vw'], } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RECORD) expect(message.action).to.equal(RECORD_ACTION.NOTIFY) expect(message.names).to.deep.equal(['car/bmw', 'car/vw']) }) it('should handle a record head', () => { const jif = { topic: 'record', action: 'head', recordName: 'car/bmw', } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.RECORD) expect(message.action).to.equal(RECORD_ACTION.HEAD) expect(message.name).to.equal('car/bmw') }) it('should only allow writes to have a path field', () => { const jifs = [ { topic: 'record', action: 'write', recordName: 'car/bmw', data: 'bla', path: 'wheel' }, { topic: 'record', action: 'read', recordName: 'car/bmw', path: 'wheel' }, { topic: 'record', action: 'head', recordName: 'car/bmw', path: 'wheel' }, { topic: 'record', action: 'delete', recordName: 'car/bmw', path: 'wheel' } ] const results = jifs.map((jif) => jifHandler.fromJIF(jif)) expect(results[0].success).to.equal(true) expect(results[1].success).to.equal(false) expect(results[2].success).to.equal(false) expect(results[3].success).to.equal(false) }) it('should only allow writes to have a data field', () => { const jifs = [ { topic: 'record', action: 'write', recordName: 'car/bmw', data: { a: 123 } }, { topic: 'record', action: 'read', recordName: 'car/bmw', data: { a: 123 } }, { topic: 'record', action: 'head', recordName: 'car/bmw', data: { a: 123 } }, { topic: 'record', action: 'delete', recordName: 'car/bmw', data: { a: 123 } } ] const results = jifs.map((jif) => jifHandler.fromJIF(jif)) expect(results[0].success).to.equal(true) expect(results[1].success).to.equal(false) expect(results[2].success).to.equal(false) expect(results[3].success).to.equal(false) }) }) describe('presence', () => { it('should handle a presence query', () => { const jif = { topic: 'presence', action: 'query' } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.PRESENCE) expect(message.action).to.equal(PRESENCE_ACTION.QUERY_ALL) }) it('should handle a presence query for some users', () => { const jif = { topic: 'presence', action: 'query', names: ['one'] } const result = jifHandler.fromJIF(jif) const message = result.message expect(result.success).to.equal(true) expect(message).to.be.an('object') expect(message.topic).to.equal(TOPIC.PRESENCE) expect(message.action).to.equal(PRESENCE_ACTION.QUERY) expect(message.names).to.deep.equal(['one']) }) }) }) describe('toJIF', () => { describe('rpcs', () => { it('should build a valid rpc response', () => { const result = jifHandler.toJIF({ topic: TOPIC.RPC, action: RPC_ACTION.RESPONSE, name: 'addTwo', correlationId: '1234', parsedData: 12 }) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.have.all.keys(['success', 'data']) expect(jif.success).to.equal(true) expect(jif.data).to.equal(12) }) it('should ignore an rpc request ack', () => { const result = jifHandler.toJIF({ topic: TOPIC.RPC, action: RPC_ACTION.REQUEST, name: 'addTwo', correlationId: 1234, isAck: true }) expect(result.done).to.equal(false) }) it('should build a valid rpc response', () => { const result = jifHandler.toJIF({ topic: TOPIC.RPC, action: RPC_ACTION.RESPONSE, name: 'addTwo', correlationId: 1234, parsedData: 12 }) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.have.all.keys(['success', 'data']) expect(jif.success).to.equal(true) expect(jif.data).to.equal(12) }) }) describe('records', () => { it('should build a valid record write ack', () => { const result = jifHandler.toJIF({ topic: TOPIC.RECORD, action: RECORD_ACTION.WRITE_ACKNOWLEDGEMENT, name: 'car/fiat', parsedData: [[2, 3], null] }) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.contain.keys(['success']) expect(jif.success).to.equal(true) }) it('should build a valid record delete success', () => { const result = jifHandler.toJIF({ topic: TOPIC.RECORD, action: RECORD_ACTION.DELETE_SUCCESS, name: 'car/fiat' }) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.have.all.keys(['success']) expect(jif.success).to.equal(true) }) it('should build a valid record read response', () => { const result = jifHandler.toJIF({ topic: TOPIC.RECORD, action: RECORD_ACTION.READ_RESPONSE, name: 'car/fiat', version: 2, parsedData: { car: true } }) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.contain.keys(['success']) expect(jif.success).to.equal(true) expect(jif.data).to.deep.equal({ car: true }) expect(jif.version).to.equal(2) }) it('should handle a valid record head response', () => { const result = jifHandler.toJIF({ topic: TOPIC.RECORD, action: RECORD_ACTION.HEAD_RESPONSE, name: 'car/fiat', version: 2 }) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.have.all.keys(['success', 'version']) expect(jif.success).to.equal(true) expect(jif.version).to.equal(2) }) it('should handle a valid record head error', () => { const result = jifHandler.errorToJIF({ topic: TOPIC.RECORD, action: RECORD_ACTION.HEAD, name: 'car/fiat' }, RECORD_ACTION.RECORD_LOAD_ERROR) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.include.all.keys(['error', 'errorEvent', 'errorTopic', 'success']) expect(jif.success).to.equal(false) expect(jif.errorTopic).to.equal('record') expect(jif.errorEvent).to.equal(RECORD_ACTION.RECORD_LOAD_ERROR) expect(jif.errorParams).to.equal('car/fiat') // TODO: review }) }) describe('presence', () => { it('should build a valid presence response', () => { const result = jifHandler.toJIF({ topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.QUERY_ALL_RESPONSE, names: ['john', 'alex', 'yasser'] }) const jif = result.message expect(result.done).to.equal(true) expect(jif).to.be.an('object') expect(jif).to.have.all.keys(['success', 'users']) expect(jif.success).to.equal(true) expect(jif.users).to.deep.equal(['john', 'alex', 'yasser']) }) }) }) }) ================================================ FILE: src/jif/jif-handler.ts ================================================ import { EVENT_ACTION, PRESENCE_ACTION, RECORD_ACTION, RPC_ACTION, TOPIC, Message, ALL_ACTIONS, ACTIONS } from '../constants' import {Ajv} from 'ajv' import { getUid, reverseMap, deepFreeze, } from '../utils/utils' import { jifSchema } from './jif-schema' import { JifMessage, DeepstreamServices, EVENT } from '@deepstream/types' const ajv = new Ajv({strict: false}) const validateJIF: any = ajv.compile(jifSchema) type JifInMessage = any // jif -> message lookup table function getJifToMsg () { const JIF_TO_MSG: any = {} JIF_TO_MSG.event = {} JIF_TO_MSG.event.emit = (msg: JifInMessage) => ({ done: true, message: { topic: TOPIC.EVENT, action: EVENT_ACTION.EMIT, name: msg.eventName, parsedData: msg.data, }, }) JIF_TO_MSG.rpc = {} JIF_TO_MSG.rpc.make = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RPC, action: RPC_ACTION.REQUEST, name: msg.rpcName, correlationId: getUid(), parsedData: msg.data, }, }) JIF_TO_MSG.record = {} JIF_TO_MSG.record.read = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.READ, name: msg.recordName, }, }) JIF_TO_MSG.record.write = (msg: JifInMessage) => ( msg.path ? JIF_TO_MSG.record.patch(msg) : JIF_TO_MSG.record.update(msg) ) JIF_TO_MSG.record.patch = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.CREATEANDPATCH, name: msg.recordName, version: msg.version || -1, path: msg.path, parsedData: msg.data, isWriteAck: true, correlationId: 0 }, }) JIF_TO_MSG.record.update = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.CREATEANDUPDATE, name: msg.recordName, version: msg.version || -1, parsedData: msg.data, isWriteAck: true, correlationId: 0 }, }) JIF_TO_MSG.record.head = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.HEAD, name: msg.recordName, }, }) JIF_TO_MSG.record.delete = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.DELETE, name: msg.recordName, }, }), JIF_TO_MSG.record.notify = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.NOTIFY, names: msg.recordNames }, }) JIF_TO_MSG.list = {} JIF_TO_MSG.list.read = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.READ, name: msg.listName, }, }) JIF_TO_MSG.list.write = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.CREATEANDUPDATE, name: msg.listName, version: msg.version || -1, parsedData: msg.data, isWriteAck: true, correlationId: 0 }, }) JIF_TO_MSG.list.delete = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.RECORD, action: RECORD_ACTION.DELETE, name: msg.listName, }, }) JIF_TO_MSG.presence = {} JIF_TO_MSG.presence.query = (msg: JifInMessage) => ( msg.names ? JIF_TO_MSG.presence.queryUsers(msg) : JIF_TO_MSG.presence.queryAll(msg) ) JIF_TO_MSG.presence.queryAll = () => ({ done: false, message: { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.QUERY_ALL, }, }) JIF_TO_MSG.presence.queryUsers = (msg: JifInMessage) => ({ done: false, message: { topic: TOPIC.PRESENCE, action: PRESENCE_ACTION.QUERY, names: msg.names, }, }) return deepFreeze(JIF_TO_MSG) } // message type enumeration const TYPE = { ACK: 'A', NORMAL: 'N' } function getMsgToJif () { // message -> jif lookup table const MSG_TO_JIF: any = {} MSG_TO_JIF[TOPIC.RPC] = {} MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.RESPONSE] = {} MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.RESPONSE][TYPE.NORMAL] = (message: Message) => ({ done: true, message: { data: message.parsedData, success: true, }, }) MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.REQUEST_ERROR] = {} MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.REQUEST_ERROR][TYPE.NORMAL] = (message: Message) => ({ done: true, message: { errorTopic: 'rpc', error: message.parsedData, success: false, }, }) MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.ACCEPT] = {} MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.ACCEPT][TYPE.NORMAL] = () => ({ done: false }) MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.REQUEST] = {} MSG_TO_JIF[TOPIC.RPC][RPC_ACTION.REQUEST][TYPE.ACK] = () => ({ done: false }) MSG_TO_JIF[TOPIC.RECORD] = {} MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.READ_RESPONSE] = {} MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.READ_RESPONSE][TYPE.NORMAL] = (message: Message) => ({ done: true, message: { version: message.version, data: message.parsedData, success: true, }, }) MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.WRITE_ACKNOWLEDGEMENT] = {} MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.WRITE_ACKNOWLEDGEMENT][TYPE.NORMAL] = (message: Message) => ({ done: true, message: { success: true, }, }) MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.DELETE] = {} MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.DELETE][TYPE.NORMAL] = () => ({ done: true, message: { success: true, }, }) MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.DELETE_SUCCESS] = {} MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.DELETE_SUCCESS][TYPE.NORMAL] = () => ({ done: true, message: { success: true, }, }) MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.NOTIFY] = {} MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.NOTIFY][TYPE.ACK] = (message: Message) => ({ done: true, message: { success: true, }, }) MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.HEAD_RESPONSE] = {} MSG_TO_JIF[TOPIC.RECORD][RECORD_ACTION.HEAD_RESPONSE][TYPE.NORMAL] = (message: Message) => ({ done: true, message: { version: message.version, success: true, }, }) MSG_TO_JIF[TOPIC.PRESENCE] = {} MSG_TO_JIF[TOPIC.PRESENCE][PRESENCE_ACTION.QUERY_ALL_RESPONSE] = {} MSG_TO_JIF[TOPIC.PRESENCE][PRESENCE_ACTION.QUERY_ALL_RESPONSE][TYPE.NORMAL] = (message: Message) => ({ done: true, message: { users: message.names, success: true, }, }) MSG_TO_JIF[TOPIC.PRESENCE][PRESENCE_ACTION.QUERY_RESPONSE] = {} MSG_TO_JIF[TOPIC.PRESENCE][PRESENCE_ACTION.QUERY_RESPONSE][TYPE.NORMAL] = (message: Message) => ({ done: true, message: { users: message.parsedData, success: true, }, }) return deepFreeze(MSG_TO_JIF) } export default class JIFHandler { private JIF_TO_MSG = getJifToMsg() private MSG_TO_JIF = getMsgToJif() private topicToKey = reverseMap(TOPIC) constructor (private services: DeepstreamServices) {} /* * Validate and convert a JIF message to a deepstream message */ public fromJIF (jifMessage: JifInMessage) { if (!validateJIF(jifMessage)) { let error = validateJIF.errors[0] switch (error.keyword) { // case 'additionalProperties': // error = `property '${error.params.additionalProperty}' // not permitted for topic '${jifMessage.topic}'` // break case 'required': error = `property '${error.params.missingProperty}' is required for topic '${jifMessage.topic}'` break case 'type': case 'minLength': error = `property '${error.dataPath}' ${error.message}` break // case 'const': // error = `value for property '${error.dataPath}' not valid for topic '${jifMessage.topic}'` // break default: error = null } return { success: false, error, done: true, } } const result = this.JIF_TO_MSG[jifMessage.topic][jifMessage.action](jifMessage) result.success = true return result } /* * Convert a deepstream response/ack message to a JIF message response * @param {Object} message deepstream message * * @returns {Object} { * {Object} message jif message * {Boolean} done false iff message should await another result/acknowledgement * } */ public toJIF (message: Message): JifMessage { let type if (message.isAck) { type = TYPE.ACK } else { type = TYPE.NORMAL } if (message.isError) { return this.errorToJIF(message, message.action) } return this.MSG_TO_JIF[message.topic][message.action][type](message) } /* * Convert a deepstream error message to a JIF message response */ public errorToJIF (message: Message, event: ALL_ACTIONS | string) { // convert topic enum to human-readable key const topicKey = this.topicToKey[message.topic] const result: any = { errorTopic: topicKey && topicKey.toLowerCase(), errorEvent: event, success: false, } if (event === ACTIONS[message.topic].MESSAGE_DENIED) { result.action = message.originalAction as number result.error = `Message denied. Action "${ACTIONS[message.topic][message.originalAction!]}" is not permitted.` } else if (message.topic === TOPIC.RECORD && event === RECORD_ACTION.VERSION_EXISTS) { result.error = `Record update failed. Version ${message.version} exists for record "${message.name}".` result.currentVersion = message.version result.currentData = message.parsedData } else if (message.topic === TOPIC.RECORD && event === RECORD_ACTION.INVALID_VERSION) { result.error = `Record update failed. Version ${message.version} is not valid for record "${message.name}".` result.currentVersion = message.version result.currentData = message.parsedData } else if (message.topic === TOPIC.RECORD && event === RECORD_ACTION.RECORD_NOT_FOUND) { result.error = `Record read failed. Record "${message.name}" could not be found.` result.errorEvent = message.action } else if (message.topic === TOPIC.RPC && event === RPC_ACTION.NO_RPC_PROVIDER) { result.error = `No provider was available to handle the RPC "${message.name}".` // message.correlationId = data[1] } else if (message.topic === TOPIC.RPC && message.action === RPC_ACTION.RESPONSE_TIMEOUT) { result.error = 'The RPC response timeout was exceeded by the provider.' } else { this.services.logger.warn( EVENT.INFO, `Unhandled request error occurred: ${TOPIC[message.topic]} ${event} ${JSON.stringify(message)}`, { message } ) result.error = `An error occurred: ${RPC_ACTION[event as number]}.` result.errorParams = message.name } return { message: result, done: true, } } } ================================================ FILE: src/jif/jif-schema.ts ================================================ export const jifSchema = { title: 'JSON Interchange Format', description: 'A JSON format for interaction with DeepstreamIO.', type: 'object', anyOf: [ { properties: { topic: { const: 'event', }, action: { const: 'emit', }, eventName: { type: 'string', minLength: 1, }, data: {}, }, required: [ 'topic', 'action', 'eventName', ], additionalProperties: false, }, { title: 'RPC', description: 'Make RPC requests.', properties: { topic: { const: 'rpc', }, action: { const: 'make', }, rpcName: { type: 'string', minLength: 1, }, data: {}, }, required: [ 'topic', 'action', 'rpcName', ], additionalProperties: false, }, { title: 'Record', description: 'Fetch and delete records.', properties: { topic: { const: 'record', }, action: { enum: [ 'read', 'head', 'delete', ], }, recordName: { type: 'string', minLength: 1, }, }, required: [ 'topic', 'action', 'recordName', ], additionalProperties: false, }, { title: 'Record Writes', description: 'Create or update a record. The full object must be specified.', properties: { topic: { const: 'record', }, action: { const: 'write', }, recordName: { type: 'string', minLength: 1, }, data: { type: ['object', 'array'], }, version: { type: 'integer', minimum: -1, }, }, required: [ 'topic', 'action', 'recordName', 'data', ], additionalProperties: false, }, { title: 'Record Write With Path', description: 'If a path is specified, a patching update will occur.', properties: { topic: { const: 'record', }, action: { const: 'write', }, recordName: { type: 'string', minLength: 1, }, data: {}, path: { type: 'string', }, version: { type: 'integer', minimum: -1, }, }, required: [ 'topic', 'action', 'recordName', 'data', 'path', ], additionalProperties: false, }, { title: 'Record Notify', description: 'Notifies deepstream that a record was written to remotely', properties: { topic: { const: 'record', }, action: { const: 'notify', }, recordNames: { type: 'array', minLength: 1, items: { type: 'string', }, }, }, required: [ 'topic', 'action', 'recordNames', ], additionalProperties: false, }, { title: 'List', description: 'Fetch and delete lists.', properties: { topic: { const: 'list', }, action: { enum: [ 'read', 'delete', ], }, listName: { type: 'string', minLength: 1, }, }, required: [ 'topic', 'action', 'listName', ], additionalProperties: false, }, { title: 'List Writes', description: 'Create or write to a list.', properties: { topic: { const: 'list', }, action: { const: 'write', }, listName: { type: 'string', minLength: 1, }, data: { type: ['array'], items: { type: 'string', }, }, version: { type: 'integer', minimum: -1, }, }, required: [ 'topic', 'action', 'listName', 'data', ], additionalProperties: false, }, { title: 'Presence', description: 'Query presence.', properties: { topic: { const: 'presence', }, action: { enum: [ 'query', ], }, names: { type: ['array'], items: { type: 'string', } } }, required: [ 'topic', 'action', ], additionalProperties: false, }, ], } ================================================ FILE: src/listen/listener-registry.spec.ts ================================================ import 'mocha' import * as C from '../constants' import ListenerTestUtils from './listener-test-utils' import 'mocha' let tu describe('listener-registry', () => { /* const ListenerRegistry = require('../../src/listen/listener-registry') const testHelper = require('../test-helper/test-helper') import SocketMock from '../test-mocks/socket-mock' const options = testHelper.getDeepstreamOptions() const msg = testHelper.msg let listenerRegistry const recordSubscriptionRegistryMock = { getNames () { return ['car/Mercedes', 'car/Abarth'] } } describe.skip('listener-registry errors', () => { beforeEach(() => { listenerRegistry = new ListenerRegistry('R', options, recordSubscriptionRegistryMock) expect(typeof listenerRegistry.handle).to.equal('function') }) it('adds a listener without message data', () => { const socketWrapper = SocketWrapperFactory.create(new SocketMock(), options) listenerRegistry.handle(socketWrapper, { topic: 'R', action: 'L', data: [] }) expect(options.logger.lastLogArguments).to.deep.equal([3, 'INVALID_MESSAGE_DATA', undefined]) expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|E|INVALID_MESSAGE_DATA|undefined+')) }) it('adds a listener with invalid message data message data', () => { const socketWrapper = new SocketWrapper(new SocketMock(), options) listenerRegistry.handle(socketWrapper, { topic: 'R', action: 'L', data: [44] }) expect(options.logger.lastLogArguments).to.deep.equal([3, 'INVALID_MESSAGE_DATA', 44]) expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|E|INVALID_MESSAGE_DATA|44+')) }) it('adds a listener with an invalid regexp', () => { const socketWrapper = new SocketWrapper(new SocketMock(), options) listenerRegistry.handle(socketWrapper, { topic: 'R', action: 'L', data: ['us('] }) expect(options.logger.lastLogArguments).to.deep.equal([3, 'INVALID_MESSAGE_DATA', 'SyntaxError: Invalid regular expression: /us(/: Unterminated group']) expect(socketWrapper.socket.lastSendMessage).to.equal(msg('R|E|INVALID_MESSAGE_DATA|SyntaxError: Invalid regular expression: /us(/: Unterminated group+')) }) }) */ describe('listener-registry-local-load-balancing', () => { beforeEach(() => { tu = new ListenerTestUtils() }) afterEach(() => { tu.complete() }) describe('with a single provider', () => { it('accepts a subscription', () => { // 1. provider does listen a/.* tu.providerListensTo(1, 'a/.*') // 3. provider will get a SP tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') // 2. clients 1 request a/1 tu.clientSubscribesTo(1, 'a/1', true) // 4. provider responds with ACCEPT tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerAccepts(1, 'a/.*', 'a/1') // // 6. clients 2 request a/1 tu.clientWillRecievePublishedUpdate(2, 'a/1', true) tu.clientSubscribesTo(2, 'a/1') // // 6. clients 3 request a/1 tu.clientWillRecievePublishedUpdate(3, 'a/1', true) tu.clientSubscribesTo(3, 'a/1') // // 9. client 1 discards a/1 tu.clientUnsubscribesTo(3, 'a/1') // // 9. client 2 discards a/1 tu.clientUnsubscribesTo(2, 'a/1') // // 10. clients discards a/1 tu.providerWillGetSubscriptionRemoved(1, 'a/.*', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', false) tu.clientUnsubscribesTo(1, 'a/1', true) // // 13. a/1 should have no active provider tu.subscriptionHasActiveProvider('a/1', false) // // 14. recieving unknown accept/reject throws an error tu.acceptMessageThrowsError(1, 'a/.*', 'a/1') tu.rejectMessageThrowsError(1, 'a/.*', 'a/1') }) it('rejects a subscription', () => { // 1. provider does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. clients request a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 4. provider responds with ACCEPT tu.providerRejects(1, 'a/.*', 'a/1') // 5. clients discards a/1 tu.clientUnsubscribesTo(1, 'a/1', true) // mocks do expecations to ensure nothing else was called }) it('rejects a subscription with a pattern for which subscriptions already exists', () => { // 0. subscription already made for b/1 tu.subscriptionAlreadyMadeFor('b/1') // 1. provider does listen a/.* tu.providerWillGetSubscriptionFound(1, 'b/.*', 'b/1') tu.providerListensTo(1, 'b/.*') // 3. provider responds with REJECT tu.providerRejects(1, 'b/.*', 'b/1') // 4. clients discards b/1 tu.clientUnsubscribesTo(1, 'b/1', true) // mocks do expecations to ensure nothing else was called }) it('accepts a subscription with a pattern for which subscriptions already exists', () => { // 0. subscription already made for b/1 tu.subscriptionAlreadyMadeFor('b/1') // 1. provider does listen a/.* tu.providerWillGetSubscriptionFound(1, 'b/.*', 'b/1') tu.providerListensTo(1, 'b/.*') // 3. provider responds with ACCEPT tu.publishUpdateWillBeSentToSubscribers('b/1', true) tu.providerAccepts(1, 'b/.*', 'b/1') // 5. clients discards b/1 tu.providerWillGetSubscriptionRemoved(1, 'b/.*', 'b/1') tu.publishUpdateWillBeSentToSubscribers('b/1', false) tu.clientUnsubscribesTo(1, 'b/1', true) // 7. send publishing=false to the clients }) it('accepts a subscription for 2 clients', () => { // 1. provider does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 4. provider responds with ACCEPT tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerAccepts(1, 'a/.*', 'a/1') // 5. send publishing=true to the clients tu.clientWillRecievePublishedUpdate(2, 'a/1', true) tu.clientSubscribesTo(2, 'a/1') // 9. client 1 discards a/1 tu.clientUnsubscribesTo(1, 'a/1') // 11. client 2 discards a/1 tu.providerWillGetSubscriptionRemoved(1, 'a/.*', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', false) tu.clientUnsubscribesTo(2, 'a/1', true) // 13. a/1 should have no active provider tu.subscriptionHasActiveProvider('a/1', false) }) }) describe('with multiple providers', () => { it('first rejects, seconds accepts, third does nothing', () => { // 1. provider 1 does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. provider 2 does listen a/[0-9] tu.providerListensTo(2, 'a/[0-9]') // 3. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 5. provider 1 responds with REJECTS tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.providerRejects(1, 'a/.*', 'a/1') // 7. provider 2 responds with ACCEPTS tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerAccepts(2, 'a/[0-9]', 'a/1') // 8. send publishing=true to the clients // 9. provider 3 does listen a/[0-9] tu.providerListensTo(3, 'a/[0-9]') // 11. client 1 unsubscribed to a/1 tu.providerWillGetSubscriptionRemoved(2, 'a/[0-9]', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', false) tu.clientUnsubscribesTo(1, 'a/1', true) }) it('first accepts, seconds does nothing', () => { // 1. provider 1 does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. provider 2 does listen a/[0-9] tu.providerListensTo(2, 'a/[0-9]') // 3. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 6. provider 1 accepts tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerAccepts(1, 'a/.*', 'a/1') // 12. send publishing=true to the clients // 6. client 1 unsubscribed to a/1 tu.providerWillGetSubscriptionRemoved(1, 'a/.*', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', false) tu.clientUnsubscribesTo(1, 'a/1', true) }) it('first rejects, seconds - which start listening after first gets SP - accepts', () => { // 1. provider 1 does listen a/.* tu.providerListensTo(1, 'a/.*') // 3. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 2. provider 2 does listen a/[0-9] tu.providerListensTo(2, 'a/[0-9]') // 6. provider 1 rejects tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.providerRejects(1, 'a/.*', 'a/1') // 6. provider 1 accepts tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerAccepts(2, 'a/[0-9]', 'a/1') // 12. send publishing=false to the clients }) it('no messages after unlisten', () => { // 1. provider 1 does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. provider 2 does listen a/[0-9] tu.providerListensTo(2, 'a/[0-9]') // 3. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 2. provider 2 does unlisten a/[0-9] tu.providerUnlistensTo(2, 'a/[0-9]') // 6. provider 1 responds with REJECTS tu.providerRejects(1, 'a/.*', 'a/1') // mock does remaining expecations }) it.skip('provider 1 accepts a subscription and disconnects then provider 2 gets a SP', () => { // 1. provider 1 does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. provider 2 does listen a/[0-9] tu.providerListensTo(2, 'a/[0-9]') // 3. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 5. provider 1 responds with ACCEPT tu.providerAccepts(1, 'a/.*', 'a/1') // 13. subscription has active provider tu.subscriptionHasActiveProvider('a/1', true) // 7. client 1 requests a/1 tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', false) tu.providerLosesItsConnection(1) // 8. send publishing=true to the clients // 9. subscription doesnt have active provider tu.subscriptionHasActiveProvider('a/1', false) tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerAccepts(2, 'a/[0-9]', 'a/1') // 12. send publishing=true to the clients // 13. subscription has active provider tu.subscriptionHasActiveProvider('a/1', true) }) }) }) describe('listener-registry-local-load-balancing does not send publishing updates for events', () => { beforeEach(() => { tu = new ListenerTestUtils(C.TOPIC.EVENT) }) afterEach(() => { tu.complete() }) it('client with provider already registered', () => { // 1. provider does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) // 4. provider responds with ACCEPT tu.providerAccepts(1, 'a/.*', 'a/1') // 6. client 2 requests a/1 tu.clientSubscribesTo(2, 'a/1') // 9. client 1 discards a/1 tu.clientUnsubscribesTo(1, 'a/1') // 11. client 2 discards a/1 tu.providerWillGetSubscriptionRemoved(1, 'a/.*', 'a/1') tu.clientUnsubscribesTo(2, 'a/1', true) // 12. provider should get a SR // 13. a/1 should have no active provider tu.subscriptionHasActiveProvider('a/1', false) }) }) describe('listener-registry-local-timeouts', () => { beforeEach(() => { tu = new ListenerTestUtils() }) afterEach(() => { tu.complete() }) beforeEach(() => { // 1. provider 1 does listen a/.* tu.providerListensTo(1, 'a/.*') // 2. provider 2 does listen a/[0-9] tu.providerListensTo(2, 'a/[0-9]') // 3. client 1 requests a/1 tu.providerWillGetSubscriptionFound(1, 'a/.*', 'a/1') tu.clientSubscribesTo(1, 'a/1', true) }) it('provider 1 times out, provider 2 accepts', (done) => { tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerWillGetListenTimeout(1, 'a/1') setTimeout(() => { tu.providerAccepts(1, 'a/[0-9]', 'a/1') tu.subscriptionHasActiveProvider('a/1', true) done() }, 40) }) it.skip('provider 1 times out and gets a RESPONSE_TIMEOUT, but will be ignored because provider 2 accepts as well', (done) => { tu.providerWillGetSubscriptionRemoved(1, 'a/.*', 'a/1') tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.providerWillGetListenTimeout(1, 'a/1') setTimeout(() => { tu.providerAcceptsButIsntAcknowledged(1, 'a/.*', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerAccepts(2, 'a/[0-9]', 'a/1') tu.subscriptionHasActiveProvider('a/1', true) done() }, 40) }) it.skip('provider 1 times out, but then it accept and will be used because provider 2 rejects', (done) => { tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.providerWillGetListenTimeout(1, 'a/1') setTimeout(() => { tu.providerAcceptsButIsntAcknowledged(1, 'a/.*', 'a/1') tu.subscriptionHasActiveProvider('a/1', false) tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerRejectsAndPreviousTimeoutProviderThatAcceptedIsUsed(2, 'a/[0-9]', 'a/1') tu.subscriptionHasActiveProvider('a/1', true) done() }, 40) }) it.skip('provider 1 and 2 times out and 3 rejects, 1 rejects and 2 accepts later and 2 wins', (done) => { tu.providerWillGetListenTimeout(1, 'a/1') tu.providerWillGetListenTimeout(2, 'a/1') tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.providerListensTo(3, 'a/[1]') setTimeout(() => { tu.providerWillGetSubscriptionFound(3, 'a/[1]', 'a/1') // first provider timeout setTimeout(() => { tu.providerRejects(1, 'a/.*', 'a/1') tu.providerAcceptsButIsntAcknowledged(2, 'a/[0-9]', 'a/1') tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerRejectsAndPreviousTimeoutProviderThatAcceptedIsUsed(3, 'a/[1]', 'a/1') done() }, 40) }, 40) }) it.skip('1 rejects and 2 accepts later and dies and 3 wins', (done) => { tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.providerListensTo(3, 'a/[1]') setTimeout(() => { tu.providerWillGetSubscriptionFound(3, 'a/[1]', 'a/1') // first provider timeout setTimeout(() => { tu.providerRejects(1, 'a/.*', 'a/1') tu.providerAcceptsButIsntAcknowledged(2, 'a/[0-9]', 'a/1') tu.providerLosesItsConnection(2, 'a/[0-9]') tu.providerRejects(3, 'a/[1]', 'a/1') done() }, 40) }, 40) }) it.skip('provider 1 and 2 times out and 3 rejects, 1 and 2 accepts later and 1 wins', (done) => { tu.providerWillGetListenTimeout(1, 'a/1') tu.providerWillGetListenTimeout(2, 'a/1') tu.providerWillGetSubscriptionFound(2, 'a/[0-9]', 'a/1') tu.providerListensTo(3, 'a/[1]') setTimeout(() => { tu.providerWillGetSubscriptionFound(3, 'a/[1]', 'a/1') // first provider setTimeout(() => { // 10. provider 1 responds with ACCEPT tu.providerAcceptsButIsntAcknowledged(1, 'a/.*', 'a/1') // 11. provider 2 responds with ACCEPT tu.providerAcceptsAndIsSentSubscriptionRemoved(2, 'a/[0-9]', 'a/1') // 12. provider 3 responds with reject tu.publishUpdateWillBeSentToSubscribers('a/1', true) tu.providerRejectsAndPreviousTimeoutProviderThatAcceptedIsUsed(3, 'a/[1]', 'a/1') done() }, 40) }, 40) }) }) }) ================================================ FILE: src/listen/listener-registry.ts ================================================ import { EVENT_ACTION, RECORD_ACTION, TOPIC, ListenMessage, STATE_REGISTRY_TOPIC } from '../constants' import { EVENT, SubscriptionListener, DeepstreamConfig, DeepstreamServices, Provider, SocketWrapper, StateRegistry, SubscriptionRegistry } from '@deepstream/types' import { shuffleArray } from '../utils/utils' interface ListenInProgress { queryProvider: Provider, remainingProviders: Provider[] } export class ListenerRegistry implements SubscriptionListener { private providerRegistry: SubscriptionRegistry private uniqueLockName = `${this.topic}_LISTEN_LOCK` private patterns = new Map() private locallyProvidedRecords = new Map() private messageTopic: TOPIC | STATE_REGISTRY_TOPIC private actions: typeof RECORD_ACTION | typeof EVENT_ACTION private listenInProgress = new Map() private unsuccesfulMatches = new Map() private clusterProvidedRecords: StateRegistry private rematchInterval!: NodeJS.Timer /** * Deepstream.io allows clients to register as listeners for subscriptions. * This allows for the creation of 'active' data-providers, * e.g. data providers that provide data on the fly, based on what clients * are actually interested in. * * When a client registers as a listener, it provides a regular expression. * It will then immediatly get a number of callbacks for existing record subscriptions * whose names match that regular expression. * * After that, whenever a record with a name matching that regular expression is subscribed * to for the first time, the listener is notified. * * Whenever the last subscription for a matching record is removed, the listener is also * notified with a SUBSCRIPTION_FOR_PATTERN_REMOVED action * * This class manages the matching of patterns and record names. The subscription / * notification logic is handled by this.providerRegistry */ constructor (private topic: TOPIC, private config: DeepstreamConfig, private services: DeepstreamServices, private clientRegistry: SubscriptionRegistry, private metaData: any = {}) { this.actions = topic === TOPIC.RECORD ? RECORD_ACTION : EVENT_ACTION this.triggerNextProvider = this.triggerNextProvider.bind(this) if (this.topic === TOPIC.RECORD) { this.providerRegistry = this.services.subscriptions.getSubscriptionRegistry( STATE_REGISTRY_TOPIC.RECORD_LISTEN_PATTERNS, STATE_REGISTRY_TOPIC.RECORD_LISTEN_PATTERNS, ) this.clusterProvidedRecords = this.services.clusterStates.getStateRegistry(STATE_REGISTRY_TOPIC.RECORD_PUBLISHED_SUBSCRIPTIONS) this.messageTopic = STATE_REGISTRY_TOPIC.RECORD_LISTENING } else { this.providerRegistry = this.services.subscriptions.getSubscriptionRegistry( STATE_REGISTRY_TOPIC.EVENT_LISTEN_PATTERNS, STATE_REGISTRY_TOPIC.EVENT_LISTEN_PATTERNS, ) this.clusterProvidedRecords = this.services.clusterStates.getStateRegistry(STATE_REGISTRY_TOPIC.EVENT_PUBLISHED_SUBSCRIPTIONS) this.messageTopic = STATE_REGISTRY_TOPIC.EVENT_LISTENING } this.providerRegistry.setAction('subscribe', this.actions.LISTEN) this.providerRegistry.setAction('unsubscribe', this.actions.UNLISTEN) this.providerRegistry.setSubscriptionListener({ onLastSubscriptionRemoved: this.removeLastPattern.bind(this), onSubscriptionRemoved: this.removePattern.bind(this), onFirstSubscriptionMade: this.addPattern.bind(this), onSubscriptionMade: this.reconcileSubscriptionsToPatterns.bind(this), }) this.clusterProvidedRecords.onAdd(this.onRecordStartProvided.bind(this)) this.clusterProvidedRecords.onRemove(this.onRecordStopProvided.bind(this)) this.services.clusterNode.subscribe( this.messageTopic, this.onIncomingMessage.bind(this), ) if (this.config.listen.rematchInterval > 1000) { this.rematchInterval = setInterval(() => { this.patterns.forEach((value, pattern) => this.reconcileSubscriptionsToPatterns(pattern)) }, this.config.listen.rematchInterval) } else { this.services.logger.warn(EVENT.INVALID_CONFIG_DATA, 'Setting listen.rematchInterval to less than a second is not permitted.') } } public async close () { clearInterval(this.rematchInterval) } /** * Returns whether or not a provider exists for * the specific subscriptionName */ public hasActiveProvider (susbcriptionName: string): boolean { return this.clusterProvidedRecords.has(susbcriptionName) } /** * The main entry point to the handle class. * Called on any of the following actions: * * 1) ACTIONS.LISTEN * 2) ACTIONS.UNLISTEN * 3) ACTIONS.LISTEN_ACCEPT * 4) ACTIONS.LISTEN_REJECT */ public handle (socketWrapper: SocketWrapper, message: ListenMessage): void { if (message.action === this.actions.LISTEN) { this.addListener(socketWrapper, message) return } if (message.action === this.actions.UNLISTEN) { this.providerRegistry.unsubscribe(message.name, message, socketWrapper) return } if (message.action === this.actions.LISTEN_ACCEPT || message.action === this.actions.LISTEN_REJECT) { this.processResponseForListenInProgress(socketWrapper, message) return } this.services.logger.warn(EVENT.UNKNOWN_ACTION, `Unknown action for topic ${TOPIC[message.topic]} action ${message.action}`) } /** * Handle messages that arrive via the message bus */ private onIncomingMessage (message: ListenMessage, serverName: string): void { if (message.action === this.actions.LISTEN_UNSUCCESSFUL) { if (this.hasActiveProvider(message.subscription) === false) { const unsuccesfulTimeStamp = this.unsuccesfulMatches.get(message.subscription) if (!unsuccesfulTimeStamp || unsuccesfulTimeStamp - Date.now() > this.config.listen.matchCooldown) { this.onFirstSubscriptionMade(message.subscription) } return } } } /** * Process an accept or reject for a listen that is currently in progress * and hasn't timed out yet. */ private processResponseForListenInProgress (socketWrapper: SocketWrapper, message: ListenMessage): void { const inProgress = this.listenInProgress.get(message.subscription) if (!inProgress || !inProgress.queryProvider) { // This should send a message saying response is invalid return } clearTimeout(inProgress.queryProvider.responseTimeout!) if (message.action === this.actions.LISTEN_ACCEPT) { this.accept(socketWrapper, message) return } if (message.action === this.actions.LISTEN_REJECT) { this.triggerNextProvider(message.subscription) return } } /** * Called by the record subscription registry whenever a subscription count goes down to zero * Part of the subscriptionListener interface. */ public onFirstSubscriptionMade (subscriptionName: string): void { this.startProviderSearch(subscriptionName) } public onSubscriptionMade (subscriptionName: string, socketWrapper: SocketWrapper): void { if (this.hasActiveProvider(subscriptionName)) { this.sendHasProviderUpdateToSingleSubscriber(true, socketWrapper, subscriptionName) return } } public onLastSubscriptionRemoved (subscriptionName: string): void { this.unsuccesfulMatches.delete(subscriptionName) const provider = this.locallyProvidedRecords.get(subscriptionName) if (!provider) { return } this.sendSubscriptionForPatternRemoved(provider, subscriptionName) this.removeActiveListener(subscriptionName) } /** * Called by the record subscription registry whenever the subscription count increments. * Part of the subscriptionListener interface. */ public onSubscriptionRemoved (subscriptionName: string, socketWrapper: SocketWrapper): void { } /** * Register callback for when the server recieves an accept message from the client */ private accept (socketWrapper: SocketWrapper, message: ListenMessage): void { const subscriptionName = message.subscription const provider = { socketWrapper, pattern: message.name, closeListener: this.removePattern.bind(this, message.name, socketWrapper) } this.locallyProvidedRecords.set(subscriptionName, provider) socketWrapper.onClose(provider.closeListener) this.clusterProvidedRecords.add(subscriptionName) this.stopProviderSearch(subscriptionName) } /** * Register a client as a listener for record subscriptions */ private addListener (socketWrapper: SocketWrapper, message: ListenMessage): void { const regExp = this.validatePattern(socketWrapper, message) if (!regExp) { // TODO: Send an invalid pattern here? return } this.providerRegistry.subscribe(message.name, message, socketWrapper) } /** * Find subscriptions that match pattern, and notify them that * they can be provided. * * We will attempt to notify all possible providers rather than * just the single provider for load balancing purposes and * so that the one listener doesnt potentially get overwhelmed. */ private reconcileSubscriptionsToPatterns (pattern: string, socketWrapper?: SocketWrapper): void { const regExp = this.patterns.get(pattern)! const names = this.clientRegistry.getNames() for (let i = 0; i < names.length; i++) { const subscriptionName = names[i] if (this.locallyProvidedRecords.has(subscriptionName)) { continue } if (!subscriptionName.match(regExp)) { continue } const listenInProgress = this.listenInProgress.get(subscriptionName) if (listenInProgress && socketWrapper) { listenInProgress.remainingProviders.push({ socketWrapper, pattern }) } else if (listenInProgress) { // A reconsile happened while listen is still in progress, ignore } else { this.startProviderSearch(subscriptionName) } } } /** * Removes the listener if it is the currently active publisher, and retriggers * another listener discovery phase */ private removeListenerIfActive (pattern: string, socketWrapper: SocketWrapper): void { for (const [subscriptionName, provider] of this.locallyProvidedRecords) { if ( provider.socketWrapper === socketWrapper && provider.pattern === pattern ) { if (provider.closeListener) { provider.socketWrapper.removeOnClose(provider.closeListener) } this.removeActiveListener(subscriptionName) if (this.clientRegistry.hasLocalSubscribers(subscriptionName)) { this.startProviderSearch(subscriptionName) } } } } /** */ private removeActiveListener (subscriptionName: string): void { this.clusterProvidedRecords.remove(subscriptionName) this.locallyProvidedRecords.delete(subscriptionName) } /** * Start discovery phase once a lock is obtained from the leader within * the cluster */ private startProviderSearch (subscriptionName: string): void { const localListenArray = this.createLocalListenArray(subscriptionName) if (localListenArray.length === 0) { return } this.services.locks.get(this.getUniqueLockName(subscriptionName), (success: boolean) => { if (!success) { return } if (this.hasActiveProvider(subscriptionName)) { this.services.locks.release(this.getUniqueLockName(subscriptionName)) return } this.startLocalDiscoveryStage(subscriptionName, localListenArray) }) } /** * Start discovery phase once a lock is obtained from the leader within * the cluster */ private startLocalDiscoveryStage (subscriptionName: string, localListenArray: Provider[]): void { this.services.logger.debug( EVENT.LOCAL_LISTEN, `started for ${TOPIC[this.topic] || STATE_REGISTRY_TOPIC[this.topic]}:${subscriptionName}`, this.metaData, ) this.triggerNextProvider(subscriptionName, localListenArray) } /** * Trigger the next provider in the map of providers capable of publishing * data to the specific subscriptionName */ private triggerNextProvider (subscriptionName: string, localListenArray?: Provider[]): void { let listenInProgress = this.listenInProgress.get(subscriptionName) let provider: Provider if (localListenArray) { provider = localListenArray.shift()! listenInProgress = { queryProvider: provider, remainingProviders: localListenArray } this.listenInProgress.set(subscriptionName, listenInProgress) } else if (listenInProgress) { if (listenInProgress.remainingProviders.length === 0) { this.stopProviderSearch(subscriptionName) return } provider = listenInProgress.remainingProviders.shift()! listenInProgress.queryProvider = provider } else { this.services.logger.warn('triggerNextProvider', 'no listen in progress', this.metaData) return } // const subscribers = this.clientRegistry.getLocalSubscribers(subscriptionName) // This stops a client from subscribing to itself, I think // if (subscribers && subscribers.has(provider!.socketWrapper)) { // this.services.logger.debug(EVENT.LOCAL_LISTEN, `Ignoring socket since it would be subscribing to itself for ${subscriptionName}`) // this.triggerNextProvider(subscriptionName) // return // } provider!.responseTimeout = setTimeout(() => { provider!.socketWrapper.sendMessage({ topic: this.topic, action: this.actions.LISTEN_RESPONSE_TIMEOUT, subscription: subscriptionName }) this.triggerNextProvider(subscriptionName) }, this.config.listen.responseTimeout) this.sendSubscriptionForPatternFound(provider!, subscriptionName) } /** * Finalises a local listener discovery stage */ private stopProviderSearch (subscriptionName: string): void { this.services.logger.debug( EVENT.LOCAL_LISTEN, `stopped for ${TOPIC[this.topic] || STATE_REGISTRY_TOPIC[this.topic]}:${subscriptionName}`, this.metaData, ) this.services.locks.release(this.getUniqueLockName(subscriptionName)) const stoppedSearch = this.listenInProgress.delete(subscriptionName) if (stoppedSearch) { if (this.hasActiveProvider(subscriptionName) === false) { this.unsuccesfulMatches.set(subscriptionName, Date.now()) this.services.clusterNode.send({ topic: this.messageTopic, action: this.actions.LISTEN_UNSUCCESSFUL, subscription: subscriptionName }) } return } this.services.logger.warn( EVENT.LOCAL_LISTEN, `nothing to stop for ${TOPIC[this.topic] || STATE_REGISTRY_TOPIC[this.topic]}:${subscriptionName}`, this.metaData, ) } /** * Triggered when a subscription is being provided by a node in the cluster */ private onRecordStartProvided (subscriptionName: string): void { this.sendHasProviderUpdate(true, subscriptionName) } /** * Triggered when a subscription is stopped being provided by a node in the cluster */ private onRecordStopProvided (subscriptionName: string): void { this.services.logger.info( 'LISTEN_PROVIDER_STOPPED', `listen provider has stopped for ${TOPIC[this.topic]}:${subscriptionName}`, this.metaData, ) this.sendHasProviderUpdate(false, subscriptionName) if (!this.hasActiveProvider(subscriptionName) && this.clientRegistry.hasName(subscriptionName)) { this.startProviderSearch(subscriptionName) } } /** * Compiles a regular expression from an incoming pattern */ private addPattern (pattern: string): void { if (!this.patterns.has(pattern)) { this.patterns.set(pattern, new RegExp(pattern)) } } /** * Deletes the pattern regex when removed */ private removePattern (pattern: string, socketWrapper: SocketWrapper): void { this.removeListenerFromInProgress(this.listenInProgress, pattern, socketWrapper) this.removeListenerIfActive(pattern, socketWrapper) } private removeLastPattern (pattern: string): void { this.patterns.delete(pattern) } /** * Remove provider from listen in progress map if it unlistens during discovery stage */ private removeListenerFromInProgress (listensCurrentlyInProgress: Map, pattern: string, socketWrapper: SocketWrapper): void { for (const [subscriptionName, listensInProgress] of listensCurrentlyInProgress) { listensInProgress.remainingProviders = listensInProgress.remainingProviders.filter((provider: Provider) => { return provider.socketWrapper === socketWrapper && provider.pattern === pattern }) if (listensInProgress.remainingProviders.length === 0) { this.stopProviderSearch(subscriptionName) } } } /** * Sends a has provider update to a single subcriber */ private sendHasProviderUpdateToSingleSubscriber (hasProvider: boolean, socketWrapper: SocketWrapper, subscriptionName: string): void { if (socketWrapper && this.topic === TOPIC.RECORD) { socketWrapper.sendMessage({ topic: this.topic, action: hasProvider ? RECORD_ACTION.SUBSCRIPTION_HAS_PROVIDER : RECORD_ACTION.SUBSCRIPTION_HAS_NO_PROVIDER, name: subscriptionName, }) } } /** * Sends a has provider update to all subcribers */ private sendHasProviderUpdate (hasProvider: boolean, subscriptionName: string): void { if (this.topic !== TOPIC.RECORD) { return } this.clientRegistry.sendToSubscribers(subscriptionName, { topic: this.topic, action: hasProvider ? RECORD_ACTION.SUBSCRIPTION_HAS_PROVIDER : RECORD_ACTION.SUBSCRIPTION_HAS_NO_PROVIDER, name: subscriptionName, }, false, null) } /** * Send a subscription found to a provider */ private sendSubscriptionForPatternFound (provider: Provider, subscriptionName: string): void { provider.socketWrapper.sendMessage({ topic: this.topic, action: this.actions.SUBSCRIPTION_FOR_PATTERN_FOUND, name: provider.pattern, subscription: subscriptionName, }) } /** * Send a subscription removed to a provider */ private sendSubscriptionForPatternRemoved (provider: Provider, subscriptionName: string): void { provider.socketWrapper.sendMessage({ topic: this.topic, action: this.actions.SUBSCRIPTION_FOR_PATTERN_REMOVED, name: provider.pattern, subscription: subscriptionName, }) } /** * Create a map of all the listeners that patterns match the subscriptionName locally */ private createLocalListenArray (subscriptionName: string): Provider[] { const providers: Provider[] = [] this.patterns.forEach((regex, pattern) => { if (regex.test(subscriptionName)) { for (const socketWrapper of this.providerRegistry.getLocalSubscribers(pattern)) { providers.push({ pattern, socketWrapper }) } } }) if (!this.config.listen.shuffleProviders) { return providers } return shuffleArray(providers) } /** * Validates that the pattern is not empty and is a valid regular expression */ private validatePattern (socketWrapper: SocketWrapper, message: ListenMessage): RegExp | null { try { return new RegExp(message.name) } catch (e) { socketWrapper.sendMessage({ topic: this.topic, action: this.actions.INVALID_LISTEN_REGEX, name: message.name }) this.services.logger.warn(this.actions[this.actions.INVALID_LISTEN_REGEX], `${e}`, this.metaData) return null } } /** * Returns the unique lock when leading a listen discovery phase */ private getUniqueLockName (subscriptionName: string) { return `${this.uniqueLockName}_${subscriptionName}` } } ================================================ FILE: src/listen/listener-test-utils.ts ================================================ import 'mocha' import { expect } from 'chai' import { ListenerRegistry } from './listener-registry' import * as testHelper from '../test/helper/test-helper' import * as C from '../constants' import { getTestMocks } from '../test/helper/test-mocks' import * as sinon from 'sinon' import { SocketWrapper, SubscriptionRegistry } from '@deepstream/types' import { TOPIC, ListenMessage } from '../constants' export default class ListenerTestUtils { private actions: any private subscribedTopics: string[] = [] private topic: TOPIC.RECORD | TOPIC.EVENT private subscribers = new Set() private clientRegistryMock: any private providers: Array<{ socketWrapper: SocketWrapper, socketWrapperMock: any }> private clients: Array<{ socketWrapper: SocketWrapper, socketWrapperMock: any }> private listenerRegistry: ListenerRegistry private clientRegistry: SubscriptionRegistry constructor (listenerTopic?: TOPIC.RECORD | TOPIC.EVENT) { const { config, services } = testHelper.getDeepstreamOptions() this.topic = listenerTopic || C.TOPIC.RECORD if (this.topic === C.TOPIC.RECORD) { this.actions = C.RECORD_ACTION } else { this.actions = C.EVENT_ACTION } const self = this this.clientRegistry = { hasName (subscriptionName: string) { return self.subscribedTopics.indexOf(subscriptionName) === -1 }, getNames () { return self.subscribedTopics }, getLocalSubscribers () { return self.subscribers }, hasLocalSubscribers () { return self.subscribers.size > 0 }, sendToSubscribers: () => {} } as never as SubscriptionRegistry this.clientRegistryMock = sinon.mock(this.clientRegistry) config.listen.responseTimeout = 30 config.listen.shuffleProviders = false // config.stateReconciliationTimeout = 10 this.clients = [ // @ts-ignore null, // to make tests start from 1 getTestMocks().getSocketWrapper('c1'), getTestMocks().getSocketWrapper('c2'), getTestMocks().getSocketWrapper('c3') ] this.providers = [ // @ts-ignore null, // to make tests start from 1 getTestMocks().getSocketWrapper('p1'), getTestMocks().getSocketWrapper('p2'), getTestMocks().getSocketWrapper('p3') ] this.listenerRegistry = new ListenerRegistry(self.topic, config, services, self.clientRegistry) expect(typeof self.listenerRegistry.handle).to.equal('function') } public complete () { this.clients.forEach((client) => { if (client) { client.socketWrapperMock.verify() } }) this.providers.forEach((provider) => { if (provider) { provider.socketWrapperMock.verify() } }) this.clientRegistryMock.verify() } /** * Provider Utils */ public providerListensTo (provider: number, pattern: string): void { this.providers[provider].socketWrapperMock .expects('sendAckMessage') .once() .withExactArgs({ topic: this.topic, action: this.actions.LISTEN, name: pattern }) this.listenerRegistry.handle(this.providers[provider].socketWrapper, { topic: this.topic, action: this.actions.LISTEN, name: pattern } as never as ListenMessage) } public providerUnlistensTo (provider: number, pattern: string) { this.providers[provider].socketWrapperMock .expects('sendAckMessage') .once() .withExactArgs({ topic: this.topic, action: this.actions.UNLISTEN, name: pattern }) this.listenerRegistry.handle(this.providers[provider].socketWrapper, { topic: this.topic, action: this.actions.UNLISTEN, name: pattern, } as never as ListenMessage) } public providerWillGetListenTimeout (provider: number, subscription: string) { this.providers[provider].socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: this.topic, action: this.actions.LISTEN_RESPONSE_TIMEOUT, subscription }) } public providerWillGetSubscriptionFound (provider: number, pattern: string, subscription: string) { this.providers[provider].socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: this.topic, action: this.actions.SUBSCRIPTION_FOR_PATTERN_FOUND, name: pattern, subscription }) } public providerWillGetSubscriptionRemoved (provider: number, pattern: string, subscription: string) { this.providers[provider].socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: this.topic, action: this.actions.SUBSCRIPTION_FOR_PATTERN_REMOVED, name: pattern, subscription }) } public providerAcceptsButIsntAcknowledged (provider: number, pattern: string, subscriptionName: string) { this.providerAccepts(provider, pattern, subscriptionName, true) } public providerAccepts (provider: number, pattern: string, subscription: string, doesnthaveActiveProvider: boolean) { this.listenerRegistry.handle(this.providers[provider].socketWrapper, { topic: this.topic, action: this.actions.LISTEN_ACCEPT, name: pattern, subscription }) expect(this.listenerRegistry.hasActiveProvider(subscription)).to.equal(!doesnthaveActiveProvider) } public providerRejectsAndPreviousTimeoutProviderThatAcceptedIsUsed (provider: number, pattern: string, subscriptionName: string) { this.providerRejects(provider, pattern, subscriptionName, true) } public providerAcceptsAndIsSentSubscriptionRemoved (provider: number, pattern: string, subscriptionName: string) { this.providerWillGetSubscriptionRemoved(provider, pattern, subscriptionName) this.providerAcceptsButIsntAcknowledged(provider, pattern, subscriptionName) } public providerRejects (provider: number, pattern: string, subscription: string, doNotCheckActiveProvider: boolean) { this.listenerRegistry.handle(this.providers[provider].socketWrapper, { topic: this.topic, action: this.actions.LISTEN_REJECT, name: pattern, subscription }) if (!doNotCheckActiveProvider) { expect(this.listenerRegistry.hasActiveProvider(subscription)).to.equal(false) } } public acceptMessageThrowsError (provider: number, pattern: string, subscription: string) { this.listenerRegistry.handle(this.providers[provider].socketWrapper, { topic: this.topic, action: this.actions.LISTEN_ACCEPT, name: pattern, subscription }) // verify( providers[ provider], this.actions.ERROR, [ C.EVENT.INVALID_MESSAGE, this.actions.LISTEN_ACCEPT, pattern, subscriptionName ] ); } public rejectMessageThrowsError (provider: number, pattern: string, subscription: string) { this.listenerRegistry.handle(this.providers[provider].socketWrapper, { topic: this.topic, action: this.actions.LISTEN_REJECT, name: pattern, subscription }) // TODO // verify( providers[ provider], this.actions.ERROR, [ C.EVENT.INVALID_MESSAGE, this.actions.LISTEN_REJECT, pattern, subscriptionName ] ); } public providerLosesItsConnection (provider: number) { // (this.providers[provider].socketWrapper as any).emit('close', this.providers[provider].socketWrapper) } /** * Subscriber Utils */ public subscriptionAlreadyMadeFor (subscriptionName: string) { this.subscribedTopics.push(subscriptionName) } public clientSubscribesTo (client: number, subscriptionName: string, firstSubscription: boolean) { if (firstSubscription) { this.listenerRegistry.onFirstSubscriptionMade(subscriptionName) } this.listenerRegistry.onSubscriptionMade(subscriptionName, this.clients[client].socketWrapper) this.subscribedTopics.push(subscriptionName) this.subscribers.add(this.clients[client].socketWrapper) } public clientUnsubscribesTo (client: number, subscriptionName: string, lastSubscription: boolean) { if (lastSubscription) { this.listenerRegistry.onLastSubscriptionRemoved(subscriptionName) } this.listenerRegistry.onSubscriptionRemoved(subscriptionName, this.clients[client].socketWrapper) this.subscribedTopics.splice(this.subscribedTopics.indexOf(subscriptionName), 1) this.subscribers.delete(this.clients[client].socketWrapper) } public clientWillRecievePublishedUpdate (client: number, subscription: string, state: boolean) { this.clients[client].socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: this.topic, action: state ? this.actions.SUBSCRIPTION_HAS_PROVIDER : this.actions.SUBSCRIPTION_HAS_NO_PROVIDER, name: subscription, }) } public publishUpdateWillBeSentToSubscribers (subscription: string, state: boolean) { this.clientRegistryMock .expects('sendToSubscribers') .once() .withExactArgs(subscription, { topic: this.topic, action: state ? this.actions.SUBSCRIPTION_HAS_PROVIDER : this.actions.SUBSCRIPTION_HAS_NO_PROVIDER, name: subscription, }, false, null) } public subscriptionHasActiveProvider (subscription: string, value: string) { expect(this.listenerRegistry.hasActiveProvider(subscription)).to.equal(value) } } ================================================ FILE: src/plugins/heap-snapshot/heap-snapshot.ts ================================================ import { DeepstreamPlugin, DeepstreamServices, EVENT } from '@deepstream/types' import { writeHeapSnapshot } from 'v8' import { existsSync, mkdirSync } from 'fs' interface HeapSnapshotOptions { interval: number, outputDir: string } /** * This plugin will log the handshake data on login/logout and send a custom event to the logged-in * client. */ export default class HeapSnapshot extends DeepstreamPlugin { public description = 'V8 Memory Analysis' private logger = this.services.logger.getNameSpace('MEMORY_ANALYSIS') private snapshotInterval!: NodeJS.Timer constructor (private options: HeapSnapshotOptions, private services: Readonly) { super() } public init () { if (typeof this.options.interval !== 'number') { this.logger.fatal(EVENT.ERROR, 'Invalid or missing "interval"') } if (this.options.interval < 10000) { this.logger.fatal(EVENT.ERROR, 'interval must be above 10000') } if (typeof this.options.outputDir !== 'string') { this.logger.fatal(EVENT.ERROR, 'Invalid or missing "outputDir"') } if (existsSync(this.options.outputDir) === false) { mkdirSync(this.options.outputDir, 0o744) } } public async whenReady (): Promise { this.snapshotInterval = setInterval(() => this.outputHeapSnapshot(), this.options.interval) } public async close (): Promise { clearInterval(this.snapshotInterval) } private outputHeapSnapshot () { writeHeapSnapshot(`${this.options.outputDir}/${Date.now()}-${process.pid}.heapsnapshot`) this.logger.info(EVENT.INFO, 'Taking a heap snapshot. This might affect your CPU usage drastically.') } } ================================================ FILE: src/service/daemon.ts ================================================ // Handle input parameters import { spawn, ChildProcess } from 'child_process' const maxMilliseconds = 5000 function _start (options: any) { let startTime: number | null = null let child!: ChildProcess let attempts = 0 let starts = 0 let wait = 1 /** * Monitor the process to make sure it is running */ function monitor () { if (!child.pid) { // If the number of periodic starts exceeds the max, kill the process if (starts >= options.maxRetries) { if ((Date.now() - startTime!) > maxMilliseconds) { console.error( `Too many restarts within the last ${maxMilliseconds / 1000} seconds. Please check the script.` ) process.exit(1) } } setTimeout(() => { wait = wait * options.growPercentage attempts += 1 if (attempts > options.maxRestarts && options.maxRestarts >= 0) { console.error( `${options.name} will not be restarted because the maximum number of total restarts has been exceeded.` ) process.exit() } else { launch() } }, wait) } else { attempts = 0 wait = options.restartDelay * 1000 } } /** * @method launch * A method to start a process. */ function launch () { // Set the start time if it's null if (startTime === null) { startTime = startTime || Date.now() setTimeout(() => { startTime = null starts = 0 }, ( maxMilliseconds ) + 2000) } starts += 1 // Fork the child process piping stdin/out/err to the parent // NOTE: ADDING process.pkg.entrypoint as first arg to the spawned process is a workaround to fix a pkg bug // https://github.com/vercel/pkg/issues/1356 // @ts-ignore child = spawn(options.processExec, [process.pkg.entrypoint, 'start'].concat(process.argv.slice(2)), { env: process.env }) child.stdout!.on('data', function (data) { process.stdout.write(data.toString()) }) child.stderr!.on('data', function (data) { process.stderr.write(data.toString()) }) // When the child dies, attempt to restart based on configuration child.on('exit', (code) => { // If an error is thrown and the process is configured to exit, then kill the parent. if (code !== 0 && options.exitOnError) { console.error(`${options.name} exited with error code ${code}`) process.exit() } // delete child.pid monitor() }) } process.on('exit', () => { child.removeAllListeners('exit') if (child) { child.kill() } }) process.on('SIGTERM', () => { child.removeAllListeners('exit') child.kill() }) process.on('SIGHUP', () => { child.removeAllListeners('exit') child.kill() }) process.on('SIGINT', () => { child.removeAllListeners('exit') child.kill() }) launch() } export const start = (daemonOptions: any) => { const options = daemonOptions || {} _start({ name: 'deepstream', exitOnError: false, growPercentage: 0.25, maxRetries: 10, restartDelay: 1, ...options }) } ================================================ FILE: src/service/service.ts ================================================ import { exec } from 'child_process' import { existsSync, unlinkSync, chmodSync, writeFileSync } from 'fs' import systemdTemplate from './template/systemd' import initdTemplate from './template/initd' /** * Returns true if system support systemd daemons * @return {Boolean} */ function hasSystemD () { return existsSync('/usr/lib/systemd/system') || existsSync('/bin/systemctl') } /** * Returns true if system support init.d daemons * @return {Boolean} */ function hasSystemV () { return existsSync('/etc/init.d') } /** * Deletes a service file from /etc/systemd/system/ */ async function deleteSystemD (name: string, callback: Function) { const filepath = `/etc/systemd/system/${name}.service` console.log(`Removing service on: ${filepath}`) const exists = existsSync(filepath) if (!exists) { callback("Service doesn't exists, nothing to uninstall") return } try { unlinkSync(filepath) const cmd = 'systemctl daemon-reload' console.log('Running %s...', cmd) exec(cmd, (e) => { callback(e, 'SystemD service removed successfully') }) } catch (e) { callback(e) } } /** * Installs a service file to /etc/systemd/system/ * * It deals with logs, restarts and by default points * to the normal system install */ async function setupSystemD (name: string, options: any, callback: Function) { options.stdOut = (options.logDir && `${options.logDir}/${name}-out.log`) || null options.stdErr = (options.logDir && `${options.logDir}/${name}-err.log`) || null const filepath = `/etc/systemd/system/${name}.service` const script = systemdTemplate(options) if (options.dryRun) { console.log(script) return } console.log(`Installing service on: ${filepath}`) const exists = existsSync(filepath) if (exists) { callback('Service already exists, please uninstall first') return } try { writeFileSync(filepath, script) chmodSync(filepath, '755') const cmd = 'systemctl daemon-reload' console.log('Running %s...', cmd) exec(cmd, (e2) => { callback(e2, 'SystemD service registered successfully') }) } catch (e) { callback(e) } } /** * Deletes a service file from /etc/init.d/ */ async function deleteSystemV (name: string, callback: Function) { const filepath = `/etc/init.d/${name}` console.log(`Removing service on: ${filepath}`) const exists = existsSync(filepath) if (!exists) { callback("Service doesn't exists, nothing to uninstall") return } try { unlinkSync(filepath) callback(null, 'SystemD service removed successfully') } catch (e) { callback(e) } } /** * Installs a service file to /etc/init.d/ * * It deals with logs, restarts and by default points * to the normal system install */ async function setupSystemV (name: string, options: any, callback: Function) { options.stdOut = (options.logDir && `${options.logDir}/${name}-out.log`) || '/dev/null' options.stdErr = (options.logDir && `${options.logDir}/${name}-err.log`) || '&1' const script = initdTemplate(options) if (options.dryRun) { console.log(script) return } const filepath = `/etc/init.d/${name}` console.log(`Installing service on: ${filepath}`) const exists = existsSync(filepath) if (exists) { callback('Service already exists, please uninstall first') return } try { writeFileSync(filepath, script) chmodSync(filepath, '755') callback(null, 'init.d service registered successfully') } catch (e) { callback(e) } } /** * Adds a service, either via systemd or init.d * @param {String} name the name of the service * @param {Object} options options to configure deepstream service * @param {Function} callback called when complete */ export const add = (name: string, options: any, callback: Function) => { options.name = name options.pidFile = options.pidFile || `/var/run/${name}.pid` options.exec = options.exec options.logDir = options.logDir || '/var/log/deepstream' options.user = options.user || 'root' options.group = options.group || 'root' if (options && !options.runLevels) { options.runLevels = [2, 3, 4, 5].join(' ') } else { options.runLevels = options.runLevels.join(' ') } if (!options.programArgs) { options.programArgs = [] } options.deepstreamArgs = ['daemon'].concat(options.programArgs).join(' ') if (hasSystemD()) { setupSystemD(name, options, callback) } else if (hasSystemV()) { setupSystemV(name, options, callback) } else { callback('Only systemd and init.d services are currently supported.') } } /** * Delete a service, either from systemd or init.d * @param {String} name the name of the service * @param {Function} callback called when complete */ export const remove = (name: string, callback: Function) => { if (hasSystemD()) { deleteSystemD(name, callback) } else if (hasSystemV()) { deleteSystemV(name, callback) } else { callback('Only systemd and init.d services are currently supported.') } } /** * Start a service, either from systemd or init.d * @param {String} name the name of the service * @param {Function} callback called when complete */ export const start = (name: string, callback: Function) => { if (hasSystemD() || hasSystemV()) { exec(`service ${name} start`, (err, stdOut, stdErr) => { callback(err || stdErr, stdOut) }) } else { callback('Only systemd and init.d services are currently supported.') } } /** * Stop a service, either from systemd or init.d * @param {String} name the name of the service * @param {Function} callback called when complete */ export const stop = (name: string, callback: Function) => { if (hasSystemD() || hasSystemV()) { exec(`service ${name} stop`, (err, stdOut, stdErr) => { callback(err || stdErr, stdOut) }) } else { callback('Only systemd and init.d services are currently supported.') } } /** * Get the status of the service, either from systemd or init.d * @param {String} name the name of the service * @param {Function} callback called when complete */ export const status = (name: string, callback: Function) => { if (hasSystemD() || hasSystemV()) { exec(`service ${name} status`, (err, stdOut, stdErr) => { callback(err || stdErr, stdOut) }) } else { callback('Only systemd and init.d services are currently supported.') } } /** * Restart the service, either from systemd or init.d * @param {String} name the name of the service * @param {Function} callback called when complete */ export const restart = (name: string, callback: Function) => { if (hasSystemD() || hasSystemV()) { exec(`service ${name} restart`, (err, stdOut, stdErr) => { callback(err || stdErr, stdOut) }) } else { callback('Only systemd and init.d services are currently supported.') } } ================================================ FILE: src/service/template/initd.ts ================================================ export default (d: any) => `#!/bin/bash ### BEGIN INIT INFO # Provides: ${d.name} # Required-Start: # Required-Stop: # Default-Start: ${d.runLevels} # Default-Stop: 0 1 6 # Short-Description: Start ${d.name} at boot time # Description: Enable ${d.name} service. ### END INIT INFO # chkconfig: ${d.runLevels} 99 1 # description: ${d.name} set_pid () { unset PID _PID=\`head -1 "${d.pidFile}" 2>/dev/null\` if [ $_PID ]; then kill -0 $_PID 2>/dev/null && PID=$_PID fi } restart () { stop start } start () { CNT=5 set_pid if [ -z "$PID" ]; then echo starting ${d.name} if [ -e "/var/deepstream/DEEPSTREAM_SETUP" ]; then bash "/var/deepstream/DEEPSTREAM_SETUP" fi if [ -e "/var/deepstream/DEEPSTREAM_ENV_VARS" ]; then source "/var/deepstream/DEEPSTREAM_ENV_VARS" fi mkdir -p ${d.logDir} "${d.exec}" ${d.deepstreamArgs} >> ${d.stdOut} 2>> ${d.stdErr} & echo $! > "${d.pidFile}" while [ : ]; do set_pid if [ -n "$PID" ]; then echo started ${d.name} break else if [ $CNT -gt 0 ]; then sleep 1 CNT=\`expr $CNT - 1\` else echo ERROR - failed to start ${d.name} break fi fi done else echo ${d.name} is already started fi } status () { set_pid if [ -z "$PID" ]; then echo ${d.name} is not running exit 1 else echo ${d.name} is running exit 0 fi } stop () { CNT=30 set_pid if [ -n "$PID" ]; then echo stopping ${d.name} kill $PID while [ : ]; do set_pid if [ -z "$PID" ]; then rm "${d.pidFile}" echo stopped ${d.name} break else if [ $CNT -gt 0 ]; then sleep 1 CNT=\`expr $CNT - 1\` else echo ERROR - failed to stop ${d.name} break fi fi done else echo ${d.name} is already stopped fi } case $1 in restart) restart ;; start) start ;; status) status ;; stop) stop ;; *) echo "usage: $0 " exit 1 ;; esac ` ================================================ FILE: src/service/template/systemd.ts ================================================ export default (d: any) => `[Unit] Description=${d.name} After=network.target [Service] Type=simple StandardOutput=${d.stdOut} StandardError=${d.stdErr} ExecStart=${d.exec} ${d.deepstreamArgs} Restart=always User=${d.user} Group=${d.group} Environment= [Install] WantedBy=multi-user.target ` ================================================ FILE: src/services/authentication/combine/combine-authentication.ts ================================================ import { DeepstreamPlugin, DeepstreamAuthenticationCombiner, DeepstreamAuthentication, UserAuthenticationCallback } from '@deepstream/types' import { JSONObject } from '../../../constants' /** * The open authentication handler allows every client to connect. * If the client specifies a username as part of its authentication * data, it will be used to identify the user internally */ export class CombineAuthentication extends DeepstreamPlugin implements DeepstreamAuthenticationCombiner { public description: string = '' constructor (private auths: DeepstreamAuthentication[]) { super() if (auths.length === 1) { this.description = auths[0].description } else { this.description = auths.map((auth, index) => `\n\t${index}) ${auth.description}`).join('') } } public async whenReady () { await Promise.all(this.auths.map((auth) => auth.whenReady())) } public async close () { await Promise.all(this.auths.map((auth) => auth.close())) } public async isValidUser (connectionData: JSONObject, authData: JSONObject, callback: UserAuthenticationCallback) { for (const auth of this.auths) { const result = await auth.isValidUser(connectionData, authData) if (result) { callback(result.isValid, result) return } } callback(false) } public onClientDisconnect (userId: string): void { for (const auth of this.auths) { if (auth.onClientDisconnect) { auth.onClientDisconnect(userId) } } } } ================================================ FILE: src/services/authentication/file/file-based-authentication.spec.ts ================================================ import { spy, assert } from 'sinon' import { expect } from 'chai' import { FileBasedAuthentication } from './file-based-authentication' import { DeepstreamServices, EVENT, MetaData } from '@deepstream/types' import { PromiseDelay } from '../../../utils/utils' import * as users from '../../../test/config/users.json' import * as usersUnhashed from '../../../test/config/users-unhashed.json' import * as invalidUsersConfig from '../../../test/config/invalid-user-config.json' import * as emptyUsersMap from '../../../test/config/empty-map-config.json' const createServices = () => { return { logger: { fatal: spy() } as never as Logger } as DeepstreamServices } const testAuthentication = async ({ username, password, handler, notFound, isValid, clientData, serverData }) => { const result = await handler.isValidUser(null, { username, password }) if (notFound) { expect(result).to.equal(null) return } expect(result.isValid).to.eq(isValid) if (isValid) { expect(result.id).to.equal(username) expect(result.clientData).to.deep.equal(clientData) expect(result.serverData).to.deep.equal(serverData) } else { expect(result).to.deep.equal({ isValid }) } } describe('file based authentication', () => { describe('does authentication for cleartext passwords', () => { let authenticationHandler beforeEach(async () => { authenticationHandler = new FileBasedAuthentication({ users: usersUnhashed, hash: false }, createServices()) await authenticationHandler.whenReady() expect(authenticationHandler.description).to.eq('File Authentication') }) it('confirms userC with valid password', async () => { await testAuthentication({ handler: authenticationHandler, username: 'userC', password: 'userCPass', isValid: true, serverData: { some: 'values' }, clientData: { all: 'othervalue' } }) }) it('confirms userD with valid password', async () => { await testAuthentication({ username: 'userD', password: 'userDPass', isValid: true, serverData: null, clientData: { all: 'client data' }, handler: authenticationHandler }) }) it('rejects userC with invalid password', async () => { await testAuthentication({ username: 'userC', password: 'userDPass', isValid: false, serverData: null, clientData: null, handler: authenticationHandler }) }) }) describe('does authentication for hashed passwords', () => { let authenticationHandler beforeEach(async () => { authenticationHandler = new FileBasedAuthentication({ users, hash: 'md5', iterations: 100, keyLength: 32, reportInvalidParameters: true }, createServices()) await authenticationHandler.whenReady() }) it('confirms userA with valid password', async () => { await testAuthentication({ handler: authenticationHandler, username: 'userA', password: 'userAPass', isValid: true, serverData: { some: 'values' }, clientData: { all: 'othervalue' } }) }) it('rejects userA with an invalid password', async () => { await testAuthentication({ username: 'userA', password: 'wrongPassword', isValid: false, handler: authenticationHandler }) }) it('rejects userA with user B\'s password', async () => { await testAuthentication({ username: 'userA', password: 'userBPass', isValid: false, handler: authenticationHandler }) }) it('accepts userB with user B\'s password', async () => { await testAuthentication({ username: 'userB', password: 'userBPass', isValid: true, serverData: null, clientData: { all: 'client data' }, handler: authenticationHandler }) }) it('returns null for userQ', async () => { await testAuthentication({ handler: authenticationHandler, username: 'userQ', password: 'userBPass', notFound: true, }) }) }) describe('errors for invalid settings', () => { const getSettings = function () { return { users, hash: 'md5', iterations: 100, keyLength: 32 } } it('accepts settings with hash = false', () => { const settings = { users: usersUnhashed, hash: false } expect(() => { // tslint:disable-next-line:no-unused-expression new FileBasedAuthentication(settings as any, createServices()) }).not.to.throw() }) it('fails for settings with hash=string that miss hashing parameters', () => { const settings = { usersUnhashed, hash: 'md5' } expect(() => { // tslint:disable-next-line:no-unused-expression new FileBasedAuthentication(settings as any) }).to.throw() }) it('fails for settings with non-existing hash algorithm', () => { const settings = getSettings() settings.hash = 'does-not-exist' expect(() => { // tslint:disable-next-line:no-unused-expression new FileBasedAuthentication(settings) }).to.throw() }) }) describe('errors for invalid configs', () => { const test = async (settings: any, errorMessage: string, expectedMetaData?: MetaData) => { const services = createServices() // tslint:disable-next-line: no-unused-expression new FileBasedAuthentication(settings, services) await PromiseDelay(10) assert.calledOnce(services.logger.fatal) if (!expectedMetaData) { assert.calledWithExactly(services.logger.fatal, EVENT.PLUGIN_INITIALIZATION_ERROR, errorMessage) return } assert.calledWith(services.logger.fatal, EVENT.PLUGIN_INITIALIZATION_ERROR, errorMessage) const actualMetadata = services.logger.fatal.getCall(0).args[2] if (expectedMetaData.error) { expect(actualMetadata.error.message).to.equal(expectedMetaData.error.message) } const actualMetadataWithoutError = { ...actualMetadata, error: null } const expectedMetadataWithoutError = { ...expectedMetaData, error: null } expect(actualMetadataWithoutError).to.deep.equal(expectedMetadataWithoutError) } it('loads a user config without password field',async () => { await test({ users: invalidUsersConfig, hash: false }, 'missing password for userB') }) it('loads a user config without users', async() => { await test({ users: emptyUsersMap, hash: false }, 'no users present in user file') }) it('loads a user config with invalid hashing parameters', async() => { await test( { users: users, hash: 'md5', iterations: '100', keyLength: 32 }, 'Validating settings failed for file auth', { error: new Error('Invalid type string for iterations') } ) }) }) describe('errors for invalid auth-data', () => { let authenticationHandler const settings = { users, hash: 'md5', iterations: 100, keyLength: 32, reportInvalidParameters: true } beforeEach(async () => { authenticationHandler = new FileBasedAuthentication(settings, createServices()) await authenticationHandler.whenReady() }) it('returns null for authData without username', async () => { const result = await authenticationHandler.isValidUser(null, { password: 'some password' }) expect(result.isValid).to.eq(false) expect(result.clientData).to.deep.eq({ error: 'missing authentication parameter: username or/and password' }) }) it('returns an error for authData without password', async () => { const result = await authenticationHandler.isValidUser(null, { username: 'some user' }) expect(result.isValid).to.eq(false) expect(result.clientData).to.deep.eq({ error: 'missing authentication parameter: username or/and password' }) }) }) }) ================================================ FILE: src/services/authentication/file/file-based-authentication.ts ================================================ import { DeepstreamPlugin, DeepstreamAuthentication, DeepstreamServices, EVENT } from '@deepstream/types' import { validateMap, createHash, validateHashingAlgorithm } from '../../../utils/utils' interface FileAuthConfig { users: any // the name of a HMAC digest algorithm, a.g. 'sha512' hash: string | false // the amount of times the algorithm should be applied iterations: number // the length of the resulting key keyLength: number, // fail authentication process if invalid login parameters are used reportInvalidParameters: boolean } /** * This authentication handler reads a list of users and their associated password (either * hashed or in cleartext ) from a json file. This can be useful to authenticate smaller amounts * of clients with static credentials, e.g. backend provider that write to publicly readable records */ export class FileBasedAuthentication extends DeepstreamPlugin implements DeepstreamAuthentication { public description: string = 'File Authentication' private base64KeyLength: number private hashSettings = { iterations: this.settings.iterations, keyLength: this.settings.keyLength, algorithm: this.settings.hash } /** * Creates the class, reads and validates the users.json file */ constructor (private settings: FileAuthConfig, private services: DeepstreamServices) { super() this.validateSettings(settings) this.base64KeyLength = 4 * Math.ceil(this.settings.keyLength / 3) if (this.settings.reportInvalidParameters === undefined) { this.settings.reportInvalidParameters = true } } public async whenReady (): Promise { } /** * Main interface. Authenticates incoming connections */ public async isValidUser (connectionData: any, authData: any) { const missingUsername = typeof authData.username !== 'string' const missingPassword = typeof authData.password !== 'string' if (missingPassword || missingUsername) { if (this.settings.reportInvalidParameters) { return { isValid: false, clientData: { error: 'missing authentication parameter: username or/and password' } } } else { return null } } const userData = this.settings.users[authData.username] if (!userData) { return null } const actualPassword = this.settings.hash ? userData.password.substr(0, this.base64KeyLength) : userData.password let expectedPassword = authData.password if (typeof this.settings.hash === 'string') { ({ hash: expectedPassword} = await createHash(authData.password, this.hashSettings as any, userData.password.substr(this.base64KeyLength))) expectedPassword = expectedPassword.toString('base64') } if (actualPassword === expectedPassword) { return { isValid: true, id: authData.username, serverData: typeof userData.serverData === 'undefined' ? null : userData.serverData, clientData: typeof userData.clientData === 'undefined' ? null : userData.clientData, } } if (this.settings.reportInvalidParameters) { return { isValid: false } } return null } /** * Called initially to validate the user provided settings */ private validateSettings (settings: FileAuthConfig) { try { if (settings.hash) { validateMap(settings, true, { hash: 'string', iterations: 'number', keyLength: 'number', }) validateHashingAlgorithm(settings.hash) } } catch (e) { this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'Validating settings failed for file auth', { error: e }) } if (Object.keys(settings.users).length === 0) { this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'no users present in user file') return } for (const username in this.settings.users) { if (typeof settings.users[username].password !== 'string') { this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, `missing password for ${username}`) } } } } ================================================ FILE: src/services/authentication/http/http-authentication.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import TestHttpServer from '../../../test/helper/test-http-server' import MockLogger from '../../../test/mock/logger-mock' import { PromiseDelay } from '../../../utils/utils'; import * as testHelper from '../../../test/helper/test-helper' import { HttpAuthentication } from './http-authentication'; import { EVENT } from '@deepstream/types' describe('it forwards authentication attempts as http post requests to a specified endpoint', () => { let authenticationHandler let server const port = TestHttpServer.getRandomPort() const { config, services} = testHelper.getDeepstreamOptions() let logSpy before((done) => { server = new TestHttpServer(port, done) logSpy = (services.logger as MockLogger).logSpy logSpy.resetHistory() }) after((done) => { server.close(done) }) before (() => { const endpointUrl = `http://localhost:${port}` authenticationHandler = new HttpAuthentication({ endpointUrl, permittedStatusCodes: [200], requestTimeout: 60, promoteToHeader: ['token'], retryAttempts: 2, retryInterval: 30, retryStatusCodes: [404, 504] }, services, config) expect(authenticationHandler.description).to.equal(`http webhook to ${endpointUrl}`) }) it('issues a request when isValidUser is called and receives 200 in return', async () => { const connectionData = { connection: 'data' } const authData = { username: 'userA' } server.once('request-received', () => { expect(server.lastRequestData).to.deep.equal({ connectionData: { connection: 'data' }, authData: { username: 'userA' } }) expect(server.lastRequestMethod).to.equal('POST') expect(server.lastRequestHeaders['content-type']).to.contain('application/json') server.respondWith(200, { serverData: { extra: 'data' }, clientData: { color: 'red' }, id: "123" }) }) const result = await authenticationHandler.isValidUser(connectionData, authData) expect(result.isValid).to.equal(true) expect(result.id).to.equal("123") expect(result.serverData).to.deep.equal({ extra: 'data' }) expect(result.clientData).to.deep.equal({ color: 'red' }) }) it('issues a request when isValidUser is called and receives 401 (denied) in return', async () => { const connectionData = { connection: 'data' } const authData = { username: 'userA' } server.once('request-received', () => { expect(server.lastRequestData).to.deep.equal({ connectionData: { connection: 'data' }, authData: { username: 'userA' } }) expect(server.lastRequestMethod).to.equal('POST') expect(server.lastRequestHeaders['content-type']).to.contain('application/json') server.respondWith(401) }) const result = await authenticationHandler.isValidUser(connectionData, authData) expect(result.isValid).to.equal(false) expect(result.serverData).to.equal(undefined) expect(result.clientData).to.equal(undefined) expect(logSpy).to.have.callCount(0) }) it('receives a positive response without data', async () => { const connectionData = { connection: 'data' } const authData = { username: 'userA' } server.once('request-received', () => { expect(server.lastRequestData).to.deep.equal({ connectionData: { connection: 'data' }, authData: { username: 'userA' } }) expect(server.lastRequestMethod).to.equal('POST') expect(server.lastRequestHeaders['content-type']).to.contain('application/json') server.respondWith(200, '') }) const result = await authenticationHandler.isValidUser(connectionData, authData) expect(result.isValid).to.equal(true) expect(result.serverData).to.equal(undefined) expect(result.clientData).to.equal(undefined) }) it('receives a positive response with only a string', async () => { const connectionData = { connection: 'data' } const authData = { username: 'userA' } server.once('request-received', () => { expect(server.lastRequestData).to.deep.equal({ connectionData: { connection: 'data' }, authData: { username: 'userA' } }) expect(server.lastRequestMethod).to.equal('POST') expect(server.lastRequestHeaders['content-type']).to.contain('application/json') server.respondWith(200, 'userA') }) const result = await authenticationHandler.isValidUser(connectionData, authData) expect(result.isValid).to.equal(true) expect(result.id).to.deep.equal('userA') }) it('receives a server error as response', async () => { const connectionData = { connection: 'data' } const authData = { username: 'userA' } server.once('request-received', () => { server.respondWith(500, 'oh dear') }) const result = await authenticationHandler.isValidUser(connectionData, authData) expect(result.isValid).to.equal(false) expect(logSpy).to.have.been.calledWith(2, EVENT.AUTH_ERROR, 'http auth server error: "oh dear"') expect(result.clientData).to.deep.equal({ error: 'oh dear' }) }) it('promotes headers from body if provides', async () => { const connectionData = { connection: 'data' } const authData = { token: 'a-token' } server.once('request-received', () => { server.respondWith(200, {}) }) const result = await authenticationHandler.isValidUser(connectionData, authData) expect(result.isValid).to.equal(true) expect(result.clientData).to.equal(undefined) expect(result.serverData).to.equal(undefined) expect(server.getRequestHeader('token')).to.equal('a-token') }) describe('retries', () => { const connectionData = { connection: 'data' } const authData = { token: 'a-token' } beforeEach(() => { server.once('request-received', () => server.respondWith(404, {})) }) it ('doesn\'t fail if the response returned is retry code', async () => { let called = false authenticationHandler.isValidUser(connectionData, authData, (result, data) => { called = true }) await PromiseDelay(20) expect(called).to.equal(false) }) it.skip ('returns true if the second attempt is valid', async () => { let done const result = new Promise((resolve) => done = resolve) authenticationHandler.isValidUser(connectionData, authData, (result, data) => { expect(result).to.equal(true) expect(data).to.deep.equal({ what: '2nd-attempt' }) done() }) await PromiseDelay(30) server.once('request-received', () => server.respondWith(200, { what: '2nd-attempt' })) await result }) // TODO: Always passing it ('returns invalid if retry attempts are exceeded', async () => { const isValidUser = authenticationHandler.isValidUser(connectionData, authData) await PromiseDelay(30) server.once('request-received', () => server.respondWith(404, {})) await PromiseDelay(30) server.once('request-received', () => server.respondWith(504, {})) const result = await isValidUser expect(result.isValid).to.equal(false) expect(result.clientData).to.deep.equal({ error: EVENT.AUTH_RETRY_ATTEMPTS_EXCEEDED }) }) }) it('times out', async () => { const connectionData = { connection: 'data' } const authData = { username: 'userA' } const response = await authenticationHandler.isValidUser(connectionData, authData) expect(response.isValid).to.equal(false) expect(logSpy).to.have.been.calledWith(2, EVENT.AUTH_ERROR, 'http auth error: Error: socket hang up') expect(response.clientData).to.deep.equal({ error: EVENT.AUTH_RETRY_ATTEMPTS_EXCEEDED }) server.respondWith(200) }) }) ================================================ FILE: src/services/authentication/http/http-authentication.ts ================================================ import { post } from 'needle' import { EVENT, DeepstreamPlugin, DeepstreamServices, DeepstreamConfig, DeepstreamAuthentication, DeepstreamAuthenticationResult } from '@deepstream/types' import { JSONObject } from '../../../constants' import { validateMap } from '../../../utils/utils' interface HttpAuthenticationHandlerSettings { // http(s) endpoint that will receive post requests endpointUrl: string // an array of http status codes that qualify as permitted permittedStatusCodes: number[] // time in milliseconds before the request times out if no reply is received requestTimeout: number // fields to copy from authData to header, useful for when endpoints authenticate using middleware promoteToHeader: string[], // any array of status codes that should be retries, useful if the server is down during a deploy // or generally unresponsive retryStatusCodes: number[], // the maximum amount of retries before returning a false login retryAttempts: number, // the time in milliseconds between retries retryInterval: number, // fail authentication process if invalid login parameters are used reportInvalidParameters: boolean } export class HttpAuthentication extends DeepstreamPlugin implements DeepstreamAuthentication { public description: string = `http webhook to ${this.settings.endpointUrl}` private retryAttempts = new Map void, attempts: number } >() private requestId = 0 constructor (private settings: HttpAuthenticationHandlerSettings, private services: DeepstreamServices, config: DeepstreamConfig) { super() this.validateSettings() if (this.settings.promoteToHeader === undefined) { this.settings.promoteToHeader = [] } if (this.settings.reportInvalidParameters === undefined) { this.settings.reportInvalidParameters = true } } public async isValidUser (connectionData: JSONObject, authData: JSONObject) { return new Promise((resolve: (result: DeepstreamAuthenticationResult | null) => void) => { this.validate(this.requestId++, connectionData, authData, resolve) }) } private validate (id: number, connectionData: JSONObject, authData: JSONObject, callback: (result: DeepstreamAuthenticationResult | null) => void): void { const options = { read_timeout: this.settings.requestTimeout, open_timeout: this.settings.requestTimeout, response_timeout: this.settings.requestTimeout, follow_max: 2, json: true, headers: {} } if (this.settings.promoteToHeader.length > 0) { options.headers = this.settings.promoteToHeader.reduce( (result, property) => { if (authData[property]) { result[property] = authData[property] } return result }, {} as JSONObject ) } post(this.settings.endpointUrl, { connectionData, authData }, options, (error, response) => { if (error) { this.services.logger.warn(EVENT.AUTH_ERROR, `http auth error: ${error}`) this.retry(id, connectionData, authData, callback) return } if (!response.statusCode) { this.services.logger.warn(EVENT.AUTH_ERROR, 'http auth server error: missing status code!') this.retryAttempts.delete(id) if (this.settings.reportInvalidParameters) { callback({ isValid: false }) } else { callback(null) } return } if (response.statusCode >= 500 && response.statusCode < 600) { this.services.logger.warn(EVENT.AUTH_ERROR, `http auth server error: ${JSON.stringify(response.body)}`) } if (this.settings.retryStatusCodes.includes(response.statusCode)) { this.retry(id, connectionData, authData, callback) return } this.retryAttempts.delete(id) if (this.settings.permittedStatusCodes.indexOf(response.statusCode) === -1) { if (this.settings.reportInvalidParameters) { if (response.body) { if (typeof response.body === 'string') { callback({ isValid: false, clientData: { error: response.body }}) } else if (typeof response.body === 'object' && Object.keys(response.body).length > 0) { callback({ isValid: false, clientData: {...response.body} }) } else { callback({ isValid: false }) } } else { callback({ isValid: false }) } } else { callback(null) } return } if (response.body && typeof response.body === 'string') { callback({ isValid: true, id: response.body }) return } callback({ isValid: true, ...response.body }) }) } private retry (id: number, connectionData: JSONObject, authData: JSONObject, callback: (result: DeepstreamAuthenticationResult | null) => void) { let retryAttempt = this.retryAttempts.get(id) if (!retryAttempt) { retryAttempt = { connectionData, authData, callback, attempts: 0 } this.retryAttempts.set(id, retryAttempt) } else { retryAttempt.attempts++ } if (retryAttempt.attempts < this.settings.retryAttempts) { setTimeout(() => this.validate(id, connectionData, authData, callback), this.settings.retryInterval) } else { this.retryAttempts.delete(id) if (this.settings.reportInvalidParameters) { callback({ isValid: false, clientData: { error: EVENT.AUTH_RETRY_ATTEMPTS_EXCEEDED } }) } else { this.services.logger.warn(EVENT.AUTH_ERROR, EVENT.AUTH_RETRY_ATTEMPTS_EXCEEDED) callback(null) } } } private validateSettings (): void { validateMap(this.settings, true, { endpointUrl: 'url', permittedStatusCodes: 'array', requestTimeout: 'number', retryStatusCodes: 'array', retryAttempts: 'number', retryInterval: 'number' }) } } ================================================ FILE: src/services/authentication/open/open-authentication.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import { spy } from 'sinon' import { OpenAuthentication } from './open-authentication' describe('open authentication handler', () => { let authenticationHandler it('creates the handler', () => { authenticationHandler = new OpenAuthentication() expect(typeof authenticationHandler.isValidUser).to.equal('function') expect(authenticationHandler.description).to.equal('Open Authentication') }) it('permissions users without auth data', async () => { const result = await authenticationHandler.isValidUser(null, {}) expect(result.isValid).to.equal(true) expect(result.id).to.equal('open') }) it('permissions users with a username', async () => { const result = await authenticationHandler.isValidUser(null, { username: 'Wolfram' }) expect(result.isValid).to.equal(true) expect(result.id).to.equal('Wolfram') }) }) ================================================ FILE: src/services/authentication/open/open-authentication.ts ================================================ import { DeepstreamPlugin, DeepstreamAuthentication } from '@deepstream/types' import { JSONObject } from '../../../constants' /** * Used for users that don't provide a username */ const OPEN: string = 'open' /** * The open authentication handler allows every client to connect. * If the client specifies a username as part of its authentication * data, it will be used to identify the user internally */ export class OpenAuthentication extends DeepstreamPlugin implements DeepstreamAuthentication { public description: string = 'Open Authentication' /** * Grants access to any user. Registers them with username or open */ public async isValidUser (connectionData: JSONObject, authData: JSONObject) { return { isValid: true, id: (authData.username && authData.username.toString()) || OPEN } } } ================================================ FILE: src/services/authentication/storage/storage-based-authentication.ts ================================================ import { DeepstreamPlugin, DeepstreamAuthentication, DeepstreamServices, EVENT, DeepstreamAuthenticationResult } from '@deepstream/types' import { v4 as uuid } from 'uuid' import { Dictionary } from 'ts-essentials' import { createHash } from '../../../utils/utils' const STRING = 'string' interface StorageAuthConfig { // fail authentication process if invalid login parameters are used reportInvalidParameters: boolean, // the table to store and lookup the users in table: string, // upsert the user if it doesn't exist in db createUser: boolean, // the name of a HMAC digest algorithm, a.g. 'sha512' hash: string // the amount of times the algorithm should be applied iterations: number // the length of the resulting key keyLength: number } type UserData = DeepstreamAuthenticationResult & { password: string, clientData: { [index: string]: any, id: string }, serverData: Dictionary } export class StorageBasedAuthentication extends DeepstreamPlugin implements DeepstreamAuthentication { public description: string = `Storage using table: ${this.settings.table}` private logger = this.services.logger.getNameSpace('STORAGE_AUTH') private hashSettings = { iterations: this.settings.iterations, keyLength: this.settings.keyLength, algorithm: this.settings.hash } private base64KeyLength = 4 * Math.ceil(this.settings.keyLength / 3) /** * Creates the class, reads and validates the users.json file */ constructor (private settings: StorageAuthConfig, private services: DeepstreamServices) { super() if (this.settings.reportInvalidParameters === undefined) { this.settings.reportInvalidParameters = true } } public async whenReady (): Promise { await this.services.storage.whenReady() } /** * Main interface. Authenticates incoming connections */ public async isValidUser (connectionData: any, authData: any): Promise { const missingUsername = typeof authData.username !== STRING const missingPassword = typeof authData.password !== STRING if (missingPassword || missingUsername) { if (this.settings.reportInvalidParameters) { return { isValid: false, clientData: { error: `missing authentication parameters: ${missingUsername && 'username'} ${missingPassword && 'password'}` } } } else { return null } } let userData: UserData const storageId = `${this.settings.table}/${authData.username}` try { userData = await new Promise((resolve, reject) => this.services.storage.get(storageId, (err, version, data) => err ? reject(err) : resolve(data))) } catch (err) { this.logger.error(EVENT.ERROR, `Error retrieving user from storage ${JSON.stringify(err)}`) return { isValid: false, clientData: { error: 'Error retrieving user from storage' } } } if (userData === null) { if (this.settings.createUser) { this.logger.info(EVENT.REGISTERING_USER, `Adding new user ${authData.username}`) const { hash, salt } = await createHash(authData.password, this.hashSettings) const clientData = { id: uuid(), } const serverData = { created: Date.now() } return await new Promise((resolve, reject) => this.services.storage.set(storageId, 1, { username: authData.username, password: hash.toString('base64') + salt, clientData, serverData }, (err) => { if (err) { this.logger.error(EVENT.ERROR, `Error creating user ${JSON.stringify(err)}`) return resolve({ isValid: false, clientData: { error: 'Error creating user' } }) } resolve({ isValid: true, id: clientData.id, clientData, serverData }) } )) } return null } const expectedHash = userData.password.substr(0, this.base64KeyLength) const { hash: actualHash } = await createHash(authData.password, this.hashSettings, userData.password.substr(this.base64KeyLength)) if (expectedHash === actualHash.toString('base64')) { return { isValid: true, id: userData.clientData.id, serverData: userData.serverData || null, clientData: userData.clientData || null, } } if (this.settings.reportInvalidParameters) { return { isValid: false } } else { return null } } } ================================================ FILE: src/services/cache/local-cache.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import {spy} from 'sinon' import { LocalCache } from './local-cache' describe('it saves values in memory', () => { let localCache before(() => { localCache = new LocalCache() }) it('has created the Local Cache', async () => { await localCache.whenReady() }) it('sets a value in the cache', (done) => { const successCallback = spy() localCache.set('firstname', 1, 'Wolfram', successCallback) setTimeout(() => { expect(successCallback).to.have.callCount(1) expect(successCallback).to.have.been.calledWith(null) done() }, 1) }) it('retrieves an existing value from the cache', (done) => { const successCallback = spy() localCache.get('firstname', successCallback) setTimeout(() => { expect(successCallback).to.have.callCount(1) expect(successCallback).to.have.been.calledWith(null, 1, 'Wolfram') done() }, 1) }) it('deletes a value from the cache', (done) => { const successCallback = spy() localCache.delete('firstname', successCallback) setTimeout(() => { expect(successCallback).to.have.callCount(1) expect(successCallback).to.have.been.calledWith(null) done() }, 1) }) it('tries to retrieve a non-existing value from the cache', (done) => { const successCallback = spy() localCache.get('firstname', successCallback) setTimeout(() => { expect(successCallback).to.have.callCount(1) expect(successCallback).to.have.been.calledWith(null, -1, null) done() }, 1) }) }) ================================================ FILE: src/services/cache/local-cache.ts ================================================ import { DeepstreamPlugin, DeepstreamCache, StorageWriteCallback, StorageReadCallback, StorageHeadBulkCallback, StorageHeadCallback } from '@deepstream/types' import { JSONValue } from '../../constants' export class LocalCache extends DeepstreamPlugin implements DeepstreamCache { public description = 'Local Cache' private data = new Map() public head (recordName: string, callback: StorageHeadCallback) { const data = this.data.get(recordName) process.nextTick(() => callback(null, data ? data.version : -1)) } public headBulk (recordNames: string[], callback: StorageHeadBulkCallback) { const versions: any = {} const missing: any = [] for (const name of recordNames) { const data = this.data.get(name) if (data) { versions[name] = data.version } else { missing.push(name) } } process.nextTick(() => callback(null, versions, missing)) } public set (key: string, version: number, data: any, callback: StorageWriteCallback) { this.data.set(key, { version, data }) process.nextTick(() => callback(null)) } public get (key: string, callback: StorageReadCallback) { const data = this.data.get(key) if (!data) { process.nextTick(() => callback(null, -1, null)) } else { process.nextTick(() => callback(null, data.version, data.data)) } } public delete (key: string, callback: StorageWriteCallback) { this.data.delete(key) process.nextTick(() => callback(null)) } public deleteBulk (keys: string[], callback: StorageWriteCallback) { keys.forEach((key) => this.data.delete(key)) process.nextTick(() => callback(null)) } } export default LocalCache ================================================ FILE: src/services/cluster-node/single-cluster-node.ts ================================================ import { TOPIC, Message } from '../../constants' import { DeepstreamClusterNode, DeepstreamPlugin } from '@deepstream/types' export class SingleClusterNode extends DeepstreamPlugin implements DeepstreamClusterNode { public description = 'Single Cluster Node' public sendDirect (serverName: string, message: Message, metaData?: any) {} public send (message: Message, metaData?: any) {} public subscribe (stateRegistryTopic: TOPIC, callback: Function) {} public async close (): Promise {} } ================================================ FILE: src/services/cluster-node/vertical-cluster-node.ts ================================================ import { Message, TOPIC } from '../../constants' import * as cluster from 'cluster' import { EventEmitter } from 'events' import { DeepstreamClusterNode, DeepstreamPlugin, DeepstreamServices, DeepstreamConfig, EVENT } from '@deepstream/types' if (cluster.isMaster) { cluster.on('message', (worker, serializedMessage: string, handle) => { for (const id in cluster.workers) { const toWorker = cluster.workers[id]! if (toWorker !== worker) { toWorker.send(serializedMessage) } } }) } export class VerticalClusterNode extends DeepstreamPlugin implements DeepstreamClusterNode { public description: string = 'Vertical Cluster Message Bus' private isReady: boolean = false public static emitter = new EventEmitter() private callbacks = new Map() constructor (pluginConfig: any, private services: DeepstreamServices, private config: DeepstreamConfig) { super() } public init () { if (cluster.isWorker) { process.on('message', (serializedMessage) => { if (this.isReady) { const { fromServer, message } = JSON.parse(serializedMessage) VerticalClusterNode.emitter.emit(TOPIC[message.topic], message, fromServer) const callbacks = this.callbacks.get(TOPIC[message.topic]) if (!callbacks || callbacks.size === 0) { this.services.logger.warn(EVENT.UNKNOWN_ACTION, `Received message for unknown topic ${TOPIC[message.topic]}`) return } callbacks.forEach((callback: Function) => callback(message, fromServer)) } }) } } async whenReady (): Promise { this.isReady = true } public send (message: Message, metaData?: any): void { process.send!(JSON.stringify({ message, fromServer: this.config.serverName })) } public sendDirect (serverName: string, message: Message, metaData?: any): void { process.send!(JSON.stringify({ toServer: serverName, fromServer: this.config.serverName, message })) } public subscribe (stateRegistryTopic: TOPIC, callback: (message: SpecificMessage, originServerName: string) => void): void { VerticalClusterNode.emitter.on(TOPIC[stateRegistryTopic], callback) let callbacks = this.callbacks.get(TOPIC[stateRegistryTopic]) if (!callbacks) { callbacks = new Set() this.callbacks.set(TOPIC[stateRegistryTopic], callbacks) } callbacks.add(callback) } public async close (): Promise { for (const [topic, callbacks] of this.callbacks) { for (const callback of callbacks) { VerticalClusterNode.emitter.off(topic, callback) } } this.callbacks.clear() this.isReady = false } } ================================================ FILE: src/services/cluster-registry/distributed-cluster-registry.ts ================================================ import { DeepstreamServices, DeepstreamConfig, ClusterRegistry, DeepstreamPlugin, EVENT } from '@deepstream/types' import { TOPIC, ClusterMessage, CLUSTER_ACTION } from '../../constants' import { EventEmitter } from 'events' /** * This class maintains a list of all nodes that are * currently present within the cluster. * * It provides status messages on a predefined interval * and keeps track of incoming status messages. */ export class DistributedClusterRegistry extends DeepstreamPlugin implements ClusterRegistry { public description: string = 'Distributed Cluster Registry' private inCluster: boolean = false private nodes = new Map() private leaderScore = Math.random() private publishInterval!: NodeJS.Timeout private checkInterval!: NodeJS.Timeout private role: string private emitter = new EventEmitter() /** * Creates the class, initialises all intervals and publishes the * initial status message that notifies other nodes within this * cluster of its presence. */ constructor (private pluginOptions: any, private services: Readonly, private config: Readonly) { super() this.role = this.pluginOptions.role || 'deepstream' } public init () { this.services.clusterNode.subscribe(TOPIC.CLUSTER, this.onMessage.bind(this)) this.leaveCluster = this.leaveCluster.bind(this) this.publishStatus() this.publishInterval = setInterval( this.publishStatus.bind(this), this.pluginOptions.keepAliveInterval ) this.checkInterval = setInterval( this.checkNodes.bind(this), this.pluginOptions.activeCheckInterval ) } public async close (): Promise { return new Promise((resolve) => { this.emitter.once('close', resolve) this.leaveCluster() }) } public onServerAdded (callback: (serverName: string) => void) { this.emitter.on('server-added', callback) } public onServerRemoved (callback: (serverName: string) => void) { this.emitter.on('server-removed', callback) } /** * Returns the serverNames of all nodes currently present within the cluster */ public getAll (): string[] { return [...this.nodes.keys()] } /** * Returns true if this node is the cluster leader */ public isLeader (): boolean { return this.config.serverName === this.getLeader() } /** * Returns the name of the current leader */ public getLeader () { let maxScore = 0 let leader = this.config.serverName for (const [serverName, node] of this.nodes) { if (node.leaderScore > maxScore) { maxScore = node.leaderScore leader = serverName } } return leader } /** * Distributes incoming messages on the cluster topic */ private onMessage (message: ClusterMessage) { if (message.action === CLUSTER_ACTION.STATUS) { this.updateNode(message) return } if (message.action === CLUSTER_ACTION.REMOVE) { this.removeNode(message.serverName) return } this.services.logger.error(EVENT.UNKNOWN_ACTION, `TOPIC: ${TOPIC[TOPIC.CLUSTER]} ${message.action}`) } /** * Called on an interval defined by clusterActiveCheckInterval to check if all nodes * within the cluster are still alive. * * Being alive is defined as having received a status message from that node less than * milliseconds ago. */ private checkNodes () { // Never remove a single node instance if (this.nodes.size === 1) { return } const now = Date.now() for (const [serverName, node] of this.nodes) { if (now - node.lastStatusTime > this.pluginOptions.nodeInactiveTimeout) { this.removeNode(serverName) } } } /** * Updates the status of a node with incoming status data and resets its lastStatusTime. * * If the remote node doesn't exist yet, it is added and an add event is emitted / logged */ private updateNode (message: ClusterMessage) { const node = this.nodes.get(message.serverName) this.nodes.set(message.serverName, { lastStatusTime: Date.now(), leaderScore: message.leaderScore! }) if (node) { return } this.services.logger.info(EVENT.CLUSTER_JOIN, message.serverName) this.services.logger.info(EVENT.CLUSTER_SIZE, `The cluster size is now ${this.nodes.size}`) this.emitter.emit('server-added', message.serverName) } /** * Removes a remote node from this registry if it exists. * Logs/emits remove */ private removeNode (serverName: string) { const deleted = this.nodes.delete(serverName) if (deleted) { this.services.logger.info(EVENT.CLUSTER_LEAVE, serverName) this.services.logger.info(EVENT.CLUSTER_SIZE, `The cluster size is now ${this.nodes.size}`) this.emitter.emit('server-removed', serverName) } } /** * Publishes this node's status on the message bus */ private publishStatus (): void { this.inCluster = true const message = { topic: TOPIC.CLUSTER, action: CLUSTER_ACTION.STATUS, serverName: this.config.serverName, leaderScore: this.leaderScore, role: this.role } as ClusterMessage this.updateNode(message) this.services.clusterNode.send(message) } /** * Prompts this node to leave the cluster, either as a result of a server.close() * call or due to the process exiting. * This sends out a leave message to all other nodes and destroys this class. */ private leaveCluster () { if (this.inCluster === false) { this.emitter.emit('close') return } this.services.logger.info(EVENT.CLUSTER_LEAVE, this.config.serverName) this.services.clusterNode.send({ topic: TOPIC.CLUSTER, action: CLUSTER_ACTION.REMOVE, name: this.config.serverName }) // TODO: If a message connector doesn't close this is required to avoid an error // being thrown during shutdown // this._options.messageConnector.unsubscribe( C.TOPIC.CLUSTER, this._onMessageFn ); process.removeListener('beforeExit', this.leaveCluster) process.removeListener('exit', this.leaveCluster) clearInterval(this.publishInterval) clearInterval(this.checkInterval) this.nodes.clear() this.inCluster = false this.emitter.emit('close') } } ================================================ FILE: src/services/cluster-registry/distributed-state-registry-factory.ts ================================================ import { DeepstreamPlugin, DeepstreamServices, DeepstreamConfig, StateRegistryFactory, StateRegistry } from '@deepstream/types' import { TOPIC } from '../../constants' import { DistributedStateRegistry, DistributedStateRegistryOptions } from '../cluster-state/distributed-state-registry' export class DistributedStateRegistryFactory extends DeepstreamPlugin implements StateRegistryFactory { public description: string = 'Distributed State Registry' private stateRegistries = new Map() constructor (private pluginConfig: DistributedStateRegistryOptions, private services: Readonly, private config: Readonly) { super() } public getStateRegistry = (topic: TOPIC) => { let stateRegistry = this.stateRegistries.get(topic) if (!stateRegistry) { stateRegistry = new DistributedStateRegistry(topic, this.pluginConfig, this.services, this.config) this.stateRegistries.set(topic, stateRegistry) } return stateRegistry } public getStateRegistries (): Map { return this.stateRegistries } } ================================================ FILE: src/services/cluster-state/distributed-state-registry-factory.ts ================================================ import { DeepstreamPlugin, DeepstreamServices, DeepstreamConfig, StateRegistryFactory, StateRegistry } from '@deepstream/types' import { TOPIC } from '../../constants' import { DistributedStateRegistry, DistributedStateRegistryOptions } from './distributed-state-registry' export class DistributedStateRegistryFactory extends DeepstreamPlugin implements StateRegistryFactory { public description: string = 'Distributed State Registry' private stateRegistries = new Map() constructor (private pluginConfig: DistributedStateRegistryOptions, private services: Readonly, private config: Readonly) { super() } public getStateRegistry = (topic: TOPIC) => { let stateRegistry = this.stateRegistries.get(topic) if (!stateRegistry) { stateRegistry = new DistributedStateRegistry(topic, this.pluginConfig, this.services, this.config) this.stateRegistries.set(topic, stateRegistry) } return stateRegistry } public getStateRegistries (): Map { return this.stateRegistries } } ================================================ FILE: src/services/cluster-state/distributed-state-registry.ts ================================================ import { TOPIC, STATE_ACTION, StateMessage } from '../../constants' import { DeepstreamServices, StateRegistry, StateRegistryCallback, DeepstreamConfig, EVENT } from '@deepstream/types' import { Dictionary } from 'ts-essentials' import { EventEmitter } from 'events' export type DistributedStateRegistryOptions = any /** * This class provides a generic mechanism that allows to maintain * a distributed state amongst the nodes of a cluster. The state is an * array of unique strings in arbitrary order. * * Whenever a string is added by any node within the cluster for the first time, * an 'add' event is emitted. Whenever its removed by the last node within the cluster, * a 'remove' event is emitted. */ export class DistributedStateRegistry implements StateRegistry { private isReady: boolean = false private data = new Map, checkSum: number }>() private reconciliationTimeouts = new Map() private checkSumTimeouts = new Map() private fullStateSent: boolean = false private initialServers = new Set() private emitter = new EventEmitter() private logger = this.services.logger.getNameSpace('DISTRIBUTED_STATE_REGISTRY') /** * Initializes the DistributedStateRegistry and subscribes to the provided cluster topic */ constructor (private topic: TOPIC, private stateOptions: any, private services: Readonly, private config: Readonly) { this.resetFullStateSent = this.resetFullStateSent.bind(this) this.services.clusterNode.subscribe(TOPIC.STATE_REGISTRY, this.processIncomingMessage.bind(this)) const serverNames = this.services.clusterRegistry.getAll() this.initialServers = new Set(serverNames) if (this.initialServers.size === 0) { this.isReady = true this.emitter.emit('ready') } this.initialServers.forEach((serverName) => { if (serverName !== this.config.serverName) { this.onServerAdded(serverName) } }) this.services.clusterRegistry.onServerAdded(this.onServerAdded.bind(this)) this.services.clusterRegistry.onServerRemoved(this.onServerRemoved.bind(this)) } public async whenReady () { if (!this.isReady) { await new Promise((resolve) => this.emitter.once('ready', resolve)) } } public onAdd (callback: StateRegistryCallback): void { this.emitter.on('add', callback) } public onRemove (callback: StateRegistryCallback): void { this.emitter.on('remove', callback) } /** * Checks if a given entry exists within the registry */ public has (name: string) { return this.data.has(name) } /** * Add a name/entry to the registry. If the entry doesn't exist yet, * this will notify the other nodes within the cluster */ public add (name: string) { const data = this.data.get(name) if (!data || !data.nodes.has(this.config.serverName)) { this.addToServer(name, this.config.serverName) this.sendMessage(name, STATE_ACTION.ADD) } else { data.localCount++ } } /** * Removes a name/entry from the registry. If the entry doesn't exist, * this will exit silently */ public remove (name: string) { const data = this.data.get(name) if (data) { data.localCount-- if (data.localCount === 0) { this.removeFromServer(name, this.config.serverName) this.sendMessage(name, STATE_ACTION.REMOVE) } } } public removeAll (serverName: string): void { throw new Error('Method not implemented.') } /** * Informs the distributed state registry a server has been added to the cluster */ public onServerAdded (serverName: string) { this._requestFullState(serverName) } /** * Removes all entries for a given serverName. This is intended to be called * whenever a node is removed from the cluster */ public onServerRemoved (serverName: string) { for (const [name, value] of this.data) { if (value.nodes.has(serverName)) { this.removeFromServer(name, serverName) } } } /** * Returns all the servers that hold a given state */ public getAllServers (name: string) { const data = this.data.get(name) if (data) { return [...data.nodes.keys()] } return [] } /** * Returns all currently registered entries */ public getAll (serverName: string): string[] { if (!serverName) { return [...this.data.keys()] } const entries: string[] = [] for (const [name, value] of this.data) { if (value.nodes.has(serverName)) { entries.push(name) } } return entries } /** * Removes an entry for a given serverName. If the serverName * was the last node that held the entry, the entire entry will * be removed and a `remove` event will be emitted */ private removeFromServer (name: string, serverName: string) { const data = this.data.get(name) if (!data) { return } data.nodes.delete(serverName) const exists = data.nodes.size !== 0 if (exists === false) { this.data.delete(name) this.emitter.emit('remove', name) } this.emitter.emit('server-removed', name, serverName) } /** * Adds a new entry to this registry, either as a result of a remote or * a local addition. Will emit an `add` event if the entry wasn't present before */ private addToServer (name: string, serverName: string) { let data = this.data.get(name) if (!data) { data = { localCount: 1, nodes: new Set(), checkSum: this.createCheckSum(name) } this.data.set(name, data) this.emitter.emit('add', name) } data.nodes.add(serverName) this.emitter.emit('server-added', name, serverName) } /** * Generic messaging function for add and remove messages */ private sendMessage (name: string, action: STATE_ACTION) { this.services.clusterNode.send({ topic: TOPIC.STATE_REGISTRY, registryTopic: this.topic, action, name }) this.getCheckSumTotal(this.config.serverName, (checksum) => this.services.clusterNode.send({ topic: TOPIC.STATE_REGISTRY, registryTopic: this.topic, action: STATE_ACTION.CHECKSUM, checksum }) ) } /** * This method calculates the total checkSum for all local entries of * a given serverName */ private getCheckSumTotal (serverName: string, callback: (checksum: number) => void): void { const callbacks = this.checkSumTimeouts.get(serverName) if (callbacks) { callbacks.push(callback) } else { this.checkSumTimeouts.set(serverName, [callback]) setTimeout(() => { let totalCheckSum = 0 for (const [, value] of this.data) { if (value.nodes.has(serverName)) { totalCheckSum += value.checkSum } } this.checkSumTimeouts.get(serverName)!.forEach((cb: (checksum: number) => void) => cb(totalCheckSum)) this.checkSumTimeouts.delete(serverName) }, this.stateOptions.checkSumBuffer) } } /** * Calculates a simple checkSum for a given name. This is done up-front and cached * to increase performance for local add and remove operations. Arguably this is a generic * method and might be moved to the utils class if we find another usecase for it. */ private createCheckSum (name: string) { let checkSum = 0 let i for (i = 0; i < name.length; i++) { // tslint:disable-next-line:no-bitwise checkSum = ((checkSum << 5) - checkSum) + name.charCodeAt(i) // eslint-disable-line } return checkSum } /** * Checks a remote checkSum for a given serverName against the * actual checksum for all local entries for the given name. * * - If the checksums match, it removes all possibly pending * reconciliationTimeouts * * - If the checksums don't match, it schedules a reconciliation request. If * another message from the remote server arrives before the reconciliation request * is send, it will be cancelled. */ private verifyCheckSum (serverName: string, remoteCheckSum: number) { this.getCheckSumTotal(serverName, (checksum: number) => { if (checksum !== remoteCheckSum) { this.reconciliationTimeouts.set(serverName, setTimeout( this._requestFullState.bind(this, serverName), this.stateOptions.stateReconciliationTimeout )) return } const timeout = this.reconciliationTimeouts.get(serverName) if (timeout) { clearTimeout(timeout) this.reconciliationTimeouts.delete(serverName) } }) } /** * Sends a reconciliation request for a server with a given name (technically, its send to * every node within the cluster, but will be ignored by all but the one with a matching name) * * The matching node will respond with a DISTRIBUTED_STATE_FULL_STATE message */ private _requestFullState (serverName: string) { this.services.clusterNode.sendDirect(serverName, { topic: TOPIC.STATE_REGISTRY, registryTopic: this.topic, action: STATE_ACTION.REQUEST_FULL_STATE, }) } /** * Creates a full state message containing an array of all local entries that * will be used to reconcile compromised states as well as provide the full state * for new nodes that joined the cluster * * When a state gets compromised, more than one remote registry might request a full state update. * This method will schedule a timeout in which no additional full state messages are sent to * make sure only a single full state message is sent in reply. */ public sendFullState (serverName: string): void { const localState: string[] = [] for (const [name, value] of this.data) { if (value.nodes.has(this.config.serverName)) { localState.push(name) } } this.services.clusterNode.sendDirect(serverName, { topic: TOPIC.STATE_REGISTRY, registryTopic: this.topic, action: STATE_ACTION.FULL_STATE, fullState: localState }) this.fullStateSent = true setTimeout(this.resetFullStateSent, this.stateOptions.stateReconciliationTimeout) } /** * This will apply the data from an incoming full state message. Entries that are not within * the incoming array will be removed for that node from the local registry and new entries will * be added. */ private applyFullState (serverName: string, names: string[]) { const namesMap: Dictionary = {} for (let i = 0; i < names.length; i++) { namesMap[names[i]] = true } Object.keys(this.data).forEach((name) => { // please note: only checking if the name exists is sufficient as the registry will just // set node[serverName] to false if the entry exists, but not for the remote server. if (!namesMap[name]) { this.removeFromServer(name, serverName) } }) names.forEach((name) => this.addToServer(name, serverName)) this.initialServers.delete(serverName) if (this.initialServers.size === 0) { this.isReady = true this.emitter.emit('ready') } } /** * Will be called after a full state message has been sent and * stateReconciliationTimeout has passed. This will allow further reconciliation * messages to be sent again. */ private resetFullStateSent (): void { this.fullStateSent = false } /** * This is the main routing point for messages coming in from * the message connector. */ private processIncomingMessage (message: StateMessage, serverName: string): void { if (message.registryTopic !== this.topic) { return } if (message.action === STATE_ACTION.ADD) { this.addToServer(message.name!, serverName) return } if (message.action === STATE_ACTION.REMOVE) { this.removeFromServer(message.name!, serverName) return } if (message.action === STATE_ACTION.REQUEST_FULL_STATE) { if (!message.data || this.fullStateSent === false) { this.sendFullState(serverName) } else { this.logger.error(EVENT.ERROR, `Ignoring a request for full state from ${serverName}`) } return } if (message.action === STATE_ACTION.FULL_STATE) { this.applyFullState(serverName, message.fullState!) } if (message.action === STATE_ACTION.CHECKSUM) { this.verifyCheckSum(serverName, message.checksum!) } } } ================================================ FILE: src/services/cluster-state/single-state-registry.ts ================================================ import { StateRegistry, DeepstreamPlugin, StateRegistryCallback } from '@deepstream/types' import { EventEmitter } from 'events' /** * This class provides a generic mechanism that allows to maintain * a distributed state amongst the nodes of a cluster. */ export class SingleStateRegistry extends DeepstreamPlugin implements StateRegistry { public description: string = 'Single State Registry' private readonly data = new Map() private emitter = new EventEmitter() /** * Checks if a given entry exists within the registry */ public has (name: string): boolean { return this.data.has(name) } public onAdd (callback: StateRegistryCallback): void { this.emitter.on('add', callback) } public onRemove (callback: StateRegistryCallback): void { this.emitter.on('remove', callback) } /** * Add a name/entry to the registry. If the entry doesn't exist yet, * this will notify the other nodes within the cluster */ public add (name: string): void { const current = this.data.get(name) if (!current) { this.data.set(name, 1) this.emitter.emit('add', name) } else { this.data.set(name, current + 1) } } /** * Removes a name/entry from the registry. If the entry doesn't exist, * this will exit silently */ public remove (name: string): void { const current = this.data.get(name)! - 1 if (current === 0) { this.data.delete(name) this.emitter.emit('remove', name) } else { this.data.set(name, current) } } /** * Returns all currently registered entries */ public getAll (): string[] { return [ ...this.data.keys() ] } /** * Returns all the servers that hold a given state */ public getAllServers (subscriptionName: string): string[] { return [] } /** * Removes all entries for a given serverName. This is intended to be called * whenever a node leaves the cluster */ public removeAll (serverName: string): void { } } ================================================ FILE: src/services/http/node/node-http.ts ================================================ import { DeepstreamPlugin, DeepstreamHTTPService, EVENT, PostRequestHandler, GetRequestHandler, DeepstreamHTTPMeta, DeepstreamHTTPResponse, SocketHandshakeData, DeepstreamServices, DeepstreamConfig, SocketWrapper, WebSocketConnectionEndpoint, SocketWrapperFactory } from '@deepstream/types' // @ts-ignore import * as httpShutdown from 'http-shutdown' import * as http from 'http' import * as https from 'https' import * as HTTPStatus from 'http-status' import * as contentType from 'content-type' import * as bodyParser from 'body-parser' import { EventEmitter } from 'events' import * as WebSocket from 'ws' import { Socket } from 'net' import { Dictionary } from 'ts-essentials' interface NodeHTTPInterface { healthCheckPath: string, host: string, port: number, allowAllOrigins: boolean, origins?: string[], maxMessageSize: number, hostUrl: string, headers: string[], ssl?: { key: string, cert: string, ca?: string } } export class NodeHTTP extends DeepstreamPlugin implements DeepstreamHTTPService { public description: string = 'NodeJS HTTP Service' private server!: http.Server | https.Server private isReady: boolean = false private methods: string[] = ['GET', 'POST', 'OPTIONS'] private methodsStr: string = this.methods.join(', ') private headers: string[] = ['X-Requested-With', 'X-HTTP-Method-Override', 'Content-Type', 'Accept'] private headersLower: string[] = this.headers.map((header) => header.toLowerCase()) private headersStr: string = this.headers.join(', ') private jsonBodyParser: any private postPaths = new Map>() private getPaths = new Map() private upgradePaths = new Map() private sortedPostPaths: string[] = [] private sortedGetPaths: string[] = [] private sortedUpgradePaths: string[] = [] private connections = new Map() private emitter = new EventEmitter() constructor (private pluginOptions: NodeHTTPInterface, private services: DeepstreamServices, config: DeepstreamConfig) { super() if (this.pluginOptions.allowAllOrigins === false) { if (this.pluginOptions.origins?.length === 0) { this.services.logger.fatal(EVENT.INVALID_CONFIG_DATA, 'HTTP allowAllOrigins set to false but no origins provided') } } this.jsonBodyParser = bodyParser.json({ inflate: true, limit: `${pluginOptions.maxMessageSize}b` }) } public async whenReady (): Promise { if (this.isReady) { return } if (!this.server) { const server: http.Server = this.createHttpServer() this.server = httpShutdown(server) this.server.on('request', this.onRequest.bind(this)) this.server.on('upgrade', this.onUpgrade.bind(this)) this.server.listen(this.pluginOptions.port, this.pluginOptions.host, () => { const serverAddress = this.server.address() as WebSocket.AddressInfo const address = serverAddress.address const port = serverAddress.port this.services.logger.info(EVENT.INFO, `Listening for http connections on ${address}:${port}`) this.services.logger.info(EVENT.INFO, `Listening for health checks on path ${this.pluginOptions.healthCheckPath}`) this.registerGetPathPrefix(this.pluginOptions.healthCheckPath, (meta: DeepstreamHTTPMeta, response: DeepstreamHTTPResponse) => { response(null) }) this.isReady = true this.emitter.emit('ready') }) } return new Promise((resolve) => this.emitter.once('ready', resolve)) } public async close (): Promise { const closePromises: Array> = [] this.connections.forEach((conn) => { if (!conn.isClosed) { closePromises.push(new Promise((resolve) => conn.onClose(resolve))) conn.destroy() } }) await Promise.all(closePromises) this.connections.clear() // @ts-ignore return new Promise((resolve) => this.server.shutdown(resolve)) } public sendWebsocketMessage (socket: WebSocket, message: any, isBinary: boolean) { socket.send(message, (err) => { if (err) { // message was not sent const socketWrapper = this.connections.get(socket)! this.services.logger.warn(EVENT.ERROR, `Failed to deliver message to ${socketWrapper.userId}, error: ${err.message}`) } }) } public getSocketWrappersForUserId (userId: string) { return [...this.connections.values()].filter((socketWrapper) => socketWrapper.userId === userId) } public registerPostPathPrefix (prefix: string, handler: PostRequestHandler) { this.postPaths.set(prefix, handler) this.sortedPostPaths = [...this.postPaths.keys()].sort().reverse() } public registerGetPathPrefix (prefix: string, handler: GetRequestHandler) { this.getPaths.set(prefix, handler) this.sortedGetPaths = [...this.getPaths.keys()].sort().reverse() } public registerWebsocketEndpoint (path: string, createSocketWrapper: SocketWrapperFactory, webSocketConnectionEndpoint: WebSocketConnectionEndpoint) { const server = new WebSocket.Server({ noServer: true, maxPayload: webSocketConnectionEndpoint.wsOptions.maxMessageSize}) server.on('connection', (websocket: WebSocket, handshakeData: SocketHandshakeData) => { websocket.on('error', (error) => { this.services.logger.error(EVENT.ERROR, `Error on websocket: ${error.message}`) }) const socketWrapper = createSocketWrapper(websocket, handshakeData, this.services, webSocketConnectionEndpoint.wsOptions, webSocketConnectionEndpoint) socketWrapper.lastMessageRecievedAt = Date.now() this.connections.set(websocket, socketWrapper) const interval = setInterval(() => { if ((Date.now() - socketWrapper.lastMessageRecievedAt) > webSocketConnectionEndpoint.wsOptions.heartbeatInterval * 2) { this.services.logger.error(EVENT.INFO, 'Heartbeat missing on websocket, terminating connection') socketWrapper.destroy() } }, webSocketConnectionEndpoint.wsOptions.heartbeatInterval) websocket.on('close', () => { clearInterval(interval) webSocketConnectionEndpoint.onSocketClose.call(webSocketConnectionEndpoint, socketWrapper) this.connections.delete(websocket) }) websocket.on('message', (msg: string) => { socketWrapper.lastMessageRecievedAt = Date.now() const messages = socketWrapper.parseMessage(msg) if (messages.length > 0) { socketWrapper.onMessage(messages) } }) webSocketConnectionEndpoint.onConnection.call(webSocketConnectionEndpoint, socketWrapper) }) this.upgradePaths.set(path, server) this.sortedUpgradePaths = [...this.upgradePaths.keys()].sort().reverse() } private createHttpServer () { if (this.pluginOptions.ssl) { const { key, cert, ca } = this.pluginOptions.ssl if (!key || !cert) { this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'To enable HTTP please provide a key and cert') } return new https.Server({ key, cert, ca }) } return new http.Server() } private onUpgrade ( request: http.IncomingMessage, socket: Socket, head: Buffer ): void { for (const path of this.sortedUpgradePaths) { if (request.url === path) { const wss = this.upgradePaths.get(path)! wss.handleUpgrade(request, socket, head, (ws) => { wss.emit('connection', ws, { remoteAddress: request.headers['x-forwarded-for'] || request.connection.remoteAddress, headers: request.headers, referer: request.headers.referer }) }) return } } socket.destroy() } private onRequest ( request: http.IncomingMessage, response: http.ServerResponse ): void { if (!this.pluginOptions.allowAllOrigins) { if (!this.verifyOrigin(request, response)) { return } } else { response.setHeader('Access-Control-Allow-Origin', '*') } switch (request.method) { case 'POST': this.handlePost(request, response) break case 'GET': this.handleGet(request, response) break case 'OPTIONS': this.handleOptions(request, response) break default: this.terminateResponse( response, HTTPStatus.METHOD_NOT_ALLOWED, `Unsupported method. Supported methods: ${this.methodsStr}` ) } } private handlePost (request: http.IncomingMessage, response: http.ServerResponse): void { let parsedContentType try { parsedContentType = contentType.parse(request) } catch (typeError) { parsedContentType = { type: null } } if (parsedContentType.type !== 'application/json') { this.terminateResponse( response, HTTPStatus.UNSUPPORTED_MEDIA_TYPE, 'Invalid "Content-Type" header. Supported media types: "application/json"' ) return } this.jsonBodyParser(request, response, (err: Error | null) => { if (err) { this.terminateResponse( response, HTTPStatus.BAD_REQUEST, `Failed to parse body of request: ${err.message}` ) return } for (const path of this.sortedPostPaths) { if (request.url!.startsWith(path)) { this.postPaths.get(path)!( (request as any).body, { headers: request.headers as Dictionary, url: request.url! }, this.sendResponse.bind(this, response) ) return } } this.terminateResponse(response, HTTPStatus.NOT_FOUND, 'Endpoint not found.') }) } private handleGet (request: http.IncomingMessage, response: http.ServerResponse) { for (const path of this.sortedGetPaths) { if (request.url!.startsWith(path)) { this.getPaths.get(path)!( { headers: request.headers as Dictionary, url: request.url! }, this.sendResponse.bind(this, response) ) return } } this.terminateResponse(response, HTTPStatus.NOT_FOUND, 'Endpoint not found.') } private handleOptions ( request: http.IncomingMessage, response: http.ServerResponse ): void { const requestMethod = request.headers['access-control-request-method'] as string | undefined if (!requestMethod) { this.terminateResponse( response, HTTPStatus.BAD_REQUEST, 'Missing header "Access-Control-Request-Method".' ) return } if (this.methods.indexOf(requestMethod) === -1) { this.terminateResponse( response, HTTPStatus.FORBIDDEN, `Method ${requestMethod} is forbidden. Supported methods: ${this.methodsStr}` ) return } const requestHeadersRaw = request.headers['access-control-request-headers'] as string | undefined if (!requestHeadersRaw) { this.terminateResponse( response, HTTPStatus.BAD_REQUEST, 'Missing header "Access-Control-Request-Headers".' ) return } const requestHeaders = requestHeadersRaw.split(',') for (let i = 0; i < requestHeaders.length; i++) { if (this.headersLower.indexOf(requestHeaders[i].trim().toLowerCase()) === -1) { this.terminateResponse( response, HTTPStatus.FORBIDDEN, `Header ${requestHeaders[i]} is forbidden. Supported headers: ${this.headersStr}` ) return } } response.setHeader('Access-Control-Allow-Methods', this.methodsStr) response.setHeader('Access-Control-Allow-Headers', this.headersStr) this.terminateResponse(response, HTTPStatus.NO_CONTENT) } private verifyOrigin ( request: http.IncomingMessage, response: http.ServerResponse ): boolean { const requestOriginUrl = request.headers.origin as string || request.headers.referer as string const requestHostUrl = request.headers.host if (this.pluginOptions.hostUrl && requestHostUrl !== this.pluginOptions.hostUrl) { this.terminateResponse(response, HTTPStatus.FORBIDDEN, 'Forbidden Host.') return false } if (this.pluginOptions.origins!.indexOf(requestOriginUrl) === -1) { if (!requestOriginUrl) { this.terminateResponse( response, HTTPStatus.FORBIDDEN, 'CORS is configured for this. All requests must set a valid "Origin" header.' ) } else { this.terminateResponse( response, HTTPStatus.FORBIDDEN, `Origin "${requestOriginUrl}" is forbidden.` ) } return false } // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin response.setHeader('Access-Control-Allow-Origin', requestOriginUrl) response.setHeader('Access-Control-Allow-Credentials', 'true') response.setHeader('Vary', 'Origin') return true } private terminateResponse (response: http.ServerResponse, code: number, message?: string) { response.setHeader('Content-Type', 'text/plain; charset=utf-8') response.writeHead(code) if (message) { response.end(`${message}\r\n\r\n`) } else { response.end() } } private sendResponse ( response: http.ServerResponse, err: { statusCode: number, message: string } | null, data: { result: string, body: object } ): void { if (err) { const statusCode = err.statusCode || HTTPStatus.BAD_REQUEST this.terminateResponse(response, statusCode, err.message) return } response.setHeader('Content-Type', 'application/json; charset=utf-8') response.writeHead(HTTPStatus.OK) if (data) { response.end(`${JSON.stringify(data)}\r\n\r\n`) } else { response.end() } } } ================================================ FILE: src/services/http/uws/uws-http.ts ================================================ import { DeepstreamPlugin, DeepstreamHTTPService, PostRequestHandler, GetRequestHandler, DeepstreamServices, DeepstreamConfig, SocketWrapper, WebSocketConnectionEndpoint, SocketWrapperFactory, EVENT, DeepstreamHTTPMeta, DeepstreamHTTPResponse } from '@deepstream/types' import { Dictionary } from 'ts-essentials' import { STATES } from '../../../constants' import { PromiseDelay } from '../../../utils/utils' import * as fileUtils from '../../../config/file-utils' import * as HTTPStatus from 'http-status' interface UWSHTTPInterface extends uws.AppOptions { healthCheckPath: string, host: string, port: number, allowAllOrigins: boolean, origins?: string[], maxMessageSize: number, maxBackpressure?: number, headers: string[], hostUrl: string } interface UserData { url: string, headers: Dictionary, referer: string } export class UWSHTTP extends DeepstreamPlugin implements DeepstreamHTTPService { public description: string = 'UWS HTTP Service' private server!: uws.TemplatedApp private isReady: boolean = false private uWS: typeof uws private connections = new Map, SocketWrapper>() private listenSocket!: uws.us_listen_socket private isGettingReady: boolean = false private maxBackpressure?: number = 1024 * 1024 private methods: string[] = ['GET', 'POST', 'OPTIONS'] private methodsStr: string = this.methods.join(', ') private headers: string[] = ['X-Requested-With', 'X-HTTP-Method-Override', 'Content-Type', 'Accept'] private headersLower: string[] = this.headers.map((header) => header.toLowerCase()) private headersStr: string = this.headers.join(', ') constructor (private pluginOptions: UWSHTTPInterface, private services: DeepstreamServices, config: DeepstreamConfig) { super() if (this.pluginOptions.allowAllOrigins === false) { if (this.pluginOptions.origins?.length === 0) { this.services.logger.fatal(EVENT.INVALID_CONFIG_DATA, 'HTTP allowAllOrigins set to false but no origins provided') } } // set maxBackpressure if defined, default is 1024*1024 if (this.pluginOptions.maxBackpressure) { this.maxBackpressure = this.pluginOptions.maxBackpressure } // alias require to trick nexe from bundling it const req = require try { this.uWS = req('uWebSockets.js') } catch (e) { this.uWS = req(fileUtils.lookupLibRequirePath('uWebSockets.js')) } const sslParams = this.getSLLParams(pluginOptions) if (sslParams) { this.server = this.uWS.SSLApp({ ...pluginOptions, ...sslParams }) } else { this.server = this.uWS.App(pluginOptions) } } public async whenReady (): Promise { if (this.isReady || this.isGettingReady) { return } this.isGettingReady = true return new Promise((resolve) => { this.server.listen(this.pluginOptions.host, this.pluginOptions.port, (token) => { /* Save the listen socket for later shut down */ this.listenSocket = token // handle options requests this.server.options('/*', (response: uws.HttpResponse, request: uws.HttpRequest) => { const baseHeaders: Dictionary = {} if (!this.pluginOptions.allowAllOrigins) { const corsValidationHeaders = this.getVerifiedOriginHeaders(response, request) if (!corsValidationHeaders) { // Verification failed and response terminated return } Object.assign(baseHeaders, corsValidationHeaders) this.handleOptions(response, request, baseHeaders) } else { baseHeaders['Access-Control-Allow-Origin'] = '*' this.handleOptions(response, request, baseHeaders) } }) // Health check path uses GET, so CORS headers will be applied by registerGetPathPrefix // The handler calls response(null), which will use sendResponse. // sendResponse will correctly order writeStatus and add Content-Type: application/json. this.registerGetPathPrefix(this.pluginOptions.healthCheckPath, (meta: DeepstreamHTTPMeta, response: DeepstreamHTTPResponse) => { response(null) }) if (!!token) { resolve() return } this.services.logger.fatal( STATES.SERVICE_INIT, `Failed to listen to port: ${this.pluginOptions.port}` ) }) }) } public async close (): Promise { const closePromises: Array> = [] this.connections.forEach((conn) => { if (!conn.isClosed) { closePromises.push(new Promise((resolve) => conn.onClose(resolve))) conn.destroy() } }) await Promise.all(closePromises) this.connections.clear() this.uWS.us_listen_socket_close(this.listenSocket) await PromiseDelay(2000) } public registerPostPathPrefix (prefix: string, handler: PostRequestHandler) { this.server.post(prefix, (response: uws.HttpResponse, request: uws.HttpRequest) => { /* Register error cb */ response.onAborted(() => { this.services.logger.warn(EVENT.ERROR, 'post request aborted') }) const meta = { headers: this.getHeaders(request), url: request.getUrl() } const accumulatedHeaders: Dictionary = {} if (!this.pluginOptions.allowAllOrigins) { const corsHeaders = this.getVerifiedOriginHeaders(response, request) if (!corsHeaders) { return // Response already terminated } Object.assign(accumulatedHeaders, corsHeaders) } else { accumulatedHeaders['Access-Control-Allow-Origin'] = '*' } readJson(response, (body: any) => { handler( body, meta, (err, data) => this.sendResponse(response, err, data, accumulatedHeaders) ) }, (code: number) => { this.terminateResponse( response, code, HTTPStatus[`${code}_MESSAGE` as keyof typeof HTTPStatus] as string, accumulatedHeaders ) }, this.pluginOptions.maxMessageSize) }) } public registerGetPathPrefix (prefix: string, handler: GetRequestHandler) { this.server.get(prefix, (response: uws.HttpResponse, request: uws.HttpRequest) => { /* Register error cb */ response.onAborted(() => { this.services.logger.warn(EVENT.ERROR, 'get request aborted') }) const accumulatedHeaders: Dictionary = {} if (!this.pluginOptions.allowAllOrigins) { const corsHeaders = this.getVerifiedOriginHeaders(response, request) if (!corsHeaders) { return // Response already terminated } Object.assign(accumulatedHeaders, corsHeaders) } else { accumulatedHeaders['Access-Control-Allow-Origin'] = '*' } handler( { headers: this.getHeaders(request), url: request.getUrl() }, // Ensure the bound sendResponse uses these accumulatedHeaders (err, data) => this.sendResponse(response, err, data, accumulatedHeaders) ) }) } public sendWebsocketMessage (socket: uws.WebSocket, message: Uint8Array | string, isBinary: boolean) { const sentStatus = socket.send(message, isBinary) if (sentStatus === 2) { // message was not sent const socketWrapper = this.connections.get(socket)! this.services.logger.error(EVENT.ERROR, `Failed to deliver message to userId ${socketWrapper.userId}, current socket backpressure ${socket.getBufferedAmount()}`) } } public getSocketWrappersForUserId (userId: string) { return [...this.connections.values()].filter((socketWrapper) => socketWrapper.userId === userId) } public registerWebsocketEndpoint (path: string, createSocketWrapper: SocketWrapperFactory, webSocketConnectionEndpoint: WebSocketConnectionEndpoint) { // uws idleTimeout is in seconds and requires it to be > 8 const idleTimeout = webSocketConnectionEndpoint.wsOptions.heartbeatInterval * 2 / 1000 > 8 ? webSocketConnectionEndpoint.wsOptions.heartbeatInterval * 2 / 1000 : 8 this.server.ws(path, { /* Options */ compression: 0, /* Maximum length of received message. If a client tries to send you a message larger than this, the connection is immediately closed.*/ maxPayloadLength: webSocketConnectionEndpoint.wsOptions.maxMessageSize, /* Maximum length of allowed backpressure per socket when sending messages. Slow receivers with too high backpressure will not receive messages */ maxBackpressure: this.maxBackpressure, idleTimeout, upgrade: (response: uws.HttpResponse, request: uws.HttpRequest, context: any) => { /* This immediately calls open handler, you must not use response after this call */ response.upgrade({ url: request.getUrl(), headers: this.getHeaders(request), referer: request.getHeader('referer') }, /* Spell these correctly */ request.getHeader('sec-websocket-key'), request.getHeader('sec-websocket-protocol'), request.getHeader('sec-websocket-extensions'), context) }, open: (websocket: uws.WebSocket) => { const handshakeData = { remoteAddress: new Uint8Array(websocket.getRemoteAddress()).join('.'), headers: websocket.getUserData().headers, referer: websocket.getUserData().referer } const socketWrapper = createSocketWrapper(websocket, handshakeData, this.services, webSocketConnectionEndpoint.wsOptions, webSocketConnectionEndpoint) this.connections.set(websocket, socketWrapper) webSocketConnectionEndpoint.onConnection.call(webSocketConnectionEndpoint, socketWrapper) }, message: (ws: uws.WebSocket, message: ArrayBuffer, isBinary: boolean) => { const socketWrapper = this.connections.get(ws)! const messages = socketWrapper.parseMessage(isBinary ? new Uint8Array(message) : Buffer.from(message).toString()) if (messages.length > 0) { socketWrapper.onMessage(messages) } }, drain: (socket: uws.WebSocket) => { const socketWrapper = this.connections.get(socket)! this.services.logger.warn(EVENT.INFO, `Socket backpressure drained for userId ${socketWrapper.userId}, current socket backpressure ${socket.getBufferedAmount()}`) }, close: (ws: uws.WebSocket) => { webSocketConnectionEndpoint.onSocketClose.call(webSocketConnectionEndpoint, this.connections.get(ws)!) this.connections.delete(ws) } } as any) } private terminateResponse (response: uws.HttpResponse, code: number, message?: string, additionalHeaders: Dictionary = {}) { response.cork(() => { response.writeStatus(code.toString()) for (const key in additionalHeaders) { if (additionalHeaders.hasOwnProperty(key)) { response.writeHeader(key, additionalHeaders[key]) } } // Only set Content-Type if there's a message body, and not for 204/304 if (message && code !== HTTPStatus.NO_CONTENT) { response.writeHeader('Content-Type', 'text/plain; charset=utf-8') response.end(`${message}\r\n\r\n`) } else { // For 204 NO_CONTENT or other cases without a message, just end. // uWS requires end() to be called. response.end() } }) } private sendResponse ( response: uws.HttpResponse, err: { statusCode: number, message: string } | null, data: { result: string, body: object }, additionalHeaders: Dictionary = {} ): void { if (err) { const statusCode = err.statusCode || HTTPStatus.BAD_REQUEST this.terminateResponse(response, statusCode, err.message, additionalHeaders) return } response.cork(() => { response.writeStatus(HTTPStatus.OK.toString()) for (const key in additionalHeaders) { if (additionalHeaders.hasOwnProperty(key)) { response.writeHeader(key, additionalHeaders[key]) } } response.writeHeader('Content-Type', 'application/json; charset=utf-8') if (data) { response.end(`${JSON.stringify(data)}\r\n\r\n`) } else { response.end() } }) } public getHeaders (req: uws.HttpRequest) { const headers: Dictionary = {} for (const wantedHeader of this.pluginOptions.headers) { headers[wantedHeader] = req.getHeader(wantedHeader).toString() } return headers } private getSLLParams (options: any) { if (!options.ssl) { return null } // tslint:disable-next-line: variable-name const { key: key_file_name, cert: cert_file_name, dhParams: dh_params_file_name, passphrase } = options.ssl if (key_file_name || cert_file_name) { if (!key_file_name) { this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'Must also include ssl key in order to use SSL') } if (!cert_file_name) { this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'Must also include ssl cert in order to use SSL') } return { key_file_name, cert_file_name, dh_params_file_name, passphrase, } } return null } // This method now either terminates the response or returns headers for the caller to use. private getVerifiedOriginHeaders (response: uws.HttpResponse, request: uws.HttpRequest): Dictionary | null { const requestOriginUrl = request.getHeader('origin') as string || request.getHeader('referer') as string const requestHostUrl = request.getHeader('host') if (this.pluginOptions.hostUrl && requestHostUrl !== this.pluginOptions.hostUrl) { this.terminateResponse(response, HTTPStatus.FORBIDDEN, 'Forbidden Host.') return null } if (this.pluginOptions.origins!.indexOf(requestOriginUrl) === -1) { if (!requestOriginUrl) { this.terminateResponse( response, HTTPStatus.FORBIDDEN, 'CORS is configured for this. All requests must set a valid "Origin" header.', // No additional headers known at this point for the error response itself ) } else { this.terminateResponse( response, HTTPStatus.FORBIDDEN, `Origin "${requestOriginUrl}" is forbidden.`, // No additional headers known at this point for the error response itself ) } return null } // If verification is successful, return the headers to be set by the caller. return { 'Access-Control-Allow-Origin': requestOriginUrl, 'Access-Control-Allow-Credentials': 'true', // Typically needed with specific origins 'Vary': 'Origin' // Good practice when Access-Control-Allow-Origin is dynamic } } private handleOptions (response: uws.HttpResponse, request: uws.HttpRequest, baseCorsHeaders: Dictionary): void { const allHeadersForResponse = { ...baseCorsHeaders } const requestMethod = request.getHeader('access-control-request-method') as string | undefined if (!requestMethod) { this.terminateResponse( response, HTTPStatus.BAD_REQUEST, 'Missing header "Access-Control-Request-Method".', allHeadersForResponse // Pass along already determined CORS headers ) return } if (this.methods.indexOf(requestMethod) === -1) { this.terminateResponse( response, HTTPStatus.FORBIDDEN, `Method ${requestMethod} is forbidden. Supported methods: ${this.methodsStr}`, allHeadersForResponse ) return } const requestHeadersRaw = request.getHeader('access-control-request-headers') as string | undefined if (!requestHeadersRaw) { // Some browsers might not send this for simple requests, but for preflight it's expected. // Depending on strictness, this could be an error or allowed. // For now, let's assume it's required for a preflight OPTIONS. this.terminateResponse( response, HTTPStatus.BAD_REQUEST, 'Missing header "Access-Control-Request-Headers".' ) return } const requestHeaders = requestHeadersRaw.split(',') for (let i = 0; i < requestHeaders.length; i++) { if (this.headersLower.indexOf(requestHeaders[i].trim().toLowerCase()) === -1) { this.terminateResponse( response, HTTPStatus.FORBIDDEN, `Header ${requestHeaders[i]} is forbidden. Supported headers: ${this.headersStr}`, allHeadersForResponse ) return } } allHeadersForResponse['Access-Control-Allow-Methods'] = this.methodsStr allHeadersForResponse['Access-Control-Allow-Headers'] = this.headersStr this.terminateResponse(response, HTTPStatus.NO_CONTENT, undefined, allHeadersForResponse) } } /* Helper function for reading a posted JSON body */ function readJson (res: uws.HttpResponse, cb: Function, err: (code: number) => void, limit: number) { let buffer: Buffer let received: number = 0 res.onData((ab, isLast) => { const chunk = Buffer.from(ab) received += chunk.length // check max length if (received > limit) { err(HTTPStatus.REQUEST_ENTITY_TOO_LARGE) return } if (isLast) { let json if (buffer) { try { json = JSON.parse(Buffer.concat([buffer, chunk]).toString()) } catch (e) { err(HTTPStatus.BAD_REQUEST) return } cb(json) } else { try { json = JSON.parse(chunk.toString()) } catch (e) { err(HTTPStatus.BAD_REQUEST) return } cb(json) } } else { if (buffer) { buffer = Buffer.concat([buffer, chunk]) } else { buffer = Buffer.concat([chunk]) } } }) } ================================================ FILE: src/services/lock/distributed-lock-registry.ts ================================================ import {EventEmitter} from 'events' import Timeout = NodeJS.Timeout import { DeepstreamPlugin, DeepstreamLockRegistry, DeepstreamServices, DeepstreamConfig, LockCallback, EVENT } from '@deepstream/types' import { TOPIC, LOCK_ACTION, LockMessage } from '../../constants' /** * The lock registry is responsible for maintaing a single source of truth * within the cluster, used mainly for issuing cluster wide locks when an operation * that stretches over multiple nodes are required. * * For example, distributed listening requires a leader to drive the nodes in sequence, * so issuing a lock prevents multiple nodes from assuming the lead. * */ export class DistributedLockRegistry extends DeepstreamPlugin implements DeepstreamLockRegistry { public description: string = 'Distributed Lock Registry' private locks = new Set() private timeouts = new Map() private responseEventEmitter = new EventEmitter() /** * The unique registry is a singleton and is only created once * within deepstream.io. It is passed via * via the options object. */ constructor (private pluginOptions: any, private services: Readonly, private config: Readonly) { super() this.onPrivateMessage = this.onPrivateMessage.bind(this) } public init () { this.services.clusterNode.subscribe(TOPIC.LOCK, this.onPrivateMessage) } /** * Requests a lock, if the leader ( whether local or distributed ) has the lock availble * it will invoke the callback with true, otherwise false. */ public get (lockName: string, callback: LockCallback) { if (this.services.clusterRegistry.isLeader()) { callback(this.getLock(lockName)) } else if (!this.timeouts.has(lockName)) { this.getRemoteLock(lockName, callback) } else { callback(false) } } /** * Release a lock, allowing other resources to request it again */ public release (lockName: string) { if (this.services.clusterRegistry.isLeader()) { this.releaseLock(lockName) } else { this.releaseRemoteLock(lockName) } } /** * Called when the current node is not the leader, issuing a lock request * via the message bus */ private getRemoteLock (lockName: string, callback: LockCallback) { const leaderServerName = this.services.clusterRegistry.getLeader() this.timeouts.set(lockName, setTimeout( this.onLockRequestTimeout.bind(this, lockName), this.pluginOptions.requestTimeout )) this.responseEventEmitter.once(lockName, callback) this.services.clusterNode.sendDirect(leaderServerName, { topic: TOPIC.LOCK, action: LOCK_ACTION.REQUEST, name: lockName }) } /** * Notifies a remote leader keeping a lock that said lock is no longer required */ private releaseRemoteLock (lockName: string) { const leaderServerName = this.services.clusterRegistry.getLeader() this.services.clusterNode.sendDirect(leaderServerName, { topic: TOPIC.LOCK, action: LOCK_ACTION.RELEASE, name: lockName }) } /** * Called when a message is received on the message bus. * This could mean the leader responded to a request or that you're currently * the leader and received a request. */ private onPrivateMessage (message: LockMessage, remoteServerName: string) { if (message.action === LOCK_ACTION.RESPONSE) { this.handleRemoteLockResponse(message.name!, message.locked) return } if (this.services.clusterRegistry.isLeader() === false) { this.services.logger.warn( EVENT.INVALID_LEADER_REQUEST, `server ${remoteServerName} assumes this node '${this.config.serverName}' is the leader` ) return } if (message.action === LOCK_ACTION.REQUEST) { this.handleRemoteLockRequest(message.name, remoteServerName) return } if (message.action === LOCK_ACTION.RELEASE) { this.releaseLock(message.name!) return } } /** * Called when a remote lock request is received */ private handleRemoteLockRequest (lockName: string, remoteServerName: string) { this.services.clusterNode.sendDirect(remoteServerName, { topic: TOPIC.LOCK, action: LOCK_ACTION.RESPONSE, name: lockName, locked: this.getLock(lockName) }) } /** * Called when a remote lock response is received */ private handleRemoteLockResponse (lockName: string, result: boolean) { clearTimeout(this.timeouts.get(lockName)!) this.timeouts.delete(lockName) this.responseEventEmitter.emit(lockName, result) } /** * Returns true if reserving lock was possible otherwise returns false */ private getLock (lockName: string) { if (this.locks.has(lockName)) { return false } this.timeouts.set(lockName, setTimeout( this.onLockTimeout.bind(this, lockName), this.pluginOptions.holdTimeout )) this.locks.add(lockName) return true } /** * Called when a lock is no longer required and can be released. This is triggered either by * a timeout if a remote release message wasn't received in time or when release was called * locally. * * Important note: Anyone can release a lock. It is assumed that the cluster is trusted * so maintaining who has the lock is not required. This may need to change going forward. */ private releaseLock (lockName: string) { clearTimeout(this.timeouts.get(lockName)!) this.timeouts.delete(lockName) this.locks.delete(lockName) } /** * Called when a timeout occurs on a lock that has been reserved for too long */ private onLockTimeout (lockName: string) { this.releaseLock(lockName) this.services.logger.warn(EVENT.LOCK_RELEASE_TIMEOUT, `lock ${lockName} released due to timeout`, { lockName }) } /** * Called when a remote request has timed out, resulting in notifying the client that * the lock wasn't able to be reserved */ private onLockRequestTimeout (lockName: string) { this.handleRemoteLockResponse(lockName, false) this.services.logger.warn(EVENT.LOCK_REQUEST_TIMEOUT, `request for lock ${lockName} timed out`, { lockName }) } } ================================================ FILE: src/services/logger/pino/pino-logger.ts ================================================ import pino, { LoggerOptions } from 'pino' import { LOG_LEVEL, DeepstreamPlugin, DeepstreamLogger, DeepstreamConfig, DeepstreamServices, NamespacedLogger, EVENT } from '@deepstream/types' const DSToPino: { [index: number]: pino.LevelWithSilent } = { [LOG_LEVEL.DEBUG]: 'debug', [LOG_LEVEL.FATAL]: 'fatal', [LOG_LEVEL.ERROR]: 'error', [LOG_LEVEL.WARN]: 'warn', [LOG_LEVEL.INFO]: 'info', [LOG_LEVEL.OFF]: 'silent', } export class PinoLogger extends DeepstreamPlugin implements DeepstreamLogger { public description = 'Pino Logger' private logger: pino.Logger constructor (pluginOptions: LoggerOptions, private services: DeepstreamServices, config: DeepstreamConfig) { super() this.logger = pino(pluginOptions) this.setLogLevel(config.logLevel) } /** * Return true if logging is enabled. This is used in deepstream to stop generating useless complex strings * that we know will never be logged. */ public shouldLog (logLevel: LOG_LEVEL): boolean { return this.logger.isLevelEnabled(DSToPino[logLevel]) } /** * Set the log level desired by deepstream. Since deepstream uses LOG_LEVEL this needs to be mapped * to whatever your libary uses (this is usually just conversion stored in a static map) */ public setLogLevel (logLevel: LOG_LEVEL): void { this.logger.level = DSToPino[logLevel] } /** * Log as info */ public info (event: EVENT, message?: string, metaData?: any): void { if (metaData) { this.logger.info({ event, message, ...metaData }) } else { this.logger.info({ event, message }) } } /** * Log as debug */ public debug (event: EVENT, message?: string, metaData?: any): void { if (metaData) { this.logger.debug({ event, message, ...metaData }) } else { this.logger.debug({ event, message }) } } /** * Log as warn */ public warn (event: EVENT, message?: string, metaData?: any): void { if (metaData) { this.logger.warn({ event, message, ...metaData }) } else { this.logger.warn({ event, message }) } } /** * Log as error */ public error (event: EVENT, message?: string, metaData?: any): void { this.services.monitoring.onErrorLog(LOG_LEVEL.ERROR, event, message!, metaData!) if (metaData) { this.logger.error({ event, message, ...metaData }) } else { this.logger.error({ event, message }) } } /** * Log as fatal */ public fatal (event: EVENT, message?: string, metaData?: any): void { this.services.monitoring.onErrorLog(LOG_LEVEL.FATAL, event, message!, metaData!) if (metaData) { this.logger.fatal({ event, message, ...metaData }) } else { this.logger.fatal({ event, message }) } this.services.notifyFatalException() } /** * Create a namespaced logger, used by plugins. This could either be a new instance of a logger * or just a thin wrapper to add the namespace at the beginning of the log method. */ public getNameSpace (namespace: string): NamespacedLogger { return { shouldLog: this.shouldLog.bind(this), fatal: this.log.bind(this, DSToPino[LOG_LEVEL.FATAL], namespace), error: this.log.bind(this, DSToPino[LOG_LEVEL.ERROR], namespace), warn: this.log.bind(this, DSToPino[LOG_LEVEL.WARN], namespace), info: this.log.bind(this, DSToPino[LOG_LEVEL.INFO], namespace), debug: this.log.bind(this, DSToPino[LOG_LEVEL.DEBUG], namespace), } } private log (logLevel: pino.LevelWithSilent, namespace: string, event: EVENT, message: string, metaData?: any ) { this.logger[logLevel]({ namespace, event, message, ...metaData }) } } ================================================ FILE: src/services/logger/std/std-out-logger.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import {spy} from 'sinon' import { StdOutLogger } from './std-out-logger' import { LOG_LEVEL, EVENT, DeepstreamConfig } from '@deepstream/types'; describe('logs to stdout and stderr', () => { const logger = new StdOutLogger({ color: false }, undefined, { logLevel: LOG_LEVEL.DEBUG } as DeepstreamConfig) const originalStdOut = process.stdout const originalStdErr = process.stderr const stdout = spy() const stderr = spy() const comp = function (std, exp) { return std.lastCall.args[0].indexOf(exp) !== -1 } before(() => { Object.defineProperty(process, 'stdout', { value: { write: stdout } }) Object.defineProperty(process, 'stderr', { value: { write: stderr } }) }) after(() => { Object.defineProperty(process, 'stdout', { value: originalStdOut }) Object.defineProperty(process, 'stderr', { value: originalStdErr }) }) it('creates the logger', async () => { await logger.whenReady() logger.info(EVENT.INFO, 'b') expect(comp(stdout, 'INFO | b')).to.equal(true) }) it('logs to stderr', () => { stdout.resetHistory() logger.error(EVENT.INFO, 'e') expect(stdout).to.have.callCount(0) expect(stderr).to.have.callCount(1) }) it('logs above log level', () => { logger.setLogLevel(LOG_LEVEL.DEBUG) stdout.resetHistory() logger.info(EVENT.INFO, 'e') expect(stdout).to.have.callCount(1) logger.setLogLevel(LOG_LEVEL.WARN) stdout.resetHistory() logger.info(EVENT.INFO, 'e') expect(stdout).to.have.callCount(0) }) }) ================================================ FILE: src/services/logger/std/std-out-logger.ts ================================================ import * as chalk from 'chalk' import { DeepstreamPlugin, DeepstreamLogger, DeepstreamServices, DeepstreamConfig, LOG_LEVEL, NamespacedLogger, EVENT, MetaData } from '@deepstream/types' import { EOL } from 'os' export class StdOutLogger extends DeepstreamPlugin implements DeepstreamLogger { public description = 'std out/err' private useColors: boolean private currentLogLevel!: LOG_LEVEL private logLevelColors: string[] = [ 'white', 'green', 'yellow', 'red', 'blue' ] /** * Logs to the operatingsystem's standard-out and standard-error streams. * * Consoles / Terminals as well as most log-managers and logging systems * consume messages from these streams */ constructor (private options: any = {}, private services: DeepstreamServices, config: DeepstreamConfig) { super() this.useColors = this.options.colors === undefined ? true : this.options.colors this.setLogLevel(config.logLevel) } public async whenReady (): Promise { this.description = `${this.description} at level ${LOG_LEVEL[this.currentLogLevel]}` } public shouldLog (logLevel: number): boolean { return this.currentLogLevel >= logLevel } public debug (event: EVENT, logMessage: string): void { this.log(LOG_LEVEL.DEBUG, '', event, logMessage) } public info (event: EVENT, logMessage: string): void { this.log(LOG_LEVEL.INFO, '', event, logMessage) } public warn (event: EVENT, logMessage: string, metaData?: MetaData): void { this.log(LOG_LEVEL.WARN, '', event, logMessage, metaData) } public error (event: EVENT, logMessage: string, metaData?: MetaData): void { this.log(LOG_LEVEL.ERROR, '', event, logMessage, metaData) } public fatal (event: EVENT, logMessage: string, metaData?: MetaData): void { this.log(LOG_LEVEL.FATAL, '', event, logMessage, metaData) this.services.notifyFatalException() } public getNameSpace (namespace: string): NamespacedLogger { return { shouldLog: this.shouldLog.bind(this), fatal: this.log.bind(this, LOG_LEVEL.FATAL, namespace), error: this.log.bind(this, LOG_LEVEL.ERROR, namespace), warn: this.log.bind(this, LOG_LEVEL.WARN, namespace), info: this.log.bind(this, LOG_LEVEL.INFO, namespace), debug: this.log.bind(this, LOG_LEVEL.DEBUG, namespace), } } /** * Sets the log-level. This can be called at runtime. */ public setLogLevel (logLevel: LOG_LEVEL) { this.currentLogLevel = logLevel } /** * Logs a line */ private log (logLevel: LOG_LEVEL, namespace: string, event: EVENT, logMessage: string, metaData: MetaData | null = null): void { if (logLevel >= LOG_LEVEL.WARN && this.services && this.services.monitoring) { this.services.monitoring.onErrorLog(logLevel, event, logMessage, metaData!) } if (this.currentLogLevel > logLevel) { return } const msg = `${namespace ? `${namespace} | ` : '' }${event} | ${logMessage}` let outputStream if (logLevel === LOG_LEVEL.ERROR || logLevel === LOG_LEVEL.WARN) { outputStream = 'stderr' } else { outputStream = 'stdout' } if (this.useColors) { (process as any)[outputStream].write((chalk as any)[this.logLevelColors[logLevel]](msg) + EOL) } else { (process as any)[outputStream].write(msg + EOL) } if (logLevel === LOG_LEVEL.FATAL) { this.services.notifyFatalException() } } } ================================================ FILE: src/services/monitoring/combine-monitoring.ts ================================================ import { DeepstreamPlugin, DeepstreamMonitoring, SocketData, LOG_LEVEL, EVENT, MetaData } from '@deepstream/types' import { Message } from '../../constants' /** * The combine monitoring handler allows multiple monitoring plugins, * this allows to develop plugins that handle independantly multiple aspects of the monitoring: audit logs, user behaviour, more complex presence logic, etc * */ export class CombineMonitoring extends DeepstreamPlugin implements DeepstreamMonitoring { public description: string = '' constructor (private monitorings: DeepstreamMonitoring[]) { super() if (monitorings.length === 1) { this.description = monitorings[0].description } else { this.description = monitorings.map((monitoring, index) => `\n\t${index}) ${monitoring.description}`).join('') } } public async whenReady () { await Promise.all(this.monitorings.map((monitoring) => monitoring.whenReady())) } public async close () { await Promise.all(this.monitorings.map((monitoring) => monitoring.close())) } public init () { this.monitorings.forEach((monitoring) => monitoring.init ? monitoring.init() : null) } public onErrorLog (loglevel: LOG_LEVEL, event: EVENT, logMessage: string, metaData: MetaData): void { // NOTE: If using another logger service other than std-out or pino, the logger service must call this endpoint when logging. this.monitorings.forEach((monitoring) => monitoring.onErrorLog(loglevel, event, logMessage, metaData)) } public onLogin (allowed: boolean, endpointType: string): void { this.monitorings.forEach((monitoring) => monitoring.onLogin(allowed, endpointType)) } public onMessageReceived (message: Message, socketData: SocketData): void { this.monitorings.forEach((monitoring) => monitoring.onMessageReceived(message, socketData)) } public onMessageSend (message: Message): void { this.monitorings.forEach((monitoring) => monitoring.onMessageSend(message)) } public onBroadcast (message: Message, count: number): void { this.monitorings.forEach((monitoring) => monitoring.onBroadcast(message, count)) } } ================================================ FILE: src/services/monitoring/http/monitoring-http.spec.ts ================================================ describe('stub', () => { it ('does nothing yet', () => { }) }) ================================================ FILE: src/services/monitoring/http/monitoring-http.ts ================================================ import { DeepstreamServices, DeepstreamHTTPMeta, DeepstreamHTTPResponse, EVENT } from '@deepstream/types' import { MonitoringBase } from '../monitoring-base' interface HTTPMonitoringOptions { url: string, headerKey: string, headerValue: string, allowOpenPermissions: boolean } export default class HTTPMonitoring extends MonitoringBase { public description = 'HTTP Monitoring' private logger = this.services.logger.getNameSpace('HTTP_MONITORING') constructor (private pluginOptions: HTTPMonitoringOptions, services: DeepstreamServices) { super(services) if (typeof pluginOptions.url !== 'string') { this.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'Missing "url" for HTTP Monitoring') } if (this.pluginOptions.allowOpenPermissions) { this.logger.warn(EVENT.PLUGIN_INITIALIZATION_ERROR, '"allowOpenPermissions" is set. Try not to deploy to production') } else if (!pluginOptions.headerKey || !pluginOptions.headerValue) { this.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'Missing "headerKey" and/or "headerValue"') } this.description += ` on ${this.pluginOptions.url}` } public async whenReady (): Promise { await this.services.httpService.whenReady() let from = Date.now() this.services.httpService.registerGetPathPrefix(this.pluginOptions.url, (metaData: DeepstreamHTTPMeta, response: DeepstreamHTTPResponse) => { if (this.pluginOptions.allowOpenPermissions !== true) { if (metaData.headers[this.pluginOptions.headerKey] !== this.pluginOptions.headerValue) { this.logger.warn(EVENT.AUTH_ERROR, 'Invalid monitoring data due to missing or invalid header values') return response({ statusCode: 404, message: 'Endpoint not found.' }) } } const to = Date.now() response(null, { from, to, ...this.getAndResetMonitoringStats() }) from = to }) } public async close (): Promise { } } ================================================ FILE: src/services/monitoring/log/monitoring-log.spec.ts ================================================ describe('stub', () => { it ('does nothing yet', () => { }) }) ================================================ FILE: src/services/monitoring/log/monitoring-log.ts ================================================ import { DeepstreamServices, NamespacedLogger } from '@deepstream/types' import { MonitoringBase } from '../monitoring-base' interface HTTPMonitoringOptions { logInterval: number, monitoringKey: string } export default class LogMonitoring extends MonitoringBase { public description = 'Log Monitoring' private logInterval!: NodeJS.Timeout private logger: NamespacedLogger constructor (private pluginOptions: HTTPMonitoringOptions, services: DeepstreamServices) { super(services) this.pluginOptions.monitoringKey = pluginOptions.monitoringKey || 'LOG_MONITORING' this.logger = this.services.logger.getNameSpace(this.pluginOptions.monitoringKey) this.pluginOptions.logInterval = pluginOptions.logInterval || 15000 this.description += ` every ${this.pluginOptions.logInterval / 1000} seconds` } public async whenReady (): Promise { let lastDate = Date.now() this.logInterval = setInterval(() => { const newDate = Date.now() this.logger.info(`Monitoring stats for ${lastDate} to ${newDate}`, JSON.stringify({ [this.pluginOptions.monitoringKey]: this.getAndResetMonitoringStats(), from: lastDate, to: newDate })) lastDate = newDate }, this.pluginOptions.logInterval) } public async close (): Promise { clearInterval(this.logInterval) } } ================================================ FILE: src/services/monitoring/monitoring-base.ts ================================================ import { Message, ACTIONS } from '@deepstream/protobuf/dist/types/messages' import { TOPIC, STATE_REGISTRY_TOPIC } from '@deepstream/protobuf/dist/types/all' import { DeepstreamMonitoring, DeepstreamPlugin, DeepstreamServices, LOG_LEVEL, EVENT } from '@deepstream/types' export abstract class MonitoringBase extends DeepstreamPlugin implements DeepstreamMonitoring { private errorLogs: { [index: string]: number } = {} private receiveStats: { [index: string]: { [index: string]: number } } = {} private sendStats: { [index: string]: { [index: string]: number } } = {} private loginStats: { [index: string]: { allowed: number, declined: number } } = {} constructor (protected services: DeepstreamServices) { super() } public onErrorLog (loglevel: LOG_LEVEL, event: EVENT, logMessage: string): void { const count = this.errorLogs[event] if (!count) { this.errorLogs[event] = 1 } else { this.errorLogs[event] = count + 1 } } /** * Called whenever a login attempt is tried and whether or not it succeeded, as well * as the connection-endpoint type, which is provided from the connection endpoint * itself */ public onLogin (allowed: boolean, endpointType: string): void { let stats = this.loginStats[endpointType] if (!stats) { stats = { allowed: 0, declined: 0 } this.loginStats[endpointType] = stats } allowed ? stats.allowed++ : stats.declined++ } public onMessageReceived (message: Message): void { let actionsMap = this.receiveStats[TOPIC[message.topic]] if (!actionsMap) { actionsMap = {} this.receiveStats[TOPIC[message.topic]] = actionsMap } const actionName = ACTIONS[message.topic][message.action!] actionsMap[actionName] = actionsMap[actionName] ? actionsMap[actionName] + 1 : 1 } public onMessageSend (message: Message): void { let actionsMap = this.sendStats[TOPIC[message.topic]] if (!actionsMap) { actionsMap = {} this.sendStats[TOPIC[message.topic]] = actionsMap } const actionName = ACTIONS[message.topic][message.action!] actionsMap[actionName] = actionsMap[actionName] ? actionsMap[actionName] + 1 : 1 } public onBroadcast (message: Message, count: number): void { let actionsMap = this.receiveStats[TOPIC[message.topic]] if (!actionsMap) { actionsMap = {} this.sendStats[TOPIC[message.topic]] = actionsMap } const actionName = ACTIONS[message.topic][message.action!] actionsMap[actionName] = actionsMap[actionName] ? actionsMap[actionName] + count : count } public getAndResetMonitoringStats () { const results = { clusterSize: this.services.clusterRegistry.getAll().length, stateMetrics: this.getStateMetrics(), errors: this.errorLogs, received: this.receiveStats, send: this.sendStats, logins: this.loginStats } this.errorLogs = {} this.receiveStats = {} this.sendStats = {} this.loginStats = {} return results } private getStateMetrics () { const result: any = {} const stateRegistries = this.services.clusterStates.getStateRegistries() for (const [topic, stateRegistry] of stateRegistries) { result[TOPIC[topic] || STATE_REGISTRY_TOPIC[topic]] = stateRegistry.getAll().length } return result } } ================================================ FILE: src/services/monitoring/noop-monitoring.ts ================================================ import { DeepstreamPlugin, DeepstreamMonitoring, SocketData, LOG_LEVEL, EVENT } from '@deepstream/types' import { Message } from '../../constants' export class NoopMonitoring extends DeepstreamPlugin implements DeepstreamMonitoring { public description: string = 'Noop Monitoring' public onErrorLog (loglevel: LOG_LEVEL, event: EVENT, logMessage: string): void { } public onLogin (allowed: boolean, endpointType: string): void { } public onMessageReceived (message: Message, socketData: SocketData): void { } public onMessageSend (message: Message): void { } public onBroadcast (message: Message, count: number): void { } } ================================================ FILE: src/services/permission/open/open-permission.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import { OpenPermission } from './open-permission' describe('open permission handler', () => { let permission it('allows any action', (done) => { permission = new OpenPermission() const message = { topic: 'This doesnt matter', action: 'Since it allows anything', data: ['anything'] } permission.canPerformAction('someone', message, (socketWrapper, msg, passItOn, error, success) => { expect(error).to.equal(null) expect(success).to.equal(true) done() }, {}) }) }) ================================================ FILE: src/services/permission/open/open-permission.ts ================================================ import { Message } from '../../../constants' import { DeepstreamPlugin, DeepstreamPermission, PermissionCallback, SocketWrapper } from '@deepstream/types' /** * The open permission handler allows any action to occur without applying * any permissions. */ export class OpenPermission extends DeepstreamPlugin implements DeepstreamPermission { public description: string = 'none' /** * Allows any action by an user */ public canPerformAction (socketWrapper: SocketWrapper, message: Message, callback: PermissionCallback, passItOn: any) { callback(socketWrapper, message, passItOn, null, true) } } ================================================ FILE: src/services/permission/valve/config-compiler.spec.ts ================================================ import 'mocha' import { expect } from 'chai' const configCompiler = require('./config-compiler') describe('compiles user entered config specs into an optimized format', () => { it('exposes a compile method', () => { expect(typeof configCompiler.compile).to.equal('function') }) it('compiles a basic config', () => { const conf = { record: { 'user/$userId': { write: '$userId === user.id' } } } const compiledConf = configCompiler.compile(conf) expect(Array.isArray(compiledConf.record)).to.equal(true) expect(compiledConf.record.length).to.equal(1) expect(compiledConf.record[0].regexp).to.not.equal(undefined) expect(compiledConf.record[0].variables).to.deep.equal(['$userId']) expect(compiledConf.record[0].rules.write.fn).to.not.equal(undefined) expect(compiledConf.record[0].rules.write.hasOldData).to.equal(false) }) }) ================================================ FILE: src/services/permission/valve/config-compiler.ts ================================================ import * as pathParser from './path-parser' import * as ruleParser from './rule-parser' import { ValveSchema } from '@deepstream/types' /** * Compiles a pre-validated config into a format that allows for quicker access * and execution */ export const compile = function (config: ValveSchema) { const compiledConfig: any = {} let compiledRuleset let section let path for (section in config) { compiledConfig[section] = [] for (path in config[section]) { compiledRuleset = compileRuleset(path, config[section][path]) compiledConfig[section].push(compiledRuleset) } } return compiledConfig } /** * Compiles an individual ruleset */ function compileRuleset (path: string, rules: any) { const ruleset = pathParser.parse(path) ruleset.rules = {} for (const ruleType in rules) { ruleset.rules[ruleType] = ruleParser.parse( rules[ruleType], ruleset.variables, ) } return ruleset } ================================================ FILE: src/services/permission/valve/config-permission-basic.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as C from '../../../constants' import { getBasePermissions } from '../../../test/helper/test-helper' import * as testHelper from '../../../test/helper/test-helper' import { ConfigPermission } from './config-permission' const options = testHelper.getDeepstreamPermissionOptions() const config = options.config const services = options.services const testPermission = testHelper.testPermission(options) const lastError = function () { return services.logger.logSpy.lastCall.args[2] } describe('permission handler applies basic permissions to incoming messages', () => { it('allows everything for a basic permission set', () => { const permissions = getBasePermissions() const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'someRecord' } expect(testPermission(permissions, message)).to.equal(true) }) it('denies reading of a private record', () => { const permissions = getBasePermissions() permissions.record['private/$userId'] = { read: 'user.id === $userId' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'private/userA' } expect(testPermission(permissions, message, 'userB')).to.equal(false) }) it('allows actions that dont need permissions for a private record', () => { const permissions = getBasePermissions() permissions.record['private/$userId'] = { read: 'user.id === $userId' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UNSUBSCRIBE, name: 'private/userA' } expect(testPermission(permissions, message, 'userB')).to.equal(true) }) it('allows reading of a private record for the right user', () => { const permissions = getBasePermissions() permissions.record['private/$userId'] = { read: 'user.id === $userId' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'private/userA' } expect(testPermission(permissions, message, 'userA')).to.equal(true) }) it('can reference the name', () => { const permissions = getBasePermissions() permissions.record['private/userA'] = { read: 'name === "private/userA"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'private/userA' } expect(testPermission(permissions, message, 'userA')).to.equal(true) }) it('denies snapshot of a private record', () => { const permissions = getBasePermissions() permissions.record['private/$userId'] = { read: 'user.id === $userId' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'private/userA' } expect(testPermission(permissions, message, 'userB')).to.equal(false) }) }) describe('permission handler applies basic permissions referencing their own data', () => { it('checks incoming data against a value for events', () => { const permissions = getBasePermissions() permissions.event['some-event'] = { publish: 'data.price < 10' } expect(testPermission(permissions, { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: '{"price":15}' })).to.equal(false) expect(testPermission(permissions, { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: '{"price":5}' })).to.equal(true) }) it('can reference data for events without a payload and fail normally', () => { const permissions = getBasePermissions() permissions.event['some-event'] = { publish: 'data.price < 10' } expect(testPermission(permissions, { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event' })).to.equal(false) }) it('checks incoming data against a value for rpcs', () => { const permissions = getBasePermissions() permissions.rpc['*'] = { request: false } permissions.rpc['trade/book'] = { request: 'user.data.role === "fx-trader" && data.assetClass === "fx"' } expect(testPermission(permissions, { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST, name: 'trade/book', correlationId: '1234', data: '{"assetClass": "equity"}' }, null, { role: 'eq-trader' })).to.equal(false) expect(testPermission(permissions, { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST, name: 'trade/book', correlationId: '1234', data: '{"assetClass": "fx"}' }, null, { role: 'fx-trader' })).to.equal(true) expect(testPermission(permissions, { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST, name: 'trade/book', correlationId: '1234', data: '{"assetClass": "fx"}' }, null, { role: 'eq-trader' })).to.equal(false) expect(testPermission(permissions, { topic: C.TOPIC.RPC, action: C.RPC_ACTION.REQUEST, name: 'trade/cancel', correlationId: '1234', data: '{"assetClass": "fx"}' }, null, { role: 'fx-trader' })).to.equal(false) }) it('checks incoming data against a value for record updates', () => { const permissions = getBasePermissions() permissions.record['cars/mercedes'] = { write: 'data.manufacturer === "mercedes-benz"' } permissions.record['cars/porsche/$model'] = { write: 'data.price > 50000 && data.model === $model' } expect(testPermission(permissions, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'cars/mercedes', version: 1, data: '{"manufacturer":"mercedes-benz"}' })).to.equal(true) expect(testPermission(permissions, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'cars/mercedes', version: 1, data: '{"manufacturer":"BMW"}' })).to.equal(false) expect(testPermission(permissions, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'cars/porsche/911', version: 1, data: '{"model": "911", "price": 60000 }' })).to.equal(true) expect(testPermission(permissions, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'cars/porsche/911', version: 1, data: '{"model": "911", "price": 40000 }' })).to.equal(false) expect(testPermission(permissions, { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'cars/porsche/911', version: 1, data: '{"model": "Boxter", "price": 70000 }' })).to.equal(false) }) it.skip('checks against existing data for non-existant record reads', (next) => { const permissions = getBasePermissions() permissions.record['non-Existing-Record'] = { read: 'oldData.xyz === "hello"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'non-Existing-Record', } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('Cannot read property \'xyz\' of null') expect(result).to.equal(false) next() } testPermission(permissions, message, 'user', null, callback) }) it.skip('checks against existing data for non-existant list reads', (next) => { const permissions = getBasePermissions() permissions.record['non-Existing-Record'] = { read: 'oldData.indexOf("hello") !== -1' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'non-Existing-Record', } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('Cannot read property \'indexOf\' of null') expect(error).to.equal(C.RECORD_ACTION.MESSAGE_PERMISSION_ERROR) expect(result).to.equal(false) next() } testPermission(permissions, message, 'user', null, callback) }) it('deals with broken messages', (next) => { const permissions = getBasePermissions() permissions.record['cars/mercedes'] = { write: 'data.manufacturer === "mercedes-benz"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'cars/mercedes', version: 1, data: '{"manufacturer":"mercedes-benz"' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('error when converting message data') expect(result).to.equal(false) next() } testPermission(permissions, message, 'user', null, callback) }) it('deals with messages with invalid types', (next) => { const permissions = getBasePermissions() permissions.event['some-event'] = { publish: 'data.manufacturer === "mercedes-benz"' } const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: 'xxx' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('error when converting message data') expect(result).to.equal(false) next() } testPermission(permissions, message, 'user', null, callback) }) }) describe('loads permissions repeatedly', () => { let permission it('creates the permission', async () => { permission = new ConfigPermission({ permissions: getBasePermissions() }, services, config) permission.setRecordHandler({ removeRecordRequest: () => {}, runWhenRecordStable: (r, c) => { c(r) } }) await permission.whenReady() }) it('requests permissions initially, causing a lookup', (next) => { const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: 'some-data' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } permission.canPerformAction('some-user', message, callback) }) it('requests permissions a second time, causing a cache retrieval', (next) => { const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: 'some-data' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } permission.canPerformAction('some-user', message, callback) }) }) ================================================ FILE: src/services/permission/valve/config-permission-create.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as C from '../../../constants' import { getBasePermissions } from '../../../test/helper/test-helper' import * as testHelper from '../../../test/helper/test-helper' const options = testHelper.getDeepstreamPermissionOptions() const services = options.services const testPermission = testHelper.testPermission(options) describe('allows to create a record without providing data, but denies updating it', () => { const permissions = getBasePermissions() permissions.record['some/*'] = { write: 'data.name === "Wolfram"' } beforeEach(() => { services.cache.set('some/tests', 0, {}, () => {}) }) it('allows creating the record', () => { const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.SUBSCRIBECREATEANDREAD, name: 'some/tests' } expect(testPermission(permissions, message)).to.equal(true) }) it('denies update', () => { const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'some/tests', version: 2, data: '{"other":"data"}' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(false) } testPermission(permissions, message, 'some-user', null, callback) }) it('denies patch', () => { const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH, name: 'some/tests', version: 2, path: 'apath', data: '"aValue"' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(false) } testPermission(permissions, message, 'some-user', null, callback) }) }) ================================================ FILE: src/services/permission/valve/config-permission-cross-reference.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as C from '../../../constants' import { getBasePermissions } from '../../../test/helper/test-helper' import * as testHelper from '../../../test/helper/test-helper' const noop = function () {} const options = testHelper.getDeepstreamPermissionOptions() const services = options.services const testPermission = testHelper.testPermission(options) const lastError = function () { return services.logger.logSpy.lastCall.args[2] } describe('permission handler loads data for cross referencing', () => { before((next) => { services.cache.set('item/doesExist', 0, { isInStock: true }, next) }) it('retrieves an existing record from a synchronous cache', (next) => { const permissions = getBasePermissions() services.cache.nextGetWillBeSynchronous = true permissions.record['purchase/$itemId'] = { read: '_("item/" + $itemId).isInStock === true' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'purchase/doesExist' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) expect(services.cache.lastRequestedKey).to.equal('item/doesExist') next() } testPermission(permissions, message, null, null, onDone) }) it('retrieves two records from the cache for cross referencing purposes', (next) => { const permissions = getBasePermissions() services.cache.set('item/itemA', 0, { isInStock: true }, noop) services.cache.set('item/itemB', 0, { isInStock: false }, noop) services.cache.nextGetWillBeSynchronous = false permissions.record['purchase/$itemId'] = { read: '_("item/" + $itemId).isInStock === true && _("item/itemB").isInStock === false' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'purchase/itemA' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } testPermission(permissions, message, null, null, onDone) }) it('retrieves and expects a non existing record', (next) => { const permissions = getBasePermissions() services.cache.nextGetWillBeSynchronous = false permissions.record['purchase/$itemId'] = { read: '_("doesNotExist") !== null && _("doesNotExist").isInStock === true' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'purchase/itemA' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) it('gets a non existant record thats not expected', (next) => { const permissions = getBasePermissions() services.cache.nextGetWillBeSynchronous = false permissions.record['purchase/$itemId'] = { read: '_("doesNotExist").isInStock === true' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'purchase/itemA' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('TypeError') .and.contain('null') .and.contain('isInStock') expect(result).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) it('mixes old data and cross references', (next) => { const permissions = getBasePermissions() services.cache.reset() services.cache.set('userA', 0, { firstname: 'Egon' }, noop) services.cache.set('userB', 0, { firstname: 'Mike' }, noop) services.cache.nextGetWillBeSynchronous = false permissions.record.userA = { read: 'oldData.firstname === "Egon" && _("userB").firstname === "Mike"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'userA' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) expect(services.cache.getCalls.length).to.equal(2) expect(services.cache.hadGetFor('userA')).to.equal(true) expect(services.cache.hadGetFor('userB')).to.equal(true) setTimeout(next, 200) } testPermission(permissions, message, null, null, onDone) }) it('retrieves keys from name', (next) => { const permissions = getBasePermissions() services.cache.set('some-event', 0, { firstname: 'Joe' }, noop) permissions.event['some-event'] = { publish: '_(name).firstname === "Joe"' } const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } testPermission(permissions, message, 'username', null, callback) }) it('retrieves keys from variables', (next) => { const permissions = getBasePermissions() services.cache.set('userX', 0, { firstname: 'Joe' }, noop) permissions.event['some-event'] = { publish: '_(data.owner).firstname === "Joe"' } const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: '{"owner":"userX"}' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } testPermission(permissions, message, 'username', null, callback) }) it('retrieves keys from variables again', (next) => { const permissions = getBasePermissions() services.cache.set('userX', 0, { firstname: 'Mike' }, noop) permissions.event['some-event'] = { publish: '_(data.owner).firstname === "Joe"' } const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: '{"owner":"userX"}' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(false) next() } testPermission(permissions, message, 'username', null, callback) }) it('handles load errors', (next) => { const permissions = getBasePermissions() permissions.event['some-event'] = { publish: '_("bla") < 10' } services.cache.nextOperationWillBeSuccessful = false const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'some-event', data: '{"price":15}' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(C.RECORD_ACTION.RECORD_LOAD_ERROR) expect(result).to.equal(false) next() } testPermission(permissions, message, 'username', null, callback) }) }) ================================================ FILE: src/services/permission/valve/config-permission-load.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import {spy, assert} from 'sinon' import * as C from '../../../constants' import { ConfigPermission } from './config-permission' import * as testHelper from '../../../test/helper/test-helper' import { PromiseDelay } from '../../../utils/utils' import { EVENT } from '@deepstream/types' import * as invalidPermissionConfig from '../../../test/config/invalid-permission-conf.json' import * as noPrivateEventsConfig from '../../../test/config/no-private-events-permission-config.json' const { config, services } = testHelper.getDeepstreamPermissionOptions() describe('permission handler loading', () => { beforeEach(() => { services.logger.fatal = spy() }) describe('permission handler is initialised correctly', () => { it('loads a valid config file upon initialization', async () => { const permission = new ConfigPermission({ permissions: testHelper.getBasePermissions(), cacheEvacuationInterval: 60000, maxRuleIterations: 10 }, services, config) assert.notCalled(services.logger.fatal) await permission.whenReady() }) it('fails to load maxRuleIterations less than zero initialization', async () => { const permission = new ConfigPermission({ permissions: testHelper.getBasePermissions(), cacheEvacuationInterval: 60000, maxRuleIterations: 0 }, services, config) assert.calledOnce(services.logger.fatal) assert.calledWithExactly(services.logger.fatal, EVENT.PLUGIN_INITIALIZATION_ERROR, 'Maximum rule iteration has to be at least one') }) it('fails when loading an invalid config file upon initialization', async () => { // tslint:disable-next-line: no-unused-expression new ConfigPermission({ permissions: invalidPermissionConfig, cacheEvacuationInterval: 60000, maxRuleIterations: 10 }, services, config) await PromiseDelay(10) assert.calledOnce(services.logger.fatal) assert.calledWithExactly(services.logger.fatal, EVENT.PLUGIN_INITIALIZATION_ERROR, 'invalid permission config - empty section "record"') }) }) describe('it loads a new config during runtime', () => { let permission: ConfigPermission const onError = spy() it('loads a valid config file upon initialization', async () => { permission = new ConfigPermission({ permissions: testHelper.getBasePermissions(), cacheEvacuationInterval: 60000, maxRuleIterations: 10 }, services, config) await permission.whenReady() }) it('allows publishing of a private event', (next) => { const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'private/event', data: 'somedata' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } permission.canPerformAction('some-user', message, callback) }) it('denies publishing of a private event', (next) => { expect(onError).to.have.callCount(0) const message = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'private/event', data: 'somedata' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(false) next() } permission.useConfig(noPrivateEventsConfig) permission.canPerformAction('some-user', message, callback, {}) }) }) }) ================================================ FILE: src/services/permission/valve/config-permission-nested-cross-reference.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as testHelper from '../../../test/helper/test-helper' import * as C from '../../../constants' const noop = function () {} const options = testHelper.getDeepstreamPermissionOptions() const services = options.services const testPermission = testHelper.testPermission(options) const lastError = function () { return services.logger.logSpy.lastCall.args[2] } describe('permission handler loads data for cross referencing', () => { it('retrieves data for a nested cross references', (next) => { const permissions = testHelper.getBasePermissions() services.cache.set('thing/x', 0, { ref: 'y' }, noop) services.cache.set('thing/y', 0, { is: 'it' }, noop) services.cache.nextGetWillBeSynchronous = false permissions.record['test-record'] = { read: '_( "thing/" + _( "thing/x" ).ref ).is === "it"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'test-record' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) expect(services.cache.getCalls.length).to.equal(2) expect(services.cache.hadGetFor('thing/x')).to.equal(true) expect(services.cache.hadGetFor('thing/y')).to.equal(true) expect(services.cache.hadGetFor('thing/z')).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) it('erors for undefined fields in crossreferences', (next) => { const permissions = testHelper.getBasePermissions() services.cache.set('thing/x', 0, { ref: 'y' }, noop) services.cache.set('thing/y', 0, { is: 'it' }, noop) services.cache.nextGetWillBeSynchronous = false permissions.record['test-record'] = { read: '_( "thing/" + _( "thing/x" ).doesNotExist ).is === "it"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'test-record' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('TypeError') .and.contain('undefined') .and.contain('is') expect(result).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) it('can use the same cross reference multiple times', (next) => { const permissions = testHelper.getBasePermissions() services.cache.reset() services.cache.set('user', 0, { firstname: 'Wolfram', lastname: 'Hempel' }, noop) services.cache.nextGetWillBeSynchronous = false permissions.record['test-record'] = { read: '_("user").firstname === "Wolfram" && _("user").lastname === "Hempel"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'test-record' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) expect(services.cache.getCalls.length).to.equal(1) expect(services.cache.hadGetFor('user')).to.equal(true) next() } testPermission(permissions, message, null, null, onDone) }) it('supports nested references to the same record', (next) => { const permissions = testHelper.getBasePermissions() services.cache.reset() services.cache.set('user', 0, { ref: 'user', firstname: 'Egon' }, noop) services.cache.nextGetWillBeSynchronous = false permissions.record['test-record'] = { read: '_(_("user").ref).firstname === "Egon"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'test-record' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) expect(services.cache.getCalls.length).to.equal(1) expect(services.cache.hadGetFor('user')).to.equal(true) next() } testPermission(permissions, message, null, null, onDone) }) it('errors for objects as cross reference arguments', (next) => { const permissions = testHelper.getBasePermissions() services.cache.reset() services.cache.set('user', 0, { ref: { bla: 'blub' } }, noop) services.cache.nextGetWillBeSynchronous = false permissions.record['test-record'] = { read: '_(_("user").ref).firstname === "Egon"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'test-record' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('crossreference got unsupported type object') expect(result).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) it('prevents nesting beyond limit', (next) => { const permissions = testHelper.getBasePermissions() services.cache.reset() services.cache.set('a', 0, 'a', noop) services.cache.set('ab', 0, 'b', noop) services.cache.set('abc', 0, 'c', noop) services.cache.set('abcd', 0, 'd', noop) services.cache.set('abcde', 0, 'e', noop) services.cache.nextGetWillBeSynchronous = false permissions.record['test-record'] = { read: '_(_(_(_(_("a")+"b")+"c")+"d")+"e")' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.READ, name: 'test-record' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('Exceeded max iteration count') expect(result).to.equal(false) expect(services.cache.getCalls.length).to.equal(3) next() } testPermission(permissions, message, null, null, onDone) }) }) ================================================ FILE: src/services/permission/valve/config-permission-other.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import { getBasePermissions } from '../../../test/helper/test-helper' import * as C from '../../../constants' import * as testHelper from '../../../test/helper/test-helper' import { ConfigPermission } from './config-permission'; const { config, services } = testHelper.getDeepstreamPermissionOptions() const testPermission = testHelper.testPermission({ config, services }) describe('supports spaces after variables and escaped quotes', () => { it('errors for read with data', () => { const permissions = getBasePermissions() permissions.record.someUser = { read: 'data.firstname === "Yasser"', write: 'data .firstname === "Yasser"' } try { // tslint:disable-next-line:no-unused-expression new ConfigPermission(config, services, permissions) } catch (e) { expect(e.toString()).to.contain('invalid permission config - rule read for record does not support data') } }) it('allows yasser', (next) => { const permissions = getBasePermissions() permissions.record.someUser = { write: 'data .firstname === "Yasser"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'someUser', version: 1, data: '{"firstname":"Yasser"}' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } testPermission(permissions, message, 'Yasser', null, callback) }) it('denies Wolfram', (next) => { const permissions = getBasePermissions() permissions.record.someUser = { write: 'data .firstname === "Yasser"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.UPDATE, name: 'someUser', version: 1, data: '{"firstname":"Wolfram"}' } const callback = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(false) next() } testPermission(permissions, message, 'Yasser', null, callback) }) }) ================================================ FILE: src/services/permission/valve/config-permission-record-patch.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as C from '../../../constants' import * as testHelper from '../../../test/helper/test-helper' const noop = function () {} const options = testHelper.getDeepstreamPermissionOptions() const services = options.services const testPermission = testHelper.testPermission(options) const lastError = function () { return services.logger.logSpy.lastCall.args[2] } describe('constructs data for patch message validation', () => { it('fails to set incorrect data', (next) => { const permissions = testHelper.getBasePermissions() services.cache.nextGetWillBeSynchronous = false permissions.record['user/wh'] = { write: 'data.firstname === "Wolfram" && data.lastname === "Hempel"' } services.cache.set('user/wh', 0, { firstname: 'Wolfram', lastname: 'Something Else' }, noop) const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH, name: 'user/wh', version: 123, path: 'lastname', data: '"Miller"' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) it('succeeds if both old and new data is correct', (next) => { const permissions = testHelper.getBasePermissions() services.cache.nextGetWillBeSynchronous = false permissions.record['user/wh'] = { write: 'data.firstname === "Wolfram" && data.lastname === "Hempel"' } services.cache.set('user/wh', 1, { firstname: 'Wolfram', lastname: 'Something Else' }, noop) const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH, name: 'user/wh', version: 123, path: 'lastname', data: '"Hempel"' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(error).to.equal(null) expect(result).to.equal(true) next() } testPermission(permissions, message, null, null, onDone) }) it('errors if the patch message has data with an invalid json', (next) => { const permissions = testHelper.getBasePermissions() services.cache.nextGetWillBeSynchronous = false permissions.record['user/wh'] = { write: 'data.firstname === "Wolfram" && data.lastname === "Hempel"' } const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH, name: 'user/wh', version: 123, path: 'lastname', data: '[' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('SyntaxError: Unexpected end of JSON input') expect(result).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) it('returns false if patch if for a non existing record', (next) => { const permissions = testHelper.getBasePermissions() services.cache.nextGetWillBeSynchronous = false permissions.record['*'].write = 'data.lastname === "Blob"' const message = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH, name: 'somerecord', version: 1, path: 'lastname', data: '"Hempel"' } const onDone = function (socketWrapper, msg, passItOn, error, result) { expect(lastError()).to.contain('Tried to apply patch to non-existant record somerecord') expect(result).to.equal(false) next() } testPermission(permissions, message, null, null, onDone) }) }) ================================================ FILE: src/services/permission/valve/config-permission.ts ================================================ import * as configCompiler from './config-compiler' import * as configValidator from './config-validator' import RuleApplication from './rule-application' import RuleCache from './rule-cache' import * as rulesMap from './rules-map' import { Message, RECORD_ACTION, EVENT_ACTION, RPC_ACTION, PRESENCE_ACTION } from '../../../constants' import RecordHandler from '../../../handlers/record/record-handler' import { DeepstreamPlugin, DeepstreamPermission, ValveConfig, DeepstreamServices, DeepstreamConfig, PermissionCallback, SocketWrapper, EVENT, ValveSchema } from '@deepstream/types' const UNDEFINED = 'undefined' export type RuleType = string export type ValveSection = string export class ConfigPermission extends DeepstreamPlugin implements DeepstreamPermission { public description = 'Valve Permissions' private ruleCache: RuleCache private permissions: any private recordHandler: RecordHandler | null = null private logger = this.services.logger.getNameSpace('PERMISSION') /** * A permission handler that reads a rules config YAML or JSON, validates * its contents, compiles it and executes the permissions that it contains * against every incoming message. * * This is the standard permission handler that deepstream exposes, in conjunction * with the default permission.yml it allows everything, but at the same time provides * a convenient starting point for permission declarations. */ constructor (private permissionOptions: ValveConfig, private services: Readonly, private config: Readonly) { super() this.ruleCache = new RuleCache(this.permissionOptions) const maxRuleIterations = permissionOptions.maxRuleIterations if (maxRuleIterations !== undefined && maxRuleIterations < 1) { this.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, 'Maximum rule iteration has to be at least one') } this.useConfig(permissionOptions.permissions) } public async whenReady (): Promise { } public async close () { this.ruleCache.close() } /** * Will be invoked with the initialized recordHandler instance by deepstream.io */ public setRecordHandler (recordHandler: RecordHandler): void { this.recordHandler = recordHandler } /** * Validates and compiles a loaded config. This can be called as the result * of a config being passed to the permission service upon initialization, * as a result of loadConfig or at runtime * * CLI useConfig */ public useConfig (permissions: ValveSchema): void { const validationResult = configValidator.validate(permissions) if (validationResult !== true) { this.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, `invalid permission config - ${validationResult}`) return } this.permissions = configCompiler.compile(permissions) this.ruleCache.reset() } /** * Implements the permission service's canPerformAction interface * method * * This is the main entry point for permissionOperations and will * be called for every incoming message. This method executes four steps * * - Check if the incoming message conforms to basic specs * - Check if the incoming message requires permissions * - Load the applicable permissions * - Apply them */ public canPerformAction (socketWrapper: SocketWrapper, message: Message, callback: PermissionCallback, passItOn: any) { const ruleSpecification = rulesMap.getRulesForMessage(message) if (ruleSpecification === null) { callback(socketWrapper, message, passItOn, null, true) return } const ruleData = this.getCompiledRulesForName(message.name!, ruleSpecification) if (!ruleData) { callback(socketWrapper, message, passItOn, null, false) return } // tslint:disable-next-line new RuleApplication({ recordHandler: this.recordHandler!, socketWrapper, userId: socketWrapper.userId, serverData: socketWrapper.serverData, path: ruleData, ruleSpecification, message, action: ruleSpecification.action as (RECORD_ACTION | EVENT_ACTION | RPC_ACTION | PRESENCE_ACTION), regexp: ruleData.regexp, rule: ruleData.rule, name: message.name!, callback, passItOn, logger: this.logger, permissionOptions: this.permissionOptions, config: this.config, services: this.services, }) } /** * Evaluates the rules within a section and returns the matching rule for a path. * Takes basic specificity (as deduced from the path length) into account and * caches frequently used rules for faster access */ private getCompiledRulesForName (name: string, ruleSpecification: any): any { const compiledRules = this.ruleCache.get(ruleSpecification.section, name, ruleSpecification.type) if (compiledRules) { return compiledRules } const sections = this.permissions[ruleSpecification.section] let pathLength = 0 let result: any = null for (let i = 0; i < sections.length; i++) { const { rules, path, regexp } = sections[i] if (typeof rules[ruleSpecification.type] !== UNDEFINED && path.length >= pathLength && regexp.test(name)) { pathLength = path.length result = { path, regexp, rule: rules[ruleSpecification.type], } } } if (result) { this.ruleCache.set(ruleSpecification.section, name, ruleSpecification.type, result) } return result } } ================================================ FILE: src/services/permission/valve/config-schema.ts ================================================ import { ConfigSchema } from '@deepstream/types' export const SCHEMA: ConfigSchema = { record: { write: true, read: true, create: true, delete: true, listen: true, notify: true, }, event: { publish: true, subscribe: true, listen: true, }, rpc: { provide: true, request: true, }, presence: { allow: true, }, } ================================================ FILE: src/services/permission/valve/config-validator.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as configValidator from './config-validator' import * as testHelper from '../../../test/helper/test-helper' describe('it validates permission.json files', () => { it('exposes a validate method', () => { expect(typeof configValidator.validate).to.equal('function') }) it('validates a basic configuration', () => { expect(configValidator.validate(testHelper.getBasePermissions())).to.equal(true) }) it('validates the type of the configuration', () => { expect(configValidator.validate()).to.equal('config should be an object literal, but was of type undefined') expect(configValidator.validate('bla')).to.equal('config should be an object literal, but was of type string') expect(configValidator.validate(testHelper.getBasePermissions())).to.equal(true) }) it('fails if a top level key is missing', () => { const conf = testHelper.getBasePermissions() delete conf.record expect(configValidator.validate(conf)).to.equal('missing configuration section "record"') }) it('fails if an unknown top level key is added', () => { const conf = testHelper.getBasePermissions() conf.bogus = {} expect(configValidator.validate(conf)).to.equal('unexpected configuration section "bogus"') }) it('fails for empty sections', () => { const conf = testHelper.getBasePermissions() conf.rpc = {} expect(configValidator.validate(conf)).to.equal('empty section "rpc"') }) it('fails if no root permissions are specified', () => { const conf = testHelper.getBasePermissions() conf.rpc = { bla: { request: 'user.id === $userId' } } expect(configValidator.validate(conf)).to.equal('missing root entry "*" for section rpc') }) it('fails for invalid paths', () => { const conf = testHelper.getBasePermissions() conf.record.a$$x = {} expect(configValidator.validate(conf)).to.equal('invalid variable name $$ for path a$$x in section record') }) it('fails for invalid rule types', () => { const conf = testHelper.getBasePermissions() conf.rpc.somepath = { write: 'a === b' } expect(configValidator.validate(conf)).to.equal('unknown rule type write in section rpc') }) it('fails for invalid rules', () => { const conf = testHelper.getBasePermissions() conf.record.somepath = { write: 'process.exit()' } expect(configValidator.validate(conf)).to.equal('function exit is not supported') }) // it( 'fails for rules referencing data that dont support it', function(){ // var conf = testHelper.getBasePermissions(); // conf.record.somepath = { read: 'data.firstname === "Egon"' }; // expect( configValidator.validate( conf ) ).to.equal( // 'data is not supported for record read - did you mean "oldData"?' \ // ); // }); }) ================================================ FILE: src/services/permission/valve/config-validator.ts ================================================ import { SCHEMA } from './config-schema' import * as pathParser from './path-parser' import * as ruleParser from './rule-parser' import { Dictionary } from 'ts-essentials' import { DeepstreamConfig } from '@deepstream/types' const validationSteps: Dictionary<(config: DeepstreamConfig) => boolean | string> = {} /** * Validates a configuration object. This method runs through multiple * individual validation steps. If any of them returns false, * the validation fails */ export const validate = function (config: any) { let validationStepResult let key for (key in validationSteps) { validationStepResult = validationSteps[key](config) if (validationStepResult !== true) { return validationStepResult } } return true } /** * Checks if the configuration is an object */ validationSteps.isValidType = function (config: any) { if (typeof config === 'object') { return true } return `config should be an object literal, but was of type ${typeof config}` } /** * Makes sure all sections (record, event, rpc) are present */ validationSteps.hasRequiredTopLevelKeys = function (config: any) { for (const key in SCHEMA) { if (typeof config[key] !== 'object') { return `missing configuration section "${key}"` } } return true } /** * Makes sure no unsupported sections were added */ validationSteps.doesNotHaveAdditionalTopLevelKeys = function (config: any) { for (const key in config) { if (typeof SCHEMA[key] === 'undefined') { return `unexpected configuration section "${key}"` } } return true } /** * Checks if the configuration contains valid path definitions */ validationSteps.doesOnlyContainValidPaths = function (config: any) { let key let path let result for (key in SCHEMA) { // Check empty if (Object.keys(config[key]).length === 0) { return `empty section "${key}"` } // Check valid for (path in config[key]) { result = pathParser.validate(path) if (result !== true) { return `${result} for path ${path} in section ${key}` } } } return true } /** * Each section must specify a generic permission ("*") that * will be applied if no other permission is applicable */ validationSteps.doesHaveRootEntries = function (config: any) { let sectionName for (sectionName in SCHEMA) { if (!config[sectionName]['*']) { return `missing root entry "*" for section ${sectionName}` } } return true } /** * Runs the rule validator against every rule in each section */ validationSteps.hasValidRules = function (config: any) { let path let ruleType let section let validationResult for (section in config) { for (path in config[section]) { for (ruleType in config[section][path]) { if (SCHEMA[section][ruleType] !== true) { return `unknown rule type ${ruleType} in section ${section}` } validationResult = ruleParser.validate(config[section][path][ruleType], section, ruleType) if (validationResult !== true) { return validationResult } } } } return true } ================================================ FILE: src/services/permission/valve/path-parser.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as pathParser from './path-parser' const isRegExp = function (val) { return typeof val === 'object' && typeof val.test === 'function' } describe('validates paths in permission.json files', () => { it('exposes a validate method', () => { expect(typeof pathParser.validate).to.equal('function') }) it('accepts a valid path', () => { expect(pathParser.validate('game-comment/$gameId/*')).to.equal(true) }) it('rejects none strings', () => { expect(pathParser.validate(3 as any)).to.equal('path must be a string') }) it('rejects empty strings', () => { expect(pathParser.validate('')).to.equal('path can\'t be empty') }) it('rejects paths starting with /', () => { expect(pathParser.validate('/bla')).to.equal('path can\'t start with /') }) it('rejects paths with invalid variable names', () => { expect(pathParser.validate('bla/$-')).to.equal('invalid variable name $-') expect(pathParser.validate('bla/$$aa')).to.equal('invalid variable name $$') }) }) describe('parses valid paths in permission.json files', () => { it('exposes a parse method', () => { expect(typeof pathParser.parse).to.equal('function') }) it('parses a simple, valid path', () => { const result = pathParser.parse('i-am-valid') expect(isRegExp(result.regexp)).to.equal(true) expect(result.regexp.toString()).to.equal('/^i-am-valid$/') expect(result.variables.length).to.equal(0) }) it('parses a valid path with a wildcard', () => { const result = pathParser.parse('i-am-valid/*') expect(isRegExp(result.regexp)).to.equal(true) expect(result.regexp.toString()).to.equal('/^i-am-valid\\/.*$/') expect(result.variables.length).to.equal(0) }) it('parses a valid path with a variable', () => { const result = pathParser.parse('game-score/$gameId') expect(isRegExp(result.regexp)).to.equal(true) expect(result.regexp.toString()).to.equal('/^game-score\\/([^\/]+)$/') expect(result.variables).to.deep.equal(['$gameId']) }) it('parses a valid path with multiple variables', () => { const result = pathParser.parse('game-comment/$gameId/$userId/$commentId') expect(isRegExp(result.regexp)).to.equal(true) expect(result.regexp.toString()).to.equal('/^game-comment\\/([^/]+)\\/([^/]+)\\/([^/]+)$/') expect(result.variables).to.deep.equal(['$gameId', '$userId', '$commentId']) }) it('parses a path with a mix of variables and wildcards', () => { const result = pathParser.parse('$recordName/*') expect(isRegExp(result.regexp)).to.equal(true) expect(result.regexp.toString()).to.equal('/^([^/]+)\\/.*$/') expect(result.variables).to.deep.equal(['$recordName']) }) }) describe('applies regexp to paths', () => { it('applies a regexp to a simple path', () => { const path = 'public/*' const result = pathParser.parse(path) expect(result.regexp.test('public/details/info')).to.equal(true) expect(result.regexp.test('private/details/info')).to.equal(false) }) it('applies a regexp and extracts a variable from a simple path', () => { const path = 'private/$userId' const name = 'private/userA' const result = pathParser.parse(path) expect(result.regexp.test(name)).to.equal(true) const r = name.match(result.regexp) || false expect(r[1]).to.equal('userA') }) it('applies a regexp and extracts variables from a more complex path', () => { const path = 'private/$userId/*/$anotherId' const name = 'private/userA/blabla/14' const result = pathParser.parse(path) expect(result.regexp.test(name)).to.equal(true) const r = name.match(result.regexp) || [] expect(r.join(',')).to.deep.equal('private/userA/blabla/14,userA,14') const reject = name.match(result.regexp) || false expect(reject[1]).to.equal('userA') }) }) ================================================ FILE: src/services/permission/valve/path-parser.ts ================================================ const WILDCARD_REGEXP = /\*/g const WILDCARD_STRING = '.*' const VARIABLE_REGEXP = /(\$[a-zA-Z0-9]+)/g const VARIABLE_STRING = '([^/]+)' const INVALID_VARIABLE_REGEXP = /(\$[^a-zA-Z0-9])/ /** * Checks a path for type and basic syntax errors * * @param {String} path The path as specified in permission.json * * @public * @returns {String|Boolean} true if path is valid, string error message if not */ export const validate = (path: string): string | boolean => { if (typeof path !== 'string') { return 'path must be a string' } if (path.length === 0) { return 'path can\'t be empty' } if (path[0] === '/') { return 'path can\'t start with /' } const invalidVariableNames = path.match(INVALID_VARIABLE_REGEXP) if (invalidVariableNames !== null) { return `invalid variable name ${invalidVariableNames[0]}` } return true } /** * Parses a path and returns a regexp matcher with capture groups for * variable names and a list of variable names in the same order. * The path is assumed to be valid when its passed to this method t */ export const parse = (path: string): any => { const variables: string[] = [] let regExp = path.replace(WILDCARD_REGEXP, WILDCARD_STRING) regExp = regExp.replace(VARIABLE_REGEXP, (variableName) => { variables.push(variableName) return VARIABLE_STRING }) return { variables, path, regexp: new RegExp(`^${regExp}$`), } } ================================================ FILE: src/services/permission/valve/rule-application.ts ================================================ import { EOL } from 'os' import { Message, RECORD_ACTION, PRESENCE_ACTION, EVENT_ACTION, RPC_ACTION, RecordData, TOPIC, RecordWriteMessage } from '../../../constants' import { PermissionCallback, ValveConfig, SocketWrapper, DeepstreamConfig, DeepstreamServices, NamespacedLogger, EVENT } from '@deepstream/types' import { recordRequest } from '../../../handlers/record/record-request' import RecordHandler from '../../../handlers/record/record-handler' import { setValue } from '../../../utils/json-path' const OPEN = 'open' const LOADING = 'loading' const ERROR = 'error' const UNDEFINED = 'undefined' const STRING = 'string' interface RuleApplicationParams { userId: string serverData: any path: string ruleSpecification: any message: Message action: RECORD_ACTION | PRESENCE_ACTION | EVENT_ACTION | RPC_ACTION regexp: RegExp rule: any name: string permissionOptions: ValveConfig logger: NamespacedLogger recordHandler: RecordHandler socketWrapper: SocketWrapper config: DeepstreamConfig services: DeepstreamServices, callback: PermissionCallback, passItOn: any, } export default class RuleApplication { private isDestroyed: boolean = false private runScheduled: boolean = false private pathVars: any private user: any private readonly maxIterationCount: number private readonly recordsData = new Map() private iterations: number /** * This class handles the evaluation of a single rule. It creates * the required variables, injects them into the rule function and * runs the function recoursively until either all cross-references, * references to old or new data is loaded, it errors or the maxIterationCount * limit is exceeded */ constructor (private params: RuleApplicationParams) { this.maxIterationCount = this.params.permissionOptions.maxRuleIterations this.run = this.run.bind(this) this.crossReference = this.crossReference.bind(this) this.createNewRecordRequest = this.createNewRecordRequest.bind(this) this.pathVars = this.getPathVars() this.user = this.getUser() this.iterations = 0 this.run() } /** * Runs the rule function. This method is initially called when this class * is constructed and recoursively from thereon whenever the loading of a record * is completed */ private run (): void { this.runScheduled = false this.iterations++ if (this.isDestroyed) { return } if (this.iterations > this.maxIterationCount) { this.onRuleError('Exceeded max iteration count') return } if (this.isDestroyed) { return } const args = this.getArguments() let result try { result = this.params.rule.fn.apply({}, args) } catch (error) { if (this.isReady()) { this.onRuleError(`${error}`) return } } if (this.isReady()) { this.params.callback(this.params.socketWrapper, this.params.message, this.params.passItOn, null, result) this.destroy() } } /** * Callback if a rule has irrecoverably errored. Rule errors due to unresolved * crossreferences are allowed as long as a loading step is in progress */ private onRuleError (error: string): void { if (this.isDestroyed === true) { return } const errorMsg = `error when executing ${this.params.rule.fn.toString()}${EOL}for ${this.params.name}: ${error.toString()}` this.params.logger.warn(EVENT_ACTION[EVENT_ACTION.MESSAGE_PERMISSION_ERROR], errorMsg, { recordName: this.params.name }) this.params.callback(this.params.socketWrapper, this.params.message, this.params.passItOn, EVENT_ACTION.MESSAGE_PERMISSION_ERROR, false) this.destroy() } /** * Called either asynchronously when data is successfully retrieved from the * cache or synchronously if its already present */ private onLoadComplete (recordName: string, version: number, data: any): void { this.recordsData.set(recordName, data) if (this.isReady()) { this.runScheduled = true process.nextTick(this.run) } } /** * Called whenever a storage or cache retrieval fails. Any kind of error during the * permission process is treated as a denied permission */ private onLoadError (event: any, errorMessage: string, recordName: string, socket: SocketWrapper | null) { this.recordsData.set(recordName, ERROR) const errorMsg = `failed to load record ${this.params.name} for permissioning:${errorMessage}` this.params.logger.error(RECORD_ACTION[RECORD_ACTION.RECORD_LOAD_ERROR], errorMsg, { recordName, socketWrapper: socket }) this.params.callback(this.params.socketWrapper, this.params.message, this.params.passItOn, RECORD_ACTION.RECORD_LOAD_ERROR, false) this.destroy() } /** * Destroys this class and nulls down values to avoid * memory leaks */ private destroy () { this.params.recordHandler.removeRecordRequest(this.params.name) this.isDestroyed = true this.runScheduled = false this.recordsData.clear() // this.params = null // this.crossReference = null // this.currentData = null this.pathVars = null this.user = null } /** * If data.someValue is used in the rule, this method retrieves or loads the * current data. This can mean different things, depending on the type of message * * the data arguments is supported for record read & write, * event publish and rpc request * * for event publish, record update and rpc request, the data is already provided * in the message and doesn't need to be loaded * * for record.patch, only a delta is part of the message. For the full data, the current value * is loaded and the patch applied on top */ private getCurrentData (): any { if (this.params.rule.hasData === false) { return null } const msg = this.params.message let result: any = false if ( (msg.topic === TOPIC.RPC) || (msg.topic === TOPIC.EVENT && msg.data) || (msg.topic === TOPIC.RECORD && msg.action === RECORD_ACTION.UPDATE) ) { result = this.params.socketWrapper.parseData(msg) if (result instanceof Error) { this.onRuleError(`error when converting message data ${result.toString()}`) } else { return msg.parsedData } } else if (msg.topic === TOPIC.RECORD && msg.action === RECORD_ACTION.PATCH) { result = this.getRecordPatchData(msg as RecordWriteMessage) if (result instanceof Error) { this.onRuleError(`error when converting message data ${result.toString()}`) } else { return result } } } /** * Loads the records current data and applies the patch data onto it * to avoid users having to distuinguish between patches and updates */ private getRecordPatchData (msg: RecordWriteMessage): any { if (!this.recordsData) { return } if (!msg.path) { this.params.logger.error(EVENT.ERROR, `Missing path for record patch ${msg.name}`, { message: msg }) return } const currentData = this.recordsData.get(this.params.name) const parseResult = this.params.socketWrapper.parseData(msg) let data if (parseResult instanceof Error) { return parseResult } if (currentData === null) { return new Error(`Tried to apply patch to non-existant record ${msg.name}`) } if (typeof currentData !== UNDEFINED && currentData !== LOADING) { data = JSON.parse(JSON.stringify(currentData)) setValue(data, msg.path, msg.parsedData) return data } this.loadRecord(this.params.name) } /** * Returns or loads the record's previous value. Only supported for record * write and read operations * * If getData encounters an error, the rule application might already be destroyed * at this point */ private getOldData (): any { if (this.isDestroyed === true || this.params.rule.hasOldData === false) { return } if (this.recordsData.has(this.params.name)) { return this.recordsData.get(this.params.name) } this.loadRecord(this.params.name) } /** * Compile the list of arguments that will be injected * into the permission function. This method is called * everytime the permission is run. This allows it to merge * patches and update the now timestamp */ private getArguments (): any[] { return [ this.crossReference, this.user, this.getCurrentData(), this.getOldData(), Date.now(), this.params ? this.params.action : null, this.params ? this.params.name : null ].concat(this.pathVars) } /** * Returns the data for the user variable. This is only done once * per rule as the user is not expected to change */ private getUser (): any { return { isAuthenticated: this.params.userId !== OPEN, id: this.params.userId, data: this.params.serverData, } } /** * Applies the compiled regexp for the path and extracts * the variables that will be made available as $variableName * within the rule * * This is only done once per rule as the path is not expected * to change */ private getPathVars (): string[] { if (!this.params.name) { return [] } const matches = this.params.name.match(this.params.regexp) if (matches) { return matches.slice(1) } else { return [] } } /** * Returns true if all loading operations that are in progress have finished * and no run has been scheduled yet */ private isReady (): boolean { let isLoading = false // @ts-ignore for (const [key, value] of this.recordsData) { if (value === LOADING) { isLoading = true break } } return isLoading === false && this.runScheduled === false } /** * Loads a record with a given name. This will either result in * a onLoadComplete or onLoadError call. This method should only * be called if the record is not already being loaded or present, * but I'll leave the additional safeguards in until absolutely sure. */ private loadRecord (recordName: string): void { const recordData = this.recordsData.get(recordName) if (recordData === LOADING) { return } if (typeof recordData !== UNDEFINED) { this.onLoadComplete(recordName, -1, recordData) return } this.recordsData.set(recordName, LOADING) this.params.recordHandler.runWhenRecordStable( recordName, this.createNewRecordRequest, ) } /** * Load the record data from the cache for permissioning. This method should be * called once the record is stable – meaning there are no remaining writes * waiting to be written to the cache. */ private createNewRecordRequest (recordName: string): void { recordRequest( recordName, this.params.config, this.params.services, null, this.onLoadComplete, this.onLoadError, this ) } /** * This method is passed to the rule function as _ to allow crossReferencing * of other records. Cross-references can be nested, leading to this method * being recoursively called until the either all cross references are loaded * or the rule has finally failed */ private crossReference (recordName: string): any | null { const type = typeof recordName const recordData = this.recordsData.get(recordName) if (type !== UNDEFINED && type !== STRING) { this.onRuleError(`crossreference got unsupported type ${type}`) } else if (type === UNDEFINED || recordName.indexOf(UNDEFINED) !== -1) { return } else if (recordData === LOADING) { return } else if (recordData === null) { return null } else if (typeof recordData === UNDEFINED) { this.loadRecord(recordName) } else { return recordData } } } ================================================ FILE: src/services/permission/valve/rule-cache.spec.ts ================================================ import 'mocha' import { expect } from 'chai' const RuleCache = require('./rule-cache').default describe('loads and retrieves values from the rule cache', () => { let ruleCache it('creates the rule cache', () => { ruleCache = new RuleCache({ cacheEvacuationInterval: 10 }) expect(ruleCache.has('record', '*', 'write')).to.equal(false) }) it('sets a value', () => { ruleCache.set('event', '*', 'write', 'ah') expect(ruleCache.has('event', '*', 'write')).to.equal(true) expect(ruleCache.get('event', '*', 'write')).to.equal('ah') }) it('sets another value', (next) => { ruleCache.set('record', '*', 'write', 'yup') expect(ruleCache.has('record', '*', 'write')).to.equal(true) expect(ruleCache.get('record', '*', 'write')).to.equal('yup') setTimeout(next, 40) }) it('sets two values for different actions', () => { ruleCache.set('record', 'somepath', 'write', true) ruleCache.set('record', 'somepath', 'read', 'bla') expect(ruleCache.has('record', 'somepath', 'write')).to.equal(true) expect(ruleCache.get('record', 'somepath', 'write')).to.equal(true) expect(ruleCache.has('record', 'somepath', 'read')).to.equal(true) expect(ruleCache.get('record', 'somepath', 'read')).to.equal('bla') }) it('has purged the cache in the meantime', () => { expect(ruleCache.has('record', '*', 'write')).to.equal(false) }) it('does not remove an entry thats repeatedly requested', (next) => { ruleCache.set('record', '*', 'write', 'yeah') let count = 0 const interval = setInterval(() => { count++ expect(ruleCache.has('record', '*', 'write')).to.equal(true) expect(ruleCache.get('record', '*', 'write')).to.equal('yeah') if (count >= 10) { clearInterval(interval) next() } }, 10) }) it('removes the entry once it stops being requested', (next) => { expect(ruleCache.has('record', '*', 'write')).to.equal(true) expect(ruleCache.get('record', '*', 'write')).to.equal('yeah') setTimeout(() => { expect(ruleCache.has('record', '*', 'write')).to.equal(false) next() }, 40) }) }) ================================================ FILE: src/services/permission/valve/rule-cache.ts ================================================ import { ValveConfig } from '@deepstream/types' interface CachedRule { rule: string, isUsed: boolean, } export default class RuleCache { private data = new Map() private purgeInterval: NodeJS.Timer /** * This cache stores rules that are frequently used. It removes * unused rules after a preset interval */ constructor (config: ValveConfig) { this.purgeInterval = setInterval(this.purge.bind(this), config.cacheEvacuationInterval) } public close () { clearInterval(this.purgeInterval) } /** * Empties the rulecache completely */ public reset (): void { this.data.clear() } /** * Checks if an entry for a specific rule in a specific section is * present */ public has (section: string, name: string, type: string): boolean { return this.data.has(toKey(section, name, type)) } /** * Resets the usage flag and returns an entry from the cache */ public get (section: string, name: string, type: string): string | undefined { const cache = this.data.get(toKey(section, name, type)) if (cache) { cache.isUsed = true return cache.rule } return undefined } /** * Adds an entry to the cache */ public set (section: string, name: string, type: string, rule: string): void { this.data.set(toKey(section, name, type), { rule, isUsed: true, }) } /** * This method is called repeatedly on an interval, defined by * cacheEvacuationInterval. * * If a rule in the cache has been used in the last interval, it sets its isUsed flag to false. * Whenever the rule is used, the isUsed flag will be set to true * Any rules that haven't been used in the next cycle will be removed from the cache */ private purge () { for (const [key, cache] of this.data) { if (cache.isUsed === true) { cache.isUsed = false } else { this.data.delete(key) } } } } /** * Creates a key from the various set parameters */ function toKey (section: string, name: string, type: string): string { return `${section}_${name}_${type}` } ================================================ FILE: src/services/permission/valve/rule-parser.spec.ts ================================================ import 'mocha' import { expect } from 'chai' const ruleParser = require('./rule-parser') describe('validates rule strings from permissions.json', () => { it('exposes a validate method', () => { expect(typeof ruleParser.validate).to.equal('function') }) it('accepts valid rules', () => { expect(ruleParser.validate('user.id === $userId')).to.equal(true) }) it('rejects non-strings', () => { expect(ruleParser.validate(3)).to.equal('rule must be a string') }) it('rejects empty strings', () => { expect(ruleParser.validate('')).to.equal('rule can\'t be empty') }) it('rejects rules that contain new as a keyword', () => { expect(ruleParser.validate('a new SomeClass')).to.equal('rule can\'t contain the new keyword') expect(ruleParser.validate('a=new SomeClass')).to.equal('rule can\'t contain the new keyword') expect(ruleParser.validate('new SomeClass')).to.equal('rule can\'t contain the new keyword') expect(ruleParser.validate(' new SomeClass')).to.equal('rule can\'t contain the new keyword') expect(ruleParser.validate('16-new Number(3)')).to.equal('rule can\'t contain the new keyword') expect(ruleParser.validate('~new SomeClass')).to.equal('rule can\'t contain the new keyword') }) it('accepts rules that contain new as part of another string or object name', () => { expect(ruleParser.validate('newData.firstname')).to.equal(true) expect(ruleParser.validate('$new = "foo"')).to.equal(true) // TODO also unicode in identifiers // expect(ruleParser.validate('a == "new"')).to.equal(true) // TODO }) it('rejects rules that define user functions', () => { expect(ruleParser.validate('(function (foo) { return foo + 1; })(20)')) .to.equal('rule can\'t contain user functions') expect(ruleParser.validate('(foo => foo + 1)(20)')).to.equal('rule can\'t contain user functions') }) it('rejects rules that call unsupported functions', () => { expect(ruleParser.validate('data.lastname.toUpperCase()', 'record', 'write')).to.equal(true) expect(ruleParser.validate('alert("bobo")')).to.equal('function alert is not supported') expect(ruleParser.validate('alert ("whoops") && console.log("nope")')) .to.equal('function alert is not supported') expect(ruleParser.validate('alert\t("whoops")')).to.equal('function alert is not supported') expect(ruleParser.validate('alert\n("whoops")')).to.equal('function alert is not supported') expect(ruleParser.validate('console["log"]("whoops")')) .to.equal('function log"] is not supported') expect(ruleParser.validate('global["con"+"sole"]["lo" + `g`] ("whoops")')) .to.equal('function g`] is not supported') expect(ruleParser.validate('data.lastname.toUpperCase() && data.lastname.substr(0,3)', 'record', 'write')).to.equal('function substr is not supported') }) it('rejects invalid cross references', () => { expect(ruleParser.validate('_("another-record" + data.userId) === $userId', 'record', 'write')).to.equal(true) }) it('rejects rules that are syntactically invalid', () => { expect(ruleParser.validate('a b')).to.match(/^SyntaxError: Unexpected identifier/) expect(ruleParser.validate('user.id.toUpperCase(')).to.equal("SyntaxError: Unexpected token '}'") }) it('rejects rules that reference old data without it being supported', () => { expect(ruleParser.validate('data.price === 500 && oldData.price < 500', 'event', 'publish')).to.equal('rule publish for event does not support oldData') }) it('rejects rules that reference data without it being supported', () => { expect(ruleParser.validate('user.id === $userId && data.price === 500', 'rpc', 'provide')).to.equal('rule provide for rpc does not support data') }) it('validates a rule referencing data as a property for a type (read) where the injected (root) data is not available', () => { const validatedRule = ruleParser.validate('user.id !== user.data.someUser', 'record', 'read') expect(typeof validatedRule).to.equal('boolean') expect(validatedRule).to.equal(true) }) }) describe('compiles rules into usable objects', () => { it('compiles boolean false', () => { const compiledRule = ruleParser.parse(false, []) expect(compiledRule.fn()).to.equal(false) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(false) expect(compiledRule.hasData).to.equal(false) }) it('compiles boolean true', () => { const compiledRule = ruleParser.parse(true, []) expect(compiledRule.fn()).to.equal(true) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(false) expect(compiledRule.hasData).to.equal(false) }) it('creates executable functions', () => { expect(ruleParser.parse('"bobo"', []).fn()).to.equal('bobo') expect(ruleParser.parse('2+2', []).fn()).to.equal(4) }) it('compiles a simple rule', () => { const compiledRule = ruleParser.parse('user.id !== "open"', []) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(false) expect(compiledRule.hasData).to.equal(false) }) it('compiles a rule referencing data', () => { const compiledRule = ruleParser.parse('user.id !== data.someUser', []) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(false) expect(compiledRule.hasData).to.equal(true) }) it('compiles a rule referencing data followed by a space', () => { const compiledRule = ruleParser.parse('data .firstname === "Yasser"', []) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(false) expect(compiledRule.hasData).to.equal(true) }) it('compiles a rule referencing oldData', () => { const compiledRule = ruleParser.parse('user.id !== oldData.someUser', []) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(true) expect(compiledRule.hasData).to.equal(false) }) it('compiles a rule referencing both data and oldData', () => { const compiledRule = ruleParser.parse('user.id !== data.someUser && oldData.price <= data.price', []) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(true) expect(compiledRule.hasData).to.equal(true) }) it('compiles a rule referencing both data and oldData as well as other records', () => { const compiledRule = ruleParser.parse('_( "private/"+ user.id ) !== data.someUser && oldData.price <= data.price', []) expect(typeof compiledRule.fn).to.equal('function') expect(compiledRule.hasOldData).to.equal(true) expect(compiledRule.hasData).to.equal(true) }) }) ================================================ FILE: src/services/permission/valve/rule-parser.ts ================================================ import * as rulesMap from './rules-map' import { ValveSection, RuleType } from './config-permission' // TODO: any of these are fine inside a string or comment context... const FUNCTION_REGEXP = /([\w]+(?:['"`]\])?)\s*\(/g const USER_FUNCTION_REGEXP = /[^\w$]function[^\w$]|=>/g const NEW_REGEXP = /(^|[^\w$])new[^\w$]/ const OLD_DATA_REGEXP = /(^|[^\w~])oldData[^\w~]/ const DATA_REGEXP = /(^|[^\w.~])data($|[^\w~])/ const SUPPORTED_FUNCTIONS = [ '_', 'startsWith', 'endsWith', 'includes', 'indexOf', 'match', 'toUpperCase', 'toLowerCase', 'trim', ] /** * Validates a rule. Makes sure that the rule is either a boolean or a string, * that it doesn't contain the new keyword or unsupported function invocations * and that it can be compiled into a javascript function */ export const validate = (rule: string | boolean, section: ValveSection, type: RuleType): boolean | string => { if (typeof rule === 'boolean') { return true } if (typeof rule !== 'string') { return 'rule must be a string' } if (rule.length === 0) { return 'rule can\'t be empty' } if (rule.match(NEW_REGEXP)) { return 'rule can\'t contain the new keyword' } if (rule.match(USER_FUNCTION_REGEXP)) { return 'rule can\'t contain user functions' } const functions = rule.match(FUNCTION_REGEXP) let functionName let i // TODO _ cross references are only supported for section record if (functions) { for (i = 0; i < functions.length; i++) { functionName = functions[i].replace(/\s*\($/, '') if (SUPPORTED_FUNCTIONS.indexOf(functionName) === -1) { return `function ${functionName} is not supported` } } } try { // tslint:disable-next-line new Function(rule) } catch (e) { return `${e}` } if (!!rule.match(OLD_DATA_REGEXP) && !rulesMap.supportsOldData(type)) { return `rule ${type} for ${section} does not support oldData` } if (!!rule.match(DATA_REGEXP) && !rulesMap.supportsData(type)) { return `rule ${type} for ${section} does not support data` } return true } /** * Cross References: * * Cross references are denoted with an underscore function _() * They can take path variables: _($someId) * variables from data: _(data.someValue) * or strings: _('user/egon') */ export const parse = (rule: boolean | string, variables: any) => { if (rule === true || rule === false) { return { fn: rule === true ? function () { return true } : function () { return false }, hasOldData: false, hasData: false, } } const ruleObj: any = {} const args = ['_', 'user', 'data', 'oldData', 'now', 'action', 'name'].concat(variables) args.push(`return ${rule};`) ruleObj.fn = Function.apply(null, args) ruleObj.hasOldData = !!rule.match(OLD_DATA_REGEXP) ruleObj.hasData = !!rule.match(DATA_REGEXP) return ruleObj } ================================================ FILE: src/services/permission/valve/rules-map.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import { getRulesForMessage } from './rules-map' import * as C from '../../../constants' describe('returns the applicable rule for a message', () => { it('exposes a getRulesForMessage method', () => { expect(typeof getRulesForMessage).to.equal('function') }) it('returns null for topics without rules', () => { const msg = { topic: C.TOPIC.AUTH } expect(getRulesForMessage(msg)).to.equal(null) }) it('returns null for actions without rules', () => { const msg = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.UNSUBSCRIBE } expect(getRulesForMessage(msg)).to.equal(null) }) it('returns ruletypes for event subscribe messages', () => { const msg = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.SUBSCRIBE } expect(getRulesForMessage(msg)).to.deep.equal({ section: 'event', type: 'subscribe', action: C.EVENT_ACTION.SUBSCRIBE }) }) it('returns ruletypes for record patch messages', () => { const msg = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.PATCH } expect(getRulesForMessage(msg)).to.deep.equal({ section: 'record', type: 'write', action: C.RECORD_ACTION.PATCH }) }) it('returns ruletypes for record notify messages', () => { const msg = { topic: C.TOPIC.RECORD, action: C.RECORD_ACTION.NOTIFY } expect(getRulesForMessage(msg)).to.deep.equal({ section: 'record', type: 'notify', action: C.RECORD_ACTION.NOTIFY }) }) }) ================================================ FILE: src/services/permission/valve/rules-map.ts ================================================ import { Dictionary } from 'ts-essentials' import { TOPIC, RECORD_ACTION, EVENT_ACTION, RPC_ACTION, PRESENCE_ACTION, Message } from '../../../constants' interface RuleType { name: string, data: boolean, oldData: boolean } /** * Different rule types support different features. Generally, all rules can * use cross referencing _() to reference records, but only record writes, incoming events * or RPC requests carry data and only existing records have a concept of oldData */ const RULE_TYPES: Dictionary = { CREATE: { name: 'create', data: false, oldData: false }, READ: { name: 'read', data: false, oldData: true }, WRITE: { name: 'write', data: true, oldData: true }, DELETE: { name: 'delete', data: false, oldData: true }, LISTEN: { name: 'listen', data: false, oldData: false }, NOTIFY: { name: 'notify', data: false, oldData: false }, PUBLISH: { name: 'publish', data: true, oldData: false }, SUBSCRIBE: { name: 'subscribe', data: true, oldData: false }, PROVIDE: { name: 'provide', data: false, oldData: false }, REQUEST: { name: 'request', data: true, oldData: false }, ALLOW: { name: 'allow', data: false, oldData: false }, } /** * This class maps topic / action combinations to applicable * rules. It combines actions of a similar character (e.g. READ, * SNAPSHOT) into high level permissions (e.g. read) * * Lower level permissioning on a per action basis can still be achieved * by virtue of using the action variable within the rule, e.g. * * { * //allow read, but not listen * 'read': 'user.id === $userId && action !== LISTEN' * } */ const RULES_MAP: Dictionary<{ section: string, actions: Dictionary }> = { [TOPIC.RECORD]: { section: 'record', actions: { [RECORD_ACTION.SUBSCRIBE]: RULE_TYPES.READ, [RECORD_ACTION.SUBSCRIBEANDHEAD]: RULE_TYPES.READ, [RECORD_ACTION.SUBSCRIBEANDREAD]: RULE_TYPES.READ, [RECORD_ACTION.READ]: RULE_TYPES.READ, [RECORD_ACTION.HEAD]: RULE_TYPES.READ, [RECORD_ACTION.LISTEN]: RULE_TYPES.LISTEN, [RECORD_ACTION.CREATE]: RULE_TYPES.CREATE, [RECORD_ACTION.UPDATE]: RULE_TYPES.WRITE, [RECORD_ACTION.PATCH]: RULE_TYPES.WRITE, [RECORD_ACTION.NOTIFY]: RULE_TYPES.NOTIFY, [RECORD_ACTION.DELETE]: RULE_TYPES.DELETE, [RECORD_ACTION.ERASE]: RULE_TYPES.DELETE }, }, [TOPIC.EVENT]: { section: 'event', actions: { [EVENT_ACTION.LISTEN]: RULE_TYPES.LISTEN, [EVENT_ACTION.SUBSCRIBE]: RULE_TYPES.SUBSCRIBE, [EVENT_ACTION.EMIT]: RULE_TYPES.PUBLISH, }, }, [TOPIC.RPC]: { section: 'rpc', actions: { [RPC_ACTION.PROVIDE]: RULE_TYPES.PROVIDE, [RPC_ACTION.REQUEST]: RULE_TYPES.REQUEST, }, }, [TOPIC.PRESENCE]: { section: 'presence', actions: { [PRESENCE_ACTION.SUBSCRIBE]: RULE_TYPES.ALLOW, [PRESENCE_ACTION.SUBSCRIBE_ALL]: RULE_TYPES.ALLOW, [PRESENCE_ACTION.QUERY]: RULE_TYPES.ALLOW, [PRESENCE_ACTION.QUERY_ALL]: RULE_TYPES.ALLOW, }, }, } /** * Returns a map of applicable rule-types for a topic * action combination */ export const getRulesForMessage = (message: Message) => { if (RULES_MAP[message.topic] === undefined) { return null } if (RULES_MAP[message.topic].actions[message.action] === undefined) { return null } return { section: RULES_MAP[message.topic].section, type: RULES_MAP[message.topic].actions[message.action].name, action: message.action, } } /** * Returns true if a given rule supports references to incoming data */ export const supportsData = function (type: string): boolean { return RULE_TYPES[type.toUpperCase()].data } /** * Returns true if a given rule supports references to existing data */ export const supportsOldData = function (type: string): boolean { return RULE_TYPES[type.toUpperCase()].oldData } ================================================ FILE: src/services/storage/noop-storage.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import {spy} from 'sinon' import { NoopStorage } from './noop-storage' describe('retuns null for all values', () => { let noopStorage before(() => { noopStorage = new NoopStorage() }) it('has created the Noop Storage', async () => { await noopStorage.whenReady() }) it('tries to retrieve a non-existing value', (done) => { const successCallback = spy() noopStorage.get('firstname', successCallback) setTimeout(() => { expect(successCallback).to.have.callCount(1) expect(successCallback).to.have.been.calledWith(null, -1, null) done() }, 1) }) it('tries to delete a value', (done) => { const successCallback = spy() noopStorage.delete('firstname', successCallback) setTimeout(() => { expect(successCallback).to.have.callCount(1) expect(successCallback).to.have.been.calledWith(null) done() }, 1) }) }) ================================================ FILE: src/services/storage/noop-storage.ts ================================================ import { DeepstreamPlugin, StorageWriteCallback, StorageReadCallback, DeepstreamStorage } from '@deepstream/types' export class NoopStorage extends DeepstreamPlugin implements DeepstreamStorage { public description = 'Noop Storage' public set (key: string, version: number, data: any, callback: StorageWriteCallback) { process.nextTick(() => callback(null)) } public get (key: string, callback: StorageReadCallback) { process.nextTick(() => callback(null, -1, null)) } public delete (key: string, callback: StorageWriteCallback) { process.nextTick(() => callback(null)) } public deleteBulk (key: string[], callback: StorageWriteCallback) { process.nextTick(() => callback(null)) } } ================================================ FILE: src/services/subscription-registry/default-subscription-registry-factory.ts ================================================ import { DefaultSubscriptionRegistry } from './default-subscription-registry' import { DeepstreamConfig, DeepstreamServices, DeepstreamPlugin, SubscriptionRegistryFactory, SubscriptionRegistry } from '@deepstream/types' import { TOPIC } from '../../constants' export class DefaultSubscriptionRegistryFactory extends DeepstreamPlugin implements SubscriptionRegistryFactory { public description: string = 'Subscription Registry' private subscriptionRegistries = new Map() constructor (private pluginOptions: any, private services: Readonly, private config: Readonly) { super() } public getSubscriptionRegistry (topic: TOPIC, clusterTopic: TOPIC) { let subscriptionRegistry = this.subscriptionRegistries.get(topic) if (!subscriptionRegistry) { subscriptionRegistry = new DefaultSubscriptionRegistry(this.pluginOptions, this.services, this.config, topic, clusterTopic) this.subscriptionRegistries.set(topic, subscriptionRegistry) } return subscriptionRegistry } public getSubscriptionRegistries () { return this.subscriptionRegistries } } ================================================ FILE: src/services/subscription-registry/default-subscription-registry.spec.ts ================================================ import 'mocha' import * as sinon from 'sinon' import {expect} from 'chai' import * as C from '../../constants' import * as testHelper from '../../test/helper/test-helper' import { getTestMocks } from '../../test/helper/test-mocks' import { SocketWrapper } from '@deepstream/types' import { DefaultSubscriptionRegistry } from './default-subscription-registry'; const options = testHelper.getDeepstreamOptions() const services = options.services const config = options.config const subscriptionListener = { onSubscriptionMade: () => {}, onSubscriptionRemoved: () => {}, onLastSubscriptionRemoved: () => {}, onFirstSubscriptionMade: () => {}, } let subscriptionRegistry: DefaultSubscriptionRegistry let subscriptionListenerMock let clientA: { socketWrapper: SocketWrapper } let clientB: { socketWrapper: SocketWrapper } let testMocks describe('subscription registry', () => { beforeEach(() => { testMocks = getTestMocks() subscriptionListenerMock = sinon.mock(subscriptionListener) subscriptionRegistry = new DefaultSubscriptionRegistry({}, services, config, C.TOPIC.EVENT, C.TOPIC.EVENT) subscriptionRegistry.setSubscriptionListener(subscriptionListener) clientA = testMocks.getSocketWrapper('client a') clientB = testMocks.getSocketWrapper('client b') }) afterEach(() => { subscriptionListenerMock.verify() clientA.socketWrapperMock.verify() clientB.socketWrapperMock.verify() }) const subscribeMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.SUBSCRIBE, name: 'someName' } const unsubscribeMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.UNSUBSCRIBE, name: 'someName' } const eventMessage = { topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.EMIT, name: 'someName' } describe('subscription-registry manages subscriptions', () => { it('subscribes to names', () => { clientA.socketWrapperMock .expects('sendAckMessage') .once() .withExactArgs(subscribeMessage) subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) // expect(socketWrapperA.socket.lastSendMessage).to.equal(_msg('E|A|S|someName+')) // subscriptionRegistry.sendToSubscribers('someName', fakeEvent('someName', 'SsomeString')) // expect(socketWrapperA.socket.lastSendMessage).to.equal(_msg('E|EVT|someName|SsomeString+')) }) it('doesn\'t subscribe twice to the same name', () => { clientA.socketWrapperMock .expects('sendAckMessage') .once() .withExactArgs(subscribeMessage) clientA.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.MULTIPLE_SUBSCRIPTIONS, originalAction: C.EVENT_ACTION.SUBSCRIBE, name: 'someName' }) subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) expect(services.logger.lastLogEvent).to.equal(C.EVENT_ACTION[C.EVENT_ACTION.MULTIPLE_SUBSCRIPTIONS]) }) it('returns the subscribed socket', () => { subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) expect(subscriptionRegistry.getLocalSubscribers('someName')).to.deep.equal(new Set([clientA.socketWrapper])) }) it('determines if it has subscriptions', () => { subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) expect(subscriptionRegistry.hasLocalSubscribers('someName')).to.equal(true) expect(subscriptionRegistry.hasLocalSubscribers('someOtherName')).to.equal(false) }) it('distributes messages to multiple subscribers', () => { subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientB.socketWrapper) clientA.socketWrapperMock .expects('sendBuiltMessage') .once() clientB.socketWrapperMock .expects('sendBuiltMessage') .once() subscriptionRegistry.sendToSubscribers('someName', eventMessage, true, null) }) it('doesn\'t send message to sender', () => { subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientB.socketWrapper) clientA.socketWrapperMock .expects('sendBuiltMessage') .never() clientB.socketWrapperMock .expects('sendBuiltMessage') .once() subscriptionRegistry.sendToSubscribers('someName', eventMessage, false, clientA.socketWrapper) }) it('unsubscribes', () => { subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) clientA.socketWrapperMock .expects('sendAckMessage') .once() .withExactArgs(unsubscribeMessage) subscriptionRegistry.unsubscribe(subscribeMessage.name, unsubscribeMessage, clientA.socketWrapper) }) it('handles unsubscribes for non existant topics', () => { clientA.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.EVENT, action: C.EVENT_ACTION.NOT_SUBSCRIBED, originalAction: C.EVENT_ACTION.UNSUBSCRIBE, name: 'someName' }) subscriptionRegistry.unsubscribe(subscribeMessage.name, unsubscribeMessage, clientA.socketWrapper) }) it.skip('removes all subscriptions on socket.close', () => { subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) subscriptionRegistry.subscribe(subscribeMessage.name, Object.assign({}, subscribeMessage, { name: 'eventB' }), clientA.socketWrapper) clientA.socketWrapperMock .expects('sendMessage') .never() subscriptionRegistry.sendToSubscribers('nameA', eventMessage, true, null) subscriptionRegistry.sendToSubscribers('nameB', eventMessage, true, null) }) }) describe('subscription-registry allows custom actions to be set', () => { beforeEach(() => { subscriptionRegistry.setAction('subscribe', 'make-aware') subscriptionRegistry.setAction('unsubscribe', 'be-unaware') subscriptionRegistry.setAction('multiple_subscriptions', 'too-aware') subscriptionRegistry.setAction('not_subscribed', 'unaware') }) it('subscribes to names', () => { clientA.socketWrapperMock .expects('sendAckMessage') .once() .withExactArgs({ topic: C.TOPIC.EVENT, action: 'make-aware', name: 'someName' }) subscriptionRegistry.subscribe( 'someName', { topic: C.TOPIC.EVENT, action: 'make-aware', name: 'someName' }, clientA.socketWrapper) }) it('doesn\'t subscribe twice to the same name', () => { clientA.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.EVENT, action: 'too-aware', originalAction: 'make-aware', name: 'someName' }) subscriptionRegistry.subscribe('someName', { topic: C.TOPIC.EVENT, action: 'make-aware', name: 'someName' }, clientA.socketWrapper) subscriptionRegistry.subscribe('someName', { topic: C.TOPIC.EVENT, action: 'make-aware', name: 'someName' }, clientA.socketWrapper) }) it('unsubscribes', () => { subscriptionRegistry.subscribe('someName', { topic: C.TOPIC.EVENT, action: 'make-aware', name: 'someName' }, clientA.socketWrapper) clientA.socketWrapperMock .expects('sendAckMessage') .once() .withExactArgs({ topic: C.TOPIC.EVENT, action: 'be-unaware', name: 'someName' }) subscriptionRegistry.unsubscribe('someName', { topic: C.TOPIC.EVENT, action: 'be-unaware', name: 'someName' }, clientA.socketWrapper) }) it('handles unsubscribes for non existant subscriptions', () => { clientA.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: C.TOPIC.EVENT, action: 'unaware', originalAction: 'be-unaware', name: 'someName' }) subscriptionRegistry.unsubscribe('someName', { topic: C.TOPIC.EVENT, action: 'be-unaware', name: 'someName' }, clientA.socketWrapper) }) }) describe('subscription-registry unbinds all events on unsubscribe', () => { it('subscribes and unsubscribes 30 times', () => { for (let i = 0; i < 30; i++) { subscriptionRegistry.subscribe(subscribeMessage.name, subscribeMessage, clientA.socketWrapper) subscriptionRegistry.unsubscribe(unsubscribeMessage.name, unsubscribeMessage, clientA.socketWrapper) } }) }) }) ================================================ FILE: src/services/subscription-registry/default-subscription-registry.ts ================================================ import { EVENT_ACTION, PRESENCE_ACTION, RECORD_ACTION, RPC_ACTION, TOPIC, MONITORING_ACTION, Message, BulkSubscriptionMessage, STATE_REGISTRY_TOPIC } from '../../constants' import { SocketWrapper, DeepstreamConfig, DeepstreamServices, SubscriptionListener, StateRegistry, SubscriptionRegistry, LOG_LEVEL, EVENT, NamespacedLogger } from '@deepstream/types' interface SubscriptionActions { MULTIPLE_SUBSCRIPTIONS: RECORD_ACTION.MULTIPLE_SUBSCRIPTIONS | EVENT_ACTION.MULTIPLE_SUBSCRIPTIONS | RPC_ACTION.MULTIPLE_PROVIDERS | PRESENCE_ACTION.MULTIPLE_SUBSCRIPTIONS NOT_SUBSCRIBED: RECORD_ACTION.NOT_SUBSCRIBED | EVENT_ACTION.NOT_SUBSCRIBED | RPC_ACTION.NOT_PROVIDED | PRESENCE_ACTION.NOT_SUBSCRIBED SUBSCRIBE: RECORD_ACTION.SUBSCRIBE | EVENT_ACTION.SUBSCRIBE | RPC_ACTION.PROVIDE | PRESENCE_ACTION.SUBSCRIBE UNSUBSCRIBE: RECORD_ACTION.UNSUBSCRIBE | EVENT_ACTION.UNSUBSCRIBE | RPC_ACTION.UNPROVIDE | PRESENCE_ACTION.UNSUBSCRIBE } interface Subscription { name: string sockets: Set } export class DefaultSubscriptionRegistry implements SubscriptionRegistry { private sockets = new Map>() private subscriptions = new Map() private subscriptionListener: SubscriptionListener | null = null private constants: SubscriptionActions private clusterSubscriptions: StateRegistry private actions: any private logger: NamespacedLogger = this.services.logger.getNameSpace('SUBSCRIPTION_REGISTRY') private invalidSockets = new Set() /** * A generic mechanism to handle subscriptions from sockets to topics. * A bit like an event-hub, only that it registers SocketWrappers rather * than functions */ constructor (private pluginConfig: any, private services: Readonly, private config: Readonly, private topic: TOPIC | STATE_REGISTRY_TOPIC, clusterTopic: TOPIC) { switch (topic) { case TOPIC.RECORD: case STATE_REGISTRY_TOPIC.RECORD_LISTEN_PATTERNS: this.actions = RECORD_ACTION break case TOPIC.EVENT: case STATE_REGISTRY_TOPIC.EVENT_LISTEN_PATTERNS: this.actions = EVENT_ACTION break case TOPIC.RPC: this.actions = RPC_ACTION break case TOPIC.PRESENCE: this.actions = PRESENCE_ACTION break case TOPIC.MONITORING: this.actions = MONITORING_ACTION break } this.constants = { MULTIPLE_SUBSCRIPTIONS: this.actions.MULTIPLE_SUBSCRIPTIONS, NOT_SUBSCRIBED: this.actions.NOT_SUBSCRIBED, SUBSCRIBE: this.actions.SUBSCRIBE, UNSUBSCRIBE: this.actions.UNSUBSCRIBE, } this.onSocketClose = this.onSocketClose.bind(this) this.clusterSubscriptions = this.services.clusterStates.getStateRegistry(clusterTopic) if (this.pluginConfig.subscriptionsSanityTimer > 0) { setInterval(this.illegalCleanup.bind(this), this.pluginConfig.subscriptionsSanityTimer) setInterval(() => this.invalidSockets.clear(), this.pluginConfig.subscriptionsSanityTimer * 100) } } public async whenReady () { await this.clusterSubscriptions.whenReady() } public async close () { await this.clusterSubscriptions.whenReady() } /** * Return all the servers that have this subscription. */ public getAllServers (subscriptionName: string): string[] { return this.clusterSubscriptions.getAllServers(subscriptionName) } /** * Return all the servers that have this subscription excluding the current * server name */ public getAllRemoteServers (subscriptionName: string): string[] { const serverNames = this.clusterSubscriptions.getAllServers(subscriptionName) const localServerIndex = serverNames.indexOf(this.config.serverName) if (localServerIndex > -1) { serverNames.splice(serverNames.indexOf(this.config.serverName), 1) } return serverNames } /** * Returns a list of all the topic this registry * currently has subscribers for */ public getNames (): string[] { return this.clusterSubscriptions.getAll() } /** * Returns true if the subscription exists somewhere * in the cluster */ public hasName (subscriptionName: string): boolean { return this.clusterSubscriptions.has(subscriptionName) } /** * This method allows you to customise the SubscriptionRegistry so that it can send * custom events and ack messages back. * For example, when using the ACTIONS.LISTEN, you would override SUBSCRIBE with * ACTIONS.SUBSCRIBE and UNSUBSCRIBE with UNSUBSCRIBE */ public setAction (name: string, value: EVENT_ACTION | RECORD_ACTION | RPC_ACTION): void { (this.constants as any)[name.toUpperCase()] = value } /** * Enqueues a message string to be broadcast to all subscribers. Broadcasts will potentially * be reordered in relation to *other* subscription names, but never in relation to the same * subscription name. */ public sendToSubscribers (name: string, message: Message, noDelay: boolean, senderSocket: SocketWrapper | null, suppressRemote: boolean = false): void { // If the senderSocket is null it means it was received via the message bus if (senderSocket !== null && suppressRemote === false) { this.services.clusterNode.send(message) } const subscription = this.subscriptions.get(name) if (!subscription) { return } const subscribers = subscription.sockets this.services.monitoring.onBroadcast(message, subscribers.size) const serializedMessages: { [index: string]: any } = {} for (const socket of subscribers) { if (socket === senderSocket) { continue } if (!serializedMessages[socket.socketType]) { if (message.parsedData) { delete message.data } this.logger.debug('SEND_TO_SUBSCRIBERS', `encoding ${name} with protocol ${socket.socketType} with data ${JSON.stringify(message)}`) serializedMessages[socket.socketType] = socket.getMessage(message) } this.logger.debug('SEND_TO_SUBSCRIBERS', `sending ${socket.socketType} payload of ${serializedMessages[socket.socketType]}`) socket.sendBuiltMessage!(serializedMessages[socket.socketType], !noDelay) } } /** * Adds a SocketWrapper as a subscriber to a topic */ public subscribeBulk (message: BulkSubscriptionMessage, socket: SocketWrapper, silent?: boolean): void { const length = message.names.length for (let i = 0; i < length; i++) { this.subscribe(message.names[i], message, socket, true) } if (!silent) { socket.sendAckMessage({ topic: message.topic, action: message.action, correlationId: message.correlationId }) } } /** * Adds a SocketWrapper as a subscriber to a topic */ public unsubscribeBulk (message: BulkSubscriptionMessage, socket: SocketWrapper, silent?: boolean): void { message.names!.forEach((name) => { this.unsubscribe(name, message, socket, true) }) if (!silent) { socket.sendAckMessage({ topic: message.topic, action: message.action, correlationId: message.correlationId }) } } /** * Adds a SocketWrapper as a subscriber to a topic */ public subscribe (name: string, message: Message, socket: SocketWrapper, silent?: boolean): void { const subscription = this.subscriptions.get(name) || { name, sockets: new Set() } if (subscription.sockets.size === 0) { this.subscriptions.set(name, subscription) } else if (subscription.sockets.has(socket)) { if (this.logger.shouldLog(LOG_LEVEL.WARN)) { const msg = `repeat subscription to "${name}" by ${socket.userId}` this.logger.warn(EVENT_ACTION[this.constants.MULTIPLE_SUBSCRIPTIONS], msg, { message, socketWrapper: socket }) } socket.sendMessage({ topic: this.topic, action: this.constants.MULTIPLE_SUBSCRIPTIONS, originalAction: message.action, name }) return } subscription.sockets.add(socket) this.addSocket(subscription, socket) if (!silent) { if (this.logger.shouldLog(LOG_LEVEL.DEBUG)) { const logMsg = `for ${TOPIC[this.topic] || STATE_REGISTRY_TOPIC[this.topic]}:${name} by ${socket.userId}` this.logger.debug(this.actions[this.constants.SUBSCRIBE], logMsg) } socket.sendAckMessage(message) } } /** * Removes a SocketWrapper from the list of subscriptions for a topic */ public unsubscribe (name: string, message: Message, socket: SocketWrapper, silent?: boolean): void { const subscription = this.subscriptions.get(name) if (!subscription || !subscription.sockets.delete(socket)) { if (!silent) { if (this.logger.shouldLog(LOG_LEVEL.WARN)) { const msg = `${socket.userId} is not subscribed to ${name}` this.logger.warn(this.actions[this.constants.NOT_SUBSCRIBED], msg, { socketWrapper: socket, message}) } if (STATE_REGISTRY_TOPIC[this.topic]) { // This isn't supported for STATE_REGISTRY_TOPIC/s return } socket.sendMessage({ topic: this.topic, action: this.constants.NOT_SUBSCRIBED, originalAction: message.action, name }) } return } this.removeSocket(subscription, socket) if (!silent) { if (this.logger.shouldLog(LOG_LEVEL.DEBUG)) { const logMsg = `for ${this.topic}:${name} by ${socket.userId}` this.logger.debug(this.actions[this.constants.UNSUBSCRIBE], logMsg) } socket.sendAckMessage(message) } } /** * Returns an array of SocketWrappers that are subscribed * to or null if there are no subscribers */ public getLocalSubscribers (name: string): Set { const subscription = this.subscriptions.get(name) return subscription ? subscription.sockets : new Set() } /** * Returns true if there are SocketWrappers that * are subscribed to or false if there * aren't any subscribers */ public hasLocalSubscribers (name: string): boolean { return this.subscriptions.has(name) } /** * Allows to set a subscriptionListener after the class had been instantiated */ public setSubscriptionListener (listener: SubscriptionListener): void { this.subscriptionListener = listener this.clusterSubscriptions.onAdd(listener.onFirstSubscriptionMade.bind(listener)) this.clusterSubscriptions.onRemove(listener.onLastSubscriptionRemoved.bind(listener)) } private addSocket (subscription: Subscription, socket: SocketWrapper): void { const subscriptions = this.sockets.get(socket) || new Set() if (subscriptions.size === 0) { this.sockets.set(socket, subscriptions) socket.onClose(this.onSocketClose) } subscriptions.add(subscription) this.clusterSubscriptions!.add(subscription.name) if (this.subscriptionListener) { this.subscriptionListener.onSubscriptionMade(subscription.name, socket) } } private removeSocket (subscription: Subscription, socket: SocketWrapper): void { if (subscription.sockets.size === 0) { this.subscriptions.delete(subscription.name) } if (this.subscriptionListener) { this.subscriptionListener.onSubscriptionRemoved(subscription.name, socket) } this.clusterSubscriptions!.remove(subscription.name) const subscriptions = this.sockets.get(socket) if (subscriptions) { subscriptions.delete(subscription) if (subscriptions.size === 0) { this.sockets.delete(socket) socket.removeOnClose(this.onSocketClose) } } else { this.logger.error(EVENT.ERROR, 'Attempting to delete a subscription that doesn\'t exist') } } /** * Called whenever a socket closes to remove all of its subscriptions */ private onSocketClose (socket: SocketWrapper): void { const subscriptions = this.sockets.get(socket) if (!subscriptions) { this.logger.error( EVENT_ACTION[this.constants.NOT_SUBSCRIBED], 'A socket has an illegal registered close callback', { socketWrapper: socket } ) return } for (const subscription of subscriptions) { subscription.sockets.delete(socket) this.removeSocket(subscription, socket) } this.sockets.delete(socket) } private illegalCleanup () { this.sockets.forEach((subscriptions, socket) => { if (socket.isClosed) { if (!this.invalidSockets.has(socket)) { this.logger.error( EVENT.CLOSED_SOCKET, `Socket ${socket.uuid} is closed but still in registry. Currently there are ${this.invalidSockets.size} sockets. If you see this please raise a github issue!` ) this.invalidSockets.add(socket) } } }) } } ================================================ FILE: src/services/telemetry/deepstreamio-telemetry.ts ================================================ import { DeepstreamTelemetry, DeepstreamPlugin, DeepstreamServices, EVENT, DeepstreamConfig } from '@deepstream/types' import { getDSInfo } from '../../config/ds-info' import { v4 as uuid } from 'uuid' import { validateUUID } from '../../utils/utils' import { Dictionary } from 'ts-essentials' import { post } from 'needle' const TELEMETRY_URL = process.env.TELEMETRY_URL || 'http://telemetry.deepstream.io:8080/api/v1/startup' const DEFAULT_UUID = '00000000-0000-0000-0000-000000000000' export interface DeepstreamIOTelemetryOptions { enabled: boolean debug: boolean deploymentId: string } export class DeepstreamIOTelemetry extends DeepstreamPlugin implements DeepstreamTelemetry { public description = 'Deepstream Telemetry' private logger = this.services.logger.getNameSpace('TELEMETRY') constructor (private pluginOptions: DeepstreamIOTelemetryOptions, private services: DeepstreamServices, private config: DeepstreamConfig) { super() } public init () { if (this.pluginOptions.enabled === false) { this.logger.info( EVENT.INFO, 'Telemetry disabled' ) return } if (this.pluginOptions.deploymentId === undefined || !validateUUID(this.pluginOptions.deploymentId)) { this.logger.error( EVENT.ERROR, `Invalid deployment id, must be uuid format. Feel free to use this one "${uuid()}"` ) this.pluginOptions.deploymentId = DEFAULT_UUID } } public async whenReady (): Promise { if (this.pluginOptions.enabled === false) { return } const info = getDSInfo() const enabledFeatures = this.config.enabledFeatures const config: any = this.config const services = Object.keys(this.config).reduce((result, key) => { if (!config[key]) { return result } if (config[key].type) { result[key] = config[key].type } else if (config[key].name) { result[key] = { name: config[key].name } } else if (config[key].path) { result[key] = 'custom' } return result }, {} as Dictionary) const analytics = { deploymentId: this.pluginOptions.deploymentId, ...info, enabledFeatures, services } if (this.pluginOptions.debug) { this.logger.info(EVENT.TELEMETRY_DEBUG, `We would have sent the following: ${JSON.stringify(analytics)}`) } else { this.sendReport(analytics) } } public async close (): Promise { } private sendReport (data: any): void { post(TELEMETRY_URL, data, { content_type: 'application/json' }, (error: any) => { if (error) { if (error.code === 'ECONNREFUSED') { this.logger.warn(EVENT.TELEMETRY_UNREACHABLE, "Can't reach telemetry endpoint") } else { console.log(error) this.logger.error(EVENT.ERROR, `Telemetry error: ${error}`) } } }) } } ================================================ FILE: src/test/common.ts ================================================ import * as chai from 'chai' import * as sinonChai from 'sinon-chai' chai.use(sinonChai) ================================================ FILE: src/test/config/basic-permission-config.json ================================================ { "presence": { "*": { "allow": true } }, "record": { "*": { "write": true, "read": true } }, "event": { "*": { "publish": true, "subscribe": true } }, "rpc": { "*": { "provide": true, "request": true } } } ================================================ FILE: src/test/config/basic-valid-json.json ================================================ { "pet": "pug" } ================================================ FILE: src/test/config/blank-config.json ================================================ ================================================ FILE: src/test/config/config-broken.js ================================================ /* eslint-disable */ foobarBreaksIt ================================================ FILE: src/test/config/config-broken.yml ================================================ asdsad: ooops: ================================================ FILE: src/test/config/config.js ================================================ module.exports = { port: 1002 } ================================================ FILE: src/test/config/config.yml ================================================ serverName: UUID port: 1337 host: 1.2.3.4 colors: false showLogo: false logLevel: ERROR multipleenvs: '${EXAMPLE_HOST}:${EXAMPLE_PORT}' thisenvironmentdoesntexist: ${DOESNT_EXIST} environmentvariable: ${ENVIRONMENT_VARIABLE_TEST_1} another: environmentvariable: ${ENVIRONMENT_VARIABLE_TEST_2} ================================================ FILE: src/test/config/empty-map-config.json ================================================ {} ================================================ FILE: src/test/config/exists-test/a-file.js ================================================ module.exports = {} ================================================ FILE: src/test/config/exists-test/a-file.yml ================================================ --- yup: thats right ================================================ FILE: src/test/config/exists-test/a-json-file.json ================================================ { "I": "exist" } ================================================ FILE: src/test/config/invalid-permission-conf.json ================================================ { "presence": { "*": { "allow": true } }, "record": {}, "event": { "*": { "publish": true, "subscribe": true } }, "rpc": { "*": { "provide": true, "request": true } } } ================================================ FILE: src/test/config/invalid-user-config.json ================================================ { "userA": { "password": "tsA+yfWGoEk9uEU/GX1JokkzteayLj6YFTwmraQrO7k=75KQ2Mzm", "serverData": {} }, "userB": { "serverData": {} } } ================================================ FILE: src/test/config/json-with-env-variables.json ================================================ { "multipleenvs": "${EXAMPLE_HOST}:${EXAMPLE_PORT}", "environmentvariable": "${ENVIRONMENT_VARIABLE_TEST_1}", "another": { "environmentvariable": "${ENVIRONMENT_VARIABLE_TEST_2}" }, "thisenvironmentdoesntexist": "${DOESNT_EXIST}" } ================================================ FILE: src/test/config/no-private-events-permission-config.json ================================================ { "presence": { "*": { "allow": true } }, "record": { "*": { "write": true, "read": true } }, "event": { "*": { "publish": true, "subscribe": true }, "private/*": { "publish": false, "subscribe": false } }, "rpc": { "*": { "provide": true, "request": true } } } ================================================ FILE: src/test/config/sslKey.pem ================================================ I'm a key ================================================ FILE: src/test/config/users-unhashed.json ================================================ { "userC": { "password": "userCPass", "serverData": { "some": "values" }, "clientData": { "all": "othervalue" } }, "userD": { "password": "userDPass", "clientData": { "all": "client data" } } } ================================================ FILE: src/test/config/users.json ================================================ { "userA": { "password": "CKgFpPLJX1+FezZR8bMsP+8wQR+WG0z7AZYRy9nz5KY=DzI79/e3yJ0Y0UvNENMXaQ==", "serverData": { "some": "values" }, "clientData": { "all": "othervalue" } }, "userB": { "password": "jHPg24EDs9SKHALytrfaoEvDyz7wJgSVEY0ANaw/LgA=m5Ilfg/3+yN5j38tx8cBfA==", "clientData": { "all": "client data" } } } ================================================ FILE: src/test/helper/start-test-server.ts ================================================ const TestServer = require('./test-http-server') const testServer = new TestServer(6004, () => {}, true) testServer.on( 'request-received', testServer.respondWith.bind( testServer, 501, { serverData: {}, clientData: { name: 'bob'} } ) ) ================================================ FILE: src/test/helper/test-helper.ts ================================================ import * as SocketWrapperFactoryMock from '../mock/socket-wrapper-factory-mock' import AuthenticationHandler from '../mock/authentication-handler-mock' import {get} from '../../default-options' import MessageConnectorMock from '../mock/message-connector-mock' import LoggerMock from '../mock/logger-mock' import StorageMock from '../mock/storage-mock' import { DeepstreamConfig, DeepstreamServices, SocketWrapper, DeepstreamMonitoring, DeepstreamPlugin, PermissionCallback, LOG_LEVEL, EVENT, ValveSchema } from '@deepstream/types' import { Message } from '../../constants' import { DefaultSubscriptionRegistryFactory } from '../../services/subscription-registry/default-subscription-registry-factory' import { DistributedStateRegistryFactory } from '../../services/cluster-state/distributed-state-registry-factory' import { DistributedClusterRegistry } from '../../services/cluster-registry/distributed-cluster-registry' export const getBasePermissions = function (): ValveSchema { return { presence: { '*': { allow: true } }, record: { '*': { write: true, read: true } }, event: { '*': { publish: true, subscribe: true } }, rpc: { '*': { provide: true, request: true } } } } export const getDeepstreamOptions = (serverName?: string): { config: DeepstreamConfig, services: DeepstreamServices } => { const config = { ...get(), ...{ serverName: serverName || 'server-name-a', cluster: { state: { options: { reconciliationTimeout: 50 } } }, permission: { options: { cacheEvacuationInterval: 60000, maxRuleIterations: 3 } }, rpc: { provideRequestorData: true, provideRequestorName: true, ackTimeout: 10, responseTimeout: 20, }, record: { cacheRetrievalTimeout: 30, storageRetrievalTimeout: 50, storageExclusionPrefixes: ['no-storage'], storageHotPathPrefixes: [], } }} as never as DeepstreamConfig class PermissionHandler extends DeepstreamPlugin implements PermissionHandler { public lastArgs: any[] public description: string public nextResult: boolean public nextError: string | null constructor () { super() this.description = 'Test Permission Handler' this.nextResult = true this.nextError = null this.lastArgs = [] } public canPerformAction (socketWrapper: SocketWrapper, message: Message, callback: PermissionCallback, passItOn: any) { this.lastArgs.push([socketWrapper.userId, message, callback]) callback(socketWrapper, message, passItOn, this.nextError, this.nextResult) } } // tslint:disable-next-line: max-classes-per-file class MonitoringMock extends DeepstreamPlugin implements DeepstreamMonitoring { public description = 'monitoring mock' public onErrorLog (loglevel: LOG_LEVEL, event: EVENT, logMessage: string): void { } public onLogin (allowed: boolean, endpointType: string): void { } public onMessageReceived (message: Message): void { } public onMessageSend (message: Message): void { } public onBroadcast (message: Message, count: number): void { } } const services: Partial = { logger: new LoggerMock(), cache: new StorageMock(), storage: new StorageMock(), clusterNode: new MessageConnectorMock(config), // @ts-ignore locks: { get (name, cb) { cb(true) }, release () {} }, monitoring: new MonitoringMock(), authenticationHandler: new AuthenticationHandler(), permission: new PermissionHandler(), connectionEndpoints: [], } services.subscriptions = new DefaultSubscriptionRegistryFactory({}, services as DeepstreamServices, config) services.clusterStates = new DistributedStateRegistryFactory({}, services as DeepstreamServices, config) services.clusterRegistry = new DistributedClusterRegistry({}, services as DeepstreamServices, config) return { config, services } as { config: DeepstreamConfig, services: DeepstreamServices} } export const getDeepstreamPermissionOptions = function () { const options = exports.getDeepstreamOptions() options.config = Object.assign(options.config, { cacheRetrievalTimeout: 500, }) return { config: options.config, services: options.services } } const ConfigPermission = require('../../services/permission/valve/config-permission').ConfigPermission export const testPermission = function (options: { config: DeepstreamConfig, services: DeepstreamServices }) { return function (permissions: any, message: Message, username?: string, userdata?: any, callback?: PermissionCallback) { options.config.permission.options.permissions = permissions const permission = new ConfigPermission(options.config.permission.options, options.services, options.config) permission.setRecordHandler({ removeRecordRequest: () => {}, runWhenRecordStable: (r: any, c: any) => { c(r) } }) let permissionResult const socketWrapper = SocketWrapperFactoryMock.createSocketWrapper() socketWrapper.userId = username || 'someUser' socketWrapper.serverData = userdata callback = callback || function (sw: SocketWrapper, msg: Message, passItOn: any, error: any, result: boolean) { permissionResult = result } permission.canPerformAction(socketWrapper, message, callback) return permissionResult } } ================================================ FILE: src/test/helper/test-http-server.ts ================================================ import * as http from 'http' import { EventEmitter } from 'events' export default class TestHttpServer extends EventEmitter { public server: any public lastRequestData: any = null public hasReceivedRequest: boolean = false public lastRequestHeaders: any = null public lastRequestMethod: any = null private response: any = null private request: any = null constructor (private port: number, private callback: Function, private doLog: boolean = false) { super() this.server = http.createServer(this.onRequest.bind(this)) this.server.listen(port, this.onListen.bind(this)) } public static getRandomPort () { return 1000 + Math.floor(Math.random() * 9000) } public getRequestHeader (key: string) { return this.request.headers[key] } public reset () { this.lastRequestData = null this.hasReceivedRequest = false this.lastRequestHeaders = null } public respondWith (statusCode: number, data: any) { if (typeof data === 'object') { data = JSON.stringify(data) // eslint-disable-line } this.response.setHeader('content-type', 'application/json') this.response.writeHead(statusCode) this.response.end(data) } public close (callback: Function) { this.server.close(callback) } private onListen () { this.log(`server listening on port ${this.port}`) this.callback() } private log (msg: string) { if (this.doLog) { console.log(msg) } } private onRequest (request: http.IncomingMessage, response: http.OutgoingMessage) { let postData = '' request.setEncoding('utf8') request.on('data', (chunk) => { postData += chunk }) request.on('end', () => { this.lastRequestData = JSON.parse(postData) this.lastRequestHeaders = request.headers this.lastRequestMethod = request.method this.emit('request-received') this.log(`received data ${postData}`) }) this.request = request this.response = response } } ================================================ FILE: src/test/helper/test-mocks.ts ================================================ import { EventEmitter } from 'events' import { Message, JSONObject } from '../../constants' import { SocketWrapper } from '@deepstream/types' const sinon = require('sinon') export const getTestMocks = () => { const subscriptionRegistry = { subscribe: () => {}, unsubscribe: () => {}, sendToSubscribers: () => {}, setSubscriptionListener: () => {}, getLocalSubscribers: () => new Set(), getAllRemoteServers: () => {}, setAction: () => {}, hasLocalSubscribers: () => true, subscribeBulk: () => {}, unsubscribeBulk: () => {} } const listenerRegistry = { handle: () => {} } const emitter = new EventEmitter() const stateRegistry = { add: () => {}, remove: () => {}, on: () => {}, emit: () => {}, getAll: () => {}, onAdd: () => {}, onRemove: () => {} } stateRegistry.on = emitter.on as any stateRegistry.emit = emitter.emit as any const recordHandler = { broadcastUpdate: () => {}, transitionComplete: () => {} } const subscriptionRegistryMock = sinon.mock(subscriptionRegistry) const listenerRegistryMock = sinon.mock(listenerRegistry) const stateRegistryMock = sinon.mock(stateRegistry) const recordHandlerMock = sinon.mock(recordHandler) function getSocketWrapper (userId: string, authData: JSONObject = {}, clientData: JSONObject = {}) { const socketWrapper = { authAttempts: 0, userId, authData, clientData, sendMessage: () => {}, sendBuiltMessage: () => {}, sendAckMessage: () => {}, uuid: Math.random(), parseData: (message: Message) => { if (message.parsedData) { return true } try { message.parsedData = JSON.parse(message.data!.toString()) return true } catch (e) { return e } }, getMessage: (message: Message) => message, parseMessage: (message: Message) => message, destroy: () => {}, getHandshakeData: () => ({}), close: () => {}, onClose: () => {}, removeOnClose: () => {} } as never as SocketWrapper return { socketWrapper, socketWrapperMock: sinon.mock(socketWrapper) } } return { subscriptionRegistry, listenerRegistry, stateRegistry, recordHandler, subscriptionRegistryMock, listenerRegistryMock, stateRegistryMock, recordHandlerMock, getSocketWrapper, } } ================================================ FILE: src/test/mock/authentication-handler-mock.ts ================================================ import { DeepstreamPlugin, DeepstreamAuthentication, DeepstreamAuthenticationResult } from '@deepstream/types' export default class AuthenticationMock extends DeepstreamPlugin implements DeepstreamAuthentication { public onClientDisconnectCalledWith: string | null = null public sendNextValidAuthWithData: boolean = false public lastUserValidationQueryArgs: IArguments | null = null public nextUserValidationResult: boolean = true public nextUserIsAnonymous: boolean = false public description: string = 'Authentication Mock' constructor () { super() this.reset() } public reset () { this.nextUserIsAnonymous = false this.nextUserValidationResult = true this.lastUserValidationQueryArgs = null this.sendNextValidAuthWithData = false this.onClientDisconnectCalledWith = null } public async isValidUser (handshakeData: any, authData: any): Promise { this.lastUserValidationQueryArgs = arguments if (this.nextUserValidationResult === true) { if (this.sendNextValidAuthWithData === true) { return { isValid: true, id: 'test-user', clientData: { value: 'test-data' } } } if (this.nextUserIsAnonymous) { return { isValid: true, id: 'open' } } return { isValid: true, id: 'test-user' } } return { isValid: false, clientData: { error: 'Invalid User' } } } public onClientDisconnect (username: string) { this.onClientDisconnectCalledWith = username } } ================================================ FILE: src/test/mock/http-mock.ts ================================================ import { EventEmitter } from 'events' export class HttpServerMock extends EventEmitter { public listening: boolean = false public closed: boolean = false private port: any private host: any public listen (port: string, host: string, callback: Function) { this.port = port this.host = host const server = this process.nextTick(() => { server.listening = true server.emit('listening') if (callback) { callback() } }) } public close (callback: Function) { this.closed = true this.emit('close') if (callback) { callback() } } public address () { return { address: this.host || 'localhost', port: this.port || 8080 } } public _simulateUpgrade (socket: any) { const head = {} const request = { url: 'https://deepstream.io/?ds=foo', headers: { 'origin': '', 'sec-websocket-key': 'xxxxxxxxxxxxxxxxxxxxxxxx' }, connection: { authorized: true } } this.emit('upgrade', request, socket, head) } } // tslint:disable-next-line:max-classes-per-file export default class HttpMock { public nextServerIsListening: boolean constructor () { this.nextServerIsListening = false } public createServer () { const server = new HttpServerMock() server.listening = this.nextServerIsListening return server } } ================================================ FILE: src/test/mock/logger-mock.ts ================================================ import {spy, SinonSpy} from 'sinon' import { DeepstreamLogger, DeepstreamPlugin, LOG_LEVEL, NamespacedLogger, EVENT } from '@deepstream/types' export default class LoggerMock extends DeepstreamPlugin implements DeepstreamLogger { public description: string = 'mock logger' public lastLogLevel: any public lastLogEvent: any public lastLogMessage: any public lastLogArguments: any public logSpy: SinonSpy constructor () { super() this.lastLogLevel = null this.lastLogEvent = null this.lastLogMessage = null this.lastLogArguments = null this.logSpy = spy() } public shouldLog (logLevel: LOG_LEVEL): boolean { return true } public warn (event: EVENT | string, message?: string, metaData?: any) { this.log(LOG_LEVEL.WARN, event, message) this.logSpy(LOG_LEVEL.WARN, event, message) } public debug (event: EVENT | string, message?: string, metaData?: any) { this.log(LOG_LEVEL.DEBUG, event, message) this.logSpy(LOG_LEVEL.DEBUG, event, message) } public info (event: EVENT | string, message?: string, metaData?: any) { this.log(LOG_LEVEL.INFO, event, message) this.logSpy(LOG_LEVEL.INFO, event, message) } public error (event: EVENT | string, message?: string, metaData?: any) { this.log(LOG_LEVEL.ERROR, event, message) this.logSpy(LOG_LEVEL.ERROR, event, message) } public fatal (event: string, message?: string | undefined, metaData?: any): void { this.log(LOG_LEVEL.FATAL, event, message) this.logSpy(LOG_LEVEL.FATAL, event, message) } public getNameSpace (namespace: string): NamespacedLogger { return this } private log (level: LOG_LEVEL, event: EVENT | string, message?: string, metaData?: any) { this.lastLogLevel = level this.lastLogEvent = event this.lastLogMessage = message this.lastLogArguments = Array.from(arguments) } public setLogLevel () { } } ================================================ FILE: src/test/mock/message-connector-mock.ts ================================================ import { EventEmitter } from 'events' import { DeepstreamPlugin, DeepstreamClusterNode } from '@deepstream/types' import { TOPIC, Message, STATE_REGISTRY_TOPIC } from '../../constants' export default class MessageConnectorMock extends DeepstreamPlugin implements DeepstreamClusterNode { public description = 'Message Connector Mock' public lastPublishedTopic: TOPIC | STATE_REGISTRY_TOPIC | null = null public lastPublishedMessage: Message | null = null public lastSubscribedTopic: TOPIC | null = null public publishedMessages: Message[] = [] public all: string[] = ['server-name-a', 'server-name-b', 'server-name-c'] public lastDirectSentMessage: any public currentLeader: string = 'server-name-a' public eventEmitter = new EventEmitter() constructor (private options: any) { super() this.eventEmitter.setMaxListeners(0) } public reset () { this.publishedMessages = [] this.lastPublishedTopic = null this.lastPublishedMessage = null this.lastSubscribedTopic = null this.all = ['server-name-a', 'server-name-b', 'server-name-c'] this.currentLeader = 'server-name-a' } public subscribe (topic: TOPIC, callback: (message: MessageType, originServerName: string) => void) { this.lastSubscribedTopic = topic this.eventEmitter.on(TOPIC[topic], callback) } public sendBroadcast () { } public send (message: Message, metaData?: any) { this.publishedMessages.push(message) this.lastPublishedTopic = message.topic this.lastPublishedMessage = message } public sendDirect (serverName: string, message: Message) { this.lastDirectSentMessage = { serverName, message } } public unsubscribe (topic: TOPIC, callback: (message: Message) => void) { this.eventEmitter.removeListener(TOPIC[topic], callback) } public simulateIncomingMessage (topic: TOPIC, msg: Message, serverName: string) { this.eventEmitter.emit(TOPIC[topic], msg, serverName) } public getAll () { return this.all } public isLeader () { return this.currentLeader === this.options.serverName } public getLeader () { return this.currentLeader } public getCurrentLeader () { return this.currentLeader } public subscribeServerDisconnect () { } } ================================================ FILE: src/test/mock/permission-handler-mock.ts ================================================ import { PermissionCallback, SocketWrapper, DeepstreamPlugin } from '@deepstream/types' import { Message } from '../../constants' export default class PermissionHandlerMock extends DeepstreamPlugin { public nextCanPerformActionResult: any public lastCanPerformActionQueryArgs: any public description = 'PermissionHandlerMock' constructor () { super() this.reset() } public reset () { this.nextCanPerformActionResult = true this.lastCanPerformActionQueryArgs = null } public canPerformAction (socketWrapper: SocketWrapper, message: Message, callback: PermissionCallback, passItOn: any) { this.lastCanPerformActionQueryArgs = arguments if (typeof this.nextCanPerformActionResult === 'string') { callback(socketWrapper, message, passItOn, this.nextCanPerformActionResult, false) } else { callback(socketWrapper, message, passItOn, null, this.nextCanPerformActionResult) } } } ================================================ FILE: src/test/mock/plugin-mock.ts ================================================ import { DeepstreamPlugin, DeepstreamServices, DeepstreamConfig } from '@deepstream/types' import { EventEmitter } from 'events' export default class PluginMock extends DeepstreamPlugin { public isReady: boolean = false public description: string = this.options.name || 'mock-plugin' private emitter = new EventEmitter() constructor (private options: any, services: DeepstreamServices, config: DeepstreamConfig) { super() } public setReady () { this.isReady = true this.emitter.emit('ready') } public async whenReady () { if (!this.isReady) { await new Promise((resolve) => { this.emitter.once('ready', resolve) setTimeout(resolve, 20) }) } } } ================================================ FILE: src/test/mock/socket-mock.ts ================================================ import { Message } from '../../constants' export default class SocketMock { public lastSendMessage: any public isDisconnected: any public sendMessages: any public autoClose: any public readyState: any public ssl: any // tslint:disable-next-line:variable-name public _handle: any constructor () { this.lastSendMessage = null this.isDisconnected = false this.sendMessages = [] this.autoClose = true this.readyState = '' this.ssl = null this._handle = {} } public send (message: Message) { this.lastSendMessage = message this.sendMessages.push(message) } public end () { } public getMsg (i: number) { return this.sendMessages[this.sendMessages.length - (i + 1)] } public getMsgSize () { return this.sendMessages.length } public close () { if (this.autoClose === true) { this.doClose() } } public destroy () { this.doClose() } public doClose () { this.isDisconnected = true this.readyState = 'closed' } } ================================================ FILE: src/test/mock/socket-wrapper-factory-mock.ts ================================================ import { EventEmitter } from 'events' import { Message } from '../../constants' class SocketWrapperMock extends EventEmitter { public static lastPreparedMessage: any public isClosed: boolean = false public authCallBack: any = null public authAttempts: number = 0 public uuid: number = Math.random() public lastSendMessage: any public userId: any = null public serverData: any constructor (private handshakeData: any) { super() } public sendAckMessage (message: Message) { this.lastSendMessage = message } public getHandshakeData () { return this.handshakeData } public sendError (/* topic, type, msg */) { } public sendMessage (message: Message) { this.lastSendMessage = message } public parseData (message: Message) { if (message.parsedData || !message.data) { return null } try { message.parsedData = JSON.parse(message.data.toString()) return true } catch (e) { return e } } public send (/* message */) { } public destroy () { this.authCallBack = null this.isClosed = true this.emit('close', this) } public close () { this.destroy() } public setUpHandshakeData () { this.handshakeData = { remoteAddress: 'remote@address' } return this.handshakeData } } export const createSocketWrapper = (options?: any) => new SocketWrapperMock(options) ================================================ FILE: src/test/mock/storage-mock.ts ================================================ import { DeepstreamStorage, DeepstreamCache, StorageWriteCallback, StorageReadCallback, DeepstreamPlugin } from '@deepstream/types' import { JSONObject } from '../../constants' export default class StorageMock extends DeepstreamPlugin implements DeepstreamStorage, DeepstreamCache { public values = new Map() public failNextSet: boolean = false public nextOperationWillBeSuccessful: boolean = true public nextOperationWillBeSynchronous: boolean = true public nextGetWillBeSynchronous: boolean = true public lastGetCallback: Function | null = null public lastRequestedKey: string | null = null public lastSetKey: string | null = null public lastSetVersion: number | null = null public lastSetValue: object | null = null public completedSetOperations: any public completedDeleteOperations: any public getCalls: any public setTimeout: any public getTimeout: any public description: string = 'Mock Storage' constructor () { super() this.reset() } public reset () { this.values.clear() this.failNextSet = false this.nextOperationWillBeSuccessful = true this.nextOperationWillBeSynchronous = true this.nextGetWillBeSynchronous = true this.lastGetCallback = null this.lastRequestedKey = null this.lastSetKey = null this.lastSetVersion = null this.lastSetValue = null this.completedSetOperations = 0 this.completedDeleteOperations = 0 this.getCalls = [] clearTimeout(this.getTimeout) clearTimeout(this.setTimeout) } public head (recordName: string, callback: any): void { throw new Error('Method not implemented.') } public headBulk (recordNames: string[], callback: any): void { throw new Error('Method not implemented.') } public deleteBulk (keys: string[], callback: StorageWriteCallback) { if (this.nextOperationWillBeSynchronous) { this.completedDeleteOperations++ if (this.nextOperationWillBeSuccessful) { keys.forEach((key) => this.values.delete(key)) callback(null) } else { callback('storageError') return } } else { setTimeout(() => { this.completedDeleteOperations++ callback(this.nextOperationWillBeSuccessful ? null : 'storageError') }, 10) } } public delete (key: string, callback: StorageWriteCallback) { if (this.nextOperationWillBeSynchronous) { this.completedDeleteOperations++ if (this.nextOperationWillBeSuccessful) { this.values.delete(key) callback(null) } else { callback('storageError') return } } else { setTimeout(() => { this.completedDeleteOperations++ callback(this.nextOperationWillBeSuccessful ? null : 'storageError') }, 10) } } public hadGetFor (key: string) { for (let i = 0; i < this.getCalls.length; i++) { if (this.getCalls[i][0] === key) { return true } } return false } public triggerLastGetCallback (errorMessage: string, value: JSONObject) { if (this.lastGetCallback) { this.lastGetCallback(errorMessage, value) } } public get (key: string, callback: StorageReadCallback) { this.getCalls.push(arguments) this.lastGetCallback = callback this.lastRequestedKey = key const set = this.values.get(key) || { version: -1, value: null } if (this.nextGetWillBeSynchronous === true) { callback(this.nextOperationWillBeSuccessful ? null : 'storageError', set.version !== undefined ? set.version : -1, set.value ? Object.assign({}, set.value) : null) } else { this.getTimeout = setTimeout(() => { callback(this.nextOperationWillBeSuccessful ? null : 'storageError', set.version !== undefined ? set.version : -1, set.value ? Object.assign({}, set.value) : null) }, 25) } } public set (key: string, version: number, value: JSONObject, callback: StorageWriteCallback) { const set = { version, value } this.lastSetKey = key this.lastSetVersion = version this.lastSetValue = value if (this.nextOperationWillBeSuccessful) { this.values.set(key, set) } if (this.nextOperationWillBeSynchronous) { this.completedSetOperations++ if (this.failNextSet) { this.failNextSet = false callback('storageError') return } callback(this.nextOperationWillBeSuccessful ? null : 'storageError') } else { this.setTimeout = setTimeout(() => { this.completedSetOperations++ callback(this.nextOperationWillBeSuccessful ? null : 'storageError') }, 50) } } } ================================================ FILE: src/utils/dependency-initialiser.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import { spy } from 'sinon' import { DependencyInitialiser } from './dependency-initialiser' import PluginMock from '../test/mock/plugin-mock' import LoggerMock from '../test/mock/logger-mock' import { LOG_LEVEL, EVENT } from '@deepstream/types'; import { PromiseDelay } from './utils'; const services = { logger: new LoggerMock() } describe('dependency-initialiser', () => { let dependencyBInitialiser: DependencyInitialiser let config: any beforeEach(() => { config = { pluginA: new PluginMock({ name:'A' }), pluginB: new PluginMock({ name: 'B'}), pluginC: new PluginMock({ name: 'C'}), brokenPlugin: {}, dependencyInitializationTimeout: 50 } services.logger.lastLogEvent = null }) it ('sets description', () => { dependencyBInitialiser = new DependencyInitialiser(config as any, services as any, config.pluginB, 'pluginB') expect(dependencyBInitialiser.getDependency().description).to.equal('B') expect(services.logger.lastLogEvent).to.equal(null) }) it('throws an error if dependency doesnt implement emitter or has isReady', () => { expect(() => { // tslint:disable-next-line:no-unused-expression new DependencyInitialiser(config as any, services as any, {} as any, 'brokenPlugin') }).to.throw() expect(services.logger.lastLogEvent).to.equal(EVENT.PLUGIN_INITIALIZATION_ERROR) }) it('notifies when the plugin is ready with when already ready', async () => { config.pluginB.isReady = true dependencyBInitialiser = new DependencyInitialiser(config as any, services as any, config.pluginB, 'pluginB') await dependencyBInitialiser.whenReady() expect(services.logger.lastLogEvent).to.equal(EVENT.INFO) }) it('notifies when the plugin is ready with when not ready', (done) => { dependencyBInitialiser = new DependencyInitialiser(config as any, services as any, config.pluginB, 'pluginB') dependencyBInitialiser.whenReady().then(() => { expect(services.logger.lastLogEvent).to.equal(EVENT.INFO) done() }) config.pluginB.setReady() }) }) describe('encounters timeouts and errors during dependency initialisations', () => { let dependencyInitialiser const onReady = spy() const originalConsoleLog = console.log const config = { plugin: new PluginMock('A'), dependencyInitializationTimeout: 1, } it('disables console.error', () => { Object.defineProperty(console, 'error', { value: services.logger.log }) }) it("creates a dependency initialiser and doesn't initialize a plugin in time", async () => { services.logger.logSpy.resetHistory() dependencyInitialiser = new DependencyInitialiser(config as any, services as any, config.plugin, 'plugin') await PromiseDelay(20) dependencyInitialiser.whenReady().then(onReady) expect(config.plugin.isReady).to.equal(false) expect(onReady).to.have.callCount(0) // expect(services.logger.logSpy).to.have.been.calledOnce // another test isn't async and bleeds into this one expect(services.logger.logSpy).to.have.been.calledWith(LOG_LEVEL.FATAL, EVENT.PLUGIN_INITIALIZATION_TIMEOUT, 'plugin wasn\'t initialised in time') }) it.skip('creates another depdendency initialiser with a plugin error', async () => { process.once('uncaughtException', () => { expect(onReady).to.have.callCount(0) expect(services.logger.logSpy).to.have.been.calledWith('Error while initialising dependency') expect(services.logger.logSpy).to.have.been.calledWith('Error while initialising plugin: something went wrong') next() }) dependencyInitialiser = new DependencyInitialiser({}, config as any, services as any, config.plugin, 'plugin') dependencyInitialiser.on('ready', onReady) try { config.plugin.emit('error', 'something went wrong') next('Fail') } catch (err) {} }) it('enable console.error', () => { Object.defineProperty(console, 'error', { value: originalConsoleLog }) }) }) ================================================ FILE: src/utils/dependency-initialiser.ts ================================================ import { DeepstreamConfig, DeepstreamServices, DeepstreamPlugin, EVENT } from '@deepstream/types' import { EventEmitter } from 'events' export class DependencyInitialiser { private isReady: boolean = false private timeout: NodeJS.Timeout | null = null private emitter = new EventEmitter() /** * This class is used to track the initialization of an individual service or plugin */ constructor (private config: DeepstreamConfig, private services: DeepstreamServices, private dependency: DeepstreamPlugin, private name: string) { if (typeof this.dependency.whenReady !== 'function') { const errorMessage = `${this.name} needs to implement async whenReady and close, please look at the DeepstreamPlugin API here` // TODO: Insert link this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_ERROR, errorMessage) this.services.notifyFatalException() return } this.timeout = setTimeout( this.onTimeout.bind(this), this.config.dependencyInitializationTimeout, ) if (this.dependency.init) { this.dependency.init() } this.dependency .whenReady() .then(this.onReady.bind(this)) } public async whenReady (): Promise { if (!this.isReady) { return new Promise((resolve) => this.emitter.once('ready', resolve)) } } /** * Returns the underlying dependency (e.g. the Logger, StorageConnector etc.) */ public getDependency (): DeepstreamPlugin { return this.dependency } /** * Callback for succesfully initialised dependencies */ private onReady (): void { if (this.timeout) { clearTimeout(this.timeout) } this.dependency.description = this.dependency.description || (this.dependency as any).type const dependencyType = this.dependency.description ? `: ${this.dependency.description}` : ': no dependency description provided' this.services.logger.info(EVENT.INFO, `${this.name} ready${dependencyType}`) this.isReady = true this.emitter.emit('ready') } /** * Callback for dependencies that weren't initialised in time */ private onTimeout (): void { const message = `${this.name} wasn't initialised in time` if (this.name === 'logger') { console.error('Error while initialising log dependency dependency') console.error(message) this.services.notifyFatalException() } this.services.logger.fatal(EVENT.PLUGIN_INITIALIZATION_TIMEOUT, message) } } ================================================ FILE: src/utils/json-path.spec.ts ================================================ import 'mocha' import { expect } from 'chai' import * as jsonPath from './json-path' describe('objects are created from paths and their value is set correctly', () => { it('sets simple values', () => { const record = {} jsonPath.setValue(record, 'firstname', 'Wolfram') expect(record).to.deep.equal({ firstname: 'Wolfram' }) }) it('setting a value to undefined deletes it', () => { const record = { a: 1, b: 2 } jsonPath.setValue(record, 'a', undefined) expect(record).to.deep.equal({ b: 2 }) }) it('sets values for nested objects', () => { const record = {} jsonPath.setValue(record, 'address.street', 'someStreet') expect(record).to.deep.equal({ address: { street: 'someStreet' } }) }) it('sets values for nested objects with numeric field names', () => { const record = {} jsonPath.setValue(record, 'address.street.1', 'someStreet') expect(record).to.deep.equal({ address: { street: { 1: 'someStreet' } } }) }) it('sets values for nested objects with multiple numeric field names', () => { const record = {} jsonPath.setValue(record, 'address.99.street.1', 'someStreet') expect(record).to.deep.equal({ address: { 99 : { street: { 1: 'someStreet' } } } }) }) it('sets values for nested objects with multiple mixed array and numeric field names', () => { const record = {} jsonPath.setValue(record, 'address[2].99.street[2].1', 'someStreet') expect(record).to.deep.equal({ address: [ undefined, undefined, { 99 : { street: [ undefined, undefined, { 1: 'someStreet' } ] } } ] }) }) it('sets first value of array', () => { const record = {} jsonPath.setValue(record, 'items[0]', 51) expect(JSON.stringify(record)).to.deep.equal(JSON.stringify({ items: [ 51 ] })) }) it('sets numeric obj member name of 0 (zero)', () => { const record = {} jsonPath.setValue(record, 'items.0', 51) expect(JSON.stringify(record)).to.deep.equal(JSON.stringify({ items: { 0 : 51 } })) }) it('sets values for arrays', () => { const record = {} jsonPath.setValue(record, 'pastAddresses[1].street', 'someStreet') expect(JSON.stringify(record)).to.deep.equal(JSON.stringify({ pastAddresses: [ undefined, { street: 'someStreet' } ] })) }) it('sets value AS arrays of arrays', () => { const record = { addresses: undefined } const arrOfArr = [ undefined, [ 'new-Street1', 'road1', 'blvd1' ], [ 'street2', 'road2', 'blvd2' ] ] jsonPath.setValue(record, 'addresses', arrOfArr) expect(JSON.stringify(record)).to.deep.equal(JSON.stringify({ addresses: [ undefined, [ 'new-Street1', 'road1', 'blvd1' ], [ 'street2', 'road2', 'blvd2' ] ] })) }) it('sets value IN arrays of arrays', () => { const record = { addresses: [ undefined, [ 'street1', 'road1', 'blvd1' ], [ 'street2', 'road2', 'blvd2' ] ] } jsonPath.setValue(record, 'addresses[1][0]', 'new-Street1') expect(JSON.stringify(record)).to.deep.equal(JSON.stringify({ addresses: [ undefined, [ 'new-Street1', 'road1', 'blvd1' ], [ 'street2', 'road2', 'blvd2' ] ] })) }) it('sets value IN deeper nested multi-dimensional arrays of arrays', () => { const record = { obj: { 101 : { addresses: [ [ undefined, [ undefined, ['street1', 'road1', 'blvd1'], ['street2', 'road2', 'blvd2'] ], [ undefined, { a: 'street1', b: 'road1', c: 'blvd1' }, { 1: 'street2', 2: 'road2', 3: 'blvd2' } ] ], undefined, [[0, 1, 2, 3], [9, 8, 7, 6], [2, 4, 6, 8]] ] } } } jsonPath.setValue(record, 'obj.101.addresses[0][1][1][0]', 'new-Street1') expect(JSON.stringify(record)).to.deep.equal(JSON.stringify({ obj: { 101 : { addresses: [ [ undefined, [ undefined, ['new-Street1', 'road1', 'blvd1'], ['street2', 'road2', 'blvd2'] ], [ undefined, { a: 'street1', b: 'road1', c: 'blvd1' }, { 1: 'street2', 2: 'road2', 3: 'blvd2' } ] ], undefined, [[0, 1, 2, 3], [9, 8, 7, 6], [2, 4, 6, 8]] ] } } })) }) it('extends existing objects', () => { const record = { firstname: 'Wolfram' } jsonPath.setValue(record, 'lastname', 'Hempel') expect(record).to.deep.equal({ firstname: 'Wolfram', lastname: 'Hempel' } as any) }) it('extends existing arrays', () => { const record = { firstname: 'Wolfram', animals: ['Bear', 'Cow', 'Ostrich'] } jsonPath.setValue(record, 'animals[ 1 ]', 'Emu') expect(record).to.deep.equal({ firstname: 'Wolfram', animals: ['Bear', 'Emu', 'Ostrich'] }) }) it('extends existing arrays with empty slot assigned a primitive', () => { const record = { firstname: 'Wolfram', animals: [undefined, 'Cow', 'Ostrich'] } jsonPath.setValue(record, 'animals[0]', 'Emu') expect(record).to.deep.equal({ firstname: 'Wolfram', animals: ['Emu', 'Cow', 'Ostrich'] }) }) it('extends existing arrays with objects', () => { const record = { firstname: 'Wolfram', animals: [undefined, 'Cow', 'Ostrich'] } jsonPath.setValue(record, 'animals[0].xxx', 'Emu') expect(record).to.deep.equal({ firstname: 'Wolfram', animals: [{ xxx: 'Emu' }, 'Cow', 'Ostrich'] } as any) }) it('treats numbers with the path such as .0. as a key value', () => { const record = {} jsonPath.setValue(record, 'animals.0.name', 'Emu') expect(record).to.deep.equal({ animals: { 0: { name: 'Emu' } } }) }) it('treats numbers with the path such as [0] as an index value', () => { const record = {} jsonPath.setValue(record, 'animals[0].name', 'Emu') expect(record).to.deep.equal({ animals: [{ name: 'Emu' }] }) }) it('handles .xyz paths into non-objects', () => { const record = { animals: 3 } jsonPath.setValue(record, 'animals.name', 'Emu') expect(record).to.deep.equal({ animals: { name: 'Emu' } } as any) }) it('handles .xyz paths through non-objects', () => { const record = { animals: 3 } jsonPath.setValue(record, 'animals.name.length', 7) expect(record).to.deep.equal({ animals: { name: { length: 7 } } } as any) }) it('handles [0] paths into non-objects', () => { const record = { animals: 3 } jsonPath.setValue(record, 'animals[0]', 7) expect(record).to.deep.equal({ animals: [7] } as any) }) }) ================================================ FILE: src/utils/json-path.ts ================================================ const SPLIT_REG_EXP = /[[\]]/g /** * Returns the value of the path or * undefined if the path can't be resolved */ export function getValue (data: any, path: string): any { const tokens = tokenize(path) let value = data for (let i = 0; i < tokens.length; i++) { if (value === undefined) { return undefined } if (typeof value !== 'object') { throw new Error('invalid data or path') } value = value[tokens[i]] } return value } /** * This class allows to set or get specific * values within a json data structure using * string-based paths */ export function setValue (root: any, path: string, value: any): void { const tokens = tokenize(path) let node = root let i for (i = 0; i < tokens.length - 1; i++) { const token = tokens[i] if (node[token] !== undefined && typeof node[token] === 'object') { node = node[token] } else if (typeof tokens[i + 1] === 'number') { node = node[token] = [] } else { node = node[token] = {} } } if (value === undefined) { delete node[tokens[i]] } else { node[tokens[i]] = value } } /** * Parses the path. Splits it into * keys for objects and indices for arrays. */ function tokenize (path: string): Array { const tokens: Array = [] const parts = path.split('.') for (let i = 0; i < parts.length; i++) { const part = parts[i].trim() if (part.length === 0) { continue } const arrayIndexes: string[] = part.split(SPLIT_REG_EXP) if (arrayIndexes.length === 0) { // TODO continue } tokens.push(arrayIndexes[0]) for (let j = 1; j < arrayIndexes.length; j++) { if (arrayIndexes[j].length === 0) { continue } tokens.push(Number(arrayIndexes[j])) } } return tokens } ================================================ FILE: src/utils/message-distributor.spec.ts ================================================ import {spy} from 'sinon' import {expect} from 'chai' import MessageDistributor from './message-distributor' import * as testHelper from '../test/helper/test-helper' import { getTestMocks } from '../test/helper/test-mocks' const options = testHelper.getDeepstreamOptions() const config = options.config const services = options.services describe('message connector distributes messages to callbacks', () => { let messageDistributor let testMocks let client let testCallback beforeEach(() => { testMocks = getTestMocks() client = testMocks.getSocketWrapper() testCallback = spy() messageDistributor = new MessageDistributor(config, services) }) afterEach(() => { client.socketWrapperMock.verify() }) it('makes remote connection', () => { expect(services.clusterNode.lastSubscribedTopic).to.equal(null) messageDistributor.registerForTopic('someTopic', testCallback) expect(services.clusterNode.lastSubscribedTopic).to.equal('someTopic') }) it('makes local connection', () => { messageDistributor.registerForTopic('someTopic', testCallback) messageDistributor.distribute(client.socketWrapper, { topic: 'someTopic' }) expect(testCallback).to.have.callCount(1) }) it.skip('routes messages from the message connector', () => { messageDistributor.registerForTopic('topicB', testCallback) services.message.simulateIncomingMessage('topicB', { topic: 'topicB' }) expect(testCallback).to.have.callCount(1) }) it('only routes matching topics', () => { messageDistributor.registerForTopic('aTopic', testCallback) messageDistributor.registerForTopic('anotherTopic', testCallback) messageDistributor.distribute(client.socketWrapper, { topic: 'aTopic' }) expect(testCallback).to.have.callCount(1) }) it('throws an error for multiple registrations to the same topic', () => { let hasErrored = false try { messageDistributor.registerForTopic('someTopic', testCallback) messageDistributor.registerForTopic('someTopic', testCallback) } catch (e) { hasErrored = true } expect(hasErrored).to.equal(true) }) }) ================================================ FILE: src/utils/message-distributor.ts ================================================ import { PARSER_ACTION, TOPIC, Message, STATE_REGISTRY_TOPIC } from '../constants' import { SocketWrapper, DeepstreamServices, DeepstreamConfig } from '@deepstream/types' /** * The MessageDistributor routes valid and permissioned messages to * various, previously registered handlers, e.g. event-, rpc- or recordHandler */ export default class MessageDistributor { private callbacks = new Map() constructor (options: DeepstreamConfig, private services: DeepstreamServices) {} /** * Accepts a socketWrapper and a parsed message as input and distributes * it to its subscriber, based on the message's topic */ public distribute (socketWrapper: SocketWrapper, message: Message) { const callback = this.callbacks.get(message.topic) if (callback === undefined) { this.services.logger.warn(PARSER_ACTION[PARSER_ACTION.UNKNOWN_TOPIC], TOPIC[message.topic], { message }) socketWrapper.sendMessage({ topic: TOPIC.PARSER, action: PARSER_ACTION.UNKNOWN_TOPIC, originalTopic: message.topic }) return } this.services.monitoring.onMessageReceived(message, socketWrapper) callback(socketWrapper, message) } /** * Allows handlers (event, rpc, record) to register for topics. Subscribes them * to both messages passed to the distribute method as well as messages received * from the messageConnector */ public registerForTopic (topic: TOPIC, callback: (message: Message) => void) { if (this.callbacks.has(topic)) { throw new Error(`Callback already registered for topic ${topic}`) } this.callbacks.set(topic, callback) this.services.clusterNode.subscribe( topic, this.onMessageConnectorMessage.bind(this, callback), ) } /** * Whenever a message from the messageConnector is received it is passed * to the relevant handler, but with null instead of * a socketWrapper as sender */ private onMessageConnectorMessage (callback: Function, message: Message, originServer: string) { callback(null, message, originServer) } } ================================================ FILE: src/utils/message-processor.spec.ts ================================================ import {expect} from 'chai' import PermissionHandlerMock from '../test/mock/permission-handler-mock' import MessageProcessor from './message-processor' import LoggerMock from '../test/mock/logger-mock' import { getTestMocks } from '../test/helper/test-mocks' import { TOPIC, CONNECTION_ACTION, RPC_ACTION, RECORD_ACTION } from '../constants'; let messageProcessor let log let lastAuthenticatedMessage = null describe('the message processor only forwards valid, authorized messages', () => { let testMocks let client let permissionMock const message = { topic: TOPIC.RECORD, action: RECORD_ACTION.READ, name: 'record/name' } beforeEach(() => { testMocks = getTestMocks() client = testMocks.getSocketWrapper('someUser') permissionMock = new PermissionHandlerMock() const loggerMock = new LoggerMock() log = loggerMock.logSpy messageProcessor = new MessageProcessor({}, { permission: permissionMock, logger: loggerMock }) messageProcessor.onAuthenticatedMessage = function (socketWrapper, authenticatedMessage) { lastAuthenticatedMessage = authenticatedMessage } }) afterEach(() => { client.socketWrapperMock.verify() }) it('handles permission errors', () => { permissionMock.nextCanPerformActionResult = 'someError' client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: TOPIC.RECORD, action: RECORD_ACTION.MESSAGE_PERMISSION_ERROR, originalAction: RECORD_ACTION.READ, name: message.name, isError: true }) messageProcessor.process(client.socketWrapper, [message]) expect(log).to.have.callCount(1) expect(log).to.have.been.calledWith(2, RECORD_ACTION[RECORD_ACTION.MESSAGE_PERMISSION_ERROR], 'someError') }) it('rpc permission errors have a correlation id', () => { permissionMock.nextCanPerformActionResult = 'someError' const rpcMessage = { topic: TOPIC.RPC, action: RPC_ACTION.REQUEST, name: 'myRPC', correlationId: '1234567890', data: Buffer.from('{}', 'utf8'), parsedData: {} } client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: TOPIC.RPC, action: RPC_ACTION.MESSAGE_PERMISSION_ERROR, originalAction: rpcMessage.action, name: rpcMessage.name, correlationId: rpcMessage.correlationId, isError: true }) messageProcessor.process(client.socketWrapper, [rpcMessage]) expect(log).to.have.callCount(1) expect(log).to.have.been.calledWith(2, RPC_ACTION[RPC_ACTION.MESSAGE_PERMISSION_ERROR], 'someError') }) it('handles denied messages', () => { permissionMock.nextCanPerformActionResult = false client.socketWrapperMock .expects('sendMessage') .once() .withExactArgs({ topic: TOPIC.RECORD, action: RECORD_ACTION.MESSAGE_DENIED, originalAction: RECORD_ACTION.READ, name: message.name, isError: true }) messageProcessor.process(client.socketWrapper, [message]) }) it('provides the correct arguments to canPerformAction', () => { permissionMock.nextCanPerformActionResult = false messageProcessor.process(client.socketWrapper, [message]) expect(permissionMock.lastCanPerformActionQueryArgs.length).to.equal(4) expect(permissionMock.lastCanPerformActionQueryArgs[0]).to.equal(client.socketWrapper) expect(permissionMock.lastCanPerformActionQueryArgs[1]).to.deep.equal(message) expect(permissionMock.lastCanPerformActionQueryArgs[3]).to.deep.equal({}) }) it('forwards validated and permissioned messages', () => { permissionMock.nextCanPerformActionResult = true messageProcessor.process(client.socketWrapper, [message]) expect(lastAuthenticatedMessage).to.equal(message as any) }) }) ================================================ FILE: src/utils/message-processor.ts ================================================ import { TOPIC, CONNECTION_ACTION, Message, ALL_ACTIONS, ACTIONS, RECORD_ACTION } from '../constants' import { SocketWrapper, DeepstreamConfig, DeepstreamServices, EVENT } from '@deepstream/types' import { getUid } from './utils' /** * The MessageProcessor consumes blocks of parsed messages emitted by the * ConnectionEndpoint, checks if they are permissioned and - if they * are - forwards them. */ export default class MessageProcessor { private bulkResults = new Map() constructor (config: DeepstreamConfig, private services: DeepstreamServices) { this.onPermissionResponse = this.onPermissionResponse.bind(this) this.onBulkPermissionResponse = this.onBulkPermissionResponse.bind(this) } /** * There will only ever be one consumer of forwarded messages. So rather than using * events - and their performance overhead - the messageProcessor exposes * this method that's expected to be overwritten. */ public onAuthenticatedMessage (socketWrapper: SocketWrapper, message: Message) { } /** * This method is the way the message processor accepts input. It receives arrays * of parsed messages, iterates through them and issues permission requests for * each individual message * * @todo The responses from the permission service might arrive in any arbitrary order - order them * @todo Handle permission handler timeouts */ public process (socketWrapper: SocketWrapper, parsedMessages: Message[]): void { const length = parsedMessages.length for (let i = 0; i < length; i++) { const message = parsedMessages[i] if (message.topic === TOPIC.CONNECTION && message.action === CONNECTION_ACTION.PING) { // respond to PING message socketWrapper.sendMessage({ topic: TOPIC.CONNECTION, action: CONNECTION_ACTION.PONG }) continue } if (message.names && message.names.length > 0) { const uuid = getUid() if (this.bulkResults.has(uuid)) { this.services.logger.error(EVENT.NOT_VALID_UUID, `Invalid uuid used twice ${uuid}`, { uuid }) } this.bulkResults.set(uuid, { total: message.names!.length, completed: 0 }) const l = message.names!.length for (let j = 0; j < l; j++) { this.services.permission.canPerformAction( socketWrapper, { ...message, name: message.names![j] }, this.onBulkPermissionResponse, { originalMessage: message, uuid } ) } continue } this.services.permission.canPerformAction( socketWrapper, message, this.onPermissionResponse, {} ) } } private onBulkPermissionResponse (socketWrapper: SocketWrapper, message: Message, passItOn: any, error: ALL_ACTIONS | Error | string | null, result: boolean) { const bulkResult = this.bulkResults.get(passItOn.uuid)! if (error !== null || result === false) { passItOn.originalMessage.names!.splice(passItOn.originalMessage.names!.indexOf(passItOn.originalMessage.name!), 1) this.processInvalidResponse(socketWrapper, message, error, result) } if (bulkResult.total !== bulkResult.completed + 1) { bulkResult.completed = bulkResult.completed + 1 return } this.bulkResults.delete(passItOn.uuid) if (message.names!.length > 0) { this.onAuthenticatedMessage(socketWrapper, passItOn.originalMessage) } } /** * Processes the response that's returned by the permission service. */ private onPermissionResponse (socketWrapper: SocketWrapper, message: Message, passItOn: any, error: ALL_ACTIONS | Error | string | null, result: boolean): void { if (error !== null || result === false) { this.processInvalidResponse(socketWrapper, message, error, result) } else { this.onAuthenticatedMessage(socketWrapper, message) } } private processInvalidResponse (socketWrapper: SocketWrapper, message: Message, error: ALL_ACTIONS | Error | string | null, result: boolean) { if (error !== null) { this.services.logger.warn(RECORD_ACTION[RECORD_ACTION.MESSAGE_PERMISSION_ERROR], error.toString(), { message }) const permissionErrorMessage: Message = { topic: message.topic, action: ACTIONS[message.topic].MESSAGE_PERMISSION_ERROR, originalAction: message.action, name: message.name, isError: true } if (message.correlationId) { permissionErrorMessage.correlationId = message.correlationId } if (message.isWriteAck) { permissionErrorMessage.isWriteAck = true } socketWrapper.sendMessage(permissionErrorMessage) return } if (result !== true) { const permissionDeniedMessage: Message = { topic: message.topic, action: ACTIONS[message.topic].MESSAGE_DENIED, originalAction: message.action, name: message.name, isError: true } if (message.correlationId) { permissionDeniedMessage.correlationId = message.correlationId } if (message.isWriteAck) { permissionDeniedMessage.isWriteAck = true } socketWrapper.sendMessage(permissionDeniedMessage) return } } } ================================================ FILE: src/utils/utils.spec.ts ================================================ import 'mocha' import {spy} from 'sinon' import {expect} from 'chai' import { EventEmitter } from 'events' import { createHash } from './utils'; const utils = require('./utils') describe('utils', () => { it('receives a different value everytime getUid is called', () => { const uidA = utils.getUid() const uidB = utils.getUid() const uidC = utils.getUid() expect(uidA).not.to.equal(uidB) expect(uidB).not.to.equal(uidC) expect(uidA).not.to.equal(uidC) }) it('reverses maps', () => { const user = { firstname: 'Wolfram', lastname: 'Hempel' } expect(utils.reverseMap(user)).to.deep.equal({ Wolfram: 'firstname', Hempel: 'lastname' }) }) describe('isOfType', () => { it('checks basic types', () => { expect(utils.isOfType('bla', 'string')).to.equal(true) expect(utils.isOfType(42, 'string')).to.equal(false) expect(utils.isOfType(42, 'number')).to.equal(true) expect(utils.isOfType(true, 'number')).to.equal(false) expect(utils.isOfType(true, 'boolean')).to.equal(true) expect(utils.isOfType({}, 'object')).to.equal(true) expect(utils.isOfType(null, 'null')).to.equal(true) expect(utils.isOfType(null, 'object')).to.equal(false) expect(utils.isOfType([], 'object')).to.equal(true) }) it('checks urls', () => { expect(utils.isOfType('bla', 'url')).to.equal(false) expect(utils.isOfType('bla:22', 'url')).to.equal(true) expect(utils.isOfType('https://deepstream.io/', 'url')).to.equal(true) }) it('checks arrays', () => { expect(utils.isOfType([], 'array')).to.equal(true) expect(utils.isOfType({}, 'array')).to.equal(false) }) }) describe('validateMap', () => { function _map () { return { 'a-string': 'bla', 'a number': 42, 'an array': ['yup'] } } function _schema () { return { 'a-string': 'string', 'a number': 'number', 'an array': 'array' } } it('validates basic maps', () => { const map = _map() const schema = _schema() expect(utils.validateMap(map, false, schema)).to.equal(true) }) it('fails validating an incorrect map', () => { const map = _map() const schema = _schema() schema['an array'] = 'number' const returnValue = utils.validateMap(map, false, schema) expect(returnValue instanceof Error).to.equal(true) }) it('fails validating an incomplete map', () => { const map = _map() const schema = _schema() delete map['an array'] const returnValue = utils.validateMap(map, false, schema) expect(returnValue instanceof Error).to.equal(true) }) it('throws errors', () => { const map = _map() const schema = _schema() schema['an array'] = 'number' expect(() => { utils.validateMap(map, true, schema) }).to.throw() }) }) describe('merges recoursively', () => { it('merges two simple objects', () => { const objA = { firstname: 'Homer', lastname: 'Simpson' } const objB = { firstname: 'Marge' } expect(utils.merge(objA, objB)).to.deep.equal({ firstname: 'Marge', lastname: 'Simpson' }) }) it('merges two nested objects', () => { const objA = { firstname: 'Homer', lastname: 'Simpson', children: { Bart: { lastname: 'Simpson' } } } const objB = { firstname: 'Marge', children: { Bart: { firstname: 'Bart' } } } expect(utils.merge(objA, objB)).to.deep.equal({ firstname: 'Marge', lastname: 'Simpson', children: { Bart: { firstname: 'Bart', lastname: 'Simpson' } } }) }) it('merges multiple objects ', () => { const objA = { pets: { birds: ['parrot', 'dove'] } } const objB = { jobs: { hunter: false } } const objC = { firstname: 'Egon' } expect(utils.merge(objA, objB, {}, objC)).to.deep.equal({ pets: { birds: ['parrot', 'dove'] }, jobs: { hunter: false }, firstname: 'Egon' }) }) it('handles null and undefined values', () => { const objA = { pets: { dog: 1, cat: 2, ape: 3 } } const objB = { pets: { cat: null, ape: undefined, zebra: 9 } } expect(utils.merge(objA, objB)).to.deep.equal({ pets: { dog: 1, cat: null, ape: 3, zebra: 9 } }) }) }) it('creates a hash', async() => { const password = 'userAPass' const settings = { algorithm: 'md5', iterations: 100, keyLength: 32 } const { hash, salt } = await createHash(password, settings) const { hash: hashCheck } = await createHash(password, settings, salt) expect(hash.toString('base64')).to.eq(hashCheck.toString('base64')) }) }) ================================================ FILE: src/utils/utils.ts ================================================ import * as url from 'url' import * as crypto from 'crypto' /** * Returns a unique identifier */ export let getUid = function (): string { return `${Date.now().toString(36)}-${(Math.random() * 10000000000000000000).toString(36)}` } /** * Takes a key-value map and returns * a map with { value: key } of the old map */ export let reverseMap = function (map: any): any { const reversedMap = {} for (const key in map) { // @ts-ignore reversedMap[map[key]] = key } return reversedMap } /** * Extended version of the typeof operator. Also supports 'array' * and 'url' to check for valid URL schemas */ export let isOfType = function (input: any, expectedType: string): boolean { if (input === null) { return expectedType === 'null' } else if (expectedType === 'array') { return Array.isArray(input) } else if (expectedType === 'url') { return !!url.parse(input).host } return typeof input === expectedType } /** * Takes a map and validates it against a basic * json schema in the form { key: type } * @returns {Boolean|Error} */ export let validateMap = function (map: any, throwError: boolean, schema: any): any { let error let key for (key in schema) { if (typeof map[key] === 'undefined') { error = new Error(`Missing key ${key}`) break } if (!isOfType(map[key], schema[key])) { error = new Error(`Invalid type ${typeof map[key]} for ${key}`) break } } if (error) { if (throwError) { throw error } else { return error } } else { return true } } /** * Multi Object recursive merge * @param {Object} multiple objects to be merged into each other recursively */ export let merge = function (...args: any[]) { const result = {} const objs = Array.prototype.slice.apply(arguments) let i const internalMerge = (objA: any, objB: any) => { let key for (key in objB) { if (objB[key] && objB[key].constructor === Object) { objA[key] = objA[key] || {} internalMerge(objA[key], objB[key]) } else if (objB[key] !== undefined) { objA[key] = objB[key] } } } for (i = 0; i < objs.length; i++) { internalMerge(result, objs[i]) } return result } export let getRandomIntInRange = function (min: number, max: number): number { return min + Math.floor(Math.random() * (max - min)) } export let spliceRandomElement = function (array: any[]): any { const randomIndex = getRandomIntInRange(0, array.length) return array.splice(randomIndex, 1)[0] } /** * Randomize array element order in-place. * Using Durstenfeld shuffle algorithm. */ export let shuffleArray = function (array: any[]): any[] { for (let i = array.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)) const temp = array[i] array[i] = array[j] array[j] = temp } return array } /* * Recursively freeze a deeply nested object * https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze */ export let deepFreeze = function (obj: any): any { // Retrieve the property names defined on obj const propNames = Object.getOwnPropertyNames(obj) // Freeze properties before freezing self propNames.forEach((name) => { const prop = obj[name] // Freeze prop if it is an object if (typeof prop === 'object' && prop !== null) { deepFreeze(prop) } }) // Freeze self (no-op if already frozen) return Object.freeze(obj) } /** * Check whether a record name should be excluded from storage */ export const isExcluded = function (exclusionPrefixes: string[], recordName: string): boolean { if (!exclusionPrefixes) { return false } for (const exclusionPrefix of exclusionPrefixes) { if (recordName.startsWith(exclusionPrefix)) { return true } } return false } export const PromiseDelay = (timeout: number) => { return new Promise((resolve) => setTimeout(resolve, timeout)) } /** * Utility method for creating hashes including salts based on * the provided parameters */ export const createHash = (password: string, settings: { iterations: number, keyLength: number, algorithm: string }, salt: string = crypto.randomBytes(16).toString('base64')): Promise<{ hash: Buffer, salt: string}> => { return new Promise((resolve, reject) => { crypto.pbkdf2( password, salt, settings.iterations, settings.keyLength, settings.algorithm, (err, hash) => { err ? reject(err) : resolve({ hash, salt }) } ) }) } export const validateHashingAlgorithm = (hash: string): void => { if (crypto.getHashes().indexOf(hash) === -1) { throw new Error(`Unknown Hash ${hash}`) } } const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/i export const validateUUID = (uuid: string): boolean => { return uuidPattern.test(uuid.toLowerCase()) } ================================================ FILE: telemetry-server/package.json ================================================ { "name": "@deepstream/telemetry-server", "version": "1.0.0", "description": "A server that simply gathers anonymous data and inserts it into postgres", "scripts": { "start": "ts-node telemetry-server.ts" }, "keywords": [ "deepstream", "telemetry" ], "author": "yasserf", "license": "MIT", "dependencies": { "body-parser": "^1.19.0", "express": "^4.17.1", "pg": "^8.1.0", "typescript": "^3.8.3" }, "devDependencies": { "@types/express": "^4.17.6", "@types/pg": "^7.14.3", "ts-node": "^8.10.1" } } ================================================ FILE: telemetry-server/telemetry-server.config.js ================================================ module.exports = { apps : [ { name: "telemetry-server", script: "npm run start", env: { PGUSER:"postgres", PGDATABASE:"deepstream", PGHOST:"localhost", PGPASSWORD:"secretpassword" } } ] } ================================================ FILE: telemetry-server/telemetry-server.ts ================================================ const port = process.env.TELEMETRY_PORT || 8080 import * as express from 'express' import * as bodyParser from 'body-parser' import { Pool } from 'pg' const app = express() const pool = new Pool() async function insertStats (stats: any) { const { record, event, presence, rpc } = stats.enabledFeatures const valueStrings = `('${stats.deploymentId}','${stats.deepstreamVersion}','${stats.nodeVersion}','${stats.platform}','${record}','${event}','${presence}','${rpc}','${JSON.stringify(stats).replace(/'/g, "''")}')` await pool.query(` INSERT INTO "public"."telemetry" (id, version, nodeVersion, platform, record, event, presence, rpc, json) VALUES ${valueStrings} ON CONFLICT (id) DO UPDATE SET version = EXCLUDED.version, nodeVersion = EXCLUDED.nodeVersion, platform = EXCLUDED.platform, record = EXCLUDED.record, event = EXCLUDED.event, presence = EXCLUDED.presence, rpc = EXCLUDED.rpc, json = EXCLUDED.json, revision = telemetry.revision + 1; `) console.log(`Updated ${stats.deploymentId}`) } // parse application/json app.use(bodyParser.json()) app.post('/api/v1/startup', async (req, res) => { await insertStats(req.body) res.json({ success: true }) }) app.listen(port, () => console.log(`Telemetry listening on port ${port}`)) ================================================ FILE: test-e2e/config/permissions-complex.json ================================================ { "presence": { "*": { "allow": "user.id === 'B'" } }, "record": { "*": { "create": true, "delete": true, "write": true, "read": true, "listen": true, "notify": "user.id === 'B'" }, "d/*": { "create": "_('perm/JohnDoe').boolean", "read": "_('perm/JohnDoe').boolean" }, "public-read-private-write/$userid": { "read": true, "create": "user.id === $userid", "write": "user.id === $userid" }, "only-increment": { "write": "!oldData.value || data.value > oldData.value", "create": true, "read": true }, "only-decrement": { "write": "!oldData.value || data.value < oldData.value", "create": true, "read": true }, "only-delete-egon-miller/$firstname/$lastname": { "delete": "$firstname.toLowerCase() === 'egon' && $lastname.toLowerCase() === 'miller'" }, "only-allows-purchase-of-products-in-stock/$purchaseId": { "create": true, "write": "_('item/' + data.itemId ).stock > 0" }, "only-a-can-read-and-create": { "create": "user.id === 'A'", "read": "user.id === 'A'" }, "forbidden": { "create": false, "delete": false, "write": false, "read": false, "listen": false }, "read-only": { "write": false, "read": true }, "deny-read":{ "create": false, "write": false, "read": false, "delete": false, "listen": false }, "deny-write":{ "create": true, "write": false, "read": true, "delete": false, "listen": false } }, "event": { "*": { "listen": true, "publish": true, "subscribe": true }, "open/*": { "listen": true, "publish": true, "subscribe": true }, "forbidden/*": { "publish": false, "subscribe": false }, "a-to-b/*": { "publish": "user.id === 'A'", "subscribe": "user.id === 'B'" }, "news/$topic": { "publish": "$topic === 'tea-cup-pigs'" }, "number": { "publish": "data > 10" }, "place/$city": { "publish": "$city.toLowerCase() === data.address.city.toLowerCase()" }, "deny-this*":{ "publish": false, "subscribe": false, "listen": false }, "admin-publish":{ "publish": "user.data.role === 'admin'" } }, "rpc": { "*": { "provide": true, "request": true }, "a-provide-b-request": { "provide": "user.id === 'A'", "request": "user.id === 'B'" }, "only-full-user-data": { "request": "typeof data.firstname === 'string' && typeof data.lastname === 'string'" }, "deny": { "provide": false, "request": false } } } ================================================ FILE: test-e2e/config/permissions-open.json ================================================ { "presence": { "*": { "allow": true } }, "record": { "*": { "create": true, "delete": true, "write": true, "read": true, "listen": true, "notify": true } }, "event": { "*": { "listen": true, "publish": true, "subscribe": true } }, "rpc": { "*": { "provide": true, "request": true } } } ================================================ FILE: test-e2e/framework/client-handler.ts ================================================ import { DeepstreamClient } from '@deepstream/client' import * as sinon from 'sinon' import { Message } from '../../src/constants' export interface E2EClient { name: string, client: DeepstreamClient, [index: string]: any } const clients: { [index: string]: E2EClient } = {} function createClient (clientName: string, server: string, options?: any) { const deepstreamUrl = global.e2eHarness.getUrl(server) // @ts-ignore const client = new DeepstreamClient(`${deepstreamUrl}-v4`, { ...options, subscriptionInterval: 5, maxReconnectInterval: 300, maxReconnectAttempts: 20, rpcAcceptTimeout: 100, rpcResponseTimeout: 300, subscriptionTimeout: 100, recordReadAckTimeout: 100, recordReadTimeout: 50, recordDeleteTimeout: 100, recordDiscardTimeout: 100, intervalTimerResolution: 1, offlineEnabled: false, offlineBufferTimeout: 10000, nativeTimerRegistry: false, initialRecordVersion: 1, socketOptions: { jsonTransportMode: false } }) clients[clientName] = { name: clientName, client, login: sinon.spy(), error: {}, connectionStateChanged: sinon.spy(), clientDataChanged: sinon.spy(), reauthenticationFailure: sinon.spy(), event: { callbacks: {}, callbacksListeners: {}, callbacksListenersSpies: {}, callbacksListenersResponse: {}, }, record: { records: { // Creates a similar structure when record is requests xxx: { record: null, discardCallback: null, deleteCallback: null, callbackError: null, subscribeCallback: null, subscribePathCallbacks: {} } }, lists: { xxx: { list: null, discardCallback: null, deleteCallback: null, callbackError: null, subscribeCallback: null, addedCallback: null, removedCallback: null, movedCallback: null } }, anonymousRecord: null, snapshotCallback: sinon.spy(), hasCallback: sinon.spy(), headCallback: sinon.spy(), callbacksListeners: {}, callbacksListenersSpies: {}, callbacksListenersResponse: {}, }, rpc: { callbacks: {}, provides: {}, callbacksListeners: {}, callbacksListenersSpies: {}, callbacksListenersResponse: {}, }, presence: { callbacks: {} } } clients[clientName].client.on('error', (message: Message, event: string, topic: number) => { if (process.env.DEBUG_LOG) { console.log('An Error occured on', clientName, message, event, topic) } if (!clients[clientName]) { return } const clientErrors = clients[clientName].error clientErrors[topic] = clientErrors[topic] || {} clientErrors[topic][event] = clientErrors[topic][event] || sinon.spy() clients[clientName].error[topic][event](message) }) clients[clientName].client.on('connectionStateChanged', (state: string) => { if (!clients[clientName]) { return } clients[clientName].connectionStateChanged(state) }) clients[clientName].client.on('clientDataChanged', (clientData: any) => { if (!clients[clientName]) { return } clients[clientName].clientDataChanged(clientData) }) clients[clientName].client.on('reauthenticationFailure', (reason: string) => { if (!clients[clientName]) { return } clients[clientName].reauthenticationFailure(reason) }) return clients[clientName] } function getClientNames (expression: string) { const clientExpression = /all clients|(?:subscriber|publisher|clients?) ([^\s']*)(?:'s)?/ const result = clientExpression.exec(expression)! if (result[0] === 'all clients') { return Object.keys(clients) } else if (result.length === 2 && result[1].indexOf(',') > -1) { return result[1].replace(/"/g, '').split(',') } else if (result.length === 2) { return [result[1].replace(/"/g, '')] } throw new Error(`Invalid expression: ${expression}`) } function getClients (expression: string) { return getClientNames(expression).map((client) => clients[client]) } function assertNoErrors (client: string) { const clientErrors = clients[client].error for (const topic in clientErrors) { for (const event in clientErrors[topic]) { sinon.assert.notCalled(clientErrors[topic][event]) } } } export const clientHandler = { clients, createClient, getClientNames, getClients, assertNoErrors } ================================================ FILE: test-e2e/framework/client.ts ================================================ // tslint:disable:no-shadowed-variable import * as sinon from 'sinon' import { clientHandler } from './client-handler' import { TOPIC, AUTH_ACTION, CONNECTION_ACTION } from '../../src/constants' export const client = { logsOut (clientExpression: string, done: Function) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.close() }) // current sync since protocol doesn't yet support async done() }, connect (clientExpression: string, server: string) { clientHandler.getClientNames(clientExpression).forEach((clientName) => { clientHandler.createClient(clientName, server) }) }, connectAndLogin (clientExpression: string, server: string, done: Function) { clientHandler.getClientNames(clientExpression).forEach((clientName) => { const client = clientHandler.createClient(clientName, server) client.client.login({ username: clientName, password: 'abcdefgh' }, (success, data) => { client.login(success, data) client.user = clientName done() }) }) }, login (clientExpression: string, username: string, password: string, done: Function) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.login({ username, password }, (success, data) => { client.login(success, data) client.user = username done() }) }) }, attemptLogin (clientExpression: string, username: string, password: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.login({ username, password }) }) }, receivedTooManyLoginAttempts (clientExpression: string) { clientHandler.getClients(clientExpression).forEach((client) => { const errorSpy = client.error[TOPIC[TOPIC.AUTH]][AUTH_ACTION[AUTH_ACTION.TOO_MANY_AUTH_ATTEMPTS]] sinon.assert.calledOnce(errorSpy) errorSpy.resetHistory() }) }, recievesNoLoginResponse (clientExpression: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.notCalled(client.login) }) }, recievesLoginResponse (clientExpression: string, loginFailed: boolean, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const loginSpy = client.login if (!loginFailed) { sinon.assert.calledOnce(loginSpy) if (data) { sinon.assert.calledWith(loginSpy, true, JSON.parse(data)) } else { sinon.assert.calledWith(loginSpy, true) } } else { sinon.assert.calledOnce(loginSpy) sinon.assert.calledWith(loginSpy, false) } loginSpy.resetHistory() }) }, connectionTimesOut (clientExpression: string, done: Function) { clientHandler.getClients(clientExpression).forEach((client) => { setTimeout(() => { const errorSpy = client.error[TOPIC[TOPIC.CONNECTION]][CONNECTION_ACTION[CONNECTION_ACTION.AUTHENTICATION_TIMEOUT]] sinon.assert.calledOnce(errorSpy) errorSpy.resetHistory() done() }, 1000) }) }, receivedErrorOnce (clientExpression: string, topicName: string, eventName: string) { const topic = topicName.toUpperCase() clientHandler.getClients(clientExpression).forEach((client) => { const errorSpy = client.error[topic][eventName] sinon.assert.called(errorSpy) errorSpy.resetHistory() }) }, receivedOneError (clientExpression: string, topicName: string, eventName: string) { // @ts-ignore const topic = TOPIC[TOPIC[topicName.toUpperCase()]] const event = eventName.toUpperCase() clientHandler.getClients(clientExpression).forEach((client) => { const errorSpy = client.error[topic][event] sinon.assert.calledOnce(errorSpy) errorSpy.resetHistory() }) }, callbackCalled (clientExpression: string, eventName: string, notCalled: boolean, once: boolean, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const spy = client[eventName] if (notCalled) { sinon.assert.notCalled(spy) } else { if (once) { sinon.assert.calledOnce(spy) } else { sinon.assert.called(spy) } if (data !== null) { sinon.assert.calledWith(spy, JSON.parse(data)) } } spy.resetHistory() }) }, receivedNoErrors (clientExpression: string) { clientHandler.getClients(clientExpression).forEach((client) => { clientHandler.assertNoErrors(client.name) }) }, hadConnectionState (clientExpression: string, had: boolean, state: boolean) { clientHandler.getClients(clientExpression).forEach((client) => { if (had) { sinon.assert.calledWith(client.connectionStateChanged, state) } else { sinon.assert.neverCalledWith(client.connectionStateChanged, state) } }) }, } ================================================ FILE: test-e2e/framework/event.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' import { parseData } from './utils' const assert = { received (clientExpression: string, doesReceive: boolean, subscriptionName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const eventSpy = client.event.callbacks[subscriptionName] if (doesReceive) { sinon.assert.calledOnce(eventSpy) sinon.assert.calledWith(eventSpy, parseData(data)) eventSpy.resetHistory() } else { sinon.assert.notCalled(eventSpy) } }) }, } export const event = { assert, publishes (clientExpression: string, subscriptionName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.event.emit(subscriptionName, parseData(data)) }) }, subscribes (clientExpression: string, subscriptionName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.event.callbacks[subscriptionName] = sinon.spy() client.client.event.subscribe(subscriptionName, client.event.callbacks[subscriptionName]) }) }, unsubscribes (clientExpression: string, subscriptionName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.event.unsubscribe(subscriptionName, client.event.callbacks[subscriptionName]) client.event.callbacks[subscriptionName].isSubscribed = false }) } } ================================================ FILE: test-e2e/framework/listening.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' import { ListenResponse } from '@deepstream/client/dist/src/util/listener' const clients = clientHandler.clients type ListenType = 'record' | 'event' export const assert = { doesNotRecieveMatch (client: string, type: ListenType, match: boolean, pattern: string) { const listenCallbackSpy = clients[client][type].callbacksListenersSpies[pattern].start sinon.assert.neverCalledWith(listenCallbackSpy, match) }, recievesMatch (client: string, count: number, type: ListenType, subscriptionName: string, pattern: string) { const listenCallbackSpy = clients[client][type].callbacksListenersSpies[pattern].start sinon.assert.callCount(listenCallbackSpy.withArgs(subscriptionName), Number(count)) }, receivedUnMatch (client: string, count: number, type: ListenType, subscriptionName: string, pattern: string) { const listenCallbackSpy = clients[client][type].callbacksListenersSpies[pattern].stop sinon.assert.callCount(listenCallbackSpy.withArgs(subscriptionName), Number(count)) } } export const listening = { assert, setupListenResponse (client: string, accepts: boolean, type: ListenType, subscriptionName: string, pattern: string) { clients[client][type].callbacksListenersSpies[pattern].start.withArgs(subscriptionName) clients[client][type].callbacksListenersSpies[pattern].stop.withArgs(subscriptionName) clients[client][type].callbacksListenersResponse[pattern] = accepts }, listens (client: string, type: ListenType, pattern: string) { if (!clients[client][type].callbacksListenersSpies[pattern]) { clients[client][type].callbacksListenersSpies[pattern] = { start: sinon.spy(), stop: sinon.spy() } } clients[client][type].callbacksListeners[pattern] = (subscriptionName: string, response: ListenResponse) => { if (clients[client][type].callbacksListenersResponse[pattern]) { response.accept() } else { response.reject() } response.onStop(clients[client][type].callbacksListenersSpies[pattern].stop) clients[client][type].callbacksListenersSpies[pattern].start(subscriptionName) } clients[client].client[type].listen(pattern, clients[client][type].callbacksListeners[pattern]) }, unlistens (client: string, type: ListenType, pattern: string) { clients[client].client[type].unlisten(pattern) clients[client][type].callbacksListeners[pattern].isListening = false } } ================================================ FILE: test-e2e/framework/presence.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' import { Dictionary } from 'ts-essentials' const subscribeEvent = 'subscribe' const queryEvent = 'query' export const assert = { notifiedUserStateChanged (notifeeExpression: string, not: boolean, notiferExpression: string, event: string) { clientHandler.getClients(notifeeExpression).forEach((notifee) => { clientHandler.getClients(notiferExpression).forEach((notifier) => { if (not) { sinon.assert.neverCalledWith(notifee.presence.callbacks[subscribeEvent], notifier.user, event === 'in') } else { sinon.assert.calledWith(notifee.presence.callbacks[subscribeEvent], notifier.user, event === 'in') } }) notifee.presence.callbacks[subscribeEvent].resetHistory() }) }, globalQueryResult (clientExpression: string, error: null | string, users?: string[]) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.presence.callbacks[queryEvent]) if (users) { sinon.assert.calledWith(client.presence.callbacks[queryEvent], error, users) } else { sinon.assert.calledWith(client.presence.callbacks[queryEvent], error) } client.presence.callbacks[queryEvent].resetHistory() }) }, queryResult (clientExpression: string, users: string[], online: boolean) { clientHandler.getClients(clientExpression).forEach((client) => { const result = users.reduce((r, user) => { r[user] = online return r }, {} as Dictionary) sinon.assert.calledOnce(client.presence.callbacks[queryEvent]) sinon.assert.calledWith(client.presence.callbacks[queryEvent], null, result) client.presence.callbacks[queryEvent].resetHistory() }) } } export const presence = { assert, subscribe (clientExpression: string, user?: string) { clientHandler.getClients(clientExpression).forEach((client) => { if (!client.presence.callbacks[subscribeEvent]) { client.presence.callbacks[subscribeEvent] = sinon.spy() } if (user) { client.client.presence.subscribe(user, client.presence.callbacks[subscribeEvent]) } else { client.client.presence.subscribe(client.presence.callbacks[subscribeEvent]) } }) }, unsubscribe (clientExpression: string, user?: string) { clientHandler.getClients(clientExpression).forEach((client) => { if (user) { client.client.presence.unsubscribe(user) } else { client.client.presence.unsubscribe() } }) }, getAll (clientExpression: string, users?: string[]) { clientHandler.getClients(clientExpression).forEach((client) => { client.presence.callbacks[queryEvent] = sinon.spy() if (users) { client.client.presence.getAll(users, client.presence.callbacks[queryEvent]) } else { client.client.presence.getAll(client.presence.callbacks[queryEvent]) } }) } } ================================================ FILE: test-e2e/framework/record.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' import * as utils from './utils' import * as assert from 'assert' function getRecordData (clientExpression: string, recordName: string) { return clientHandler.getClients(clientExpression).map((client) => client.record.records[recordName]) } function getListData (clientExpression: string, listName: string) { return clientHandler.getClients(clientExpression).map((client) => client.record.lists[listName]) } const assert2 = { deleted (clientExpression: string, recordName: string, called: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (called) { sinon.assert.calledOnce(recordData.deleteCallback) recordData.deleteCallback.resetHistory() } else { sinon.assert.notCalled(recordData.deleteCallback) } recordData.deleteCallback.resetHistory() }) }, discarded (clientExpression: string, recordName: string, called: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (called) { sinon.assert.calledOnce(recordData.discardCallback) recordData.discardCallback.resetHistory() } else { sinon.assert.notCalled(recordData.discardCallback) } }) }, receivedUpdate (clientExpression: string, recordName: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.calledOnce(recordData.subscribeCallback) sinon.assert.calledWith(recordData.subscribeCallback, data) recordData.subscribeCallback.resetHistory() }) }, receivedUpdateForPath (clientExpression: string, recordName: string, path: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.calledOnce(recordData.subscribePathCallbacks[path]) sinon.assert.calledWith(recordData.subscribePathCallbacks[path], data) recordData.subscribePathCallbacks[path].resetHistory() }) }, receivedNoUpdate (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.notCalled(recordData.subscribeCallback) }) }, receivedNoUpdateForPath (clientExpression: string, recordName: string, path: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.notCalled(recordData.subscribePathCallbacks[path]) }) }, receivedRecordError (clientExpression: string, error: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.calledWith(recordData.errorCallback, error) recordData.errorCallback.resetHistory() }) }, hasData (clientExpression: string, recordName: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { assert.deepEqual(recordData.record.get(), data) }) }, hasProviders (clientExpression: string, recordName: string, without: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { assert.deepEqual(recordData.record.hasProvider, !without) }) }, hasDataAtPath (clientExpression: string, recordName: string, path: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { assert.deepEqual(recordData.record.get(path), data) }) }, writeAckSuccess (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (!recordData) { return } sinon.assert.calledOnce(recordData.setCallback) sinon.assert.calledWith(recordData.setCallback, null) recordData.setCallback.resetHistory() }) clientHandler.getClients(clientExpression).forEach((client) => { if (!client.record.writeAcks) { return } sinon.assert.calledOnce(client.record.writeAcks[recordName]) sinon.assert.calledWith(client.record.writeAcks[recordName], null) client.record.writeAcks[recordName].resetHistory() }) }, writeAckError (clientExpression: string, recordName: string, errorMessage: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (!recordData) { return } sinon.assert.calledOnce(recordData.setCallback) sinon.assert.calledWith(recordData.setCallback, errorMessage) recordData.setCallback.resetHistory() }) clientHandler.getClients(clientExpression).forEach((client) => { if (!client.record.writeAcks) { return } sinon.assert.calledOnce(client.record.writeAcks[recordName]) sinon.assert.calledWith(client.record.writeAcks[recordName], errorMessage) client.record.writeAcks[recordName].resetHistory() }) }, snapshotSuccess (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.snapshotCallback) sinon.assert.calledWith(client.record.snapshotCallback, null, utils.parseData(data)) client.record.snapshotCallback.resetHistory() }) }, snapshotError (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.snapshotCallback) sinon.assert.calledWith(client.record.snapshotCallback, data.replace(/"/g, '')) client.record.snapshotCallback.resetHistory() }) }, headSuccess (clientExpression: string, recordName: string, data: number) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.headCallback) sinon.assert.calledWith(client.record.headCallback, null, data) client.record.headCallback.resetHistory() }) }, headError (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.headCallback) sinon.assert.calledWith(client.record.headCallback, data.replace(/"/g, '')) client.record.snapshotCallback.resetHistory() }) }, has (clientExpression: string, recordName: string, expected: boolean) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.hasCallback) sinon.assert.calledWith(client.record.hasCallback, null, expected) client.record.hasCallback.resetHistory() }) }, hasEntries (clientExpression: string, listName: string, data: string) { data = utils.parseData(data) getListData(clientExpression, listName).forEach((listData) => { assert.deepEqual(listData.list.getEntries(), data) }) }, addedNotified (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { sinon.assert.calledWith(listData.addedCallback, entryName) }) }, removedNotified (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { sinon.assert.calledWith(listData.removedCallback, entryName) }) }, movedNotified (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { sinon.assert.calledWith(listData.movedNotified, entryName) }) }, listChanged (clientExpression: string, listName: string, data: string) { data = utils.parseData(data) getListData(clientExpression, listName).forEach((listData) => { // sinon.assert.calledOnce( listData.subscribeCallback ); sinon.assert.calledWith(listData.subscribeCallback, data) }) }, anonymousRecordContains (clientExpression: string, data: string) { data = utils.parseData(data) clientHandler.getClients(clientExpression).forEach((client) => { assert.deepEqual(client.record.anonymousRecord.get(), data) }) } } export const record = { assert: assert2, getRecord (clientExpression: string, recordName: string) { const clients = clientHandler.getClients(clientExpression) clients.forEach((client) => { const recordData = { record: client.client.record.getRecord(recordName), discardCallback: sinon.spy(), deleteSuccessCallback: sinon.spy(), deleteCallback: sinon.spy(), callbackError: sinon.spy(), subscribeCallback: sinon.spy(), errorCallback: sinon.spy(), setCallback: undefined, subscribePathCallbacks: {} } recordData.record.on('delete', recordData.deleteCallback) recordData.record.on('error', recordData.errorCallback) recordData.record.on('discard', recordData.discardCallback) client.record.records[recordName] = recordData }) }, subscribe (clientExpression: string, recordName: string, immediate: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.subscribe(recordData.subscribeCallback, !!immediate) }) }, unsubscribe (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.unsubscribe(recordData.subscribeCallback) }) }, subscribeWithPath (clientExpression: string, recordName: string, path: string, immediate: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.subscribePathCallbacks[path] = sinon.spy() recordData.record.subscribe(path, recordData.subscribePathCallbacks[path], !!immediate) }) }, unsubscribeFromPath (clientExpression: string, recordName: string, path: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.unsubscribe(path, recordData.subscribePathCallbacks[path]) }) }, discard (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.discard() }) }, delete (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.delete(recordData.deleteSuccessCallback) }) }, setupWriteAck (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.record.records[recordName].setCallback = sinon.spy() }) }, set (clientExpression: string, recordName: string, data: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (recordData.setCallback) { recordData.record.set(utils.parseData(data), recordData.setCallback) } else { recordData.record.set(utils.parseData(data)) } }) }, setWithPath (clientExpression: string, recordName: string, path: string, data: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (recordData.setCallback) { recordData.record.set(path, utils.parseData(data), recordData.setCallback) } else { recordData.record.set(path, utils.parseData(data)) } }) }, erase (clientExpression: string, recordName: string, path: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (recordData.setCallback) { recordData.record.erase(path, recordData.setCallback) } else { recordData.record.erase(path) } }) }, setData (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.setData(recordName, utils.parseData(data)) }) }, setDataWithWriteAck (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { if (!client.record.writeAcks) { client.record.writeAcks = {} } client.record.writeAcks[recordName] = sinon.spy() client.client.record.setData(recordName, utils.parseData(data), client.record.writeAcks[recordName]) }) }, setDataWithPath (clientExpression: string, recordName: string, path: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.setData(recordName, path, utils.parseData(data)) }) }, snapshot (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.snapshot(recordName, client.record.snapshotCallback) }) }, has (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.has(recordName, client.record.hasCallback) }) }, head (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.head(recordName, client.record.headCallback) }) }, /** ****************************************************************************************************************************** *********************************************************** Lists ************************************************************ ********************************************************************************************************************************/ getList (clientExpression: string, listName: string) { clientHandler.getClients(clientExpression).forEach((client) => { const listData = { list: client.client.record.getList(listName), discardCallback: sinon.spy(), deleteCallback: sinon.spy(), callbackError: sinon.spy(), subscribeCallback: sinon.spy(), addedCallback: sinon.spy(), removedCallback: sinon.spy(), movedCallback: sinon.spy() } listData.list.on('discard', listData.discardCallback) listData.list.on('delete', listData.deleteCallback) listData.list.on('entry-added', listData.addedCallback) listData.list.on('entry-removed', listData.removedCallback) listData.list.on('entry-moved', listData.movedCallback) listData.list.subscribe(listData.subscribeCallback) client.record.lists[listName] = listData }) }, setEntries (clientExpression: string, listName: string, data: string) { const entries = utils.parseData(data) getListData(clientExpression, listName).forEach((listData) => { listData.list.setEntries(entries) }) }, addEntry (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { listData.list.addEntry(entryName) }) }, removeEntry (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { listData.list.removeEntry(entryName) }) }, /** ****************************************************************************************************************************** *********************************************************** ANONYMOUS RECORDS ************************************************************ ********************************************************************************************************************************/ getAnonymousRecord (clientExpression: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.record.anonymousRecord = client.client.record.getAnonymousRecord() }) }, setName (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.record.anonymousRecord.setName(recordName) }) } } ================================================ FILE: test-e2e/framework/rpc.ts ================================================ // tslint:disable:no-shadowed-variable import * as sinon from 'sinon' import { clientHandler, E2EClient } from './client-handler' import { RPCResponse } from '@deepstream/client/dist/src/rpc/rpc-response' let rejected = false const rpcs: { [index: string]: (client: E2EClient, data: any, response: RPCResponse) => void } = { 'addTwo': (client, data, response) => { client.rpc.provides.addTwo() response.send(data.numA + data.numB) }, 'double': (client, data, response) => { client.rpc.provides.double() response.send(data * 2) }, 'stringify': (client, data, response) => { client.rpc.provides.stringify() response.send(typeof data === 'object' ? JSON.stringify(data) : String(data)) }, 'a-provide-b-request': (client, data, response) => { client.rpc.provides['a-provide-b-request']() response.send(data * 3) }, 'only-full-user-data': (client, data, response) => { client.rpc.provides['only-full-user-data']() response.send('ok') }, 'alwaysReject': (client, data, response) => { client.rpc.provides.alwaysReject() response.reject() }, 'alwaysError': (client, data, response) => { client.rpc.provides.alwaysError() response.error('always errors') }, 'neverRespond': (client) => { client.rpc.provides.neverRespond() }, 'clientBRejects': (client, data, response) => { client.rpc.provides.clientBRejects() if (client.name === 'B') { response.reject() } else { response.send(data.root * data.root) } }, 'deny': (client, data, response) => { // permissions always deny }, 'rejectOnce': (client, data, response) => { client.rpc.provides.rejectOnce(data) if (rejected) { response.send('ok') rejected = false } else { response.reject() rejected = true } } } const assert = { recievesResponse (clientExpression: string, rpc: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.rpc.callbacks[rpc]) sinon.assert.calledWith(client.rpc.callbacks[rpc], null, JSON.parse(data).toString()) client.rpc.callbacks[rpc].resetHistory() }) }, recievesResponseWithError (clientExpression: string, eventually: boolean, rpc: string, error: string, done: Function) { setTimeout(() => { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.rpc.callbacks[rpc]) sinon.assert.calledWith(client.rpc.callbacks[rpc], error) client.rpc.callbacks[rpc].resetHistory() done() }) }, eventually ? 150 : 0) }, providerCalled (clientExpression: string, rpc: string, timesCalled: number, data?: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.callCount(client.rpc.provides[rpc], timesCalled) if (data) { sinon.assert.calledWith(client.rpc.provides[rpc], JSON.parse(data)) } client.rpc.provides[rpc].resetHistory() }) } } export const rpc = { assert, provide (clientExpression: string, rpc: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.rpc.provides[rpc] = sinon.spy() client.client.rpc.provide(rpc, rpcs[rpc].bind(null, client)) }) }, unprovide (clientExpression: string, rpc: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.rpc.unprovide(rpc) }) }, make (clientExpression: string, rpc: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const callback = client.rpc.callbacks[rpc] = sinon.spy() client.client.rpc.make(rpc, JSON.parse(data), (error, result) => { callback(error, result && result.toString()) }) }) } } ================================================ FILE: test-e2e/framework/utils.ts ================================================ export const defaultDelay: number = Number(process.env.DEFAULT_DELAY) || 10 export const parseData = (data: string) => { if (data === undefined || data === 'undefined') { return undefined } else if (data === 'null') { return null } try { return JSON.parse(data) } catch (e) { return data } } ================================================ FILE: test-e2e/framework/world.ts ================================================ // tslint:disable:no-shadowed-variable import { clientHandler } from './client-handler' export const world = { endTest (done: (...args: any[]) => void) { const clients = clientHandler.clients for (const client in clients) { clientHandler.assertNoErrors(client) for (const event in clients[client].event.callbacks) { if (clients[client].event.callbacks[event].isSubscribed !== false) { clients[client].client.event.unsubscribe(event, clients[client].event.callbacks[event]) } } setTimeout(function (client: string) { for (const pattern in clients[client].event.callbacksListeners) { if (clients[client].event.callbacksListeners[pattern].isListening !== false) { clients[client].client.event.unlisten(pattern) } } }.bind(null, client), 1) setTimeout(function (client: string) { clients[client].client.close() delete clients[client] }.bind(null, client), 50) } setTimeout(done, 100) } } ================================================ FILE: test-e2e/framework-v3/client-handler.ts ================================================ import * as deepstream from 'deepstream.io-client-js' import * as sinon from 'sinon' import { Message } from '../../src/constants' export interface E2EClient { name: string, client: any, [index: string]: any } const clients: { [index: string]: E2EClient } = {} function createClient (clientName: string, server: string, options?: any) { const deepstreamUrl = global.e2eHarness.getUrl(server) // @ts-ignore const client = deepstream(`${deepstreamUrl}-v3`, { ...options, silentDeprecation: true, subscriptionInterval: 5, maxReconnectInterval: 300, maxReconnectAttempts: 20, rpcAcceptTimeout: 100, rpcResponseTimeout: 300, subscriptionTimeout: 100, recordReadAckTimeout: 100, recordReadTimeout: 50, recordDeleteTimeout: 100, recordDiscardTimeout: 100, timerResolution: 1, offlineEnabled: false, offlineBufferTimeout: 10000 }) clients[clientName] = { name: clientName, client, login: sinon.spy(), error: {}, connectionStateChanged: sinon.spy(), clientDataChanged: sinon.spy(), reauthenticationFailure: sinon.spy(), event: { callbacks: {}, callbacksListeners: {}, callbacksListenersSpies: {}, callbacksListenersResponse: {}, }, record: { records: { // Creates a similar structure when record is requests xxx: { record: null, discardCallback: null, deleteCallback: null, callbackError: null, subscribeCallback: null, subscribePathCallbacks: {} } }, lists: { xxx: { list: null, discardCallback: null, deleteCallback: null, callbackError: null, subscribeCallback: null, addedCallback: null, removedCallback: null, movedCallback: null } }, anonymousRecord: null, snapshotCallback: sinon.spy(), hasCallback: sinon.spy(), headCallback: sinon.spy(), callbacksListeners: {}, callbacksListenersSpies: {}, callbacksListenersResponse: {}, }, rpc: { callbacks: {}, provides: {}, callbacksListeners: {}, callbacksListenersSpies: {}, callbacksListenersResponse: {}, }, presence: { callbacks: {} } } clients[clientName].client.on('error', (message: Message, event: string, topic: number) => { if (process.env.DEBUG_LOG) { console.log('An Error occured on', clientName, message, event, topic) } if (event === 'MULTIPLE_SUBSCRIPTIONS' || event === 'UNSOLICITED_MESSAGE') { return } if (!clients[clientName]) { return } const clientErrors = clients[clientName].error clientErrors[topic] = clientErrors[topic] || {} clientErrors[topic][event] = clientErrors[topic][event] || sinon.spy() clients[clientName].error[topic][event](message) }) clients[clientName].client.on('connectionStateChanged', (state: string) => { if (!clients[clientName]) { return } clients[clientName].connectionStateChanged(state) }) clients[clientName].client.on('clientDataChanged', (clientData: any) => { if (!clients[clientName]) { return } clients[clientName].clientDataChanged(clientData) }) clients[clientName].client.on('reauthenticationFailure', (reason: string) => { if (!clients[clientName]) { return } clients[clientName].reauthenticationFailure(reason) }) return clients[clientName] } function getClientNames (expression: string) { const clientExpression = /all clients|(?:subscriber|publisher|clients?) ([^\s']*)(?:'s)?/ const result = clientExpression.exec(expression)! if (result[0] === 'all clients') { return Object.keys(clients) } else if (result.length === 2 && result[1].indexOf(',') > -1) { return result[1].replace(/"/g, '').split(',') } else if (result.length === 2) { return [result[1].replace(/"/g, '')] } throw new Error(`Invalid expression: ${expression}`) } function getClients (expression: string) { return getClientNames(expression).map((client) => clients[client]) } function assertNoErrors (client: string) { const clientErrors = clients[client].error for (const topic in clientErrors) { for (const event in clientErrors[topic]) { sinon.assert.notCalled(clientErrors[topic][event]) } } } export const clientHandler = { clients, createClient, getClientNames, getClients, assertNoErrors } ================================================ FILE: test-e2e/framework-v3/client.ts ================================================ // tslint:disable:no-shadowed-variable import * as sinon from 'sinon' import { clientHandler } from './client-handler' const compat: any = { CONNECTION: 'C', EVENT: 'E', RECORD: 'R', PRESENCE: 'U', RPC: 'P', AUTH: 'A', CONNECTION_ERROR: 'connectionError', RECORD_NOT_FOUND: 'RECORD_NOT_FOUND' } export const client = { logsOut (clientExpression: string, done: Function) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.close() }) // current sync since protocol doesn't yet support async done() }, connect (clientExpression: string, server: string) { clientHandler.getClientNames(clientExpression).forEach((clientName) => { clientHandler.createClient(clientName, server) }) }, connectAndLogin (clientExpression: string, server: string, done: Function) { clientHandler.getClientNames(clientExpression).forEach((clientName) => { const client = clientHandler.createClient(clientName, server) client.client.login({ username: clientName, password: 'abcdefgh' }, (success: boolean, data: any) => { client.login(success, data) client.user = clientName done() }) }) }, login (clientExpression: string, username: string, password: string, done: Function) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.login({ username, password }, (success: boolean, data: any) => { client.login(success, data) client.user = username done() }) }) }, attemptLogin (clientExpression: string, username: string, password: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.login({ username, password }) }) }, receivedTooManyLoginAttempts (clientExpression: string) { // Not a < V4 feature // clientHandler.getClients(clientExpression).forEach((client) => { // const errorSpy = client.error.A.TOO_MANY_AUTH_ATTEMPTS // sinon.assert.calledOnce(errorSpy) // errorSpy.resetHistory() // }) }, recievesNoLoginResponse (clientExpression: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.notCalled(client.login) }) }, recievesLoginResponse (clientExpression: string, loginFailed: boolean, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const loginSpy = client.login if (!loginFailed) { sinon.assert.calledOnce(loginSpy) if (data) { sinon.assert.calledWith(loginSpy, true, JSON.parse(data)) } else { sinon.assert.calledWith(loginSpy, true) } } else { sinon.assert.called(loginSpy) sinon.assert.calledWith(loginSpy, false) } loginSpy.resetHistory() }) }, connectionTimesOut (clientExpression: string, done: Function) { clientHandler.getClients(clientExpression).forEach((client) => { setTimeout(() => { const errorSpy = client.error.C.CONNECTION_AUTHENTICATION_TIMEOUT sinon.assert.calledOnce(errorSpy) errorSpy.resetHistory() done() }, 1000) }) }, receivedErrorOnce (clientExpression: string, topicName: string, eventName: string) { const topic = topicName.toUpperCase() clientHandler.getClients(clientExpression).forEach((client) => { const errorSpy = client.error[compat[topic]][compat[eventName]] sinon.assert.called(errorSpy) errorSpy.resetHistory() }) }, receivedOneError (clientExpression: string, topicName: string, eventName: string) { let topic = compat[topicName] const event = eventName.toUpperCase() clientHandler.getClients(clientExpression).forEach((client) => { if (event === 'IS_CLOSED') { topic = 'X' } const errorSpy = client.error[topic][event] sinon.assert.calledOnce(errorSpy) errorSpy.resetHistory() }) }, callbackCalled (clientExpression: string, eventName: string, notCalled: boolean, once: boolean, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const spy = client[eventName] if (notCalled) { sinon.assert.notCalled(spy) } else { if (once) { sinon.assert.calledOnce(spy) } else { sinon.assert.called(spy) } if (data !== undefined) { sinon.assert.calledWith(spy, JSON.parse(data)) } } spy.resetHistory() }) }, receivedNoErrors (clientExpression: string) { clientHandler.getClients(clientExpression).forEach((client) => { clientHandler.assertNoErrors(client.name) }) }, hadConnectionState (clientExpression: string, had: boolean, state: boolean) { clientHandler.getClients(clientExpression).forEach((client) => { if (had) { sinon.assert.calledWith(client.connectionStateChanged, state) } else { sinon.assert.neverCalledWith(client.connectionStateChanged, state) } }) }, } ================================================ FILE: test-e2e/framework-v3/event.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' import { parseData } from './utils' const assert = { received (clientExpression: string, doesReceive: boolean, subscriptionName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const eventSpy = client.event.callbacks[subscriptionName] if (doesReceive) { sinon.assert.calledOnce(eventSpy) sinon.assert.calledWith(eventSpy, parseData(data)) eventSpy.resetHistory() } else { sinon.assert.notCalled(eventSpy) } }) }, } export const event = { assert, publishes (clientExpression: string, subscriptionName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.event.emit(subscriptionName, parseData(data)) }) }, subscribes (clientExpression: string, subscriptionName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.event.callbacks[subscriptionName] = sinon.spy() client.client.event.subscribe(subscriptionName, client.event.callbacks[subscriptionName]) }) }, unsubscribes (clientExpression: string, subscriptionName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.event.unsubscribe(subscriptionName, client.event.callbacks[subscriptionName]) client.event.callbacks[subscriptionName].isSubscribed = false }) } } ================================================ FILE: test-e2e/framework-v3/listening.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' const clients = clientHandler.clients type ListenType = 'record' | 'event' export const assert = { doesNotRecieveMatch (client: string, type: ListenType, match: boolean, pattern: string) { const listenCallbackSpy = clients[client][type].callbacksListenersSpies[pattern].start sinon.assert.neverCalledWith(listenCallbackSpy, match) }, recievesMatch (client: string, count: number, type: ListenType, subscriptionName: string, pattern: string) { const listenCallbackSpy = clients[client][type].callbacksListenersSpies[pattern].start sinon.assert.callCount(listenCallbackSpy.withArgs(subscriptionName), Number(count)) }, receivedUnMatch (client: string, count: number, type: ListenType, subscriptionName: string, pattern: string) { const listenCallbackSpy = clients[client][type].callbacksListenersSpies[pattern].stop sinon.assert.callCount(listenCallbackSpy.withArgs(subscriptionName), Number(count)) } } export const listening = { assert, setupListenResponse (client: string, accepts: boolean, type: ListenType, subscriptionName: string, pattern: string) { clients[client][type].callbacksListenersSpies[pattern].start.withArgs(subscriptionName) clients[client][type].callbacksListenersSpies[pattern].stop.withArgs(subscriptionName) clients[client][type].callbacksListenersResponse[pattern] = accepts }, listens (client: string, type: ListenType, pattern: string) { if (!clients[client][type].callbacksListenersSpies[pattern]) { clients[client][type].callbacksListenersSpies[pattern] = { start: sinon.spy(), stop: sinon.spy() } } clients[client][type].callbacksListeners[pattern] = (subscriptionName: string, isSubscribed: boolean, response: any) => { if (isSubscribed) { if (clients[client][type].callbacksListenersResponse[pattern]) { response.accept() } else { response.reject() } clients[client][type].callbacksListenersSpies[pattern].start(subscriptionName) } else { clients[client][type].callbacksListenersSpies[pattern].stop(subscriptionName) } } clients[client].client[type].listen(pattern, clients[client][type].callbacksListeners[pattern]) }, unlistens (client: string, type: ListenType, pattern: string) { clients[client].client[type].unlisten(pattern) clients[client][type].callbacksListeners[pattern].isListening = false } } ================================================ FILE: test-e2e/framework-v3/presence.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' import { client as cl } from './client' import { Dictionary } from 'ts-essentials' const subscribeEvent = 'subscribe' const queryEvent = 'query' export const assert = { notifiedUserStateChanged (notifeeExpression: string, not: boolean, notiferExpression: string, event: string) { clientHandler.getClients(notifeeExpression).forEach((notifee) => { clientHandler.getClients(notiferExpression).forEach((notifier) => { if (not) { sinon.assert.neverCalledWith(notifee.presence.callbacks[subscribeEvent], notifier.user, event === 'in') } else { sinon.assert.calledWith(notifee.presence.callbacks[subscribeEvent], notifier.user, event === 'in') } }) notifee.presence.callbacks[subscribeEvent].resetHistory() }) }, globalQueryResult (clientExpression: string, error: null | string, users?: string[]) { if (error) { cl.receivedOneError(clientExpression, 'PRESENCE', error) return } clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.presence.callbacks[queryEvent]) if (users) { sinon.assert.calledWith(client.presence.callbacks[queryEvent], users) } else { sinon.assert.calledWith(client.presence.callbacks[queryEvent]) } client.presence.callbacks[queryEvent].resetHistory() }) }, queryResult (clientExpression: string, users: string[], online: boolean) { clientHandler.getClients(clientExpression).forEach((client) => { const result = users.reduce((r, user) => { r[user] = online return r }, {} as Dictionary) sinon.assert.calledOnce(client.presence.callbacks[queryEvent]) sinon.assert.calledWith(client.presence.callbacks[queryEvent], result) client.presence.callbacks[queryEvent].resetHistory() }) } } export const presence = { assert, subscribe (clientExpression: string, user?: string) { clientHandler.getClients(clientExpression).forEach((client) => { if (!client.presence.callbacks[subscribeEvent]) { client.presence.callbacks[subscribeEvent] = sinon.spy() } if (user) { client.client.presence.subscribe(user, (...args: any[]) => { client.presence.callbacks[subscribeEvent](args[1], args[0]) }) } else { client.client.presence.subscribe((...args: any[]) => { client.presence.callbacks[subscribeEvent](...args) }) } }) }, unsubscribe (clientExpression: string, user?: string) { clientHandler.getClients(clientExpression).forEach((client) => { if (user) { client.client.presence.unsubscribe(user) } else { client.client.presence.unsubscribe() } }) }, getAll (clientExpression: string, users?: string[]) { clientHandler.getClients(clientExpression).forEach((client) => { client.presence.callbacks[queryEvent] = sinon.spy() if (users) { client.client.presence.getAll(users, (...args: any[]) => { client.presence.callbacks[queryEvent](...args) }) } else { client.client.presence.getAll((...args: any[]) => { client.presence.callbacks[queryEvent](...args) }) } }) } } ================================================ FILE: test-e2e/framework-v3/record.ts ================================================ import * as sinon from 'sinon' import { clientHandler } from './client-handler' import { client as cl } from './client' import * as utils from './utils' import * as assert from 'assert' function getRecordData (clientExpression: string, recordName: string) { return clientHandler.getClients(clientExpression).map((client) => client.record.records[recordName]) } function getListData (clientExpression: string, listName: string) { return clientHandler.getClients(clientExpression).map((client) => client.record.lists[listName]) } const assert2 = { deleted (clientExpression: string, recordName: string, called: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (called) { sinon.assert.calledOnce(recordData.deleteCallback) recordData.deleteCallback.resetHistory() } else { sinon.assert.notCalled(recordData.deleteCallback) } recordData.deleteCallback.resetHistory() }) }, discarded (clientExpression: string, recordName: string, called: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (called) { sinon.assert.calledOnce(recordData.discardCallback) recordData.discardCallback.resetHistory() } else { sinon.assert.notCalled(recordData.discardCallback) } }) }, receivedUpdate (clientExpression: string, recordName: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.calledOnce(recordData.subscribeCallback) sinon.assert.calledWith(recordData.subscribeCallback, data) recordData.subscribeCallback.resetHistory() }) }, receivedUpdateForPath (clientExpression: string, recordName: string, path: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.calledOnce(recordData.subscribePathCallbacks[path]) sinon.assert.calledWith(recordData.subscribePathCallbacks[path], data) recordData.subscribePathCallbacks[path].resetHistory() }) }, receivedNoUpdate (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.notCalled(recordData.subscribeCallback) }) }, receivedNoUpdateForPath (clientExpression: string, recordName: string, path: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { sinon.assert.notCalled(recordData.subscribePathCallbacks[path]) }) }, receivedRecordError (clientExpression: string, error: string, recordName: string) { cl.receivedOneError(clientExpression, 'RECORD', error) // getRecordData(clientExpression, recordName).forEach((recordData) => { // sinon.assert.calledWith(recordData.errorCallback, error) // recordData.errorCallback.resetHistory() // }) }, hasData (clientExpression: string, recordName: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { assert.deepEqual(recordData.record.get(), data) }) }, hasProviders (clientExpression: string, recordName: string, without: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { assert.deepEqual(recordData.record.hasProvider, !without) }) }, hasDataAtPath (clientExpression: string, recordName: string, path: string, data: string) { data = utils.parseData(data) getRecordData(clientExpression, recordName).forEach((recordData) => { assert.deepEqual(recordData.record.get(path), data) }) }, writeAckSuccess (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (!recordData) { return } sinon.assert.calledOnce(recordData.setCallback) sinon.assert.calledWith(recordData.setCallback, null) recordData.setCallback.resetHistory() }) clientHandler.getClients(clientExpression).forEach((client) => { if (!client.record.writeAcks) { return } sinon.assert.calledOnce(client.record.writeAcks[recordName]) sinon.assert.calledWith(client.record.writeAcks[recordName], null) client.record.writeAcks[recordName].resetHistory() }) }, writeAckError (clientExpression: string, recordName: string, errorMessage: string) { cl.receivedOneError(clientExpression, 'RECORD', errorMessage) // getRecordData(clientExpression, recordName).forEach((recordData) => { // if (!recordData) { return } // sinon.assert.calledOnce(recordData.setCallback) // sinon.assert.calledWith(recordData.setCallback, errorMessage) // recordData.setCallback.resetHistory() // }) // clientHandler.getClients(clientExpression).forEach((client) => { // if (!client.record.writeAcks) { return } // sinon.assert.calledOnce(client.record.writeAcks[recordName]) // sinon.assert.calledWith(client.record.writeAcks[recordName], errorMessage) // client.record.writeAcks[recordName].resetHistory() // }) }, snapshotSuccess (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.snapshotCallback) sinon.assert.calledWith(client.record.snapshotCallback, null, utils.parseData(data)) client.record.snapshotCallback.resetHistory() }) }, snapshotError (clientExpression: string, recordName: string, data: string) { cl.receivedOneError(clientExpression, 'RECORD', data) // clientHandler.getClients(clientExpression).forEach((client) => { // sinon.assert.calledOnce(client.record.snapshotCallback) // sinon.assert.calledWith(client.record.snapshotCallback, data.replace(/"/g, '')) // client.record.snapshotCallback.resetHistory() // }) }, headSuccess (clientExpression: string, recordName: string, data: number) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.headCallback) sinon.assert.calledWith(client.record.headCallback, null, data) client.record.headCallback.resetHistory() }) }, headError (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.headCallback) sinon.assert.calledWith(client.record.headCallback, data.replace(/"/g, '')) client.record.snapshotCallback.resetHistory() }) }, has (clientExpression: string, recordName: string, expected: boolean) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.record.hasCallback) sinon.assert.calledWith(client.record.hasCallback, null, expected) client.record.hasCallback.resetHistory() }) }, hasEntries (clientExpression: string, listName: string, data: string) { data = utils.parseData(data) getListData(clientExpression, listName).forEach((listData) => { assert.deepEqual(listData.list.getEntries(), data) }) }, addedNotified (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { sinon.assert.calledWith(listData.addedCallback, entryName) }) }, removedNotified (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { sinon.assert.calledWith(listData.removedCallback, entryName) }) }, movedNotified (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { sinon.assert.calledWith(listData.movedNotified, entryName) }) }, listChanged (clientExpression: string, listName: string, data: string) { data = utils.parseData(data) getListData(clientExpression, listName).forEach((listData) => { // sinon.assert.calledOnce( listData.subscribeCallback ); sinon.assert.calledWith(listData.subscribeCallback, data) }) }, anonymousRecordContains (clientExpression: string, data: string) { data = utils.parseData(data) clientHandler.getClients(clientExpression).forEach((client) => { assert.deepEqual(client.record.anonymousRecord.get(), data) }) } } export const record = { assert: assert2, getRecord (clientExpression: string, recordName: string) { const clients = clientHandler.getClients(clientExpression) clients.forEach((client) => { const recordData = { record: client.client.record.getRecord(recordName), discardCallback: sinon.spy(), deleteSuccessCallback: sinon.spy(), deleteCallback: sinon.spy(), callbackError: sinon.spy(), subscribeCallback: sinon.spy(), errorCallback: sinon.spy(), setCallback: undefined, subscribePathCallbacks: {} } recordData.record.on('delete', recordData.deleteCallback) recordData.record.on('error', recordData.errorCallback) recordData.record.on('discard', recordData.discardCallback) client.record.records[recordName] = recordData }) }, subscribe (clientExpression: string, recordName: string, immediate: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.subscribe(recordData.subscribeCallback, !!immediate) }) }, unsubscribe (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.unsubscribe(recordData.subscribeCallback) }) }, subscribeWithPath (clientExpression: string, recordName: string, path: string, immediate: boolean) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.subscribePathCallbacks[path] = sinon.spy() recordData.record.subscribe(path, recordData.subscribePathCallbacks[path], !!immediate) }) }, unsubscribeFromPath (clientExpression: string, recordName: string, path: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.unsubscribe(path, recordData.subscribePathCallbacks[path]) }) }, discard (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.discard() }) }, delete (clientExpression: string, recordName: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { recordData.record.delete(recordData.deleteSuccessCallback) }) }, setupWriteAck (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.record.records[recordName].setCallback = sinon.spy() }) }, set (clientExpression: string, recordName: string, data: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (recordData.setCallback) { recordData.record.set(utils.parseData(data), recordData.setCallback) } else { recordData.record.set(utils.parseData(data)) } }) }, setWithPath (clientExpression: string, recordName: string, path: string, data: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (recordData.setCallback) { recordData.record.set(path, utils.parseData(data), recordData.setCallback) } else { recordData.record.set(path, utils.parseData(data)) } }) }, erase (clientExpression: string, recordName: string, path: string) { getRecordData(clientExpression, recordName).forEach((recordData) => { if (recordData.setCallback) { recordData.record.set(path, undefined, recordData.setCallback) } else { recordData.record.set(path, undefined) } }) }, setData (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.setData(recordName, utils.parseData(data)) }) }, setDataWithWriteAck (clientExpression: string, recordName: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { if (!client.record.writeAcks) { client.record.writeAcks = {} } client.record.writeAcks[recordName] = sinon.spy() client.client.record.setData(recordName, utils.parseData(data), client.record.writeAcks[recordName]) }) }, setDataWithPath (clientExpression: string, recordName: string, path: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.setData(recordName, path, utils.parseData(data)) }) }, snapshot (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.snapshot(recordName, client.record.snapshotCallback) }) }, has (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.has(recordName, client.record.hasCallback) }) }, head (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.record.head(recordName, client.record.headCallback) }) }, /** ****************************************************************************************************************************** *********************************************************** Lists ************************************************************ ********************************************************************************************************************************/ getList (clientExpression: string, listName: string) { clientHandler.getClients(clientExpression).forEach((client) => { const listData = { list: client.client.record.getList(listName), discardCallback: sinon.spy(), deleteCallback: sinon.spy(), callbackError: sinon.spy(), subscribeCallback: sinon.spy(), addedCallback: sinon.spy(), removedCallback: sinon.spy(), movedCallback: sinon.spy() } listData.list.on('discard', listData.discardCallback) listData.list.on('delete', listData.deleteCallback) listData.list.on('entry-added', listData.addedCallback) listData.list.on('entry-removed', listData.removedCallback) listData.list.on('entry-moved', listData.movedCallback) listData.list.subscribe(listData.subscribeCallback) client.record.lists[listName] = listData }) }, setEntries (clientExpression: string, listName: string, data: string) { const entries = utils.parseData(data) getListData(clientExpression, listName).forEach((listData) => { listData.list.setEntries(entries) }) }, addEntry (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { listData.list.addEntry(entryName) }) }, removeEntry (clientExpression: string, listName: string, entryName: string) { getListData(clientExpression, listName).forEach((listData) => { listData.list.removeEntry(entryName) }) }, /** ****************************************************************************************************************************** *********************************************************** ANONYMOUS RECORDS ************************************************************ ********************************************************************************************************************************/ getAnonymousRecord (clientExpression: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.record.anonymousRecord = client.client.record.getAnonymousRecord() }) }, setName (clientExpression: string, recordName: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.record.anonymousRecord.setName(recordName) }) } } ================================================ FILE: test-e2e/framework-v3/rpc.ts ================================================ // tslint:disable:no-shadowed-variable import * as sinon from 'sinon' import { clientHandler, E2EClient } from './client-handler' import { RPCResponse } from '@deepstream/client/dist/src/rpc/rpc-response' let rejected = false const rpcs: { [index: string]: (client: E2EClient, data: any, response: RPCResponse) => void } = { 'addTwo': (client, data, response) => { client.rpc.provides.addTwo() response.send(data.numA + data.numB) }, 'double': (client, data, response) => { client.rpc.provides.double() response.send(data * 2) }, 'stringify': (client, data, response) => { client.rpc.provides.stringify() response.send(typeof data === 'object' ? JSON.stringify(data) : String(data)) }, 'a-provide-b-request': (client, data, response) => { client.rpc.provides['a-provide-b-request']() response.send(data * 3) }, 'only-full-user-data': (client, data, response) => { client.rpc.provides['only-full-user-data']() response.send('ok') }, 'alwaysReject': (client, data, response) => { client.rpc.provides.alwaysReject() response.reject() }, 'alwaysError': (client, data, response) => { client.rpc.provides.alwaysError() response.error('always errors') }, 'neverRespond': (client) => { client.rpc.provides.neverRespond() }, 'clientBRejects': (client, data, response) => { client.rpc.provides.clientBRejects() if (client.name === 'B') { response.reject() } else { response.send(data.root * data.root) } }, 'deny': (client, data, response) => { // permissions always deny }, 'rejectOnce': (client, data, response) => { client.rpc.provides.rejectOnce(data) if (rejected) { response.send('ok') rejected = false } else { response.reject() rejected = true } } } const assert = { recievesResponse (clientExpression: string, rpc: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.rpc.callbacks[rpc]) sinon.assert.calledWith(client.rpc.callbacks[rpc], null, JSON.parse(data).toString()) client.rpc.callbacks[rpc].resetHistory() }) }, recievesResponseWithError (clientExpression: string, eventually: boolean, rpc: string, error: string, done: Function) { setTimeout(() => { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.calledOnce(client.rpc.callbacks[rpc]) sinon.assert.calledWith(client.rpc.callbacks[rpc], error) client.rpc.callbacks[rpc].resetHistory() done() }) }, eventually ? 150 : 0) }, providerCalled (clientExpression: string, rpc: string, timesCalled: number, data?: string) { clientHandler.getClients(clientExpression).forEach((client) => { sinon.assert.callCount(client.rpc.provides[rpc], timesCalled) if (data) { sinon.assert.calledWith(client.rpc.provides[rpc], JSON.parse(data)) } client.rpc.provides[rpc].resetHistory() }) } } export const rpc = { assert, provide (clientExpression: string, rpc: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.rpc.provides[rpc] = sinon.spy() client.client.rpc.provide(rpc, rpcs[rpc].bind(null, client)) }) }, unprovide (clientExpression: string, rpc: string) { clientHandler.getClients(clientExpression).forEach((client) => { client.client.rpc.unprovide(rpc) }) }, make (clientExpression: string, rpc: string, data: string) { clientHandler.getClients(clientExpression).forEach((client) => { const callback = client.rpc.callbacks[rpc] = sinon.spy() client.client.rpc.make(rpc, JSON.parse(data), (error: any, result: any) => { callback(error, result && result.toString()) }) }) } } ================================================ FILE: test-e2e/framework-v3/utils.ts ================================================ export const defaultDelay: number = Number(process.env.DEFAULT_DELAY) || 10 export const parseData = (data: string) => { if (data === undefined || data === 'undefined') { return undefined } else if (data === 'null') { return null } try { return JSON.parse(data) } catch (e) { return data } } ================================================ FILE: test-e2e/framework-v3/world.ts ================================================ // tslint:disable:no-shadowed-variable import { clientHandler } from './client-handler' export const world = { endTest (done: (...args: any[]) => void) { const clients = clientHandler.clients for (const client in clients) { clientHandler.assertNoErrors(client) for (const event in clients[client].event.callbacks) { if (clients[client].event.callbacks[event].isSubscribed !== false) { clients[client].client.event.unsubscribe(event, clients[client].event.callbacks[event]) } } setTimeout(function (client: string) { for (const pattern in clients[client].event.callbacksListeners) { if (clients[client].event.callbacksListeners[pattern].isListening !== false) { clients[client].client.event.unlisten(pattern) } } }.bind(null, client), 1) setTimeout(function (client: string) { clients[client].client.close() delete clients[client] }.bind(null, client), 50) } setTimeout(done, 100) } } ================================================ FILE: test-e2e/steps/client/client-definition-step.ts ================================================ import {Before, After} from 'cucumber' const { world } = require(`../../framework${process.env.V3 ? '-v3' : ''}/world`) Before((/* scenario*/) => { // client are connecting via "Background" explictly }) After((scenario, done) => { world.endTest(done!) }) ================================================ FILE: test-e2e/steps/client/connection-steps.ts ================================================ import { defaultDelay } from '../../framework/utils' import {When, Then, Given} from 'cucumber' const { client } = require(`../../framework${process.env.V3 ? '-v3' : ''}/client`) Then(/^(.+) receives? at least one "([^"]*)" error "([^"]*)"$/, client.receivedErrorOnce) Then(/^(.+) receives? "([^"]*)" error "([^"]*)"$/, client.receivedOneError) Then(/^(.+) received? no errors$/, client.receivedNoErrors) Given(/^(.+) logs? out$/, (clientExpression: string, done) => { client.logsOut(clientExpression, () => {}) setTimeout(done, defaultDelay) }) Given(/^(.+) connects? to server (\d+)$/, client.connect) Then(/^(.+) connections? times? out$/, client.connectionTimesOut) // Then(/^(.+) has connection state "([^"]*)"$/, (clientExpression: string, state) => // client.hasConnectionState(clientExpression: string, state)) Then(/^(.+) had a connection state change to "([^"]*)"$/, (clientExpression: string, state) => client.hadConnectionState(clientExpression, true, state)) Then(/^(.+) did not have a connection state change to "([^"]*)"$/, (clientExpression: string, state) => client.hadConnectionState(clientExpression, false, state)) Given(/^(.+) connects? and logs? into server (\d+)$/, (clientExpression: string, server, done) => { client.connectAndLogin(clientExpression, server, () => { setTimeout(done, defaultDelay) }) }) Given(/^(.+) logs? in with username "([^"]*)" and password "([^"]*)"$/, (clientExpression: string, username, password, done) => { client.login(clientExpression, username, password, () => { setTimeout(done, defaultDelay) }) }) When(/^(.+) attempts? to login with username "([^"]*)" and password "([^"]*)"$/, client.attemptLogin) Then(/^(.+) (?:is|are) notified of too many login attempts$/, client.receivedTooManyLoginAttempts) Then(/^(.+) receives? no login response$/, (clientExpression) => { client.recievesNoLoginResponse(clientExpression) }) Then(/^(.+) receives? an (un)?authenticated login response(?: with data ({.*}))?$/, (clientExpression: string, unauth, data) => { client.recievesLoginResponse(clientExpression, unauth, data) }) Then(/^(.+) "([^"]*)" callback was( not)? called( once)?( with ({.*}))?$/, (clientExpression: string, eventName, notCalled, once, data) => { if (eventName !== 'clientDataChanged' && eventName !== 'reauthenticationFailure') { client.callbackCalled(clientExpression, eventName, notCalled, false, data) } }) ================================================ FILE: test-e2e/steps/client/event-steps.ts ================================================ import { defaultDelay } from '../../framework/utils' import {When, Then, Given} from 'cucumber' const { event } = require(`../../framework${process.env.V3 ? '-v3' : ''}/event`) When(/^(.+) publishes? (?:an|the) event "([^"]*)"(?: with data ("[^"]*"|\d+|{.*}))?$/, (clientExpression: string, subscriptionName, data, done) => { event.publishes(clientExpression, subscriptionName, data) setTimeout(done, defaultDelay) }) Then(/^(.+) receives? (the|no) event "([^"]*)"(?: with data (.+))?$/, (clientExpression: string, theNo, subscriptionName, data) => { event.assert.received(clientExpression, !theNo.match(/^no$/), subscriptionName, data) }) Given(/^(.+) subscribes? to (?:an|the) event "([^"]*)"$/, (clientExpression: string, subscriptionName, done) => { event.subscribes(clientExpression, subscriptionName) setTimeout(done, defaultDelay * 10) }) When(/^(.+) unsubscribes from (?:an|the) event "([^"]*)"$/, (clientExpression: string, subscriptionName, done) => { event.unsubscribes(clientExpression, subscriptionName) setTimeout(done, defaultDelay) }) ================================================ FILE: test-e2e/steps/client/listening-steps.ts ================================================ import { defaultDelay } from '../../framework/utils' import {When, Then} from 'cucumber' const { listening } = require(`../../framework${process.env.V3 ? '-v3' : ''}/listening`) When(/^publisher (\S*) (accepts|rejects) (?:a|an) (event|record) match "([^"]*)" for pattern "([^"]*)"$/, (client, action, type, subscriptionName, pattern) => { listening.setupListenResponse(client, action === 'accepts', type, subscriptionName, pattern) }) When(/^publisher (\S*) listens to (?:a|an) (event|record) with pattern "([^"]*)"$/, (client, type, pattern, done) => { listening.listens(client, type, pattern) setTimeout(done, defaultDelay) }) When(/^publisher (\S*) unlistens to the (event|record) pattern "([^"]*)"$/, (client, type, pattern, done) => { listening.unlistens(client, type, pattern) setTimeout(done, defaultDelay) }) Then(/^publisher (\S*) does not receive (?:a|an) (event|record) match "([^"]*)" for pattern "([^"]*)"$/, listening.assert.doesNotRecieveMatch) Then(/^publisher (\S*) receives (\d+) (event|record) (?:match|matches) "([^"]*)" for pattern "([^"]*)"$/, listening.assert.recievesMatch) Then(/^publisher (\S*) removed (\d+) (event|record) (?:match|matches) "([^"]*)" for pattern "([^"]*)"$/, listening.assert.receivedUnMatch) ================================================ FILE: test-e2e/steps/client/presence-steps.ts ================================================ import { defaultDelay } from '../../framework/utils' import {When, Then, Given} from 'cucumber' const { presence } = require(`../../framework${process.env.V3 ? '-v3' : ''}/presence`) Given(/^(.+) subscribes to presence events$/, (clientExpression: string, done) => { presence.subscribe(clientExpression) setTimeout(done, defaultDelay) }) Given(/^(.+) unsubscribes to presence events$/, (clientExpression: string, done) => { presence.unsubscribe(clientExpression) setTimeout(done, defaultDelay) }) Given(/^(.+) subscribes to presence events for "([^"]*)"$/, (clientExpression: string, users: string, done) => { users.split(',').forEach((user) => presence.subscribe(clientExpression, user)) setTimeout(done, defaultDelay * 3) }) Given(/^(.+) unsubscribes to presence events for "([^"]*)"$/, (clientExpression: string, users: string, done) => { users.split(',').forEach((user) => presence.unsubscribe(clientExpression, user)) setTimeout(done, defaultDelay) }) When(/^(.+) queries for connected clients$/, (clientExpression: string, done) => { presence.getAll(clientExpression) setTimeout(done, defaultDelay) }) When(/^(.+) queries for clients "([^"]*)"$/, (clientExpression: string, clients: string, done) => { presence.getAll(clientExpression, clients.split(',')) setTimeout(done, defaultDelay) }) Then(/^(.+) (?:is|are) (not )?notified that (.+) logged ([^"]*)$/, presence.assert.notifiedUserStateChanged) Then(/^(.+) is notified that (?:clients|client) "([^"]*)" (?:are|is) connected$/, (clientExpression: string, connectedClients) => { presence.assert.globalQueryResult(clientExpression, null, connectedClients.split(',')) }) Then(/^(.+) receives a "([^"]*)" error on their query$/, (clientExpression: string, error) => { presence.assert.globalQueryResult(clientExpression, error) }) Then(/^(.+) (?:is|are) notified that (?:clients|client) "([^"]*)" (?:are|is) online$/, (clientExpression: string, clients) => { presence.assert.queryResult(clientExpression, clients.split(','), true) }) Then(/^(.+) (?:is|are) notified that (?:clients|client) "([^"]*)" (?:are|is) offline$/, (clientExpression: string, clients) => { presence.assert.queryResult(clientExpression, clients.split(','), false) }) Then(/^(.+) is notified that no clients are connected$/, (clientExpression: string) => { presence.assert.globalQueryResult(clientExpression, null, []) }) ================================================ FILE: test-e2e/steps/client/record-steps.ts ================================================ import { defaultDelay } from '../../framework/utils' import {When, Then, Given} from 'cucumber' const { record } = require(`../../framework${process.env.V3 ? '-v3' : ''}/record`) const { client } = require(`../../framework${process.env.V3 ? '-v3' : ''}/client`) When(/(.+) gets? the record "([^"]*)"$/, (clientExpression: string, recordName: string, done) => { record.getRecord(clientExpression, recordName) setTimeout(done, defaultDelay * 3) }) When(/(.+) sets the merge strategy to (remote|local)$/, (clientExpression: string, recordName: string) => { // not implemented }) Then(/^(.+) (gets?|is not) notified of record "([^"]*)" getting (discarded|deleted)$/, (clientExpression: string, notified, recordName, action) => { const called = notified.indexOf('is not') !== -1 ? false : true if (action === 'discarded') { // record.assert.discarded(clientExpression, recordName, called) } else { record.assert.deleted(clientExpression, recordName, called) } }) Then(/^(.+) receives? an? "([^"]*)" error on record "([^"]*)"$/, (clientExpression: string, error: string, recordName: string, done) => { record.assert.receivedRecordError(clientExpression, error, recordName) if (recordName ==='only-a-can-read-and-create') client.receivedOneError(clientExpression, 'record', error) setTimeout(done, defaultDelay) }) Then(/^(.+) receives? an update for record "([^"]*)" with data '([^']+)'$/, record.assert.receivedUpdate) Then(/^(.+) receives? an update for record "([^"]*)" and path "([^"]*)" with data '([^']+)'$/, record.assert.receivedUpdateForPath) Then(/^(.+) (?:don't|doesn't|does not) receive an update for record "([^"]*)"$/, record.assert.receivedNoUpdate) Then(/^(.+) don't receive an update for record "([^"]*)" and path "([^"]*)"$/, record.assert.receivedNoUpdateForPath) Given(/^(.+) subscribes? to record "([^"]*)"( with immediate flag)?$/, record.subscribe) Given(/^(.+) unsubscribes? to record "([^"]*)"$/, record.unsubscribe) Given(/^(.+) subscribes? to record "([^"]*)" with path "([^"]*)"( with immediate flag)?$/, record.subscribeWithPath) Given(/^(.+) unsubscribes? to record "([^"]*)" with path "([^"]*)"$/, record.unsubscribeFromPath) Then(/^(.+) (?:have|has) record "([^"]*)" with data '([^']+)'$/, record.assert.hasData) Then(/^(.+) (?:have|has) record "([^"]*)" with(out)? providers$/, record.assert.hasProviders) Then(/^(.+) (?:have|has) record "([^"]*)" with path "([^"]*)" and data '([^']+)'$/, record.assert.hasDataAtPath) Given(/^(.+) discards record "([^"]*)"$/, (clientExpression: string, recordName: string, done) => { record.discard(clientExpression, recordName) setTimeout(done, defaultDelay) }) Given(/^(.+) deletes record "([^"]*)"$/, (clientExpression: string, recordName: string, done) => { record.delete(clientExpression, recordName) setTimeout(done, defaultDelay) }) When(/^(.+) requires? write acknowledgements for record "([^"]*)"$/, (clientExpression: string, recordName: string) => { record.setupWriteAck(clientExpression, recordName) }) When(/^(.+) sets? the record "([^"]*)" with data '([^']+)'$/, (clientExpression: string, recordName: string, data: string, done) => { record.set(clientExpression, recordName, data) setTimeout(done, defaultDelay) }) When(/^(.+) sets? the record "([^"]*)" without being subscribed with data '([^']+)'$/, (clientExpression: string, recordName: string, data: string, done) => { record.setData(clientExpression, recordName, data) setTimeout(done, defaultDelay) }) When(/^(.+) sets? the record "([^"]*)" without being subscribed with data '([^']+)' and requires write acknowledgement$/, (clientExpression: string, recordName: string, data: string, done) => { record.setDataWithWriteAck(clientExpression, recordName, data) setTimeout(done, defaultDelay) }) When( /^(.+) sets? the record "([^"]*)" without being subscribed with path "([^"]*)" and data '([^']+)'$/, (clientExpression: string, recordName: string, path, data, done) => { record.setDataWithPath(clientExpression, recordName, path, data) setTimeout(done, defaultDelay) }) When(/^(.+) sets? the record "([^"]*)" and path "([^"]*)" with data '([^']+)'$/, (clientExpression: string, recordName: string, path, data, done) => { record.setWithPath(clientExpression, recordName, path, data) setTimeout(done, defaultDelay) }) When(/^(.+) erases the path "([^"]*)" on record "([^"]*)"$/, (clientExpression: string, path, recordName, done) => { record.erase(clientExpression, recordName, path) setTimeout(done, defaultDelay) }) Then(/^(.+) is told that the record "([^"]*)" was set without error$/, record.assert.writeAckSuccess) Then(/^(.+) is told that the record "([^"]*)" experienced error "([^"]*)" while setting$/, (clientExpression: string, recordName: string, errorMessage, done) => { setTimeout(() => { record.assert.writeAckError(clientExpression, recordName, errorMessage) done() }, 100) }) Given(/^(.+) requests? a snapshot of record "([^"]*)"$/, (clientExpression: string, recordName: string, done) => { record.snapshot(clientExpression, recordName) setTimeout(done, defaultDelay) }) Then(/^(.+) gets? a snapshot response for "([^"]*)" with data '([^']+)'$/, record.assert.snapshotSuccess) Then(/^(.+) gets? a snapshot response for "([^"]*)" with error '([^']+)'$/, record.assert.snapshotError) Given(/^(.+) asks? if record "([^"]*)" exists$/, (clientExpression: string, recordName: string, done) => { record.has(clientExpression, recordName) setTimeout(done, defaultDelay) }) Then(/^(.+) gets? told record "([^"]*)" (.*)exists?$/, (clientExpression: string, recordName: string, adjective) => { record.assert.has(clientExpression, recordName, (adjective || '').indexOf('not') === -1) }) Then(/^(.+) asks? for the version of record "([^"]*)"$/, (clientExpression: string, recordName: string, done) => { record.head(clientExpression, recordName) setTimeout(done, defaultDelay) }) Then(/^(.+) gets? told record "([^"]*)" has version (.*)$/, (clientExpression: string, recordName: string, version: string) => { record.assert.headSuccess(clientExpression, recordName, Number(version)) }) Then(/^(.+) gets? a head response for "([^"]*)" with error '([^']+)'$/, record.assert.headError) /** ****************************************************************************************************************************** *********************************************************** Lists ************************************************************ ********************************************************************************************************************************/ When(/(.+) gets? the list "([^"]*)"$/, (clientExpression: string, listName, done) => { record.getList(clientExpression, listName) setTimeout(done, defaultDelay * 3) }) Given(/^(.+) sets the entries on the list "([^"]*)" to '([^']*)'$/, (clientExpression: string, listName: string, data: string, done) => { record.setEntries(clientExpression, listName, data) setTimeout(done, defaultDelay) }) Given(/^(.+) (adds|removes) an entry "([^"]*)" (?:to|from) "([^""]*)"$/, (clientExpression: string, action, entryName, listName, done) => { if (action === 'adds') { record.addEntry(clientExpression, listName, entryName) } else { record.removeEntry(clientExpression, listName, entryName) } setTimeout(done, defaultDelay) }) Then(/^(.+) have a list "([^"]*)" with entries '([^']*)'$/, record.assert.hasEntries) Then(/^(.+) gets? notified of "([^"]*)" being (added|removed|moved) (?:to|in|from) "([^""]*)"$/, (clientExpression: string, entryName, action, listName) => { if (action === 'added') { record.assert.addedNotified(clientExpression, listName, entryName) } else if (action === 'removed') { record.assert.removedNotified(clientExpression, listName, entryName) } else { record.assert.movedNotified(clientExpression, listName, entryName) } }) Then(/^(.+) gets? notified of list "([^"]*)" entries changing to '([^']*)'$/, record.assert.listChanged) /** ****************************************************************************************************************************** *********************************************************** ANONYMOUS RECORDS ************************************************************ ********************************************************************************************************************************/ When(/(.+) gets? a anonymous record$/, record.getAnonymousRecord) When(/(.+) sets? the underlying record to "([^"]*)" on the anonymous record$/, (clientExpression: string, recordName: string, done) => { record.setName(clientExpression, recordName) setTimeout(done, defaultDelay) }) Then(/(.+) anonymous record data is '([^']*)'$/, (clientExpression: string, data: string) => { record.assert.anonymousRecordContains(clientExpression, data) }) ================================================ FILE: test-e2e/steps/client/rpc-steps.ts ================================================ import { defaultDelay } from '../../framework/utils' import {When, Then, Given} from 'cucumber' const { rpc } = require(`../../framework${process.env.V3 ? '-v3' : ''}/rpc`) Given(/^(.+) provides? the RPC "([^"]*)"$/, (clientExpression: string, rpcName, done) => { rpc.provide(clientExpression, rpcName) setTimeout(done, defaultDelay) }) Given(/^(.+) unprovides? the RPC "([^"]*)"$/, (clientExpression: string, rpcName, done) => { rpc.unprovide(clientExpression, rpcName) setTimeout(done, defaultDelay) }) When(/^(.+) calls? the RPC "([^"]*)" with arguments? ("[^"]*"|\d+|{.*})$/, (clientExpression: string, rpcName, args, done) => { rpc.make(clientExpression, rpcName, args) setTimeout(done, defaultDelay) }) Then(/(.+) receives? a response for RPC "([^"]*)" with data ("[^"]*"|\d+|{.*})$/, rpc.assert.recievesResponse) Then(/(.+) (eventually )?receives? a response for RPC "([^"]*)" with error "([^"]*)"$/, rpc.assert.recievesResponseWithError) Then(/(.+) RPCs? "([^"]*)" (?:is|are) never called$/, (clientExpression: string, rpcName) => { rpc.assert.providerCalled(clientExpression, rpcName, 0) }) Then(/(.+) RPCs? "([^"]*)" (?:is|are) called once( with data ("[^"]*"|\d+|{.*}))?$/, (clientExpression: string, rpcName, data) => { rpc.assert.providerCalled(clientExpression, rpcName, 1, data) }) Then(/(.+) RPCs? "([^"]*)" is called (\d+) times$/, (clientExpression: string, rpcName, numTimes) => { rpc.assert.providerCalled(clientExpression, rpcName, numTimes) }) ================================================ FILE: test-e2e/steps/http/http-steps.ts ================================================ import * as sinon from 'sinon' import {Given, When, Then, After } from 'cucumber' import { expect } from 'chai' import * as needle from 'needle' import { parseData, defaultDelay } from '../../framework/utils' const { clientHandler } = require(`../../framework${process.env.V3 ? '-v3' : ''}/client-handler`) let httpClients: { [index: string]: any } = {} Given(/^(.+) authenticates? with http server (\d+)$/, (clientExpression: string, server, done) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { let serverUrl if (global.e2eHarness.getAuthUrl) { serverUrl = global.e2eHarness.getAuthUrl(server) } else { serverUrl = global.e2eHarness.getHttpUrl(server) } const message = { username: clientName, password: 'abcdefgh' } needle.post(serverUrl, message, { json: true }, (err, response) => { process.nextTick(done) expect(err).to.equal(null) expect(response.statusCode).to.be.within(200, 299) expect(response.body.token).to.be.a('string') httpClients[clientName] = { token: response.body.token, serverUrl: global.e2eHarness.getHttpUrl(server - 1, clientName), queue: [], lastResponse: Object.assign({}, response, { isAuthResponse: true }), resultChecked: false } }) }) }) Given(/^(.+) authenticates? with http server (\d+) with details ("[^"]*"|\d+|{.*})?$/, (clientExpression: string, server, data, done) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { let serverUrl if (global.e2eHarness.getAuthUrl) { serverUrl = global.e2eHarness.getAuthUrl(server - 1, clientName) } else { serverUrl = global.e2eHarness.getHttpUrl(server - 1, clientName) } const credentials = JSON.parse(data) needle.post(serverUrl, credentials, { json: true }, (err, response) => { process.nextTick(done) expect(err).to.equal(null) httpClients[clientName] = { token: response.body.token, serverUrl: global.e2eHarness.getHttpUrl(server - 1, clientName), queue: [], lastResponse: Object.assign({}, response, { isAuthResponse: true }), resultChecked: false } }) }) }) Then(/^the last response (.+) received contained the properties "([^"]*)"$/, (clientExpression: string, properties) => { const propertyArray = properties.split(',') clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] expect(client.lastResponse.body).to.contain.keys(propertyArray) expect(Object.keys(client.lastResponse.body).length).to.equal(propertyArray.length) client.resultChecked = true }) }) When(/^(.+) queues? (?:an|the|"(\d+)") events? "([^"]*)"(?: with data ("[^"]*"|\d+|{.*}))?$/, (clientExpression: string, numEvents, eventName, rawData) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = { topic: 'event', action: 'emit', eventName, data: null } if (rawData !== null) { jifMessage.data = parseData(rawData) } if (numEvents === null) { client.queue.push(jifMessage) } else { for (let i = 0; i < numEvents; i++) { client.queue.push(jifMessage) } } }) }) When(/^(.+) sends the data ("[^"]*"|\d+|{.*})$/, (clientExpression: string, rawData, done) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] needle.post(`${client.serverUrl}`, JSON.parse(rawData), { json: true }, (err, response) => { setTimeout(done, defaultDelay) client.lastResponse = response }) }) }) When(/^(.+) queues "(\d+)" random messages$/, (clientExpression: string, numMessages) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] for (let i = 0; i < numMessages; i++) { const r = Math.random() let message if (r < 0.3) { message = { topic: 'event', action: 'emit', eventName: 'eventName' } } else if (r < 0.5) { message = { topic: 'record', action: 'read', recordName: 'recordName' } } else if (r < 0.6) { message = { topic: 'record', action: 'write', recordName: 'recordName', path: 'r', data: r } } else if (r < 0.7) { message = { topic: 'record', action: 'head', recordName: 'recordName', } } else if (r < 0.8) { message = { topic: 'rpc', action: 'make', rpcName: 'addTwo', data: { numA: (r * 1011) % 77, numB: (r * 9528) % 63 } } } else { message = { topic: 'presence', action: 'query', } } client.queue.push(message) } }) }) When(/^(.+) queues a presence query$/, (clientExpression) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = { topic: 'presence', action: 'query' } client.queue.push(jifMessage) }) }) When(/^(.+) queues? (?:an|the) RPC call to "([^"]*)"(?: with arguments ("[^"]*"|\d+|{.*}))?$/, (clientExpression: string, rpcName, rawData) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = { topic: 'rpc', action: 'make', rpcName, data: null } if (rawData !== null) { jifMessage.data = parseData(rawData) } client.queue.push(jifMessage) }) }) When(/^(.+) queues? a fetch for (record|list) "([^"]*)"$/, (clientExpression: string, recordOrList, recordName) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = recordOrList === 'record' ? { topic: 'record', action: 'read', recordName } : { topic: 'list', action: 'read', listName: recordName } client.queue.push(jifMessage) }) }) When(/^(.+) queues? a write to (record|list) "([^"]*)"(?: and path "([^"]*)")? with data '([^']*)'(?: and version "(-?\d+)")?$/, (clientExpression: string, recordOrList, recordName, path, rawData, version) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = recordOrList === 'record' ? { topic: 'record', action: 'write', recordName } : { topic: 'list', action: 'write', listName: recordName } if (path !== null) { // @ts-ignore jifMessage.path = path } if (rawData !== null) { // @ts-ignore jifMessage.data = parseData(rawData) } if (version !== null) { // @ts-ignore jifMessage.version = parseInt(version, 10) } client.queue.push(jifMessage) }) }) When(/^(.+) queues? a notify for records? '([^"]*)'$/, (clientExpression: string, recordNames) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = { topic: 'record', action: 'notify', recordNames: recordNames.split(',') } client.queue.push(jifMessage) }) }) When(/^(.+) queues? a delete for (record|list) "([^"]*)"$/, (clientExpression: string, recordOrList, recordName) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = recordOrList === 'record' ? { topic: 'record', action: 'delete', recordName } : { topic: 'list', action: 'delete', listName: recordName } client.queue.push(jifMessage) }) }) When(/^(.+) queues? a head for record "([^"]*)"$/, (clientExpression: string, recordName: string) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const jifMessage = { topic: 'record', action: 'head', recordName, } client.queue.push(jifMessage) }) }) When(/^(.+) flushe?s? their http queues?$/, (clientExpression: string, done) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const message = { token: client.token, body: client.queue } client.queue = [] needle.post(`${client.serverUrl}`, message, { json: true }, (err, response) => { client.lastResponse = response setTimeout(done, defaultDelay) }) }) }) Then(/^(.+) last response said that clients? "([^"]*)" (?:is|are) connected(?: at index "(\d+)")?$/, (clientExpression: string, connectedClients, rawIndex) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const responseIndex = rawIndex === null ? 0 : rawIndex const client = httpClients[clientName] const lastResponse = client.lastResponse expect(lastResponse).not.to.equal(null) expect(lastResponse.body).to.be.an('object') expect(lastResponse.body.body).to.be.an('array') const result = lastResponse.body.body[responseIndex] expect(result).to.be.an('object') expect(result.success).to.equal(true) expect(result.users).to.have.members(connectedClients.split(',')) }) }) Then(/^(.+) last response said that no clients are connected(?: at index "(\d+)")?$/, (clientExpression: string, rawIndex) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const responseIndex = rawIndex === null ? 0 : rawIndex const client = httpClients[clientName] const lastResponse = client.lastResponse expect(lastResponse).not.to.equal(null) expect(lastResponse.body).to.be.an('object') expect(lastResponse.body.body).to.be.an('array') const result = lastResponse.body.body[responseIndex] expect(result).to.be.an('object') expect(result.success).to.equal(true) expect(result.users).to.deep.equal([]) }) }) Then(/^(.+) receives? an RPC response(?: with data ("[^"]*"|\d+|{.*}))?(?: at index "(\d+)")?$/, (clientExpression: string, rawData, rawIndex) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const responseIndex = rawIndex === null ? 0 : rawIndex const client = httpClients[clientName] const lastResponse = client.lastResponse expect(lastResponse).not.to.equal(null) expect(lastResponse.body).to.be.an('object') expect(lastResponse.body.body).to.be.an('array') const result = lastResponse.body.body[responseIndex] expect(result).to.be.an('object') expect(result.success).to.equal(true) if (rawData !== null && rawData !== '"undefined"') { expect(result.data).to.deep.equal(parseData(rawData)) } }) }) Then(/^(.+) receives? the (?:record|list) (?:head )?"([^"]*)"(?: with data '([^']+)')?(?: (?:with|and) version "(\d+)")?(?: at index "(\d+)")?$/, (clientExpression: string, recordName: string, rawData, version, rawIndex) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const responseIndex = rawIndex === null ? 0 : rawIndex const client = httpClients[clientName] const lastResponse = client.lastResponse expect(lastResponse).not.to.equal(null) expect(lastResponse.body).to.be.an('object') expect(lastResponse.body.body).to.be.an('array') const result = lastResponse.body.body[responseIndex] expect(result).to.be.an('object') expect(result.success).to.equal(true) if (rawData !== null) { expect(result.data).to.deep.equal(parseData(rawData)) } if (version !== null) { expect(result.version).to.equal(parseInt(version, 10)) } }) }) Then(/^(.+) last response was a "(\S*)"(?: with length "(\d+)")?$/, (clientExpression: string, result, length) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const lastResponse = client.lastResponse const failures = client.lastResponse.body.body.filter((res: any) => !res.success) const failuresStr = JSON.stringify(failures, null, 2) expect(lastResponse.body.result).to.equal(result, failuresStr) if (length !== null) { expect(lastResponse.body.body.length).to.equal(parseInt(length, 10)) } // by default, clients are expected to have a SUCCESS response last, so mark as already // checked client.resultChecked = true }) }) Then(/^(.+) (eventually )?receives "(\d+)" events? "([^"]*)"(?: with data (.+))?$/, (clientExpression: string, eventually, numEvents, subscriptionName, data, done) => { setTimeout(() => { clientHandler.getClients(clientExpression).forEach((client: any) => { const eventSpy = client.event.callbacks[subscriptionName] expect(eventSpy.callCount).to.equal(parseInt(numEvents, 10)) sinon.assert.calledWith(eventSpy, parseData(data)) eventSpy.resetHistory() done() }) }, eventually ? 350 : 0) }) Then(/^(.+) last response had a success(?: at index "(\d+)")?$/, (clientExpression: string, rawIndex) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const responseIndex = rawIndex === null ? 0 : rawIndex const client = httpClients[clientName] const lastResponse = client.lastResponse expect(lastResponse).not.to.equal(null) expect(lastResponse.body).to.be.an('object') expect(lastResponse.body.body).to.be.an('array') const result = lastResponse.body.body[responseIndex] expect(result).to.be.an('object') expect(result.success).to.equal(true) client.resultChecked = true }) }) Then(/^(.+) last response had an? "([^"]*)" error matching "([^"]*)"(?: at index "(\d+)")?$/, (clientExpression: string, topic: string, message: string, rawIndex: number) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const responseIndex = rawIndex === null ? 0 : rawIndex const client = httpClients[clientName] const lastResponse = client.lastResponse expect(lastResponse).not.to.equal(null) expect(lastResponse.body).to.be.an('object') expect(lastResponse.body.body).to.be.an('array') const result = lastResponse.body.body[responseIndex] expect(result).to.be.an('object') expect(result.success).to.equal(false) expect(result.errorTopic).to.equal(topic) expect(result.error).to.match(new RegExp(message, 'i')) client.resultChecked = true }) }) Then(/^(.+) last response had an error matching "([^"]*)"$/, (clientExpression: string, message) => { clientHandler.getClientNames(clientExpression).forEach((clientName: string) => { const client = httpClients[clientName] const lastResponse = client.lastResponse expect(lastResponse).not.to.equal(null) expect(lastResponse.body).to.be.an('string') expect(lastResponse.body).to.match(new RegExp(message, 'i')) client.resultChecked = true }) }) After(() => { for (const clientName in httpClients) { const client = httpClients[clientName] if (client.lastResponse && !client.resultChecked && !client.lastResponse.isAuthResponse) { const failures = client.lastResponse.body.body.filter((res: any) => !res.success) const failuresStr = JSON.stringify(failures, null, 2) expect(client.lastResponse.body.result).to.equal('SUCCESS', failuresStr) } client.lastResponse = null } httpClients = {} }) ================================================ FILE: test-e2e/steps/server/step-definition-server.ts ================================================ import {Given, When, Before, BeforeAll } from 'cucumber' import { PromiseDelay } from '../../../src/utils/utils' import { E2EHarness } from '../../tools/e2e-harness' Before(async (scenarioResult) => { await global.e2eHarness.updatePermissions('open') }) Given(/"([^"]*)" permissions are used$/, async (permissionType) => { await global.e2eHarness.updatePermissions(permissionType) }) When('storage remote updates {string} to {string} with version {int}', async (recordName, data, version) => { await global.e2eHarness.updateStorageDirectly(recordName, version, data) }) When('storage remote deletes {string}', async (recordName) => { await global.e2eHarness.deleteFromStorageDirectly(recordName) }) When(/^server (\S)* goes down$/, async (server) => { if (global.e2eHarness.started === false) { return } await global.e2eHarness.stopServer(server) }) When(/^all servers go down$/, async () => { if (global.e2eHarness.started === false) { return } await global.e2eHarness.stop() await PromiseDelay(200) }) When(/^server (\S)* comes back up$/, async (server) => { if (global.e2eHarness.started) { return } await global.e2eHarness.startServer(server) }) When(/^all servers come back up$/, async () => { if (global.e2eHarness.started === true) { return } await global.e2eHarness.start() await PromiseDelay(200) }) Given(/^a small amount of time passes$/, (done) => { setTimeout(done, 500) }) BeforeAll(async () => { global.e2eHarness = new E2EHarness([6001, 7001, 8001], process.env.ENABLE_LOGGING === 'true') await global.e2eHarness.start() await PromiseDelay(200) }) // AfterAll((callback) => { // setTimeout(() => { // global.e2eHarness.once('stopped', () => { // callback() // }) // global.e2eHarness.stop() // }, 500) // }) ================================================ FILE: test-e2e/tools/e2e-authentication.ts ================================================ import { DeepstreamPlugin, DeepstreamAuthentication } from '@deepstream/types' import { JSONObject } from '../../src/constants' interface AuthData { token?: string, username?: string, password?: string } export class E2EAuthentication extends DeepstreamPlugin implements DeepstreamAuthentication { public description: string = 'E2E Authentication' public tokens = new Map() public onlyLoginOnceUser: boolean = false public async isValidUser (headers: JSONObject, authData: AuthData) { if (authData.token) { if (authData.token === 'letmein') { return { isValid: true, id: 'A' } } // authenticate token const response = this.tokens.get(authData.token) if (response && response.id) { return { isValid: true, ...response } } } const username = authData.username const token = Math.random().toString() let clientData: any = null const serverData: any = {} let success // authenticate auth data const users = ['A', 'B', 'C', 'D', 'E', 'F', 'W', '1', '2', '3', '4', 'OPEN'] if (users.indexOf(username!) !== -1 && authData.password === 'abcdefgh') { success = true } else if (username === 'userA' && authData.password === 'abcdefgh') { success = true serverData.role = 'user' } else if (username === 'userB' && authData.password === '123456789') { success = true clientData = { 'favorite color': 'orange', 'id': username } serverData.role = 'admin' } else if (username === 'randomClientData') { success = true clientData = { value : Math.random() } } else if (username === 'onlyLoginOnce' && !this.onlyLoginOnceUser) { this.onlyLoginOnceUser = true success = true } else { success = false } const authResponseData = { id: username, token, clientData, serverData } if (success) { this.tokens.set(token, authResponseData) return { isValid: true, ...authResponseData } } else { return { isValid: false } } } } ================================================ FILE: test-e2e/tools/e2e-cluster-node.ts ================================================ import { Message, TOPIC, STATE_REGISTRY_TOPIC } from '../../src/constants' import { EventEmitter } from 'events' import { DeepstreamServices, DeepstreamConfig, DeepstreamPlugin, DeepstreamClusterNode } from '@deepstream/types' export class E2EClusterNode extends DeepstreamPlugin implements DeepstreamClusterNode { public description: string = 'E2EClusterNode' private static emitters = new Map() constructor (options: any, services: DeepstreamServices, private config: DeepstreamConfig) { super() E2EClusterNode.emitters.set(this.config.serverName, new EventEmitter()) } public sendDirect (toServer: string, message: Message, metaData?: any): void { const msg = { ...message } process.nextTick(() => { E2EClusterNode.emitters.get(toServer)!.emit(TOPIC[message.topic], this.config.serverName, msg) }) } public send (message: Message, metaData?: any): void { const msg = { ...message } process.nextTick(() => { for (const [serverName, emitter] of E2EClusterNode.emitters) { if (serverName !== this.config.serverName) { emitter.emit(TOPIC[message.topic], this.config.serverName, { ...msg }) } } }) } public subscribe (topic: STATE_REGISTRY_TOPIC, callback: (message: SpecificMessage, serverName: string) => void): void { E2EClusterNode.emitters.get(this.config.serverName)!.on(TOPIC[topic], (fromServer, message) => { if (fromServer === this.config.serverName) { throw new Error('Cyclic message was sent!') } callback(message, fromServer) }) } public async close () { E2EClusterNode.emitters.delete(this.config.serverName) } } ================================================ FILE: test-e2e/tools/e2e-harness.ts ================================================ import { EventEmitter } from 'events' import { PromiseDelay } from '../../src/utils/utils' import { Deepstream } from '../../src/deepstream.io' import { E2EAuthentication } from './e2e-authentication' import { getServerConfig } from './e2e-server-config' import { E2ELogger } from './e2e-logger' import { STATES, JSONValue } from '../../src/constants' import { LocalCache } from '../../src/services/cache/local-cache' import { ConfigPermission } from '../../src/services/permission/valve/config-permission' import { E2EClusterNode } from './e2e-cluster-node' import * as openPermissions from '../config/permissions-open.json' import * as complexPermissions from '../config/permissions-complex.json' const cache = new LocalCache() const SERVER_STOP_OR_START_DURATION = 200 const authenticationHandler = new E2EAuthentication() export class E2EHarness extends EventEmitter { private servers: Deepstream[] = [] constructor (private ports: number[], private enableLogging: boolean = false) { super() this.start() } public getServerName (serverId: number) { return `server-${this.ports[serverId - 1]}` } public getUrl (serverId: number) { return `localhost:${this.ports[serverId - 1]}/e2e` } public getHttpUrl (serverId: number) { return `localhost:${this.ports[serverId - 1]}/api` } public getAuthUrl (serverId: number) { return `localhost:${this.ports[serverId - 1]}/api/auth` } public async updateStorageDirectly (recordName: string, version: number, data: JSONValue) { this.servers.forEach((server) => { server.getServices().storage.set(recordName, version, data, () => {}) }) return new Promise((resolve) => setTimeout(resolve, 10)) } public async deleteFromStorageDirectly (recordName: string) { this.servers.forEach((server) => { server.getServices().storage.delete(recordName, () => {}) }) return new Promise((resolve) => setTimeout(resolve, 10)) } public async start () { for (let i = 1; i <= this.ports.length; i++) { this.startServer(i) } await this.whenReady() } public async stop () { this.servers.forEach((server) => server.stop()) await this.whenStopped() } public async updatePermissions (type: string) { const promises = this.servers.map((server) => { const permission = server.getServices().permission as never as ConfigPermission permission.useConfig(type === 'open' ? openPermissions : complexPermissions) }) await Promise.all(promises) } public stopServer (serverId: number) { return new Promise(async (resolve) => { const server = this.servers[serverId - 1] if (!server) { throw new Error(`Server ${serverId} not found`) } if (server.isRunning() === false) { // Single node resolve() return } server.on('stopped', async () => { await PromiseDelay(SERVER_STOP_OR_START_DURATION) // @ts-ignore this.servers[serverId - 1] = null resolve() }) server.stop() }) } public async startServer (serverId: number) { if (this.servers[serverId - 1]) { await PromiseDelay(SERVER_STOP_OR_START_DURATION) return } const server = new Deepstream(getServerConfig(this.ports[serverId - 1])) as any this.servers[serverId - 1] = server const startedPromise = new Promise((resolve) => server.on('started', resolve)) if (this.enableLogging !== true) { server.set('logger', new E2ELogger()) } server.set('cache', cache) server.set('authentication', authenticationHandler) // @ts-ignore server.set('clusterNode', new E2EClusterNode({}, server.services, server.config)) server.start() await startedPromise await PromiseDelay(SERVER_STOP_OR_START_DURATION * 2) } public async whenReady () { const startedPromises = this.servers.reduce((result, server) => { if (!server.isRunning()) { result.push(new Promise((resolve) => server.on('started', resolve))) } return result }, [] as Array>) await Promise.all(startedPromises) await PromiseDelay(SERVER_STOP_OR_START_DURATION) } public async whenStopped () { const stopPromises = this.servers.reduce((result, server) => { // @ts-ignore if (server.currentState !== STATES.STOPPED) { result.push(new Promise((resolve) => server.on('stopped', resolve))) } return result }, [] as Array>) await Promise.all(stopPromises) await PromiseDelay(SERVER_STOP_OR_START_DURATION) this.servers = [] } } ================================================ FILE: test-e2e/tools/e2e-logger.ts ================================================ import { DeepstreamLogger, DeepstreamPlugin, LOG_LEVEL, NamespacedLogger, EVENT } from '@deepstream/types' interface Log { level: number, event: string, message: string } export class E2ELogger extends DeepstreamPlugin implements DeepstreamLogger { public description = 'Test Logger' public logs: Log[] = [] public lastLog: Log | null = null public setLogLevel (logLevel: LOG_LEVEL): void { throw new Error('Method not implemented.') } public shouldLog () { return true } public error (event: EVENT, logMessage: string) { this.log(LOG_LEVEL.ERROR, event, logMessage) } public warn (event: EVENT, logMessage: string) { this.log(LOG_LEVEL.WARN, event, logMessage) } public info (event: EVENT, logMessage: string) { this.log(LOG_LEVEL.INFO, event, logMessage) } public debug (event: EVENT, logMessage: string) { this.log(LOG_LEVEL.DEBUG, event, logMessage) } public fatal (event: EVENT, logMessage: string) { this.log(LOG_LEVEL.FATAL, event, logMessage) } public getNameSpace (namespace: string): NamespacedLogger { return { shouldLog: this.shouldLog.bind(this), fatal: this.fatal.bind(this), error: this.error.bind(this), warn: this.warn.bind(this), info: this.info.bind(this), debug: this.debug.bind(this), } } private log (logLevel: LOG_LEVEL, event: EVENT, logMessage: string) { const log = { level: logLevel, event, message: logMessage } this.logs.push(log) this.lastLog = log switch (logLevel) { case 3: throw new Error(`Critical error occured on deepstream ${event} ${logMessage}`) break case 2: // console.log('Warning:', event, logMessage) break } } } ================================================ FILE: test-e2e/tools/e2e-server-config.ts ================================================ import { PartialDeepstreamConfig, LOG_LEVEL } from '@deepstream/types' import * as permissions from '../config/permissions-open.json' export const getServerConfig = (port: number): PartialDeepstreamConfig => ({ serverName : `server-${port}`, showLogo : false, rpc: { // This shouldn't be more than response, // but it solves issues in E2E tests for HTTP bulk requests for now ackTimeout: 100, responseTimeout: 100, }, listen: { shuffleProviders: false, responseTimeout: 2000, rematchInterval: 60000, matchCooldown: 10000 }, permission: { type : 'config', options : { permissions } as any }, httpServer: { type: process.env.uws ? 'uws' : 'default', options: { port } }, connectionEndpoints: [ { type: 'ws-binary', options: { urlPath: '/e2e-v4', maxAuthAttempts : 2, unauthenticatedClientTimeout : 200, heartbeatInterval: 10000 } as any }, { type: 'ws-text', options: { urlPath: '/e2e-v3', maxAuthAttempts : 2, unauthenticatedClientTimeout : 200, heartbeatInterval: 10000 } as any }, { type: 'http', options: { allowAuthData: true, enableAuthEndpoint: true, } as any } ], monitoring: { type: 'http', options: { url: '/monitoring', allowOpenPermissions: false, headerKey: 'deepstream-password', headerValue: 'deepstream-secret' } as any }, telemetry: { type: 'deepstreamIO', options: { enabled: false } }, logger: { type: 'default', options: { logLevel: LOG_LEVEL.WARN } }, locks: { type: 'default', options: { holdTimeout : 1500, requestTimeout : 1500, } as any }, clusterNode: { type: 'default', options: { } as any }, clusterRegistry: { type: 'default', options: { keepAliveInterval: 20, activeCheckInterval: 200 } as any }, clusterStates: { type: 'default', options: { reconciliationTimeout : 100, } as any }, storage: { path: './src/services/cache/local-cache', options: { } as any } }) ================================================ FILE: tsconfig.json ================================================ { "compilerOptions": { "target": "ES2018", "module": "commonjs", "outDir": "dist", "declaration": true, "sourceMap": true, "allowJs": false, "resolveJsonModule": true, "noUnusedLocals": true, "strict": true, "lib": ["es2018"], "types": [ "node", "mocha" ] }, "include": [ "protocol/**/*", "bin/*", "src/**/*", "src/**/*.json", "test-e2e/**/*", "package.json", "types/uws.d.ts" ], "exclude": [ "**/*.spec.ts", "node_modules" ], "files": [ "types/global.d.ts", "types/uws.d.ts" ], } ================================================ FILE: tslint.json ================================================ { "defaultSeverity": "error", "extends": [ "tslint:recommended" ], "jsRules": {}, "rules": { "arrow-parens": [true], "ordered-imports": false, "trailing-comma": false, "array-type": [true, "array-simple"], "prefer-const": true, "new-parens": true, "no-consecutive-blank-lines": true, "no-trailing-whitespace": true, "no-unnecessary-initializer": true, "one-variable-per-declaration": true, "space-before-function-paren": [true, "always"], "interface-name": [true, "never-prefix"], "forin": false, "indent": [true, "spaces", 2], "jsdoc-format": false, "object-literal-sort-keys": false, "member-ordering": false, "prefer-for-of": false, "max-line-length": false, "only-arrow-functions": false, "ban-types": false, "no-console": false, "no-empty": false, "semicolon": [true, "never"], "no-var-requires": false, "quotemark": [true, "single", "avoid-escape", "avoid-template"], "unified-signatures": false }, "rulesDirectory": [] } ================================================ FILE: types/global.d.ts ================================================ declare namespace NodeJS { export interface Global { deepstreamCLI: any deepstreamLibDir: string | null deepstreamConfDir: string | null require (path: string): any e2eHarness: any // Used by e2e tests } } ================================================ FILE: types/uws.d.ts ================================================ /* * Authored by Alex Hultman, 2018-2021. * Intellectual property of third-party. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ declare namespace uws { /** Native type representing a raw uSockets struct us_listen_socket_t. * Careful with this one, it is entirely unchecked and native so invalid usage will blow up. */ export interface us_listen_socket { } /** Native type representing a raw uSockets struct us_socket_t. * Careful with this one, it is entirely unchecked and native so invalid usage will blow up. */ export interface us_socket { } /** Native type representing a raw uSockets struct us_socket_context_t. * Used while upgrading a WebSocket manually. */ export interface us_socket_context_t { } /** Recognized string types, things C++ can read and understand as strings. * "String" does not have to mean "text", it can also be "binary". * * Ironically, JavaScript strings are the least performant of all options, to pass or receive to/from C++. * This because we expect UTF-8, which is packed in 8-byte chars. JavaScript strings are UTF-16 internally meaning extra copies and reinterpretation are required. * * That's why all events pass data by ArrayBuffer and not JavaScript strings, as they allow zero-copy data passing. * * You can always do Buffer.from(arrayBuffer).toString(), but keeping things binary and as ArrayBuffer is preferred. */ export type RecognizedString = string | ArrayBuffer | Uint8Array | Int8Array | Uint16Array | Int16Array | Uint32Array | Int32Array | Float32Array | Float64Array; /** A WebSocket connection that is valid from open to close event. * Read more about this in the user manual. */ export interface WebSocket { /** Sends a message. Returns 1 for success, 2 for dropped due to backpressure limit, and 0 for built up backpressure that will drain over time. You can check backpressure before or after sending by calling getBufferedAmount(). * * Make sure you properly understand the concept of backpressure. Check the backpressure example file. */ send(message: RecognizedString, isBinary?: boolean, compress?: boolean) : number; /** Returns the bytes buffered in backpressure. This is similar to the bufferedAmount property in the browser counterpart. * Check backpressure example. */ getBufferedAmount() : number; /** Gracefully closes this WebSocket. Immediately calls the close handler. * A WebSocket close message is sent with code and shortMessage. */ end(code?: number, shortMessage?: RecognizedString) : void; /** Forcefully closes this WebSocket. Immediately calls the close handler. * No WebSocket close message is sent. */ close() : void; /** Sends a ping control message. Returns sendStatus similar to WebSocket.send (regarding backpressure). This helper function correlates to WebSocket::send(message, uWS::OpCode::PING, ...) in C++. */ ping(message?: RecognizedString) : number; /** Subscribe to a topic. */ subscribe(topic: RecognizedString) : boolean; /** Unsubscribe from a topic. Returns true on success, if the WebSocket was subscribed. */ unsubscribe(topic: RecognizedString) : boolean; /** Returns whether this websocket is subscribed to topic. */ isSubscribed(topic: RecognizedString) : boolean; /** Returns a list of topics this websocket is subscribed to. */ getTopics() : string[]; /** Publish a message under topic. Backpressure is managed according to maxBackpressure, closeOnBackpressureLimit settings. * Order is guaranteed since v20. */ publish(topic: RecognizedString, message: RecognizedString, isBinary?: boolean, compress?: boolean) : boolean; /** See HttpResponse.cork. Takes a function in which the socket is corked (packing many sends into one single syscall/SSL block) */ cork(cb: () => void) : WebSocket; /** Returns the remote IP address. Note that the returned IP is binary, not text. * * IPv4 is 4 byte long and can be converted to text by printing every byte as a digit between 0 and 255. * IPv6 is 16 byte long and can be converted to text in similar ways, but you typically print digits in HEX. * * See getRemoteAddressAsText() for a text version. */ getRemoteAddress() : ArrayBuffer; /** Returns the remote IP address as text. See RecognizedString. */ getRemoteAddressAsText() : ArrayBuffer; /** Returns the UserData object. */ getUserData() : UserData; } /** An HttpResponse is valid until either onAborted callback or any of the .end/.tryEnd calls succeed. You may attach user data to this object. */ export interface HttpResponse { /** Writes the HTTP status message such as "200 OK". * This has to be called first in any response, otherwise * it will be called automatically with "200 OK". * * If you want to send custom headers in a WebSocket * upgrade response, you have to call writeStatus with * "101 Switching Protocols" before you call writeHeader, * otherwise your first call to writeHeader will call * writeStatus with "200 OK" and the upgrade will fail. * * As you can imagine, we format outgoing responses in a linear * buffer, not in a hash table. You can read about this in * the user manual under "corking". */ /** Pause http body streaming (throttle) */ pause() : void; /** Resume http body streaming (unthrottle) */ resume() : void; writeStatus(status: RecognizedString) : HttpResponse; /** Writes key and value to HTTP response. * See writeStatus and corking. */ writeHeader(key: RecognizedString, value: RecognizedString) : HttpResponse; /** Enters or continues chunked encoding mode. Writes part of the response. End with zero length write. Returns true if no backpressure was added. */ write(chunk: RecognizedString) : boolean; /** Ends this response by copying the contents of body. */ end(body?: RecognizedString, closeConnection?: boolean) : HttpResponse; /** Ends this response without a body. */ endWithoutBody(reportedContentLength?: number, closeConnection?: boolean) : HttpResponse; /** Ends this response, or tries to, by streaming appropriately sized chunks of body. Use in conjunction with onWritable. Returns tuple [ok, hasResponded].*/ tryEnd(fullBodyOrChunk: RecognizedString, totalSize: number) : [boolean, boolean]; /** Immediately force closes the connection. Any onAborted callback will run. */ close() : HttpResponse; /** Returns the global byte write offset for this response. Use with onWritable. */ getWriteOffset() : number; /** Registers a handler for writable events. Continue failed write attempts in here. * You MUST return true for success, false for failure. * Writing nothing is always success, so by default you must return true. */ onWritable(handler: (offset: number) => boolean) : HttpResponse; /** Every HttpResponse MUST have an attached abort handler IF you do not respond * to it immediately inside of the callback. Returning from an Http request handler * without attaching (by calling onAborted) an abort handler is ill-use and will terminate. * When this event emits, the response has been aborted and may not be used. */ onAborted(handler: () => void) : HttpResponse; /** Handler for reading data from POST and such requests. You MUST copy the data of chunk if isLast is not true. We Neuter ArrayBuffers on return, making it zero length.*/ onData(handler: (chunk: ArrayBuffer, isLast: boolean) => void) : HttpResponse; /** Returns the remote IP address in binary format (4 or 16 bytes). */ getRemoteAddress() : ArrayBuffer; /** Returns the remote IP address as text. */ getRemoteAddressAsText() : ArrayBuffer; /** Returns the remote IP address in binary format (4 or 16 bytes), as reported by the PROXY Protocol v2 compatible proxy. */ getProxiedRemoteAddress() : ArrayBuffer; /** Returns the remote IP address as text, as reported by the PROXY Protocol v2 compatible proxy. */ getProxiedRemoteAddressAsText() : ArrayBuffer; /** Corking a response is a performance improvement in both CPU and network, as you ready the IO system for writing multiple chunks at once. * By default, you're corked in the immediately executing top portion of the route handler. In all other cases, such as when returning from * await, or when being called back from an async database request or anything that isn't directly executing in the route handler, you'll want * to cork before calling writeStatus, writeHeader or just write. Corking takes a callback in which you execute the writeHeader, writeStatus and * such calls, in one atomic IO operation. This is important, not only for TCP but definitely for TLS where each write would otherwise result * in one TLS block being sent off, each with one send syscall. * * Example usage: * * ``` * res.cork(() => { * res.writeStatus("200 OK").writeHeader("Some", "Value").write("Hello world!"); * }); * ``` */ cork(cb: () => void) : HttpResponse; /** Upgrades a HttpResponse to a WebSocket. See UpgradeAsync, UpgradeSync example files. */ upgrade(userData : UserData, secWebSocketKey: RecognizedString, secWebSocketProtocol: RecognizedString, secWebSocketExtensions: RecognizedString, context: us_socket_context_t) : void; /** Arbitrary user data may be attached to this object */ [key: string]: any; } /** An HttpRequest is stack allocated and only accessible during the callback invocation. */ export interface HttpRequest { /** Returns the lowercased header value or empty string. */ getHeader(lowerCaseKey: RecognizedString) : string; /** Returns the parsed parameter at index. Corresponds to route. */ getParameter(index: number) : string; /** Returns the URL including initial /slash */ getUrl() : string; /** Returns the lowercased HTTP method, useful for "any" routes. */ getMethod() : string; /** Returns the HTTP method as-is. */ getCaseSensitiveMethod() : string; /** Returns the raw querystring (the part of URL after ? sign) or empty string. */ getQuery() : string; /** Returns a decoded query parameter value or empty string. */ getQuery(key: string) : string; /** Loops over all headers. */ forEach(cb: (key: string, value: string) => void) : void; /** Setting yield to true is to say that this route handler did not handle the route, causing the router to continue looking for a matching route handler, or fail. */ setYield(_yield: boolean) : HttpRequest; } /** A structure holding settings and handlers for a WebSocket URL route handler. */ export interface WebSocketBehavior { /** Maximum length of received message. If a client tries to send you a message larger than this, the connection is immediately closed. Defaults to 16 * 1024. */ maxPayloadLength?: number; /** Whether or not we should automatically close the socket when a message is dropped due to backpressure. Defaults to false. */ closeOnBackpressureLimit?: number; /** Maximum number of minutes a WebSocket may be connected before being closed by the server. 0 disables the feature. */ maxLifetime?: number; /** Maximum amount of seconds that may pass without sending or getting a message. Connection is closed if this timeout passes. Resolution (granularity) for timeouts are typically 4 seconds, rounded to closest. * Disable by using 0. Defaults to 120. */ idleTimeout?: number; /** What permessage-deflate compression to use. uWS.DISABLED, uWS.SHARED_COMPRESSOR or any of the uWS.DEDICATED_COMPRESSOR_xxxKB. Defaults to uWS.DISABLED. */ compression?: CompressOptions; /** Maximum length of allowed backpressure per socket when publishing or sending messages. Slow receivers with too high backpressure will be skipped until they catch up or timeout. Defaults to 64 * 1024. */ maxBackpressure?: number; /** Whether or not we should automatically send pings to uphold a stable connection given whatever idleTimeout. */ sendPingsAutomatically?: boolean; /** Upgrade handler used to intercept HTTP upgrade requests and potentially upgrade to WebSocket. * See UpgradeAsync and UpgradeSync example files. */ upgrade?: (res: HttpResponse, req: HttpRequest, context: us_socket_context_t) => void | Promise; /** Handler for new WebSocket connection. WebSocket is valid from open to close, no errors. */ open?: (ws: WebSocket) => void | Promise; /** Handler for a WebSocket message. Messages are given as ArrayBuffer no matter if they are binary or not. Given ArrayBuffer is valid during the lifetime of this callback (until first await or return) and will be neutered. */ message?: (ws: WebSocket, message: ArrayBuffer, isBinary: boolean) => void | Promise; /** Handler for when WebSocket backpressure drains. Check ws.getBufferedAmount(). Use this to guide / drive your backpressure throttling. */ drain?: (ws: WebSocket) => void; /** Handler for close event, no matter if error, timeout or graceful close. You may not use WebSocket after this event. Do not send on this WebSocket from within here, it is closed. */ close?: (ws: WebSocket, code: number, message: ArrayBuffer) => void; /** Handler for received ping control message. You do not need to handle this, pong messages are automatically sent as per the standard. */ ping?: (ws: WebSocket, message: ArrayBuffer) => void; /** Handler for received pong control message. */ pong?: (ws: WebSocket, message: ArrayBuffer) => void; /** Handler for subscription changes. */ subscription?: (ws: WebSocket, topic: ArrayBuffer, newCount: number, oldCount: number) => void; } /** Options used when constructing an app. Especially for SSLApp. * These are options passed directly to uSockets, C layer. */ export interface AppOptions { key_file_name?: RecognizedString; cert_file_name?: RecognizedString; ca_file_name?: RecognizedString; passphrase?: RecognizedString; dh_params_file_name?: RecognizedString; ssl_ciphers?: RecognizedString; /** This translates to SSL_MODE_RELEASE_BUFFERS */ ssl_prefer_low_memory_usage?: boolean; } export enum ListenOptions { LIBUS_LISTEN_DEFAULT = 0, LIBUS_LISTEN_EXCLUSIVE_PORT = 1 } /** TemplatedApp is either an SSL or non-SSL app. See App for more info, read user manual. */ export interface TemplatedApp { /** Listens to hostname & port. Callback hands either false or a listen socket. */ listen(host: RecognizedString, port: number, cb: (listenSocket: us_listen_socket | false) => void | Promise) : TemplatedApp; /** Listens to port. Callback hands either false or a listen socket. */ listen(port: number, cb: (listenSocket: us_listen_socket | false) => void | Promise) : TemplatedApp; /** Listens to port and sets Listen Options. Callback hands either false or a listen socket. */ listen(port: number, options: ListenOptions, cb: (listenSocket: us_listen_socket | false) => void | Promise) : TemplatedApp; /** Listens to unix socket. Callback hands either false or a listen socket. */ listen_unix(cb: (listenSocket: us_listen_socket) => void | Promise, path: RecognizedString) : TemplatedApp; /** Registers an HTTP GET handler matching specified URL pattern. */ get(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP POST handler matching specified URL pattern. */ post(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP OPTIONS handler matching specified URL pattern. */ options(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP DELETE handler matching specified URL pattern. */ del(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP PATCH handler matching specified URL pattern. */ patch(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP PUT handler matching specified URL pattern. */ put(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP HEAD handler matching specified URL pattern. */ head(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP CONNECT handler matching specified URL pattern. */ connect(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP TRACE handler matching specified URL pattern. */ trace(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers an HTTP handler matching specified URL pattern on any HTTP method. */ any(pattern: RecognizedString, handler: (res: HttpResponse, req: HttpRequest) => void | Promise) : TemplatedApp; /** Registers a handler matching specified URL pattern where WebSocket upgrade requests are caught. */ ws(pattern: RecognizedString, behavior: WebSocketBehavior) : TemplatedApp; /** Publishes a message under topic, for all WebSockets under this app. See WebSocket.publish. */ publish(topic: RecognizedString, message: RecognizedString, isBinary?: boolean, compress?: boolean) : boolean; /** Returns number of subscribers for this topic. */ numSubscribers(topic: RecognizedString) : number; /** Adds a server name. */ addServerName(hostname: string, options: AppOptions) : TemplatedApp; /** Browse to SNI domain. Used together with .get, .post and similar to attach routes under SNI domains. */ domain(domain: string) : TemplatedApp; /** Removes a server name. */ removeServerName(hostname: string) : TemplatedApp; /** Registers a synchronous callback on missing server names. See /examples/ServerName.js. */ missingServerName(cb: (hostname: string) => void) : TemplatedApp; } /** Constructs a non-SSL app. An app is your starting point where you attach behavior to URL routes. * This is also where you listen and run your app, set any SSL options (in case of SSLApp) and the like. */ export function App(options?: AppOptions) : TemplatedApp; /** Constructs an SSL app. See App. */ export function SSLApp(options: AppOptions) : TemplatedApp; /** Closes a uSockets listen socket. */ export function us_listen_socket_close(listenSocket: us_listen_socket) : void; /** Gets local port of socket (or listenSocket) or -1. */ export function us_socket_local_port(socket: us_socket) : number; export interface MultipartField { data: ArrayBuffer; name: string; type?: string; filename?: string; } /** Takes a POSTed body and contentType, and returns an array of parts if the request is a multipart request */ export function getParts(body: RecognizedString, contentType: RecognizedString) : MultipartField[] | undefined; /** WebSocket compression options. Combine any compressor with any decompressor using bitwise OR. */ export type CompressOptions = number; /** No compression (always a good idea if you operate using an efficient binary protocol) */ export var DISABLED: CompressOptions; /** Zero memory overhead compression. */ export var SHARED_COMPRESSOR: CompressOptions; /** Zero memory overhead decompression. */ export var SHARED_DECOMPRESSOR: CompressOptions; /** Sliding dedicated compress window, requires 3KB of memory per socket */ export var DEDICATED_COMPRESSOR_3KB: CompressOptions; /** Sliding dedicated compress window, requires 4KB of memory per socket */ export var DEDICATED_COMPRESSOR_4KB: CompressOptions; /** Sliding dedicated compress window, requires 8KB of memory per socket */ export var DEDICATED_COMPRESSOR_8KB: CompressOptions; /** Sliding dedicated compress window, requires 16KB of memory per socket */ export var DEDICATED_COMPRESSOR_16KB: CompressOptions; /** Sliding dedicated compress window, requires 32KB of memory per socket */ export var DEDICATED_COMPRESSOR_32KB: CompressOptions; /** Sliding dedicated compress window, requires 64KB of memory per socket */ export var DEDICATED_COMPRESSOR_64KB: CompressOptions; /** Sliding dedicated compress window, requires 128KB of memory per socket */ export var DEDICATED_COMPRESSOR_128KB: CompressOptions; /** Sliding dedicated compress window, requires 256KB of memory per socket */ export var DEDICATED_COMPRESSOR_256KB: CompressOptions; /** Sliding dedicated decompress window, requires 32KB of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR_32KB: CompressOptions; /** Sliding dedicated decompress window, requires 16KB of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR_16KB: CompressOptions; /** Sliding dedicated decompress window, requires 8KB of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR_8KB: CompressOptions; /** Sliding dedicated decompress window, requires 4KB of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR_4KB: CompressOptions; /** Sliding dedicated decompress window, requires 2KB of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR_2KB: CompressOptions; /** Sliding dedicated decompress window, requires 1KB of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR_1KB: CompressOptions; /** Sliding dedicated decompress window, requires 512B of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR_512B: CompressOptions; /** Sliding dedicated decompress window, requires 32KB of memory per socket (plus about 23KB) */ export var DEDICATED_DECOMPRESSOR: CompressOptions; }