Repository: OriginTrail/ot-node Branch: v8/develop Commit: a8156d63132d Files: 436 Total size: 1.5 MB Directory structure: gitextract_wywppseo/ ├── .codex/ │ ├── review-prompt.md │ └── review-schema.json ├── .eslintrc.cjs ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ └── bug_report_v8.md │ ├── actions/ │ │ └── setup/ │ │ └── action.yml │ ├── pull_request_template.md │ ├── release-drafter-template.yml │ └── workflows/ │ ├── check-package-lock.yml │ ├── checks.yml │ ├── codex-review.yml │ ├── release-drafter-config.yml │ └── update-cache.yml ├── .gitignore ├── .husky/ │ ├── .gitignore │ └── pre-commit ├── .lintstagedrc.json ├── .prettierrc ├── Alpine.Dockerfile ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Debian.Dockerfile ├── LICENSE ├── README.md ├── Ubuntu.Dockerfile ├── bin/ │ ├── darwin/ │ │ ├── arm64/ │ │ │ └── .gitkeep │ │ └── x64/ │ │ └── .gitkeep │ ├── linux/ │ │ ├── arm64/ │ │ │ └── .gitkeep │ │ └── x64/ │ │ └── .gitkeep │ └── win32/ │ └── x64/ │ └── .gitkeep ├── blazegraph-migration/ │ ├── README.md │ ├── check_quad_num.sh │ ├── export.sh │ ├── import.sh │ └── job_pool.sh ├── config/ │ ├── config.json │ └── papertrail.yml ├── cucumber.js ├── dependencies.md ├── docker/ │ ├── docker-compose-alpine-blazegraph.yaml │ ├── docker-compose-alpine-graphdb.yaml │ ├── docker-compose-debian-blazegraph.yaml │ ├── docker-compose-debian-graphdb.yaml │ ├── docker-compose-ubuntu-blazegraph.yaml │ └── docker-compose-ubuntu-graphdb.yaml ├── docs/ │ ├── openapi/ │ │ └── DKGv8.yaml │ └── postman/ │ └── DKGv8.postman_collection.json ├── index.js ├── installer/ │ ├── README.md │ └── installer.sh ├── ot-node.js ├── package.json ├── scripts/ │ ├── copy-assertions.js │ ├── set-ask.js │ ├── set-operator-fee.js │ ├── set-stake.js │ └── utils.js ├── src/ │ ├── commands/ │ │ ├── blockchain-event-listener/ │ │ │ ├── blockchain-event-listener-command.js │ │ │ └── event-listener-command.js │ │ ├── cleaners/ │ │ │ ├── ask-cleaner-command.js │ │ │ ├── ask-response-cleaner-command.js │ │ │ ├── batch-get-cleaner-command.js │ │ │ ├── blockchain-event-cleaner-command.js │ │ │ ├── cleaner-command.js │ │ │ ├── commands-cleaner-command.js │ │ │ ├── finality-cleaner-command.js │ │ │ ├── finality-response-cleaner-command.js │ │ │ ├── get-cleaner-command.js │ │ │ ├── get-response-cleaner-command.js │ │ │ ├── operation-id-cleaner-command.js │ │ │ ├── pending-storage-cleaner-command.js │ │ │ ├── publish-cleaner-command.js │ │ │ ├── publish-response-cleaner-command.js │ │ │ ├── update-cleaner-command.js │ │ │ └── update-response-cleaner-command.js │ │ ├── command-executor.js │ │ ├── command-resolver.js │ │ ├── command.js │ │ ├── common/ │ │ │ ├── dial-peers-command.js │ │ │ ├── log-public-addresses-command.js │ │ │ ├── otnode-update-command.js │ │ │ ├── send-telemetry-command.js │ │ │ ├── send-transaction-command.js │ │ │ ├── sharding-table-check-command.js │ │ │ └── validate-asset-command.js │ │ ├── paranet/ │ │ │ ├── paranet-sync-command.js │ │ │ └── start-paranet-sync-commands.js │ │ ├── protocols/ │ │ │ ├── ask/ │ │ │ │ ├── receiver/ │ │ │ │ │ └── v1.0.0/ │ │ │ │ │ └── v1-0-0-handle-ask-request-command.js │ │ │ │ └── sender/ │ │ │ │ ├── ask-find-shard-command.js │ │ │ │ ├── ask-schedule-messages-command.js │ │ │ │ ├── network-ask-command.js │ │ │ │ └── v1.0.0/ │ │ │ │ └── v1-0-0-ask-request-command.js │ │ │ ├── common/ │ │ │ │ ├── find-curated-paranet-nodes-command.js │ │ │ │ ├── find-shard-command.js │ │ │ │ ├── handle-protocol-message-command.js │ │ │ │ ├── network-protocol-command.js │ │ │ │ ├── protocol-message-command.js │ │ │ │ ├── protocol-request-command.js │ │ │ │ ├── protocol-schedule-messages-command.js │ │ │ │ └── validate-assertion-metadata-command.js │ │ │ ├── finality/ │ │ │ │ ├── receiver/ │ │ │ │ │ ├── publish-finality-save-ack-command.js │ │ │ │ │ └── v1.0.0/ │ │ │ │ │ └── v1-0-0-handle-finality-request-command.js │ │ │ │ └── sender/ │ │ │ │ ├── finality-schedule-messages-command.js │ │ │ │ ├── find-publisher-node-command.js │ │ │ │ ├── network-finality-command.js │ │ │ │ └── v1.0.0/ │ │ │ │ └── v1-0-0-finality-request-command.js │ │ │ ├── get/ │ │ │ │ ├── receiver/ │ │ │ │ │ └── v1.0.0/ │ │ │ │ │ ├── v1-0-0-handle-batch-get-request-command.js │ │ │ │ │ └── v1-0-0-handle-get-request-command.js │ │ │ │ └── sender/ │ │ │ │ ├── batch-get-command.js │ │ │ │ └── get-command.js │ │ │ ├── publish/ │ │ │ │ ├── publish-finalization-command.js │ │ │ │ ├── receiver/ │ │ │ │ │ └── v1.0.0/ │ │ │ │ │ └── v1-0-0-handle-store-request-command.js │ │ │ │ └── sender/ │ │ │ │ └── publish-replication-command.js │ │ │ └── update/ │ │ │ ├── receiver/ │ │ │ │ └── v1.0.0/ │ │ │ │ └── v1-0-0-handle-update-request-command.js │ │ │ ├── sender/ │ │ │ │ ├── network-update-command.js │ │ │ │ ├── update-find-shard-command.js │ │ │ │ ├── update-schedule-messages-command.js │ │ │ │ ├── update-validate-asset-command.js │ │ │ │ └── v1.0.0/ │ │ │ │ └── v1-0-0-update-request-command.js │ │ │ ├── update-assertion-command.js │ │ │ └── update-validate-assertion-metadata-command.js │ │ └── query/ │ │ └── query-command.js │ ├── constants/ │ │ └── constants.js │ ├── controllers/ │ │ ├── http-api/ │ │ │ ├── base-http-api-controller.js │ │ │ ├── http-api-router.js │ │ │ ├── v0/ │ │ │ │ ├── bid-suggestion-http-api-controller-v0.js │ │ │ │ ├── get-http-api-controller-v0.js │ │ │ │ ├── info-http-api-controller-v0.js │ │ │ │ ├── local-store-http-api-controller-v0.js │ │ │ │ ├── publish-http-api-controller-v0.js │ │ │ │ ├── query-http-api-controller-v0.js │ │ │ │ ├── request-schema/ │ │ │ │ │ ├── bid-suggestion-schema-v0.js │ │ │ │ │ ├── get-schema-v0.js │ │ │ │ │ ├── local-store-schema-v0.js │ │ │ │ │ ├── publish-schema-v0.js │ │ │ │ │ ├── query-schema-v0.js │ │ │ │ │ └── update-schema-v0.js │ │ │ │ ├── result-http-api-controller-v0.js │ │ │ │ └── update-http-api-controller-v0.js │ │ │ └── v1/ │ │ │ ├── .gitkeep │ │ │ ├── ask-http-api-controller-v1.js │ │ │ ├── direct-query-http-api-controller-v1.js │ │ │ ├── finality-http-api-controller-v1.js │ │ │ ├── get-http-api-controller-v1.js │ │ │ ├── info-http-api-controller-v1.js │ │ │ ├── local-store-http-api-controller-v1.js │ │ │ ├── publish-http-api-controller-v1.js │ │ │ ├── query-http-api-controller-v1.js │ │ │ ├── request-schema/ │ │ │ │ ├── .gitkeep │ │ │ │ ├── ask-schema-v1.js │ │ │ │ ├── direct-query-schema-v1.js │ │ │ │ ├── finality-schema-v1.js │ │ │ │ ├── get-schema-v1.js │ │ │ │ ├── local-store-schema-v1.js │ │ │ │ ├── publish-schema-v1.js │ │ │ │ └── query-schema-v1.js │ │ │ └── result-http-api-controller-v1.js │ │ └── rpc/ │ │ ├── ask-rpc-controller.js │ │ ├── base-rpc-controller.js │ │ ├── batch-get-rpc-controller.js │ │ ├── finality-rpc-controller.js │ │ ├── get-rpc-controller.js │ │ ├── publish-rpc-controller.js │ │ ├── rpc-router.js │ │ └── update-rpc-controller.js │ ├── logger/ │ │ └── logger.js │ ├── migration/ │ │ ├── base-migration.js │ │ ├── migration-executor.js │ │ ├── redis-setup-migration.js │ │ └── triple-store-user-configuration-migration.js │ ├── modules/ │ │ ├── auto-updater/ │ │ │ ├── auto-updater-module-manager.js │ │ │ └── implementation/ │ │ │ └── ot-auto-updater.js │ │ ├── base-module-manager.js │ │ ├── blockchain/ │ │ │ ├── blockchain-module-manager.js │ │ │ └── implementation/ │ │ │ ├── base/ │ │ │ │ └── base-service.js │ │ │ ├── eth/ │ │ │ │ └── eth-service.js │ │ │ ├── gnosis/ │ │ │ │ └── gnosis-service.js │ │ │ ├── hardhat/ │ │ │ │ └── hardhat-service.js │ │ │ ├── ot-parachain/ │ │ │ │ └── ot-parachain-service.js │ │ │ ├── polygon/ │ │ │ │ └── polygon-service.js │ │ │ ├── web3-service-validator.js │ │ │ └── web3-service.js │ │ ├── blockchain-events/ │ │ │ ├── blockchain-events-module-manager.js │ │ │ └── implementation/ │ │ │ ├── blockchain-events-service.js │ │ │ └── ot-ethers/ │ │ │ └── ot-ethers.js │ │ ├── http-client/ │ │ │ ├── http-client-module-manager.js │ │ │ └── implementation/ │ │ │ ├── express-http-client.js │ │ │ └── middleware/ │ │ │ ├── authentication-middleware.js │ │ │ ├── authorization-middleware.js │ │ │ ├── blockchain-id-midleware.js │ │ │ ├── rate-limiter-middleware.js │ │ │ └── request-validation-middleware.js │ │ ├── module-config-validation.js │ │ ├── network/ │ │ │ ├── implementation/ │ │ │ │ └── libp2p-service.js │ │ │ └── network-module-manager.js │ │ ├── repository/ │ │ │ ├── implementation/ │ │ │ │ └── sequelize/ │ │ │ │ ├── migrations/ │ │ │ │ │ ├── 20211117005500-create-commands.js │ │ │ │ │ ├── 20211117005504-create-operation_ids.js │ │ │ │ │ ├── 20220620100000-create-publish.js │ │ │ │ │ ├── 20220620100005-create-publish-response.js │ │ │ │ │ ├── 20220623125000-create-get.js │ │ │ │ │ ├── 20220623125001-create-get-response.js │ │ │ │ │ ├── 20220624020509-create-event.js │ │ │ │ │ ├── 20220624103229-create-ability.js │ │ │ │ │ ├── 20220624103610-create-role.js │ │ │ │ │ ├── 20220624103615-create-user.js │ │ │ │ │ ├── 20220624103658-create-token.js │ │ │ │ │ ├── 20220624113659-create-role-ability.js │ │ │ │ │ ├── 20220628113824-add-predefined-auth-entities.js │ │ │ │ │ ├── 20221025120253-create-blockchain-event.js │ │ │ │ │ ├── 20221025212800-create-shard.js │ │ │ │ │ ├── 20221028125900-create-blockchain.js │ │ │ │ │ ├── 20221114115524-update-publish-add-agreement-data.js │ │ │ │ │ ├── 20221206183634-update-shard-types.js │ │ │ │ │ ├── 20221214110050-update-commands-types.js │ │ │ │ │ ├── 20221215130500-update-event-types.js │ │ │ │ │ ├── 20230216112400-add-abilities.js │ │ │ │ │ ├── 20230227094500-create-update.js │ │ │ │ │ ├── 20230303131200-update-publish-remove-agreement-data.js │ │ │ │ │ ├── 20230303131400-create-update-response.js │ │ │ │ │ ├── 20230413194400-update-command-period-type.js │ │ │ │ │ ├── 20230419140000-create-service-agreements.js │ │ │ │ │ ├── 20230502110300-add-blockchain-event-index.js │ │ │ │ │ ├── 20231201140100-event-add-blockchain-id.js │ │ │ │ │ ├── 20231221131300-update-abilities.js │ │ │ │ │ ├── 20233010122500-update-blockchain-id.js │ │ │ │ │ ├── 20233011121700-remove-blockchain-info.js │ │ │ │ │ ├── 20240126120000-shard-add-sha256blobl.js │ │ │ │ │ ├── 20240201100000-remove-sha256Blob.js │ │ │ │ │ ├── 20240221162000-add-service-agreement-data-source.js │ │ │ │ │ ├── 20240429083058-create-paranet.js │ │ │ │ │ ├── 20240529070000-create-missed-paranet-asset.js │ │ │ │ │ ├── 20240923195000-create-publish-paranet.js │ │ │ │ │ ├── 20240924161700-create-paranet-synced-asset.js │ │ │ │ │ ├── 20240924205500-create-publish-paranet-response.js │ │ │ │ │ ├── 20240927110000-change-paranet-synced-asset-nullable-assertions.js │ │ │ │ │ ├── 20240930113000-add-error-message.js │ │ │ │ │ ├── 20241011112100-remove-knowledge-asset-id.js │ │ │ │ │ ├── 20241014164500-paranet-synced-asset-optional-fileds.js │ │ │ │ │ ├── 20241023170300-add-synced-data-source.js │ │ │ │ │ ├── 20241105150000-change-data-source-col-type-in-paranet-synced-asset.js │ │ │ │ │ ├── 20241105160000-add-indexes-to-tables.js │ │ │ │ │ ├── 20241125151200-rename-keyword-column-to-datasetroot-in-responses.js │ │ │ │ │ ├── 20241126114400-add-commands-priority.js │ │ │ │ │ ├── 20241129120000-add-commands-is_blocking.js │ │ │ │ │ ├── 20241129125800-remove-datasetroot-response-table.js │ │ │ │ │ ├── 20241201152000-update-blockchain-events.js │ │ │ │ │ ├── 20241202214500-update-blockchain-table.js │ │ │ │ │ ├── 20241203125000-create-finality.js │ │ │ │ │ ├── 20241203125001-create-finality-response.js │ │ │ │ │ ├── 20241211204400-rename-ask.js │ │ │ │ │ ├── 20241211205400-create-finality-response.js │ │ │ │ │ ├── 20241211205400-create-finality-status.js │ │ │ │ │ ├── 20241211205400-create-finality.js │ │ │ │ │ ├── 20241212122200-add-min-acks-reached-column.js │ │ │ │ │ ├── 20241215122200-create-paranet-kc.js │ │ │ │ │ ├── 20241226151800-prune-commands.js │ │ │ │ │ ├── 20250401123500-truncate-commands-table.js │ │ │ │ │ ├── 20250401155600-create-random-sampling-chanalage.js │ │ │ │ │ ├── 20250408164300-create-triples-inserted-count-table.js │ │ │ │ │ ├── 20250422150500-add-tx-hash-blockchain-event.js │ │ │ │ │ ├── 20250509142900-create-batch-get.js │ │ │ │ │ ├── 20250509142901-create-latest-synced-kc.js │ │ │ │ │ └── 20250509142902-create-sync-missed-kc.js │ │ │ │ ├── models/ │ │ │ │ │ ├── ability.js │ │ │ │ │ ├── ask-response.js │ │ │ │ │ ├── ask.js │ │ │ │ │ ├── base-sync-missed-kc.js │ │ │ │ │ ├── batch-get.js │ │ │ │ │ ├── blockchain-event.js │ │ │ │ │ ├── blockchain.js │ │ │ │ │ ├── commands.js │ │ │ │ │ ├── event.js │ │ │ │ │ ├── finality-response.js │ │ │ │ │ ├── finality-status.js │ │ │ │ │ ├── finality.js │ │ │ │ │ ├── get-response.js │ │ │ │ │ ├── get.js │ │ │ │ │ ├── gnosis-sync-missed-kc.js │ │ │ │ │ ├── hardhat1-sync-missed-kc.js │ │ │ │ │ ├── hardhat2-sync-missed-kc.js │ │ │ │ │ ├── latest-synced-kc.js │ │ │ │ │ ├── missed-paranet-asset.js │ │ │ │ │ ├── operation_ids.js │ │ │ │ │ ├── otp-sync-missed-kc.js │ │ │ │ │ ├── paranet-kc.js │ │ │ │ │ ├── paranet-synced-asset.js │ │ │ │ │ ├── paranet.js │ │ │ │ │ ├── publish-paranet-response.js │ │ │ │ │ ├── publish-paranet.js │ │ │ │ │ ├── publish-response.js │ │ │ │ │ ├── publish.js │ │ │ │ │ ├── random-sampling-challenge.js │ │ │ │ │ ├── role-ability.js │ │ │ │ │ ├── role.js │ │ │ │ │ ├── shard.js │ │ │ │ │ ├── token.js │ │ │ │ │ ├── triples-inserted-count.js │ │ │ │ │ ├── update-response.js │ │ │ │ │ ├── update.js │ │ │ │ │ └── user.js │ │ │ │ ├── repositories/ │ │ │ │ │ ├── blockchain-event-repository.js │ │ │ │ │ ├── blockchain-missed-kc-repository.js │ │ │ │ │ ├── blockchain-repository.js │ │ │ │ │ ├── command-repository.js │ │ │ │ │ ├── event-repository.js │ │ │ │ │ ├── finality-status-repository.js │ │ │ │ │ ├── inserted-triples-repository.js │ │ │ │ │ ├── latest-synced-kc-repository.js │ │ │ │ │ ├── missed-paranet-asset-repository.js │ │ │ │ │ ├── operation-id-repository.js │ │ │ │ │ ├── operation-repository.js │ │ │ │ │ ├── operation-response.js │ │ │ │ │ ├── paranet-kc-repository.js │ │ │ │ │ ├── paranet-repository.js │ │ │ │ │ ├── paranet-synced-asset-repository.js │ │ │ │ │ ├── random-sampling-challenge-repository.js │ │ │ │ │ ├── shard-repository.js │ │ │ │ │ ├── token-repository.js │ │ │ │ │ └── user-repository.js │ │ │ │ ├── sequelize-migrator.js │ │ │ │ └── sequelize-repository.js │ │ │ └── repository-module-manager.js │ │ ├── telemetry/ │ │ │ ├── implementation/ │ │ │ │ └── quest-telemetry.js │ │ │ └── telemetry-module-manager.js │ │ ├── triple-store/ │ │ │ ├── implementation/ │ │ │ │ ├── ot-blazegraph/ │ │ │ │ │ └── ot-blazegraph.js │ │ │ │ ├── ot-fuseki/ │ │ │ │ │ └── ot-fuseki.js │ │ │ │ ├── ot-graphdb/ │ │ │ │ │ └── ot-graphdb.js │ │ │ │ ├── ot-neptune/ │ │ │ │ │ └── ot-neptune.js │ │ │ │ └── ot-triple-store.js │ │ │ └── triple-store-module-manager.js │ │ └── validation/ │ │ ├── implementation/ │ │ │ └── merkle-validation.js │ │ └── validation-module-manager.js │ └── service/ │ ├── ask-service.js │ ├── auth-service.js │ ├── batch-get-service.js │ ├── blockchain-events-service.js │ ├── claim-rewards-service.js │ ├── crypto-service.js │ ├── dependency-injection.js │ ├── file-service.js │ ├── finality-service.js │ ├── get-service.js │ ├── json-schema-service.js │ ├── messaging-service.js │ ├── operation-id-service.js │ ├── operation-service.js │ ├── paranet-service.js │ ├── pending-storage-service.js │ ├── proofing-service.js │ ├── protocol-service.js │ ├── publish-service.js │ ├── sharding-table-service.js │ ├── signature-service.js │ ├── sync-service.js │ ├── triple-store-service.js │ ├── ual-service.js │ ├── update-service.js │ ├── util/ │ │ ├── jwt-util.js │ │ └── string-util.js │ └── validation-service.js ├── test/ │ ├── assertions/ │ │ └── assertions.js │ ├── bdd/ │ │ ├── features/ │ │ │ ├── bid-suggestion.feature │ │ │ ├── get-errors.feature │ │ │ ├── publish-errors.feature │ │ │ ├── publish.feature │ │ │ ├── smoke.feature │ │ │ └── update-errors.feature │ │ ├── run-bdd.sh │ │ └── steps/ │ │ ├── api/ │ │ │ ├── bid-suggestion.mjs │ │ │ ├── get.mjs │ │ │ ├── info.mjs │ │ │ ├── publish.mjs │ │ │ ├── resolve.mjs │ │ │ └── update.mjs │ │ ├── blockchain.mjs │ │ ├── common.mjs │ │ ├── hooks.mjs │ │ └── lib/ │ │ ├── local-blockchain.mjs │ │ └── ot-node-process.mjs │ ├── modules/ │ │ └── telemetry/ │ │ ├── config.json │ │ └── telemetry.js │ ├── unit/ │ │ ├── api/ │ │ │ └── http-api-router.test.js │ │ ├── commands/ │ │ │ └── operation-id-cleaner-command.test.js │ │ ├── controllers/ │ │ │ └── publish-http-api-controller-v1.test.js │ │ ├── middleware/ │ │ │ ├── authentication-middleware.test.js │ │ │ └── authorization-middleware.test.js │ │ ├── mock/ │ │ │ ├── blockchain-module-manager-mock.js │ │ │ ├── command-executor-mock.js │ │ │ ├── event-emitter-mock.js │ │ │ ├── http-client-module-manager-mock.js │ │ │ ├── json-schema-service-mock.js │ │ │ ├── network-module-manager-mock.js │ │ │ ├── operation-id-service-mock.js │ │ │ ├── repository-module-manager-mock.js │ │ │ └── validation-module-manager-mock.js │ │ ├── modules/ │ │ │ ├── repository/ │ │ │ │ ├── config.json │ │ │ │ └── repository.test.js │ │ │ ├── triple-store/ │ │ │ │ ├── config.json │ │ │ │ └── triple-store.test.js │ │ │ └── validation/ │ │ │ ├── config.json │ │ │ └── validation-module-manager.test.js │ │ ├── service/ │ │ │ ├── auth-service.test.js │ │ │ ├── get-service.test.js │ │ │ ├── operation-id-service-cache.test.js │ │ │ ├── operation-service.test.js │ │ │ ├── publish-service.test.js │ │ │ ├── sharding-table-service.test.js │ │ │ ├── update-service.test.js │ │ │ ├── util/ │ │ │ │ └── jwt-util.test.js │ │ │ └── validation-service.test.js │ │ └── sparlql-query-service.test.js │ └── utilities/ │ ├── dkg-client-helper.mjs │ ├── http-api-helper.mjs │ ├── steps-utils.mjs │ └── utilities.js ├── tools/ │ ├── local-network-setup/ │ │ ├── .origintrail_noderc_template.json │ │ ├── README.md │ │ ├── generate-config-files.js │ │ ├── run-local-blockchain.js │ │ ├── setup-linux-environment.sh │ │ └── setup-macos-environment.sh │ ├── ot-parachain-account-mapping/ │ │ └── create-account-mapping-signature.js │ ├── substrate-accounts-mapping/ │ │ ├── README.md │ │ └── accounts-mapping.js │ └── token-generation.js └── v8-data-migration/ ├── abi/ │ ├── ContentAssetStorage.json │ └── ContentAssetStorageV2.json ├── blockchain-utils.js ├── constants.js ├── logger.js ├── run-data-migration.sh ├── sqlite-utils.js ├── triple-store-utils.js ├── v8-data-migration-utils.js ├── v8-data-migration.js └── validation.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .codex/review-prompt.md ================================================ # PR Review Instructions You are a senior code reviewer for the OriginTrail DKG Engine (ot-node). Your job is to review a pull request diff and produce structured, actionable feedback as inline comments on specific changed lines. You review like a staff engineer who cares deeply about code quality, readability, and simplicity. ## Context Files Read these files before reviewing: 1. **`pr-diff.patch`** — The PR diff (generated at runtime). This is the primary input. You may read other files in the repository **only** to understand how code changed in the diff is called or referenced. Do not review, comment on, or mention code in files that are not part of the diff. All review comments and the summary must be strictly scoped to changes introduced by this PR's diff — nothing else. ## Project Architecture - **Node.js** application (ESM modules, `.js` and `.mjs` files) - **Awilix** dependency injection container for service management - **libp2p** for peer-to-peer networking and message passing - **Ethers.js / Web3.js** for multi-chain blockchain interactions (NeuroWeb, Gnosis, Base) - **Sequelize** ORM for local SQLite database - **Blazegraph** triple store for RDF/SPARQL knowledge graph operations - **Pino** for structured logging - **Command pattern** for async operations (publish, get, query) - **BDD tests** using Cucumber.js with Gherkin feature files ### Key Directories - `src/commands/` — Command implementations (publish, get, query protocols) - `src/modules/` — Core modules (blockchain, network, repository, triple-store) - `src/service/` — Service layer (pending-storage, operation, validation) - `src/constants/` — System-wide constants and error definitions - `test/bdd/` — Cucumber BDD tests (features, steps, utilities) ## Review Philosophy Most PR issues in this codebase are maintainability problems — bloat, poor naming, scattered validation, hardcoded values, pattern drift. These matter a lot. However, review priority is always **severity-first**: 1. **Blockers first** — correctness, security, auth, data integrity, blockchain safety. 2. **Then maintainability** — readability, simplicity, pattern conformance. When both exist, report blockers first. ### Review Method Do three passes: 1. **Context + risk-map pass (mandatory)** — Start from diff hunks, then read surrounding or full touched files when needed to evaluate maintainability, coupling, naming, and extraction opportunities. Use this context to assess changed behavior, not to run unrelated file-wide audits. 2. **Blockers pass** — Scan for correctness bugs, security issues, blockchain transaction safety, gas handling issues, data integrity risks, and missing tests for changed behavior. These are `🔴 Bug` comments. 3. **Maintainability pass** — Scan for code bloat, readability issues, naming problems, pattern violations, hardcoded values, and architecture drift in touched areas. These are `🟡 Issue`, `🔵 Nit`, or `💡 Suggestion` comments. ### Comment Gate Before posting any comment, verify all four conditions: 1. **Introduced by this diff** — The issue is introduced or materially worsened by the changes in this PR, not pre-existing. 2. **Materially impactful** — The issue affects correctness, security, readability, or maintainability in a meaningful way. Not a theoretical concern. 3. **Concrete fix direction** — You can suggest a specific fix or clear direction. If you can only say "this seems off" without a concrete suggestion, do not comment. 4. **Scope fit** — If the issue is mainly in pre-existing code, the PR must touch the same function/module and fixing it must directly simplify, de-risk, or de-duplicate the new/changed code. If any check fails, skip the comment. Every comment must be traceable to changed behavior in this PR and anchored to a right-side line present in `pr-diff.patch`. Prefer added/modified lines; use nearby unchanged hunk lines only when necessary to explain a directly related issue. **Uncertainty guard:** If you are not certain an issue is real and cannot verify it from the diff and allowed context, do not label it `🔴 Bug`. Downgrade to `🟡 Issue` or `💡 Suggestion`, or skip it entirely. **Deduplication:** One comment per root cause. If the same pattern repeats across multiple lines, comment on the first occurrence and note "same pattern at lines X, Y, Z." Aim for a maximum of ~10 comments, highest impact first. ## What to Review ### Pass 1: Blockers #### Correctness - Logic errors, off-by-one, null/undefined handling, incorrect assumptions, race conditions. - Boundary conditions — empty arrays, null inputs, zero values, maximum values. - Error handling — swallowed errors, missing error propagation, unhelpful error messages. Do not flag missing error handling for internal code that cannot reasonably fail. - Async/await correctness — unhandled promise rejections, missing awaits, race conditions in concurrent operations. - Nonce management — verify blockchain nonce allocation and retry logic does not create orphan transactions or nonce gaps. #### Security - Injection risks (SQL, command, XSS) when handling user input. - Hardcoded secrets — API keys, passwords, private keys, tokens in code. Private keys must never appear in source. - Missing input validation at system boundaries (user input, external APIs, RPC responses). Not for internal function calls. - Auth bypass, privilege escalation, or missing authorization checks. - RPC endpoint exposure — verify no private/paid RPC URLs or API keys are hardcoded in committed code. #### Blockchain Safety - Gas handling — verify gas price calculations, multipliers, and buffers are reasonable and consistent across testnet/mainnet. - Transaction retry logic — ensure retries don't waste gas, create duplicate transactions, or cause nonce conflicts. - Wallet/key management — no hardcoded private keys, proper key isolation between environments. - Multi-chain consistency — changes affecting one chain should be verified for impact on other supported chains (NeuroWeb, Gnosis, Base). - BigNumber handling — verify arithmetic operations use BigNumber-safe methods, no precision loss from floating point. #### Tests for Changed Behavior - New behavior must have corresponding tests covering core functionality and error handling. - Bug fixes must include a regression test that would have caught the original bug. - Changed behavior must have updated tests reflecting the new expectations. - If tests are present but brittle (testing implementation details rather than behavior), flag it. Missing tests for changed behavior are blockers (`🔴 Bug`) only when the change affects user-facing behavior, API contracts, or data integrity. Missing tests for internal refactors or trivial changes are `🟡 Issue`. ### Pass 2: Maintainability #### Code Bloat and Unnecessary Complexity - **Excessive code** — More lines than necessary. Could this be done in fewer lines without sacrificing clarity? - **Over-engineering** — Abstractions, helpers, or utilities for one-time operations. Premature generalization. - **Dead code** — Unused variables, unreachable branches, commented-out code, leftover debug logging. - **Duplicate code** — Same logic repeated instead of extracted. Do not suggest extraction for only 2-3 similar lines unless the repeated logic encodes a correctness invariant across multiple paths. #### Readability and Naming - **Confusing variable/function names** — Names that don't describe what the thing is or does. Generic names like `data`, `result`, `item`, `temp`, `val` when a specific name would be clearer. - **Misleading names** — Names that suggest different behavior than what the code does. - **Inconsistent naming** — Not following conventions in the rest of the codebase. - **Long functions** — Functions doing too many things. If you need a comment to explain a section, it should probably be its own function. - **Deep nesting** — More than 2-3 levels. Suggest early returns, guard clauses, or extraction. - **Unclear control flow** — Complex conditionals that could be simplified or decomposed. #### Hardcoded Values and Magic Constants Flag only when the value is: - **Reused 3+ times** in touched files or the diff — should be a named constant. - **Domain-significant** — timeout values, retry counts, gas multipliers, RPC URLs, network message timeouts. Even if used once, these belong in constants or configuration. Do not flag one-off numeric literals that are self-explanatory in context (e.g., `array.slice(0, 2)`, `Math.round(x * 100) / 100`). #### Performance (Only Obvious Issues) - N+1 queries — database queries inside loops. - Blocking operations in async contexts — synchronous I/O in async code. - Unnecessary work in hot paths — redundant allocations, repeated computations. - Memory leaks — Maps/Sets/caches that grow unboundedly without cleanup. ## What NOT to Review - Formatting or style — ESLint and Prettier handle this. - Things that are clearly intentional design choices backed by existing patterns. - Pre-existing issues in unchanged code outside the diff. - Pre-existing issues in touched files when the PR does not introduce/worsen them. - Adding documentation unless a public API is clearly undocumented. - Repository-wide or file-wide audits not required by the changed behavior. - Test configuration files (cucumber.js, .eslintrc) unless they introduce issues. ## Comment Format Use severity prefixes: - `🔴 Bug:` — Correctness error, security issue, blockchain safety issue, data integrity risk. Will cause incorrect behavior. - `🟡 Issue:` — Code quality problem that should be fixed. Bloated code, bad naming, pattern violation, missing tests. - `🔵 Nit:` — Minor improvement, optional. - `💡 Suggestion:` — Alternative approach worth considering. Be specific, be concise, explain why. One clear sentence with a concrete fix is better than a paragraph of theory. ## Output Format Return raw JSON only. No markdown fences, no prose before or after the JSON object. Your output MUST be valid JSON matching the provided output schema. Example: ```json { "summary": "This PR improves blockchain error handling but introduces a potential gas waste issue in the retry loop and has leftover debug logging.", "comments": [ { "path": "src/modules/blockchain/implementation/web3-service.js", "line": 142, "body": "🔴 Bug: Gas price is bumped on every retry including network errors, which wastes gas. Only bump for nonce conflicts and execution errors. Add a `shouldBumpGas` guard." }, { "path": "src/commands/protocols/publish/sender/publish-replication-command.js", "line": 58, "body": "🟡 Issue: `console.log` debug statement left in production code. Use `this.logger.debug()` instead or remove it." } ] } ``` The `line` field must refer to the line number in the new version of the file (right side of the diff), and it must be a line that actually appears in the diff hunks. Do not comment on lines outside the diff. ## Summary Write a brief (2–4 sentence) overall assessment in the `summary` field covering **only** what this PR's diff changes. Do not mention code, packages, or behavior outside the diff. Lead with blockers if any exist. Mention whether the PR is clean/minimal or has code quality issues. Include one sentence on maintainability direction in touched areas (improved / neutral / worsened, and why). If the PR looks good, say so. ================================================ FILE: .codex/review-schema.json ================================================ { "type": "object", "properties": { "summary": { "type": "string", "description": "Brief overall assessment of the PR (2-4 sentences)" }, "comments": { "type": "array", "description": "Inline review comments on specific changed lines", "items": { "type": "object", "properties": { "path": { "type": "string", "description": "File path relative to repository root" }, "line": { "type": "integer", "minimum": 1, "description": "Line number in the new version of the file (must be within the diff)" }, "body": { "type": "string", "description": "Review comment with severity prefix" } }, "required": ["path", "line", "body"], "additionalProperties": false } } }, "required": ["summary", "comments"], "additionalProperties": false } ================================================ FILE: .eslintrc.cjs ================================================ module.exports = { env: { es6: true, node: true, }, extends: ['airbnb/base', 'prettier'], parserOptions: { sourceType: 'module', ecmaVersion: 'latest', }, rules: { 'linebreak-style': ['error', 'unix'], 'class-methods-use-this': 0, 'consistent-return': 0, 'no-restricted-syntax': 0, 'guard-for-in': 0, 'no-console': 'warn', 'no-continue': 0, 'no-underscore-dangle': 0, 'import/extensions': 0, }, overrides: [ { files: ['*.test.js', '*.spec.js'], rules: { 'no-unused-expressions': 'off', }, }, { files: ['*-mock.js', '*.test.js'], rules: { 'no-empty-function': 'off', 'no-unused-vars': 'off', }, }, ], }; ================================================ FILE: .github/CODEOWNERS ================================================ * @branarakic @u-hubar @Mihajlo-Pavlovic ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report_v8.md ================================================ --- name: Bug report for V8 ot-node about: Create an issue report title: '' labels: '' assignees: '' --- ## Issue description ## Expected behavior ## Actual behavior ## Steps to reproduce the problem 1. 2. 3. ## Specifications - Node version: - Platform: - Node wallet: - Node libp2p identity: ## Contact details - Email: ## Error logs ## Disclaimer Please be aware that the issue reported on a public repository allows everyone to see your node logs, node details, and contact details. If you have any sensitive information, feel free to share it by sending an email to [tech@origin-trail.com](tech@origin-trail.com). ================================================ FILE: .github/actions/setup/action.yml ================================================ name: setup runs: using: composite steps: - name: Setup NodeJS id: nodejs uses: actions/setup-node@v3 with: node-version: 20.x cache: npm - name: Cache node modules id: cache-node-modules uses: actions/cache@v3 with: path: '**/node_modules' key: ${{ runner.os }}-node-modules-${{ hashFiles('**/package-lock.json') }} restore-keys: | ${{ runner.os }}-node-modules-${{ hashFiles('**/package-lock.json') }} ${{ runner.os }}-node-modules- - name: Cache Blazegraph id: cache-blazegraph uses: actions/cache@v3 with: path: blazegraph.jar key: ${{ runner.os }}-blazegraph-${{ hashFiles('blazegraph.jar') }} restore-keys: ${{ runner.os }}-blazegraph- - if: steps.cache-node-modules.outputs.cache-hit != 'true' name: Install dependencies & compile contracts shell: bash run: | npm install npm install --save-dev rollup@4.40.0 npm explore dkg-evm-module -- npm run compile - if: steps.cache-blazegraph.outputs.cache-hit != 'true' name: Download Blazegraph shell: bash run: wget https://github.com/blazegraph/database/releases/latest/download/blazegraph.jar ================================================ FILE: .github/pull_request_template.md ================================================ # Description Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. Fixes # (issue) ## Type of change Please delete options that are not relevant. - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update # How Has This Been Tested? Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration - [ ] Test A - [ ] Test B # Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules ================================================ FILE: .github/release-drafter-template.yml ================================================ name-template: 'OriginTrail Release $NEXT_PATCH_VERSION' tag-template: "$NEXT_PATCH_VERSION" version-template: "v$MAJOR.$MINOR.$PATCH" categories: - title: '🚀 Features' labels: - 'enhancement' - title: '🐛 Bug Fixes' labels: - 'bug' - title: '🧰 Maintenance' labels: - 'internal process' - title: '⚠️ Breaking changes' labels: - 'breaking change' change-template: '- $TITLE (#$NUMBER)' template: | # Changes $CHANGES ================================================ FILE: .github/workflows/check-package-lock.yml ================================================ name: Check Package Lock File permissions: contents: read concurrency: group: check-package-lock-${{ github.ref }} cancel-in-progress: true on: push: branches: - main pull_request: branches: - "**" jobs: verify-package-lock: name: Verify package-lock.json runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check if package.json dependencies were changed id: check-changes run: | if [ "${{ github.event_name }}" = "pull_request" ]; then BASE_SHA="${{ github.event.pull_request.base.sha }}" else BASE_SHA="${{ github.event.before }}" fi if ! git diff --name-only "$BASE_SHA" HEAD | grep -q '^package\.json$'; then echo "package_json_changed=false" >> "$GITHUB_OUTPUT" echo "package.json was NOT changed, skipping lock file validation" exit 0 fi echo "package_json_changed=true" >> "$GITHUB_OUTPUT" echo "package.json was changed, will validate lock file" - name: Check if package-lock.json exists run: | if [ ! -f "package-lock.json" ]; then echo "ERROR: package-lock.json file is missing from the repository" echo "This file is required to ensure consistent dependency versions across all environments" echo "Please ensure package-lock.json is committed with your changes" exit 1 fi echo "SUCCESS: package-lock.json file is present" - name: Verify package-lock.json is not empty run: | if [ ! -s "package-lock.json" ]; then echo "ERROR: package-lock.json file exists but is empty" echo "Please run 'npm install' to regenerate the lock file" exit 1 fi echo "SUCCESS: package-lock.json file is valid and not empty" - name: Setup Node.js if: steps.check-changes.outputs.package_json_changed == 'true' uses: actions/setup-node@v4 with: node-version: '22' - name: Validate package-lock.json is in sync if: steps.check-changes.outputs.package_json_changed == 'true' run: npm ci --dry-run --ignore-scripts ================================================ FILE: .github/workflows/checks.yml ================================================ name: checks on: pull_request: types: [opened, reopened, synchronize] push: branches: - '**' permissions: contents: read env: REPOSITORY_PASSWORD: password JWT_SECRET: aTx13FzDG+85j9b5s2G7IBEc5SJNJZZLPLe7RF8hu1xKgRKj46YFRx/z7fJi7iF2NnL7SHcxTzq7TySuPKWkdg/AYKEMD2p1I++qPYFHqg8KQeLArGjCYiqtf43i1Fgtya8z9qJXyegogMz/jYori2BJ8v6b4K3GkAw3XxiO7VaaEYktOp8qsRDcN3b+bITMZqztDvZdWp4EnViGjoES7fRFhKm/d/2C8URnQyGm6xgTR3xTfAjy7+milGmoPA0KU0nu+GsZIhOfeVc9Z2nfxOK/1JQykpjeBhNDYTOr31yW/xdvoW0Kq0PZ6JmM+yezLoyQXcYjavZ+X7cXjbREQg== concurrency: group: checks-${{ github.ref }} cancel-in-progress: true jobs: lint: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3 - name: Set up environment uses: ./.github/actions/setup - name: Run linter run: npm run lint bdd-smoke: if: github.event_name == 'push' runs-on: ubuntu-latest services: mysql: image: mysql:5.7 env: MYSQL_DATABASE: operationaldb MYSQL_USER: node MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: password ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 redis: image: redis:7 ports: - 6379:6379 options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout repository uses: actions/checkout@v3 - name: Set up environment uses: ./.github/actions/setup - name: Start Blazegraph run: /usr/bin/java -Djava.awt.headless=true -jar blazegraph.jar & - name: Wait for Blazegraph run: | for i in $(seq 1 30); do if curl -sf http://localhost:9999/blazegraph/status > /dev/null; then echo "Blazegraph is ready" exit 0 fi sleep 2 done echo "Blazegraph did not start in time" exit 1 - name: Run BDD smoke tests run: | npx cucumber-js \ --config cucumber.js \ --tags "@smoke" \ --format progress \ test/bdd/ \ --import test/bdd/steps/ \ --exit - name: Upload log files if: '!cancelled()' uses: actions/upload-artifact@v4 with: name: bdd-smoke-logs path: ./test/bdd/log/ bdd-tests: if: github.event_name == 'pull_request' runs-on: ubuntu-latest services: mysql: image: mysql:5.7 env: MYSQL_DATABASE: operationaldb MYSQL_USER: node MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: password ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 redis: image: redis:7 ports: - 6379:6379 options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout repository uses: actions/checkout@v3 - name: Set up environment uses: ./.github/actions/setup - name: Start Blazegraph run: /usr/bin/java -Djava.awt.headless=true -jar blazegraph.jar & - name: Wait for Blazegraph run: | for i in $(seq 1 30); do if curl -sf http://localhost:9999/blazegraph/status > /dev/null; then echo "Blazegraph is ready" exit 0 fi sleep 2 done echo "Blazegraph did not start in time" exit 1 - name: Run full BDD tests run: | npx cucumber-js \ --config cucumber.js \ --tags "not @ignore" \ --format progress \ test/bdd/ \ --import test/bdd/steps/ \ --exit - name: Upload log files if: '!cancelled()' uses: actions/upload-artifact@v4 with: name: bdd-tests-logs path: ./test/bdd/log/ ================================================ FILE: .github/workflows/codex-review.yml ================================================ name: Codex PR Review on: pull_request: types: [opened, synchronize, reopened] concurrency: group: codex-review-${{ github.event.pull_request.number }} cancel-in-progress: true permissions: contents: read pull-requests: write jobs: review: name: Codex Review runs-on: ubuntu-latest timeout-minutes: 15 # Skip fork PRs — they cannot access repository secrets if: github.event.pull_request.head.repo.full_name == github.repository steps: - name: Checkout PR merge commit uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: refs/pull/${{ github.event.pull_request.number }}/merge fetch-depth: 0 - name: Generate PR diff run: git diff ${{ github.event.pull_request.base.sha }}...HEAD > pr-diff.patch - name: Allow unprivileged user namespaces for bubblewrap sandbox run: | if sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns 2>/dev/null; then sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 fi - name: Run Codex review id: codex uses: openai/codex-action@f5c0ca71642badb34c1e66321d8d85685a0fa3dc # v1 with: openai-api-key: ${{ secrets.OPENAI_API_KEY }} prompt-file: .codex/review-prompt.md output-schema-file: .codex/review-schema.json effort: high sandbox: read-only - name: Post PR review with inline comments uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 env: REVIEW_JSON: ${{ steps.codex.outputs.final-message }} with: script: | let review; try { review = JSON.parse(process.env.REVIEW_JSON); } catch (e) { console.error('Failed to parse Codex output:', e.message); console.error('Raw output:', process.env.REVIEW_JSON?.slice(0, 500)); await github.rest.pulls.createReview({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, body: '⚠️ Codex review failed to produce valid JSON output. Check the [workflow logs](' + `${process.env.GITHUB_SERVER_URL}/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}) for details.`, event: 'COMMENT', comments: [], }); return; } // Fetch all changed files (paginated for large PRs) const files = await github.paginate( github.rest.pulls.listFiles, { owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, per_page: 100, } ); // Build set of valid (path:line) pairs from right-side diff hunk lines // (added + context). This keeps comments bound to changed areas. const validLines = new Set(); for (const file of files) { // Skip binary/large/truncated files with no patch if (!file.patch) continue; const lines = file.patch.split('\n'); let currentLine = 0; for (const line of lines) { const hunkMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)/); if (hunkMatch) { currentLine = parseInt(hunkMatch[1], 10); continue; } // Added lines are valid comment targets if (line.startsWith('+')) { validLines.add(`${file.filename}:${currentLine}`); currentLine++; continue; } // Deleted lines don't exist in the new file if (line.startsWith('-')) continue; // Ignore hunk metadata lines if (line.startsWith('\\')) continue; // Context lines on the right side are also valid targets validLines.add(`${file.filename}:${currentLine}`); currentLine++; } } // Partition comments into valid (on right-side diff lines) and dropped const comments = Array.isArray(review.comments) ? review.comments : []; const validComments = []; const droppedComments = []; for (const comment of comments) { const key = `${comment.path}:${comment.line}`; if (validLines.has(key)) { validComments.push({ path: comment.path, line: comment.line, body: comment.body, side: 'RIGHT', }); } else { droppedComments.push(comment); } } // Build review body from summary only. // Intentionally do NOT publish out-of-diff comments. let body = review.summary || 'Codex review complete.'; // Post the review await github.rest.pulls.createReview({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, body, event: 'COMMENT', comments: validComments, }); console.log(`Review posted: ${validComments.length} inline comments, ${droppedComments.length} dropped out-of-diff comments`); ================================================ FILE: .github/workflows/release-drafter-config.yml ================================================ name: release-drafter on: push: branches: - develop jobs: update_release_draft: runs-on: ubuntu-latest steps: # Drafts your next Release notes as Pull Requests are merged into "master" - uses: release-drafter/release-drafter@v6 with: # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml config-name: release-drafter-template.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/update-cache.yml ================================================ name: update-cache on: push: branches: - v8/develop concurrency: group: update-cache-${{ github.ref }} cancel-in-progress: true jobs: build-cache: name: Build Cache runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3 - name: Set up environment uses: ./.github/actions/setup ================================================ FILE: .gitignore ================================================ # Logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* lerna-debug.log* .idea .origintrail_noderc .*_origintrail_noderc.json # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage *.lcov # nyc test coverage .nyc_output # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # TypeScript v1 declaration files typings/ # TypeScript cache *.tsbuildinfo # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ .rts2_cache_es/ .rts2_cache_umd/ # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variables file .env .env.test # parcel-bundler cache (https://parceljs.org/) .cache # Next.js build output .next # Nuxt.js build / generate output .nuxt dist # Gatsby files .cache/ # Comment in the public line in if your project uses Gatsby and *not* Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output .vuepress/dist # Serverless directories .serverless/ # FuseBox cache .fusebox/ # DynamoDB Local files .dynamodb/ # TernJS port file .tern-port .DS_Store # Test data folders test-data* # Data folders data* # VS code files .vscode/launch.json # KAs Distribution Simulation Script Plots tools/knowledge-assets-distribution-simulation/plots/**/*jpg node_modules .env # Hardhat files /cache /artifacts # TypeChain files /typechain /typechain-types # solidity-coverage files /coverage /coverage.json # Hardhat Ignition default folder for deployments against a local node ignition/deployments/chain-31337 # Redis dump.rdb # Blazegraph journal files *.jnl ================================================ FILE: .husky/.gitignore ================================================ _ ================================================ FILE: .husky/pre-commit ================================================ #!/bin/sh . "$(dirname "$0")/_/husky.sh" npm run lint-staged ================================================ FILE: .lintstagedrc.json ================================================ { "*.{js, json}": ["prettier --write", "eslint"] } ================================================ FILE: .prettierrc ================================================ { "semi": true, "printWidth": 100, "singleQuote": true, "trailingComma": "all", "arrowParens": "always", "tabWidth": 4, "bracketSpacing": true } ================================================ FILE: Alpine.Dockerfile ================================================ FROM node:14-alpine3.15 LABEL maintainer="OriginTrail" ENV NODE_ENV=testnet #Install Papertrail RUN wget https://github.com/papertrail/remote_syslog2/releases/download/v0.20/remote_syslog_linux_amd64.tar.gz RUN tar xzf ./remote_syslog_linux_amd64.tar.gz && cd remote_syslog && cp ./remote_syslog /usr/local/bin COPY config/papertrail.yml /etc/log_files.yml #Install git & forever RUN npm install forever -g RUN apk add git WORKDIR /ot-node COPY . . #Install nppm RUN npm install ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience * Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at tech@origin-trail.com. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning **Community Impact**: A violation through a single incident or series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing ## Rules There are a few basic ground-rules for contributors (including the maintainer(s) of the project): - **No `--force` pushes** or modifying the Git history in any way. If you need to rebase, ensure you do it in your own repo. - **All modifications** must be made in a **pull-request** to solicit feedback from other contributors. ### Reviewing pull requests When reviewing a pull request, the end-goal is to suggest useful changes to the author. Reviews should finish with approval unless there are issues that would result in: - Buggy behavior. - Undue maintenance burden. - Breaking with house coding style. - Pessimization (i.e. reduction of speed as measured in the projects benchmarks). - Feature reduction (i.e. it removes some aspect of functionality that a significant minority of users rely on). - Uselessness (i.e. it does not strictly add a feature or fix a known issue). ### Reviews may not be used as an effective veto for a PR because - There exists a somewhat cleaner/better/faster way of accomplishing the same feature/fix. - It does not fit well with some other contributors' longer-term vision for the project. ## Releases Declaring formal releases remains the prerogative of the project maintainer(s). ## Changes to this arrangement This is an experiment and feedback is welcome! This document may also be subject to pull-requests or changes by contributors where you believe you have something valuable to add or change. ## Heritage These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: ================================================ FILE: Debian.Dockerfile ================================================ #base image FROM node:14.18.3-bullseye MAINTAINER OriginTrail LABEL maintainer="OriginTrail" ENV NODE_ENV=testnet #Mysql-server installation ARG DEBIAN_FRONTEND=noninteractive ARG PASSWORD=password RUN apt-get update RUN apt-get install -y lsb-release RUN apt-get install -y wget gnupg curl RUN curl -LO https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb RUN dpkg -i ./mysql-apt-config_0.8.20-1_all.deb RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 RUN { \ echo mysql-server mysql-server/root_password password $PASSWORD ''; \ echo mysql-server mysql-server/root_password_again password $PASSWORD ''; \ } | debconf-set-selections \ && apt-get update && apt-get install -y default-mysql-server default-mysql-server-core RUN apt-get -qq -y install git RUN apt-get -qq -y install make python #Install Papertrail RUN wget https://github.com/papertrail/remote_syslog2/releases/download/v0.20/remote_syslog_linux_amd64.tar.gz RUN tar xzf ./remote_syslog_linux_amd64.tar.gz && cd remote_syslog && cp ./remote_syslog /usr/local/bin COPY config/papertrail.yml /etc/log_files.yml #Install forever RUN npm install forever -g WORKDIR /ot-node COPY . . #Install nppm RUN npm install #Mysql intialization RUN service mariadb start && mysql -u root -e "CREATE DATABASE operationaldb /*\!40100 DEFAULT CHARACTER SET utf8 */; SET PASSWORD FOR root@localhost = PASSWORD(''); FLUSH PRIVILEGES;" ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2018 OriginTrail Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ ---
OriginTrail Node Banner

OT-Node


OriginTrail Docs · Report Bug · Request Feature


Table of Contents
  1. 📚 About The Project
  2. 🚀 Getting Started
  3. 📄 License
  4. 🤝 Contributing
  5. 📰 Social Media
---
## 📚 About The Project
### **What is the Decentralized Knowledge Graph?**
Knowledge Asset
OriginTrail Decentralized Knowledge Graph (DKG), hosted on the OriginTrail Decentralized Network (ODN) as trusted knowledge infrastructure, is shared global Knowledge Graph of Knowledge Assets. Running on the basis of the permissionless multi-chain OriginTrail protocol, it combines blockchains and knowledge graph technology to enable trusted AI applications based on key W3C standards.
### **The OriginTrail DKG Architecture**
The OriginTrail tech stack is a three layer structure, consisting of the multi-chain consensus layer (OriginTrail layer 1, running on multiple blockchains), the Decentralized Knowledge Graph layer (OriginTrail Layer 2, hosted on the ODN) and Trusted Knowledge applications in the application layer.
DKG Architecture
Further, the architecture differentiates between **the public, replicated knowledge graph** shared by all network nodes according to the protocol, and **private Knowledge graphs** hosted separately by each of the OriginTrail nodes. **Anyone can run an OriginTrail node and become part of the ODN, contributing to the network capacity and hosting the OriginTrail DKG. The OriginTrail node is the ultimate data service for data and knowledge intensive Web3 applications and is used as the key backbone for trusted AI applications (see https://chatdkg.ai)**
### **What is a Knowledge Asset?**
Knowledge Asset
**Knowledge Asset is the new, AI‑ready resource for the Internet** Knowledge Assets are verifiable containers of structured knowledge that live on the OriginTrail DKG and provide: - **Discoverability - UAL is the new URL**. Uniform Asset Locators (UALs, based on the W3C Decentralized Identifiers) are a new Web3 knowledge identifier (extensions of the Uniform Resource Locators - URLs) which identify a specific piece of knowledge and make it easy to find and connect with other Knowledge Assets. - **Ownership - NFTs enable ownership**. Each Knowledge Asset contains an NFT token that enables ownership, knowledge asset administration and market mechanisms. - **Verifiability - On-chain information origin and verifiable trail**. The blockchain tech increases trust, security, transparency, and the traceability of information. By their nature, Knowledge Assets are semantic resources (following the W3C Semantic Web set of standards), and through their symbolic representations inherently AI ready. See more at https://chatdkg.ai
**Discover Knowledge Assets with the DKG Explorer:**
Knowledge Assets Graph 1
Supply Chains 		Knowledge Assets Graph 2
Construction 		Knowledge Assets Graph 3
Life sciences and healthcare 		Knowledge Assets Graph 3
Metaverse

(back to top)


## 🚀 Getting Started --- ### Prerequisites
- **Node.js** 20.18 - **npm** 10.8.2 ---
### Local Network Setup
First, clone the repo: ```bash git clone https://github.com/OriginTrail/ot-node.git cd ot-node ``` Switch the branch to `v8/develop`: ```bash git checkout v8/develop ``` Install dependencies using `npm`: ```bash npm install ``` Create the .env file inside the "ot-node" directory: ```bash nano .env ``` and paste the following content inside (save and close): ```bash NODE_ENV=development RPC_ENDPOINT_BC1=http://localhost:8545 RPC_ENDPOINT_BC2=http://localhost:9545 REPOSITORY_PASSWORD= ``` Run the Triple Store. To use default Triple Store (`blazegraph`), download the exec file and run it with the following command in the separate process: ```bash java -server -Xmx6g -jar blazegraph.jar ``` It's highly recommended to use a larger heap size (6GB or 8GB), as the DKG node will require a lot of memory. Ensure your MySQL instance is running on port 3306 with the password matching REPOSITORY_PASSWORD in your .env file. Additionally, set up Redis on its default port 6379. Both are required for the nodes to start properly. Then, depending on the OS, use one of the scripts in order to run the local network with provided number of nodes (minimal amount of nodes should be 6): **MacOS** ```bash bash ./tools/local-network-setup/setup-macos-environment.sh --nodes=6 ``` **Linux** ```bash ./tools/local-network-setup/setup-linux-environment.sh --nodes=6 ``` ---
### DKG Node Setup
In order to run a DKG node on the **V8 Testnet**, please read the official documentation: https://docs.origintrail.io/dkg-v8-upcoming-version/run-a-v8-core-node-on-testnet ---
### Build on DKG
The OriginTrail SDKs are client libraries for your applications, used to interact and connect with the OriginTrail Decentralized Knowledge Graph. From an architectural standpoint, the SDK libraries are application interfaces into the DKG, enabling you to create and manage Knowledge Assets through your apps, as well as perform network queries (such as search, or SPARQL queries), as illustrated below.
SDK
The OriginTrail SDK libraries are being built in various languages by the team and the community, as listed below: - dkg.js - V8 JavaScript SDK implementation - [Github repository](https://github.com/OriginTrail/dkg.js/tree/v8/develop) - [Documentation](https://docs.origintrail.io/dkg-v8-upcoming-version/v8-dkg-sdk/dkg-v8-js-client) - dkg.py - V8 Python SDK implementation - [Github repository](https://github.com/OriginTrail/dkg.py/tree/v8/develop) - [Documentation](https://docs.origintrail.io/dkg-v8-upcoming-version/v8-dkg-sdk/dkg-v8-py-client) ---

(back to top)

## 📄 License Distributed under the Apache-2.0 License. See `LICENSE` file for more information.

(back to top)

## 🤝 Contributing Contributions are what make the open source community such an amazing place to learn, inspire, and create. Any contributions you make are **greatly appreciated**. If you have a suggestion that would make this better, please fork the repo and create a pull request. You can also simply open an issue with the tag "enhancement". Don't forget to give the project a star! Thanks again! 1. Fork the Project 2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`) 3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`) 4. Push to the Branch (`git push origin feature/AmazingFeature`) 5. Open a Pull Request

(back to top)

## 📰 Social Media
Medium Badge Telegram Badge X Badge YouTube Badge LinkedIn Badge Discord Badge Reddit Badge Coinmarketcap Badge
--- ================================================ FILE: Ubuntu.Dockerfile ================================================ #base image FROM ubuntu:20.04 MAINTAINER OriginTrail LABEL maintainer="OriginTrail" ENV NODE_ENV=testnet #Install git, nodejs, mysql, python RUN apt-get -qq update && apt-get -qq -y install curl RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - RUN apt-get -qq update RUN apt-get -qq -y install wget apt-transport-https RUN apt-get -qq -y install git nodejs RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends mysql-server RUN apt-get -qq -y install unzip nano RUN apt-get -qq -y install make python #Install Papertrail RUN wget https://github.com/papertrail/remote_syslog2/releases/download/v0.20/remote_syslog_linux_amd64.tar.gz RUN tar xzf ./remote_syslog_linux_amd64.tar.gz && cd remote_syslog && cp ./remote_syslog /usr/local/bin COPY config/papertrail.yml /etc/log_files.yml #Install forever RUN npm install -g forever WORKDIR /ot-node COPY . . #Install npm RUN npm install #Intialize mysql RUN usermod -d /var/lib/mysql/ mysql RUN echo "disable_log_bin" >> /etc/mysql/mysql.conf.d/mysqld.cnf RUN service mysql start && mysql -u root -e "CREATE DATABASE operationaldb /*\!40100 DEFAULT CHARACTER SET utf8 */; update mysql.user set plugin = 'mysql_native_password' where User='root'/*\!40100 DEFAULT CHARACTER SET utf8 */; flush privileges;" ================================================ FILE: bin/darwin/arm64/.gitkeep ================================================ ================================================ FILE: bin/darwin/x64/.gitkeep ================================================ ================================================ FILE: bin/linux/arm64/.gitkeep ================================================ ================================================ FILE: bin/linux/x64/.gitkeep ================================================ ================================================ FILE: bin/win32/x64/.gitkeep ================================================ ================================================ FILE: blazegraph-migration/README.md ================================================ The migration is manual and split into two scripts: export and import, which you must run yourself. Your node will be offline during export (several hours), but usable during import. Core nodes won’t be affected staking-wise. Import time varies based on data and hardware and can take hours to days. After finishing both export and import processes, the check_quad_num.sh should be ran. It will return info on the outcome of the migration. We recommend migrating before the 8.0.6 release, as it will add more data and increase future migration time. The process removes blazegraph.jnl and rebuilds it from exported DKG and paranet repositories — so if you have custom data, review the script and back up your journal file first. Before running the migration, make sure the blazegraph.jnl you are migrating is in the ot-node directory and that blazegraph is running. Export script (to export dkg and paranet namespaces): ```bash nohup ./current/blazegraph-migration/export.sh /path_to_ot_node/ot-node dkg $(curl -s http://localhost:9999/blazegraph/namespace | grep -oP ']*>\K[^<]+' | grep '^paranet-') | tee export_migration.log & ``` Import script: ```bash ./current/blazegraph-migration/import.sh ``` Check quad number: ```bash ./current/blazegraph-migration/check_quad_num.sh ``` ================================================ FILE: blazegraph-migration/check_quad_num.sh ================================================ #!/bin/bash get_mysql_password() { local base_dir=$(dirname "$0") grep ^REPOSITORY_PASSWORD= "./current/.env" | cut -d '=' -f2 } get_inserted_triples_count() { local repo_pw=$1 mysql -u root -p"$repo_pw" operationaldb -e "SELECT count FROM triples_insert_count WHERE id = 1;" | tail -n 1 } get_blazegraph_count() { local namespace=$1 local BLAZEGRAPH_URL="http://localhost:9999/blazegraph/namespace/$namespace/sparql" QUAD_COUNT=$(curl -s -X POST "$BLAZEGRAPH_URL" \ -H "Accept: text/tab-separated-values" \ --data-urlencode 'query=SELECT (COUNT(*) AS ?total) WHERE { GRAPH ?g { ?s ?p ?o } }' \ | tail -n 1) if ! [[ "$QUAD_COUNT" =~ ^[0-9]+$ ]]; then echo "Error: Failed to get valid count from Blazegraph" >&2 return 1 fi echo "$QUAD_COUNT" return 0 } get_old_count() { local old_count_file=$1 if [ ! -f "$old_count_file" ]; then echo "Error: $old_count_file not found" >&2 return 1 fi cat "$old_count_file" } get_uninserted_count() { local namespace=$1 local chunks_dir="${namespace}/chunks" if [ ! -d "$chunks_dir" ]; then echo "Error: Chunks directory not found at $chunks_dir" >&2 return 1 fi local total_lines=0 for chunk_file in "$chunks_dir"/chunk*; do if [ -f "$chunk_file" ]; then local lines=$(wc -l < "$chunk_file") total_lines=$((total_lines + lines)) fi done echo "$total_lines" return 0 } check_quad_count() { local namespace=$1 local old_count_file="OLD_QUAD_COUNT_${namespace}.txt" local old_count=$(get_old_count "$old_count_file") || return 1 local repo_pw=$(get_mysql_password) local current_count=$(get_blazegraph_count "$namespace") local inserted_triples=$(get_inserted_triples_count "$repo_pw") local uninserted_count=0 if uninserted=$(get_uninserted_count "$namespace"); then uninserted_count=$uninserted echo "[WARN] There are ${uninserted_count#-} uninserted quads in the chunks folder." echo "Feel free to run the import.sh script again." echo "*Note: Some chunks may need to be manually imported" fi local actual_count=$((current_count - inserted_triples + uninserted_count)) local expected_total=$((old_count)) percentage_diff=$(echo "scale=6; 100 * ($actual_count - $expected_total) / $expected_total" | bc) abs_diff=$(echo "$percentage_diff" | sed 's/-//') within_threshold=$(echo "$abs_diff <= 0.05" | bc) if [ "$within_threshold" -eq 1 ]; then if [ $uninserted_count -eq 0 ]; then echo "[SUCCESS] The migration has been completed successfully! There are no uninserted quads." else echo "[SUCCESS] The migration has been completed successfully thus far." fi return 0 else local difference=$((actual_count - expected_total)) if [ $difference -gt 0 ]; then echo "[ERROR] There are $difference more quads than expected" echo "*Note: Errors can happen during importing, if this number is of a small value, it can be ignored" else echo "[ERROR] There are ${difference#-} fewer quads than expected" echo "*Note: Errors can happen during importing, if this number is of a small value, it can be ignored" fi return 1 fi } check_quad_count "dkg" ================================================ FILE: blazegraph-migration/export.sh ================================================ #!/bin/bash set -e # Configs BLAZEGRAPH_JAR="blazegraph.jar" if [ "$#" -lt 2 ]; then echo "Usage: $0 [namespace2 ...]" exit 1 fi BASE_DIR="$1" shift log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" } # Ensure Blazegraph and Otnode are started on script exit (success or error) cleanup() { log "Ensuring Blazegraph service is running..." sudo systemctl start blazegraph.service log "Ensuring Otnode service is running..." sudo systemctl restart otnode.service } trap cleanup EXIT log "Stopping otnode service..." sudo systemctl stop otnode.service sleep 10 for NAMESPACE in "$@"; do FOLDER_NAME=$(echo "$NAMESPACE" | tr '-' '_') EXPORT_DIR="$BASE_DIR/$FOLDER_NAME" EXPORT_FILE="$EXPORT_DIR/$FOLDER_NAME/data.nq.gz" QUAD_COUNT_FILE="OLD_QUAD_COUNT_${NAMESPACE}.txt" PROPERTIES_FILE="$EXPORT_DIR/${NAMESPACE}.properties" BLAZEGRAPH_URL="http://localhost:9999/blazegraph/namespace/$NAMESPACE/sparql" log "Processing namespace: $NAMESPACE" log "Querying quad count from Blazegraph..." QUAD_COUNT=$(curl -s -X POST "$BLAZEGRAPH_URL" \ -H "Accept: text/tab-separated-values" \ --data-urlencode 'query=SELECT (COUNT(*) AS ?total) WHERE { GRAPH ?g { ?s ?p ?o } }' \ | tail -n 1) if ! [[ "$QUAD_COUNT" =~ ^[0-9]+$ ]]; then log "❌ Invalid quad count received: '$QUAD_COUNT'" exit 1 fi echo "$QUAD_COUNT" > "$QUAD_COUNT_FILE" log "Quad count: $QUAD_COUNT (saved to $QUAD_COUNT_FILE)" log "Stopping blazegraph service..." sudo systemctl stop blazegraph.service sleep 10 log "Creating properties file for $NAMESPACE..." mkdir -p "$EXPORT_DIR" cat > "$PROPERTIES_FILE" < "$EXPORT_DIR/output.log" 2>&1 log "Export complete. Output should be in $EXPORT_FILE" if [ ! -f "$EXPORT_FILE" ]; then log "❌ Export file not found: $EXPORT_FILE" exit 1 fi log "Validating line count..." LINE_COUNT=$(zcat "$EXPORT_FILE" | wc -l) if [ "$LINE_COUNT" -eq "$QUAD_COUNT" ]; then log "✅ Line count matches quad count: $LINE_COUNT lines" else log "❌ MISMATCH for $NAMESPACE: exported $LINE_COUNT lines, expected $QUAD_COUNT" exit 1 fi log "Starting blazegraph service..." sudo systemctl start blazegraph.service until nc -z localhost 9999; do sleep 1 done done log "Stopping blazegraph service to remove old journal..." sudo systemctl stop blazegraph.service sleep 10 log "Removing old Blazegraph journal..." rm -f "${BASE_DIR}/blazegraph.jnl" log "Creating new journal on blazegraph start..." sudo systemctl start blazegraph.service log "Resetting triple log in MySQL..." REPO_PW=$(grep ^REPOSITORY_PASSWORD= "$BASE_DIR/current/.env" | cut -d '=' -f2) mysql -u root -p"$REPO_PW" operationaldb -e "UPDATE triples_insert_count SET count = 0 WHERE id = 1;" log "✅ Migration completed for all namespaces." ================================================ FILE: blazegraph-migration/import.sh ================================================ #!/bin/bash LOG_FILE="migration_import_$(date +%Y%m%d_%H%M%S).log" CURL_TIMEOUT=300 MAX_RETRIES=3 CHUNK_DELAY=2 log_message() { local level="$1" local message="$2" local timestamp=$(date "+%Y-%m-%d %H:%M:%S") echo "[$timestamp] [$level] $message" | tee -a "$LOG_FILE" } convert_namespace() { local input="$1" echo "${input//_/-}" } find_target_folders() { local folders=() if [ -d "dkg" ]; then folders+=("dkg") fi for folder in paranet*; do if [ -d "$folder" ]; then folders+=("$folder") fi done echo "${folders[@]}" } split_into_chunks() { local input_dir="$1" local chunk_size="$2" local input_file="$input_dir/$input_dir/data.nq.gz" local output_dir="$input_dir/chunks" mkdir -p "$output_dir" log_message "INFO" "Starting file split for $input_file" gunzip -c "$input_file" | split -l "$chunk_size" --additional-suffix=.nq - "${output_dir}/chunk_" log_message "INFO" "File split completed. Output directory: $output_dir" } process_chunk() { local chunk_file="$1" local chunk_num="$2" local blazegraph_url="$3" local total_chunks="$4" local retry_count=0 local success=false while [ $retry_count -lt $MAX_RETRIES ] && [ "$success" = false ]; do if [ $retry_count -gt 0 ]; then log_message "INFO" "Retry $retry_count for chunk $chunk_num" fi local chunk_lines=$(wc -l < "$chunk_file") local response=$(curl -s --max-time $CURL_TIMEOUT -X POST \ -H "Content-Type: text/x-nquads" \ --data-binary @"$chunk_file" \ "$blazegraph_url" 2>&1) if [[ $response == *" ${job_queue} while [[ "${cmd}" != "${job_pool_end_of_jobs}" && -e "${job_queue}" ]]; do # workers block on the exclusive lock to read the job queue flock --exclusive 7 IFS=$'\v' read cmd args <${job_queue} set -- ${args} unset IFS flock --unlock 7 # the worker should exit if it sees the end-of-job marker or run the # job otherwise and save its exit code to the result log. if [[ "${cmd}" == "${job_pool_end_of_jobs}" ]]; then # write it one more time for the next sibling so that everyone # will know we are exiting. echo "${cmd}" >&7 else _job_pool_echo "### _job_pool_worker-${id}: ${cmd}" # run the job { ${cmd} "$@" ; } # now check the exit code and prepend "ERROR" to the result log entry # which we will use to count errors and then strip out later. local result=$? local status= if [[ "${result}" != "0" ]]; then status=ERROR fi # now write the error to the log, making sure multiple processes # don't trample over each other. exec 8<> ${result_log} flock --exclusive 8 _job_pool_echo "${status}job_pool: exited ${result}: ${cmd} $@" >> ${result_log} flock --unlock 8 exec 8>&- _job_pool_echo "### _job_pool_worker-${id}: exited ${result}: ${cmd} $@" fi done exec 7>&- } # \brief sends message to worker processes to stop function _job_pool_stop_workers() { # send message to workers to exit, and wait for them to stop before # doing cleanup. echo ${job_pool_end_of_jobs} >> ${job_pool_job_queue} wait } # \brief fork off the workers # \param[in] job_queue the fifo used to send jobs to the workers # \param[in] result_log the temporary log file to write exit codes to function _job_pool_start_workers() { local job_queue=$1 local result_log=$2 for ((i=0; i<${job_pool_pool_size}; i++)); do _job_pool_worker ${i} ${job_queue} ${result_log} & done } ################################################################################ # public functions ################################################################################ # \brief initializes the job pool # \param[in] pool_size number of parallel jobs allowed # \param[in] echo_command 1 to turn on echo, 0 to turn off function job_pool_init() { local pool_size=$1 local echo_command=$2 # set the global attibutes job_pool_pool_size=${pool_size:=1} job_pool_echo_command=${echo_command:=0} # create the fifo job queue and create the exit code log rm -rf ${job_pool_job_queue} ${job_pool_result_log} mkfifo ${job_pool_job_queue} touch ${job_pool_result_log} # fork off the workers _job_pool_start_workers ${job_pool_job_queue} ${job_pool_result_log} } # \brief waits for all queued up jobs to complete and shuts down the job pool function job_pool_shutdown() { _job_pool_stop_workers _job_pool_print_result_log _job_pool_cleanup } # \brief run a job in the job pool function job_pool_run() { if [[ "${job_pool_pool_size}" == "-1" ]]; then job_pool_init fi printf "%s\v" "$@" >> ${job_pool_job_queue} echo >> ${job_pool_job_queue} } # \brief waits for all queued up jobs to complete before starting new jobs # This function actually fakes a wait by telling the workers to exit # when done with the jobs and then restarting them. function job_pool_wait() { _job_pool_stop_workers _job_pool_start_workers ${job_pool_job_queue} ${job_pool_result_log} } ######################################### # End of Job Pool ######################################### ================================================ FILE: config/config.json ================================================ { "development": { "modules": { "autoUpdater": { "enabled": false, "implementation": { "ot-auto-updater": { "enabled": false, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v8/develop" } } } }, "httpClient": { "enabled": true, "implementation": { "express-http-client": { "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, "port": 8900, "sslKeyPath": "/root/certs/privkey.pem", "sslCertificatePath": "/root/certs/fullchain.pem", "rateLimiter": { "timeWindowSeconds": 60, "maxRequests": 10 } } } } }, "network": { "enabled": true, "implementation": { "libp2p-service": { "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { "kBucketSize": 20 }, "nat": { "enabled": false, "externalIp": null }, "connectionManager": { "autoDial": true, "autoDialInterval": 10e3, "dialTimeout": 2e3 }, "peerRouting": { "refreshManager": { "enabled": true, "interval": 6e5, "bootDelay": 2e3 } }, "port": 9100, "bootstrap": [] } } } }, "repository": { "enabled": true, "implementation": { "sequelize-repository": { "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", "user": "root", "password": "", "port": "3306", "host": "localhost", "dialect": "mysql", "logging": false, "pool": { "max": 120, "min": 0, "acquire": 60000, "idle": 10000, "evict": 1000 } } } } }, "tripleStore": { "enabled": true, "timeout": { "query": 60000, "get": 10000, "batchGet": 10000, "insert": 300000, "ask": 10000 }, "implementation": { "ot-blazegraph": { "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", "config": {} }, "ot-fuseki": { "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", "config": {} }, "ot-graphdb": { "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", "config": {} }, "ot-neptune": { "enabled": false, "package": "./triple-store/implementation/ot-neptune/ot-neptune.js", "config": {} } } }, "validation": { "enabled": true, "implementation": { "merkle-validation": { "enabled": true, "package": "./validation/implementation/merkle-validation.js", "config": {} } } }, "blockchain": { "enabled": true, "implementation": { "hardhat1:31337": { "enabled": true, "package": "./blockchain/implementation/hardhat/hardhat-service.js", "config": { "hubContractAddress": "0x5FbDB2315678afecb367f032d93F642f64180aa3", "rpcEndpoints": ["http://localhost:8545"], "evmManagementPublicKey": "0x1B420da5f7Be66567526E32bc68ab29F1A63765A", "initialStakeAmount": 50000, "initialAskAmount": 0.2, "operatorFee": 0 } }, "hardhat2:31337": { "enabled": true, "package": "./blockchain/implementation/hardhat/hardhat-service.js", "config": { "hubContractAddress": "0x5FbDB2315678afecb367f032d93F642f64180aa3", "rpcEndpoints": ["http://localhost:9545"], "evmManagementPublicKey": "0x1B420da5f7Be66567526E32bc68ab29F1A63765A", "initialStakeAmount": 50000, "initialAskAmount": 0.2, "operatorFee": 0 } } } }, "telemetry": { "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { "localEndpoint": "http::addr=localhost:10000", "signalingServiceEndpoint": "", "sendToSignalingService": false } } } }, "blockchainEvents": { "enabled": true, "implementation": { "ot-ethers": { "enabled": true, "package": "./blockchain-events/implementation/ot-ethers/ot-ethers.js", "config": { "blockchains": ["hardhat1:31337", "hardhat2:31337"], "rpcEndpoints": { "hardhat1:31337": ["http://localhost:8545"], "hardhat2:31337": ["http://localhost:9545"] }, "hubContractAddress": { "hardhat1:31337": "0x5FbDB2315678afecb367f032d93F642f64180aa3", "hardhat2:31337": "0x5FbDB2315678afecb367f032d93F642f64180aa3" } } } } } }, "maximumAssertionSizeInKb": 500000, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logging": { "defaultLevel": "trace", "enableExperimentalScopes": true }, "assetSync": { "syncDKG": { "enabled": true, "syncBatchSize": 5, "doneThreshold": 95 }, "syncParanets": [] }, "auth": { "ipBasedAuthEnabled": true, "tokenBasedAuthEnabled": false, "loggingEnabled": true, "ipWhitelist": ["::1", "127.0.0.1"], "publicOperations": [], "bothIpAndTokenAuthRequired": false } }, "test": { "modules": { "autoUpdater": { "enabled": false, "implementation": { "ot-auto-updater": { "enabled": false, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v8/develop" } } } }, "httpClient": { "enabled": true, "implementation": { "express-http-client": { "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, "sslKeyPath": "/root/certs/privkey.pem", "sslCertificatePath": "/root/certs/fullchain.pem", "rateLimiter": { "timeWindowSeconds": 60, "maxRequests": 10 } } } } }, "network": { "enabled": true, "implementation": { "libp2p-service": { "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { "kBucketSize": 20 }, "nat": { "enabled": false, "externalIp": null }, "connectionManager": { "autoDial": true, "autoDialInterval": 10e3, "dialTimeout": 2e3 }, "peerRouting": { "refreshManager": { "enabled": true, "interval": 6e5, "bootDelay": 2e3 } }, "port": 9000, "bootstrap": [ "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" ] } } } }, "validation": { "enabled": true, "implementation": { "enabled": true, "merkle-validation": { "package": "./validation/implementation/merkle-validation.js", "config": {} } } }, "repository": { "enabled": true, "implementation": { "sequelize-repository": { "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", "user": "root", "password": "", "port": "3306", "host": "localhost", "dialect": "mysql", "logging": false, "pool": { "max": 120, "min": 0, "acquire": 60000, "idle": 10000, "evict": 1000 } } } } }, "blockchain": { "enabled": true, "implementation": { "hardhat1:31337": { "enabled": true, "package": "./blockchain/implementation/hardhat/hardhat-service.js", "config": { "hubContractAddress": "0x5FbDB2315678afecb367f032d93F642f64180aa3", "rpcEndpoints": ["http://localhost:8545"], "initialStakeAmount": 50000, "initialAskAmount": 0.2, "operatorFee": 0 } }, "hardhat2:31337": { "enabled": true, "package": "./blockchain/implementation/hardhat/hardhat-service.js", "config": { "hubContractAddress": "0x5FbDB2315678afecb367f032d93F642f64180aa3", "rpcEndpoints": ["http://localhost:9545"], "initialStakeAmount": 50000, "initialAskAmount": 0.2, "operatorFee": 0 } } } }, "tripleStore": { "enabled": true, "timeout": { "query": 60000, "get": 10000, "batchGet": 10000, "insert": 300000, "ask": 10000 }, "implementation": { "ot-blazegraph": { "enabled": true, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", "config": {} }, "ot-neptune": { "enabled": false, "package": "./triple-store/implementation/ot-neptune/ot-neptune.js", "config": {} } } }, "telemetry": { "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { "localEndpoint": "http::addr=localhost:10000", "signalingServiceEndpoint": "", "sendToSignalingService": false } } } }, "blockchainEvents": { "enabled": true, "implementation": { "ot-ethers": { "enabled": true, "package": "./blockchain-events/implementation/ot-ethers/ot-ethers.js", "config": { "blockchains": ["hardhat1:31337", "hardhat2:31337"], "rpcEndpoints": { "hardhat1:31337": ["http://localhost:8545"], "hardhat2:31337": ["http://localhost:9545"] }, "hubContractAddress": { "hardhat1:31337": "0x5FbDB2315678afecb367f032d93F642f64180aa3", "hardhat2:31337": "0x5FbDB2315678afecb367f032d93F642f64180aa3" } } } } } }, "maximumAssertionSizeInKb": 500000, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logging": { "defaultLevel": "info", "enableExperimentalScopes": true }, "assetSync": { "syncDKG": { "enabled": false, "syncBatchSize": 20, "doneThreshold": 95 }, "syncParanets": [] }, "auth": { "ipBasedAuthEnabled": true, "tokenBasedAuthEnabled": false, "loggingEnabled": true, "ipWhitelist": ["::1", "127.0.0.1"], "publicOperations": [], "bothIpAndTokenAuthRequired": false } }, "testnet": { "modules": { "autoUpdater": { "enabled": true, "implementation": { "ot-auto-updater": { "enabled": true, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v6/release/testnet" } } } }, "network": { "enabled": true, "implementation": { "libp2p-service": { "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { "kBucketSize": 20 }, "nat": { "enabled": true, "externalIp": null }, "connectionManager": { "autoDial": true, "autoDialInterval": 10e3, "dialTimeout": 2e3 }, "peerRouting": { "refreshManager": { "enabled": true, "interval": 6e5, "bootDelay": 2e3 } }, "port": 9000, "bootstrap": [ "/ip4/164.92.138.30/tcp/9000/p2p/QmbiZQm18JefDizrQwbRhPgkaLykTLyrUEpeMWuKJHXuUM", "/ip4/139.59.145.152/tcp/9000/p2p/Qme2oF6afixBjLYjF5CYeC73d5dygsTq8P7BPQp31NVkye" ] } } } }, "httpClient": { "enabled": true, "implementation": { "express-http-client": { "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, "port": 8900, "sslKeyPath": "/root/certs/privkey.pem", "sslCertificatePath": "/root/certs/fullchain.pem", "rateLimiter": { "timeWindowSeconds": 60, "maxRequests": 10 } } } } }, "repository": { "enabled": true, "implementation": { "sequelize-repository": { "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", "user": "root", "password": "password", "port": "3306", "host": "localhost", "dialect": "mysql", "logging": false, "pool": { "max": 120, "min": 0, "acquire": 60000, "idle": 10000, "evict": 1000 } } } } }, "blockchain": { "enabled": true, "implementation": { "otp:20430": { "enabled": false, "package": "./blockchain/implementation/ot-parachain/ot-parachain-service.js", "config": { "hubContractAddress": "0xe233b5b78853a62b1e11ebe88bf083e25b0a57a6", "rpcEndpoints": [ "https://lofar-testnet.origin-trail.network", "https://lofar-testnet.origintrail.network" ], "operatorFee": 0 } }, "gnosis:10200": { "enabled": false, "package": "./blockchain/implementation/gnosis/gnosis-service.js", "config": { "hubContractAddress": "0x2c08AC4B630c009F709521e56Ac385A6af70650f", "gasPriceOracleLink": "https://blockscout.chiadochain.net/api/v1/gas-price-oracle", "rpcEndpoints": ["https://rpc.chiadochain.net"], "operatorFee": 0 } }, "base:84532": { "enabled": false, "package": "./blockchain/implementation/base/base-service.js", "config": { "hubContractAddress": "0xf21CE8f8b01548D97DCFb36869f1ccB0814a4e05", "rpcEndpoints": ["https://sepolia.base.org"], "operatorFee": 0 } } } }, "validation": { "enabled": true, "implementation": { "merkle-validation": { "enabled": true, "package": "./validation/implementation/merkle-validation.js", "config": {} } } }, "tripleStore": { "enabled": true, "timeout": { "query": 60000, "get": 10000, "batchGet": 10000, "insert": 300000, "ask": 10000 }, "implementation": { "ot-blazegraph": { "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", "config": {} }, "ot-fuseki": { "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", "config": {} }, "ot-graphdb": { "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", "config": {} }, "ot-neptune": { "enabled": false, "package": "./triple-store/implementation/ot-neptune/ot-neptune.js", "config": {} } } }, "telemetry": { "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { "localEndpoint": "http::addr=localhost:10000", "signalingServiceEndpoint": "", "sendToSignalingService": false } } } }, "blockchainEvents": { "enabled": true, "implementation": { "ot-ethers": { "enabled": true, "package": "./blockchain-events/implementation/ot-ethers/ot-ethers.js", "config": { "blockchains": ["otp:20430", "gnosis:10200", "base:84532"], "rpcEndpoints": { "base:84532": ["https://sepolia.base.org"], "otp:20430": [ "https://lofar-testnet.origin-trail.network", "https://lofar-testnet.origintrail.network" ], "gnosis:10200": ["https://rpc.chiadochain.net"] }, "hubContractAddress": { "base:84532": "0xf21CE8f8b01548D97DCFb36869f1ccB0814a4e05", "otp:20430": "0xe233b5b78853a62b1e11ebe88bf083e25b0a57a6", "gnosis:10200": "0x2c08AC4B630c009F709521e56Ac385A6af70650f" } } } } } }, "maximumAssertionSizeInKb": 500000, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logging": { "defaultLevel": "trace", "enableExperimentalScopes": true }, "assetSync": { "syncDKG": { "enabled": false, "syncBatchSize": 20, "doneThreshold": 95 }, "syncParanets": [] }, "auth": { "ipBasedAuthEnabled": true, "tokenBasedAuthEnabled": false, "loggingEnabled": true, "ipWhitelist": ["::1", "127.0.0.1"], "publicOperations": [], "bothIpAndTokenAuthRequired": false } }, "devnet": { "modules": { "autoUpdater": { "enabled": true, "implementation": { "ot-auto-updater": { "enabled": true, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v8/develop" } } } }, "network": { "enabled": true, "implementation": { "libp2p-service": { "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { "kBucketSize": 20 }, "nat": { "enabled": true, "externalIp": null }, "connectionManager": { "autoDial": true, "autoDialInterval": 10e3, "dialTimeout": 2e3 }, "peerRouting": { "refreshManager": { "enabled": true, "interval": 6e5, "bootDelay": 2e3 } }, "port": 9000, "bootstrap": [ "/ip4/64.225.99.151/tcp/9000/p2p/QmawsTRqaLPyLQ5PfStpFcpQW4bvNQ59zV1by2G5aJHuVn" ] } } } }, "httpClient": { "enabled": true, "implementation": { "express-http-client": { "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, "port": 8900, "sslKeyPath": "/root/certs/privkey.pem", "sslCertificatePath": "/root/certs/fullchain.pem", "rateLimiter": { "timeWindowSeconds": 60, "maxRequests": 10 } } } } }, "repository": { "enabled": true, "implementation": { "sequelize-repository": { "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", "user": "root", "password": "password", "port": "3306", "host": "localhost", "dialect": "mysql", "logging": false, "pool": { "max": 120, "min": 0, "acquire": 60000, "idle": 10000, "evict": 1000 } } } } }, "blockchain": { "enabled": true, "implementation": { "base:84532": { "enabled": false, "package": "./blockchain/implementation/base/base-service.js", "config": { "hubContractAddress": "0xE043daF4cC8ae2c720ef95fc82574a37a429c40A", "rpcEndpoints": ["https://sepolia.base.org"], "operatorFee": 0 } } } }, "validation": { "enabled": true, "implementation": { "merkle-validation": { "enabled": true, "package": "./validation/implementation/merkle-validation.js", "config": {} } } }, "tripleStore": { "enabled": true, "timeout": { "query": 60000, "get": 10000, "batchGet": 10000, "insert": 300000, "ask": 10000 }, "implementation": { "ot-blazegraph": { "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", "config": {} }, "ot-fuseki": { "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", "config": {} }, "ot-graphdb": { "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", "config": {} }, "ot-neptune": { "enabled": false, "package": "./triple-store/implementation/ot-neptune/ot-neptune.js", "config": {} } } }, "telemetry": { "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { "localEndpoint": "http::addr=localhost:10000", "signalingServiceEndpoint": "", "sendToSignalingService": false } } } }, "blockchainEvents": { "enabled": true, "implementation": { "ot-ethers": { "enabled": true, "package": "./blockchain-events/implementation/ot-ethers/ot-ethers.js", "config": { "blockchains": ["base:84532"], "rpcEndpoints": { "base:84532": ["https://sepolia.base.org"] }, "hubContractAddress": { "base:84532": "0xE043daF4cC8ae2c720ef95fc82574a37a429c40A" } } } } } }, "maximumAssertionSizeInKb": 500000, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logging": { "defaultLevel": "trace", "enableExperimentalScopes": false }, "assetSync": { "syncDKG": { "enabled": false, "syncBatchSize": 20, "doneThreshold": 95 }, "syncParanets": [] }, "auth": { "ipBasedAuthEnabled": true, "tokenBasedAuthEnabled": false, "loggingEnabled": true, "ipWhitelist": ["::1", "127.0.0.1"], "publicOperations": [], "bothIpAndTokenAuthRequired": false } }, "mainnet": { "modules": { "autoUpdater": { "enabled": true, "implementation": { "ot-auto-updater": { "enabled": true, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v6/release/mainnet" } } } }, "network": { "enabled": true, "implementation": { "libp2p-service": { "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { "kBucketSize": 20 }, "nat": { "enabled": true, "externalIp": null }, "connectionManager": { "autoDial": true, "autoDialInterval": 10e3, "dialTimeout": 2e3 }, "peerRouting": { "refreshManager": { "enabled": true, "interval": 6e5, "bootDelay": 2e3 } }, "port": 9000, "bootstrap": [ "/ip4/157.230.96.194/tcp/9000/p2p/QmZFcns6eGUosD96beHyevKu1jGJ1bA56Reg2f1J4q59Jt", "/ip4/18.132.135.102/tcp/9000/p2p/QmemqyXyvrTAm7PwrcTcFiEEFx69efdR92GSZ1oQprbdja" ] } } } }, "httpClient": { "enabled": true, "implementation": { "express-http-client": { "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, "port": 8900, "sslKeyPath": "/root/certs/privkey.pem", "sslCertificatePath": "/root/certs/fullchain.pem", "rateLimiter": { "timeWindowSeconds": 60, "maxRequests": 10 } } } } }, "repository": { "enabled": true, "implementation": { "sequelize-repository": { "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", "user": "root", "password": "password", "port": "3306", "host": "localhost", "dialect": "mysql", "logging": false, "pool": { "max": 120, "min": 0, "acquire": 60000, "idle": 10000, "evict": 1000 } } } } }, "blockchain": { "enabled": true, "defaultImplementation": "otp:2043", "implementation": { "otp:2043": { "enabled": false, "package": "./blockchain/implementation/ot-parachain/ot-parachain-service.js", "config": { "hubContractAddress": "0x0957e25BD33034948abc28204ddA54b6E1142D6F", "rpcEndpoints": [ "https://astrosat-parachain-rpc.origin-trail.network", "https://astrosat.origintrail.network/", "https://astrosat-2.origintrail.network/" ], "operatorFee": 0 } }, "gnosis:100": { "enabled": false, "package": "./blockchain/implementation/gnosis/gnosis-service.js", "config": { "hubContractAddress": "0x882D0BF07F956b1b94BBfe9E77F47c6fc7D4EC8f", "gasPriceOracleLink": "https://gnosis.blockscout.com/api/v1/gas-price-oracle", "operatorFee": 0 } }, "base:8453": { "enabled": false, "package": "./blockchain/implementation/base/base-service.js", "config": { "hubContractAddress": "0x99Aa571fD5e681c2D27ee08A7b7989DB02541d13", "operatorFee": 0 } } } }, "tripleStore": { "enabled": true, "timeout": { "query": 60000, "get": 10000, "batchGet": 10000, "insert": 300000, "ask": 10000 }, "implementation": { "ot-blazegraph": { "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", "config": {} }, "ot-fuseki": { "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", "config": {} }, "ot-graphdb": { "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", "config": {} }, "ot-neptune": { "enabled": false, "package": "./triple-store/implementation/ot-neptune/ot-neptune.js", "config": {} } } }, "validation": { "enabled": true, "implementation": { "merkle-validation": { "enabled": true, "package": "./validation/implementation/merkle-validation.js", "config": {} } } }, "telemetry": { "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { "localEndpoint": "http::addr=localhost:10000", "signalingServiceEndpoint": "", "sendToSignalingService": false } } } }, "blockchainEvents": { "enabled": true, "implementation": { "ot-ethers": { "enabled": true, "package": "./blockchain-events/implementation/ot-ethers/ot-ethers.js", "config": { "blockchains": ["otp:2043", "gnosis:100", "base:8453"], "rpcEndpoints": { "otp:2043": [ "https://astrosat-parachain-rpc.origin-trail.network", "https://astrosat.origintrail.network/", "https://astrosat-2.origintrail.network/" ] }, "hubContractAddress": { "otp:2043": "0x0957e25BD33034948abc28204ddA54b6E1142D6F", "gnosis:100": "0x882D0BF07F956b1b94BBfe9E77F47c6fc7D4EC8f", "base:8453": "0x99Aa571fD5e681c2D27ee08A7b7989DB02541d13" } } } } } }, "maximumAssertionSizeInKb": 500000, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logging": { "defaultLevel": "trace", "enableExperimentalScopes": false }, "assetSync": { "syncDKG": { "enabled": false, "syncBatchSize": 20, "doneThreshold": 95 }, "syncParanets": [] }, "auth": { "ipBasedAuthEnabled": true, "tokenBasedAuthEnabled": false, "loggingEnabled": true, "ipWhitelist": ["::1", "127.0.0.1"], "publicOperations": [], "bothIpAndTokenAuthRequired": false } } } ================================================ FILE: config/papertrail.yml ================================================ files: - /ot-node/complete-node.log destination: host: logs4.papertrailapp.com port: 39178 protocol: tls ================================================ FILE: cucumber.js ================================================ export default { retry: 1, failFast: false, backtrace: true, }; ================================================ FILE: dependencies.md ================================================ # OT-node dependencies ## dev dependencies ##### [@cucumber/cucumber](https://www.npmjs.com/package/@cucumber/cucumber) - **version**: ^8.5.2 - **description**: used to execute bdd tests ##### [chai](https://www.npmjs.com/package/chai) - **version**: ^4.3.6 - **description**: assertion library for bdd tests ##### [dkg.js](https://www.npmjs.com/package/dkg.js) - **version**: ^6.0.2 - **description**: dkg client used in bdd tests ##### [eslint](https://www.npmjs.com/package/eslint) - **version**: ^8.23.0 - **description**: code linter ##### [eslint-config-airbnb](https://www.npmjs.com/package/eslint-config-airbnb) - **version**: ^19.0.4 - **description**: linter plugin ##### [eslint-config-prettier](https://www.npmjs.com/package/eslint-config-prettier) - **version**: ^8.5.0 - **description**: linter plugin ##### [husky](https://www.npmjs.com/package/husky) - **version**: ^8.0.1 - **description**: used to run lint-staged as pre commit ##### [lint-staged](https://www.npmjs.com/package/lint-staged) - **version**: ^13.0.3 - **description**: code linter for pre commits ##### [mocha](https://www.npmjs.com/package/mocha) - **version**: ^10.0.0 - **description**: test framework used in unit tests ##### [nyc](https://www.npmjs.com/package/nyc) - **version**: ^15.1.0 - **description**: command line interface used for running mocha ##### [prettier](https://www.npmjs.com/package/prettier) - **version**: ^2.7.1 - **description**: code formatter ##### [sinon](https://www.npmjs.com/package/sinon) - **version**: ^14.0.0 - **description**: used to create sandboxes in unit tests ##### [slugify](https://www.npmjs.com/package/slugify) - **version**: ^1.6.5 - **description**: used to stringify cucumber test scenarios ## dependencies ##### [@comunica/query-sparql](https://www.npmjs.com/package/@comunica/query-sparql) - **version**: ^2.4.3 - **description**: sparql query engine ##### [@ethersproject/bytes](https://www.npmjs.com/package/@ethersproject/bytes) - **version**: ^5.7.0 - **description**: Used for creating substrate and evm accounts mapping signatures in `create-account-mapping-signature.js` ##### [@ethersproject/hash](https://www.npmjs.com/package/@ethersproject/hash) - **version**: ^5.7.0 - **description**: Used for creating substrate and evm accounts mapping signatures in `create-account-mapping-signature.js` ##### [@ethersproject/wallet](https://www.npmjs.com/package/@ethersproject/wallet) - **version**: ^5.7.0 - **description**: Used for creating substrate and evm accounts mapping signatures in `create-account-mapping-signature.js` ##### [@polkadot/api](https://www.npmjs.com/package/@polkadot/api) - **version**: ^9.3.2 - **description**: used to interact with substrate nodes ##### [@polkadot/keyring](https://www.npmjs.com/package/@polkadot/keyring) - **version**: ^10.1.7 - **description**: Used for creating substrate and evm accounts mapping signatures in `create-account-mapping-signature.js` ##### [@polkadot/util](https://www.npmjs.com/package/@polkadot/util) - **version**: ^10.1.7 - **description**: Used for creating substrate and evm accounts mapping signatures in `create-account-mapping-signature.js` ##### [@polkadot/util-crypto](https://www.npmjs.com/package/@polkadot/util-crypto) - **version**: ^10.1.7 - **description**: Used for creating substrate and evm accounts mapping signatures in `create-account-mapping-signature.js` ##### [app-root-path](https://www.npmjs.com/package/app-root-path) - **version**: ^3.1.0 - **description**: used to determine root path ##### [assertion-tools](https://www.npmjs.com/package/assertion-tools) - **version**: ^2.0.2 - **description**: various functions used by both dkg.js and ot-node ##### [async](https://www.npmjs.com/package/async) - **version**: ^3.2.4 - **description**: used in `command-executor.js` to create an async queue to manage commands ##### [async-mutex](https://www.npmjs.com/package/async-mutex) - **version**: ^0.3.2 - **description**: used to avoid race conditions when updating sql repository ##### [awilix](https://www.npmjs.com/package/awilix) - **version**: ^7.0.3 - **description**: dependency injection container ##### [axios](https://www.npmjs.com/package/axios) - **version**: ^0.27.2 - **description**: http client used to make http requests ##### [cors](https://www.npmjs.com/package/cors) - **version**: ^2.8.5 - **description**: cors express middleware ##### [deep-extend](https://www.npmjs.com/package/deep-extend) - **version**: ^0.6.0 - **description**: used to merge users config and default config ##### [dkg-evm-module](https://www.npmjs.com/package/dkg-evm-module) - **version**: ^4.0.5 - **description**: used to import latest ot-node smart contracts abis ##### [dotenv](https://www.npmjs.com/package/dotenv) - **version**: ^16.0.1 - **description**: used for NODE_ENV variable ##### [ethers](https://www.npmjs.com/package/ethers) - **version**: ^5.7.2 - **description**: used to interact with blockchain nodes ##### [express](https://www.npmjs.com/package/express) - **version**: ^4.18.1 - **description**: used to handle http requests ##### [express-fileupload](https://www.npmjs.com/package/express-fileupload) - **version**: ^1.4.0 - **description**: express middleware **review required** ##### [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) - **version**: ^6.5.2 - **description**: used to rate limit rpc requests ##### [fs-extra](https://www.npmjs.com/package/fs-extra) - **version**: ^10.1.0 - **description**: used for file system methods ##### [graphdb](https://www.npmjs.com/package/graphdb) - **version**: ^2.0.2 - **description**: used to create graphdb repositories if they don't exist ##### [ip](https://www.npmjs.com/package/ip) - **version**: ^1.1.8 - **description**: used to compare ip addresses ##### [it-length-prefixed](https://www.npmjs.com/package/it-length-prefixed) - **version**: ^5.0.3 - **description**: used to encode and decode streamed buffers in libp2p ##### [it-map](https://www.npmjs.com/package/it-map) - **version**: ^1.0.6 - **description**: used to map values received yielded by libp2p async iterators ##### [it-pipe](https://www.npmjs.com/package/it-pipe) - **version**: ^1.1.0 - **description**: stream pipeline that supports libp2p duplex streams. Used for streaming messages between nodes ##### [jsonld](https://www.npmjs.com/package/jsonld) - **version**: ^8.1.0 - **description**: used to canonize n-quads retrieved from db. **Should be moved to assertion-tools dependency** ##### [jsonschema](https://www.npmjs.com/package/jsonschema) - **version**: ^1.4.1 - **description**: used to validate ot-node rpc requests' bodies ##### [jsonwebtoken](https://www.npmjs.com/package/jsonwebtoken) - **version**: ^9.0.0 - **description**: used to generate, validate and decode JWTs ##### [libp2p](https://www.npmjs.com/package/libp2p) - **version**: ^0.32.4 - **description**: used for p2p network communication ##### [libp2p-bootstrap](https://www.npmjs.com/package/libp2p-bootstrap) - **version**: ^0.13.0 - **description**: used to define libp2p bootstrap nodes ##### [libp2p-kad-dht](https://www.npmjs.com/package/libp2p-kad-dht) - **version**: ^0.24.2 - **description**: used for libp2p kad dht initialisation ##### [libp2p-mplex](https://www.npmjs.com/package/libp2p-mplex) - **version**: ^0.10.7 - **description**: used for libp2p mplex initialisation ##### [libp2p-noise](https://www.npmjs.com/package/libp2p-noise) - **version**: ^4.0.0 - **description**: used for libp2p message encription ##### [libp2p-tcp](https://www.npmjs.com/package/libp2p-tcp) - **version**: ^0.17.2 - **description**: used for libp2p tcp communication ##### [minimist](https://www.npmjs.com/package/minimist) - **version**: ^1.2.7 - **description**: used to parse process arguments ##### [ms](https://www.npmjs.com/package/ms) - **version**: ^2.1.3 - **description**: convert expiration time to milliseconds in `token-generation.js` ##### [mysql2](https://www.npmjs.com/package/mysql2) - **version**: ^3.3.0 - **description**: ##### [peer-id](https://www.npmjs.com/package/peer-id) - **version**: ^0.15.3 - **description**: used to create network id ##### [pino](https://www.npmjs.com/package/pino) - **version**: ^8.4.2 - **description**: ot-node logger implementation ##### [pino-pretty](https://www.npmjs.com/package/pino-pretty) - **version**: ^9.1.0 - **description**: prettifier for pino logger ##### [rc](https://www.npmjs.com/package/rc) - **version**: ^1.2.8 - **description**: configuration loader ##### [rolling-rate-limiter](https://www.npmjs.com/package/rolling-rate-limiter) - **version**: ^0.2.13 - **description**: used to limit network requests ##### [semver](https://www.npmjs.com/package/semver) - **version**: ^7.3.7 - **description**: used to compare ot-node versions during auto update ##### [sequelize](https://www.npmjs.com/package/sequelize) - **version**: ^6.29.0 - **description**: used to communicate with sql repository ##### [timeout-abort-controller](https://www.npmjs.com/package/timeout-abort-controller) - **version**: ^3.0.0 - **description**: timeout network messages ##### [toobusy-js](https://www.npmjs.com/package/toobusy-js) - **version**: ^0.5.1 - **description**: used to check nodejs event loop lag ##### [uint8arrays](https://www.npmjs.com/package/uint8arrays) - **version**: ^3.1.0 - **description**: used to convert from string to buffer and from buffer to string ##### [umzug](https://www.npmjs.com/package/umzug) - **version**: ^3.2.1 - **description**: sequelize migration tool ##### [unzipper](https://www.npmjs.com/package/unzipper) - **version**: ^0.10.11 - **description**: unzip file during autoupdate ##### [uuid](https://www.npmjs.com/package/uuid) - **version**: ^8.3.2 - **description**: uuid generation ================================================ FILE: docker/docker-compose-alpine-blazegraph.yaml ================================================ version: '3' services: blazegraph: container_name: blazegraph image: origintrail/ot-node:blazegraph network_mode: host mysql: container_name: mysql image: mysql:8.0.17 environment: MYSQL_ALLOW_EMPTY_PASSWORD: 1 MYSQL_ROOT_PASSWORD: null MYSQL_DATABASE: operationaldb expose: - 3306 network_mode: host ot-node: container_name: ot-node image: origintrail/ot-node:v6.0.0-beta.1-alpine depends_on: - blazegraph - mysql expose: - 8900 - 9000 command: - '/bin/sh' - '-c' - '/bin/sleep 25 && forever index.js' volumes: - ${PWD}/.origintrail_noderc:/ot-node/.origintrail_noderc - ~/certs/:/root/certs/ network_mode: host ================================================ FILE: docker/docker-compose-alpine-graphdb.yaml ================================================ version: '3' services: graphdb: container_name: graphdb image: khaller/graphdb-free:latest network_mode: host mysql: container_name: mysql image: mysql:8.0.17 environment: MYSQL_ALLOW_EMPTY_PASSWORD: 1 MYSQL_ROOT_PASSWORD: null MYSQL_DATABASE: operationaldb expose: - 3306 network_mode: host ot-node: container_name: ot-node image: origintrail/ot-node:v6.0.0-beta.1-alpine depends_on: - graphdb - mysql expose: - 8900 - 9000 command: - '/bin/sh' - '-c' - '/bin/sleep 25 && forever index.js' volumes: - ${PWD}/.origintrail_noderc:/ot-node/.origintrail_noderc - ~/certs/:/root/certs/ network_mode: host ================================================ FILE: docker/docker-compose-debian-blazegraph.yaml ================================================ version: '3.8' services: blazegraph: container_name: blazegraph image: origintrail/ot-node:blazegraph network_mode: host ot-node: container_name: ot-node image: origintrail/ot-node:v6.0.0-beta.1-debian depends_on: - blazegraph expose: - 8900 - 9000 command: > bash -c " /bin/sleep 30 service mariadb start && forever index.js " volumes: - ${PWD}/.origintrail_noderc:/ot-node/.origintrail_noderc - ~/certs/:/root/certs/ network_mode: host ================================================ FILE: docker/docker-compose-debian-graphdb.yaml ================================================ version: '3.8' services: graphdb: container_name: graphdb image: khaller/graphdb-free:latest network_mode: host ot-node: container_name: ot-node image: origintrail/ot-node:v6.0.0-beta.1-debian depends_on: - graphdb expose: - 8900 - 9000 command: > bash -c " /bin/sleep 30 service mariadb start && forever index.js " volumes: - ${PWD}/.origintrail_noderc:/ot-node/.origintrail_noderc - ~/certs/:/root/certs/ network_mode: host ================================================ FILE: docker/docker-compose-ubuntu-blazegraph.yaml ================================================ version: '3.8' services: blazegraph: container_name: blazegraph image: origintrail/ot-node:blazegraph network_mode: host ot-node: container_name: ot-node image: origintrail/ot-node:v6.0.0-beta.1-ubuntu depends_on: - blazegraph expose: - 8900 - 9000 command: > bash -c " /bin/sleep 35 service mysql restart && forever index.js " volumes: - ${PWD}/.origintrail_noderc:/ot-node/.origintrail_noderc - ~/certs/:/root/certs/ network_mode: host ================================================ FILE: docker/docker-compose-ubuntu-graphdb.yaml ================================================ version: '3.8' services: graphdb: container_name: graphdb image: khaller/graphdb-free:latest network_mode: host ot-node: container_name: ot-node image: origintrail/ot-node:v6.0.0-beta.1-ubuntu depends_on: - graphdb expose: - 8900 - 9000 command: > bash -c " /bin/sleep 35 service mysql restart && forever index.js " volumes: - ${PWD}/.origintrail_noderc:/ot-node/.origintrail_noderc - ~/certs/:/root/certs/ network_mode: host ================================================ FILE: docs/openapi/DKGv8.yaml ================================================ openapi: 3.0.3 info: title: DKGv8 description: DKG V8 API Collection. version: 1.0.0 contact: {} servers: - url: localhost paths: /info: get: tags: - old summary: Node Info description: Get the node information. operationId: nodeInfo responses: '200': description: Node Info headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '20' Date: schema: type: string example: Thu, 17 Aug 2023 12:43:07 GMT ETag: schema: type: string example: W/"14-Rq/28W5aGKCGXmXfM1+eW1LAbb4" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: version: type: string example: 6.0.13 examples: Node Info: value: version: 6.0.13 /bid-suggestion: get: tags: - old summary: Get Bid Suggestion description: Get bid suggestion based on provided parameters. operationId: getBidSuggestion parameters: - name: blockchain in: query schema: type: string example: hardhat - name: epochsNumber in: query schema: type: string example: '5' - name: assertionSize in: query schema: type: string example: '299' - name: contentAssetStorageAddress in: query schema: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' - name: firstAssertionId in: query schema: type: string example: '0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf' - name: hashFunctionId in: query schema: type: string example: '1' requestBody: content: text/plain: example: '' responses: '200': description: Get Bid Suggestion headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '38' Date: schema: type: string example: Thu, 17 Aug 2023 12:42:31 GMT ETag: schema: type: string example: W/"26-UrjseieOcIBnowM9obJae/FG7xc" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: bidSuggestion: type: string example: '903051579928002449' examples: Get Bid Suggestion: value: bidSuggestion: '903051579928002449' /local-store: post: tags: - old summary: Local Store description: Store locally. operationId: localStore requestBody: content: application/json: schema: type: array items: type: object properties: assertion: type: array items: type: string example: . example: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: type: string example: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: type: string example: hardhat contract: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: type: string example: TRIPLE tokenId: type: number example: 0 example: - assertion: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 - assertion: - '11000' . - 'Belgrade' . - 'Smith' . - 'Adam' . assertionId: >- 0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9 blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 example: - assertion: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 - assertion: - '11000' . - 'Belgrade' . - 'Smith' . - 'Adam' . assertionId: >- 0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9 blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 responses: '202': description: Local Store headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 12:49:45 GMT ETag: schema: type: string example: W/"36-uF3l7SNXwSBVObRCAJxOmp8OJGc" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 7d499975-ce42-4d84-9092-0ac2a62f5151 examples: Local Store: value: operationId: 7d499975-ce42-4d84-9092-0ac2a62f5151 /publish: post: tags: - old summary: Publish Knowledge Asset description: Publish assertion. operationId: publishKnowledgeAsset requestBody: content: application/json: schema: type: object properties: assertion: type: array items: type: string example: . example: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: type: string example: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: type: string example: hardhat contract: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: type: number example: 1 tokenId: type: number example: 0 example: assertion: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: 1 tokenId: 0 responses: '202': description: Publish Knowledge Asset headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 13:07:57 GMT ETag: schema: type: string example: W/"36-SQS1f7vf+HLSUHZ6wvE9UUwksSY" Keep-Alive: schema: type: string example: timeout=5 RateLimit-Limit: schema: type: string example: '10' RateLimit-Remaining: schema: type: string example: '9' RateLimit-Reset: schema: type: string example: '22' X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 8270c131-91b8-4573-a69e-504ff388a8b6 examples: Publish Knowledge Asset: value: operationId: 8270c131-91b8-4573-a69e-504ff388a8b6 /get: post: tags: - old summary: Get Knowledge Asset description: Get an assertion. operationId: getKnowledgeAsset requestBody: content: application/json: schema: type: object properties: hashFunctionId: type: number example: 1 id: type: string example: did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0 state: type: string example: LATEST example: hashFunctionId: 1 id: did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0 state: LATEST responses: '202': description: Get Knowledge Asset headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 13:16:39 GMT ETag: schema: type: string example: W/"36-tXDgcL88Mx02VotKK9H3zPuWwf8" Keep-Alive: schema: type: string example: timeout=5 RateLimit-Limit: schema: type: string example: '10' RateLimit-Remaining: schema: type: string example: '9' RateLimit-Reset: schema: type: string example: '12' X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 3a6df062-b3ce-4cac-aefa-77b1e8b9a4db examples: Get Knowledge Asset: value: operationId: 3a6df062-b3ce-4cac-aefa-77b1e8b9a4db /update: post: tags: - old summary: Update Knowledge Asset description: Update assertion. operationId: updateKnowledgeAsset requestBody: content: application/json: schema: type: object properties: assertion: type: array items: type: string example: . example: - . - 'TL' . - . - >- _:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' . assertionId: type: string example: >- 0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0 blockchain: type: string example: hardhat contract: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: type: number example: 1 tokenId: type: number example: 0 example: assertion: - . - 'TL' . - . - >- _:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' . assertionId: >- 0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0 blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: 1 tokenId: 0 responses: '202': description: Update Knowledge Asset headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 13:17:54 GMT ETag: schema: type: string example: W/"36-CjvPRlFINYIIcvR2H5gFBcOkNH8" Keep-Alive: schema: type: string example: timeout=5 RateLimit-Limit: schema: type: string example: '10' RateLimit-Remaining: schema: type: string example: '9' RateLimit-Reset: schema: type: string example: '57' X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 0d4c3efc-0f0b-435d-b9a3-402748dbbb2f examples: Update Knowledge Asset: value: operationId: 0d4c3efc-0f0b-435d-b9a3-402748dbbb2f /query: post: tags: - old summary: Query DKG description: Execute a query. operationId: queryDkg requestBody: content: application/json: schema: type: object properties: query: type: string example: >- CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}} repository: type: string example: privateCurrent type: type: string example: CONSTRUCT example: query: >- CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}} repository: privateCurrent type: CONSTRUCT responses: '202': description: Query DKG headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 13:20:16 GMT ETag: schema: type: string example: W/"36-WRBDN6AcKKCbVi3DGfI6FvESm5w" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 746992ba-e607-4858-8deb-5cffc2541859 examples: Query DKG: value: operationId: 746992ba-e607-4858-8deb-5cffc2541859 /{operation}/{operationId}: get: tags: - v0 summary: '[v0] Get Operation Result' description: Get result of a specific operation by its ID. operationId: v0GetOperationResult responses: '200': description: '' parameters: - name: operation in: path required: true schema: type: string - name: operationId in: path required: true schema: type: string /v0/info: get: tags: - v0 summary: '[v0] Node Info' description: Get the node information. operationId: v0NodeInfo responses: '200': description: '[v0] Node Info' headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '20' Date: schema: type: string example: Thu, 17 Aug 2023 13:27:58 GMT ETag: schema: type: string example: W/"14-Rq/28W5aGKCGXmXfM1+eW1LAbb4" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: version: type: string example: 6.0.13 examples: '[v0] Node Info': value: version: 6.0.13 /v0/bid-suggestion: get: tags: - v0 summary: '[v0] Get Bid Suggestion' description: Get bid suggestion based on provided parameters. operationId: v0GetBidSuggestion parameters: - name: blockchain in: query schema: type: string example: hardhat - name: epochsNumber in: query schema: type: string example: '5' - name: assertionSize in: query schema: type: string example: '299' - name: contentAssetStorageAddress in: query schema: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' - name: firstAssertionId in: query schema: type: string example: '0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf' - name: hashFunctionId in: query schema: type: string example: '1' requestBody: content: text/plain: example: '' responses: '200': description: '[v0] Get Bid Suggestion' headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '39' Date: schema: type: string example: Thu, 17 Aug 2023 13:59:02 GMT ETag: schema: type: string example: W/"27-ieFm/6t4DZwm0kFCMq71s37uy/g" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: bidSuggestion: type: string example: '1122511549276000025' examples: '[v0] Get Bid Suggestion': value: bidSuggestion: '1122511549276000025' /v0/local-store: post: tags: - v0 summary: '[v0] Local Store' description: Store locally. operationId: v0LocalStore requestBody: content: application/json: schema: type: array items: type: object properties: assertion: type: array items: type: string example: . example: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: type: string example: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: type: string example: hardhat contract: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: type: string example: TRIPLE tokenId: type: number example: 0 example: - assertion: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 - assertion: - '11000' . - 'Belgrade' . - 'Smith' . - 'Adam' . assertionId: >- 0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9 blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 example: - assertion: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 - assertion: - '11000' . - 'Belgrade' . - 'Smith' . - 'Adam' . assertionId: >- 0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9 blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' storeType: TRIPLE tokenId: 0 responses: '202': description: '[v0] Local Store' headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 13:59:11 GMT ETag: schema: type: string example: W/"36-fpQtTlhbbWO7tqbMGm3CkKmOqaI" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 0a4ee669-95bb-41cd-a2e8-3382361e80d9 examples: '[v0] Local Store': value: operationId: 0a4ee669-95bb-41cd-a2e8-3382361e80d9 /v0/publish: post: tags: - v0 summary: '[v0] Publish Knowledge Asset' description: Publish assertion. operationId: v0PublishKnowledgeAsset requestBody: content: application/json: schema: type: object properties: assertion: type: array items: type: string example: . example: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: type: string example: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: type: string example: hardhat contract: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: type: number example: 1 tokenId: type: number example: 0 example: assertion: - . - 'OT' . - . - >- _:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' . assertionId: >- 0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: 1 tokenId: 0 responses: '202': description: '[v0] Publish Knowledge Asset' headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 13:59:54 GMT ETag: schema: type: string example: W/"36-wKIhHpa0/tdVYh1Y8D2yINolruA" Keep-Alive: schema: type: string example: timeout=5 RateLimit-Limit: schema: type: string example: '10' RateLimit-Remaining: schema: type: string example: '9' RateLimit-Reset: schema: type: string example: '52' X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 476fb996-db1a-47b8-8da4-80d71411feb3 examples: '[v0] Publish Knowledge Asset': value: operationId: 476fb996-db1a-47b8-8da4-80d71411feb3 /v0/get: post: tags: - v0 summary: '[v0] Get Knowledge Asset' description: Get an assertion. operationId: v0GetKnowledgeAsset requestBody: content: application/json: schema: type: object properties: hashFunctionId: type: number example: 1 id: type: string example: did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0 state: type: string example: LATEST example: hashFunctionId: 1 id: did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0 state: LATEST responses: '202': description: '[v0] Get Knowledge Asset' headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 14:00:02 GMT ETag: schema: type: string example: W/"36-/27PH/ZH74wwVBrYDxWSzCk4yA0" Keep-Alive: schema: type: string example: timeout=5 RateLimit-Limit: schema: type: string example: '10' RateLimit-Remaining: schema: type: string example: '9' RateLimit-Reset: schema: type: string example: '44' X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 5b34c048-2d08-4696-b3c4-c37c831b89ce examples: '[v0] Get Knowledge Asset': value: operationId: 5b34c048-2d08-4696-b3c4-c37c831b89ce /v0/update: post: tags: - v0 summary: '[v0] Update Knowledge Asset' description: Update assertion. operationId: v0UpdateKnowledgeAsset requestBody: content: application/json: schema: type: object properties: assertion: type: array items: type: string example: . example: - . - 'TL' . - . - >- _:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' . assertionId: type: string example: >- 0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0 blockchain: type: string example: hardhat contract: type: string example: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: type: number example: 1 tokenId: type: number example: 0 example: assertion: - . - 'TL' . - . - >- _:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' . assertionId: >- 0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0 blockchain: hardhat contract: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07' hashFunctionId: 1 tokenId: 0 responses: '202': description: '[v0] Update Knowledge Asset' headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 14:00:09 GMT ETag: schema: type: string example: W/"36-77qAdgCc/SEN47aETAww86PM04w" Keep-Alive: schema: type: string example: timeout=5 RateLimit-Limit: schema: type: string example: '10' RateLimit-Remaining: schema: type: string example: '9' RateLimit-Reset: schema: type: string example: '38' X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: f0d34032-6910-49b4-a2a8-71c9f58feb58 examples: '[v0] Update Knowledge Asset': value: operationId: f0d34032-6910-49b4-a2a8-71c9f58feb58 /v0/query: post: tags: - v0 summary: '[v0] Query DKG' description: Execute a query. operationId: v0QueryDkg requestBody: content: application/json: schema: type: object properties: query: type: string example: >- CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}} repository: type: string example: privateCurrent type: type: string example: CONSTRUCT example: query: >- CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}} repository: privateCurrent type: CONSTRUCT responses: '202': description: '[v0] Query DKG' headers: Access-Control-Allow-Origin: schema: type: string example: '*' Connection: schema: type: string example: keep-alive Content-Length: schema: type: string example: '54' Date: schema: type: string example: Thu, 17 Aug 2023 14:00:19 GMT ETag: schema: type: string example: W/"36-kqaRe64EoJygoEadXJWRekiCs4s" Keep-Alive: schema: type: string example: timeout=5 X-Powered-By: schema: type: string example: Express content: application/json: schema: type: object properties: operationId: type: string example: 4d371ffb-a620-452f-8d16-3e427bafeae2 examples: '[v0] Query DKG': value: operationId: 4d371ffb-a620-452f-8d16-3e427bafeae2 tags: - name: old - name: v0 ================================================ FILE: docs/postman/DKGv8.postman_collection.json ================================================ { "info": { "_postman_id": "550b0443-cd47-482a-9c56-2b1229422426", "name": "DKGv8", "description": "DKG V8 API Collection.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "item": [ { "name": "old", "item": [ { "name": "Node Info", "request": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "url": { "raw": "{{host}}:{{port}}/info", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "info" ] }, "description": "Get the node information." }, "response": [ { "name": "Node Info", "originalRequest": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "url": { "raw": "{{host}}:{{port}}/info", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "info" ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "20" }, { "key": "ETag", "value": "W/\"14-Rq/28W5aGKCGXmXfM1+eW1LAbb4\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 12:43:07 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"version\": \"6.0.13\"\n}" } ] }, { "name": "Get Bid Suggestion", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "body": { "mode": "raw", "raw": "" }, "url": { "raw": "{{host}}:{{port}}/bid-suggestion?blockchain={{blockchain}}&epochsNumber={{epochsNumber}}&assertionSize={{assertionSize}}&contentAssetStorageAddress={{contentAssetStorageAddress}}&firstAssertionId={{firstAssertionId}}&hashFunctionId={{hashFunctionId}}", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "bid-suggestion" ], "query": [ { "key": "blockchain", "value": "{{blockchain}}" }, { "key": "epochsNumber", "value": "{{epochsNumber}}" }, { "key": "assertionSize", "value": "{{assertionSize}}" }, { "key": "contentAssetStorageAddress", "value": "{{contentAssetStorageAddress}}" }, { "key": "firstAssertionId", "value": "{{firstAssertionId}}" }, { "key": "hashFunctionId", "value": "{{hashFunctionId}}" } ] }, "description": "Get bid suggestion based on provided parameters." }, "response": [ { "name": "Get Bid Suggestion", "originalRequest": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "body": { "mode": "raw", "raw": "" }, "url": { "raw": "{{host}}:{{port}}/bid-suggestion?blockchain={{blockchain}}&epochsNumber={{epochsNumber}}&assertionSize={{assertionSize}}&contentAssetStorageAddress={{contentAssetStorageAddress}}&firstAssertionId={{firstAssertionId}}&hashFunctionId={{hashFunctionId}}", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "bid-suggestion" ], "query": [ { "key": "blockchain", "value": "{{blockchain}}" }, { "key": "epochsNumber", "value": "{{epochsNumber}}" }, { "key": "assertionSize", "value": "{{assertionSize}}" }, { "key": "contentAssetStorageAddress", "value": "{{contentAssetStorageAddress}}" }, { "key": "firstAssertionId", "value": "{{firstAssertionId}}" }, { "key": "hashFunctionId", "value": "{{hashFunctionId}}" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "38" }, { "key": "ETag", "value": "W/\"26-UrjseieOcIBnowM9obJae/FG7xc\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 12:42:31 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"bidSuggestion\": \"903051579928002449\"\n}" } ] }, { "name": "Local Store", "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{{assertions}}" }, "url": { "raw": "{{host}}:{{port}}/local-store", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "local-store" ] }, "description": "Store locally." }, "response": [ { "name": "Local Store", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{{assertions}}" }, "url": { "raw": "{{host}}:{{port}}/local-store", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "local-store" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-uF3l7SNXwSBVObRCAJxOmp8OJGc\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 12:49:45 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"7d499975-ce42-4d84-9092-0ac2a62f5151\"\n}" } ] }, { "name": "Publish Knowledge Asset", "protocolProfileBehavior": { "disabledSystemHeaders": { "content-type": true } }, "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf\",\n \"assertion\": [\n \" .\",\n \" 'OT' .\",\n \" .\",\n \"_:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/publish", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "publish" ] }, "description": "Publish assertion." }, "response": [ { "name": "Publish Knowledge Asset", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf\",\n \"assertion\": [\n \" .\",\n \" 'OT' .\",\n \" .\",\n \"_:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/publish", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "publish" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "RateLimit-Limit", "value": "10" }, { "key": "RateLimit-Remaining", "value": "9" }, { "key": "RateLimit-Reset", "value": "22" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-SQS1f7vf+HLSUHZ6wvE9UUwksSY\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:07:57 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"8270c131-91b8-4573-a69e-504ff388a8b6\"\n}" } ] }, { "name": "Get Knowledge Asset", "protocolProfileBehavior": { "disabledSystemHeaders": {} }, "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"id\": \"did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0\",\n \"state\": \"LATEST\",\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/get", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "get" ] }, "description": "Get an assertion." }, "response": [ { "name": "Get Knowledge Asset", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"id\": \"did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0\",\n \"state\": \"LATEST\",\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/get", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "get" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "RateLimit-Limit", "value": "10" }, { "key": "RateLimit-Remaining", "value": "9" }, { "key": "RateLimit-Reset", "value": "12" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-tXDgcL88Mx02VotKK9H3zPuWwf8\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:16:39 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"3a6df062-b3ce-4cac-aefa-77b1e8b9a4db\"\n}" } ] }, { "name": "Update Knowledge Asset", "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0\",\n \"assertion\": [\n \" .\",\n \" 'TL' .\",\n \" .\",\n \"_:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/update", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "update" ] }, "description": "Update assertion." }, "response": [ { "name": "Update Knowledge Asset", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0\",\n \"assertion\": [\n \" .\",\n \" 'TL' .\",\n \" .\",\n \"_:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/update", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "update" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "RateLimit-Limit", "value": "10" }, { "key": "RateLimit-Remaining", "value": "9" }, { "key": "RateLimit-Reset", "value": "57" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-CjvPRlFINYIIcvR2H5gFBcOkNH8\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:17:54 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"0d4c3efc-0f0b-435d-b9a3-402748dbbb2f\"\n}" } ] }, { "name": "Query DKG", "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"query\": \"CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}}\",\n \"type\": \"CONSTRUCT\",\n \"repository\": \"privateCurrent\"\n}" }, "url": { "raw": "{{host}}:{{port}}/query", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "query" ] }, "description": "Execute a query." }, "response": [ { "name": "Query DKG", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"query\": \"CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}}\",\n \"type\": \"CONSTRUCT\",\n \"repository\": \"privateCurrent\"\n}" }, "url": { "raw": "{{host}}:{{port}}/query", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "query" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-WRBDN6AcKKCbVi3DGfI6FvESm5w\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:20:16 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"746992ba-e607-4858-8deb-5cffc2541859\"\n}" } ] }, { "name": "Get Operation Result", "request": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "url": { "raw": "{{host}}:{{port}}/{{operation}}/{{operationId}}", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "{{operation}}", "{{operationId}}" ] }, "description": "Get result of a specific operation by its ID." }, "response": [] } ] }, { "name": "v0", "item": [ { "name": "[v0] Node Info", "request": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "url": { "raw": "{{host}}:{{port}}/v0/info", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "info" ] }, "description": "Get the node information." }, "response": [ { "name": "[v0] Node Info", "originalRequest": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "url": { "raw": "{{host}}:{{port}}/v0/info", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "info" ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "20" }, { "key": "ETag", "value": "W/\"14-Rq/28W5aGKCGXmXfM1+eW1LAbb4\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:27:58 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"version\": \"6.0.13\"\n}" } ] }, { "name": "[v0] Get Bid Suggestion", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "body": { "mode": "raw", "raw": "" }, "url": { "raw": "{{host}}:{{port}}/v0/bid-suggestion?blockchain={{blockchain}}&epochsNumber={{epochsNumber}}&assertionSize={{assertionSize}}&contentAssetStorageAddress={{contentAssetStorageAddress}}&firstAssertionId={{firstAssertionId}}&hashFunctionId={{hashFunctionId}}", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "bid-suggestion" ], "query": [ { "key": "blockchain", "value": "{{blockchain}}" }, { "key": "epochsNumber", "value": "{{epochsNumber}}" }, { "key": "assertionSize", "value": "{{assertionSize}}" }, { "key": "contentAssetStorageAddress", "value": "{{contentAssetStorageAddress}}" }, { "key": "firstAssertionId", "value": "{{firstAssertionId}}" }, { "key": "hashFunctionId", "value": "{{hashFunctionId}}" } ] }, "description": "Get bid suggestion based on provided parameters." }, "response": [ { "name": "[v0] Get Bid Suggestion", "originalRequest": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "body": { "mode": "raw", "raw": "" }, "url": { "raw": "{{host}}:{{port}}/v0/bid-suggestion?blockchain={{blockchain}}&epochsNumber={{epochsNumber}}&assertionSize={{assertionSize}}&contentAssetStorageAddress={{contentAssetStorageAddress}}&firstAssertionId={{firstAssertionId}}&hashFunctionId={{hashFunctionId}}", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "bid-suggestion" ], "query": [ { "key": "blockchain", "value": "{{blockchain}}" }, { "key": "epochsNumber", "value": "{{epochsNumber}}" }, { "key": "assertionSize", "value": "{{assertionSize}}" }, { "key": "contentAssetStorageAddress", "value": "{{contentAssetStorageAddress}}" }, { "key": "firstAssertionId", "value": "{{firstAssertionId}}" }, { "key": "hashFunctionId", "value": "{{hashFunctionId}}" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "39" }, { "key": "ETag", "value": "W/\"27-ieFm/6t4DZwm0kFCMq71s37uy/g\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:59:02 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"bidSuggestion\": \"1122511549276000025\"\n}" } ] }, { "name": "[v0] Local Store", "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{{assertions}}" }, "url": { "raw": "{{host}}:{{port}}/v0/local-store", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "local-store" ] }, "description": "Store locally." }, "response": [ { "name": "[v0] Local Store", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{{assertions}}" }, "url": { "raw": "{{host}}:{{port}}/v0/local-store", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "local-store" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-fpQtTlhbbWO7tqbMGm3CkKmOqaI\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:59:11 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"0a4ee669-95bb-41cd-a2e8-3382361e80d9\"\n}" } ] }, { "name": "[v0] Publish Knowledge Asset", "protocolProfileBehavior": { "disabledSystemHeaders": { "content-type": true } }, "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf\",\n \"assertion\": [\n \" .\",\n \" 'OT' .\",\n \" .\",\n \"_:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/publish", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "publish" ] }, "description": "Publish assertion." }, "response": [ { "name": "[v0] Publish Knowledge Asset", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf\",\n \"assertion\": [\n \" .\",\n \" 'OT' .\",\n \" .\",\n \"_:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/publish", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "publish" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "RateLimit-Limit", "value": "10" }, { "key": "RateLimit-Remaining", "value": "9" }, { "key": "RateLimit-Reset", "value": "52" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-wKIhHpa0/tdVYh1Y8D2yINolruA\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 13:59:54 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"476fb996-db1a-47b8-8da4-80d71411feb3\"\n}" } ] }, { "name": "[v0] Get Knowledge Asset", "protocolProfileBehavior": { "disabledSystemHeaders": {} }, "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"id\": \"did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0\",\n \"state\": \"LATEST\",\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/get", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "get" ] }, "description": "Get an assertion." }, "response": [ { "name": "[v0] Get Knowledge Asset", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"id\": \"did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0\",\n \"state\": \"LATEST\",\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/get", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "get" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "RateLimit-Limit", "value": "10" }, { "key": "RateLimit-Remaining", "value": "9" }, { "key": "RateLimit-Reset", "value": "44" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-/27PH/ZH74wwVBrYDxWSzCk4yA0\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 14:00:02 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"5b34c048-2d08-4696-b3c4-c37c831b89ce\"\n}" } ] }, { "name": "[v0] Update Knowledge Asset", "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0\",\n \"assertion\": [\n \" .\",\n \" 'TL' .\",\n \" .\",\n \"_:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/update", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "update" ] }, "description": "Update assertion." }, "response": [ { "name": "[v0] Update Knowledge Asset", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"assertionId\": \"0xef0adc464c3dcb1d353567db5972de8d47f44d6621326645324f9730f2c83cf0\",\n \"assertion\": [\n \" .\",\n \" 'TL' .\",\n \" .\",\n \"_:c14n0 '0xa3acb6d57097f316b973e9e33d303cf411b8d62d7d589576e348d0d7049e3b63' .\"\n ],\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"hashFunctionId\": 1\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/update", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "update" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "RateLimit-Limit", "value": "10" }, { "key": "RateLimit-Remaining", "value": "9" }, { "key": "RateLimit-Reset", "value": "38" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-77qAdgCc/SEN47aETAww86PM04w\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 14:00:09 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"f0d34032-6910-49b4-a2a8-71c9f58feb58\"\n}" } ] }, { "name": "[v0] Query DKG", "request": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"query\": \"CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}}\",\n \"type\": \"CONSTRUCT\",\n \"repository\": \"privateCurrent\"\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/query", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "query" ] }, "description": "Execute a query." }, "response": [ { "name": "[v0] Query DKG", "originalRequest": { "method": "POST", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" }, { "key": "Content-Type", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"query\": \"CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}}\",\n \"type\": \"CONSTRUCT\",\n \"repository\": \"privateCurrent\"\n}" }, "url": { "raw": "{{host}}:{{port}}/v0/query", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "v0", "query" ] } }, "status": "Accepted", "code": 202, "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", "value": "Express" }, { "key": "Access-Control-Allow-Origin", "value": "*" }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", "value": "54" }, { "key": "ETag", "value": "W/\"36-kqaRe64EoJygoEadXJWRekiCs4s\"" }, { "key": "Date", "value": "Thu, 17 Aug 2023 14:00:19 GMT" }, { "key": "Connection", "value": "keep-alive" }, { "key": "Keep-Alive", "value": "timeout=5" } ], "cookie": [], "body": "{\n \"operationId\": \"4d371ffb-a620-452f-8d16-3e427bafeae2\"\n}" } ] }, { "name": "[v0] Get Operation Result", "request": { "method": "GET", "header": [ { "key": "Authorization", "value": "Bearer {{authToken}}" } ], "url": { "raw": "{{host}}:{{port}}/{{operation}}/{{operationId}}", "host": [ "{{host}}" ], "port": "{{port}}", "path": [ "{{operation}}", "{{operationId}}" ] }, "description": "Get result of a specific operation by its ID." }, "response": [] } ] } ], "auth": { "type": "bearer", "bearer": [ { "key": "token", "value": "{{authToken}}", "type": "string" } ] }, "event": [ { "listen": "prerequest", "script": { "type": "text/javascript", "exec": [ "" ] } }, { "listen": "test", "script": { "type": "text/javascript", "exec": [ "" ] } } ], "variable": [ { "key": "host", "value": "localhost" }, { "key": "port", "value": "8900", "type": "number" }, { "key": "authToken", "value": "", "type": "string" }, { "key": "blockchain", "value": "hardhat" }, { "key": "epochsNumber", "value": "5" }, { "key": "assertionSize", "value": "299" }, { "key": "contentAssetStorageAddress", "value": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07" }, { "key": "hashFunctionId", "value": "1" }, { "key": "firstAssertionId", "value": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" }, { "key": "assertions", "value": "[\n {\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"assertionId\": \"0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf\",\n \"assertion\": [\n \" .\",\n \" 'OT' .\",\n \" .\",\n \"_:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' .\"\n ],\n \"storeType\": \"TRIPLE\"\n },\n {\n \"blockchain\": \"hardhat\",\n \"contract\": \"0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07\",\n \"tokenId\": 0,\n \"assertionId\": \"0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9\",\n \"assertion\": [\n \" '11000' .\",\n \" 'Belgrade' .\",\n \" 'Smith' .\",\n \" 'Adam' .\"\n ],\n \"storeType\": \"TRIPLE\"\n }\n]", "type": "string" }, { "key": "assertionId", "value": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" }, { "key": "assertion", "value": "[\n \" .\",\n \" 'OT' .\",\n \" .\",\n \"_:c14n0 '0xcfab2d364fe01757d7a83d3b32284395d87b1c379adabb1e28a16666e0a4fca9' .\"\n]", "type": "string" }, { "key": "contract", "value": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07" }, { "key": "tokenId", "value": "1" }, { "key": "UAL", "value": "did:dkg:hardhat/0xb0d4afd8879ed9f52b28595d31b441d079b2ca07/0" }, { "key": "state", "value": "LATEST" }, { "key": "query", "value": "CONSTRUCT { ?s ?p ?o } WHERE {{GRAPH { ?s ?p ?o . }}}" }, { "key": "type", "value": "CONSTRUCT" }, { "key": "repository", "value": "privateCurrent" } ] } ================================================ FILE: index.js ================================================ /* eslint-disable no-console */ import 'dotenv/config'; import fs from 'fs-extra'; import OTNode from './ot-node.js'; import { NODE_ENVIRONMENTS } from './src/constants/constants.js'; process.env.NODE_ENV = process.env.NODE_ENV && Object.values(NODE_ENVIRONMENTS).includes(process.env.NODE_ENV) ? process.env.NODE_ENV : NODE_ENVIRONMENTS.DEVELOPMENT; (async () => { let userConfig = null; try { if (process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT && process.argv.length === 3) { const configurationFilename = process.argv[2]; userConfig = JSON.parse(await fs.promises.readFile(process.argv[2])); userConfig.configFilename = configurationFilename; } } catch (error) { console.log('Unable to read user configuration from file: ', process.argv[2]); process.exit(1); } try { const node = new OTNode(userConfig); await node.start(); } catch (e) { console.error(`Error occurred while start ot-node, error message: ${e}. ${e.stack}`); // console.error(`Trying to recover from older version`); // if (process.env.NODE_ENV !== NODE_ENVIRONMENTS.DEVELOPMENT) { // const rootPath = path.join(appRootPath.path, '..'); // const oldVersionsDirs = (await fs.promises.readdir(rootPath, { withFileTypes: true })) // .filter((dirent) => dirent.isDirectory()) // .map((dirent) => dirent.name) // .filter((name) => semver.valid(name) && !appRootPath.path.includes(name)); // // if (oldVersionsDirs.length === 0) { // console.error( // `Failed to start OT-Node, no backup code available. Error message: ${e.message}`, // ); // process.exit(1); // } // // const oldVersion = oldVersionsDirs.sort(semver.compare).pop(); // const oldversionPath = path.join(rootPath, oldVersion); // execSync(`ln -sfn ${oldversionPath} ${rootPath}/current`); // await fs.promises.rm(appRootPath.path, { force: true, recursive: true }); // } process.exit(1); } })(); process.on('unhandledRejection', (err) => { // Handle specific libp2p peer lookup failures that escape try-catch blocks if (err && err.code === 'ERR_LOOKUP_FAILED') { console.warn(`Peer lookup failed (ERR_LOOKUP_FAILED): ${err.message}`); return; // Don't crash for peer lookup failures } // Handle ECONNRESET errors gracefully - these are common network issues if (err && (err.code === 'ECONNRESET' || err.errno === -104)) { console.warn(`Network connection reset (ECONNRESET): ${err.message}`); return; // Don't crash for connection reset errors } // Handle ERR_UNSUPPORTED_PROTOCOL errors gracefully if (err && err.code === 'ERR_UNSUPPORTED_PROTOCOL') { console.warn(`Unsupported protocol error (ERR_UNSUPPORTED_PROTOCOL): ${err.message}`); return; // Don't crash for protocol errors } // Handle EPIPE (broken pipe) errors gracefully if (err && (err.code === 'EPIPE' || err.errno === -32)) { console.warn(`Broken pipe error (EPIPE): ${err.message}`); return; // Don't crash for broken pipe errors } // Handle ETIMEDOUT errors gracefully - these are common database connection timeouts if (err && (err.code === 'ETIMEDOUT' || err.errno === -110)) { console.warn(`Connection timeout error (ETIMEDOUT): ${err.message}`); return; // Don't crash for timeout errors } // Handle Sequelize "Got timeout reading communication packets" errors gracefully if (err && err.message && err.message.includes('Got timeout reading communication packets')) { console.warn(`Sequelize communication timeout error: ${err.message}`); return; // Don't crash for database communication timeout errors } // For all other unhandled rejections, crash the node console.error('Something went really wrong! OT-node shutting down...', err); process.exit(1); }); process.on('uncaughtException', (err) => { // Handle ERR_UNSUPPORTED_PROTOCOL errors gracefully if (err && err.code === 'ERR_UNSUPPORTED_PROTOCOL') { console.warn(`Unsupported protocol error (ERR_UNSUPPORTED_PROTOCOL): ${err.message}`); return; // Don't crash for protocol errors } // Handle EPIPE (broken pipe) errors gracefully if (err && (err.code === 'EPIPE' || err.errno === -32)) { console.warn(`Broken pipe error (EPIPE): ${err.message}`); return; // Don't crash for broken pipe errors } // Handle ECONNRESET errors gracefully if (err && (err.code === 'ECONNRESET' || err.errno === -104)) { console.warn(`Network connection reset (ECONNRESET): ${err.message}`); return; // Don't crash for connection reset errors } // Handle ETIMEDOUT errors gracefully - these are common database connection timeouts if (err && (err.code === 'ETIMEDOUT' || err.errno === -110)) { console.warn(`Connection timeout error (ETIMEDOUT): ${err.message}`); return; // Don't crash for timeout errors } // Handle Sequelize "Got timeout reading communication packets" errors gracefully if (err && err.message && err.message.includes('Got timeout reading communication packets')) { console.warn(`Sequelize communication timeout error: ${err.message}`); return; // Don't crash for database communication timeout errors } console.error('Something went really wrong! OT-node shutting down...', err); process.exit(1); }); ================================================ FILE: installer/README.md ================================================ Installs the V8 Node 2. Login to the server as root. You __cannot__ use sudo and run this script. The command "npm install" __will__ fail. 3. Execute the following command: ``` cd /root/ && curl https://raw.githubusercontent.com/OriginTrail/ot-node/v8/release/testnet/installer/installer.sh --output installer.sh && chmod +x installer.sh ``` ================================================ FILE: installer/installer.sh ================================================ #!/bin/bash OTNODE_DIR="/root/ot-node" text_color() { GREEN='\033[0;32m' BGREEN='\033[1;32m' RED='\033[0;31m' BRED='\033[1;31m' YELLOW='\033[0;33m' BYELLOW='\033[1;33m' BOLD='\033[1m' NC='\033[0m' # No Color # Detect if this is an error, warning or success message local message="$@" if [[ "$message" == *"$RED"* || "$message" == *"$BRED"* ]]; then echo -e "❌ $@$NC" elif [[ "$message" == *"$YELLOW"* || "$message" == *"$BYELLOW"* ]]; then echo -e "⚠️ $@$NC" elif [[ "$message" == *"$GREEN"* || "$message" == *"$BGREEN"* ]]; then echo -e "✅ $@$NC" else echo -e "$@$NC" fi } header_color() { LIGHTCYAN='\033[1;36m' NC='\033[0m' # No Color local header_text="$@" local line=$(printf '═%.0s' $(seq 1 ${#header_text})) echo "" echo -e "${LIGHTCYAN}╔═${line}═╗${NC}" echo -e "${LIGHTCYAN}║ ${header_text} ║${NC}" echo -e "${LIGHTCYAN}╚═${line}═╝${NC}" echo "" } perform_step() { N1=$'\n' echo -n "⏳ ${@: -1}: " OUTPUT=$(${@:1:$#-1} 2>&1) if [[ $? -ne 0 ]]; then text_color $BOLD$RED "FAILED" echo -e "${N1}❌ Step failed. Output of error is:${N1}${N1}$OUTPUT" echo -e "${BRED}Press Enter to exit the installer.${NC}" read exit 1 else text_color $BOLD$GREEN "OK" fi } # Function to display a notification box notification_box() { local message="$1" local type="$2" local RED='\033[0;31m' local GREEN='\033[0;32m' local YELLOW='\033[0;33m' local BLUE='\033[0;34m' local BOLD='\033[1m' local NC='\033[0m' local color="$BLUE" local icon="ℹ️" if [[ "$type" == "error" ]]; then color="$RED" icon="❌" elif [[ "$type" == "warning" ]]; then color="$YELLOW" icon="⚠️" elif [[ "$type" == "success" ]]; then color="$GREEN" icon="✅" fi local line=$(printf '─%.0s' $(seq 1 60)) echo -e "${color}$line${NC}" echo -e "${color}${BOLD} $icon $message${NC}" echo -e "${color}$line${NC}" if [[ "$type" == "error" ]]; then echo -e "${BRED}Press Enter to exit the installer.${NC}" read fi } # Check Ubuntu version check_ubuntu_version() { UBUNTU_VERSION=$(lsb_release -r -s) if [[ "$UBUNTU_VERSION" != "20.04" && "$UBUNTU_VERSION" != "22.04" && "$UBUNTU_VERSION" != "24.04" ]]; then notification_box "Error: OriginTrail node installer currently requires Ubuntu 20.04 LTS, 22.04 LTS or 24.04 LTS versions in order to execute successfully. You are installing on Ubuntu $UBUNTU_VERSION." echo -e "${BRED}Please make sure that you get familiar with the requirements before setting up your OriginTrail node! Documentation: docs.origintrail.io${NC}" exit 1 fi } # Check if script is running as root check_root() { if [[ $EUID -ne 0 ]]; then notification_box "Error: This script must be run as root." echo -e "${BRED}Please re-run the script as root using 'sudo'.${NC}" exit 1 fi } install_aliases() { if [[ -f "/root/.bashrc" ]]; then if grep -Fxq "alias otnode-restart='systemctl restart otnode.service'" ~/.bashrc; then echo "Aliases found, skipping." else echo "alias otnode-restart='systemctl restart otnode.service'" >> ~/.bashrc echo "alias otnode-stop='systemctl stop otnode.service'" >> ~/.bashrc echo "alias otnode-start='systemctl start otnode.service'" >> ~/.bashrc echo "alias otnode-logs='journalctl -u otnode --output cat -f'" >> ~/.bashrc echo "alias otnode-config='nano ~/ot-node/.origintrail_noderc'" >> ~/.bashrc fi else echo "bashrc does not exist. Proceeding with OriginTrail node installation." fi } install_directory() { ARCHIVE_REPOSITORY_URL="github.com/OriginTrail/ot-node/archive" echo "" echo -e "${CYAN}┌─────────────────────────────────────────────┐${RESET}" echo -e "${CYAN}│ NODE ENVIRONMENT SELECTION │${RESET}" echo -e "${CYAN}└─────────────────────────────────────────────┘${RESET}" echo "" echo -e "Please select the environment for your OriginTrail node:" echo -e " [M] ${GREEN}Mainnet${RESET} - Production environment" echo -e " [T] ${YELLOW}Testnet${RESET} - Testing environment" echo "" read -p "▶ Your choice [M/T/E to exit]: " choice case "$choice" in [tT]* ) nodeEnv="testnet"; BRANCH="v6/release/testnet"; BRANCH_DIR="/root/ot-node-6-release-testnet";; [mM]* ) nodeEnv="mainnet"; BRANCH="v6/release/mainnet"; BRANCH_DIR="/root/ot-node-6-release-mainnet";; [eE]* ) text_color $RED "Installer stopped by user"; exit;; * ) nodeEnv="mainnet"; BRANCH="v6/release/mainnet"; BRANCH_DIR="/root/ot-node-6-release-mainnet";; esac text_color $GREEN "Selected environment: $nodeEnv with branch: $BRANCH" perform_step wget https://$ARCHIVE_REPOSITORY_URL/$BRANCH.zip "Downloading node files" perform_step unzip *.zip "Unzipping node files" perform_step rm *.zip "Removing zip file" OTNODE_VERSION=$(jq -r '.version' $BRANCH_DIR/package.json) perform_step mkdir $OTNODE_DIR "Creating new ot-node directory" perform_step mkdir $OTNODE_DIR/$OTNODE_VERSION "Creating new ot-node version directory" perform_step mv $BRANCH_DIR/* $OTNODE_DIR/$OTNODE_VERSION/ "Moving downloaded node files to ot-node version directory" OUTPUT=$(mv $BRANCH_DIR/.* $OTNODE_DIR/$OTNODE_VERSION/ 2>&1) perform_step rm -rf $BRANCH_DIR "Removing old directories" perform_step ln -sfn $OTNODE_DIR/$OTNODE_VERSION $OTNODE_DIR/current "Creating symlink from $OTNODE_DIR/$OTNODE_VERSION to $OTNODE_DIR/current" echo "NODE_ENV=$nodeEnv" >> $OTNODE_DIR/current/.env # Save selected environment for later use export SELECTED_NODE_ENV=$nodeEnv } install_prereqs() { export DEBIAN_FRONTEND=noninteractive NODEJS_VER="20" perform_step install_aliases "Updating .bashrc file with OriginTrail node aliases" > /dev/null 2>&1 perform_step rm -rf /var/lib/dpkg/lock-frontend "Removing any frontend locks" > /dev/null 2>&1 perform_step apt update "Updating Ubuntu package repository" > /dev/null 2>&1 perform_step apt upgrade -y "Updating Ubuntu to the latest version" > /dev/null 2>&1 perform_step apt install unzip jq -y "Installing unzip, jq" > /dev/null 2>&1 perform_step apt install default-jre -y "Installing default-jre" > /dev/null 2>&1 perform_step apt install build-essential -y "Installing build-essential" > /dev/null 2>&1 # Install nodejs 20 (via NVM). wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash > /dev/null 2>&1 export NVM_DIR="$HOME/.nvm" # This loads nvm [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm bash_completion [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" nvm install $NODEJS_VER > /dev/null 2>&1 nvm use $NODEJS_VER > /dev/null 2>&1 # Set nodejs 20 as default and link node to /usr/bin/ nvm alias default $NODEJS_VER > /dev/null 2>&1 sudo ln -s $(which node) /usr/bin/ > /dev/null 2>&1 sudo ln -s $(which npm) /usr/bin/ > /dev/null 2>&1 apt remove unattended-upgrades -y > /dev/null 2>&1 perform_step apt remove unattended-upgrades -y "Remove unattended upgrades" > /dev/null 2>&1 } install_fuseki() { FUSEKI_VER="apache-jena-fuseki-$(git ls-remote --tags https://github.com/apache/jena | grep -o 'refs/tags/jena-[0-9]*\.[0-9]*\.[0-9]*' | sort -r | head -n 1 | grep -o '[^\/-]*$')" FUSEKI_PREV_VER="apache-jena-fuseki-$(git ls-remote --tags https://github.com/apache/jena | grep -o 'refs/tags/jena-[0-9]*\.[0-9]*\.[0-9]*' | sort -r | head -n 3 | tail -n 1 | grep -o '[^\/-]*$')" wget -q --spider https://dlcdn.apache.org/jena/binaries/$FUSEKI_VER.zip if [[ $? -ne 0 ]]; then FUSEKI_VER=$FUSEKI_PREV_VER fi perform_step wget https://dlcdn.apache.org/jena/binaries/$FUSEKI_VER.zip "Downloading Fuseki" perform_step unzip $FUSEKI_VER.zip "Unzipping Fuseki" perform_step rm /root/$FUSEKI_VER.zip "Removing Fuseki zip file" perform_step mkdir /root/ot-node/fuseki "Making /root/ot-node/fuseki directory" perform_step cp /root/$FUSEKI_VER/fuseki-server.jar /root/ot-node/fuseki/ "Copying Fuseki files to $OTNODE_DIR/fuseki/ 1/2" perform_step cp -r /root/$FUSEKI_VER/webapp/ /root/ot-node/fuseki/ "Copying Fuseki files to $OTNODE_DIR/fuseki/ 1/2" perform_step rm -r /root/$FUSEKI_VER "Removing the remaining /root/$FUSEKI_VER directory" perform_step cp $OTNODE_DIR/installer/data/fuseki.service /lib/systemd/system/ "Copying Fuseki service file" systemctl daemon-reload perform_step systemctl enable fuseki "Enabling Fuseki" perform_step systemctl start fuseki "Starting Fuseki" perform_step systemctl status fuseki "Fuseki status" } install_blazegraph() { perform_step wget https://github.com/blazegraph/database/releases/latest/download/blazegraph.jar "Downloading Blazegraph" perform_step cp $OTNODE_DIR/installer/data/blazegraph.service /lib/systemd/system/ "Copying Blazegraph service file" mv blazegraph.jar $OTNODE_DIR/../blazegraph.jar systemctl daemon-reload perform_step systemctl enable blazegraph "Enabling Blazegrpah" perform_step systemctl start blazegraph "Starting Blazegraph" perform_step systemctl status blazegraph "Blazegraph status" } install_sql() { # Replace the SQL database selection with a more user-friendly interface text_color $BYELLOW "╔════════════════════════════════════════════════════════════════╗" text_color $BYELLOW "║ IMPORTANT: SQL Database Selection ║" text_color $BYELLOW "╚════════════════════════════════════════════════════════════════╝" text_color $YELLOW " To avoid potential migration issues, please select the SQL type" text_color $YELLOW " you are currently using. For first installations, both choices" text_color $YELLOW " are valid. If unsure, select option [1]." echo "" while true; do echo -e "${CYAN}Available SQL database options:${RESET}" echo -e " [1] ${GREEN}MySQL${RESET} - Default choice" echo -e " [2] ${GREEN}MariaDB${RESET} - Alternative option" echo -e " [E] ${RED}Exit${RESET} - Cancel installation" echo "" read -p "▶ Your choice: " choice case "$choice" in [2]* ) text_color $GREEN "✅ MariaDB selected. Proceeding with installation." sql=mariadb perform_step apt-get install curl software-properties-common dirmngr ca-certificates apt-transport-https -y "Installing mariadb dependencies" curl -LsS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | sudo bash -s -- --mariadb-server-version=10.8 perform_step apt-get install mariadb-server -y "Installing mariadb-server" break;; [Ee]* ) text_color $RED "❌ Installer stopped by user"; exit;; * ) text_color $GREEN "✅ MySQL selected. Proceeding with installation." sql=mysql mysql_native_password=" WITH mysql_native_password" perform_step apt-get install tcllib mysql-server -y "Installing mysql-server" break;; esac done #check old sql password OUTPUT=$($sql -u root -e "status;" 2>&1) if [[ $? -ne 0 ]]; then while true; do read -s -p "Enter your old sql password: " oldpassword echo echo -n "Password check: " OUTPUT=$(MYSQL_PWD=$oldpassword $sql -u root -e "status;" 2>&1) if [[ $? -ne 0 ]]; then text_color $YELLOW"ERROR - The sql repository password provided does not match your sql password. Please try again." else text_color $GREEN "OK" break fi done fi #check operationaldb if [[ -d "/var/lib/mysql/operationaldb/" ]]; then read -p "Old operationaldb repository detected. Would you like to overwrite it ? (Default: No) [Y]es [N]o [E]xit " choice case "$choice" in [yY]* ) perform_step $(MYSQL_PWD=$oldpassword $sql -u root -e "DROP DATABASE IF EXISTS operationaldb;") "Overwritting slq repository";; [eE]* ) text_color $RED"Installer stopped by user"; exit;; * ) text_color $GREEN"Keeping previous sql repository"; NEW_DB=FALSE;; esac fi #check sql new password read -p "Would you like to change your sql password or add one ? (Default: Yes) [Y]es [N]o [E]xit " choice case "$choice" in [nN]* ) text_color $GREEN"Keeping previous sql password"; password=$oldpassword;; [eE]* ) text_color $RED"Installer stopped by user"; exit;; * ) while true; do read -s -p "Enter your new sql password: " password echo read -s -p "Please confirm your new sql password: " password2 echo [[ $password = $password2 ]] && break text_color $YELLOW "Password entered do not match. Please try again." done perform_step $(MYSQL_PWD=$oldpassword $sql -u root -e "ALTER USER 'root'@'localhost' IDENTIFIED$mysql_native_password BY '$password';") "Changing sql password";; esac perform_step $(echo "REPOSITORY_PASSWORD=$password" >> $OTNODE_DIR/.env) "Adding sql password to .env" if [[ $NEW_DB != FALSE ]]; then perform_step $(MYSQL_PWD=$password $sql -u root -e "CREATE DATABASE operationaldb /*\!40100 DEFAULT CHARACTER SET utf8 */;") "Creating new sql repository" fi if [[ $sql = mysql ]]; then perform_step sed -i 's|max_binlog_size|#max_binlog_size|' /etc/mysql/mysql.conf.d/mysqld.cnf "Setting max log size" perform_step $(echo -e "disable_log_bin\nwait_timeout = 31536000\ninteractive_timeout = 31536000" >> /etc/mysql/mysql.conf.d/mysqld.cnf) "Adding disable_log_bin, wait_timeout, interactive_timeout to sql config" fi if [[ $sql = mariadb ]]; then perform_step sed -i 's|max_binlog_size|#max_binlog_size|' /etc/mysql/mariadb.conf.d/50-server.cnf "Setting max log size" perform_step $(echo -e "disable_log_bin\nwait_timeout = 31536000\ninteractive_timeout = 31536000" >> /etc/mysql/mariadb.conf.d/50-server.cnf) "Adding disable_log_bin, wait_timeout, interactive_timeout to sql config" fi perform_step systemctl restart $sql "Restarting $sql" } # Define wallet configuration functions request_operational_wallet_keys() { WALLET_ADDRESSES=() WALLET_PRIVATE_KEYS=() echo "" echo -e "${CYAN}┌─────────────────────────────────────────────────────────┐${RESET}" echo -e "${CYAN}│ OPERATIONAL WALLET CONFIGURATION │${RESET}" echo -e "${CYAN}└─────────────────────────────────────────────────────────┘${RESET}" echo "" echo -e "${YELLOW}You'll now be asked to input your operational wallets for $1.${RESET}" echo -e "${YELLOW}(Press ENTER without typing to skip/finish adding wallets)${RESET}" echo "" wallet_no=1 while true; do echo -e "${CYAN}=== Wallet #$wallet_no Configuration ===${RESET}" read -p "▶ Address for $1 operational wallet #$wallet_no: " address [[ -z $address ]] && break text_color $GREEN "✅ EVM operational wallet address for $blockchain wallet #$wallet_no: $address" read -s -p "▶ Private key for $1 operational wallet #$wallet_no: " private_key echo # Add newline after hidden input [[ -z $private_key ]] && break text_color $GREEN "✅ EVM operational wallet private key stored successfully!" WALLET_ADDRESSES+=($address) WALLET_PRIVATE_KEYS+=($private_key) wallet_no=$((wallet_no + 1)) echo "" done OP_WALLET_KEYS_JSON=$(jq -n ' [ $ARGS.positional as $args | ($args | length / 2) as $upto | range(0; $upto) as $start | [{ evmAddress: $args[$start], privateKey: $args[$start + $upto] }] ] | add ' --args "${WALLET_ADDRESSES[@]}" "${WALLET_PRIVATE_KEYS[@]}") echo -e "${GREEN}✅ Wallet configuration completed${RESET}" } # Enhanced validate_operator_fees function with better UI validate_operator_fees() { local blockchain=$1 echo "" echo -e "${CYAN}┌─────────────────────────────────────────────┐${RESET}" echo -e "${CYAN}│ OPERATOR FEE CONFIGURATION │${RESET}" echo -e "${CYAN}└─────────────────────────────────────────────┘${RESET}" echo "" echo -e "${YELLOW}The operator fee is the percentage of rewards you will receive (0-100).${RESET}" while true; do read -p "▶ Enter operator fee for $blockchain: " OPERATOR_FEE if [[ "$OPERATOR_FEE" =~ ^[0-9]+$ ]] && [ "$OPERATOR_FEE" -ge 0 ] && [ "$OPERATOR_FEE" -le 100 ]; then print_color $GREEN "✅ Operator fee for $blockchain set to: $OPERATOR_FEE%" break else print_color $RED "⚠️ Invalid input. Please enter a number between 0 and 100." fi done } # Define color codes RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' MAGENTA='\033[0;35m' CYAN='\033[0;36m' RESET='\033[0m' # Function to print colored text print_color() { local color=$1 local text=$2 echo -e "${color}${text}${RESET}" } install_node() { # Change directory to ot-node/current cd $OTNODE_DIR # Set blockchain options based on the selected environment if [ "$SELECTED_NODE_ENV" == "mainnet" ]; then blockchain_options=("Neuroweb" "Gnosis" "Base") otp_blockchain_id=2043 gnosis_blockchain_id=100 base_blockchain_id=8453 else blockchain_options=("Neuroweb" "Gnosis" "Base-Sepolia") otp_blockchain_id=20430 gnosis_blockchain_id=10200 base_blockchain_id=84532 fi # Ask user which blockchains to connect to selected_blockchains=() checkbox_states=() for _ in "${blockchain_options[@]}"; do checkbox_states+=("[ ]") done while true; do clear # Clear the screen for a cleaner display echo "" echo -e "${CYAN}┌─────────────────────────────────────────────┐${RESET}" echo -e "${CYAN}│ BLOCKCHAIN SELECTION │${RESET}" echo -e "${CYAN}└─────────────────────────────────────────────┘${RESET}" echo "" echo -e "Please select the blockchains you want to connect your node to:" echo "" for i in "${!blockchain_options[@]}"; do echo -e " ${checkbox_states[$i]} $((i+1)). ${blockchain_options[$i]}" done echo -e " [ ] $((${#blockchain_options[@]}+1)). All Blockchains" echo "" echo -e "${YELLOW}Enter the number to toggle selection, or 'd' to finish.${RESET}" echo "" # Use read -n 1 to read a single character without requiring Enter read -n 1 -p "▶ Your choice: " choice echo # Add a newline after the selection if [[ "$choice" == "d" ]]; then if [ ${#selected_blockchains[@]} -eq 0 ]; then echo "" print_color $RED "You must select at least one blockchain. Please try again." read -n 1 -p "Press any key to continue..." continue else break fi elif [[ "$choice" =~ ^[1-${#blockchain_options[@]}]$ ]]; then index=$((choice-1)) if [[ "${checkbox_states[$index]}" == "[ ]" ]]; then checkbox_states[$index]="[x]" selected_blockchains+=("${blockchain_options[$index]}") else checkbox_states[$index]="[ ]" selected_blockchains=(${selected_blockchains[@]/${blockchain_options[$index]}}) fi elif [[ "$choice" == "$((${#blockchain_options[@]}+1))" ]]; then if [[ "${checkbox_states[-1]}" == "[ ]" ]]; then for i in "${!checkbox_states[@]}"; do checkbox_states[$i]="[x]" done selected_blockchains=("${blockchain_options[@]}") else for i in "${!checkbox_states[@]}"; do checkbox_states[$i]="[ ]" done selected_blockchains=() fi else echo "" print_color $RED "Invalid choice. Please enter a number between 1 and $((${#blockchain_options[@]}+1))." read -n 1 -p "Press any key to continue..." fi done print_color $GREEN "✅ Final blockchain selection: ${selected_blockchains[*]}" CONFIG_DIR=$OTNODE_DIR/.. perform_step touch $CONFIG_DIR/.origintrail_noderc "Configuring node config file" perform_step $(jq --null-input '{"logLevel": "trace", "auth": {"ipWhitelist": ["::1", "127.0.0.1"]}, "modules": {"blockchain": {"implementation": {}}}}' > $CONFIG_DIR/.origintrail_noderc) "Adding initial config to node config file" perform_step $(jq --arg tripleStore "$tripleStore" --arg tripleStoreUrl "$tripleStoreUrl" '.modules.tripleStore.implementation[$tripleStore] |= { "enabled": "true", "config": { "repositories": { "dkg": { "url": $tripleStoreUrl, "name": "dkg", "username": "admin", "password": "" }, "privateCurrent": { "url": $tripleStoreUrl, "name": "private-current", "username": "admin", "password": "" }, "publicCurrent": { "url": $tripleStoreUrl, "name": "public-current", "username": "admin", "password": "" } } } } + .' $CONFIG_DIR/.origintrail_noderc > $CONFIG_DIR/origintrail_noderc_tmp) "Adding triple store config to node config file" perform_step mv $CONFIG_DIR/origintrail_noderc_tmp $CONFIG_DIR/.origintrail_noderc "Finalizing initial node config file" # Function to configure a blockchain configure_blockchain() { local blockchain=$1 local blockchain_id=$2 request_operational_wallet_keys $blockchain local EVM_OP_WALLET_KEYS=$OP_WALLET_KEYS_JSON read -p "Enter your EVM management wallet address for $blockchain: " EVM_MANAGEMENT_WALLET text_color $GREEN "EVM management wallet address for $blockchain: $EVM_MANAGEMENT_WALLET" read -p "$(print_color $YELLOW "Enter your profile node name : ")" NODE_NAME print_color $GREEN "✅ Profile node name : $NODE_NAME" validate_operator_fees $blockchain local RPC_ENDPOINT="" if [ "$blockchain" == "gnosis" ] || [ "$blockchain" == "base" ]; then read -p "Enter your $blockchain RPC endpoint: " RPC_ENDPOINT text_color $GREEN "$blockchain RPC endpoint: $RPC_ENDPOINT" # Store RPC endpoint in a global associative array for later use declare -g "${blockchain}_rpc_endpoint=$RPC_ENDPOINT" fi local jq_filter=$(cat < $CONFIG_DIR/origintrail_noderc_tmp mv $CONFIG_DIR/origintrail_noderc_tmp $CONFIG_DIR/.origintrail_noderc } # Function to configure blockchain events services configure_blockchain_events_services() { local blockchain=$1 local blockchain_id=$2 print_color $CYAN "🔧 Configuring Blockchain Events Service for $blockchain (ID: $blockchain_id)..." # Get previously stored RPC endpoint instead of asking again local stored_rpc_var="${blockchain}_rpc_endpoint" local RPC_ENDPOINT="${!stored_rpc_var}" # If no stored RPC endpoint is found (which shouldn't happen), ask for it if [ -z "$RPC_ENDPOINT" ]; then read -p "$(print_color $YELLOW "Enter your RPC endpoint for $blockchain: ")" RPC_ENDPOINT else print_color $GREEN "✅ Using previously provided RPC endpoint for $blockchain" fi print_color $GREEN "✅ RPC endpoint: $RPC_ENDPOINT" # Correct `jq` usage to safely initialize and update the configuration local jq_filter=' .modules |= (if .blockchainEvents == null then .blockchainEvents = {implementation: {}} else . end) | .modules.blockchainEvents.implementation |= (if .["ot-ethers"] == null then .["ot-ethers"] = {enabled: false, config: {}} else . end) | .modules.blockchainEvents.implementation["ot-ethers"].enabled = true | .modules.blockchainEvents.implementation["ot-ethers"].config |= (if .blockchains == null then .blockchains = [] else . end) | .modules.blockchainEvents.implementation["ot-ethers"].config |= (if .rpcEndpoints == null then .rpcEndpoints = {} else . end) | .modules.blockchainEvents.implementation["ot-ethers"].config.blockchains += ["'"$blockchain:$blockchain_id"'"] | .modules.blockchainEvents.implementation["ot-ethers"].config.rpcEndpoints["'"$blockchain:$blockchain_id"'"] = ["'"$RPC_ENDPOINT"'"] ' # Apply the configuration changes if jq "$jq_filter" "$CONFIG_DIR/.origintrail_noderc" > "$CONFIG_DIR/.origintrail_noderc_tmp"; then mv "$CONFIG_DIR/.origintrail_noderc_tmp" "$CONFIG_DIR/.origintrail_noderc" chmod 600 "$CONFIG_DIR/.origintrail_noderc" print_color $GREEN "✅ Successfully configured Blockchain Events Service for $blockchain (ID: $blockchain_id)." else print_color $RED "❌ Failed to configure Blockchain Events Service for $blockchain (ID: $blockchain_id)." exit 1 fi } # Configure blockchain events service for Base Sepolia for blockchain in "${selected_blockchains[@]}"; do case "$blockchain" in "Neuroweb") configure_blockchain "otp" $otp_blockchain_id ;; "Gnosis") configure_blockchain "gnosis" $gnosis_blockchain_id ;; "Base" | "Base-Sepolia") configure_blockchain "base" $base_blockchain_id ;; esac done for blockchain in "${selected_blockchains[@]}"; do case "$blockchain" in "Gnosis") configure_blockchain_events_services "gnosis" $gnosis_blockchain_id ;; "Base" | "Base-Sepolia") configure_blockchain_events_services "base" $base_blockchain_id ;; esac done # Now execute npm install after configuring wallets print_color $CYAN "📦 Installing npm packages..." perform_step npm ci --omit=dev --ignore-scripts "Executing npm install" print_color $CYAN "🔧 Setting up system service..." perform_step cp $OTNODE_DIR/installer/data/otnode.service /lib/systemd/system/ "Copying otnode service file" print_color $CYAN "🚀 Starting OriginTrail node..." systemctl daemon-reload perform_step systemctl enable otnode "Enabling otnode" perform_step systemctl start otnode "Starting otnode" perform_step systemctl status otnode "Checking otnode status" print_color $GREEN "✅ OriginTrail node installation complete!" } #For Arch Linux installation if [[ ! -z $(grep "arch" "/etc/os-release") ]]; then source <(curl -s https://raw.githubusercontent.com/OriginTrail/ot-node/v8/develop/installer/data/archlinux) fi # Perform checks header_color "Checking Ubuntu version" check_ubuntu_version header_color "Checking root privilege" check_root #### INSTALLATION START #### clear cd /root header_color $BGREEN"Welcome to the OriginTrail Installer. Please sit back while the installer runs. " header_color $BGREEN"Installing OriginTrail node pre-requisites..." install_prereqs header_color $BGREEN"Preparing OriginTrail node directory..." if [[ -d "$OTNODE_DIR" ]]; then read -p "Previous ot-node directory detected. Would you like to overwrite it? (Default: Yes) [Y]es [N]o [E]xit " choice case "$choice" in [nN]* ) text_color $GREEN"Keeping previous ot-node directory.";; [eE]* ) text_color $RED"Installer stopped by user"; exit;; * ) text_color $GREEN"Reconfiguring ot-node directory."; systemctl is-active --quiet otnode && systemctl stop otnode; perform_step rm -rf $OTNODE_DIR "Deleting $OTNODE_DIR"; install_directory;; esac else install_directory fi OTNODE_DIR=$OTNODE_DIR/current header_color $BGREEN"Installing Triplestore (Graph Database)..." echo "" echo -e "${CYAN}┌─────────────────────────────────────────────┐${RESET}" echo -e "${CYAN}│ TRIPLESTORE SELECTION │${RESET}" echo -e "${CYAN}└─────────────────────────────────────────────┘${RESET}" echo "" echo -e "Please select the database you would like to use for your graph data:" echo -e " [1] ${GREEN}Blazegraph${RESET} - Default choice, recommended for most users" echo -e " [2] ${GREEN}Fuseki${RESET} - Alternative option" echo -e " [E] ${RED}Exit${RESET} - Cancel installation" echo "" read -p "▶ Your choice: " choice case "$choice" in [2] ) text_color $GREEN "✅ Fuseki selected. Proceeding with installation."; tripleStore=ot-fuseki; tripleStoreUrl="http://localhost:3030";; [Ee] ) text_color $RED "❌ Installer stopped by user"; exit;; * ) text_color $GREEN "✅ Blazegraph selected. Proceeding with installation."; tripleStore=ot-blazegraph; tripleStoreUrl="http://localhost:9999";; esac if [[ $tripleStore = "ot-fuseki" ]]; then if [[ -d "$OTNODE_DIR/../fuseki" ]]; then read -p "Previously installed Fuseki triplestore detected. Would you like to overwrite it? (Default: Yes) [Y]es [N]o [E]xit " choice case "$choice" in [nN]* ) text_color $GREEN"Keeping previous Fuseki installation.";; [eE]* ) text_color $RED"Installer stopped by user"; exit;; * ) text_color $GREEN"Reinstalling Fuseki."; perform_step rm -rf $OTNODE_DIR/../fuseki "Removing previous Fuseki installation"; install_fuseki;; esac else install_fuseki fi fi if [[ $tripleStore = "ot-blazegraph" ]]; then if [[ -f "blazegraph.jar" ]]; then read -p "Previously installed Blazegraph triplestore detected. Would you like to overwrite it? (Default: Yes) [Y]es [N]o [E]xit " choice case "$choice" in [nN]* ) text_color $GREEN"Keeping old Blazegraph Installation.";; [eE]* ) text_color $RED"Installer stopped by user"; exit;; * ) text_color $GREEN"Reinstalling Blazegraph."; perform_step rm -rf blazegraph* "Removing previous Blazegraph installation"; install_blazegraph;; esac else install_blazegraph fi fi header_color $BGREEN"Installing SQL..." install_sql header_color $BGREEN"Configuring OriginTrail node..." install_node header_color $BGREEN"INSTALLATION COMPLETE!" # Create a more visually appealing summary echo -e "${GREEN}╔═══════════════════════════════════════════════════════════╗${RESET}" echo -e "${GREEN}║ ║${RESET}" echo -e "${GREEN}║ 🎉 OriginTrail Node Successfully Installed! 🎉 ║${RESET}" echo -e "${GREEN}║ ║${RESET}" echo -e "${GREEN}╚═══════════════════════════════════════════════════════════╝${RESET}" echo "" echo -e "${CYAN}📊 Node Information:${RESET}" echo -e " • Environment: ${YELLOW}$SELECTED_NODE_ENV${RESET}" echo -e " • Triple Store: ${YELLOW}$tripleStore${RESET}" echo -e " • SQL Database: ${YELLOW}$sql${RESET}" echo "" echo -e "${CYAN}📋 Node Management Commands:${RESET}" echo -e " • ${YELLOW}otnode-restart${RESET} - Restart the node service" echo -e " • ${YELLOW}otnode-stop${RESET} - Stop the node service" echo -e " • ${YELLOW}otnode-start${RESET} - Start the node service" echo -e " • ${YELLOW}otnode-logs${RESET} - View real-time node logs" echo -e " • ${YELLOW}otnode-config${RESET} - Edit node configuration" echo "" echo -e "${CYAN}💡 To start using these commands, run:${RESET}" echo -e " ${YELLOW}source ~/.bashrc${RESET}" echo "" echo -e "${CYAN}📜 Logs will be displayed below. Press ${BOLD}Ctrl+C${RESET}${CYAN} to exit the logs.${RESET}" echo -e "${CYAN} The node will continue running in the background.${RESET}" echo "" echo -e "${YELLOW}⚠️ If logs do not appear or the screen freezes, press Ctrl+C to exit${RESET}" echo -e "${YELLOW} and then reboot your server.${RESET}" echo "" read -p "▶ Press Enter to view logs..." systemctl restart systemd-journald journalctl -u otnode --output cat -fn 200 text_color $GREEN " New aliases added: otnode-restart otnode-stop otnode-start otnode-logs otnode-config To start using aliases, run: source ~/.bashrc " text_color $YELLOW"Logs will be displayed. Press ctrl+c to exit the logs. The node WILL stay running after you return to the command prompt. If the logs do not show and the screen hangs, press ctrl+c to exit the installation and reboot your server. " read -p "Press enter to continue..." ================================================ FILE: ot-node.js ================================================ import DeepExtend from 'deep-extend'; import rc from 'rc'; import EventEmitter from 'events'; import { createRequire } from 'module'; import { execSync } from 'child_process'; import DependencyInjection from './src/service/dependency-injection.js'; import Logger from './src/logger/logger.js'; import { MIN_NODE_VERSION, PARANET_ACCESS_POLICY } from './src/constants/constants.js'; import FileService from './src/service/file-service.js'; import OtnodeUpdateCommand from './src/commands/common/otnode-update-command.js'; import OtAutoUpdater from './src/modules/auto-updater/implementation/ot-auto-updater.js'; import MigrationExecutor from './src/migration/migration-executor.js'; const require = createRequire(import.meta.url); const { setTimeout } = require('timers/promises'); const pjson = require('./package.json'); const configjson = require('./config/config.json'); class OTNode { constructor(config) { this.initializeConfiguration(config); this.initializeLogger(); this.initializeFileService(); this.initializeAutoUpdaterModule(); this.checkNodeVersion(); // Set up process event listeners process.on('SIGINT', () => this.handleExit()); // Ctrl+C process.on('SIGTERM', () => this.handleExit()); // kill command or Docker stop } async start() { await this.checkForUpdate(); await this.removeUpdateFile(); await MigrationExecutor.executeTripleStoreUserConfigurationMigration( this.container, this.logger, this.config, ); await MigrationExecutor.executeRedisSetupMigration( this.container, this.logger, this.config, ); this.logger.info('██████╗ ██╗ ██╗ ██████╗ ██╗ ██╗ █████╗ '); this.logger.info('██╔══██╗██║ ██╔╝██╔════╝ ██║ ██║██╔══██╗'); this.logger.info('██║ ██║█████╔╝ ██║ ███╗ ██║ ██║╚█████╔╝'); this.logger.info('██║ ██║██╔═██╗ ██║ ██║ ╚██╗ ██╔╝██╔══██╗'); this.logger.info('██████╔╝██║ ██╗╚██████╔╝ ╚████╔╝ ╚█████╔╝'); this.logger.info('╚═════╝ ╚═╝ ╚═╝ ╚═════╝ ╚═══╝ ╚════╝ '); this.logger.info('======================================================'); this.logger.info(` OriginTrail Node v${pjson.version}`); this.logger.info('======================================================'); this.logger.info(`Node is running in ${process.env.NODE_ENV} environment`); await this.initializeDependencyContainer(); this.initializeEventEmitter(); await this.initializeModules(); this.initializeBlockchainEventsService(); await this.initializeShardingTableService(); await this.initializeParanets(); await this.createProfiles(); await this.initializeCommandExecutor(); await this.initializeRouters(); await this.startNetworkModule(); this.resumeCommandExecutor(); await this.initializeProofing(); await this.initializeClaimRewards(); await this.initializeSyncService(); this.logger.info('Node is up and running!'); } checkNodeVersion() { const nodeMajorVersion = process.versions.node.split('.')[0]; this.logger.warn('======================================================'); this.logger.warn(`Using node.js version: ${process.versions.node}`); if (nodeMajorVersion < MIN_NODE_VERSION) { this.logger.warn( `This node was tested with node.js version 16. To make sure that your node is running properly please update your node version!`, ); } this.logger.warn('======================================================'); } initializeLogger() { this.logger = new Logger(this.config.logging.defaultLevel); } initializeFileService() { this.fileService = new FileService({ config: this.config, logger: this.logger }); } initializeAutoUpdaterModule() { this.autoUpdaterModuleManager = new OtAutoUpdater(); this.autoUpdaterModuleManager.initialize( this.config.modules.autoUpdater.implementation['ot-auto-updater'].config, this.logger, ); } initializeConfiguration(userConfig) { const defaultConfig = JSON.parse(JSON.stringify(configjson[process.env.NODE_ENV])); if (userConfig) { this.config = DeepExtend(defaultConfig, userConfig); } else { this.config = rc(pjson.name, defaultConfig); } if (!this.config.configFilename) { // set default user configuration filename this.config.configFilename = '.origintrail_noderc'; } } async initializeDependencyContainer() { this.container = await DependencyInjection.initialize(); DependencyInjection.registerValue(this.container, 'config', this.config); DependencyInjection.registerValue(this.container, 'logger', this.logger); this.logger.info('Dependency injection module is initialized'); } async initializeModules() { const initializationPromises = []; for (const moduleName in this.config.modules) { const moduleManagerName = `${moduleName}ModuleManager`; const moduleManager = this.container.resolve(moduleManagerName); initializationPromises.push(moduleManager.initialize()); } try { await Promise.all(initializationPromises); this.logger.info(`All modules initialized!`); } catch (e) { this.logger.error(`Module initialization failed. Error message: ${e.message}`); this.stop(1); } } initializeEventEmitter() { const eventEmitter = new EventEmitter(); DependencyInjection.registerValue(this.container, 'eventEmitter', eventEmitter); this.logger.info('Event emitter initialized'); } async initializeRouters() { try { this.logger.info('Initializing http api and rpc router'); const routerNames = ['httpApiRouter', 'rpcRouter']; await Promise.all( routerNames.map(async (routerName) => { const router = this.container.resolve(routerName); try { await router.initialize(); } catch (error) { this.logger.error( `${routerName} initialization failed. Error message: ${error.message}, ${error.stackTrace}`, ); this.stop(1); } }), ); this.logger.info('Routers initialized successfully'); } catch (error) { this.logger.error( `Failed to initialize routers: ${error.message}, ${error.stackTrace}`, ); this.stop(1); } } async createProfiles() { const cryptoService = this.container.resolve('cryptoService'); const blockchainModuleManager = this.container.resolve('blockchainModuleManager'); const networkModuleManager = this.container.resolve('networkModuleManager'); const peerId = networkModuleManager.getPeerId().toB58String(); const createProfilesPromises = blockchainModuleManager .getImplementationNames() .map(async (blockchain) => { try { const identityExists = await blockchainModuleManager.identityIdExists( blockchain, ); if (!identityExists) { this.logger.info(`Creating profile on network: ${blockchain}`); await blockchainModuleManager.createProfile(blockchain, peerId); if ( process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test' ) { const blockchainConfig = blockchainModuleManager.getModuleConfiguration(blockchain); execSync( `npm run set-ask -- --rpcEndpoint=${ blockchainConfig.rpcEndpoints[0] } --ask=${1 + Math.random() * 0.5} --privateKey=${ blockchainConfig.operationalWallets[0].privateKey } --hubContractAddress=${blockchainConfig.hubContractAddress}`, { stdio: 'inherit' }, ); await setTimeout(10000 + Math.random() * 10000); execSync( `npm run set-stake -- --rpcEndpoint=${blockchainConfig.rpcEndpoints[0]} --stake=${blockchainConfig.initialStakeAmount} --operationalWalletPrivateKey=${blockchainConfig.operationalWallets[0].privateKey} --managementWalletPrivateKey=${blockchainConfig.evmManagementWalletPrivateKey} --hubContractAddress=${blockchainConfig.hubContractAddress}`, { stdio: 'inherit' }, ); } } const identityId = await blockchainModuleManager.getIdentityId(blockchain); this.logger.info(`Identity ID: ${identityId}`); if (identityExists) { const onChainNodeId = await blockchainModuleManager.getNodeId( blockchain, identityId, ); const onChainPeerId = cryptoService.convertHexToAscii(onChainNodeId); if (peerId !== onChainPeerId) { this.logger.warn( `Local peer id: ${peerId} doesn't match on chain peer id: ${onChainPeerId} for blockchain: ${blockchain}, identity id: ${identityId}.`, ); blockchainModuleManager.removeImplementation(blockchain); } } } catch (error) { this.logger.warn( `Unable to create ${blockchain} blockchain profile. Removing implementation. Error: ${error.message}`, ); blockchainModuleManager.removeImplementation(blockchain); } }); await Promise.all(createProfilesPromises); if (!blockchainModuleManager.getImplementationNames().length) { this.logger.error(`Unable to create blockchain profiles. OT-node shutting down...`); this.stop(1); } } async initializeCommandExecutor() { try { const commandExecutor = this.container.resolve('commandExecutor'); await commandExecutor.pauseCommandExecutor(); await commandExecutor.addDefaultCommands(); // commandExecutor // .replayOldCommands() // .then(() => this.logger.info('Finished replaying old commands')); } catch (e) { this.logger.error( `Command executor initialization failed. Error message: ${e.message}`, ); this.stop(1); } } resumeCommandExecutor() { try { const commandExecutor = this.container.resolve('commandExecutor'); commandExecutor.resumeCommandExecutor(); } catch (e) { this.logger.error( `Unable to resume command executor queue. Error message: ${e.message}`, ); this.stop(1); } } async startNetworkModule() { const networkModuleManager = this.container.resolve('networkModuleManager'); await networkModuleManager.start(); } async initializeShardingTableService() { try { const shardingTableService = this.container.resolve('shardingTableService'); await shardingTableService.initialize(); this.logger.info('Sharding Table Service initialized successfully'); } catch (error) { this.logger.error( `Unable to initialize sharding table service. Error message: ${error.message} OT-node shutting down...`, ); this.stop(1); } } initializeBlockchainEventsService() { try { const blockchainEventsService = this.container.resolve('blockchainEventsService'); blockchainEventsService.initializeBlockchainEventsServices(); this.logger.info('Blockchain Events Service initialized successfully'); } catch (error) { this.logger.error( `Unable to initialize Blockchain Events Service. Error message: ${error.message} OT-node shutting down...`, ); this.stop(1); } } async removeUpdateFile() { const updateFilePath = this.fileService.getUpdateFilePath(); await this.fileService.removeFile(updateFilePath).catch((error) => { this.logger.warn(`Unable to remove update file. Error: ${error}`); }); this.config.otNodeUpdated = true; } async checkForUpdate() { const autoUpdaterCommand = new OtnodeUpdateCommand({ logger: this.logger, config: this.config, fileService: this.fileService, autoUpdaterModuleManager: this.autoUpdaterModuleManager, }); await autoUpdaterCommand.execute(); } async initializeParanets() { const blockchainModuleManager = this.container.resolve('blockchainModuleManager'); const tripleStoreService = this.container.resolve('tripleStoreService'); const paranetService = this.container.resolve('paranetService'); const ualService = this.container.resolve('ualService'); const validParanets = []; const syncParanets = this.config.assetSync && this.config.assetSync.syncParanets ? this.config.assetSync.syncParanets : []; for (const paranetUAL of syncParanets) { if (!ualService.isUAL(paranetUAL)) { this.logger.warn( `Unable to initialize Paranet with id ${paranetUAL} because of invalid UAL format`, ); continue; } const { blockchain, contract, knowledgeCollectionId, knowledgeAssetId } = ualService.resolveUAL(paranetUAL); if (!knowledgeAssetId) { this.logger.warn( `Invalid paranet UAL: ${paranetUAL} . Knowledge asset token id is required!`, ); continue; } if (!blockchainModuleManager.getImplementationNames().includes(blockchain)) { this.logger.warn( `Unable to initialize Paranet with id ${paranetUAL} because of unsupported blockchain implementation`, ); continue; } const paranetId = paranetService.constructParanetId( contract, knowledgeCollectionId, knowledgeAssetId, ); // eslint-disable-next-line no-await-in-loop const paranetExists = await blockchainModuleManager.paranetExists( blockchain, paranetId, ); if (!paranetExists) { this.logger.warn( `Unable to initialize Paranet with id ${paranetUAL} because it doesn't exist`, ); continue; } // eslint-disable-next-line no-await-in-loop const nodesAccessPolicy = await blockchainModuleManager.getNodesAccessPolicy( blockchain, paranetId, ); if (nodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { // eslint-disable-next-line no-await-in-loop const identityId = await blockchainModuleManager.getIdentityId(blockchain); // eslint-disable-next-line no-await-in-loop const isPermissionedNode = await blockchainModuleManager.isPermissionedNode( blockchain, paranetId, identityId, ); if (!isPermissionedNode) { this.logger.warn( `Unable to initialize Paranet with id ${paranetUAL} because node with id ${identityId} is not a permissioned node`, ); continue; } } validParanets.push(paranetUAL); // eslint-disable-next-line no-await-in-loop await paranetService.initializeParanetRecord(blockchain, paranetId); } this.config.assetSync.syncParanets = validParanets; tripleStoreService.initializeRepositories(); } async initializeProofing() { const proofingService = this.container.resolve('proofingService'); await proofingService.initialize(); } async initializeClaimRewards() { const claimRewardsService = this.container.resolve('claimRewardsService'); await claimRewardsService.initialize(); } async initializeSyncService() { const syncService = this.container.resolve('syncService'); await syncService.initialize(); } stop(code = 0) { this.logger.info('Stopping node...'); process.exit(code); } async handleExit() { this.logger.info('SIGINT or SIGTERM received. Shutting down...'); const commandExecutor = this.container.resolve('commandExecutor'); await commandExecutor.commandExecutorShutdown(); process.exit(0); } } export default OTNode; ================================================ FILE: package.json ================================================ { "name": "origintrail_node", "version": "8.2.6", "description": "OTNode V8", "main": "index.js", "type": "module", "scripts": { "compile-contracts": "npm explore dkg-evm-module -- npm run compile", "bootstrap-node": "node index.js tools/local-network-setup/.bootstrap_origintrail_noderc", "start": "node index.js", "prepare": "husky install", "lint-staged": "lint-staged", "create-account-mapping-signature": "node tools/ot-parachain-account-mapping/create-account-mapping-signature.js ", "start:local_blockchain": "npm explore dkg-evm-module -- npm run dev -- --port", "start:local_blockchain:v1": "npm explore dkg-evm-module -- npm run dev:v1 -- --port", "start:local_blockchain:v2": "npm explore dkg-evm-module -- npm run dev:v2 -- --port", "kill:local_blockchain": "npx kill-port --port", "test:bdd": "bash test/bdd/run-bdd.sh", "test:unit": "nyc --all mocha --exit $(find test/unit -name '*.js')", "test:modules": "nyc --all mocha --exit $(find test/unit/modules -name '*.js')", "test:bdd:release": "cucumber-js --tags=@release --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ --import test/bdd/steps/", "test:bdd:publish-errors": "cucumber-js --tags=@publish-errors --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ --import test/bdd/steps/", "test:bdd:update-errors": "cucumber-js --tags=@update-errors --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ --import test/bdd/steps/", "test:bdd:get-errors": "cucumber-js --tags=@get-errors --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ --import test/bdd/steps/", "lint": "eslint .", "set-ask": "node scripts/set-ask.js", "set-stake": "node scripts/set-stake.js", "set-operator-fee": "node scripts/set-operator-fee.js" }, "repository": { "type": "git", "url": "git+https://github.com/OriginTrail/ot-node.git" }, "keywords": [ "ot-node", "v8" ], "author": "TraceLabs", "license": "ISC", "bugs": { "url": "https://github.com/OriginTrail/ot-node/issues" }, "homepage": "https://origintrail.io/", "engines": { "node": ">=16.0.0", "npm": ">=8.0.0" }, "devDependencies": { "@cucumber/cucumber": "^11.2.0", "chai": "^4.3.6", "concurrently": "^9.1.2", "d3": "^7.8.5", "d3-node": "^3.0.0", "dkg.js": "^8.0.8", "eslint": "^8.23.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-prettier": "^8.5.0", "hardhat": "^2.22.19", "husky": "^8.0.1", "lint-staged": "^13.0.3", "mocha": "^10.0.0", "nyc": "^15.1.0", "prettier": "^2.7.1", "rollup": "^4.40.0", "sharp": "^0.32.6", "sinon": "^14.0.0", "slugify": "^1.6.5" }, "dependencies": { "@comunica/query-sparql": "^4.0.2", "@ethersproject/bytes": "^5.7.0", "@ethersproject/hash": "^5.7.0", "@ethersproject/wallet": "^5.7.0", "@polkadot/api": "^9.3.2", "@polkadot/keyring": "^10.1.7", "@polkadot/util": "^10.1.7", "@polkadot/util-crypto": "^10.1.7", "@questdb/nodejs-client": "^3.0.0", "app-root-path": "^3.1.0", "assertion-tools": "^8.0.6", "async": "^3.2.4", "async-mutex": "^0.3.2", "awilix": "^7.0.3", "axios": "^1.6.0", "bullmq": "^5.56.1", "cors": "^2.8.5", "deep-extend": "^0.6.0", "dkg-evm-module": "git+https://github.com/OriginTrail/dkg-evm-module.git#main", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", "express-fileupload": "^1.4.0", "express-rate-limit": "^6.5.2", "fs-extra": "^10.1.0", "graphdb": "^2.0.2", "ip": "^1.1.8", "it-length-prefixed": "^5.0.3", "it-map": "^1.0.6", "it-pipe": "^1.1.0", "jsonld": "^8.1.0", "jsonschema": "^1.4.1", "jsonwebtoken": "^9.0.0", "libp2p": "^0.32.4", "libp2p-bootstrap": "^0.13.0", "libp2p-kad-dht": "^0.24.2", "libp2p-mplex": "^0.10.7", "libp2p-noise": "^4.0.0", "libp2p-tcp": "^0.17.2", "mcl-wasm": "^1.7.0", "minimist": "^1.2.7", "ms": "^2.1.3", "mysql2": "^3.3.0", "peer-id": "^0.15.3", "pino": "^9.7.0", "pino-pretty": "^13.0.0", "rc": "^1.2.8", "rolling-rate-limiter": "^0.2.13", "semver": "^7.5.2", "sequelize": "^6.29.0", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", "timeout-abort-controller": "^3.0.0", "toobusy-js": "^0.5.1", "uint8arrays": "^3.1.0", "umzug": "^3.2.1", "unzipper": "^0.10.11", "uuid": "^8.3.2" } } ================================================ FILE: scripts/copy-assertions.js ================================================ import 'dotenv/config'; import fs from 'fs-extra'; import rc from 'rc'; import appRootPath from 'app-root-path'; import path from 'path'; import { TRIPLE_STORE_REPOSITORIES, SCHEMA_CONTEXT } from '../src/constants/constants.js'; import TripleStoreModuleManager from '../src/modules/triple-store/triple-store-module-manager.js'; import DataService from '../src/service/data-service.js'; import Logger from '../src/logger/logger.js'; const { readFile } = fs; const generalConfig = JSON.parse(await readFile(path.join(appRootPath.path, 'config/config.json'))); const pjson = JSON.parse(await readFile(path.join(appRootPath.path, 'package.json'))); const defaultConfig = generalConfig[process.env.NODE_ENV]; const config = rc(pjson.name, defaultConfig); const logger = new Logger(config.loglevel); const tripleStoreModuleManager = new TripleStoreModuleManager({ config, logger }); await tripleStoreModuleManager.initialize(); const dataService = new DataService({ config, logger }); const repositoryImplementations = {}; for (const implementationName of tripleStoreModuleManager.getImplementationNames()) { for (const repository in tripleStoreModuleManager.getImplementation(implementationName).module .repositories) { repositoryImplementations[repository] = implementationName; } } const fromRepository = TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT; const fromImplementation = repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT]; const fromRepositoryName = tripleStoreModuleManager.getImplementation(fromImplementation).module.repositories[ fromRepository ].name; const toRepository = TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT; const toImplementation = repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT]; const toRepositoryName = tripleStoreModuleManager.getImplementation(toImplementation).module.repositories[toRepository] .name; async function getAssertions(implementation, repository) { const graphs = await tripleStoreModuleManager.select( implementation, repository, `SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o } }`, ); return (graphs ?? []).filter(({ g }) => g.startsWith('assertion:')).map(({ g }) => g); } function logPercentage(index, max) { const previousPercentage = (Math.max(0, index - 1) / max) * 100; const currentPercentage = (index / max) * 100; if (Math.floor(currentPercentage) - Math.floor(previousPercentage) < 1) return; logger.debug(`Migration at ${Math.floor(currentPercentage * 10) / 10}%`); } let toRepositoryAssertions = await getAssertions(toImplementation, toRepository); logger.info( `${toRepositoryAssertions.length} assertions found in ${toRepository} repository before migration`, ); logger.info( `Starting to copy assertions from ${fromImplementation} repository ${fromRepository} with name ${fromRepositoryName} to repository ${toImplementation} repository ${toRepository} with name ${toRepositoryName}`, ); const fromRepositoryAssertions = await getAssertions(fromImplementation, fromRepository); logger.info(`${fromRepositoryAssertions.length} assertions found in ${fromRepository}`); let completed = 0; const copyAssertion = async (g) => { if (!toRepositoryAssertions.includes(g)) { let nquads; try { nquads = await tripleStoreModuleManager.construct( fromImplementation, fromRepository, `PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } WHERE { { GRAPH <${g}> { ?s ?p ?o . } } }`, ); nquads = await dataService.toNQuads(nquads, 'application/n-quads'); } catch (error) { logger.error( `Error while getting assertion ${g.substring( 'assertion:'.length, )} from ${fromImplementation} repository ${fromRepository} with name ${fromRepositoryName}. Error: ${ error.message }`, ); process.exit(1); } try { await tripleStoreModuleManager.insertKnowledgeCollection( toImplementation, toRepository, g.substring('assertion:'.length), nquads.join('\n'), ); } catch (error) { logger.error( `Error while inserting assertion ${g.substring( 'assertion:'.length, )} with nquads: ${nquads} in ${toImplementation} repository ${toRepository} with name ${toRepositoryName}. Error: ${ error.message }`, ); process.exit(1); } } completed += 1; logPercentage(completed, fromRepositoryAssertions.length); }; const start = Date.now(); const concurrency = 10; let promises = []; for (let i = 0; i < fromRepositoryAssertions.length; i += 1) { promises.push(copyAssertion(fromRepositoryAssertions[i])); if (promises.length > concurrency) { // eslint-disable-next-line no-await-in-loop await Promise.all(promises); promises = []; } } await Promise.all(promises); const end = Date.now(); logger.info(`Migration completed! Lasted ${(end - start) / 1000} seconds.`); toRepositoryAssertions = await getAssertions(toImplementation, toRepository); logger.info( `${toRepositoryAssertions.length} assertions found in ${toRepository} repository after migration`, ); ================================================ FILE: scripts/set-ask.js ================================================ /* eslint-disable no-console */ import { ethers } from 'ethers'; import axios from 'axios'; import { createRequire } from 'module'; import { TRANSACTION_POLLING_TIMEOUT_MILLIS, TRANSACTION_CONFIRMATIONS, } from '../src/constants/constants.js'; import validateArguments from './utils.js'; const require = createRequire(import.meta.url); const Profile = require('dkg-evm-module/abi/Profile.json'); const IdentityStorage = require('dkg-evm-module/abi/IdentityStorage.json'); const Hub = require('dkg-evm-module/abi/Hub.json'); const argv = require('minimist')(process.argv.slice(1), { string: ['ask', 'privateKey', 'hubContractAddress', 'gasPriceOracleLink'], }); async function getGasPrice(gasPriceOracleLink, hubContractAddress, provider) { try { if (!gasPriceOracleLink) { return provider.getGasPrice(); } let gasPrice; const response = await axios.get(gasPriceOracleLink); if ( gasPriceOracleLink === 'https://api.gnosisscan.io/api?module=proxy&action=eth_gasPrice' ) { gasPrice = Number(response.data.result, 10); } else if ( gasPriceOracleLink === 'https://blockscout.chiadochain.net/api/v1/gas-price-oracle' ) { gasPrice = Math.round(response.data.average * 1e9); } else { gasPrice = Math.round(response.result * 1e9); } this.logger.debug(`Gas price: ${gasPrice}`); return gasPrice; } catch (error) { return undefined; } } async function setAsk(rpcEndpoint, ask, walletPrivateKey, hubContractAddress, gasPriceOracleLink) { const provider = new ethers.providers.JsonRpcProvider(rpcEndpoint); const wallet = new ethers.Wallet(walletPrivateKey, provider); const hubContract = new ethers.Contract(hubContractAddress, Hub, provider); const profileAddress = await hubContract.getContractAddress('Profile'); const profile = new ethers.Contract(profileAddress, Profile, wallet); const identityStorageAddress = await hubContract.getContractAddress('IdentityStorage'); const identityStorage = new ethers.Contract(identityStorageAddress, IdentityStorage, provider); const identityId = await identityStorage.getIdentityId(wallet.address); const askWei = ethers.utils.parseEther(ask); const gasPrice = await getGasPrice(gasPriceOracleLink, hubContractAddress, provider); const tx = await profile.updateAsk(identityId, askWei, { gasPrice: gasPrice ?? 8, gasLimit: 500_000, }); await provider.waitForTransaction( tx.hash, TRANSACTION_CONFIRMATIONS, TRANSACTION_POLLING_TIMEOUT_MILLIS, ); } const expectedArguments = ['rpcEndpoint', 'ask', 'privateKey', 'hubContractAddress']; if (validateArguments(argv, expectedArguments)) { setAsk( argv.rpcEndpoint, argv.ask, argv.privateKey, argv.hubContractAddress, argv.gasPriceOracleLink, ) .then(() => { console.log('Set ask completed'); process.exit(0); }) .catch((error) => { console.log('Error while setting ask. Error: ', error); process.exit(1); }); } else { console.log('Wrong arguments sent in script.'); console.log( 'Example: npm run set-ask -- --rpcEndpoint= --ask= --privateKey= --hubContractAddress=', ); } ================================================ FILE: scripts/set-operator-fee.js ================================================ /* eslint-disable no-console */ import { ethers } from 'ethers'; import { createRequire } from 'module'; import { NODE_ENVIRONMENTS, TRANSACTION_POLLING_TIMEOUT_MILLIS, TRANSACTION_CONFIRMATIONS, } from '../src/constants/constants.js'; import validateArguments from './utils.js'; const require = createRequire(import.meta.url); const Staking = require('dkg-evm-module/abi/Staking.json'); const IdentityStorage = require('dkg-evm-module/abi/IdentityStorage.json'); const Hub = require('dkg-evm-module/abi/Hub.json'); const argv = require('minimist')(process.argv.slice(1), { string: ['operatorFee', 'privateKey', 'hubContractAddress'], }); async function setOperatorFee(rpcEndpoint, operatorFee, walletPrivateKey, hubContractAddress) { const provider = new ethers.providers.JsonRpcProvider(rpcEndpoint); const wallet = new ethers.Wallet(walletPrivateKey, provider); const hubContract = new ethers.Contract(hubContractAddress, Hub.abi, provider); const stakingContractAddress = await hubContract.getContractAddress('Staking'); const stakingContract = new ethers.Contract(stakingContractAddress, Staking.abi, wallet); const identityStorageAddress = await hubContract.getContractAddress('IdentityStorage'); const identityStorage = new ethers.Contract( identityStorageAddress, IdentityStorage.abi, provider, ); const identityId = await identityStorage.getIdentityId(wallet.address); const tx = await stakingContract.setOperatorFee(identityId, operatorFee, { gasPrice: process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT ? undefined : 8, gasLimit: 500_000, }); await provider.waitForTransaction( tx.hash, TRANSACTION_CONFIRMATIONS, TRANSACTION_POLLING_TIMEOUT_MILLIS, ); } const expectedArguments = ['rpcEndpoint', 'operatorFee', 'privateKey', 'hubContractAddress']; if (validateArguments(argv, expectedArguments)) { setOperatorFee(argv.rpcEndpoint, argv.operatorFee, argv.privateKey, argv.hubContractAddress) .then(() => { console.log('Set operator fee completed'); }) .catch((error) => { console.log('Error while setting operator fee. Error: ', error); }); } else { console.log('Wrong arguments sent in script.'); console.log( 'Example: npm run set-operator-fee -- --rpcEndpoint= --operatorFee= --privateKey= --hubContractAddress=', ); } ================================================ FILE: scripts/set-stake.js ================================================ /* eslint-disable no-console */ import { ethers } from 'ethers'; import { createRequire } from 'module'; import axios from 'axios'; import { NODE_ENVIRONMENTS, TRANSACTION_POLLING_TIMEOUT_MILLIS, TRANSACTION_CONFIRMATIONS, } from '../src/constants/constants.js'; import validateArguments from './utils.js'; const require = createRequire(import.meta.url); const Staking = require('dkg-evm-module/abi/Staking.json'); const IdentityStorage = require('dkg-evm-module/abi/IdentityStorage.json'); const ERC20Token = require('dkg-evm-module/abi/Token.json'); const Hub = require('dkg-evm-module/abi/Hub.json'); const argv = require('minimist')(process.argv.slice(1), { string: [ 'stake', 'operationalWalletPrivateKey', 'managementWalletPrivateKey', 'hubContractAddress', 'gasPriceOracleLink', ], }); const devEnvironment = process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST; async function getGasPrice(gasPriceOracleLink, hubContractAddress, provider) { try { if (!gasPriceOracleLink) { if ( hubContractAddress === '0x6C861Cb69300C34DfeF674F7C00E734e840C29C0' || hubContractAddress === '0x144eDa5cbf8926327cb2cceef168A121F0E4A299' || hubContractAddress === '0xaBfcf2ad1718828E7D3ec20435b0d0b5EAfbDf2c' ) { return provider.getGasPrice(); } return devEnvironment ? undefined : 8; } let gasPrice; const response = await axios.get(gasPriceOracleLink); if ( gasPriceOracleLink === 'https://api.gnosisscan.io/api?module=proxy&action=eth_gasPrice' ) { gasPrice = Number(response.data.result, 10); } else if ( gasPriceOracleLink === 'https://blockscout.chiadochain.net/api/v1/gas-price-oracle' ) { gasPrice = Math.round(response.data.average * 1e9); } else { gasPrice = Math.round(response.result * 1e9); } this.logger.debug(`Gas price: ${gasPrice}`); return gasPrice; } catch (error) { return undefined; } } async function setStake( rpcEndpoint, stake, operationalWalletPrivateKey, managementWalletPrivateKey, hubContractAddress, gasPriceOracleLink, ) { const provider = new ethers.providers.JsonRpcProvider(rpcEndpoint); const operationalWallet = new ethers.Wallet(operationalWalletPrivateKey, provider); const managementWallet = new ethers.Wallet(managementWalletPrivateKey, provider); const hubContract = new ethers.Contract(hubContractAddress, Hub, provider); const stakingContractAddress = await hubContract.getContractAddress('Staking'); const stakingContract = new ethers.Contract(stakingContractAddress, Staking, managementWallet); const identityStorageAddress = await hubContract.getContractAddress('IdentityStorage'); const identityStorage = new ethers.Contract(identityStorageAddress, IdentityStorage, provider); const identityId = await identityStorage.getIdentityId(operationalWallet.address); const tokenContractAddress = await hubContract.getContractAddress('Token'); const tokenContract = new ethers.Contract(tokenContractAddress, ERC20Token, managementWallet); const stakeWei = ethers.utils.parseEther(stake); const gasPrice = await getGasPrice(gasPriceOracleLink, hubContractAddress, provider); let tx = await tokenContract.increaseAllowance(stakingContractAddress, stakeWei, { gasPrice, gasLimit: 500_000, }); await provider.waitForTransaction( tx.hash, TRANSACTION_CONFIRMATIONS, TRANSACTION_POLLING_TIMEOUT_MILLIS, ); // TODO: Add ABI instead of hard-coded function definition tx = await stakingContract['stake(uint72,uint96)'](identityId, stakeWei, { gasPrice: gasPrice ? gasPrice * 100 : undefined, gasLimit: 3_000_000, }); await provider.waitForTransaction( tx.hash, TRANSACTION_CONFIRMATIONS, TRANSACTION_POLLING_TIMEOUT_MILLIS, ); } const expectedArguments = [ 'rpcEndpoint', 'stake', 'operationalWalletPrivateKey', 'managementWalletPrivateKey', 'hubContractAddress', ]; if (validateArguments(argv, expectedArguments)) { setStake( argv.rpcEndpoint, argv.stake, argv.operationalWalletPrivateKey, argv.managementWalletPrivateKey, argv.hubContractAddress, argv.gasPriceOracleLink, ) .then(() => { console.log('Set stake completed'); process.exit(0); }) .catch((error) => { console.log('Error while setting stake. Error: ', error); process.exit(1); }); } else { console.log('Wrong arguments sent in script.'); console.log( 'Example: npm run set-stake -- --rpcEndpoint= --stake= --operationalWalletPrivateKey= --managementWalletPrivateKey= --hubContractAddress=', ); } ================================================ FILE: scripts/utils.js ================================================ export default function validateArguments(received, expected) { for (const arg of expected) { if (!received[arg]) { return false; } } return true; } ================================================ FILE: src/commands/blockchain-event-listener/blockchain-event-listener-command.js ================================================ import Command from '../command.js'; import { CONTRACTS, MONITORED_CONTRACT_EVENTS, CONTRACT_INDEPENDENT_EVENTS, ERROR_TYPE, OPERATION_ID_STATUS, MONITORED_CONTRACTS, MONITORED_EVENTS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class BlockchainEventListenerCommand extends Command { constructor(ctx) { super(ctx); this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.ualService = ctx.ualService; this.blockchainEventsService = ctx.blockchainEventsService; this.fileService = ctx.fileService; this.operationIdService = ctx.operationIdService; this.commandExecutor = ctx.commandExecutor; this.invalidatedContracts = new Set(); this.errorType = ERROR_TYPE.BLOCKCHAIN_EVENT_LISTENER_ERROR; } async execute(command) { const { blockchainId } = command.data; const repositoryTransaction = await this.repositoryModuleManager.transaction(); try { await this.fetchAndHandleBlockchainEvents(blockchainId, repositoryTransaction); await repositoryTransaction.commit(); } catch (e) { this.logger.error( `Failed to fetch and process blockchain events for blockchain: ${blockchainId}. Error: ${e}`, ); await repositoryTransaction.rollback(); return Command.repeat(); } await this.repositoryModuleManager.markAllBlockchainEventsAsProcessed(blockchainId); return Command.empty(); } async fetchAndHandleBlockchainEvents(blockchainId, repositoryTransaction) { const currentBlock = (await this.blockchainEventsService.getBlock(blockchainId)).number - 2; const lastCheckedBlockRecord = await this.repositoryModuleManager.getLastCheckedBlock( blockchainId, { transaction: repositoryTransaction }, ); const { events: newEvents, eventsMissed } = await this.blockchainEventsService.getPastEvents( blockchainId, MONITORED_CONTRACTS, MONITORED_EVENTS, lastCheckedBlockRecord?.lastCheckedBlock ?? 0, currentBlock, ); if (eventsMissed) { const missedFrom = (lastCheckedBlockRecord?.lastCheckedBlock ?? 0) + 1; this.logger.warn( `[EVENT LISTENER] Blockchain events missed on ${blockchainId}! ` + `Gap too large: blocks ${missedFrom}–${currentBlock} ` + `(${currentBlock - missedFrom + 1} blocks). ` + `Publish finality for assets created during this window will not complete.`, ); } if (newEvents.length !== 0) { this.logger.trace( `Storing ${newEvents.length} new events for blockchain ${blockchainId} in the database.`, ); await this.repositoryModuleManager.insertBlockchainEvents(newEvents, { transaction: repositoryTransaction, }); } await this.repositoryModuleManager.updateLastCheckedBlock( blockchainId, currentBlock, Date.now(), { transaction: repositoryTransaction }, ); const unprocessedEvents = await this.repositoryModuleManager.getAllUnprocessedBlockchainEvents( blockchainId, MONITORED_EVENTS, { transaction: repositoryTransaction }, ); if (unprocessedEvents.length > 0) { this.logger.trace( `Handling ${unprocessedEvents.length} unprocessed blockchain events.`, ); } this.independentEvents = []; this.dependentEvents = []; for (const event of unprocessedEvents) { if (this.isIndependentEvent(event.contract, event.event)) { this.independentEvents.push(event); } else { this.dependentEvents.push(event); } } this.dependentEvents.sort((a, b) => { if (a.blockNumber !== b.blockNumber) { return a.blockNumber - b.blockNumber; } if (a.transactionIndex !== b.transactionIndex) { return a.transactionIndex - b.transactionIndex; } return a.logIndex - b.logIndex; }); await Promise.all([ this.processIndependentEvents(currentBlock, repositoryTransaction), this.processDependentEvents(currentBlock, repositoryTransaction), ]); } isIndependentEvent(contractName, eventName) { const contractIndependentEvents = CONTRACT_INDEPENDENT_EVENTS[contractName] || []; return contractIndependentEvents.includes(eventName); } async processIndependentEvents(currentBlock, repositoryTransaction) { await Promise.all( this.independentEvents.map((event) => this.processEvent(event, currentBlock, repositoryTransaction), ), ); } async processDependentEvents(currentBlock, repositoryTransaction) { let index = 0; while (index < this.dependentEvents.length) { const event = this.dependentEvents[index]; // Step 1: Handle invalidated contracts if (this.invalidatedContracts.has(event.contractAddress)) { this.logger.info( `Skipping event ${event.event} for blockchain: ${event.blockchain}, ` + `invalidated contract: ${event.contract} (${event.contractAddress})`, ); this.dependentEvents.splice(index, 1); // Remove the invalidated event continue; // Restart the loop with the updated array } // Step 2: Handle new dependent events if (this.newDependentEvents?.length > 0) { this.logger.info( `Adding ${this.newDependentEvents.length} new dependent events before processing.`, ); // Merge new events into the unprocessed part of the array const combinedEvents = [ ...this.dependentEvents.slice(index), // Unprocessed events ...this.newDependentEvents, // New events ].sort((a, b) => { if (a.blockNumber !== b.blockNumber) { return a.blockNumber - b.blockNumber; } if (a.transactionIndex !== b.transactionIndex) { return a.transactionIndex - b.transactionIndex; } return a.logIndex - b.logIndex; }); // Update dependentEvents: add back processed events + sorted combined events this.dependentEvents = [...this.dependentEvents.slice(0, index), ...combinedEvents]; // Reset the new events buffer this.newDependentEvents = []; } // Step 3: Process the current event // eslint-disable-next-line no-await-in-loop await this.processEvent(event, currentBlock, repositoryTransaction); index += 1; // Move to the next event } // Clear invalidated contracts after processing this.invalidatedContracts.clear(); } async processEvent(event, currentBlock, repositoryTransaction) { const handlerFunctionName = `handle${event.event}Event`; if (typeof this[handlerFunctionName] !== 'function') { this.logger.warn(`No handler for event type: ${event.event}`); return; } this.logger.trace(`Processing event ${event.event} in block ${event.blockNumber}.`); try { await this[handlerFunctionName](event, currentBlock, repositoryTransaction); } catch (error) { this.logger.error( `Error processing event ${event.event} in block ${event.blockNumber}: ${error.message}`, ); } } async handleParameterChangedEvent(event) { const { blockchain, contract, data } = event; const { parameterName, parameterValue } = JSON.parse(data); switch (contract) { case CONTRACTS.PARAMETERS_STORAGE: this.blockchainModuleManager.setContractCallCache( blockchain, CONTRACTS.PARAMETERS_STORAGE, parameterName, parameterValue, ); break; default: this.logger.warn( `Unable to handle parameter changed event. Unknown contract name ${event.contract}`, ); } } async handleNewContractEvent(event, currentBlock, repositoryTransaction) { const { contractName, newContractAddress } = JSON.parse(event.data); const blockchchainModuleContractAddress = this.blockchainModuleManager.getContractAddress( event.blockchain, contractName, ); if (newContractAddress !== blockchchainModuleContractAddress) { this.blockchainModuleManager.initializeContract( event.blockchain, contractName, newContractAddress, ); } const blockchainEventsServiceContractAddress = this.blockchainEventsService.getContractAddress(event.blockchain, contractName); if ( blockchainEventsServiceContractAddress && newContractAddress !== blockchainEventsServiceContractAddress ) { this.blockchainEventsService.updateContractAddress( event.blockchain, contractName, newContractAddress, ); this.invalidatedContracts.add(blockchainEventsServiceContractAddress); await this.repositoryModuleManager.removeContractEventsAfterBlock( event.blockchain, contractName, event.contractAddress, event.blockNumber, event.transactionIndex, { transaction: repositoryTransaction }, ); const { events: newEvents } = await this.blockchainEventsService.getPastEvents( event.blockchain, [contractName], MONITORED_CONTRACT_EVENTS[contractName], event.blockNumber, currentBlock, ); if (newEvents.length !== 0) { this.logger.trace( `Storing ${newEvents.length} new events for blockchain ${event.blockchain} in the database.`, ); await this.repositoryModuleManager.insertBlockchainEvents(newEvents, { transaction: repositoryTransaction, }); this.newDependentEvents = newEvents; } } } async handleContractChangedEvent(event, currentBlock, repositoryTransaction) { const { contractName, newContractAddress } = JSON.parse(event.data); const blockchchainModuleContractAddress = this.blockchainModuleManager.getContractAddress( event.blockchain, contractName, ); if (newContractAddress !== blockchchainModuleContractAddress) { this.blockchainModuleManager.initializeContract( event.blockchain, contractName, newContractAddress, ); } const blockchainEventsServiceContractAddress = this.blockchainEventsService.getContractAddress(event.blockchain, contractName); if ( blockchainEventsServiceContractAddress && newContractAddress !== blockchainEventsServiceContractAddress ) { this.blockchainEventsService.updateContractAddress( event.blockchain, contractName, newContractAddress, ); this.invalidatedContracts.add(blockchainEventsServiceContractAddress); await this.repositoryModuleManager.removeContractEventsAfterBlock( event.blockchain, contractName, event.contractAddress, event.blockNumber, event.transactionIndex, { transaction: repositoryTransaction }, ); const { events: newEvents } = await this.blockchainEventsService.getPastEvents( event.blockchain, [contractName], MONITORED_CONTRACT_EVENTS[contractName], event.blockNumber, currentBlock, ); if (newEvents.length !== 0) { this.logger.trace( `Storing ${newEvents.length} new events for blockchain ${event.blockchain} in the database.`, ); await this.repositoryModuleManager.insertBlockchainEvents(newEvents, { transaction: repositoryTransaction, }); this.newDependentEvents = newEvents; } } } async handleNewAssetStorageEvent(event, currentBlock, repositoryTransaction) { const { contractName, newContractAddress } = JSON.parse(event.data); const blockchchainModuleContractAddress = this.blockchainModuleManager.getContractAddress( event.blockchain, contractName, ); if (newContractAddress !== blockchchainModuleContractAddress) { this.blockchainModuleManager.initializeAssetStorageContract( event.blockchain, newContractAddress, ); } const blockchainEventsServiceContractAddress = this.blockchainEventsService.getContractAddress(event.blockchain, contractName); if ( blockchainEventsServiceContractAddress && newContractAddress !== blockchainEventsServiceContractAddress ) { this.blockchainEventsService.updateContractAddress( event.blockchain, contractName, newContractAddress, ); this.invalidatedContracts.add(blockchainEventsServiceContractAddress); await this.repositoryModuleManager.removeContractEventsAfterBlock( event.blockchain, contractName, event.contractAddress, event.blockNumber, event.transactionIndex, { transaction: repositoryTransaction }, ); const { events: newEvents } = await this.blockchainEventsService.getPastEvents( event.blockchain, [contractName], MONITORED_CONTRACT_EVENTS[contractName], event.blockNumber, currentBlock, ); if (newEvents.length !== 0) { this.logger.trace( `Storing ${newEvents.length} new events for blockchain ${event.blockchain} in the database.`, ); await this.repositoryModuleManager.insertBlockchainEvents(newEvents, { transaction: repositoryTransaction, }); this.newDependentEvents = newEvents; } } } async handleAssetStorageChangedEvent(event, currentBlock, repositoryTransaction) { const { contractName, newContractAddress } = JSON.parse(event.data); const blockchchainModuleContractAddress = this.blockchainModuleManager.getContractAddress( event.blockchain, contractName, ); if (newContractAddress !== blockchchainModuleContractAddress) { this.blockchainModuleManager.initializeAssetStorageContract( event.blockchain, newContractAddress, ); } const blockchainEventsServiceContractAddress = this.blockchainEventsService.getContractAddress(event.blockchain, contractName); if ( blockchainEventsServiceContractAddress && newContractAddress !== blockchainEventsServiceContractAddress ) { this.blockchainEventsService.updateContractAddress( event.blockchain, contractName, newContractAddress, ); this.invalidatedContracts.add(blockchainEventsServiceContractAddress); await this.repositoryModuleManager.removeContractEventsAfterBlock( event.blockchain, contractName, event.contractAddress, event.blockNumber, event.transactionIndex, { transaction: repositoryTransaction }, ); const { events: newEvents } = await this.blockchainEventsService.getPastEvents( event.blockchain, [contractName], MONITORED_CONTRACT_EVENTS[contractName], event.blockNumber, currentBlock, ); if (newEvents.length !== 0) { this.logger.trace( `Storing ${newEvents.length} new events for blockchain ${event.blockchain} in the database.`, ); await this.repositoryModuleManager.insertBlockchainEvents(newEvents, { transaction: repositoryTransaction, }); this.newDependentEvents = newEvents; } } } async handleKnowledgeCollectionCreatedEvent(event) { await this.commandExecutor.add({ name: 'publishFinalizationCommand', sequence: [], data: { event, }, priority: COMMAND_PRIORITY.HIGHEST, transactional: false, }); } // TODO: Adjust after new contracts are released async handleAssetUpdatedEvent(event) { const eventData = JSON.parse(event.data); // TODO: Add correct name for assetStateIndex from event currently it's placeholder const { assetContract, tokenId, state, updateOperationId, assetStateIndex } = eventData; const { blockchain } = event; const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.UPDATE_FINALIZATION.UPDATE_FINALIZATION_START, blockchain, ); let datasetPath; let cachedData; try { datasetPath = this.fileService.getPendingStorageDocumentPath(updateOperationId); cachedData = await this.fileService.readFile(datasetPath, true); } catch (error) { this.operationIdService.markOperationAsFailed( operationId, blockchain, `Unable to read cached data from ${datasetPath}, error: ${error.message}`, ERROR_TYPE.PUBLISH_FINALIZATION.PUBLISH_FINALIZATION_NO_CACHED_DATA, ); } const ual = this.ualService.deriveUAL(blockchain, assetContract, tokenId); await this.commandExecutor.add({ name: 'updateValidateAssertionMetadataCommand', sequence: ['updateAssertionCommand'], delay: 0, data: { operationId, ual, blockchain, contract: assetContract, tokenId, assetStateIndex, merkleRoot: state, assertion: cachedData.assertion, cachedMerkleRoot: cachedData.merkleRoot, }, transactional: false, }); } /** * Recover system from failure * @param error */ async recover() { return Command.repeat(); } /** * Builds default BlockchainEventListenerCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'blockchainEventListenerCommand', data: {}, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default BlockchainEventListenerCommand; ================================================ FILE: src/commands/blockchain-event-listener/event-listener-command.js ================================================ import Command from '../command.js'; import { CONTRACT_EVENT_FETCH_INTERVALS, NODE_ENVIRONMENTS, ERROR_TYPE, COMMAND_PRIORITY, MAXIMUM_FETCH_EVENTS_FAILED_COUNT, } from '../../constants/constants.js'; class EventListenerCommand extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.blockchainModuleManager = ctx.blockchainModuleManager; this.errorType = ERROR_TYPE.EVENT_LISTENER_ERROR; } calculateCommandPeriod() { const isDevEnvironment = [NODE_ENVIRONMENTS.DEVELOPMENT, NODE_ENVIRONMENTS.TEST].includes( process.env.NODE_ENV, ); return isDevEnvironment ? CONTRACT_EVENT_FETCH_INTERVALS.DEVELOPMENT : CONTRACT_EVENT_FETCH_INTERVALS.MAINNET; } async execute() { this.logger.info('Event Listener: Starting event listener command.'); await Promise.all( this.blockchainModuleManager.getImplementationNames().map(async (blockchainId) => { const commandData = { blockchainId }; return this.commandExecutor.add({ name: `blockchainEventListenerCommand`, data: commandData, retries: MAXIMUM_FETCH_EVENTS_FAILED_COUNT, priority: COMMAND_PRIORITY.HIGHEST, isBlocking: true, transactional: false, }); }), ); if (!this.blockchainModuleManager.getImplementationNames().length) { this.logger.error(`No blockchain implementations. OT-node shutting down...`); process.exit(1); } return Command.repeat(); } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to execute ${command.name}. Error: ${command.message}`); return Command.repeat(); } /** * Builds default eventListenerCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'eventListenerCommand', delay: 0, data: {}, transactional: false, period: this.calculateCommandPeriod(), priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default EventListenerCommand; ================================================ FILE: src/commands/cleaners/ask-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, ASK_CLEANUP_TIME_DELAY, ASK_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class AskCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationRecords( OPERATIONS.ASK, nowTimestamp - ASK_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'askCleanerCommand', data: {}, period: ASK_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default AskCleanerCommand; ================================================ FILE: src/commands/cleaners/ask-response-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, ASK_RESPONSE_CLEANUP_TIME_DELAY, ASK_RESPONSE_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class AskResponseCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationResponse( OPERATIONS.ASK, nowTimestamp - ASK_RESPONSE_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'askResponseCleanerCommand', data: {}, period: ASK_RESPONSE_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default AskResponseCleanerCommand; ================================================ FILE: src/commands/cleaners/batch-get-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, GET_CLEANUP_TIME_DELAY, GET_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class BatchGetCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationRecords( OPERATIONS.BATCH_GET, nowTimestamp - GET_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'batchGetCleanerCommand', data: {}, period: GET_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default BatchGetCleanerCommand; ================================================ FILE: src/commands/cleaners/blockchain-event-cleaner-command.js ================================================ import { PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY, COMMAND_PRIORITY, } from '../../constants/constants.js'; import CleanerCommand from './cleaner-command.js'; class BlockchainEventCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedEvents( nowTimestamp - PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'blockchainEventCleanerCommand', data: {}, period: PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default BlockchainEventCleanerCommand; ================================================ FILE: src/commands/cleaners/cleaner-command.js ================================================ import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER } from '../../constants/constants.js'; import Command from '../command.js'; class CleanerCommand extends Command { constructor(ctx) { super(ctx); this.repositoryModuleManager = ctx.repositoryModuleManager; } /** * Executes command and produces one or more events * @param command */ async execute() { let deletedRowsCount; do { const nowTimestamp = Date.now(); // eslint-disable-next-line no-await-in-loop deletedRowsCount = await this.deleteRows(nowTimestamp); } while (deletedRowsCount === REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER); return Command.repeat(); } // eslint-disable-next-line no-unused-vars async deleteRows(nowTimestamp) { throw Error('deleteRows not implemented'); } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to clean operational db data: error: ${command.message}`); return Command.repeat(); } } export default CleanerCommand; ================================================ FILE: src/commands/cleaners/commands-cleaner-command.js ================================================ import { FINALIZED_COMMAND_CLEANUP_TIME_MILLS, FINALIZED_COMMAND_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, } from '../../constants/constants.js'; import CleanerCommand from './cleaner-command.js'; class CommandsCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveFinalizedCommands( nowTimestamp - FINALIZED_COMMAND_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'commandsCleanerCommand', data: {}, period: FINALIZED_COMMAND_CLEANUP_TIME_MILLS, transactional: false, }; Object.assign(command, map); return command; } } export default CommandsCleanerCommand; ================================================ FILE: src/commands/cleaners/finality-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, FINALITY_CLEANUP_TIME_DELAY, FINALITY_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class FinalityCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationRecords( OPERATIONS.FINALITY, nowTimestamp - FINALITY_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'finalityCleanerCommand', data: {}, period: FINALITY_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default FinalityCleanerCommand; ================================================ FILE: src/commands/cleaners/finality-response-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, FINALITY_RESPONSE_CLEANUP_TIME_DELAY, FINALITY_RESPONSE_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class FinalityResponseCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationResponse( OPERATIONS.FINALITY, nowTimestamp - FINALITY_RESPONSE_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'finalityResponseCleanerCommand', data: {}, period: FINALITY_RESPONSE_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default FinalityResponseCleanerCommand; ================================================ FILE: src/commands/cleaners/get-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, GET_CLEANUP_TIME_DELAY, GET_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class GetCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationRecords( OPERATIONS.GET, nowTimestamp - GET_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'getCleanerCommand', data: {}, period: GET_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default GetCleanerCommand; ================================================ FILE: src/commands/cleaners/get-response-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, GET_RESPONSE_CLEANUP_TIME_DELAY, GET_RESPONSE_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class GetResponseCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationResponse( OPERATIONS.GET, nowTimestamp - GET_RESPONSE_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'getResponseCleanerCommand', data: {}, period: GET_RESPONSE_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default GetResponseCleanerCommand; ================================================ FILE: src/commands/cleaners/operation-id-cleaner-command.js ================================================ import Command from '../command.js'; import { BYTES_IN_KILOBYTE, OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS, OPERATION_ID_STATUS, COMMAND_PRIORITY, } from '../../constants/constants.js'; /** * Increases approval for Bidding contract on blockchain */ class OperationIdCleanerCommand extends Command { constructor(ctx) { super(ctx); this.logger = ctx.logger; this.repositoryModuleManager = ctx.repositoryModuleManager; this.fileService = ctx.fileService; } /** * Executes command and produces one or more events * @param command */ async execute() { let memoryBytes = 0; let fileBytes = 0; try { memoryBytes = this.operationIdService.getOperationIdMemoryCacheSizeBytes(); } catch (error) { this.logger.warn(`Unable to read memory cache footprint: ${error.message}`); } try { fileBytes = await this.operationIdService.getOperationIdFileCacheSizeBytes(); } catch (error) { this.logger.warn(`Unable to read file cache footprint: ${error.message}`); } const bytesInMegabyte = 1024 * 1024; this.logger.debug( `Operation cache footprint before cleanup: memory=${( memoryBytes / bytesInMegabyte ).toFixed(2)}MB, files=${(fileBytes / bytesInMegabyte).toFixed(2)}MB`, ); this.logger.debug('Starting command for removal of expired cache files'); const timeToBeDeleted = Date.now() - OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS; await this.repositoryModuleManager.removeOperationIdRecord(timeToBeDeleted, [ OPERATION_ID_STATUS.COMPLETED, OPERATION_ID_STATUS.FAILED, ]); let removed = await this.operationIdService.removeExpiredOperationIdMemoryCache( OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS, ); if (removed) { this.logger.debug( `Successfully removed ${ removed / BYTES_IN_KILOBYTE } Kbs expired cached operation entries from memory`, ); } removed = await this.operationIdService.removeExpiredOperationIdFileCache( OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, ); if (removed) { this.logger.debug(`Successfully removed ${removed} expired cached operation files`); } return Command.repeat(); } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to clean operation ids table: error: ${command.message}`); return Command.repeat(); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'operationIdCleanerCommand', period: OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS, data: {}, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default OperationIdCleanerCommand; ================================================ FILE: src/commands/cleaners/pending-storage-cleaner-command.js ================================================ import Command from '../command.js'; import { PUBLISH_STORAGE_MEMORY_CLEANUP_COMMAND_CLEANUP_TIME_MILLS, PUBLISH_STORAGE_FILE_CLEANUP_COMMAND_CLEANUP_TIME_MILLS, PENDING_STORAGE_FILES_FOR_REMOVAL_MAX_NUMBER, COMMAND_PRIORITY, } from '../../constants/constants.js'; /** * Cleans memory cache in the pending storage service */ class PendingStorageCleanerCommand extends Command { constructor(ctx) { super(ctx); this.logger = ctx.logger; this.pendingStorageService = ctx.pendingStorageService; } /** * Executes command and produces one or more events * @param command */ async execute() { this.logger.debug('Starting command for removal of expired pending storage entries'); const removed = await this.pendingStorageService.removeExpiredFileCache( PUBLISH_STORAGE_FILE_CLEANUP_COMMAND_CLEANUP_TIME_MILLS, PENDING_STORAGE_FILES_FOR_REMOVAL_MAX_NUMBER, ); if (removed) { this.logger.debug(`Successfully removed ${removed} expired cached operation files`); } return Command.repeat(); } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to clean pending storage: error: ${command.message}`); return Command.repeat(); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'pendingStorageCleanerCommand', period: PUBLISH_STORAGE_MEMORY_CLEANUP_COMMAND_CLEANUP_TIME_MILLS, data: {}, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default PendingStorageCleanerCommand; ================================================ FILE: src/commands/cleaners/publish-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, PUBLISH_CLEANUP_TIME_DELAY, PUBLISH_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class PublishCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationRecords( OPERATIONS.PUBLISH, nowTimestamp - PUBLISH_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'publishCleanerCommand', data: {}, period: PUBLISH_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default PublishCleanerCommand; ================================================ FILE: src/commands/cleaners/publish-response-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, PUBLISH_RESPONSE_CLEANUP_TIME_DELAY, PUBLISH_RESPONSE_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class PublishResponseCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationResponse( OPERATIONS.PUBLISH, nowTimestamp - PUBLISH_RESPONSE_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'publishResponseCleanerCommand', data: {}, period: PUBLISH_RESPONSE_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default PublishResponseCleanerCommand; ================================================ FILE: src/commands/cleaners/update-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, UPDATE_CLEANUP_TIME_DELAY, UPDATE_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class UpdateCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationRecords( OPERATIONS.UPDATE, nowTimestamp - UPDATE_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'updateCleanerCommand', data: {}, period: UPDATE_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default UpdateCleanerCommand; ================================================ FILE: src/commands/cleaners/update-response-cleaner-command.js ================================================ import CleanerCommand from './cleaner-command.js'; import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, OPERATIONS, UPDATE_RESPONSE_CLEANUP_TIME_DELAY, UPDATE_RESPONSE_CLEANUP_TIME_MILLS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class UpdateResponseCleanerCommand extends CleanerCommand { async deleteRows(nowTimestamp) { return this.repositoryModuleManager.findAndRemoveProcessedOperationResponse( OPERATIONS.UPDATE, nowTimestamp - UPDATE_RESPONSE_CLEANUP_TIME_DELAY, REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, ); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'updateResponseCleanerCommand', data: {}, period: UPDATE_RESPONSE_CLEANUP_TIME_MILLS, transactional: false, priority: COMMAND_PRIORITY.LOWEST, }; Object.assign(command, map); return command; } } export default UpdateResponseCleanerCommand; ================================================ FILE: src/commands/command-executor.js ================================================ import { Queue, Worker } from 'bullmq'; import { PERMANENT_COMMANDS, DEFAULT_COMMAND_DELAY_IN_MILLS, GENERAL_COMMAND_QUEUE_PARALLELISM, BATCH_GET_COMMAND_QUEUE_PARALLELISM, DEFAULT_COMMAND_PRIORITY, MAX_COMMAND_LIFETIME, } from '../constants/constants.js'; /** * Queues and processes commands */ class CommandExecutor { constructor(ctx) { this.logger = ctx.logger; this.commandResolver = ctx.commandResolver; this.operationIdService = ctx.operationIdService; this.verboseLoggingEnabled = ctx.config.commandExecutorVerboseLoggingEnabled; const env = process.env.NODE_ENV; const queueName = env === 'development' ? `command-executor-${ctx.config.modules.blockchain.implementation['hardhat1:31337'].config.nodeName}` : 'command-executor'; const batchGetQueueName = env === 'development' ? `batchGetQueue-${ctx.config.modules.blockchain.implementation['hardhat1:31337'].config.nodeName}` : 'batchGetQueue'; this.queue = new Queue(queueName, { connection: { host: 'localhost', port: 6379, }, }); this.queueBatchGet = new Queue(batchGetQueueName, { connection: { host: 'localhost', port: 6379, }, }); this.batchGetWorker = new Worker( batchGetQueueName, async (job) => { const commandData = job.data; const createdTime = new Date(job.timestamp).getTime(); const now = Date.now(); if (now - createdTime > MAX_COMMAND_LIFETIME) { throw new Error('Command is too old'); } this.logger.trace(`Command started ${job.name}, ${job.id}`); const commandName = job.name; const handler = this.commandResolver.resolve(commandName); if (!handler) { throw new Error(`Command will not be executed ${job.name}, missing handler`); } await handler.execute({ data: commandData }); }, { connection: { host: 'localhost', port: 6379, }, maxStalledCount: 0, lockDuration: 3 * 60 * 1000, stalledInterval: 3 * 60 * 1000, concurrency: BATCH_GET_COMMAND_QUEUE_PARALLELISM, }, ); this.worker = new Worker( queueName, async (job) => { const commandData = job.data; const createdTime = new Date(job.timestamp).getTime(); const now = Date.now(); if (now - createdTime > MAX_COMMAND_LIFETIME) { throw new Error('Command is too old'); } this.logger.trace(`Command started ${job.name}, ${job.id}`); let commandName = job.name; if (job.name.startsWith('paranetSyncCommand')) { commandName = `paranetSyncCommand`; } const handler = this.commandResolver.resolve(commandName); if (!handler) { throw new Error(`Command will not be executed ${job.name}, missing handler`); } await handler.execute({ data: commandData }); }, { connection: { host: 'localhost', port: 6379, }, maxStalledCount: 0, lockDuration: 3 * 60 * 1000, stalledInterval: 3 * 60 * 1000, concurrency: GENERAL_COMMAND_QUEUE_PARALLELISM, }, ); this.worker.on('completed', async (job) => { this.logger.trace( `Job with ID ${job.id}, ${job.name} has been completed. Duration: ${ job.finishedOn - job.timestamp }`, ); }); this.batchGetWorker.on('completed', async (job) => { this.logger.trace( `BatchGetJob with ID ${job.id}, ${job.name} has been completed. Duration: ${ job.finishedOn - job.timestamp }`, ); }); this.worker.on('failed', (job, err) => { this.logger.error( `Job with ID ${job.id}, ${job.name} has failed with error: ${err.message}, ${err.stack}`, ); }); this.batchGetWorker.on('failed', (job, err) => { this.logger.error( `BatchGetJob with ID ${job.id}, ${job.name} has failed with error: ${err.message}, ${err.stack}`, ); }); this.queue.on('error', (err) => { this.logger.error(`Queue error: ${err.message}, ${err.stack}`); }); this.queueBatchGet.on('error', (err) => { this.logger.error(`BatchGetQueue error: ${err.message}, ${err.stack}`); }); this.worker.on('error', (err) => { this.logger.error(`Worker error: ${err.message}, ${err.stack}`); }); this.batchGetWorker.on('error', (err) => { this.logger.error(`BatchGetWorker error: ${err.message}, ${err.stack}`); }); this.queueBatchGet.on('closed', () => { this.logger.trace('BatchGetQueue has been closed.'); }); this.queue.on('closed', () => { this.logger.trace('Queue has been closed.'); }); setInterval(async () => { const generalQueueCount = await this.queue.count(); const batchGetQueueCount = await this.queueBatchGet.count(); this.logger.trace( `General queue count: ${generalQueueCount}, Batch get queue count: ${batchGetQueueCount}`, ); this.operationIdService.emitChangeEvent( 'COMMAND_EXECUTOR_QUEUE_COUNT', `command-executor-queue-count-${Date.now()}`, null, generalQueueCount, batchGetQueueCount, ); }, 5 * 60 * 1000); } /** * Initialize executor * @returns {Promise} */ async addDefaultCommands() { await Promise.all(PERMANENT_COMMANDS.map((command) => this._addDefaultCommand(command))); this.logger.trace('Command executor has been initialized...'); } /** * Resumes the command executor */ async resumeCommandExecutor() { if (this.verboseLoggingEnabled) { this.logger.trace('Command executor has been resumed...'); } await this.queue.resume(); await this.queueBatchGet.resume(); this.worker.resume(); this.batchGetWorker.resume(); } /** * Pause the command executor queue */ async pauseCommandExecutor() { this.logger.trace('Command executor queue has been paused...'); await this.queue.pause(); await this.worker.pause(); await this.queueBatchGet.pause(); await this.batchGetWorker.pause(); } /** * Starts the default command by name * @param name - Command name * @return {Promise} * @private */ async _addDefaultCommand(name) { const handler = this.commandResolver.resolve(name); if (!handler) { // Add command name to the log this.logger.warn(`Command will not be executed.`); return; } await this.removePeriodicCommand(['paranetSyncCommand']); if (['eventListenerCommand', 'shardingTableCheckCommand'].includes(name)) { await this.add(handler.default(), 0); } else { await this.add(handler.default(), DEFAULT_COMMAND_DELAY_IN_MILLS); } if (this.verboseLoggingEnabled) { handler.logger.trace(`Permanent command created.`); } } // TODO: Add function that removes periodic command async removePeriodicCommand(commandNames) { const periodicCommands = await this.queue.getJobSchedulers(); // Find if command with this prefix exist in repeatable commands const periodicCommandsToRemove = periodicCommands.filter((command) => commandNames.some((name) => command.name.startsWith(name)), ); await Promise.all( periodicCommandsToRemove.map((command) => this.queue.removeJobScheduler(command.name)), ); } /** * Adds single command to queue * @param command * @param delay * @param insert */ async add(addCommand, addDelay) { const command = addCommand; const delay = addDelay ?? 0; const commandPriority = command.priority ?? DEFAULT_COMMAND_PRIORITY; const jobOptions = { removeOnComplete: true, removeOnFail: true }; if (delay > 0) { jobOptions.delay = delay; } jobOptions.priority = commandPriority; if (command.period && command.period > 0) { await this.queue.upsertJobScheduler( command.name, { every: command.period }, { name: command.name, data: command.data, opts: jobOptions }, ); } else if ( command.name.toLowerCase().endsWith('batchgetcommand') || command.name.toLowerCase().endsWith('batchgetrequestcommand') ) { await this.queueBatchGet.add(command.name, command.data, jobOptions); } else { await this.queue.add(command.name, command.data, jobOptions); } } async commandExecutorShutdown() { await this.worker.close(); await this.queue.close(); await this.queueBatchGet.close(); await this.batchGetWorker.close(); } } export default CommandExecutor; ================================================ FILE: src/commands/command-resolver.js ================================================ /** * Resolves command handlers based on command names */ class CommandResolver { constructor(ctx) { this.ctx = ctx; this.logger = ctx.logger; } /** * Gets command handler based on command name * @param name * @return {*} */ resolve(name) { try { return this.ctx[`${name}`]; } catch (e) { this.logger.warn(`No handler defined for command '${name}'`); } } } export default CommandResolver; ================================================ FILE: src/commands/command.js ================================================ import { OPERATION_ID_STATUS } from '../constants/constants.js'; /** * Describes one command handler */ class Command { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; this.commandResolver = ctx.commandResolver; this.operationIdService = ctx.operationIdService; } /** * Executes command and produces one or more events */ async execute() { return Command.empty(); } /** * Recover system from failure * @param command * @param err */ async recover(command) { const { operationId, blockchain } = command.data; await this.handleError(operationId, blockchain, command.message, this.errorType, true); return Command.empty(); } /** * Execute strategy when event is too late */ async expired() { return Command.empty(); } /** * Pack data for DB * @param data */ pack(data) { return data; } /** * Unpack data from DB * @param data */ unpack(data) { return data; } /** * Makes command from sequence and continues it * @param data - Command data * @param [sequence] - Optional command sequence * @param [opts] - Optional command options */ continueSequence(data, sequence, opts) { if (!sequence || sequence.length === 0) { return Command.empty(); } const [name] = sequence; const newSequence = sequence.slice(1); const handler = this.commandResolver.resolve(name); const command = handler.default(); const commandData = command.data ? command.data : {}; Object.assign(command, { data: Object.assign(commandData, data), sequence: newSequence, }); if (opts) { Object.assign(command, opts); } return { commands: [command], }; } /** * Builds command * @param name - Command name * @param data - Command data * @param [sequence] - Optional command sequence * @param [opts] - Optional command options * @returns {*} */ build(name, data, sequence, opts) { const command = this.commandResolver.resolve(name).default(); const commandData = command.data ? command.data : {}; Object.assign(command, { data: Object.assign(commandData, data), sequence, }); if (opts) { Object.assign(command, opts); } return command; } async retryFinished(command) { this.logger.trace(`Max retry count for command: ${command.name} reached!`); } /** * Error handler for command * @param operationId - Operation operation id * @param error - Error object * @param errorName - Name of error * @param markFailed - Update operation status to failed * @returns {*} */ async handleError(operationId, blockchain, errorMessage, errorName, markFailed) { this.logger.error(`Command error (${errorName}): ${errorMessage}`); if (markFailed) { await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.FAILED, errorMessage, errorName, ); } } /** * Builds default command * @returns {{add, data: *, delay: *, deadline: *}} */ default() { return {}; } /** * Halt execution * @returns {{repeat: boolean, commands: Array}} */ static empty() { return { commands: [], }; } /** * Returns repeat info * @returns {{repeat: boolean, commands: Array}} */ static repeat() { return { repeat: true, }; } /** * Returns retry info * @returns {{retry: boolean, commands: Array}} */ static retry() { return { retry: true, }; } } export default Command; ================================================ FILE: src/commands/common/dial-peers-command.js ================================================ import Command from '../command.js'; import { DIAL_PEERS_COMMAND_FREQUENCY_MILLS, DIAL_PEERS_CONCURRENCY, MIN_DIAL_FREQUENCY_MILLIS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class DialPeersCommand extends Command { constructor(ctx) { super(ctx); this.shardingTableService = ctx.shardingTableService; this.repositoryModuleManager = ctx.repositoryModuleManager; } /** * Executes command and produces one or more events * @param command */ async execute() { const peersToDial = await this.repositoryModuleManager.getPeersToDial( DIAL_PEERS_CONCURRENCY, MIN_DIAL_FREQUENCY_MILLIS, ); if (peersToDial.length) { this.logger.trace(`Dialing ${peersToDial.length} remote peers`); await Promise.all( peersToDial.map(({ peerId }) => this.shardingTableService.dial(peerId)), ); } return Command.repeat(); } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to dial peers: error: ${command.message}`); return Command.repeat(); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'dialPeersCommand', data: {}, period: DIAL_PEERS_COMMAND_FREQUENCY_MILLS, priority: COMMAND_PRIORITY.MEDIUM, transactional: false, }; Object.assign(command, map); return command; } } export default DialPeersCommand; ================================================ FILE: src/commands/common/log-public-addresses-command.js ================================================ import ip from 'ip'; import Command from '../command.js'; import { NODE_ENVIRONMENTS } from '../../constants/constants.js'; class LogPublicAddressesCommand extends Command { constructor(ctx) { super(ctx); this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.networkModuleManager = ctx.networkModuleManager; } /** * Executes command and produces one or more events * @param command */ async execute() { if ( process.env.NODE_ENV !== NODE_ENVIRONMENTS.DEVELOPMENT && process.env.NODE_ENV !== NODE_ENVIRONMENTS.DEVNET ) return Command.empty(); const publicAddressesMap = {}; await Promise.all( this.blockchainModuleManager.getImplementationNames().map(async (blockchain) => { const peers = await this.repositoryModuleManager.getAllPeerRecords(blockchain); await Promise.all( peers.map(async (p) => { let peerInfo = await this.networkModuleManager .getPeerInfo(p.peerId) .catch(() => ({ addresses: [] })); if (!peerInfo?.addresses.length) { peerInfo = { addresses: [] }; } publicAddressesMap[p.peerId] = peerInfo.addresses .map((addr) => addr.multiaddr) .filter((addr) => addr.isThinWaistAddress()) .filter((addr) => !ip.isPrivate(addr.toString().split('/')[2])); }), ); }), ); this.logger.debug( `Found public addresses for sharding table peers: ${JSON.stringify( publicAddressesMap, null, 2, )}`, ); return Command.repeat(); } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to log public addresses: error: ${command.message}`); return Command.repeat(); } /** * Builds default command * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'logPublicAddressesCommand', data: {}, period: 60 * 1000, transactional: false, }; Object.assign(command, map); return command; } } export default LogPublicAddressesCommand; ================================================ FILE: src/commands/common/otnode-update-command.js ================================================ import semver from 'semver'; import Command from '../command.js'; import { COMMAND_PRIORITY } from '../../constants/constants.js'; class OtnodeUpdateCommand extends Command { constructor(ctx) { super(ctx); this.logger = ctx.logger; this.config = ctx.config; this.autoUpdaterModuleManager = ctx.autoUpdaterModuleManager; this.fileService = ctx.fileService; } /** * Performs code update by fetching new code from github repo * @param command */ async execute() { if (!this.config.modules.autoUpdater.enabled) { return Command.empty(); } try { this.logger.info('Checking for new updates...'); const { upToDate, currentVersion, remoteVersion } = await this.autoUpdaterModuleManager.compareVersions(); if (!upToDate) { if (semver.lt(semver.valid(remoteVersion), semver.valid(currentVersion))) { this.logger.info( 'Remote version less than current version, update will be skipped', ); return Command.repeat(); } const success = await this.autoUpdaterModuleManager.update(); if (success) { const updateFolderPath = this.fileService.getDataFolderPath(); await this.fileService.writeContentsToFile( updateFolderPath, 'UPDATED', 'UPDATED', ); this.logger.info('Node will now restart!'); process.exit(1); } this.logger.info('Unable to update ot-node to new version.'); } else { this.logger.info('Your node is running on the latest version!'); } } catch (error) { await this.handleError(error.message); } return Command.repeat(); } async recover(command) { await this.handleError(command.message); return Command.repeat(); } async handleError(errorMessage) { this.logger.error(`Error in update command: ${errorMessage}`); } /** * Builds default otnodeUpdateCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'otnodeUpdateCommand', delay: 0, period: 15 * 60 * 1000, transactional: false, priority: COMMAND_PRIORITY.HIGH, }; Object.assign(command, map); return command; } } export default OtnodeUpdateCommand; ================================================ FILE: src/commands/common/send-telemetry-command.js ================================================ import { createRequire } from 'module'; import Command from '../command.js'; import { SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES, COMMAND_PRIORITY, } from '../../constants/constants.js'; const require = createRequire(import.meta.url); const pjson = require('../../../package.json'); class SendTelemetryCommand extends Command { constructor(ctx) { super(ctx); this.logger = ctx.logger; this.config = ctx.config; this.networkModuleManager = ctx.networkModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.telemetryModuleManager = ctx.telemetryModuleManager; } /** * Performs code update by fetching new code from github repo * @param command */ async execute() { if ( !this.config.modules.telemetry.enabled || !this.telemetryModuleManager.getModuleConfiguration().sendToSignalingService ) { return Command.empty(); } try { const events = (await this.getUnpublishedEvents()) || []; const blockchainsNodeInfo = []; const blockchainImplementations = this.blockchainModuleManager.getImplementationNames(); for (const implementation of blockchainImplementations) { const blockchainInfo = { blockchain_id: implementation, // eslint-disable-next-line no-await-in-loop identity_id: await this.blockchainModuleManager.getIdentityId(implementation), operational_wallet: this.blockchainModuleManager.getPublicKeys(implementation)[0], management_wallet: this.blockchainModuleManager.getManagementKey(implementation), }; blockchainsNodeInfo.push(blockchainInfo); } const tripleStoreNodeInfo = []; const tripleStoreImplementations = this.tripleStoreModuleManager.getImplementationNames(); for (const implementation of tripleStoreImplementations) { const tripleStoreInfo = { implementationName: implementation, }; tripleStoreNodeInfo.push(tripleStoreInfo); } const nodeData = { version: pjson.version, identity: this.networkModuleManager.getPeerId().toB58String(), hostname: this.config.hostname, triple_stores: tripleStoreNodeInfo, auto_update_enabled: this.config.modules.autoUpdater.enabled, multiaddresses: this.networkModuleManager.getMultiaddrs(), blockchains: blockchainsNodeInfo, }; const isDataSuccessfullySent = await this.telemetryModuleManager.sendTelemetryData( nodeData, events, ); if (isDataSuccessfullySent && events?.length > 0) { await this.removePublishedEvents(events); } } catch (error) { await this.handleError(error.message); } return Command.repeat(); } async recover(command) { await this.handleError(command.message); return Command.repeat(); } async handleError(errorMessage) { this.logger.error(`Error in send telemetry command: ${errorMessage}`); } /** * Builds default sendTelemetryCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'sendTelemetryCommand', delay: 0, data: {}, period: SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES * 60 * 1000, transactional: false, priority: COMMAND_PRIORITY.MEDIUM, }; Object.assign(command, map); return command; } async getUnpublishedEvents() { return this.repositoryModuleManager.getUnpublishedEvents(); } async removePublishedEvents(events) { const ids = events.map((event) => event.id); await this.repositoryModuleManager.destroyEvents(ids); } } export default SendTelemetryCommand; ================================================ FILE: src/commands/common/send-transaction-command.js ================================================ import Command from '../command.js'; import { EXPECTED_TRANSACTION_ERRORS, OPERATION_ID_STATUS } from '../../constants/constants.js'; class SendTransactionCommand extends Command { async sendTransactionAndHandleResult(transactionCompletePromise, data, command) { const { blockchain, agreementId, epoch, operationId, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, contract, tokenId, stateIndex, txGasPrice, } = data; const sendTransactionOperationId = this.operationIdService.generateId(); let txSuccess; let msgBase; try { this.operationIdService.emitChangeEvent( this.txStartStatus, sendTransactionOperationId, blockchain, agreementId, epoch, operationId, ); txSuccess = await transactionCompletePromise; } catch (error) { this.logger.warn( `Failed to execute ${command.name}, Error Message: ${error.message} for the Service Agreement ` + `with the ID: ${agreementId}, Blockchain: ${blockchain}, Contract: ${contract}, ` + `Token ID: ${tokenId},` + `Epoch: ${epoch}, State Index: ${stateIndex}, Operation ID: ${operationId}, ` + `Closest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + `Right neighborhood edge: ${rightNeighborhoodEdge}, ` + `Retry number: ${this.commandRetryNumber - command.retries + 1}.`, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, sendTransactionOperationId, blockchain, error.message, this.txErrorType, ); txSuccess = false; if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.NODE_ALREADY_SUBMITTED_COMMIT)) { msgBase = 'Node has already submitted commit. Finishing'; } else if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.NODE_ALREADY_REWARDED)) { msgBase = 'Node already rewarded. Finishing'; } else if ( error.message.includes(EXPECTED_TRANSACTION_ERRORS.SERVICE_AGREEMENT_DOESNT_EXIST) ) { msgBase = 'Service agreement doesnt exist. Finishing'; } else if ( error.message.includes( EXPECTED_TRANSACTION_ERRORS.INVALID_PROXIMITY_SCORE_FUNCTIONS_PAIR_ID, ) ) { msgBase = 'Invalid proximity score functions pair id. Finishing'; } else if ( error.message.includes(EXPECTED_TRANSACTION_ERRORS.INVALID_SCORE_FUNCTION_ID) ) { msgBase = 'Invalid score function id. Finishing'; } else if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.COMMIT_WINDOW_CLOSED)) { msgBase = 'Commit window closed. Finishing'; } else if ( error.message.includes(EXPECTED_TRANSACTION_ERRORS.NODE_NOT_IN_SHARDING_TABLE) ) { msgBase = 'Node not in sharding table. Finishing'; } else if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.PROOF_WINDOW_CLOSED)) { msgBase = 'Proof window closed. Finishing'; } else if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.NODE_NOT_AWARDED)) { msgBase = 'Node not awarded. Finishing'; } else if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.WRONG_MERKLE_PROOF)) { msgBase = 'Wrong merkle proof. Finishing'; } else if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.INSUFFICIENT_FUNDS)) { msgBase = 'Insufficient funds. Finishing'; if (this.insufficientFundsErrorReceived) { await this.insufficientFundsErrorReceived(command.data); } } else { let newGasPrice; if ( error.message.includes(EXPECTED_TRANSACTION_ERRORS.TIMEOUT_EXCEEDED) || error.message.includes(EXPECTED_TRANSACTION_ERRORS.TOO_LOW_PRIORITY) ) { newGasPrice = Math.ceil(txGasPrice * this.txGasIncreaseFactor); } else { newGasPrice = null; } Object.assign(command, { data: { ...command.data, gasPrice: newGasPrice }, message: error.message, }); return Command.retry(); } } if (txSuccess) { this.operationIdService.emitChangeEvent( this.txEndStatus, sendTransactionOperationId, blockchain, agreementId, epoch, operationId, ); msgBase = 'Successfully executed'; this.operationIdService.emitChangeEvent( this.operationEndStatus, operationId, blockchain, agreementId, epoch, ); } this.logger.trace( `${msgBase} ${command.name} for the Service Agreement with the ID: ${agreementId}, ` + `Blockchain: ${blockchain}, Contract: ${contract}, Token ID: ${tokenId}, ` + `Epoch: ${epoch}, ` + `State Index: ${stateIndex}, Operation ID: ${operationId}, ` + `Closest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + `Right neighborhood edge: ${rightNeighborhoodEdge}, ` + `Retry number: ${this.commandRetryNumber - command.retries + 1}`, ); return Command.empty(); } /** * Builds default sendTransactionCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'sendTransactionCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default SendTransactionCommand; ================================================ FILE: src/commands/common/sharding-table-check-command.js ================================================ import Command from '../command.js'; import { COMMAND_PRIORITY, SHARDING_TABLE_CHECK_COMMAND_FREQUENCY_MILLS, } from '../../constants/constants.js'; class ShardingTableCheckCommand extends Command { constructor(ctx) { super(ctx); this.logger = ctx.logger; this.config = ctx.config; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.shardingTableService = ctx.shardingTableService; } /** * Checks sharding table size on blockchain and compares to local * If not equal, removes local and pulls new from blockchain * @param command */ async execute() { const repositoryTransaction = await this.repositoryModuleManager.transaction(); try { const promises = this.blockchainModuleManager .getImplementationNames() .map(async (blockchainId) => { this.logger.debug( `Performing sharding table check for blockchain ${blockchainId}.`, ); const shardingTableLength = await this.blockchainModuleManager.getShardingTableLength(blockchainId); const totalNodesNumber = await this.repositoryModuleManager.getPeersCount( blockchainId, ); if (shardingTableLength !== totalNodesNumber) { return this.shardingTableService.pullBlockchainShardingTable( blockchainId, repositoryTransaction, ); } }); await Promise.all(promises); await repositoryTransaction.commit(); } catch (error) { await repositoryTransaction.rollback(); await this.handleError(error.message); } return Command.repeat(); } async recover(command) { await this.handleError(command.message); return Command.repeat(); } async handleError(errorMessage) { this.logger.error(`Error in sharding table check command: ${errorMessage}`); } /** * Builds default shardingTableCheckCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'shardingTableCheckCommand', delay: 0, data: {}, period: SHARDING_TABLE_CHECK_COMMAND_FREQUENCY_MILLS, priority: COMMAND_PRIORITY.HIGHEST, isBlocking: true, transactional: false, }; Object.assign(command, map); return command; } } export default ShardingTableCheckCommand; ================================================ FILE: src/commands/common/validate-asset-command.js ================================================ import Command from '../command.js'; import { ERROR_TYPE, OPERATION_ID_STATUS, LOCAL_STORE_TYPES, PARANET_ACCESS_POLICY, } from '../../constants/constants.js'; class ValidateAssetCommand extends Command { constructor(ctx) { super(ctx); this.blockchainModuleManager = ctx.blockchainModuleManager; this.ualService = ctx.ualService; this.dataService = ctx.dataService; this.validationService = ctx.validationService; this.paranetService = ctx.paranetService; this.errorType = ERROR_TYPE.VALIDATE_ASSET_ERROR; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { operationId, blockchain, contract, tokenId, storeType = LOCAL_STORE_TYPES.TRIPLE, paranetUAL, } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.VALIDATE_ASSET_START, ); // TODO: Validate number of triplets and other stuff we did before so it matches like we did it in v6 const cachedData = await this.operationIdService.getCachedOperationIdData(operationId); const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); // backwards compatibility const cachedAssertion = cachedData.datasetRoot || cachedData.public.assertionId; const cachedDataset = cachedData.dataset || cachedData.public.assertion; // V0 backwards compatibility if (cachedData.private?.assertionId && cachedData.private?.assertion) { this.logger.info( `Validating asset's private assertion with id: ${cachedData.private.assertionId} ual: ${ual}`, ); try { await this.validationService.validateDatasetRoot( cachedData.private.assertion, cachedData.private.assertionId, ); } catch (error) { await this.handleError( operationId, blockchain, error.message, this.errorType, true, ); return Command.empty(); } } await this.validationService.validateDatasetRoot(cachedDataset, cachedAssertion); let paranetId; if (storeType === LOCAL_STORE_TYPES.TRIPLE_PARANET) { try { const { blockchain: paranetBlockchain, contract: paranetContract, knowledgeCollectionId: paranetKnowledgeCollectionId, knowledgeAssetId: paranetKnowledgeAssetId, } = this.ualService.resolveUAL(paranetUAL); if (!paranetKnowledgeAssetId) { await this.handleError( operationId, blockchain, `Invalid paranet UAL: ${paranetUAL} . Knowledge asset token id is required!`, this.errorType, true, ); return Command.empty(); } paranetId = this.paranetService.constructParanetId( paranetContract, paranetKnowledgeCollectionId, paranetKnowledgeAssetId, ); const paranetExists = await this.blockchainModuleManager.paranetExists( paranetBlockchain, paranetId, ); if (!paranetExists) { await this.handleError( operationId, blockchain, `Paranet: ${paranetId} doesn't exist.`, this.errorType, true, ); return Command.empty(); } const nodesAccessPolicy = await this.blockchainModuleManager.getNodesAccessPolicy( blockchain, paranetId, ); if (nodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const isPermissionedNode = await this.blockchainModuleManager.isPermissionedNode( blockchain, paranetId, identityId, ); if (!isPermissionedNode) { await this.handleError( operationId, blockchain, `Node is not part of permissioned paranet ${paranetId} because node with id ${identityId} is not a permissioned node.`, this.errorType, true, ); return Command.empty(); } } else { await this.handleError( operationId, blockchain, `Paranet ${paranetId} is not permissioned paranet.`, this.errorType, true, ); return Command.empty(); } } catch (error) { await this.handleError( operationId, blockchain, error.message, this.errorType, true, ); return Command.empty(); } } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.VALIDATE_ASSET_END, ); return this.continueSequence( { ...command.data, paranetId, retry: undefined, period: undefined }, command.sequence, ); } async retryFinished(command) { const { blockchain, contract, tokenId, operationId } = command.data; const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); await this.handleError( operationId, blockchain, `Max retry count for command: ${command.name} reached! Unable to validate ual: ${ual}`, this.errorType, true, ); } /** * Builds default validateAssetCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'validateAssetCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default ValidateAssetCommand; ================================================ FILE: src/commands/paranet/paranet-sync-command.js ================================================ /* eslint-disable no-await-in-loop */ import { setTimeout } from 'timers/promises'; import Command from '../command.js'; import { ERROR_TYPE, PARANET_SYNC_FREQUENCY_MILLS, OPERATION_ID_STATUS, PARANET_SYNC_PARAMETERS, PARANET_SYNC_KA_COUNT, PARANET_SYNC_RETRIES_LIMIT, PARANET_SYNC_RETRY_DELAY_MS, OPERATION_STATUS, PARANET_NODES_ACCESS_POLICIES, PARANET_ACCESS_POLICY, TRIPLES_VISIBILITY, DKG_METADATA_PREDICATES, TRIPLE_STORE_REPOSITORIES, COMMAND_PRIORITY, } from '../../constants/constants.js'; class ParanetSyncCommand extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.blockchainModuleManager = ctx.blockchainModuleManager; this.tripleStoreService = ctx.tripleStoreService; this.ualService = ctx.ualService; this.paranetService = ctx.paranetService; this.getService = ctx.getService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.errorType = ERROR_TYPE.PARANET.PARANET_SYNC_ERROR; } // TODO: Fix logs? Use word 'Knowledge Collection' or 'Collection' instead of 'Asset'. async execute(command) { const { blockchain, operationId, paranetUAL, paranetId, nodesAccessPolicy } = command.data; const paranetNodesAccessPolicy = PARANET_NODES_ACCESS_POLICIES[nodesAccessPolicy]; this.logger.info( `Paranet sync: Starting paranet sync for paranet: ${paranetUAL} (${paranetId}), operation ID: ${operationId}, access policy ${paranetNodesAccessPolicy}`, ); const countContract = ( await this.blockchainModuleManager.getParanetKnowledgeCollectionCount( blockchain, paranetId, ) ).toNumber(); const countDatabase = await this.repositoryModuleManager.getParanetKcCount(paranetUAL); const missingUALs = ( await this.blockchainModuleManager.getParanetKnowledgeCollectionLocatorsWithPagination( blockchain, paranetId, countDatabase, countContract, ) ).map(({ knowledgeCollectionStorageContract, knowledgeCollectionTokenId }) => this.ualService.deriveUAL( blockchain, knowledgeCollectionStorageContract, knowledgeCollectionTokenId, ), ); await this.repositoryModuleManager.createParanetKcRecords( paranetUAL, blockchain, missingUALs, ); const countSynced = await this.repositoryModuleManager.getParanetKcSyncedCount(paranetUAL); const countUnsynced = await this.repositoryModuleManager.getParanetKcUnsyncedCount( paranetUAL, ); this.logger.info( `Paranet sync: Paranet: ${paranetUAL} (${paranetId}) Total count of Paranet KAs in the contract: ${countContract}; Synced KAs count: ${countSynced}; Total count of missed KAs: ${countUnsynced}`, ); if (countUnsynced === 0) { this.logger.info( `Paranet sync: No new assets to sync for paranet: ${paranetUAL} (${paranetId}), operation ID: ${operationId}!`, ); return Command.repeat(); } // #region Sync batch; const syncBatch = await this.repositoryModuleManager.getParanetKcSyncBatch( paranetUAL, PARANET_SYNC_RETRIES_LIMIT, PARANET_SYNC_RETRY_DELAY_MS, PARANET_SYNC_KA_COUNT, ); this.logger.info( `Paranet sync: Attempting to sync ${syncBatch.length} missed assets for paranet: ${paranetUAL} (${paranetId}), operation ID: ${operationId}!`, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.PARANET.PARANET_SYNC_MISSED_KAS_SYNC_START, ); const syncResults = await Promise.all( syncBatch.map(({ ual }) => this.syncKc(paranetUAL, ual, paranetId, nodesAccessPolicy, operationId), ), ); const countSyncSuccessful = syncResults.filter((err) => !err).length; const countSyncFailed = syncResults.length - countSyncSuccessful; this.logger.info( `Paranet sync: Successful missed assets syncs: ${countSyncSuccessful}; ` + `Failed missed assets syncs: ${countSyncFailed} for paranet: ${paranetUAL} ` + `(${paranetId}), operation ID: ${operationId}!`, ); // #endregion await this.operationIdService.updateOperationIdStatusWithValues( operationId, blockchain, OPERATION_ID_STATUS.PARANET.PARANET_SYNC_MISSED_KAS_SYNC_END, countSyncSuccessful, countSyncFailed, ); return Command.repeat(); } /** **NOTE:** Throws errors! */ async syncKcState( paranetUAL, ual, stateIndex, assertionId, paranetId, paranetNodesAccessPolicy, ) { const { blockchain, contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); const getOperationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.GET.GET_START, blockchain, ); // #region GET (LOCAL) this.operationIdService.updateOperationIdStatus( getOperationId, blockchain, OPERATION_ID_STATUS.GET.GET_INIT_START, ); this.repositoryModuleManager.createOperationRecord( this.getService.getOperationName(), getOperationId, OPERATION_STATUS.IN_PROGRESS, ); this.logger.debug( `Paranet sync: Get for ${ual} with operation id ${getOperationId} initiated.`, ); const maxAttempts = PARANET_SYNC_PARAMETERS.GET_RESULT_POLLING_MAX_ATTEMPTS; const pollingInterval = PARANET_SYNC_PARAMETERS.GET_RESULT_POLLING_INTERVAL_MILLIS; let attempt = 0; let getResult; const contentType = paranetNodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED ? TRIPLES_VISIBILITY.ALL : TRIPLES_VISIBILITY.PUBLIC; await this.commandExecutor.add({ name: 'getCommand', sequence: [], delay: 0, data: { operationId: getOperationId, id: ual, blockchain, contract, knowledgeCollectionId, state: assertionId, ual: this.ualService.deriveUAL(blockchain, contract, knowledgeCollectionId), includeMetadata: true, contentType, paranetId, paranetUAL, paranetNodesAccessPolicy, paranetSync: true, }, transactional: false, }); attempt = 0; do { await setTimeout(pollingInterval); getResult = await this.operationIdService.getOperationIdRecord(getOperationId); attempt += 1; } while ( attempt < maxAttempts && getResult?.status !== OPERATION_ID_STATUS.FAILED && getResult?.status !== OPERATION_ID_STATUS.COMPLETED ); // #endregion NETWORK END if (getResult?.status !== OPERATION_ID_STATUS.COMPLETED) { throw new Error( `Unable to sync Knowledge Collection Id: ${knowledgeCollectionId}, for contract: ${contract}, state index: ${stateIndex}, blockchain: ${blockchain}, GET result: ${JSON.stringify( getResult, )}`, ); } const data = await this.operationIdService.getCachedOperationIdData(getOperationId); this.logger.debug( `Paranet sync: ${ data.assertion.public.length + (data.assertion?.private?.length || 0) } nquads found for asset with ual: ${ual}, state index: ${stateIndex}, assertionId: ${assertionId}`, ); const metadata = {}; data.metadata.forEach((line) => { for (const predicate of Object.values(DKG_METADATA_PREDICATES)) { if (line.includes(predicate)) { switch (predicate) { case DKG_METADATA_PREDICATES.PUBLISHED_BY: metadata.publisherKey = line .split(' ')[2] .split('/')[1] .replaceAll('>', ''); break; case DKG_METADATA_PREDICATES.PUBLISHED_AT_BLOCK: metadata.blockNumber = line.split(' ')[2].trim().replaceAll('"', ''); break; case DKG_METADATA_PREDICATES.PUBLISH_TX: metadata.txHash = line.split(' ')[2].trim().replaceAll('"', ''); break; case DKG_METADATA_PREDICATES.BLOCK_TIME: metadata.blockTimestamp = new Date( line .split(' ')[2] .trim() .replaceAll('"', '') .replaceAll( '^^', '', ), ).getTime() / 1000; break; default: break; } } } }); // Delete old insert time as it's updated on each sync both paranet triples and private data after permissioned sync await this.tripleStoreService.deletePublishTimestampMetadata( TRIPLE_STORE_REPOSITORIES.DKG, ual, ); await this.tripleStoreService.insertKnowledgeCollection( TRIPLE_STORE_REPOSITORIES.DKG, ual, data.assertion, metadata, 5, 50, paranetUAL, contentType, ); } /** * Syncs all states ("merkle roots") of a Knowledge Collection in a paranet. * * @param {string} paranetUAL Universal Asset Locator of the paranet * @param {string} ual Universal Asset Locator of the Knowledge Collection * @param {string} paranetId Id of paranet, stored on-chain. Provided in command options. * @param {'OPEN'|'PERMISSIONED'} paranetNodesAccessPolicy Node access policy, enum string indicating paranet type. * @param {string} operationId Local database id of sync operation. Needed for logging. * * @returns {Promise} Returns `null` if sync of all states was successful, otherwise `Error` which broke the operation. */ async syncKc(paranetUAL, ual, paranetId, paranetNodesAccessPolicy, operationId) { try { this.logger.info( `Paranet sync: Syncing asset: ${ual} for paranet: ${paranetId}, operation ID: ${operationId}`, ); const { blockchain, contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); const merkleRoots = await this.blockchainModuleManager.getKnowledgeCollectionMerkleRoots( blockchain, contract, knowledgeCollectionId, ); for (let stateIndex = 0; stateIndex < merkleRoots.length; stateIndex += 1) { this.logger.debug( `Paranet sync: Fetching state: ${merkleRoots[stateIndex].merkleRoot} index: ${ stateIndex + 1 } of ${merkleRoots.length} for asset with ual: ${ual}.`, ); await this.syncKcState( paranetUAL, ual, stateIndex, merkleRoots[stateIndex].merkleRoot, paranetId, paranetNodesAccessPolicy, ); } await this.repositoryModuleManager.paranetKcMarkAsSynced(paranetUAL, ual); return null; } catch (error) { this.logger.warn( `Paranet sync: Failed to sync asset: ${ual} for paranet: ${paranetId}, error: ${error}`, ); await this.repositoryModuleManager.paranetKcIncrementRetries( paranetUAL, ual, `${error}`, ); return error; } } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to execute ${command.name}. Error: ${command.message}`); return Command.repeat(); } /** * Builds default paranetSyncCommands * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'paranetSyncCommands', data: {}, transactional: false, period: PARANET_SYNC_FREQUENCY_MILLS, priority: COMMAND_PRIORITY.LOW, }; Object.assign(command, map); return command; } } export default ParanetSyncCommand; ================================================ FILE: src/commands/paranet/start-paranet-sync-commands.js ================================================ import Command from '../command.js'; import { ERROR_TYPE, PARANET_SYNC_FREQUENCY_MILLS, OPERATION_ID_STATUS, COMMAND_PRIORITY, } from '../../constants/constants.js'; class StartParanetSyncCommands extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.ualService = ctx.ualService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.paranetService = ctx.paranetService; this.errorType = ERROR_TYPE.PARANET.START_PARANET_SYNC_ERROR; } async execute() { const promises = []; this.config.assetSync?.syncParanets.forEach(async (paranetUAL) => { const operationId = this.operationIdService.generateId( OPERATION_ID_STATUS.PARANET.PARANET_SYNC_START, ); const { blockchain, contract, knowledgeCollectionId, knowledgeAssetId } = this.ualService.resolveUAL(paranetUAL); if (!knowledgeAssetId) { this.logger.error( `Invalid paranet UAL: ${paranetUAL} . Knowledge asset token id is required!`, ); return Command.empty(); } const paranetId = this.paranetService.constructParanetId( contract, knowledgeCollectionId, knowledgeAssetId, ); const nodesAccessPolicy = await this.blockchainModuleManager.getNodesAccessPolicy( blockchain, paranetId, ); const commandData = { blockchain, contract, knowledgeCollectionId, knowledgeAssetId, paranetUAL, paranetId, nodesAccessPolicy, operationId, }; promises.push( this.commandExecutor.add({ name: `paranetSyncCommand-${paranetId}`, data: commandData, period: PARANET_SYNC_FREQUENCY_MILLS, }), ); }); await Promise.all(promises); return Command.empty(); } /** * Recover system from failure * @param command * @param error */ async recover(command) { this.logger.warn(`Failed to execute ${command.name}. Error: ${command.message}`); return Command.repeat(); } /** * Builds default startParanetSyncCommands * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'startParanetSyncCommands', data: {}, transactional: false, priority: COMMAND_PRIORITY.LOW, }; Object.assign(command, map); return command; } } export default StartParanetSyncCommands; ================================================ FILE: src/commands/protocols/ask/receiver/v1.0.0/v1-0-0-handle-ask-request-command.js ================================================ import HandleProtocolMessageCommand from '../../../common/handle-protocol-message-command.js'; import { ERROR_TYPE, NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS, } from '../../../../../constants/constants.js'; class HandleAskRequestCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); this.operationService = ctx.askService; this.tripleStoreService = ctx.tripleStoreService; this.pendingStorageService = ctx.pendingStorageService; this.paranetService = ctx.paranetService; this.errorType = ERROR_TYPE.ASK.ASK_REQUEST_REMOTE_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.ASK.ASK_REMOTE_START; this.operationEndEvent = OPERATION_ID_STATUS.ASK.ASK_REMOTE_END; } async prepareMessage(commandData) { const { ual, operationId, blockchain } = commandData; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.ASK.ASK_REMOTE_START, ); const knowledgeCollectionExistsInUnifiedGraph = await this.tripleStoreService.checkIfKnowledgeCollectionExistsInUnifiedGraph(ual); if (knowledgeCollectionExistsInUnifiedGraph) { await this.operationService.markOperationAsCompleted( operationId, blockchain, knowledgeCollectionExistsInUnifiedGraph, [ OPERATION_ID_STATUS.ASK.ASK_FETCH_FROM_NODES_END, OPERATION_ID_STATUS.ASK.ASK_END, OPERATION_ID_STATUS.COMPLETED, ], ); } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.ASK.ASK_REMOTE_END, ); return knowledgeCollectionExistsInUnifiedGraph ? { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: { knowledgeCollectionExistsInUnifiedGraph }, } : { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `Unable to find knowledge collection ${ual}` }, }; } /** * Builds default handleAskRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0HandleAskRequestCommand', transactional: false, errorType: ERROR_TYPE.ASK.ASK_REQUEST_REMOTE_ERROR, }; Object.assign(command, map); return command; } } export default HandleAskRequestCommand; ================================================ FILE: src/commands/protocols/ask/sender/ask-find-shard-command.js ================================================ import FindShardCommand from '../../common/find-shard-command.js'; import { ERROR_TYPE, OPERATION_ID_STATUS } from '../../../../constants/constants.js'; class AskFindShardCommand extends FindShardCommand { constructor(ctx) { super(ctx); this.operationService = ctx.askService; this.errorType = ERROR_TYPE.FIND_SHARD.ASK_FIND_SHARD_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.ASK.ASK_FIND_NODES_START; this.operationEndEvent = OPERATION_ID_STATUS.ASK.ASK_FIND_NODES_END; } // eslint-disable-next-line no-unused-vars getOperationCommandSequence(nodePartOfShard, commandData) { return []; } /** * Builds default askFindShardCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'askFindShardCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default AskFindShardCommand; ================================================ FILE: src/commands/protocols/ask/sender/ask-schedule-messages-command.js ================================================ import ProtocolScheduleMessagesCommand from '../../common/protocol-schedule-messages-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../../constants/constants.js'; class AskScheduleMessagesCommand extends ProtocolScheduleMessagesCommand { constructor(ctx) { super(ctx); this.operationService = ctx.askService; this.errorType = ERROR_TYPE.ASK.ASK_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.ASK.ASK_FETCH_FROM_NODES_START; this.operationEndEvent = OPERATION_ID_STATUS.ASK.ASK_FETCH_FROM_NODES_END; } getNextCommandData(command) { return { ...super.getNextCommandData(command), ual: command.data.ual, operationId: command.data.operationId, minimumNumberOfNodeReplications: command.data.minimumNumberOfNodeReplications, }; } /** * Builds default askScheduleMessagesCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'askScheduleMessagesCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default AskScheduleMessagesCommand; ================================================ FILE: src/commands/protocols/ask/sender/network-ask-command.js ================================================ import NetworkProtocolCommand from '../../common/network-protocol-command.js'; import { ERROR_TYPE } from '../../../../constants/constants.js'; class NetworkAskCommand extends NetworkProtocolCommand { constructor(ctx) { super(ctx); this.operationService = ctx.askService; this.ualService = ctx.ualService; this.errorType = ERROR_TYPE.ASK.ASK_NETWORK_ERROR; } /** * Builds default networkGetCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'networkAskCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default NetworkAskCommand; ================================================ FILE: src/commands/protocols/ask/sender/v1.0.0/v1-0-0-ask-request-command.js ================================================ import ProtocolRequestCommand from '../../../common/protocol-request-command.js'; import { NETWORK_MESSAGE_TIMEOUT_MILLS, ERROR_TYPE, OPERATION_REQUEST_STATUS, OPERATION_STATUS, } from '../../../../../constants/constants.js'; class AskRequestCommand extends ProtocolRequestCommand { constructor(ctx) { super(ctx); this.operationService = ctx.askService; this.operationIdService = ctx.operationIdService; this.errorType = ERROR_TYPE.ASK.ASK_REQUEST_ERROR; } async shouldSendMessage(command) { const { operationId } = command.data; const { status } = await this.operationService.getOperationStatus(operationId); if (status === OPERATION_STATUS.IN_PROGRESS) { return true; } this.logger.trace( `${command.name} skipped for operationId: ${operationId} with status ${status}`, ); return false; } async prepareMessage(command) { const { ual, operationId, numberOfFoundNodes, blockchain } = command.data; return { ual, operationId, numberOfFoundNodes, blockchain, }; } messageTimeout() { return NETWORK_MESSAGE_TIMEOUT_MILLS.ASK.REQUEST; } async handleAck(command, responseData) { if (responseData?.knowledgeCollectionExistsInUnifiedGraph) { await this.operationService.processResponse( command, OPERATION_REQUEST_STATUS.COMPLETED, responseData, ); return ProtocolRequestCommand.empty(); } return this.handleNack(command, responseData); } /** * Builds default askRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0AskRequestCommand', delay: 0, retries: 0, transactional: false, }; Object.assign(command, map); return command; } } export default AskRequestCommand; ================================================ FILE: src/commands/protocols/common/find-curated-paranet-nodes-command.js ================================================ import Command from '../../command.js'; import { OPERATION_ID_STATUS } from '../../../constants/constants.js'; class FindCuratedParanetNodesCommand extends Command { constructor(ctx) { super(ctx); this.operationService = ctx.getService; this.networkModuleManager = ctx.networkModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.shardingTableService = ctx.shardingTableService; this.cryptoService = ctx.cryptoService; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { operationId, blockchain, errorType, networkProtocols, paranetId, minAckResponses } = command.data; this.errorType = errorType; this.logger.debug( `Searching for paranet (${paranetId}) node(s) for operationId: ${operationId}`, ); // TODO: protocol selection const paranetNodes = []; const foundNodes = await this.findNodes(blockchain, operationId, paranetId); for (const node of foundNodes) { if (node.id !== this.networkModuleManager.getPeerId().toB58String()) { paranetNodes.push({ id: node.id, protocol: networkProtocols[0] }); } } this.logger.debug( `Found ${paranetNodes.length} paranet (${paranetId}) node(s) for operationId: ${operationId}`, ); this.logger.trace( `Found paranet (${paranetId}) nodes: ${JSON.stringify( paranetNodes.map((node) => node.id), null, 2, )}`, ); if (paranetNodes.length < minAckResponses) { await this.handleError( operationId, blockchain, `Unable to find enough paranet (${paranetId}) nodes for operationId: ${operationId}. Minimum number of nodes required: ${minAckResponses}`, this.errorType, true, ); return Command.empty(); } return this.continueSequence( { ...command.data, leftoverNodes: paranetNodes, numberOfFoundNodes: paranetNodes.length, }, command.sequence, ); } async findNodes(blockchainId, operationId, paranetId) { await this.operationIdService.updateOperationIdStatus( operationId, blockchainId, OPERATION_ID_STATUS.FIND_CURATED_PARANET_NODES_START, ); const paranetCuratedNodes = await this.blockchainModuleManager.getParanetCuratedNodes( blockchainId, paranetId, ); const paranetCuratedPeerIds = paranetCuratedNodes.map((node) => this.cryptoService.convertHexToAscii(node.nodeId), ); const paranetCuratedNodePeerRecords = await this.repositoryModuleManager.getPeerRecordsByIds( blockchainId, paranetCuratedPeerIds, ); const availableParanetNodes = paranetCuratedNodePeerRecords.filter( (node) => node.lastSeen >= node.lastDialed, ); const nodesFound = await Promise.all( availableParanetNodes.map(({ peerId }) => this.shardingTableService.findPeerAddressAndProtocols(peerId), ), ); await this.operationIdService.updateOperationIdStatus( operationId, blockchainId, OPERATION_ID_STATUS.FIND_CURATED_PARANET_NODES_END, ); return nodesFound; } /** * Builds default findCuratedParanetNodesCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'findCuratedParanetNodesCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default FindCuratedParanetNodesCommand; ================================================ FILE: src/commands/protocols/common/find-shard-command.js ================================================ import Command from '../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../constants/constants.js'; class FindShardCommand extends Command { constructor(ctx) { super(ctx); this.networkModuleManager = ctx.networkModuleManager; this.shardingTableService = ctx.shardingTableService; this.errorType = ERROR_TYPE.FIND_SHARD.FIND_SHARD_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.FIND_NODES_START; this.operationEndEvent = OPERATION_ID_STATUS.FIND_NODES_END; } // eslint-disable-next-line no-unused-vars getOperationCommandSequence(nodePartOfShard, commandData) { return []; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { operationId, blockchain, datasetRoot, minimumNumberOfNodeReplications } = command.data; this.logger.debug( `Searching for shard for operationId: ${operationId}, dataset root: ${datasetRoot}`, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, this.operationStartEvent, ); this.minAckResponses = this.operationService.getMinAckResponses( minimumNumberOfNodeReplications, ); const networkProtocols = this.operationService.getNetworkProtocols(); const shardNodes = []; let nodePartOfShard = false; const currentPeerId = this.networkModuleManager.getPeerId().toB58String(); const foundNodes = await this.findShardNodes(blockchain); for (const node of foundNodes) { if (node.id === currentPeerId) { nodePartOfShard = true; } else { shardNodes.push({ id: node.id, protocol: networkProtocols[0] }); } } const commandSequence = this.getOperationCommandSequence(nodePartOfShard, command.data); command.sequence.push(...commandSequence); this.logger.debug( `Found ${ shardNodes.length + (nodePartOfShard ? 1 : 0) } node(s) for operationId: ${operationId}`, ); // TODO: Log local node this.logger.trace( `Found shard: ${JSON.stringify( shardNodes.map((node) => node.id), null, 2, )}`, ); if (shardNodes.length + (nodePartOfShard ? 1 : 0) < this.minAckResponses) { await this.handleError( operationId, blockchain, `Unable to find enough nodes for operationId: ${operationId}. Minimum number of nodes required: ${this.minAckResponses}`, this.errorType, true, ); return Command.empty(); } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, this.operationEndEvent, ); return this.continueSequence( { ...command.data, nodePartOfShard, leftoverNodes: shardNodes, numberOfFoundNodes: shardNodes.length + (nodePartOfShard ? 1 : 0), }, command.sequence, ); } async findShardNodes(blockchainId) { const shardNodes = await this.shardingTableService.findShard( blockchainId, true, // filter inactive nodes ); // TODO: Optimize this so it's returned by shardingTableService.findShard const nodesFound = await Promise.all( shardNodes.map(({ peerId }) => this.shardingTableService.findPeerAddressAndProtocols(peerId), ), ); return nodesFound; } /** * Builds default findShardCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'findShardCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default FindShardCommand; ================================================ FILE: src/commands/protocols/common/handle-protocol-message-command.js ================================================ import Command from '../../command.js'; import { NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS } from '../../../constants/constants.js'; class HandleProtocolMessageCommand extends Command { constructor(ctx) { super(ctx); this.ualService = ctx.ualService; this.networkModuleManager = ctx.networkModuleManager; this.operationIdService = ctx.operationIdService; this.shardingTableService = ctx.shardingTableService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.operationStartEvent = OPERATION_ID_STATUS.HANDLE_PROTOCOL_MESSAGE_START; this.operationEndEvent = OPERATION_ID_STATUS.HANDLE_PROTOCOL_MESSAGE_END; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { remotePeerId, operationId, protocol, blockchain } = command.data; this.operationIdService.emitChangeEvent(this.operationStartEvent, operationId, blockchain); try { const { messageType, messageData } = await this.prepareMessage(command.data); await this.networkModuleManager.sendMessageResponse( protocol, remotePeerId, messageType, operationId, messageData, ); } catch (error) { if (command.retries) { this.logger.warn(error.message); return Command.retry(); } await this.handleError(error.message, command); } this.networkModuleManager.removeCachedSession(operationId, remotePeerId); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, this.operationEndEvent, ); return Command.empty(); } async prepareMessage() { throw Error('prepareMessage not implemented'); } async validateShard(blockchain) { const peerId = this.networkModuleManager.getPeerId().toB58String(); const isNodePartOfShard = await this.shardingTableService.isNodePartOfShard( blockchain, peerId, ); return isNodePartOfShard; } async validateAssertionId(blockchain, contract, tokenId, assertionId, ual) { const blockchainAssertionId = await this.blockchainModuleManager.getKnowledgeCollectionMerkleRoot( blockchain, contract, tokenId, ); if (blockchainAssertionId !== assertionId) { throw Error( `Invalid assertion id for asset ${ual}. Received value from blockchain: ${blockchainAssertionId}, received value from request: ${assertionId}`, ); } } async validateReceivedData(operationId, datasetRoot, dataset, blockchain, isOperationV0) { this.logger.trace(`Validating shard for datasetRoot: ${datasetRoot}`); const isShardValid = await this.validateShard(blockchain); if (!isShardValid) { this.logger.warn( `Invalid shard on blockchain: ${blockchain}, operationId: ${operationId}`, ); return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: 'Invalid neighbourhood' }, }; } if (!isOperationV0) { try { await this.validationService.validateDatasetRoot(dataset, datasetRoot); } catch (error) { return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: error.message, }, }; } } return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: {} }; } async handleError(errorMessage, command) { const { operationId, blockchain, remotePeerId, protocol } = command.data; this.logger.error(`Command error (${this.errorType}): ${errorMessage}`); if (errorMessage !== null) { this.logger.debug(`Marking operation id ${operationId} as failed`); await this.operationIdService.removeOperationIdCache(operationId); } this.operationIdService.emitChangeEvent( this.errorType, operationId, blockchain, errorMessage, this.errorType, ); try { await this.networkModuleManager.sendMessageResponse( protocol, remotePeerId, NETWORK_MESSAGE_TYPES.RESPONSES.NACK, operationId, { errorMessage }, ); } catch (sendErr) { this.logger.debug( `Failed to send NACK to ${remotePeerId} for operation ${operationId}: ${sendErr.message}`, ); } this.networkModuleManager.removeCachedSession(operationId, remotePeerId); } } export default HandleProtocolMessageCommand; ================================================ FILE: src/commands/protocols/common/network-protocol-command.js ================================================ import Command from '../../command.js'; import { ERROR_TYPE } from '../../../constants/constants.js'; class NetworkProtocolCommand extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.blockchainModuleManager = ctx.blockchainModuleManager; this.errorType = ERROR_TYPE.NETWORK_PROTOCOL_ERROR; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { minimumNumberOfNodeReplications, batchSize } = command.data; const batchSizePar = this.operationService.getBatchSize(batchSize); const minAckResponses = this.operationService.getMinAckResponses( minimumNumberOfNodeReplications, ); const commandSequence = [ `${this.operationService.getOperationName()}ScheduleMessagesCommand`, ]; await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), delay: 0, data: { ...command.data, batchSize: batchSizePar, minAckResponses, errorType: this.errorType, }, transactional: false, }); return Command.empty(); } getBatchSize() { throw Error('getBatchSize not implemented'); } getMinAckResponses() { throw Error('getMinAckResponses not implemented'); } /** * Builds default protocolNetworkCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'protocolNetworkCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default NetworkProtocolCommand; ================================================ FILE: src/commands/protocols/common/protocol-message-command.js ================================================ import Command from '../../command.js'; import { NETWORK_MESSAGE_TYPES, OPERATION_REQUEST_STATUS } from '../../../constants/constants.js'; class ProtocolMessageCommand extends Command { constructor(ctx) { super(ctx); this.networkModuleManager = ctx.networkModuleManager; } async executeProtocolMessageCommand(command, messageType) { if (!(await this.shouldSendMessage(command))) { return Command.empty(); } const message = await this.prepareMessage(command); return this.sendProtocolMessage(command, message, messageType); } async shouldSendMessage() { return true; } async prepareMessage() { throw Error('prepareMessage not implemented'); } async sendProtocolMessage(command, message, messageType) { const { node, operationId } = command.data; const response = await this.networkModuleManager.sendMessage( node.protocol, node.id, messageType, operationId, message, this.messageTimeout(), ); this.networkModuleManager.removeCachedSession(operationId, node.id); switch (response.header.messageType) { case NETWORK_MESSAGE_TYPES.RESPONSES.BUSY: return this.handleBusy(command, response.data); case NETWORK_MESSAGE_TYPES.RESPONSES.NACK: return this.handleNack(command, response.data); case NETWORK_MESSAGE_TYPES.RESPONSES.ACK: return this.handleAck(command, response.data); default: await this.markResponseAsFailed( command, `Received unknown message type from node during ${command.name}`, ); return Command.empty(); } } messageTimeout() { throw Error('messageTimeout not implemented'); } async handleAck(command) { return this.continueSequence(command.data, command.sequence); } async handleBusy() { return Command.retry(); } async handleNack(command, responseData) { await this.markResponseAsFailed( command, `Received NACK response from node during ${command.name}. Error message: ${responseData.errorMessage}`, ); return Command.empty(); } async recover(command) { const { node, operationId } = command.data; this.networkModuleManager.removeCachedSession(operationId, node.id); await this.markResponseAsFailed(command, command.message); return Command.empty(); } async markResponseAsFailed(command, errorMessage) { await this.operationService.processResponse(command, OPERATION_REQUEST_STATUS.FAILED, { errorMessage, }); } async retryFinished(command) { await this.markResponseAsFailed( command, `Max number of retries for protocol message ${command.name} reached`, ); } } export default ProtocolMessageCommand; ================================================ FILE: src/commands/protocols/common/protocol-request-command.js ================================================ import Command from '../../command.js'; import ProtocolMessageCommand from './protocol-message-command.js'; import { NETWORK_MESSAGE_TYPES, OPERATION_REQUEST_STATUS } from '../../../constants/constants.js'; class ProtocolRequestCommand extends ProtocolMessageCommand { async execute(command) { const result = await this.executeProtocolMessageCommand( command, NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST, ); return result; } async handleAck(command, responseData) { await this.operationService.processResponse( command, OPERATION_REQUEST_STATUS.COMPLETED, responseData, ); return Command.empty(); } } export default ProtocolRequestCommand; ================================================ FILE: src/commands/protocols/common/protocol-schedule-messages-command.js ================================================ import Command from '../../command.js'; import { OPERATION_ID_STATUS } from '../../../constants/constants.js'; class ProtocolScheduleMessagesCommand extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.protocolService = ctx.protocolService; this.operationStartEvent = OPERATION_ID_STATUS.PROTOCOL_SCHEDULE_MESSAGE_START; this.operationEndEvent = OPERATION_ID_STATUS.PROTOCOL_SCHEDULE_MESSAGE_END; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { operationId, batchSize, leftoverNodes, numberOfFoundNodes, blockchain, minAckResponses, } = command.data; const currentBatchNodes = leftoverNodes.slice(0, batchSize); const currentBatchLeftoverNodes = batchSize < leftoverNodes.length ? leftoverNodes.slice(batchSize) : []; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, this.operationStartEvent, ); this.logger.debug( `Trying to ${this.operationService.getOperationName()} to batch of ${ currentBatchNodes.length }, leftover for retry: ${currentBatchLeftoverNodes.length}`, ); const nextCommandData = this.getNextCommandData(command); const addCommandPromises = currentBatchNodes.map(async (node) => { const commandSequence = this.protocolService.getSenderCommandSequence(node.protocol); await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), delay: 0, data: { ...nextCommandData, blockchain, operationId, node, numberOfFoundNodes, batchSize, minAckResponses, leftoverNodes: currentBatchLeftoverNodes, }, period: 5000, retries: 3, transactional: false, }); }); await Promise.all(addCommandPromises); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, this.operationEndEvent, ); return Command.empty(); } getNextCommandData(command) { const { datasetRoot, blockchain } = command.data; return { blockchain, datasetRoot, }; } /** * Builds default protocolScheduleMessagesCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'protocolScheduleMessagesCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default ProtocolScheduleMessagesCommand; ================================================ FILE: src/commands/protocols/common/validate-assertion-metadata-command.js ================================================ import Command from '../../command.js'; import { OPERATION_ID_STATUS } from '../../../constants/constants.js'; class ValidateAssertionMetadataCommand extends Command { constructor(ctx) { super(ctx); this.operationIdService = ctx.operationIdService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.dataService = ctx.dataService; this.operationStartEvent = OPERATION_ID_STATUS.VALIDATE_ASSERTION_METADATA_START; this.operationEndEvent = OPERATION_ID_STATUS.VALIDATE_ASSERTION_METADATA_END; } async execute(command) { const { operationId, ual, blockchain, merkleRoot, cachedMerkleRoot, byteSize, assertion } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, this.operationStartEvent, ); try { if (merkleRoot !== cachedMerkleRoot) { await this.handleError( operationId, blockchain, `Invalid Merkle Root for Knowledge Collection with UAL: ${ual}. Received value from blockchain: ${merkleRoot}, Cached value from publish operation: ${cachedMerkleRoot}`, this.errorType, true, ); } const calculatedAssertionSize = this.dataService.calculateAssertionSize( assertion.public ?? assertion, ); if (byteSize.toString() !== calculatedAssertionSize.toString()) { await this.handleError( operationId, blockchain, `Invalid Assertion Size for Knowledge Collection with UAL: ${ual}. Received value from blockchain: ${byteSize}, Calculated value: ${calculatedAssertionSize}`, this.errorType, true, ); } } catch (e) { await this.handleError(operationId, blockchain, e.message, this.errorType, true); return Command.empty(); } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, this.operationEndEvent, ); return this.continueSequence(command.data, command.sequence); } /** * Builds default validateAssertionMetadataCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'validateAssertionMetadataCommand', delay: 0, retries: 0, transactional: false, }; Object.assign(command, map); return command; } } export default ValidateAssertionMetadataCommand; ================================================ FILE: src/commands/protocols/finality/receiver/publish-finality-save-ack-command.js ================================================ import { COMMAND_PRIORITY, NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS, } from '../../../../constants/constants.js'; import Command from '../../../command.js'; class PublishFinalitySaveAckCommand extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.protocolService = ctx.protocolService; this.operationService = ctx.finalityService; this.networkModuleManager = ctx.networkModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { ual, publishOperationId, blockchain, operationId, remotePeerId, state } = command.data; let ualWithState = ual; if (state) { ualWithState = `${ual}:${state}`; } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.FINALITY.PUBLISH_FINALITY_REMOTE_START, ); let response; let success; try { await this.repositoryModuleManager.saveFinalityAck( publishOperationId, ualWithState, remotePeerId, ); success = true; response = { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: { message: `Acknowledged storing of ${ualWithState}.` }, }; } catch (err) { success = false; response = { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `Failed to acknowledge storing of ${ualWithState}.` }, }; } await this.operationService.markOperationAsCompleted(operationId, blockchain, success, [ OPERATION_ID_STATUS.FINALITY.PUBLISH_FINALITY_REMOTE_END, OPERATION_ID_STATUS.COMPLETED, ]); return this.continueSequence({ ...command.data, response }, command.sequence); } /** * Builds default publishFinalitySaveAckCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'publishFinalitySaveAckCommand', delay: 0, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default PublishFinalitySaveAckCommand; ================================================ FILE: src/commands/protocols/finality/receiver/v1.0.0/v1-0-0-handle-finality-request-command.js ================================================ import HandleProtocolMessageCommand from '../../../common/handle-protocol-message-command.js'; import { ERROR_TYPE, OPERATION_ID_STATUS, COMMAND_PRIORITY, NETWORK_MESSAGE_TYPES, } from '../../../../../constants/constants.js'; class HandleFinalityRequestCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); this.operationService = ctx.finalityService; this.tripleStoreService = ctx.tripleStoreService; this.pendingStorageService = ctx.pendingStorageService; this.paranetService = ctx.paranetService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.commandExecutor = ctx.commandExecutor; this.protocolService = ctx.protocolService; this.operationService = ctx.finalityService; this.networkModuleManager = ctx.networkModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.errorType = ERROR_TYPE.FINALITY.FINALITY_REQUEST_REMOTE_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.FINALITY.FINALITY_REMOTE_START; this.operationEndEvent = OPERATION_ID_STATUS.FINALITY.FINALITY_REMOTE_END; } async prepareMessage(commandData) { return commandData.response; } async execute(command) { const { ual, publishOperationId, blockchain, operationId, remotePeerId, state } = command.data; let ualWithState = ual; if (state) { ualWithState = `${ual}:${state}`; } this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FINALITY.PUBLISH_FINALITY_REMOTE_START, operationId, blockchain, ); let response; let success; try { await this.repositoryModuleManager.saveFinalityAck( publishOperationId, ualWithState, remotePeerId, ); success = true; response = { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: { message: `Acknowledged storing of ${ualWithState}.` }, }; } catch (err) { success = false; response = { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `Failed to acknowledge storing of ${ualWithState}.` }, }; } await this.operationService.markOperationAsCompleted(operationId, blockchain, success, [ OPERATION_ID_STATUS.FINALITY.PUBLISH_FINALITY_FETCH_FROM_NODES_END, OPERATION_ID_STATUS.FINALITY.PUBLISH_FINALITY_END, OPERATION_ID_STATUS.COMPLETED, ]); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FINALITY.PUBLISH_FINALITY_REMOTE_END, operationId, blockchain, ); // eslint-disable-next-line no-param-reassign command.data.response = response; super.execute(command); return HandleFinalityRequestCommand.empty(); } /** * Builds default handleFinalityRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0HandleFinalityRequestCommand', delay: 0, transactional: false, errorType: ERROR_TYPE.FINALITY.FINALITY_REQUEST_REMOTE_ERROR, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default HandleFinalityRequestCommand; ================================================ FILE: src/commands/protocols/finality/sender/finality-schedule-messages-command.js ================================================ import ProtocolScheduleMessagesCommand from '../../common/protocol-schedule-messages-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, COMMAND_PRIORITY, } from '../../../../constants/constants.js'; class FinalityScheduleMessagesCommand extends ProtocolScheduleMessagesCommand { constructor(ctx) { super(ctx); this.operationService = ctx.finalityService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.operationStartEvent = OPERATION_ID_STATUS.FINALITY.FINALITY_REPLICATE_START; this.operationEndEvent = OPERATION_ID_STATUS.FINALITY.FINALITY_REPLICATE_END; this.errorType = ERROR_TYPE.FINALITY.FINALITY_START_ERROR; } getNextCommandData(command) { return { ...super.getNextCommandData(command), ual: command.data.ual, publishOperationId: command.data.publishOperationId, }; } /** * Builds default finalityScheduleMessagesCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'finalityScheduleMessagesCommand', delay: 0, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default FinalityScheduleMessagesCommand; ================================================ FILE: src/commands/protocols/finality/sender/find-publisher-node-command.js ================================================ import { COMMAND_PRIORITY } from '../../../../constants/constants.js'; import Command from '../../../command.js'; class FindPublisherNodeCommand extends Command { constructor(ctx) { super(ctx); this.operationService = ctx.finalityService; } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { remotePeerId } = command.data; const networkProtocols = this.operationService.getNetworkProtocols(); const leftoverNodes = [{ id: remotePeerId, protocol: networkProtocols[0] }]; return this.continueSequence( { ...command.data, leftoverNodes, numberOfFoundNodes: leftoverNodes.length, }, command.sequence, ); } /** * Builds default findPublisherNodeCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'findPublisherNodeCommand', delay: 0, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default FindPublisherNodeCommand; ================================================ FILE: src/commands/protocols/finality/sender/network-finality-command.js ================================================ import Command from '../../../command.js'; import NetworkProtocolCommand from '../../common/network-protocol-command.js'; import { COMMAND_PRIORITY, ERROR_TYPE, OPERATION_ID_STATUS, } from '../../../../constants/constants.js'; class NetworkFinalityCommand extends NetworkProtocolCommand { constructor(ctx) { super(ctx); this.operationService = ctx.finalityService; this.ualService = ctx.ualService; this.errorType = ERROR_TYPE.FINALITY.FINALITY_NETWORK_ERROR; } async execute(command) { await super.execute(command); const { operationId, blockchain } = command.data; await this.operationService.markOperationAsCompleted(operationId, blockchain, null, [ OPERATION_ID_STATUS.PUBLISH_FINALIZATION.PUBLISH_FINALIZATION_END, ]); return Command.empty(); } /** * Builds default networkFinalityCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'networkFinalityCommand', delay: 0, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default NetworkFinalityCommand; ================================================ FILE: src/commands/protocols/finality/sender/v1.0.0/v1-0-0-finality-request-command.js ================================================ import Command from '../../../../command.js'; import ProtocolRequestCommand from '../../../common/protocol-request-command.js'; import { NETWORK_MESSAGE_TIMEOUT_MILLS, ERROR_TYPE, OPERATION_ID_STATUS, COMMAND_PRIORITY, } from '../../../../../constants/constants.js'; class FinalityRequestCommand extends ProtocolRequestCommand { constructor(ctx) { super(ctx); this.operationService = ctx.finalityService; this.operationIdService = ctx.operationIdService; this.errorType = ERROR_TYPE.FINALITY.FINALITY_REQUEST_ERROR; } async prepareMessage(command) { const { ual, publishOperationId, blockchain, operationId } = command.data; return { ual, publishOperationId, blockchain, operationId }; } async handleAck(command) { const { operationId, blockchain } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.COMPLETED, ); return ProtocolRequestCommand.empty(); } async handleNack(command, responseData) { const { operationId, blockchain } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.COMPLETED, ); await this.markResponseAsFailed( command, `Received NACK response from node during ${command.name}. Error message: ${responseData.errorMessage}`, ); return Command.empty(); } messageTimeout() { return NETWORK_MESSAGE_TIMEOUT_MILLS.FINALITY.REQUEST; } /** * Builds default finalityRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0FinalityRequestCommand', delay: 0, retries: 0, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default FinalityRequestCommand; ================================================ FILE: src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js ================================================ import fs from 'fs/promises'; import path from 'path'; import HandleProtocolMessageCommand from '../../../common/handle-protocol-message-command.js'; import { ERROR_TYPE, NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS, MIGRATION_FLAG_PATH, TRIPLE_STORE_REPOSITORY, TRIPLES_VISIBILITY, BATCH_GET_UAL_MAX_LIMIT, COMMAND_PRIORITY, } from '../../../../../constants/constants.js'; class HandleBatchGetRequestCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); this.logger = ctx.config.logging.enableExperimentalScopes ? ctx.logger.child({ scope: 'HandleBatchGetRequestCommand', }) : ctx.logger; this.tripleStoreService = ctx.tripleStoreService; this.paranetService = ctx.paranetService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.networkModuleManager = ctx.networkModuleManager; this.cryptoService = ctx.cryptoService; this.errorType = ERROR_TYPE.BATCH_GET.BATCH_GET_REQUEST_REMOTE_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_REMOTE_START; this.operationEndEvent = OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_REMOTE_END; } async prepareMessage(commandData) { const { operationId, blockchain, includeMetadata } = commandData; let { uals, tokenIds } = commandData; this.logger.startTimer( `HandleBatchGetRequestCommand [PREPARE]: ${operationId} ${uals.length}`, ); await this.operationIdService.emitChangeEvent( this.operationStartEvent, operationId, blockchain, ); // Trim uals and tokenIds to the max limit of BATCH_GET_UAL_MAX_LIMIT uals = uals.slice(0, BATCH_GET_UAL_MAX_LIMIT); tokenIds = Object.fromEntries(Object.entries(tokenIds).slice(0, BATCH_GET_UAL_MAX_LIMIT)); const promises = []; let migrationFlag = '0'; const migrationFlagPath = path.join(process.cwd(), MIGRATION_FLAG_PATH); try { migrationFlag = await fs.readFile(migrationFlagPath, 'utf8'); migrationFlag = migrationFlag.trim(); } catch (error) { if (error.code === 'ENOENT') { this.logger.warn( `Migration flag file not found at ${migrationFlagPath}, using default value '${migrationFlag}'`, ); } else { throw error; } } this.logger.endTimer( `HandleBatchGetRequestCommand [PREPARE]: ${operationId} ${uals.length}`, ); this.logger.startTimer( `HandleBatchGetRequestCommand [PROCESSING]: ${operationId} ${uals.length}`, ); const assertionPromise = this.tripleStoreService.getAssertionsInBatch( TRIPLE_STORE_REPOSITORY.DKG, uals, tokenIds, TRIPLES_VISIBILITY.PUBLIC, operationId, ); promises.push(assertionPromise); if (includeMetadata) { const metadataPromise = this.tripleStoreService.getAssertionMetadataBatch( uals, tokenIds, ); promises.push(metadataPromise); } const [assertions, metadata] = await Promise.all(promises); const responseData = { assertions, ...(includeMetadata && metadata && { metadata }), }; this.logger.endTimer( `HandleBatchGetRequestCommand [PROCESSING]: ${operationId} ${uals.length}`, ); this.logger.startTimer( `HandleBatchGetRequestCommand [RESPONSE]: ${operationId} ${uals.length}`, ); if (assertions?.length) { await this.operationIdService.emitChangeEvent( this.operationEndEvent, operationId, blockchain, ); } this.logger.endTimer( `HandleBatchGetRequestCommand [RESPONSE]: ${operationId} ${uals.length}`, ); return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: responseData }; } /** * Builds default handleGetRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0HandleBatchGetRequestCommand', transactional: false, priority: COMMAND_PRIORITY.MEDIUM, errorType: this.errorType, }; Object.assign(command, map); return command; } } export default HandleBatchGetRequestCommand; ================================================ FILE: src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js ================================================ import HandleProtocolMessageCommand from '../../../common/handle-protocol-message-command.js'; import { ERROR_TYPE, NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS, TRIPLES_VISIBILITY, PARANET_ACCESS_POLICY, COMMAND_PRIORITY, } from '../../../../../constants/constants.js'; class HandleGetRequestCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); this.operationService = ctx.getService; this.tripleStoreService = ctx.tripleStoreService; this.pendingStorageService = ctx.pendingStorageService; this.paranetService = ctx.paranetService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.networkModuleManager = ctx.networkModuleManager; this.cryptoService = ctx.cryptoService; this.errorType = ERROR_TYPE.GET.GET_REQUEST_REMOTE_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.GET.GET_REMOTE_START; this.operationEndEvent = OPERATION_ID_STATUS.GET.GET_REMOTE_END; } async prepareMessage(commandData) { const { operationId, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, tokenIds, ual, includeMetadata, paranetUAL, remotePeerId, migrationFlag, repository, } = commandData; if (paranetUAL) { const { contract: paranetContract, knowledgeCollectionId: paranetKnowledgeCollectionId, knowledgeAssetId: paranetKnowledgeAssetId, } = this.ualService.resolveUAL(paranetUAL); const paranetId = this.paranetService.constructParanetId( paranetContract, paranetKnowledgeCollectionId, paranetKnowledgeAssetId, ); const paranetNodeAccessPolicy = await this.blockchainModuleManager.getNodesAccessPolicy( blockchain, paranetId, ); if (paranetNodeAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { const knowledgeCollectionOnchainId = this.cryptoService.keccak256EncodePacked( ['address', 'uint256'], [contract, knowledgeCollectionId], ); const [isKCInParanet, paranetPermissionedNodes] = await Promise.all([ this.blockchainModuleManager.isKnowledgeCollectionRegistered( blockchain, paranetId, knowledgeCollectionOnchainId, ), this.blockchainModuleManager.getPermissionedNodes(blockchain, paranetId), ]); if (!isKCInParanet) { return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `Knowledge collection ${knowledgeCollectionId} is not registered in the Paranet (${paranetId}) with UAL: ${paranetUAL}`, }, }; } const paranetPermissionedPeerIds = paranetPermissionedNodes.map((node) => this.cryptoService.convertHexToAscii(node.nodeId), ); if (!paranetPermissionedPeerIds.includes(remotePeerId)) { return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `Remote peer ${remotePeerId} is not a part of the Paranet (${paranetId}) with UAL: ${paranetUAL}`, }, }; } const currentPeerId = this.networkModuleManager.getPeerId().toB58String(); if (!paranetPermissionedPeerIds.includes(currentPeerId)) { return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `This node is not a part of the Paranet (${paranetId}) with UAL: ${paranetUAL}`, }, }; } const promises = []; promises.push( this.tripleStoreService.getAssertion( blockchain, contract, knowledgeCollectionId, knowledgeAssetId, tokenIds, migrationFlag, TRIPLES_VISIBILITY.ALL, repository, ), ); if (includeMetadata) { const metadataPromise = this.tripleStoreService.getAssertionMetadata( blockchain, contract, knowledgeCollectionId, ); promises.push(metadataPromise); } const [assertion, metadata] = await Promise.all(promises); if (assertion?.public?.length) { return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: { assertion, metadata }, }; } return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `Unable to find assertion ${ual} for Paranet (${paranetId}) with UAL: ${paranetUAL}`, }, }; } } const promises = []; const assertionPromise = this.tripleStoreService.getAssertion( blockchain, contract, knowledgeCollectionId, knowledgeAssetId, tokenIds, migrationFlag, TRIPLES_VISIBILITY.PUBLIC, repository, ); promises.push(assertionPromise); if (includeMetadata) { const metadataPromise = this.tripleStoreService.getAssertionMetadata( blockchain, contract, knowledgeCollectionId, ); promises.push(metadataPromise); } const [assertion, metadata] = await Promise.all(promises); const responseData = { assertion, ...(includeMetadata && metadata && { metadata }), }; if (assertion?.public?.length || assertion?.length) { await this.operationIdService.emitChangeEvent( this.operationEndEvent, operationId, blockchain, ); } return assertion?.public?.length || assertion?.length ? { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: responseData } : { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: `Unable to find assertion ${ual}` }, }; } /** * Builds default handleGetRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0HandleGetRequestCommand', delay: 0, transactional: false, priority: COMMAND_PRIORITY.HIGH, errorType: ERROR_TYPE.GET.GET_REQUEST_REMOTE_ERROR, }; Object.assign(command, map); return command; } } export default HandleGetRequestCommand; ================================================ FILE: src/commands/protocols/get/sender/batch-get-command.js ================================================ import { kcTools } from 'assertion-tools'; import fs from 'fs/promises'; import path from 'path'; import Command from '../../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, TRIPLE_STORE_REPOSITORIES, NETWORK_MESSAGE_TYPES, NETWORK_MESSAGE_TIMEOUT_MILLS, MIGRATION_FLAG_PATH, PRIVATE_HASH_SUBJECT_PREFIX, OPERATION_STATUS, BATCH_GET_BATCH_SIZE as BATCH_SIZE, TRIPLE_STORE_REPOSITORY, TRIPLES_VISIBILITY, COMMAND_PRIORITY, } from '../../../../constants/constants.js'; class BatchGetCommand extends Command { constructor(ctx) { super(ctx); this.logger = ctx.config.logging.enableExperimentalScopes ? ctx.logger.child({ scope: 'BatchGetCommand', }) : ctx.logger; this.operationIdService = ctx.operationIdService; this.ualService = ctx.ualService; this.operationService = ctx.batchGetService; this.validationService = ctx.validationService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.paranetService = ctx.paranetService; this.tripleStoreService = ctx.tripleStoreService; this.networkModuleManager = ctx.networkModuleManager; this.shardingTableService = ctx.shardingTableService; this.cryptoService = ctx.cryptoService; this.messagingService = ctx.messagingService; this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.DONE_THRESHOLD = ctx.config.assetSync.syncDKG.doneThreshold; } async handleError(operationId, blockchain, errorMessage, errorType) { await this.operationService.markOperationAsFailed( operationId, blockchain, errorMessage, errorType, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_FAILED, operationId, blockchain, errorMessage, errorType, ); } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { operationId, blockchain, uals, // paranetUAL, // paranetSync, contentType, includeMetadata, paranetNodesAccessPolicy, } = command.data; this.logger.startTimer(`BatchGetCommand [PREPARE]: ${operationId} ${uals.length}`); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_START, ); await this.repositoryModuleManager.createOperationRecord( this.operationService.getOperationName(), operationId, OPERATION_STATUS.IN_PROGRESS, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_VALIDATE_ASSET_START, ); const { isValid, errorMessage } = await this.validateUALs(operationId, blockchain, uals); this.logger.endTimer(`BatchGetCommand [PREPARE]: ${operationId} ${uals.length}`); if (!isValid) { await this.handleError( operationId, blockchain, errorMessage, ERROR_TYPE.BATCH_GET.BATCH_GET_VALIDATE_ASSET_ERROR, ); return Command.empty(); } this.logger.startTimer(`BatchGetCommand [NETWORK_INIT]: ${operationId} ${uals.length}`); const currentPeerId = this.networkModuleManager.getPeerId().toB58String(); // let paranetId; const repository = TRIPLE_STORE_REPOSITORIES.DKG; let migrationFlag = '0'; const migrationFlagPath = path.join(process.cwd(), MIGRATION_FLAG_PATH); try { migrationFlag = await fs.readFile(migrationFlagPath, 'utf8'); migrationFlag = migrationFlag.trim(); } catch (error) { if (error.code === 'ENOENT') { this.logger.warn( `Migration flag file not found at ${migrationFlagPath}, using default value '${migrationFlag}'`, ); } else { throw error; } } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_VALIDATE_ASSET_END, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_LOCAL_START, ); this.logger.endTimer(`BatchGetCommand [NETWORK_INIT]: ${operationId} ${uals.length}`); this.logger.startTimer(`BatchGetCommand [TOKEN_IDS]: ${operationId} ${uals.length}`); const tokenIds = {}; const tokenIdPromises = uals.map(async (ual) => { const { contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); try { tokenIds[ual] = await this.blockchainModuleManager.getKnowledgeAssetsRange( blockchain, contract, knowledgeCollectionId, ); } catch (error) { // Asset created on old content asset storage contract tokenIds[ual] = { startTokenId: 1, endTokenId: 1, burned: [], }; } }); await Promise.all(tokenIdPromises); this.logger.endTimer(`BatchGetCommand [TOKEN_IDS]: ${operationId} ${uals.length}`); this.logger.startTimer(`BatchGetCommand [LOCAL_BATCH_GET]: ${operationId} ${uals.length}`); const promises = []; const assertionPromise = this.tripleStoreService.getAssertionsInBatch( TRIPLE_STORE_REPOSITORY.DKG, uals, tokenIds, TRIPLES_VISIBILITY.PUBLIC, operationId, ); promises.push(assertionPromise); const [batchAssertions] = await Promise.all(promises); const finalResult = { local: [], remote: {}, metadata: {} }; this.logger.endTimer(`BatchGetCommand [LOCAL_BATCH_GET]: ${operationId} ${uals.length}`); this.logger.startTimer( `BatchGetCommand [LOCAL_BATCH_GET_VALIDATE]: ${operationId} ${uals.length}`, ); const localGetResultValid = await this.validateBatchResponse( batchAssertions, blockchain, paranetNodesAccessPolicy, contentType, finalResult, ); // Filter what we have locally and add those ual to finalResult local const ualPresentLocally = Object.keys(localGetResultValid).filter( (ual) => localGetResultValid[ual], ); const ualNotPresentLocally = Object.keys(localGetResultValid).filter( (ual) => !localGetResultValid[ual], ); this.logger.endTimer( `BatchGetCommand [LOCAL_BATCH_GET_VALIDATE]: ${operationId} ${uals.length}`, ); this.logger.startTimer(`BatchGetCommand [LOCAL]: ${operationId} ${uals.length}`); ualPresentLocally.forEach((ual) => { finalResult.local.push(ual); delete tokenIds[ual]; }); if (ualNotPresentLocally.length === 0) { await this.operationService.markOperationAsCompleted( operationId, blockchain, finalResult, [ OPERATION_ID_STATUS.GET.GET_LOCAL_END, OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED, ], ); return Command.empty(); } this.logger.endTimer(`BatchGetCommand [LOCAL]: ${operationId} ${uals.length}`); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.GET.GET_LOCAL_END, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_FIND_SHARD_START, ); this.logger.startTimer(`BatchGetCommand [FIND_SHARD]: ${operationId} ${uals.length}`); let nodesInfo = []; // if (paranetNodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { // const onChainNodes = await this.blockchainModuleManager.getPermissionedNodes( // blockchain, // paranetId, // ); // const foundNodes = await Promise.all( // onChainNodes.map(async (node) => // this.shardingTableService.findPeerAddressAndProtocols( // this.cryptoService.convertHexToAscii(node.nodeId), // ), // ), // ); // const networkProtocols = this.operationService.getNetworkProtocols(); // for (const node of foundNodes) { // if (node.id !== currentPeerId) { // nodesInfo.push({ id: node.id, protocol: networkProtocols[0] }); // } // } // } else { nodesInfo = await this.findShardNodes(operationId, blockchain, currentPeerId); // Make order of nodes random, shuffle the array nodesInfo = nodesInfo.sort(() => Math.random() - 0.5); // } if (nodesInfo.length < 1) { await this.handleError( operationId, blockchain, `Unable to find enough nodes for operationId: ${operationId}. Minimum number of nodes required: 1`, ERROR_TYPE.FIND_SHARD.BATCH_GET_FIND_SHARD_ERROR, true, ); return Command.empty(); } this.logger.endTimer(`BatchGetCommand [FIND_SHARD]: ${operationId} ${uals.length}`); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_FIND_SHARD_END, ); this.logger.startTimer(`BatchGetCommand [NETWORK]: ${operationId} ${uals.length}`); let index = 0; let commandCompleted = false; const initialMissing = ualNotPresentLocally.length; const hasReachedThreshold = () => { if (initialMissing === 0) { return true; } const retrieved = initialMissing - ualNotPresentLocally.length; const ratio = (retrieved / initialMissing) * 100; return ratio >= this.DONE_THRESHOLD; }; while (index < nodesInfo.length && ualNotPresentLocally.length > 0 && !commandCompleted) { const batch = nodesInfo.slice(index, index + BATCH_SIZE); const message = { blockchain, tokenIds, includeMetadata, uals: ualNotPresentLocally, repository, }; // eslint-disable-next-line no-loop-func const messagePromises = batch.map(async (node) => { try { this.logger.startTimer( `BatchGetCommand [NETWORK_SEND_MESSAGE]: ${operationId} ${uals.length} ${node.id}`, ); const result = await this.sendMessage(node, operationId, message); this.logger.endTimer( `BatchGetCommand [NETWORK_SEND_MESSAGE]: ${operationId} ${uals.length} ${node.id}`, ); if (commandCompleted || !result.success) { return; } this.logger.startTimer( `BatchGetCommand [NETWORK_VALIDATE_RESPONSE]: ${operationId} ${uals.length} ${node.id}`, ); const validationResult = await this.validateBatchResponse( result.responseData.assertions, blockchain, paranetNodesAccessPolicy, contentType, finalResult, [OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED], ); this.logger.endTimer( `BatchGetCommand [NETWORK_VALIDATE_RESPONSE]: ${operationId} ${uals.length} ${node.id}`, ); if (commandCompleted) { return; } for (const [ual, isKCValid] of Object.entries(validationResult)) { if (isKCValid) { finalResult.remote[ual] = result.responseData.assertions[ual]; finalResult.metadata[ual] = result.responseData.metadata[ual]; const idx = ualNotPresentLocally.indexOf(ual); if (idx !== -1) { ualNotPresentLocally.splice(idx, 1); } } } if (hasReachedThreshold() && !commandCompleted) { commandCompleted = true; this.logger.startTimer( `BatchGetCommand [NETWORK_MARK_AS_COMPLETED]: ${operationId} ${uals.length} ${node.id}`, ); await this.operationService.markOperationAsCompleted( operationId, blockchain, finalResult, [OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED], ); this.logger.endTimer( `BatchGetCommand [NETWORK_MARK_AS_COMPLETED]: ${operationId} ${uals.length} ${node.id}`, ); } } catch (err) { this.logger.warn(`Node ${node.id} failed: ${err.message}`); } }); // eslint-disable-next-line no-await-in-loop, no-loop-func await new Promise((resolve) => { let settledPromises = 0; const countSettledAndMaybeResolve = () => { settledPromises += 1; const allSettled = settledPromises === messagePromises.length; if ( commandCompleted || allSettled // Safety net to stop infinite hang ) { resolve(); } }; // eslint-disable-next-line no-loop-func messagePromises.forEach((p) => p.finally(countSettledAndMaybeResolve)); }); index += BATCH_SIZE; } // Just in case we finish outside early-exit if (!commandCompleted) { await this.operationService.markOperationAsCompleted( operationId, blockchain, finalResult, [OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED], ); } this.logger.endTimer(`BatchGetCommand [NETWORK]: ${operationId} ${uals.length}`); return Command.empty(); } async validateUALs(operationId, blockchain, uals) { if (uals.length === 0) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, UALs: ${uals}: no UALs provided.`, }; } const validationPromises = uals.map(async (ual) => { const isUAL = this.ualService.isUAL(ual); if (!isUAL) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, UAL: ${ual}: is not a UAL.`, }; } const { contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); const isValidUal = await this.validationService.validateUal( blockchain, contract, knowledgeCollectionId, ); if (!isValidUal) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, UAL: ${ual}: there is no asset with this UAL.`, }; } return { isValid: true, errorMessage: null, }; }); const results = await Promise.all(validationPromises); // Find the first invalid result if any const invalidResult = results.find((result) => !result.isValid); if (invalidResult) { return invalidResult; } return { isValid: true, errorMessage: null, }; } // async validateParanet( // operationId, // paranetUAL, // paranetBlockchain, // paranetKnowledgeAssetId, // paranetNodeAccessPolicy, // paranetId, // blockchain, // uals, // ) { // if (!paranetKnowledgeAssetId) { // return { // isValid: false, // errorMessage: `Invalid paranet UAL: ${paranetUAL} . Paranet knowledge asset token id is required!`, // }; // } // const isParanetUAL = this.ualService.isUAL(paranetUAL); // if (!isParanetUAL) { // return { // isValid: false, // errorMessage: `Get for operation id: ${operationId}, Paranet UAL: ${paranetUAL}: is not a UAL.`, // }; // } // const [paranetExists, chainParanetNodesAccessPolicy] = await Promise.all([ // this.blockchainModuleManager.paranetExists(paranetBlockchain, paranetId), // this.blockchainModuleManager.getNodesAccessPolicy(paranetBlockchain, paranetId), // ]); // if (!paranetExists) { // return { // isValid: false, // errorMessage: `Get for operation id: ${operationId}, Paranet UAL: ${paranetUAL}: paranet does not exist.`, // }; // } // if (paranetNodeAccessPolicy !== chainParanetNodesAccessPolicy) { // return { // isValid: false, // errorMessage: `Get for operation id: ${operationId}, Paranet UAL: ${paranetUAL}: onchain paranet access policy does not match the requested paranet access policy.`, // }; // } // const validationPromises = uals.map(async (ual) => { // const { contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); // const knowledgeCollectionOnchainId = this.cryptoService.keccak256EncodePacked( // ['address', 'uint256'], // [contract, knowledgeCollectionId], // ); // const paranetContainsKnowledgeCollection = // await this.blockchainModuleManager.isKnowledgeCollectionRegistered( // blockchain, // paranetId, // knowledgeCollectionOnchainId, // ); // if (!paranetContainsKnowledgeCollection) { // return { // isValid: false, // errorMessage: `Paranet UAL: ${paranetUAL} does not contain Knowledge Collection: ${ual}`, // }; // } // return { // isValid: true, // errorMessage: null, // }; // }); // const results = await Promise.all(validationPromises); // // Find the first invalid result if any // const invalidResult = results.find((result) => !result.isValid); // if (invalidResult) { // return invalidResult; // } // return { // isValid: true, // errorMessage: null, // }; // } async findShardNodes(operationId, blockchain, currentPeerId) { this.logger.debug(`Searching for shard for operationId: ${operationId}`); const networkProtocols = this.operationService.getNetworkProtocols(); const shardNodes = await this.shardingTableService.findShard(blockchain, true); // TODO: Optimize this so it's returned by shardingTableService.findShard const foundNodes = await Promise.all( shardNodes.map(({ peerId }) => this.shardingTableService.findPeerAddressAndProtocols(peerId), ), ); const nodesInfo = []; for (const node of foundNodes) { if (node.id !== currentPeerId) { nodesInfo.push({ id: node.id, protocol: networkProtocols[0] }); } } this.logger.debug(`Found ${nodesInfo.length} node(s) for operationId: ${operationId}`); this.logger.trace( `Found shard: ${JSON.stringify( nodesInfo.map((node) => node.id), null, 2, )}`, ); return nodesInfo; } async sendMessage(node, operationId, message) { const response = await this.messagingService.sendProtocolMessage( node, operationId, message, NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST, NETWORK_MESSAGE_TIMEOUT_MILLS.BATCH_GET.REQUEST, ); return { success: response.header.messageType === NETWORK_MESSAGE_TYPES.RESPONSES.ACK, responseData: response.data, }; } async validateBatchResponse( responseData, blockchain, paranetNodesAccessPolicy, contentType, finalResult, ) { const validationResults = {}; await Promise.all( Object.entries(responseData).map(async ([ual, assertion]) => { // Already received and validate this assertion if (finalResult.remote[ual]) { return; } if (contentType === 'private') { validationResults[ual] = true; return; } const filteredPublic = []; const privateHashTriples = []; // Separate public vs private hash triples if (!assertion.public || assertion.public.length === 0) { validationResults[ual] = false; return; } assertion.public.forEach((triple) => { if (triple.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { privateHashTriples.push(triple); } else { filteredPublic.push(triple); } }); // Group triples by subject const publicKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( filteredPublic, true, ); publicKnowledgeAssetsTriplesGrouped.push( ...kcTools.groupNquadsBySubject(privateHashTriples, true), ); try { // Validate public dataset const { contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); await this.validationService.validateDatasetOnBlockchain( publicKnowledgeAssetsTriplesGrouped.map((t) => t.sort()).flat(), blockchain, contract, knowledgeCollectionId, ); // If not permissioned and there are private triples, validate if (assertion?.private?.length) { await this.validationService.validatePrivateMerkleRoot( assertion.public, assertion.private, ); } validationResults[ual] = true; } catch (e) { this.logger.error(`Validation failed for UAL ${ual}: ${e.name}, ${e.message}`); validationResults[ual] = false; } }), ); return validationResults; } /** * Builds default GetCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'batchGetCommand', delay: 0, transactional: false, priority: COMMAND_PRIORITY.MEDIUM, }; Object.assign(command, map); return command; } } export default BatchGetCommand; ================================================ FILE: src/commands/protocols/get/sender/get-command.js ================================================ import { kcTools } from 'assertion-tools'; import fs from 'fs/promises'; import path from 'path'; import Command from '../../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, PARANET_ACCESS_POLICY, TRIPLE_STORE_REPOSITORIES, NETWORK_MESSAGE_TYPES, NETWORK_MESSAGE_TIMEOUT_MILLS, PRIVATE_ASSERTION_PREDICATE, PRIVATE_HASH_SUBJECT_PREFIX, MIGRATION_FLAG_PATH, COMMAND_PRIORITY, } from '../../../../constants/constants.js'; class GetCommand extends Command { constructor(ctx) { super(ctx); this.operationIdService = ctx.operationIdService; this.ualService = ctx.ualService; this.operationService = ctx.getService; this.validationService = ctx.validationService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.paranetService = ctx.paranetService; this.tripleStoreService = ctx.tripleStoreService; this.networkModuleManager = ctx.networkModuleManager; this.shardingTableService = ctx.shardingTableService; this.cryptoService = ctx.cryptoService; this.messagingService = ctx.messagingService; this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; this.pendingStorageService = ctx.pendingStorageService; } async handleError(operationId, blockchain, errorMessage, errorType) { await this.operationService.markOperationAsFailed( operationId, blockchain, errorMessage, errorType, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.GET.GET_FAILED, operationId, blockchain, ); } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { operationId, blockchain, contract, knowledgeCollectionId, ual, paranetUAL, paranetSync, contentType, includeMetadata, paranetNodesAccessPolicy, knowledgeAssetId, minimumNumberOfNodeReplications, } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.GET.GET_VALIDATE_ASSET_START, ); const maxGetRetries = 3; const getRetryDelayMs = 5_000; let ualValidationPassed = false; let ualValidationError = null; for (let attempt = 1; attempt <= maxGetRetries; attempt += 1) { try { // eslint-disable-next-line no-await-in-loop const { isValid, errorMessage: valMsg } = await this.validateUAL( operationId, blockchain, contract, knowledgeCollectionId, ual, ); if (isValid) { ualValidationPassed = true; break; } ualValidationError = valMsg; } catch (err) { ualValidationError = `UAL validation failed: ${err.message}`; } if (!ualValidationPassed) { try { // eslint-disable-next-line no-await-in-loop const cachedResult = await this._tryCacheFallback( blockchain, contract, knowledgeCollectionId, knowledgeAssetId, ual, operationId, paranetNodesAccessPolicy, contentType, ); if (cachedResult) { return cachedResult; } } catch (_cacheErr) { // cache fallback also failed, will retry } } if (attempt < maxGetRetries) { this.logger.debug( `Get validation/cache attempt ${attempt}/${maxGetRetries} failed for ${ual}, retrying in ${getRetryDelayMs}ms`, ); // eslint-disable-next-line no-await-in-loop await new Promise((resolve) => { setTimeout(resolve, getRetryDelayMs); }); } } if (!ualValidationPassed) { await this.handleError( operationId, blockchain, ualValidationError, ERROR_TYPE.GET.GET_VALIDATE_ASSET_ERROR, ); return Command.empty(); } const currentPeerId = this.networkModuleManager.getPeerId().toB58String(); let paranetId; let repository = TRIPLE_STORE_REPOSITORIES.DKG; let migrationFlag = '0'; const migrationFlagPath = path.join(process.cwd(), MIGRATION_FLAG_PATH); try { migrationFlag = await fs.readFile(migrationFlagPath, 'utf8'); migrationFlag = migrationFlag.trim(); } catch (error) { if (error.code === 'ENOENT') { this.logger.warn( `Migration flag file not found at ${migrationFlagPath}, using default value '${migrationFlag}'`, ); } else { throw error; } } if (paranetUAL) { const { blockchain: paranetBlockchain, contract: paranetContract, knowledgeCollectionId: paranetKnowledgeCollectionId, knowledgeAssetId: paranetKnowledgeAssetId, } = this.ualService.resolveUAL(paranetUAL); paranetId = this.paranetService.constructParanetId( paranetContract, paranetKnowledgeCollectionId, paranetKnowledgeAssetId, ); if (!paranetSync && migrationFlag === '0') { // query the paranet repository if the migration is not yet finished repository = this.paranetService.getParanetRepositoryName(paranetUAL); const repositoryExists = this.tripleStoreModuleManager.repositoryInitilized(repository); if (!repositoryExists) { repository = TRIPLE_STORE_REPOSITORIES.DKG; } } const { isValid: paranetIsValid, errorMessage: paranetErrorMessage } = await this.validateParanet( operationId, paranetUAL, paranetBlockchain, paranetKnowledgeAssetId, paranetNodesAccessPolicy, paranetId, knowledgeCollectionId, blockchain, contract, ual, ); if (!paranetIsValid) { await this.handleError( operationId, blockchain, paranetErrorMessage, ERROR_TYPE.GET.GET_VALIDATE_ASSET_ERROR, ); return Command.empty(); } } this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.GET.GET_VALIDATE_ASSET_END, operationId, blockchain, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.GET.GET_LOCAL_START, ); let tokenIds; if (!knowledgeAssetId) { try { tokenIds = await this.blockchainModuleManager.getKnowledgeAssetsRange( blockchain, contract, knowledgeCollectionId, ); } catch (error) { // Asset created on old content asset storage contract tokenIds = { startTokenId: 1, endTokenId: 1, burned: [], }; } } else { // kaId is number, so transform it to range tokenIds = { startTokenId: knowledgeAssetId, endTokenId: knowledgeAssetId, burned: [], }; } const promises = []; const assertionPromise = this.tripleStoreService.getAssertion( blockchain, contract, knowledgeCollectionId, knowledgeAssetId, tokenIds, migrationFlag, contentType, repository, ); promises.push(assertionPromise); if (includeMetadata) { const metadataPromise = this.tripleStoreService.getAssertionMetadata( blockchain, contract, knowledgeCollectionId, repository, ); promises.push(metadataPromise); } const [assertion, metadata] = await Promise.all(promises); const responseData = { assertion, ...(includeMetadata && metadata && { metadata }), }; let localGetPassed = true; if (paranetNodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { if (Array.isArray(assertion?.public)) { const assertionShouldHavePrivateTriples = assertion?.public?.some((triple) => triple.includes(`${PRIVATE_ASSERTION_PREDICATE}`), ); if (assertionShouldHavePrivateTriples) { localGetPassed = assertion?.private?.length > 0; } } else { localGetPassed = false; } } const localGetResultValid = await this.validateResponse( { assertion }, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, paranetNodesAccessPolicy, contentType, ); if ( localGetPassed && localGetResultValid && (assertion?.public?.length || assertion?.private?.length || assertion?.length) ) { await this.operationService.markOperationAsCompleted( operationId, blockchain, responseData, [ OPERATION_ID_STATUS.GET.GET_LOCAL_END, OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED, ], ); return Command.empty(); } this.logger.debug(`Could not find asset with UAL: ${ual} locally`); try { const cachedResult = await this._tryCacheFallback( blockchain, contract, knowledgeCollectionId, knowledgeAssetId, ual, operationId, paranetNodesAccessPolicy, contentType, ); if (cachedResult) { return cachedResult; } } catch (cacheErr) { this.logger.debug( `Pending storage cache fallback failed for ${ual}: ${cacheErr.message}`, ); } await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.GET.GET_LOCAL_END, operationId, blockchain, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.GET.GET_SHARD_START, ); let nodesInfo = []; if (paranetNodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { const onChainNodes = await this.blockchainModuleManager.getPermissionedNodes( blockchain, paranetId, ); const foundNodes = await Promise.all( onChainNodes.map(async (node) => this.shardingTableService.findPeerAddressAndProtocols( this.cryptoService.convertHexToAscii(node.nodeId), ), ), ); const networkProtocols = this.operationService.getNetworkProtocols(); for (const node of foundNodes) { if (node.id !== currentPeerId) { nodesInfo.push({ id: node.id, protocol: networkProtocols[0] }); } } } else { nodesInfo = await this.findShardNodes(operationId, blockchain, currentPeerId); } this.minAckResponses = this.operationService.getMinAckResponses( minimumNumberOfNodeReplications, ); if (nodesInfo.length < this.minAckResponses) { await this.handleError( operationId, blockchain, `Unable to find enough nodes for operationId: ${operationId}. Minimum number of nodes required: ${this.minAckResponses}`, ERROR_TYPE.FIND_SHARD.GET_FIND_SHARD_ERROR, true, ); return Command.empty(); } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.GET.GET_SHARD_END, ); const message = { blockchain, contract, knowledgeCollectionId, knowledgeAssetId, tokenIds, includeMetadata, ual, paranetUAL, migrationFlag, repository, }; const BATCH_SIZE = 5; let index = 0; // Process shard nodes in batches while (index < nodesInfo.length) { // Slice out a batch of nodes const batch = nodesInfo.slice(index, index + BATCH_SIZE); // Send messages in parallel to all nodes in the current batch // eslint-disable-next-line no-await-in-loop const results = await Promise.all( batch.map((node) => this.sendMessage(node, operationId, message)), ); const succsesfulResult = []; const failedResults = []; results.forEach((result) => { if (result.success) { succsesfulResult.push(result); } else { failedResults.push(result); } }); for (const result of succsesfulResult) { // eslint-disable-next-line no-await-in-loop const isResponseValid = await this.validateResponse( result.responseData, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, paranetNodesAccessPolicy, contentType, ); if (isResponseValid) { this.operationService.markOperationAsCompleted( operationId, blockchain, result.responseData, [OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED], ); return Command.empty(); } } // Otherwise, continue with the next batch index += BATCH_SIZE; } await this.handleError( operationId, blockchain, `No node responded successfully for GET for ${ual}. Minimum required responses: ${this.minAckResponses}. Operation id: ${operationId}`, ERROR_TYPE.FIND_SHARD.GET_ERROR, ); return Command.empty(); } async _tryCacheFallback( blockchain, contract, knowledgeCollectionId, knowledgeAssetId, ual, operationId, paranetNodesAccessPolicy, contentType, ) { if (knowledgeAssetId) return null; const latestMerkleRoot = await this.blockchainModuleManager.getKnowledgeCollectionLatestMerkleRoot( blockchain, contract, knowledgeCollectionId, ); if (!latestMerkleRoot) return null; const publishOpId = this.pendingStorageService.getOperationIdByMerkleRoot(latestMerkleRoot); if (!publishOpId) return null; const cachedAssertion = await this.pendingStorageService.getCachedDataset(publishOpId); if ( !cachedAssertion || (!cachedAssertion.public?.length && !cachedAssertion.private?.length) ) { return null; } const filteredAssertion = this._filterAssertionByContentType(cachedAssertion, contentType); if (!filteredAssertion.public?.length && !filteredAssertion.private?.length) { return null; } let cachePassed = true; if (paranetNodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { if (Array.isArray(filteredAssertion.public)) { const shouldHavePrivate = filteredAssertion.public.some((triple) => triple.includes(`${PRIVATE_ASSERTION_PREDICATE}`), ); if (shouldHavePrivate) { cachePassed = filteredAssertion.private?.length > 0; } } else { cachePassed = false; } } if (!cachePassed) return null; const cachedResponseData = { assertion: filteredAssertion }; const isValid = await this.validateResponse( cachedResponseData, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, paranetNodesAccessPolicy, contentType, ); if (!isValid) return null; this.logger.info( `Serving asset ${ual} from pending storage cache (merkleRoot: ${latestMerkleRoot})`, ); await this.operationService.markOperationAsCompleted( operationId, blockchain, cachedResponseData, [ OPERATION_ID_STATUS.GET.GET_LOCAL_END, OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED, ], ); return Command.empty(); } _filterAssertionByContentType(assertion, contentType) { if (!contentType || contentType === 'all') return assertion; if (contentType === 'public') { return { public: assertion.public || [] }; } if (contentType === 'private') { return { private: assertion.private || [] }; } return assertion; } async validateUAL(operationId, blockchain, contract, knowledgeCollectionId, ual) { const isUAL = this.ualService.isUAL(ual); if (!isUAL) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, UAL: ${ual}: is not a UAL.`, }; } const isValidUal = await this.validationService.validateUal( blockchain, contract, knowledgeCollectionId, ); if (!isValidUal) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, UAL: ${ual}: there is no asset with this UAL.`, }; } return { isValid: true, errorMessage: null, }; } async validateParanet( operationId, paranetUAL, paranetBlockchain, paranetKnowledgeAssetId, paranetNodeAccessPolicy, paranetId, knowledgeCollectionId, blockchain, contract, ual, ) { if (!paranetKnowledgeAssetId) { return { isValid: false, errorMessage: `Invalid paranet UAL: ${paranetUAL} . Paranet knowledge asset token id is required!`, }; } const isParanetUAL = this.ualService.isUAL(paranetUAL); if (!isParanetUAL) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, Paranet UAL: ${paranetUAL}: is not a UAL.`, }; } const [paranetExists, chainParanetNodesAccessPolicy] = await Promise.all([ this.blockchainModuleManager.paranetExists(paranetBlockchain, paranetId), this.blockchainModuleManager.getNodesAccessPolicy(paranetBlockchain, paranetId), ]); const knowledgeCollectionOnchainId = this.cryptoService.keccak256EncodePacked( ['address', 'uint256'], [contract, knowledgeCollectionId], ); const paranetContainsKnowledgeCollection = await this.blockchainModuleManager.isKnowledgeCollectionRegistered( blockchain, paranetId, knowledgeCollectionOnchainId, ); if (!paranetContainsKnowledgeCollection) { return { isValid: false, errorMessage: `Paranet UAL: ${paranetUAL} does not contain Knowledge Collection: ${ual}`, }; } if (!paranetExists) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, Paranet UAL: ${paranetUAL}: paranet does not exist.`, }; } if (paranetNodeAccessPolicy !== chainParanetNodesAccessPolicy) { return { isValid: false, errorMessage: `Get for operation id: ${operationId}, Paranet UAL: ${paranetUAL}: onchain paranet access policy does not match the requested paranet access policy.`, }; } return { isValid: true, errorMessage: null, }; } async findShardNodes(operationId, blockchain, currentPeerId) { this.logger.debug(`Searching for shard for operationId: ${operationId}`); const networkProtocols = this.operationService.getNetworkProtocols(); const shardNodes = await this.shardingTableService.findShard(blockchain, true); // TODO: Optimize this so it's returned by shardingTableService.findShard const foundNodes = await Promise.all( shardNodes.map(({ peerId }) => this.shardingTableService.findPeerAddressAndProtocols(peerId), ), ); const nodesInfo = []; for (const node of foundNodes) { if (node.id !== currentPeerId) { nodesInfo.push({ id: node.id, protocol: networkProtocols[0] }); } } this.logger.debug(`Found ${nodesInfo.length} node(s) for operationId: ${operationId}`); this.logger.trace( `Found shard: ${JSON.stringify( nodesInfo.map((node) => node.id), null, 2, )}`, ); return nodesInfo; } async sendMessage(node, operationId, message) { const response = await this.messagingService.sendProtocolMessage( node, operationId, message, NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST, NETWORK_MESSAGE_TIMEOUT_MILLS.GET.REQUEST, ); return { success: response.header.messageType === NETWORK_MESSAGE_TYPES.RESPONSES.ACK, responseData: response.data, }; } async validateResponse( responseData, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, paranetNodesAccessPolicy, contentType, ) { if (knowledgeAssetId) { return true; } if (responseData?.assertion?.public) { // We can only validate whole collection not particular KA if ( !knowledgeAssetId || (typeof knowledgeAssetId === 'object' && Object.keys(knowledgeAssetId).length === 3 && 'startTokenId' in knowledgeAssetId && 'endTokenId' in knowledgeAssetId && 'burned' in knowledgeAssetId && Array.isArray(knowledgeAssetId.burned)) ) { const publicAssertion = responseData?.assertion?.public; const filteredPublic = []; const privateHashTriples = []; publicAssertion.forEach((triple) => { if (triple.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { privateHashTriples.push(triple); } else { filteredPublic.push(triple); } }); const publicKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( filteredPublic, true, ); publicKnowledgeAssetsTriplesGrouped.push( ...kcTools.groupNquadsBySubject(privateHashTriples, true), ); try { await this.validationService.validateDatasetOnBlockchain( publicKnowledgeAssetsTriplesGrouped.map((t) => t.sort()).flat(), blockchain, contract, knowledgeCollectionId, ); if (paranetNodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { if (Array.isArray(responseData?.assertion?.public)) { const assertionShouldHavePrivateTriples = responseData?.assertion?.public?.some((triple) => triple.includes(`${PRIVATE_ASSERTION_PREDICATE}`), ); if (assertionShouldHavePrivateTriples) { if (responseData?.assertion?.private?.length > 0) { await this.validationService.validatePrivateMerkleRoot( responseData.assertion.public, responseData.assertion.private, ); return true; } } } return false; } if (responseData.assertion?.private?.length) { await this.validationService.validatePrivateMerkleRoot( responseData.assertion.public, responseData.assertion.private, ); return true; } } catch (e) { return false; } } return true; } if ( !responseData?.assertion?.public && responseData?.assertion?.private && contentType === 'private' ) { // if there is only private part skip validation return true; } return false; } /** * Builds default GetCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'getCommand', transactional: false, priority: COMMAND_PRIORITY.HIGH, }; Object.assign(command, map); return command; } } export default GetCommand; ================================================ FILE: src/commands/protocols/publish/publish-finalization-command.js ================================================ import Command from '../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, MAX_RETRIES_READ_CACHED_PUBLISH_DATA, RETRY_DELAY_READ_CACHED_PUBLISH_DATA, TRIPLE_STORE_REPOSITORIES, NETWORK_MESSAGE_TYPES, NETWORK_MESSAGE_TIMEOUT_MILLS, COMMAND_PRIORITY, } from '../../../constants/constants.js'; class PublishFinalizationCommand extends Command { constructor(ctx) { super(ctx); this.ualService = ctx.ualService; this.fileService = ctx.fileService; this.messagingService = ctx.messagingService; this.operationService = ctx.finalityService; this.errorType = ERROR_TYPE.STORE_ASSERTION_ERROR; this.tripleStoreService = ctx.tripleStoreService; this.operationIdService = ctx.operationIdService; this.networkModuleManager = ctx.networkModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.dataService = ctx.dataService; this.blockchainModuleManager = ctx.blockchainModuleManager; } async execute(command) { const { event } = command.data; const eventData = JSON.parse(event.data); const { txHash, blockNumber } = event; const { id, publishOperationId, merkleRoot, byteSize } = eventData; const { blockchain, contractAddress } = event; const operationId = this.operationIdService.generateId(); const ual = this.ualService.deriveUAL(blockchain, contractAddress, id); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH_FINALIZATION.PUBLISH_FINALIZATION_START, operationId, blockchain, publishOperationId, ); let transaction; let blockTimestamp; try { [transaction, blockTimestamp] = await Promise.all([ this.blockchainModuleManager.getTransaction(blockchain, txHash), this.blockchainModuleManager.getBlockTimestamp(blockchain, blockNumber), ]); } catch (error) { this.logger.error(`Failed to get transaction or block timestamp: ${error.message}`); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, blockchain, publishOperationId, ); return Command.empty(); } const metadata = { publisherKey: transaction.from.toLowerCase(), blockNumber, txHash, blockTimestamp, }; let publisherPeerId; let cachedMerkleRoot; let assertion; try { const result = await this.readWithRetries(publishOperationId); cachedMerkleRoot = result.merkleRoot; assertion = result.assertion; publisherPeerId = result.remotePeerId; } catch (_error) { this.logger.warn( `[Cache] Failed to read cached publish data for UAL ${ual} (publishOperationId: ${publishOperationId}, txHash: ${txHash}, operationId: ${operationId})`, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, blockchain, publishOperationId, ); return Command.empty(); } try { await this.validatePublishData(merkleRoot, cachedMerkleRoot, byteSize, assertion, ual); } catch (e) { this.logger.error(`Failed to validate publish data: ${e.message}`); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, blockchain, publishOperationId, ); return Command.empty(); } try { await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH_FINALIZATION.PUBLISH_FINALIZATION_STORE_ASSERTION_START, operationId, blockchain, ); const totalTriples = await this.tripleStoreService.insertKnowledgeCollection( TRIPLE_STORE_REPOSITORIES.DKG, ual, assertion, metadata, ); await this.repositoryModuleManager.incrementInsertedTriples(totalTriples ?? 0); this.logger.info(`Number of triples added to the database +${totalTriples}`); await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH_FINALIZATION.PUBLISH_FINALIZATION_STORE_ASSERTION_END, operationId, blockchain, ); const myPeerId = this.networkModuleManager.getPeerId().toB58String(); if (publisherPeerId === myPeerId) { await this.repositoryModuleManager.saveFinalityAck( publishOperationId, ual, publisherPeerId, ); for (const status of this.operationService.completedStatuses) { this.operationIdService.emitChangeEvent(status, operationId, blockchain); } } else { const networkProtocols = this.operationService.getNetworkProtocols(); const node = { id: publisherPeerId, protocol: networkProtocols[0] }; const message = { ual, publishOperationId, blockchain, operationId }; const maxFinalityAttempts = 3; const backoffDelays = [0, 5_000, 10_000]; let response; let lastError; for (let attempt = 0; attempt < maxFinalityAttempts; attempt += 1) { if (backoffDelays[attempt] > 0) { // eslint-disable-next-line no-await-in-loop await new Promise((r) => { setTimeout(r, backoffDelays[attempt]); }); } try { // eslint-disable-next-line no-await-in-loop response = await this.messagingService.sendProtocolMessage( node, operationId, message, NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST, NETWORK_MESSAGE_TIMEOUT_MILLS.FINALITY.REQUEST, ); lastError = null; break; } catch (err) { lastError = err; this.logger.warn( `Finality request to publisher ${publisherPeerId} failed ` + `(attempt ${attempt + 1}/${maxFinalityAttempts}): ${err.message}`, ); } } if (lastError) { throw lastError; } await this.messagingService.handleProtocolResponse( response, this.operationService, blockchain, operationId, ); } } catch (e) { this.logger.error(`Command error (${this.errorType}): ${e.message}`); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, blockchain, publishOperationId, ); } return Command.empty(); } async validatePublishData(merkleRoot, cachedMerkleRoot, byteSize, assertion, ual) { if (merkleRoot !== cachedMerkleRoot) { const errorMessage = `Invalid Merkle Root for Knowledge Collection: ${ual}. Received value from blockchain: ${merkleRoot}, Cached value from publish operation: ${cachedMerkleRoot}`; throw new Error(errorMessage); } const calculatedAssertionSize = this.dataService.calculateAssertionSize( assertion.public ?? assertion, ); if (byteSize.toString() !== calculatedAssertionSize.toString()) { const errorMessage = `Invalid Assertion Size for Knowledge Collection: ${ual}. Received value from blockchain: ${byteSize}, Calculated value: ${calculatedAssertionSize}`; throw new Error(errorMessage); } } async readWithRetries(publishOperationId) { let attempt = 0; const datasetPath = this.fileService.getPendingStorageDocumentPath(publishOperationId); while (attempt < MAX_RETRIES_READ_CACHED_PUBLISH_DATA) { try { // eslint-disable-next-line no-await-in-loop const cachedData = await this.fileService.readFile(datasetPath, true); return cachedData; } catch (error) { attempt += 1; if (attempt < MAX_RETRIES_READ_CACHED_PUBLISH_DATA) { this.logger.debug( `[Cache] Read attempt ${attempt}/${MAX_RETRIES_READ_CACHED_PUBLISH_DATA} ` + `failed for publishOperationId: ${publishOperationId}, retrying in ${RETRY_DELAY_READ_CACHED_PUBLISH_DATA}ms...`, ); // eslint-disable-next-line no-await-in-loop await new Promise((resolve) => { setTimeout(resolve, RETRY_DELAY_READ_CACHED_PUBLISH_DATA); }); } } } this.logger.warn( `[Cache] Exhausted retries reading cached publish data (publishOperationId: ${publishOperationId}, path: ${datasetPath}).`, ); throw new Error('Failed to read cached publish data'); } /** * Builds default readCachedPublishDataCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'publishFinalizationCommand', transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default PublishFinalizationCommand; ================================================ FILE: src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js ================================================ import HandleProtocolMessageCommand from '../../../common/handle-protocol-message-command.js'; import { NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS, ERROR_TYPE, COMMAND_PRIORITY, } from '../../../../../constants/constants.js'; class HandleStoreRequestCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); this.validationService = ctx.validationService; this.operationService = ctx.publishService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.tripleStoreService = ctx.tripleStoreService; this.ualService = ctx.ualService; this.pendingStorageService = ctx.pendingStorageService; this.operationIdService = ctx.operationIdService; this.pendingStorageService = ctx.pendingStorageService; this.signatureService = ctx.signatureService; this.errorType = ERROR_TYPE.PUBLISH.PUBLISH_LOCAL_STORE_REMOTE_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_REMOTE_START; this.operationEndEvent = OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_REMOTE_END; } async prepareMessage(commandData) { const { blockchain, operationId, datasetRoot, remotePeerId, isOperationV0 } = commandData; await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_VALIDATE_ASSET_REMOTE_START, operationId, blockchain, ); const { dataset } = await this.operationIdService.getCachedOperationIdData(operationId); const validationResult = await this.validateReceivedData( operationId, datasetRoot, dataset, blockchain, isOperationV0, ); await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_VALIDATE_ASSET_REMOTE_END, operationId, blockchain, ); if (validationResult.messageType === NETWORK_MESSAGE_TYPES.RESPONSES.NACK) { return validationResult; } await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_REMOTE_CACHE_DATASET_START, operationId, blockchain, ); if (isOperationV0) { const { contract, tokenId } = commandData; const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); await this.tripleStoreService.createV6KnowledgeCollection(dataset, ual); } else { await this.pendingStorageService.cacheDataset( operationId, datasetRoot, dataset, remotePeerId, ); } await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_REMOTE_CACHE_DATASET_END, operationId, blockchain, ); const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const { v, r, s, vs } = await this.signatureService.signMessage(blockchain, datasetRoot); await this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_VALIDATE_ASSET_REMOTE_END, operationId, blockchain, ); return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: { identityId, v, r, s, vs }, }; } /** * Builds default handleStoreRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0HandleStoreRequestCommand', priority: COMMAND_PRIORITY.HIGHEST, transactional: false, }; Object.assign(command, map); return command; } } export default HandleStoreRequestCommand; ================================================ FILE: src/commands/protocols/publish/sender/publish-replication-command.js ================================================ import { Semaphore } from 'async-mutex'; import { OPERATION_ID_STATUS, ERROR_TYPE, OPERATION_REQUEST_STATUS, NETWORK_MESSAGE_TYPES, NETWORK_SIGNATURES_FOLDER, PUBLISHER_NODE_SIGNATURES_FOLDER, NETWORK_MESSAGE_TIMEOUT_MILLS, COMMAND_PRIORITY, } from '../../../../constants/constants.js'; import Command from '../../../command.js'; const replicationSemaphore = new Semaphore(3); class PublishReplicationCommand extends Command { constructor(ctx) { super(ctx); this.operationIdService = ctx.operationIdService; this.operationService = ctx.publishService; this.shardingTableService = ctx.shardingTableService; this.networkModuleManager = ctx.networkModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.signatureService = ctx.signatureService; this.cryptoService = ctx.cryptoService; this.messagingService = ctx.messagingService; this.pendingStorageService = ctx.pendingStorageService; this.errorType = ERROR_TYPE.LOCAL_STORE.LOCAL_STORE_ERROR; } async execute(command) { const { operationId, blockchain, datasetRoot, minimumNumberOfNodeReplications, batchSize } = command.data; this.logger.debug( `Searching for shard for operationId: ${operationId}, dataset root: ${datasetRoot}`, ); try { await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.FIND_NODES_START, ); const minAckResponses = this.operationService.getMinAckResponses( minimumNumberOfNodeReplications, ); const networkProtocols = this.operationService.getNetworkProtocols(); const shardNodes = []; let nodePartOfShard = false; const currentPeerId = this.networkModuleManager.getPeerId().toB58String(); const foundNodes = await this.findShardNodes(blockchain); for (const node of foundNodes) { if (node.id === currentPeerId) { nodePartOfShard = true; } else { shardNodes.push({ id: node.id, protocol: networkProtocols[0] }); } } this.logger.debug( `Found ${ shardNodes.length + (nodePartOfShard ? 1 : 0) } node(s) for operationId: ${operationId}`, ); this.logger.trace( `Found shard: ${JSON.stringify( shardNodes.map((node) => node.id), null, 2, )}`, ); if (shardNodes.length + (nodePartOfShard ? 1 : 0) < minAckResponses) { await this.handleError( operationId, blockchain, `Unable to find enough nodes for operationId: ${operationId}. Minimum number of nodes required: ${minAckResponses}`, this.errorType, true, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, blockchain, ); return Command.empty(); } try { await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.PUBLISH.PUBLISH_REPLICATE_START, ); const batchSizePar = this.operationService.getBatchSize(batchSize); const { identityId, v, r, s, vs } = await this.createSignatures( blockchain, nodePartOfShard, datasetRoot, operationId, ); const updatedData = { ...command.data, batchSize: batchSizePar, minAckResponses, numberOfFoundNodes: shardNodes.length + (nodePartOfShard ? 1 : 0), }; // eslint-disable-next-line no-param-reassign command.data = updatedData; if (nodePartOfShard) { await this.operationService.processResponse( { ...command, data: updatedData }, OPERATION_REQUEST_STATUS.COMPLETED, { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: { identityId, v, r, s, vs }, }, null, ); } } catch (e) { await this.handleError(operationId, blockchain, e.message, this.errorType, true); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, blockchain, ); return Command.empty(); } const { dataset } = await this.operationIdService.getCachedOperationIdData(operationId); await this.pendingStorageService.cacheDataset( operationId, datasetRoot, dataset, currentPeerId, ); const message = { dataset: dataset.public, datasetRoot, blockchain, }; const replicationBatchSize = minAckResponses + 2; await replicationSemaphore.runExclusive(async () => { this.logger.info( `[REPLICATION] Starting for operationId: ${operationId}, ` + `shard: ${shardNodes.length} nodes, batch: ${replicationBatchSize}, min ACKs: ${minAckResponses}`, ); for (let i = 0; i < shardNodes.length; i += replicationBatchSize) { if (i > 0) { // eslint-disable-next-line no-await-in-loop const record = await this.operationIdService.getOperationIdRecord( operationId, ); if (record?.minAcksReached) { this.logger.info( `[REPLICATION] Minimum replication reached after ${i} nodes, ` + `skipping remaining ${ shardNodes.length - i } for operationId: ${operationId}`, ); break; } } const batch = shardNodes.slice(i, i + replicationBatchSize); this.logger.debug( `Sending replication batch ${Math.floor(i / replicationBatchSize) + 1} ` + `(${batch.length} nodes) for operationId: ${operationId}`, ); // eslint-disable-next-line no-await-in-loop await Promise.all( batch.map((node) => this.sendAndHandleMessage(node, operationId, message, command), ), ); } }); } catch (e) { await this.handleError(operationId, blockchain, e.message, this.errorType, true); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, blockchain, ); return Command.empty(); } return Command.empty(); } async sendAndHandleMessage(node, operationId, message, command) { try { let response = await this.messagingService.sendProtocolMessage( node, operationId, message, NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST, NETWORK_MESSAGE_TIMEOUT_MILLS.PUBLISH.REQUEST, ); if (response.header.messageType !== NETWORK_MESSAGE_TYPES.RESPONSES.ACK) { const preRetryRecord = await this.operationIdService.getOperationIdRecord( operationId, ); if (preRetryRecord?.minAcksReached) return; this.logger.info( `[REPLICATION] Peer ${node.id} NACK for operationId: ${operationId}: ` + `${response.data?.errorMessage || 'unknown reason'}, retrying...`, ); response = await this.messagingService.sendProtocolMessage( node, operationId, message, NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST, NETWORK_MESSAGE_TIMEOUT_MILLS.PUBLISH.REQUEST, ); } const responseData = response.data; if (response.header.messageType === NETWORK_MESSAGE_TYPES.RESPONSES.ACK) { await this.signatureService.addSignatureToStorage( NETWORK_SIGNATURES_FOLDER, operationId, responseData.identityId, responseData.v, responseData.r, responseData.s, responseData.vs, ); await this.operationService.processResponse( command, OPERATION_REQUEST_STATUS.COMPLETED, responseData, ); } else { this.logger.warn( `[REPLICATION] Peer ${node.id} failed after retry for operationId: ${operationId}: ` + `${responseData?.errorMessage || 'unknown reason'}`, ); await this.operationService.processResponse( command, OPERATION_REQUEST_STATUS.FAILED, responseData, ); } } catch (error) { this.logger.warn( `[REPLICATION] Peer ${node.id} error for operationId: ${operationId}: ${error.message}`, ); await this.operationService.processResponse(command, OPERATION_REQUEST_STATUS.FAILED, { errorMessage: error.message, }); } } async findShardNodes(blockchainId) { const shardNodes = await this.shardingTableService.findShard( blockchainId, true, // filter inactive nodes ); // TODO: Optimize this so it's returned by shardingTableService.findShard const nodesFound = await Promise.all( shardNodes.map(({ peerId }) => this.shardingTableService.findPeerAddressAndProtocols(peerId), ), ); return nodesFound; } async createSignatures(blockchain, nodePartOfShard, datasetRoot, operationId) { let v; let r; let s; let vs; const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); if (nodePartOfShard) { ({ v, r, s, vs } = await this.signatureService.signMessage(blockchain, datasetRoot)); await this.signatureService.addSignatureToStorage( NETWORK_SIGNATURES_FOLDER, operationId, identityId, v, r, s, vs, ); } const { v: publisherNodeV, r: publisherNodeR, s: publisherNodeS, vs: publisherNodeVS, } = await this.signatureService.signMessage( blockchain, this.cryptoService.keccak256EncodePacked( ['uint72', 'bytes32'], [identityId, datasetRoot], ), ); await this.signatureService.addSignatureToStorage( PUBLISHER_NODE_SIGNATURES_FOLDER, operationId, identityId, publisherNodeV, publisherNodeR, publisherNodeS, publisherNodeVS, ); return { identityId, v, r, s, vs }; } /** * Builds default localStoreCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'publishReplicationCommand', transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }; Object.assign(command, map); return command; } } export default PublishReplicationCommand; ================================================ FILE: src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js ================================================ import HandleProtocolMessageCommand from '../../../common/handle-protocol-message-command.js'; import { NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS, ERROR_TYPE, } from '../../../../../constants/constants.js'; class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); this.operationService = ctx.updateService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.pendingStorageService = ctx.pendingStorageService; this.operationIdService = ctx.operationIdService; this.pendingStorageService = ctx.pendingStorageService; this.signatureService = ctx.signatureService; this.errorType = ERROR_TYPE.UPDATE.UPDATE_LOCAL_STORE_REMOTE_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.UPDATE.UPDATE_LOCAL_STORE_REMOTE_START; this.operationEndEvent = OPERATION_ID_STATUS.UPDATE.UPDATE_LOCAL_STORE_REMOTE_END; } async prepareMessage(commandData) { const { blockchain, operationId, datasetRoot } = commandData; const { dataset } = await this.operationIdService.getCachedOperationIdData(operationId); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_VALIDATE_ASSET_REMOTE_START, ); const validationResult = await this.validateReceivedData( operationId, datasetRoot, dataset, blockchain, ); this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_VALIDATE_ASSET_REMOTE_END, ); if (validationResult.messageType === NETWORK_MESSAGE_TYPES.RESPONSES.NACK) { return validationResult; } await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_LOCAL_STORE_REMOTE_CACHE_DATASET_START, ); await this.pendingStorageService.cacheDataset(operationId, datasetRoot, dataset); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_LOCAL_STORE_REMOTE_CACHE_DATASET_END, ); const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const { v, r, s, vs } = await this.signatureService.signMessage(blockchain, datasetRoot); return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: { identityId, v, r, s, vs }, }; } /** * Builds default handleUpdateRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0HandleUpdateRequestCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default HandleUpdateRequestCommand; ================================================ FILE: src/commands/protocols/update/sender/network-update-command.js ================================================ import NetworkProtocolCommand from '../../common/network-protocol-command.js'; import { ERROR_TYPE } from '../../../../constants/constants.js'; class NetworkUpdateCommand extends NetworkProtocolCommand { constructor(ctx) { super(ctx); this.blockchainModuleManager = ctx.blockchainModuleManager; // can we remove this this.ualService = ctx.ualService; // can we remove this this.errorType = ERROR_TYPE.UPDATE.UPDATE_NETWORK_START_ERROR; } /** * Builds default networkUpdateCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'networkUpdateCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default NetworkUpdateCommand; ================================================ FILE: src/commands/protocols/update/sender/update-find-shard-command.js ================================================ import FindShardCommand from '../../common/find-shard-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../../constants/constants.js'; class UpdateFindShardCommand extends FindShardCommand { constructor(ctx) { super(ctx); this.operationService = ctx.upateService; this.errorType = ERROR_TYPE.FIND_SHARD.UPDATE_FIND_SHARD_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.UPDATE.UPDATE_FIND_NODES_START; this.operationEndEvent = OPERATION_ID_STATUS.UPDATE.UPDATE_FIND_NODES_END; } getOperationCommandSequence(nodePartOfShard) { const sequence = []; sequence.push('updateValidateAssetCommand'); if (nodePartOfShard) { sequence.push('localUpdateCommand'); } sequence.push('networkUpdateCommand'); return sequence; } /** * Builds default updateFindShardCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'updateFindShardCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default UpdateFindShardCommand; ================================================ FILE: src/commands/protocols/update/sender/update-schedule-messages-command.js ================================================ import ProtocolScheduleMessagesCommand from '../../common/protocol-schedule-messages-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../../constants/constants.js'; class UpdateScheduleMessagesCommand extends ProtocolScheduleMessagesCommand { constructor(ctx) { super(ctx); this.blockchainModuleManager = ctx.blockchainModuleManager; // can this be removed this.repositoryModuleManager = ctx.repositoryModuleManager; // can this be removed this.operationStartEvent = OPERATION_ID_STATUS.UPDATE.UPDATE_REPLICATE_START; this.operationEndEvent = OPERATION_ID_STATUS.UPDATE.UPDATE_REPLICATE_END; this.errorType = ERROR_TYPE.UPDATE.UPDATE_START_ERROR; } /** * Builds default updateScheduleMessagesCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'updateScheduleMessagesCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default UpdateScheduleMessagesCommand; ================================================ FILE: src/commands/protocols/update/sender/update-validate-asset-command.js ================================================ import ValidateAssetCommand from '../../../common/validate-asset-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../../constants/constants.js'; class UpdateValidateAssetCommand extends ValidateAssetCommand { constructor(ctx) { super(ctx); this.operationService = ctx.updateService; this.errorType = ERROR_TYPE.UPDATE.UPDATE_VALIDATE_ASSET_ERROR; } async handleError(operationId, blockchain, errorMessage, errorType) { await this.operationService.markOperationAsFailed( operationId, blockchain, errorMessage, errorType, ); } /** * Executes command and produces one or more events * @param command */ async execute(command) { const { operationId, blockchain, datasetRoot } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_VALIDATE_ASSET_START, ); const cachedData = await this.operationIdService.getCachedOperationIdData(operationId); await this.validationService.validateDatasetRoot(cachedData.dataset, datasetRoot); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_VALIDATE_ASSET_END, ); return this.continueSequence( { ...command.data, retry: undefined, period: undefined }, command.sequence, ); } /** * Builds default updateValidateAssetCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'updateValidateAssetCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default UpdateValidateAssetCommand; ================================================ FILE: src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js ================================================ import ProtocolRequestCommand from '../../../common/protocol-request-command.js'; import { NETWORK_MESSAGE_TIMEOUT_MILLS, ERROR_TYPE, NETWORK_SIGNATURES_FOLDER, } from '../../../../../constants/constants.js'; class PublishRequestCommand extends ProtocolRequestCommand { constructor(ctx) { super(ctx); this.operationService = ctx.updateService; this.signatureService = ctx.signatureService; this.operationIdService = ctx.operationIdService; this.errorType = ERROR_TYPE.UPDATE.UPDATE_STORE_REQUEST_ERROR; } async prepareMessage(command) { const { datasetRoot, operationId } = command.data; // TODO: Backwards compatibility, send blockchain without chainId const { blockchain } = command.data; const { dataset } = await this.operationIdService.getCachedOperationIdData(operationId); return { dataset, datasetRoot, blockchain, }; } messageTimeout() { return NETWORK_MESSAGE_TIMEOUT_MILLS.UPDATE.REQUEST; } async handleAck(command, responseData) { const { operationId } = command.data; await this.signatureService.addSignatureToStorage( NETWORK_SIGNATURES_FOLDER, operationId, responseData.identityId, responseData.v, responseData.r, responseData.s, responseData.vs, ); return super.handleAck(command, responseData); } /** * Builds default publishRequestCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'v1_0_0PublishRequestCommand', delay: 0, transactional: false, }; Object.assign(command, map); return command; } } export default PublishRequestCommand; ================================================ FILE: src/commands/protocols/update/update-assertion-command.js ================================================ import { kcTools } from 'assertion-tools'; import Command from '../../command.js'; import { // OPERATION_ID_STATUS, ERROR_TYPE, TRIPLE_STORE_REPOSITORY, TRIPLES_VISIBILITY, } from '../../../constants/constants.js'; class UpdateAssertionCommand extends Command { constructor(ctx) { super(ctx); this.operationIdService = ctx.operationIdService; this.ualService = ctx.ualService; this.dataService = ctx.dataService; this.tripleStoreService = ctx.tripleStoreService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.errorType = ERROR_TYPE.UPDATE.UPDATE_ASSERTION_ERROR; } async execute(command) { const { operationId, ual, blockchain, assertion, firstNewKAIndex, updateStateIndex } = command.data; const validateCurrentData = this.validateCurrentData(ual); if (this.validateCurrentData(validateCurrentData)) { const preUpdateUalNamedGraphs = // Old subjects old ual from select returned here probably {s, g} await this.tripleStoreService.moveToHistoricAndDeleteAssertion( ual, updateStateIndex - 1, ); await this.tripleStoreService.insertUpdatedKnowledgeCollection( preUpdateUalNamedGraphs, ual, assertion, firstNewKAIndex, ); } else { await this.handleError( operationId, blockchain, `Data in current DKG doesn't match pre update data for ${ual}.`, ERROR_TYPE.UPDATE_FINALIZATION.UPDATE_FINALIZATION_NO_OLD_DATA, true, ); } return Command.empty(); } // TODO: Move maybe outside of the command into metadata validation command (but it's not metadata) async validateCurrentData(ual) { const { blockchain, contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); const assertionIds = await this.blockchainModuleManager.getKnowledgeCollectionMerkleRoot( blockchain, contract, knowledgeCollectionId, ); const assertionIdOfCurrent = assertionIds[assertionIds.length() - 2]; const preUpdateAssertion = await this.tripleStoreService.getKnowledgeAssetNamedGraph( TRIPLE_STORE_REPOSITORY.DKG, ual, TRIPLES_VISIBILITY.PUBLIC, ); const preUpdateMerkleRoot = kcTools.calculateMerkleRoot(preUpdateAssertion); return assertionIdOfCurrent === preUpdateMerkleRoot; } /** * Builds default updateAssertionCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'updateAssertionCommand', delay: 0, retries: 0, transactional: false, }; Object.assign(command, map); return command; } } export default UpdateAssertionCommand; ================================================ FILE: src/commands/protocols/update/update-validate-assertion-metadata-command.js ================================================ import ValidateAssertionMetadataCommand from '../common/validate-assertion-metadata-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../constants/constants.js'; class UpdateValidateAssertionMetadataCommand extends ValidateAssertionMetadataCommand { constructor(ctx) { super(ctx); this.operationIdService = ctx.operationIdService; this.errorType = ERROR_TYPE.UPDATE.UPDATE_VALIDATE_ASSERTION_METADATA_ERROR; this.operationStartEvent = OPERATION_ID_STATUS.UPDATE_FINALIZATION.UPDATE_FINALIZATION_METADATA_VALIDATION_START; this.operationEndEvent = OPERATION_ID_STATUS.UPDATE_FINALIZATION.UPDATE_FINALIZATION_METADATA_VALIDATION_END; } /** * Builds default updateValidateAssertionMetadataCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'updateValidateAssertionMetadataCommand', delay: 0, retries: 0, transactional: false, }; Object.assign(command, map); return command; } } export default UpdateValidateAssertionMetadataCommand; ================================================ FILE: src/commands/query/query-command.js ================================================ import Command from '../command.js'; import { TRIPLE_STORE_REPOSITORIES, QUERY_TYPES, OPERATION_ID_STATUS, ERROR_TYPE, COMMAND_PRIORITY, } from '../../constants/constants.js'; class QueryCommand extends Command { constructor(ctx) { super(ctx); this.dataService = ctx.dataService; this.tripleStoreService = ctx.tripleStoreService; this.paranetService = ctx.paranetService; this.ualService = ctx.ualService; this.errorType = ERROR_TYPE.QUERY.LOCAL_QUERY_ERROR; } async execute(command) { const { operationId, queryType, paranetUAL } = command.data; let { query, repository } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.QUERY.QUERY_START, ); let data; if (paranetUAL) { repository = this.paranetService.getParanetRepositoryName(paranetUAL); } // TODO: Review federated query logic for V8 // check if it's federated query const pattern = /SERVICE\s+<([^>]+)>/g; const matches = query.match(pattern); if (matches?.length > 0) { for (const match of matches) { const repositoryInOriginalQuery = match.split('<')[1].split('>')[0]; const repositoryName = this.validateRepositoryName(repositoryInOriginalQuery); const federatedQueryRepositoryEndpoint = this.tripleStoreService.getRepositorySparqlEndpoint(repositoryName); query = query.replace(repositoryInOriginalQuery, federatedQueryRepositoryEndpoint); } } try { switch (queryType) { case QUERY_TYPES.CONSTRUCT: { if (Array.isArray(repository)) { const dataV6 = await this.tripleStoreService.construct( query, repository[0], ); const dataV8 = await this.tripleStoreService.construct( query, repository[1], ); data = this.dataService.removeDuplicateObjectsFromArray([ ...dataV6, ...dataV8, ]); } else { data = await this.tripleStoreService.construct(query, repository); } break; } case QUERY_TYPES.SELECT: { if (Array.isArray(repository)) { const dataV6 = await this.tripleStoreService.select(query, repository[0]); const dataV8 = await this.tripleStoreService.select(query, repository[1]); data = this.dataService.removeDuplicateObjectsFromArray([ ...dataV6, ...dataV8, ]); } else { data = await this.tripleStoreService.select(query, repository); } break; } default: throw new Error(`Unknown query type ${queryType}`); } await this.operationIdService.cacheOperationIdDataToMemory(operationId, data); await this.operationIdService.cacheOperationIdDataToFile(operationId, data); await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.QUERY.QUERY_END, ); await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.COMPLETED, ); } catch (e) { await this.handleError(operationId, null, e.message, this.errorType, true); } return Command.empty(); } validateRepositoryName(repository) { let isParanetRepoValid = false; if (this.ualService.isUAL(repository)) { const paranetRepoName = this.paranetService.getParanetRepositoryName(repository); isParanetRepoValid = this.config.assetSync?.syncParanets.includes(repository); if (isParanetRepoValid) { return paranetRepoName; } } const isTripleStoreRepoValid = Object.values(TRIPLE_STORE_REPOSITORIES).includes(repository); if (isTripleStoreRepoValid) { return repository; } if (!isParanetRepoValid && !isTripleStoreRepoValid) { throw new Error(`Query failed! Repository with name: ${repository} doesn't exist`); } } /** * Builds default queryCommand * @param map * @returns {{add, data: *, delay: *, deadline: *}} */ default(map) { const command = { name: 'queryCommand', priority: COMMAND_PRIORITY.HIGHEST, transactional: false, }; Object.assign(command, map); return command; } } export default QueryCommand; ================================================ FILE: src/constants/constants.js ================================================ import { BigNumber, ethers } from 'ethers'; import { createRequire } from 'module'; export const WS_RPC_PROVIDER_PRIORITY = 2; export const HTTP_RPC_PROVIDER_PRIORITY = 1; export const FALLBACK_PROVIDER_QUORUM = 1; export const PUBLISH_BATCH_SIZE = 20; export const PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS = 3; export const GET_BATCH_SIZE = 2; export const GET_MIN_NUM_OF_NODE_REPLICATIONS = 1; export const FINALITY_BATCH_SIZE = 1; export const FINALITY_MIN_NUM_OF_NODE_REPLICATIONS = 1; export const ASK_BATCH_SIZE = 20; export const RPC_PROVIDER_STALL_TIMEOUT = 60 * 1000; export const SIX_HOURS_IN_MS = 6 * 60 * 60 * 1000; export const UINT256_MAX_BN = ethers.constants.MaxUint256; export const UINT128_MAX_BN = BigNumber.from(2).pow(128).sub(1); export const UINT64_MAX_BN = BigNumber.from(2).pow(64).sub(1); export const UINT40_MAX_BN = BigNumber.from(2).pow(40).sub(1); export const UINT32_MAX_BN = BigNumber.from(2).pow(32).sub(1); export const ONE_ETHER = BigNumber.from('1000000000000000000'); export const HASH_RING_SIZE = ethers.constants.MaxUint256; export const STAKE_UINT256_MULTIPLIER_BN = UINT256_MAX_BN.div(500000000); export const UINT256_UINT32_DIVISOR_BN = UINT256_MAX_BN.div(UINT32_MAX_BN); export const ZERO_PREFIX = '0x'; export const ZERO_BYTES32 = ethers.constants.HashZero; export const ZERO_ADDRESS = ethers.constants.AddressZero; export const SCHEMA_CONTEXT = 'http://schema.org/'; export const PRIVATE_ASSERTION_PREDICATE = 'https://ontology.origintrail.io/dkg/1.0#privateMerkleRoot'; export const TRIPLE_ANNOTATION_LABEL_PREDICATE = 'https://ontology.origintrail.io/dkg/1.0#label'; export const PRIVATE_RESOURCE_PREDICATE = 'https://ontology.origintrail.io/dkg/1.0#representsPrivateResource'; export const DKG_METADATA_PREDICATES = { PUBLISHED_BY: 'https://ontology.origintrail.io/dkg/1.0#publishedBy', PUBLISHED_AT_BLOCK: 'https://ontology.origintrail.io/dkg/1.0#publishedAtBlock', PUBLISH_TX: 'https://ontology.origintrail.io/dkg/1.0#publishTx', PUBLISH_TIME: 'https://ontology.origintrail.io/dkg/1.0#publishTime', BLOCK_TIME: 'https://ontology.origintrail.io/dkg/1.0#blockTime', }; export const PRIVATE_HASH_SUBJECT_PREFIX = 'https://ontology.origintrail.io/dkg/1.0#metadata-hash:'; export const UAL_PREDICATE = ''; export const COMMIT_BLOCK_DURATION_IN_BLOCKS = 5; export const COMMITS_DELAY_BETWEEN_NODES_IN_BLOCKS = 5; export const TRANSACTION_POLLING_TIMEOUT_MILLIS = 300 * 1000; export const SOLIDITY_ERROR_STRING_PREFIX = '0x08c379a0'; export const SOLIDITY_PANIC_CODE_PREFIX = '0x4e487b71'; export const SOLIDITY_PANIC_REASONS = { 0x1: 'Assertion error', 0x11: 'Arithmetic operation underflowed or overflowed outside of an unchecked block', 0x12: 'Division or modulo division by zero', 0x21: 'Tried to convert a value into an enum, but the value was too big or negative', 0x22: 'Incorrectly encoded storage byte array', 0x31: '.pop() was called on an empty array', 0x32: 'Array accessed at an out-of-bounds or negative index', 0x41: 'Too much memory was allocated, or an array was created that is too large', 0x51: 'Called a zero-initialized variable of internal function type', }; export const LIBP2P_KEY_DIRECTORY = 'libp2p'; export const LIBP2P_KEY_FILENAME = 'privateKey'; export const BLS_KEY_DIRECTORY = 'bls'; export const BLS_KEY_FILENAME = 'secretKey'; export const TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; export const COMMAND_PRIORITY = { HIGHEST: 0, HIGH: 1, MEDIUM: 5, LOW: 10, LOWEST: 20, }; export const DEFAULT_COMMAND_PRIORITY = COMMAND_PRIORITY.MEDIUM; export const DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS = 15 * 24 * 60 * 60 * 1000; // 15 days export const MAX_BLOCKCHAIN_EVENT_SYNC_OF_HISTORICAL_BLOCKS_IN_MILLS = 60 * 60 * 1000; // 1 hour export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 50; export const TRANSACTION_QUEUE_CONCURRENCY = 1; export const TRIPLE_STORE_CONNECT_RETRY_FREQUENCY = 10; export const MAX_FILE_SIZE = 10000000; export const GET_STATES = { LATEST: 'LATEST', FINALIZED: 'LATEST_FINALIZED' }; export const BYTES_IN_KILOBYTE = 1024; export const BYTES_IN_MEGABYTE = BYTES_IN_KILOBYTE * BYTES_IN_KILOBYTE; export const PUBLISH_TYPES = { ASSERTION: 'assertion', ASSET: 'asset', INDEX: 'index' }; export const DEFAULT_GET_STATE = GET_STATES.LATEST; export const PEER_OFFLINE_LIMIT = 24 * 60 * 60 * 1000; export const CONTENT_ASSET_HASH_FUNCTION_ID = 1; export const CHUNK_BYTE_SIZE = 32; export const PARANET_SYNC_KA_COUNT = 50; export const PARANET_SYNC_RETRIES_LIMIT = 3; export const PARANET_SYNC_RETRY_DELAY_MS = 60 * 1000; export const PARANET_ACCESS_POLICY = { OPEN: 0, PERMISSIONED: 1, }; export const TRIPLE_STORE_REPOSITORIES = { DKG: 'dkg', PUBLIC_CURRENT: 'publicCurrent', PRIVATE_CURRENT: 'privateCurrent', }; export const BASE_NAMED_GRAPHS = { UNIFIED: 'unified:graph', HISTORICAL_UNIFIED: 'historical-unified:graph', METADATA: 'metadata:graph', CURRENT: 'current:graph', }; export const REQUIRED_MODULES = [ 'repository', 'httpClient', 'network', 'validation', 'blockchain', 'tripleStore', 'blockchainEventsService', ]; /** * Triple store media types * @type {{APPLICATION_JSON: string, N_QUADS: string, SPARQL_RESULTS_JSON: string, LD_JSON: string}} */ export const MEDIA_TYPES = { LD_JSON: 'application/ld+json', N_QUADS: 'application/n-quads', SPARQL_RESULTS_JSON: 'application/sparql-results+json', JSON: 'application/json', }; /** * XML data types * @type {{FLOAT: string, DECIMAL: string, DOUBLE: string, BOOLEAN: string, INTEGER: string}} */ export const XML_DATA_TYPES = { DECIMAL: 'http://www.w3.org/2001/XMLSchema#decimal', FLOAT: 'http://www.w3.org/2001/XMLSchema#float', DOUBLE: 'http://www.w3.org/2001/XMLSchema#double', INTEGER: 'http://www.w3.org/2001/XMLSchema#integer', BOOLEAN: 'http://www.w3.org/2001/XMLSchema#boolean', }; export const MIN_NODE_VERSION = 16; export const NETWORK_API_RATE_LIMIT = { TIME_WINDOW_MILLS: 1 * 60 * 1000, MAX_NUMBER: 100, }; export const NETWORK_API_SPAM_DETECTION = { TIME_WINDOW_MILLS: 1 * 60 * 1000, MAX_NUMBER: 150, }; export const NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES = 60; export const HIGH_TRAFFIC_OPERATIONS_NUMBER_PER_HOUR = 16000; export const SHARDING_TABLE_CHECK_COMMAND_FREQUENCY_MILLS = 10 * 1000; // 10 seconds export const PARANET_SYNC_FREQUENCY_MILLS = 1 * 60 * 1000; export const SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES = 15; export const PEER_RECORD_UPDATE_DELAY = 30 * 60 * 1000; // 30 minutes export const DEFAULT_COMMAND_CLEANUP_TIME_MILLS = 4 * 24 * 60 * 60 * 1000; export const REMOVE_SESSION_COMMAND_DELAY = 2 * 60 * 1000; export const OPERATION_IDS_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; export const GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS = 30 * 1000; export const DIAL_PEERS_COMMAND_FREQUENCY_MILLS = 30 * 1000; export const DIAL_PEERS_CONCURRENCY = 10; export const MIN_DIAL_FREQUENCY_MILLIS = 60 * 60 * 1000; export const PERMANENT_COMMANDS = [ 'eventListenerCommand', 'otnodeUpdateCommand', 'sendTelemetryCommand', 'startParanetSyncCommands', 'dialPeersCommand', 'shardingTableCheckCommand', 'commandsCleanerCommand', 'operationIdCleanerCommand', 'blockchainEventCleanerCommand', 'getCleanerCommand', 'getResponseCleanerCommand', 'publishCleanerCommand', 'publishResponseCleanerCommand', 'pendingStorageCleanerCommand', 'finalityCleanerCommand', 'finalityResponseCleanerCommand', 'askCleanerCommand', 'askResponseCleanerCommand', 'batchGetCleanerCommand', ]; export const MAX_COMMAND_DELAY_IN_MILLS = 14400 * 60 * 1000; // 10 days export const DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS = 5000; // 5 seconds export const DEFAULT_COMMAND_DELAY_IN_MILLS = 60 * 1000; // 60 seconds export const TRANSACTION_PRIORITY = { HIGHEST: 0, HIGH: 1, MEDIUM: 5, LOW: 10, LOWEST: 20, }; export const V0_PRIVATE_ASSERTION_PREDICATE = 'https://ontology.origintrail.io/dkg/1.0#privateAssertionID'; export const DKG_PREDICATE = 'https://ontology.origintrail.io/dkg/1.0#'; export const HAS_NAMED_GRAPH_SUFFIX = 'hasNamedGraph'; export const HAS_KNOWLEDGE_ASSET_SUFFIX = 'hasKnowledgeAsset'; const require = createRequire(import.meta.url); export const ABIs = { KnowledgeCollection: require('dkg-evm-module/abi/KnowledgeCollection.json'), KnowledgeCollectionStorage: require('dkg-evm-module/abi/KnowledgeCollectionStorage.json'), Staking: require('dkg-evm-module/abi/Staking.json'), Token: require('dkg-evm-module/abi/Token.json'), Hub: require('dkg-evm-module/abi/Hub.json'), IdentityStorage: require('dkg-evm-module/abi/IdentityStorage.json'), ParametersStorage: require('dkg-evm-module/abi/ParametersStorage.json'), Profile: require('dkg-evm-module/abi/Profile.json'), ProfileStorage: require('dkg-evm-module/abi/ProfileStorage.json'), ShardingTable: require('dkg-evm-module/abi/ShardingTable.json'), ShardingTableStorage: require('dkg-evm-module/abi/ShardingTableStorage.json'), ParanetsRegistry: require('dkg-evm-module/abi/ParanetsRegistry.json'), ParanetKnowledgeCollectionsRegistry: require('dkg-evm-module/abi/ParanetKnowledgeCollectionsRegistry.json'), AskStorage: require('dkg-evm-module/abi/AskStorage.json'), Chronos: require('dkg-evm-module/abi/Chronos.json'), Paranet: require('dkg-evm-module/abi/Paranet.json'), RandomSampling: require('dkg-evm-module/abi/RandomSampling.json'), RandomSamplingStorage: require('dkg-evm-module/abi/RandomSamplingStorage.json'), DelegatorsInfo: require('dkg-evm-module/abi/DelegatorsInfo.json'), }; export const CONTRACT_FUNCTION_PRIORITY = {}; export const COMMAND_RETRIES = {}; export const SIMPLE_ASSET_SYNC_PARAMETERS = { GET_RESULT_POLLING_INTERVAL_MILLIS: 1 * 1000, GET_RESULT_POLLING_MAX_ATTEMPTS: 30, }; export const PARANET_SYNC_PARAMETERS = { GET_RESULT_POLLING_INTERVAL_MILLIS: 1 * 1000, GET_RESULT_POLLING_MAX_ATTEMPTS: 300, }; export const COMMAND_TX_GAS_INCREASE_FACTORS = { SUBMIT_COMMIT: 1.2, SUBMIT_UPDATE_COMMIT: 1.2, SUBMIT_PROOFS: 1.2, }; export const MIGRATION_FLAG_PATH = '.enrichment_migration_done_dkg'; export const CONTRACT_FUNCTION_GAS_LIMIT_INCREASE_FACTORS = {}; export const GNOSIS_DEFAULT_GAS_PRICE = { TESTNET: 1, MAINNET: 1, }; export const NEURO_DEFAULT_GAS_PRICE = { TESTNET: 8, MAINNET: 8, }; export const CONTRACT_FUNCTION_FIXED_GAS_PRICE = {}; export const WEBSOCKET_PROVIDER_OPTIONS = { reconnect: { auto: true, delay: 1000, // ms maxAttempts: 3, }, clientConfig: { keepalive: true, keepaliveInterval: 30 * 1000, // ms }, }; export const TRIPLE_STORE_IMPLEMENTATION = { BLAZEGRAPH: 'Blazegraph', GRAPHDB: 'GraphDB', FUSEKI: 'Fuseki', NEPTUNE: 'Neptune', }; export const NETWORK_MESSAGE_TYPES = { REQUESTS: { PROTOCOL_REQUEST: 'PROTOCOL_REQUEST', }, RESPONSES: { ACK: 'ACK', NACK: 'NACK', BUSY: 'BUSY', }, }; export const PARANET_NODES_ACCESS_POLICIES = ['OPEN', 'PERMISSIONED']; export const NETWORK_MESSAGE_TIMEOUT_MILLS = { PUBLISH: { REQUEST: 60 * 1000, }, UPDATE: { REQUEST: 60 * 1000, }, GET: { REQUEST: 15 * 1000, }, ASK: { REQUEST: 60 * 1000, }, FINALITY: { REQUEST: 60 * 1000, }, BATCH_GET: { REQUEST: 30 * 1000, }, }; export const MAX_OPEN_SESSIONS = 10; export const ERROR_TYPE = { EVENT_LISTENER_ERROR: 'EventListenerError', BLOCKCHAIN_EVENT_LISTENER_ERROR: 'BlockchainEventListenerError', DIAL_PROTOCOL_ERROR: 'DialProtocolError', VALIDATE_ASSET_ERROR: 'ValidateAssetError', NETWORK_PROTOCOL_ERROR: 'NetworkProtocolError', PUBLISH: { PUBLISH_START_ERROR: 'PublishStartError', PUBLISH_ROUTE_ERROR: 'PublishRouteError', PUBLISH_NETWORK_START_ERROR: 'PublishNetworkStartError', PUBLISH_VALIDATE_ASSET_ERROR: 'PublishValidateAssetError', PUBLISH_LOCAL_STORE_ERROR: 'PublishLocalStoreError', PUBLISH_LOCAL_STORE_REMOTE_ERROR: 'PublishLocalStoreRemoteError', PUBLISH_FIND_NODES_ERROR: 'PublishFindNodesError', PUBLISH_STORE_REQUEST_ERROR: 'PublishStoreRequestError', PUBLISH_VALIDATE_ASSERTION_METADATA_ERROR: 'PublishValidateAssertionMetadataError', PUBLISH_ERROR: 'PublishError', }, STORE_ASSERTION_ERROR: 'StoreAssertionError', UPDATE: { UPDATE_INIT_ERROR: 'UpdateInitError', UPDATE_REQUEST_ERROR: 'UpdateRequestError', UPDATE_START_ERROR: 'UpdateStartError', UPDATE_ROUTE_ERROR: 'UpdateRouteError', UPDATE_LOCAL_STORE_ERROR: 'UpdateLocalStoreError', UPDATE_LOCAL_STORE_REMOTE_ERROR: 'UpdateLocalStoreRemoteError', UPDATE_ERROR: 'UpdateError', UPDATE_STORE_INIT_ERROR: 'UpdateStoreInitError', UPDATE_REMOTE_ERROR: 'UpdateRemoteError', UPDATE_DELETE_PENDING_STATE_ERROR: 'UpdateDeletePendingStateError', UPDATE_VALIDATE_ASSET_ERROR: 'UpdateValidateAssetError', UPDATE_STORE_REQUEST_ERROR: 'UpdateStoreRequestError', UPDATE_VALIDATE_ASSERTION_METADATA_ERROR: 'UpadateValidateAssertionMetadataError', UPDATE_ASSERTION_ERROR: 'UpdateAssertionError', UPDATE_NETWORK_START_ERROR: 'UpdateNetworkStartError', }, GET: { GET_ROUTE_ERROR: 'GetRouteError', GET_ASSERTION_ID_ERROR: 'GetAssertionIdError', GET_PRIVATE_ASSERTION_ID_ERROR: 'GetPrivateAssertionIdError', GET_VALIDATE_ASSET_ERROR: 'GetValidateAssetError', GET_LOCAL_ERROR: 'GetLocalError', GET_NETWORK_ERROR: 'GetNetworkError', GET_CURATED_PARANET_NETWORK_ERROR: 'GetCuratedParanetNetworkError', GET_START_ERROR: 'GetStartError', GET_INIT_ERROR: 'GetInitError', GET_REQUEST_ERROR: 'GetRequestError', GET_INIT_REMOTE_ERROR: 'GetInitRemoteError', GET_REQUEST_REMOTE_ERROR: 'GetRequestRemoteError', GET_ERROR: 'GetError', }, BATCH_GET: { BATCH_GET_ERROR: 'BatchGetError', }, LOCAL_STORE: { LOCAL_STORE_ERROR: 'LocalStoreError', }, QUERY: { LOCAL_QUERY_ERROR: 'LocalQueryError', }, GET_BID_SUGGESTION: { UNSUPPORTED_BID_SUGGESTION_RANGE_ERROR: 'UnsupportedBidSuggestionRangeError', }, PARANET: { START_PARANET_SYNC_ERROR: 'StartParanetSyncError', PARANET_SYNC_ERROR: 'ParanetSyncError', }, FIND_SHARD: { FIND_SHARD_ERROR: 'FindShardError', PUBLISH_FIND_SHARD_ERROR: 'PublishFindShardError', UPDATE_FIND_SHARD_ERROR: 'UpdateFindShardError', GET_FIND_SHARD_ERROR: 'GetFindShardError', BATCH_GET_FIND_SHARD_ERROR: 'BatchGetFindShardError', }, ASK: { ASK_ERROR: 'AskError', ASK_NETWORK_ERROR: 'AskNetworkError', ASK_REQUEST_ERROR: 'AskRequestError', ASK_REQUEST_REMOTE_ERROR: 'AskRequestRemoteError', ASK_FIND_SHARD_ERROR: 'AskFindShardError', }, PUBLISH_FINALIZATION: { PUBLISH_FINALIZATION_NO_CACHED_DATA: 'PublishFinalizationNoCachedData', }, UPDATE_FINALIZATION: { UPDATE_FINALIZATION_NO_CACHED_DATA: 'UpdateFinalizationNoCachedData', UPDATE_FINALIZATION_NO_OLD_DATA: 'UpdateFinalizationNoOldData', }, FINALITY: { FINALITY_ERROR: 'FinalityError', FINALITY_NETWORK_ERROR: 'FinalityNetworkError', FINALITY_REQUEST_ERROR: 'FinalityRequestError', FINALITY_REQUEST_REMOTE_ERROR: 'FinalityRequestRemoteError', FINALITY_START_ERROR: 'FinalityStartError', }, }; export const OPERATION_ID_STATUS = { PENDING: 'PENDING', FAILED: 'FAILED', COMPLETED: 'COMPLETED', FIND_NODES_START: 'FIND_NODES_START', FIND_NODES_END: 'FIND_NODES_END', FIND_CURATED_PARANET_NODES_START: 'FIND_CURATED_PARANET_NODES_START', FIND_CURATED_PARANET_NODES_END: 'FIND_CURATED_PARANET_NODES_END', DIAL_PROTOCOL_START: 'DIAL_PROTOCOL_START', DIAL_PROTOCOL_END: 'DIAL_PROTOCOL_END', VALIDATE_ASSET_START: 'VALIDATE_ASSET_START', VALIDATE_ASSET_END: 'VALIDATE_ASSET_END', VALIDATE_ASSET_BLOCKCHAIN_START: 'VALIDATE_ASSET_BLOCKCHAIN_START', VALIDATE_ASSET_BLOCKCHAIN_END: 'VALIDATE_ASSET_BLOCKCHAIN_END', VALIDATE_ASSERTION_METADATA_START: 'VALIDATE_ASSERTION_METADATA_START', VALIDATE_ASSERTION_METADATA_END: 'VALIDATE_ASSERTION_METADATA_END', PROTOCOL_SCHEDULE_MESSAGE_START: 'PROTOCOL_SCHEDULE_MESSAGE_START', PROTOCOL_SCHEDULE_MESSAGE_END: 'PROTOCOL_SCHEDULE_MESSAGE_END', HANDLE_PROTOCOL_MESSAGE_START: 'HANDLE_PROTOCOL_MESSAGE_START', HANDLE_PROTOCOL_MESSAGE_END: 'HANDLE_PROTOCOL_MESSAGE_END', PUBLISH_LOCAL_STORE_REMOTE_SEND_MESSAGE_RESPONSE_START: 'PUBLISH_LOCAL_STORE_REMOTE_SEND_MESSAGE_RESPONSE_START', PUBLISH_LOCAL_STORE_REMOTE_SEND_MESSAGE_RESPONSE_END: 'PUBLISH_LOCAL_STORE_REMOTE_SEND_MESSAGE_RESPONSE_END', PUBLISH: { VALIDATING_PUBLISH_ASSERTION_REMOTE_START: 'VALIDATING_PUBLISH_ASSERTION_REMOTE_START', VALIDATING_PUBLISH_ASSERTION_REMOTE_END: 'VALIDATING_PUBLISH_ASSERTION_REMOTE_END', PUBLISH_VALIDATE_ASSET_START: 'PUBLISH_VALIDATE_ASSET_START', PUBLISH_VALIDATE_ASSET_END: 'PUBLISH_VALIDATE_ASSET_END', PUBLISH_VALIDATE_ASSET_PARANET_EXISTS_START: 'PUBLISH_VALIDATE_ASSET_PARANET_EXISTS_START', PUBLISH_VALIDATE_ASSET_PARANET_EXISTS_END: 'PUBLISH_VALIDATE_ASSET_PARANET_EXISTS_END', PUBLISH_VALIDATE_ASSET_NODES_ACCESS_POLICY_CHECK_START: 'PUBLISH_VALIDATE_ASSET_NODES_ACCESS_POLICY_CHECK_START', PUBLISH_VALIDATE_ASSET_NODES_ACCESS_POLICY_CHECK_END: 'PUBLISH_VALIDATE_ASSET_NODES_ACCESS_POLICY_CHECK_END', INSERTING_ASSERTION: 'INSERTING_ASSERTION', PUBLISHING_ASSERTION: 'PUBLISHING_ASSERTION', PUBLISH_START: 'PUBLISH_START', PUBLISH_INIT_START: 'PUBLISH_INIT_START', PUBLISH_INIT_END: 'PUBLISH_INIT_END', PUBLISH_LOCAL_STORE_REMOTE_CACHE_DATASET_START: 'PUBLISH_LOCAL_STORE_REMOTE_CACHE_DATASET_START', PUBLISH_LOCAL_STORE_REMOTE_CACHE_DATASET_END: 'PUBLISH_LOCAL_STORE_REMOTE_CACHE_DATASET_END', PUBLISH_REPLICATE_START: 'PUBLISH_REPLICATE_START', PUBLISH_REPLICATE_END: 'PUBLISH_REPLICATE_END', PUBLISH_FIND_NODES_START: 'PUBLISH_FIND_NODES_START', PUBLISH_FIND_NODES_END: 'PUBLISH_FIND_NODES_END', PUBLISH_END: 'PUBLISH_END', PUBLISH_LOCAL_STORE_REMOTE_START: 'PUBLISH_LOCAL_STORE_REMOTE_START', PUBLISH_LOCAL_STORE_REMOTE_END: 'PUBLISH_LOCAL_STORE_REMOTE_END', PUBLISH_VALIDATE_ASSET_REMOTE_START: 'VALIDATE_ASSET_REMOTE_START', PUBLISH_VALIDATE_ASSET_REMOTE_END: 'VALIDATE_ASSET_REMOTE_END', PUBLISH_FAILED: 'PUBLISH_FAILED', }, PUBLISH_FINALIZATION: { PUBLISH_FINALIZATION_START: 'PUBLISH_FINALIZATION_START', PUBLISH_FINALIZATION_METADATA_VALIDATION_START: 'PUBLISH_FINALIZATION_METADATA_VALIDATION_START', PUBLISH_FINALIZATION_METADATA_VALIDATION_END: 'PUBLISH_FINALIZATION_METADATA_VALIDATION_END', PUBLISH_FINALIZATION_STORE_ASSERTION_START: 'PUBLISH_FINALIZATION_STORE_ASSERTION_START', PUBLISH_FINALIZATION_STORE_ASSERTION_END: 'PUBLISH_FINALIZATION_STORE_ASSERTION_END', PUBLISH_FINALIZATION_END: 'PUBLISH_FINALIZATION_END', PUBLISH_FINALIZATION_FAILED: 'PUBLISH_FINALIZATION_FAILED', }, UPDATE_FINALIZATION: { UPDATE_FINALIZATION_START: 'UPDATE_FINALIZATION_START', UPDATE_FINALIZATION_METADATA_VALIDATION_START: 'UPDATE_FINALIZATION_METADATA_VALIDATION_START', UPDATE_FINALIZATION_METADATA_VALIDATION_END: 'UPDATE_FINALIZATION_METADATA_VALIDATION_END', UPDATE_FINALIZATION_STORE_ASSERTION_START: 'UPDATE_FINALIZATION_STORE_ASSERTION_START', UPDATE_FINALIZATION_STORE_ASSERTION_END: 'UPDATE_FINALIZATION_STORE_ASSERTION_END', UPDATE_FINALIZATION_END: 'UPDATE_FINALIZATION_END', }, UPDATE: { UPDATE_START: 'UPDATE_START', UPDATE_INIT_START: 'UPDATE_INIT_START', UPDATE_INIT_END: 'UPDATE_INIT_END', UPDATE_REPLICATE_START: 'UPDATE_REPLICATE_START', UPDATE_REPLICATE_END: 'UPDATE_REPLICATE_END', UPDATE_FIND_NODES_START: 'UPDATE_FIND_NODES_START', UPDATE_FIND_NODES_END: 'UPDATE_FIND_NODES_END', VALIDATING_UPDATE_ASSERTION_REMOTE_START: 'VALIDATING_UPDATE_ASSERTION_REMOTE_START', VALIDATING_UPDATE_ASSERTION_REMOTE_END: 'VALIDATING_UPDATE_ASSERTION_REMOTE_END', UPDATE_END: 'UPDATE_END', UPDATE_VALIDATE_ASSET_START: 'UPDATE_VALIDATE_ASSET_START', UPDATE_VALIDATE_ASSET_END: 'UPDATE_VALIDATE_ASSET_END', UPDATE_NETWORK_START_ERROR: 'UPDATE_NETWORK_START_ERROR', UPDATE_LOCAL_STORE_REMOTE_START: 'UPDATE_LOCAL_STORE_REMOTE_START', UPDATE_LOCAL_STORE_REMOTE_END: 'UPDATE_LOCAL_STORE_REMOTE_END', UPDATE_VALIDATE_ASSET_REMOTE_START: 'UPDATE_VALIDATE_ASSET_REMOTE_START', UPDATE_VALIDATE_ASSET_REMOTE_END: 'UPDATE_VALIDATE_ASSET_REMOTE_END', UPDATE_LOCAL_STORE_REMOTE_CACHE_DATASET_START: 'UPDATE_LOCAL_STORE_REMOTE_CACHE_DATASET_START', UPDATE_LOCAL_STORE_REMOTE_CACHE_DATASET_END: 'UPDATE_LOCAL_STORE_REMOTE_CACHE_DATASET_END', }, GET: { ASSERTION_EXISTS_LOCAL_START: 'ASSERTION_EXISTS_LOCAL_START', ASSERTION_EXISTS_LOCAL_END: 'ASSERTION_EXISTS_LOCAL_END', GET_START: 'GET_START', GET_INIT_START: 'GET_INIT_START', GET_INIT_END: 'GET_INIT_END', GET_VALIDATE_ASSET_START: 'GET_VALIDATE_ASSET_START', GET_VALIDATE_ASSET_END: 'GET_VALIDATE_ASSET_END', GET_LOCAL_START: 'GET_LOCAL_START', GET_LOCAL_END: 'GET_LOCAL_END', GET_REMOTE_START: 'GET_REMOTE_START', GET_REMOTE_END: 'GET_REMOTE_END', GET_FETCH_FROM_NODES_START: 'GET_FETCH_FROM_NODES_START', GET_FETCH_FROM_NODES_END: 'GET_FETCH_FROM_NODES_END', GET_FIND_NODES_START: 'GET_FIND_NODES_START', GET_FIND_NODES_END: 'PUBLISH_FIND_NODES_END', GET_END: 'GET_END', GET_FAILED: 'GET_FAILED', }, BATCH_GET: { BATCH_GET_INIT: 'BATCH_GET_INIT', BATCH_GET_START: 'BATCH_GET_START', BATCH_GET_END: 'BATCH_GET_END', BATCH_GET_FAILED: 'BATCH_GET_FAILED', BATCH_GET_VALIDATE_ASSET_START: 'BATCH_GET_VALIDATE_ASSET_START', BATCH_GET_VALIDATE_ASSET_END: 'BATCH_GET_VALIDATE_ASSET_END', BATCH_GET_VALIDATE_ASSET_ERROR: 'BATCH_GET_VALIDATE_ASSET_ERROR', BATCH_GET_LOCAL_START: 'BATCH_GET_LOCAL_START', BATCH_GET_LOCAL_END: 'BATCH_GET_LOCAL_END', BATCH_GET_REMOTE_START: 'BATCH_GET_REMOTE_START', BATCH_GET_REMOTE_END: 'BATCH_GET_REMOTE_END', BATCH_GET_REQUEST_REMOTE_ERROR: 'BATCH_GET_REQUEST_REMOTE_ERROR', BATCH_GET_FIND_SHARD_START: 'BATCH_GET_FIND_SHARD_START', BATCH_GET_FIND_SHARD_END: 'BATCH_GET_FIND_SHARD_END', }, QUERY: { QUERY_INIT_START: 'QUERY_INIT_START', QUERY_INIT_END: 'QUERY_INIT_END', QUERY_START: 'QUERY_START', QUERY_END: 'QUERY_END', QUERY_FAILED: 'QUERY_FAILED', }, LOCAL_STORE: { LOCAL_STORE_INIT_START: 'LOCAL_STORE_INIT_START', LOCAL_STORE_INIT_END: 'LOCAL_STORE_INIT_END', LOCAL_STORE_START: 'LOCAL_STORE_START', LOCAL_STORE_END: 'LOCAL_STORE_END', LOCAL_STORE_PROCESS_RESPONSE_START: 'LOCAL_STORE_PROCESS_RESPONSE_START', LOCAL_STORE_PROCESS_RESPONSE_END: 'LOCAL_STORE_PROCESS_RESPONSE_END', }, PARANET: { PARANET_SYNC_START: 'PARANET_SYNC_START', PARANET_SYNC_END: 'PARANET_SYNC_END', PARANET_SYNC_MISSED_KAS_SYNC_START: 'PARANET_SYNC_MISSED_KAS_SYNC_START', PARANET_SYNC_MISSED_KAS_SYNC_END: 'PARANET_SYNC_MISSED_KAS_SYNC_END', PARANET_SYNC_NEW_KAS_SYNC_START: 'PARANET_SYNC_NEW_KAS_SYNC_START', PARANET_SYNC_NEW_KAS_SYNC_END: 'PARANET_SYNC_NEW_KAS_SYNC_END', }, ASK: { ASK_START: 'ASK_START', ASK_END: 'ASK_END', ASK_REMOTE_START: 'ASK_REMOTE_START', ASK_REMOTE_END: 'ASK_REMOTE_START', ASK_FIND_NODES_START: 'ASK_FIND_NODES_START', ASK_FIND_NODES_END: 'ASK_FIND_NODES_END', ASK_FETCH_FROM_NODES_START: 'ASK_FETCH_FROM_NODES_START', ASK_FETCH_FROM_NODES_END: 'ASK_FETCH_FROM_NODES_END', }, FINALITY: { FINALITY_START: 'FINALITY_START', FINALITY_END: 'FINALITY_END', FINALITY_REMOTE_START: 'FINALITY_REMOTE_START', FINALITY_REMOTE_END: 'FINALITY_REMOTE_START', FINALITY_REPLICATE_START: 'FINALITY_REPLICATE_START', FINALITY_REPLICATE_END: 'FINALITY_REPLICATE_END', FINALITY_FETCH_FROM_NODES_START: 'FINALITY_FETCH_FROM_NODES_START', FINALITY_FETCH_FROM_NODES_END: 'FINALITY_FETCH_FROM_NODES_END', PUBLISH_FINALITY_REMOTE_START: 'PUBLISH_FINALITY_REMOTE_START', PUBLISH_FINALITY_REMOTE_END: 'PUBLISH_FINALITY_REMOTE_END', PUBLISH_FINALITY_END: 'PUBLISH_FINALITY_END', PUBLISH_FINALITY_FETCH_FROM_NODES_END: 'PUBLISH_FINALITY_FETCH_FROM_NODES_END', }, SYNC: { SYNC_START: 'SYNC_START', SYNC_NEW_START: 'SYNC_NEW_START', SYNC_MISSED_START: 'SYNC_MISSED_START', SYNC_END: 'SYNC_END', SYNC_NEW_END: 'SYNC_NEW_END', SYNC_MISSED_END: 'SYNC_MISSED_END', SYNC_PROGRESS_STATUS: 'SYNC_PROGRESS_STATUS', SYNC_FAILED: 'SYNC_FAILED', SYNC_NEW_FAILED: 'SYNC_NEW_FAILED', SYNC_MISSED_FAILED: 'SYNC_MISSED_FAILED', }, }; export const OPERATIONS = { PUBLISH: 'publish', FINALITY: 'finality', // UPDATE: 'update', GET: 'get', BATCH_GET: 'batchGet', ASK: 'ask', }; export const SERVICE_AGREEMENT_START_TIME_DELAY_FOR_COMMITS_SECONDS = { mainnet: 5 * 60, testnet: 5 * 60, devnet: 3 * 60, test: 10, development: 10, }; export const EXPECTED_TRANSACTION_ERRORS = { INSUFFICIENT_FUNDS: 'InsufficientFunds', NODE_ALREADY_SUBMITTED_COMMIT: 'NodeAlreadySubmittedCommit', TIMEOUT_EXCEEDED: 'timeout exceeded', TOO_LOW_PRIORITY: 'TooLowPriority', NODE_ALREADY_REWARDED: 'NodeAlreadyRewarded', SERVICE_AGREEMENT_DOESNT_EXIST: 'ServiceAgreementDoesntExist', INVALID_SCORE_FUNCTION_ID: 'InvalidScoreFunctionId', COMMIT_WINDOW_CLOSED: 'CommitWindowClosed', NODE_NOT_IN_SHARDING_TABLE: 'NodeNotInShardingTable', PROOF_WINDOW_CLOSED: 'ProofWindowClosed', NODE_NOT_AWARDED: 'NodeNotAwarded', WRONG_MERKLE_PROOF: 'WrongMerkleProof', NO_MINTED_ASSETS: 'NoMintedAssets', NONCE_TOO_LOW: 'nonce too low', REPLACEMENT_UNDERPRICED: 'replacement transaction underpriced', ALREADY_KNOWN: 'already known', EXECUTION_FAILED: 'transaction execution fails', FEE_TOO_LOW: 'feetoolow', SOCKET_HANG_UP: 'socket hang up', ECONNRESET: 'econnreset', ECONNREFUSED: 'econnrefused', SERVER_ERROR: 'server error', BAD_GATEWAY: '502', SERVICE_UNAVAILABLE: '503', EXPECT_BLOCK_NUMBER: 'expect block number from id', }; /** * @constant {number} OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS - * operation id command cleanup interval time 24h */ export const OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; /** * @constant {number} OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS - * operation id memory cleanup interval time 1h */ export const OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS = 60 * 60 * 1000; /** * @constant {number} FINALIZED_COMMAND_CLEANUP_TIME_MILLS - Command cleanup interval time * finalized commands command cleanup interval time 24h */ export const PUBLISH_STORAGE_MEMORY_CLEANUP_COMMAND_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const PUBLISH_STORAGE_FILE_CLEANUP_COMMAND_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const FINALIZED_COMMAND_CLEANUP_TIME_MILLS = 1 * 60 * 60 * 1000; export const FINALIZED_COMMAND_CLEANUP_TIME_DELAY = 1 * 60 * 60 * 1000; export const GET_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const GET_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; export const GET_RESPONSE_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const GET_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; export const PUBLISH_CLEANUP_TIME_MILLS = 1 * 60 * 60 * 1000; export const PUBLISH_CLEANUP_TIME_DELAY = 1 * 60 * 60 * 1000; export const PUBLISH_RESPONSE_CLEANUP_TIME_MILLS = 1 * 60 * 60 * 1000; export const PUBLISH_RESPONSE_CLEANUP_TIME_DELAY = 1 * 60 * 60 * 1000; export const UPDATE_CLEANUP_TIME_MILLS = 1 * 60 * 60 * 1000; export const UPDATE_CLEANUP_TIME_DELAY = 1 * 60 * 60 * 1000; export const UPDATE_RESPONSE_CLEANUP_TIME_MILLS = 1 * 60 * 60 * 1000; export const UPDATE_RESPONSE_CLEANUP_TIME_DELAY = 1 * 60 * 60 * 1000; export const ASK_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const ASK_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; export const ASK_RESPONSE_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const ASK_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; export const FINALITY_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const FINALITY_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; export const FINALITY_RESPONSE_CLEANUP_TIME_MILLS = 12 * 60 * 60 * 1000; export const FINALITY_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; export const PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS = 1 * 60 * 60 * 1000; export const PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY = 1 * 60 * 60 * 1000; /** * @constant {number} COMMAND_STATUS - * Status for commands */ export const COMMAND_STATUS = { FAILED: 'FAILED', EXPIRED: 'EXPIRED', UNKNOWN: 'UNKNOWN', STARTED: 'STARTED', PENDING: 'PENDING', COMPLETED: 'COMPLETED', REPEATING: 'REPEATING', }; export const PENDING_STORAGE_FILES_FOR_REMOVAL_MAX_NUMBER = 100_000; export const OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER = 100; export const REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER = 50_000; export const MIGRATION_FOLDER = 'migrations'; export const PUBLISHER_NODE_SIGNATURES_FOLDER = 'publisher'; export const NETWORK_SIGNATURES_FOLDER = 'network'; /** * How many commands will run in parallel * @type {number} */ export const GENERAL_COMMAND_QUEUE_PARALLELISM = 100; export const BATCH_GET_COMMAND_QUEUE_PARALLELISM = 20; export const GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE = 50; export const GET_ASSERTION_IDS_MAX_RETRY_COUNT = 5; export const GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS = 2; export const GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID = 1; /** * @constant {object} HTTP_API_ROUTES - * HTTP API Routes with parameters */ export const HTTP_API_ROUTES = { v0: { // publish: { // method: 'post', // path: '/publish', // options: { rateLimit: true }, // }, // update: { // method: 'post', // path: '/update', // options: { rateLimit: true }, // }, query: { method: 'post', path: '/query', options: {}, }, // 'local-store': { // method: 'post', // path: '/local-store', // options: {}, // }, get: { method: 'post', path: '/get', options: { rateLimit: true }, }, result: { method: 'get', path: '/:operation/:operationId', options: {}, }, info: { method: 'get', path: '/info', options: {}, }, 'bid-suggestion': { method: 'get', path: '/bid-suggestion', options: {}, }, }, v1: { publish: { method: 'post', path: '/publish', options: { rateLimit: true }, }, finality: { method: 'get', path: '/finality', options: {}, }, // update: { // method: 'post', // path: '/update', // options: { rateLimit: true }, // }, query: { method: 'post', path: '/query', options: {}, }, get: { method: 'post', path: '/get', options: { rateLimit: true }, }, result: { method: 'get', path: '/:operation/:operationId', options: {}, }, info: { method: 'get', path: '/info', options: {}, }, ask: { method: 'post', path: '/ask', options: {}, }, 'direct-query': { method: 'post', path: '/direct-query', options: {}, }, 'local-store': { method: 'post', path: '/local-store', options: {}, }, }, }; /** * @constant {object} NETWORK_PROTOCOLS - * Network protocols */ export const NETWORK_PROTOCOLS = { STORE: ['/store/1.0.0'], // UPDATE: ['/update/1.0.0'], GET: ['/get/1.0.0'], BATCH_GET: ['/batch-get/1.0.0'], ASK: ['/ask/1.0.0'], FINALITY: ['/finality/1.0.0'], }; export const OPERATION_STATUS = { IN_PROGRESS: 'IN_PROGRESS', FAILED: 'FAILED', COMPLETED: 'COMPLETED', }; export const AGREEMENT_STATUS = { ACTIVE: 'ACTIVE', EXPIRED: 'EXPIRED', }; export const OPERATION_REQUEST_STATUS = { FAILED: 'FAILED', COMPLETED: 'COMPLETED', }; /** * Local query types * @type {{CONSTRUCT: string, SELECT: string}} */ export const QUERY_TYPES = { SELECT: 'SELECT', CONSTRUCT: 'CONSTRUCT', }; /** * Local store types * @type {{TRIPLE: string, PENDING: string}} */ export const LOCAL_STORE_TYPES = { TRIPLE: 'TRIPLE', TRIPLE_PARANET: 'TRIPLE_PARANET', }; /** * Contract names * @type {{SHARDING_TABLE: string}} */ export const CONTRACTS = { SHARDING_TABLE: 'ShardingTable', STAKING: 'Staking', PROFILE: 'Profile', HUB: 'Hub', PARAMETERS_STORAGE: 'ParametersStorage', IDENTITY_STORAGE: 'IdentityStorage', LOG2PLDSF: 'Log2PLDSF', LINEAR_SUM: 'LinearSum', PARANETS_REGISTRY: 'ParanetsRegistry', }; export const MONITORED_CONTRACT_EVENTS = { Hub: ['NewContract', 'ContractChanged', 'NewAssetStorage', 'AssetStorageChanged'], ParametersStorage: ['ParameterChanged'], KnowledgeCollectionStorage: ['KnowledgeCollectionCreated'], }; export const MONITORED_CONTRACTS = Object.keys(MONITORED_CONTRACT_EVENTS); export const MONITORED_EVENTS = Object.values(MONITORED_CONTRACT_EVENTS).flatMap( (events) => events, ); export const CONTRACT_INDEPENDENT_EVENTS = {}; export const NODE_ENVIRONMENTS = { DEVELOPMENT: 'development', TEST: 'test', DEVNET: 'devnet', TESTNET: 'testnet', MAINNET: 'mainnet', }; export const MAXIMUM_FETCH_EVENTS_FAILED_COUNT = 1000; export const CONTRACT_EVENT_FETCH_INTERVALS = { MAINNET: 10 * 1000, DEVELOPMENT: 4 * 1000, }; export const TRANSACTION_CONFIRMATIONS = 1; export const SERVICE_AGREEMENT_SOURCES = { BLOCKCHAIN: 'blockchain', EVENT: 'event', CLIENT: 'client', NODE: 'node', }; export const CACHE_DATA_TYPES = { NUMBER: 'number', ANY: 'any', }; export const PARANET_SYNC_SOURCES = { SYNC: 'sync', LOCAL_STORE: 'local_store', }; /** * CACHED_FUNCTIONS: * ContractName: { * functionName: returnType * } * @type {{IdentityStorageContract: [{name: string, type: string}], ParametersStorageContract: *}} */ export const CACHED_FUNCTIONS = { ParametersStorageContract: { r0: CACHE_DATA_TYPES.NUMBER, r1: CACHE_DATA_TYPES.NUMBER, r2: CACHE_DATA_TYPES.NUMBER, finalizationCommitsNumber: CACHE_DATA_TYPES.NUMBER, updateCommitWindowDuration: CACHE_DATA_TYPES.NUMBER, commitWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, proofWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, epochLength: CACHE_DATA_TYPES.NUMBER, minimumStake: CACHE_DATA_TYPES.ANY, maximumStake: CACHE_DATA_TYPES.ANY, minProofWindowOffsetPerc: CACHE_DATA_TYPES.NUMBER, maxProofWindowOffsetPerc: CACHE_DATA_TYPES.NUMBER, }, IdentityStorageContract: { getIdentityId: CACHE_DATA_TYPES.NUMBER, }, }; export const LOW_BID_SUGGESTION = 'low'; export const MED_BID_SUGGESTION = 'med'; export const HIGH_BID_SUGGESTION = 'high'; export const ALL_BID_SUGGESTION = 'all'; export const BID_SUGGESTION_RANGE_ENUM = [ LOW_BID_SUGGESTION, MED_BID_SUGGESTION, HIGH_BID_SUGGESTION, ALL_BID_SUGGESTION, ]; export const LOW_BID_SUGGESTION_OFFSET = 9; export const MED_BID_SUGGESTION_OFFSET = 11; export const HIGH_BID_SUGGESTION_OFFSET = 14; export const LOCAL_INSERT_FOR_ASSET_SYNC_MAX_ATTEMPTS = 5; export const LOCAL_INSERT_FOR_ASSET_SYNC_RETRY_DELAY = 1000; export const LOCAL_INSERT_FOR_CURATED_PARANET_MAX_ATTEMPTS = 5; export const LOCAL_INSERT_FOR_CURATED_PARANET_RETRY_DELAY = 1000; export const MAX_RETRIES_READ_CACHED_PUBLISH_DATA = 5; export const RETRY_DELAY_READ_CACHED_PUBLISH_DATA = 5 * 1000; export const TRIPLE_STORE_REPOSITORY = { DKG: 'dkg', DKG_HISTORIC: 'dkg-historic', }; export const TRIPLES_VISIBILITY = { PUBLIC: 'public', PRIVATE: 'private', ALL: 'all', }; export const V6_CONTENT_STORAGE_MAP = { BASE_MAINNET: '0x3bdfA81079B2bA53a25a6641608E5E1E6c464597', BASE_TESTNET: '0x9e3071Dc0730CB6dd0ce42969396D716Ea33E7e1', BASE_DEVNET: '0xBe08A25dcF2B68af88501611e5456571f50327B4', GNOSIS_MAINNET: '0xf81a8C0008DE2DCdb73366Cf78F2b178616d11DD', GNOSIS_TESTNET: '0xeA3423e02c8d231532dab1BCE5D034f3737B3638', GNOSIS_DEVNET: '0x3db64dD0Ac054610d1e2Af9Cca0fbCB1A7f4C2d8', OTP_MAINNET: '0x5cAC41237127F94c2D21dAe0b14bFeFa99880630', OTP_TESTNET: '0x1A061136Ed9f5eD69395f18961a0a535EF4B3E5f', OTP_DEVNET: '0xABd59A9aa71847F499d624c492d3903dA953d67a', }; export const PROOFING_INTERVAL = 5 * 60 * 1000; export const PROOFING_MAX_ATTEMPTS = 120; export const REORG_PROOFING_BUFFER = 60 * 1000; export const CHUNK_SIZE = 32; export const CLAIM_REWARDS_INTERVAL = 60 * 60 * 1000; export const CLAIM_REWARDS_BATCH_SIZE = 10; export const BATCH_GET_BATCH_SIZE = 2; export const BATCH_GET_UAL_MAX_LIMIT = 1000; export const SYNC_INTERVAL = 12 * 1000; export const SYNC_BATCH_GET_WAIT_TIME = 1000; export const SYNC_BATCH_GET_MAX_ATTEMPTS = 15 * 60; export const MAX_TOKEN_ID_PER_GET_PAGE = 50; export const BLAZEGRAPH_HEALTH_INTERVAL = 60 * 1000; export const MAX_COMMAND_LIFETIME = 15 * 60 * 1000; ================================================ FILE: src/controllers/http-api/base-http-api-controller.js ================================================ class BaseController { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; } returnResponse(res, status, data) { res.status(status).send(data); } } export default BaseController; ================================================ FILE: src/controllers/http-api/http-api-router.js ================================================ import stringUtil from '../../service/util/string-util.js'; import { HTTP_API_ROUTES } from '../../constants/constants.js'; class HttpApiRouter { constructor(ctx) { this.httpClientModuleManager = ctx.httpClientModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.apiRoutes = HTTP_API_ROUTES; this.apiVersions = Object.keys(this.apiRoutes); this.routers = {}; for (const version of this.apiVersions) { this.routers[version] = this.httpClientModuleManager.createRouterInstance(); const operations = Object.keys(this.apiRoutes[version]); for (const operation of operations) { const versionedController = `${stringUtil.toCamelCase( operation, )}HttpApiController${stringUtil.capitalize(version)}`; this[versionedController] = ctx[versionedController]; } } this.routers.latest = this.httpClientModuleManager.createRouterInstance(); this.jsonSchemaService = ctx.jsonSchemaService; } async initialize() { this.initializeBeforeMiddlewares(); await this.initializeVersionedListeners(); this.initializeRouters(); this.initializeAfterMiddlewares(); await this.httpClientModuleManager.listen(); } initializeBeforeMiddlewares() { const blockchainImplementations = this.blockchainModuleManager.getImplementationNames(); this.httpClientModuleManager.initializeBeforeMiddlewares(blockchainImplementations); } async initializeVersionedListeners() { const descendingOrderedVersions = this.apiVersions.sort((a, b) => b.localeCompare(a)); const mountedLatestRoutes = new Set(); for (const version of descendingOrderedVersions) { for (const [name, route] of Object.entries(this.apiRoutes[version])) { const { method, path, options } = route; const camelRouteName = stringUtil.toCamelCase(name); const controller = `${camelRouteName}HttpApiController${stringUtil.capitalize( version, )}`; const schema = `${camelRouteName}Schema`; if ( schema in this.jsonSchemaService && typeof this.jsonSchemaService[schema] === 'function' ) { // eslint-disable-next-line no-await-in-loop options.requestSchema = await this.jsonSchemaService[schema](version); } const middlewares = this.httpClientModuleManager.selectMiddlewares(options); const callback = (req, res) => { this[controller].handleRequest(req, res); }; this.routers[version][method](path, ...middlewares, callback); if (!mountedLatestRoutes.has(route.name)) { this.routers.latest[method](path, ...middlewares, callback); mountedLatestRoutes.add(route.name); } } } } initializeRouters() { for (const version of this.apiVersions) { this.httpClientModuleManager.use(`/${version}`, this.routers[version]); } this.httpClientModuleManager.use('/latest', this.routers.latest); this.httpClientModuleManager.use('/', this.routers.v0); } initializeAfterMiddlewares() { this.httpClientModuleManager.initializeAfterMiddlewares(); } } export default HttpApiRouter; ================================================ FILE: src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js ================================================ import { BigNumber } from 'ethers'; import BaseController from '../base-http-api-controller.js'; import { ONE_ETHER } from '../../../constants/constants.js'; class BidSuggestionController extends BaseController { constructor(ctx) { super(ctx); this.blockchainModuleManager = ctx.blockchainModuleManager; } async handleRequest(req, res) { try { const { blockchain, epochsNumber, assertionSize } = req.query; const promises = [ this.blockchainModuleManager.getTimeUntilNextEpoch(blockchain), this.blockchainModuleManager.getEpochLength(blockchain), this.blockchainModuleManager.getStakeWeightedAverageAsk(blockchain), ]; const [timeUntilNextEpoch, epochLength, stakeWeightedAverageAsk] = await Promise.all( promises, ); const timeUntilNextEpochScaled = BigNumber.from(timeUntilNextEpoch) .mul(ONE_ETHER) .div(BigNumber.from(epochLength)); const epochsNumberScaled = BigNumber.from(epochsNumber).mul(ONE_ETHER); const storageTime = timeUntilNextEpochScaled.add(epochsNumberScaled); const bidSuggestion = BigNumber.from(stakeWeightedAverageAsk) .mul(storageTime) .mul(BigNumber.from(assertionSize)) .div(ONE_ETHER); this.returnResponse(res, 200, { bidSuggestion: bidSuggestion.toString() }); } catch (error) { this.logger.error(`Unable to get bid suggestion. Error: ${error}`); this.returnResponse(res, 500, { code: 500, message: 'Unable to calculate bid suggestion', }); } } } export default BidSuggestionController; ================================================ FILE: src/controllers/http-api/v0/get-http-api-controller-v0.js ================================================ import { OPERATION_ID_STATUS, OPERATION_STATUS, ERROR_TYPE, TRIPLES_VISIBILITY, V6_CONTENT_STORAGE_MAP, } from '../../../constants/constants.js'; import BaseController from '../base-http-api-controller.js'; class GetController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.operationService = ctx.getService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.ualService = ctx.ualService; this.validationService = ctx.validationService; this.fileService = ctx.fileService; } async handleRequest(req, res) { const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.GET.GET_START, ); await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.GET.GET_INIT_START, ); this.returnResponse(res, 202, { operationId, }); await this.repositoryModuleManager.createOperationRecord( this.operationService.getOperationName(), operationId, OPERATION_STATUS.IN_PROGRESS, ); let tripleStoreMigrationAlreadyExecuted = false; try { tripleStoreMigrationAlreadyExecuted = (await this.fileService.readFile( '/root/ot-node/data/migrations/v8DataMigration', )) === 'MIGRATED'; } catch (e) { this.logger.warn(`No triple store migration file error: ${e}`); } let blockchain; let contract; let knowledgeCollectionId; let knowledgeAssetId; try { const { paranetUAL, includeMetadata, contentType } = req.body; let { id } = req.body; ({ blockchain, contract, knowledgeCollectionId, knowledgeAssetId } = this.ualService.resolveUAL(id)); contract = contract.toLowerCase(); id = this.ualService.deriveUAL(blockchain, contract, knowledgeCollectionId); this.logger.info(`Get for ${id} with operation id ${operationId} initiated.`); // Get assertionId - datasetRoot // const isV6Contract = Object.values(V6_CONTENT_STORAGE_MAP).some((ca) => ca.toLowerCase().includes(contract.toLowerCase()), ); const commandSequence = []; if (!tripleStoreMigrationAlreadyExecuted && isV6Contract) { commandSequence.push('getAssertionMerkleRootCommand'); } commandSequence.push('getFindShardCommand'); await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), delay: 0, data: { ual: id, includeMetadata, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, operationId, paranetUAL, isV6Contract, contentType: contentType ?? TRIPLES_VISIBILITY.ALL, isOperationV0: true, }, transactional: false, }); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.GET.GET_INIT_END, ); } catch (error) { this.logger.error(`Error while initializing get data: ${error.message}.`); await this.operationService.markOperationAsFailed( operationId, blockchain, 'Unable to get data, Failed to process input data!', ERROR_TYPE.GET.GET_ROUTE_ERROR, ); } } } export default GetController; ================================================ FILE: src/controllers/http-api/v0/info-http-api-controller-v0.js ================================================ import { createRequire } from 'module'; import BaseController from '../base-http-api-controller.js'; const require = createRequire(import.meta.url); const { version } = require('../../../../package.json'); class InfoController extends BaseController { handleRequest(_, res) { this.returnResponse(res, 200, { version, }); } } export default InfoController; ================================================ FILE: src/controllers/http-api/v0/local-store-http-api-controller-v0.js ================================================ import BaseController from '../base-http-api-controller.js'; import { OPERATION_ID_STATUS } from '../../../constants/constants.js'; class LocalStoreController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.dataService = ctx.dataService; this.fileService = ctx.fileService; } async handleRequest(req, res) { const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.LOCAL_STORE.LOCAL_STORE_INIT_START, ); this.returnResponse(res, 202, { operationId, }); await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.LOCAL_STORE.LOCAL_STORE_INIT_END, ); let assertions; const { filePath } = req.body; if (filePath) { assertions = JSON.parse(await this.fileService.readFile(filePath)); } else { assertions = req.body; } const cachedAssertions = { public: {}, private: {}, }; switch (assertions.length) { case 1: { const { assertion, assertionId } = assertions[0]; cachedAssertions.public = { assertion, assertionId }; break; } case 2: { const isFirstPublic = this.dataService.getPrivateAssertionId(assertions[0].assertion) != null; const publicAssertionData = isFirstPublic ? assertions[0] : assertions[1]; const privateAssertionData = isFirstPublic ? assertions[1] : assertions[0]; cachedAssertions.public = { assertion: publicAssertionData.assertion, assertionId: publicAssertionData.assertionId, }; cachedAssertions.private = { assertion: privateAssertionData.assertion, assertionId: privateAssertionData.assertionId, }; break; } default: throw Error('Unexpected number of assertions in local store'); } this.logger.info( `Received assertion with assertion ids: ${assertions.map( (reqObject) => reqObject.assertionId, )}. Operation id: ${operationId}`, ); await this.operationIdService.cacheOperationIdDataToMemory(operationId, cachedAssertions); await this.operationIdService.cacheOperationIdDataToFile(operationId, cachedAssertions); const commandSequence = ['localStoreCommand']; await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), delay: 0, data: { operationId, blockchain: assertions[0].blockchain, contract: assertions[0].contract, tokenId: assertions[0].tokenId, storeType: assertions[0].storeType, paranetUAL: assertions[0].paranetUAL, isOperationV0: true, }, transactional: false, }); } } export default LocalStoreController; ================================================ FILE: src/controllers/http-api/v0/publish-http-api-controller-v0.js ================================================ import BaseController from '../base-http-api-controller.js'; import { ERROR_TYPE, OPERATION_ID_STATUS, OPERATION_STATUS, LOCAL_STORE_TYPES, } from '../../../constants/constants.js'; class PublishController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationService = ctx.publishService; this.operationIdService = ctx.operationIdService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; // this is not used this.pendingStorageService = ctx.pendingStorageService; } async handleRequest(req, res) { const { assertion: dataset, assertionId: datasetRoot, blockchain, contract, tokenId, } = req.body; this.logger.info( `Received asset with dataset root: ${datasetRoot}, blockchain: ${blockchain}`, ); const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.PUBLISH.PUBLISH_START, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.PUBLISH.PUBLISH_INIT_START, ); this.returnResponse(res, 202, { operationId, }); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.PUBLISH.PUBLISH_INIT_END, ); await this.repositoryModuleManager.createOperationRecord( this.operationService.getOperationName(), operationId, OPERATION_STATUS.IN_PROGRESS, ); try { await this.operationIdService.cacheOperationIdDataToMemory(operationId, { dataset, datasetRoot, }); await this.operationIdService.cacheOperationIdDataToFile(operationId, { dataset, datasetRoot, }); await this.pendingStorageService.cacheDataset(operationId, datasetRoot, dataset); const commandSequence = ['publishFindShardCommand']; await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), delay: 0, period: 5000, retries: 3, data: { datasetRoot, blockchain, operationId, contract, tokenId, isOperationV0: true, storeType: LOCAL_STORE_TYPES.TRIPLE, }, transactional: false, }); } catch (error) { this.logger.error( `Error while initializing publish data: ${error.message}. ${error.stack}`, ); await this.operationService.markOperationAsFailed( operationId, blockchain, 'Unable to publish data, Failed to process input data!', ERROR_TYPE.PUBLISH.PUBLISH_ROUTE_ERROR, ); } } } export default PublishController; ================================================ FILE: src/controllers/http-api/v0/query-http-api-controller-v0.js ================================================ import BaseController from '../base-http-api-controller.js'; import { OPERATION_ID_STATUS, TRIPLE_STORE_REPOSITORIES } from '../../../constants/constants.js'; class QueryController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.fileService = ctx.fileService; } async handleRequest(req, res) { const { query, type: queryType, repository } = req.body; const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.QUERY.QUERY_INIT_START, ); this.returnResponse(res, 202, { operationId, }); let tripleStoreMigrationAlreadyExecuted = false; try { tripleStoreMigrationAlreadyExecuted = (await this.fileService.readFile( '/root/ot-node/data/migrations/v8DataMigration', )) === 'MIGRATED'; } catch (e) { this.logger.warn(`No triple store migration file error: ${e}`); } await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.QUERY.QUERY_INIT_END, ); await this.commandExecutor.add({ name: 'queryCommand', sequence: [], delay: 0, data: { query, queryType, repository: !tripleStoreMigrationAlreadyExecuted && repository ? [repository, TRIPLE_STORE_REPOSITORIES.DKG] : TRIPLE_STORE_REPOSITORIES.DKG, operationId, }, transactional: false, }); } } export default QueryController; ================================================ FILE: src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js ================================================ import { BID_SUGGESTION_RANGE_ENUM } from '../../../../constants/constants.js'; export default (argumentsObject) => ({ type: 'object', required: [ 'blockchain', 'epochsNumber', 'assertionSize', 'contentAssetStorageAddress', 'firstAssertionId', 'hashFunctionId', ], properties: { blockchain: { enum: argumentsObject.blockchainImplementationNames, }, epochsNumber: { type: 'number', minimum: 1, }, assertionSize: { type: 'number', minimum: 1, }, contentAssetStorageAddress: { type: 'string', minLength: 42, maxLength: 42, }, firstAssertionId: { type: 'string', minLength: 66, maxLength: 66, }, hashFunctionId: { type: 'number', minimum: 1, maximum: 1, }, proximityScoreFunctionsPairId: { type: 'number', minimum: 1, maximum: 2, }, bidSuggestionRange: { type: 'string', enum: BID_SUGGESTION_RANGE_ENUM, }, }, }); ================================================ FILE: src/controllers/http-api/v0/request-schema/get-schema-v0.js ================================================ export default () => ({ type: 'object', required: ['id'], properties: { id: { type: 'string', }, contentType: { type: 'string', }, includeMetadata: { type: 'boolean', }, hashFunctionId: { type: 'number', minimum: 1, }, paranetUAL: { type: ['string', 'null'], }, }, }); ================================================ FILE: src/controllers/http-api/v0/request-schema/local-store-schema-v0.js ================================================ import { LOCAL_STORE_TYPES } from '../../../../constants/constants.js'; export default (argumentsObject) => ({ type: ['object', 'array'], items: { oneOf: [ { type: 'object', required: ['assertionId', 'assertion'], properties: { assertionId: { type: 'string', minLength: 66, maxLength: 66, }, assertion: { type: 'array', items: { type: 'string', }, minItems: 1, }, blockchain: { enum: argumentsObject.blockchainImplementationNames, }, contract: { type: 'string', minLength: 42, maxLength: 42, }, tokenId: { type: 'number', minimum: 0, }, storeType: { enum: [LOCAL_STORE_TYPES.TRIPLE, LOCAL_STORE_TYPES.TRIPLE_PARANET], }, paranetUAL: { type: 'string', }, }, minItems: 1, maxItems: 2, }, { type: 'object', required: ['filePath'], properties: { filePath: { type: 'string', }, paranetUAL: { type: 'string', }, blockchain: { enum: argumentsObject.blockchainImplementationNames, }, contract: { type: 'string', minLength: 42, maxLength: 42, }, tokenId: { type: 'number', minimum: 0, }, storeType: { enum: [LOCAL_STORE_TYPES.TRIPLE, LOCAL_STORE_TYPES.TRIPLE_PARANET], }, }, }, ], }, }); ================================================ FILE: src/controllers/http-api/v0/request-schema/publish-schema-v0.js ================================================ export default (argumentsObject) => ({ type: 'object', required: ['assertionId', 'assertion', 'blockchain', 'contract', 'tokenId'], properties: { assertionId: { type: 'string', minLength: 66, maxLength: 66, }, assertion: { type: 'array', items: { type: 'string', }, minItems: 1, }, blockchain: { enum: argumentsObject.blockchainImplementationNames, }, contract: { type: 'string', minLength: 42, maxLength: 42, }, tokenId: { type: 'number', minimum: 0, }, hashFunctionId: { type: 'number', minimum: 1, }, }, }); ================================================ FILE: src/controllers/http-api/v0/request-schema/query-schema-v0.js ================================================ import { QUERY_TYPES } from '../../../../constants/constants.js'; export default () => ({ type: 'object', required: ['type', 'query'], properties: { type: { enum: [QUERY_TYPES.CONSTRUCT, QUERY_TYPES.SELECT], }, query: { type: 'string', }, // repository: { // type: 'string', // }, }, }); ================================================ FILE: src/controllers/http-api/v0/request-schema/update-schema-v0.js ================================================ export default (argumentsObject) => ({ type: 'object', required: ['assertionId', 'assertion', 'blockchain', 'contract', 'tokenId'], properties: { assertionId: { type: 'string', minLength: 66, maxLength: 66, }, assertion: { type: 'array', items: { type: 'string', }, minItems: 1, }, blockchain: { enum: argumentsObject.blockchainImplementationNames, }, contract: { type: 'string', minLength: 42, maxLength: 42, }, tokenId: { type: 'number', minimum: 0, }, hashFunctionId: { type: 'number', minimum: 1, }, }, }); ================================================ FILE: src/controllers/http-api/v0/result-http-api-controller-v0.js ================================================ import { OPERATION_ID_STATUS } from '../../../constants/constants.js'; import BaseController from '../base-http-api-controller.js'; class ResultController extends BaseController { constructor(ctx) { super(ctx); this.operationIdService = ctx.operationIdService; this.availableOperations = ['publish', 'get', 'query', 'local-store', 'update']; } async handleRequest(req, res) { if (!this.availableOperations.includes(req.params.operation)) { return this.returnResponse(res, 400, { code: 400, message: `Unsupported operation: ${req.params.operation}, available operations are: ${this.availableOperations}`, }); } const { operationId, operation } = req.params; if (!this.operationIdService.operationIdInRightFormat(operationId)) { return this.returnResponse(res, 400, { code: 400, message: `Operation id: ${operationId} is in wrong format`, }); } try { const handlerRecord = await this.operationIdService.getOperationIdRecord(operationId); if (handlerRecord) { const response = { status: handlerRecord.status, }; if (handlerRecord.status === OPERATION_ID_STATUS.FAILED) { response.data = JSON.parse(handlerRecord.data); } switch (operation) { case 'get': case 'publish': case 'query': case 'local-store': case 'update': if (handlerRecord.status === OPERATION_ID_STATUS.COMPLETED) { response.data = await this.operationIdService.getCachedOperationIdData( operationId, ); } break; default: break; } return this.returnResponse(res, 200, response); } return this.returnResponse(res, 400, { code: 400, message: `Handler with id: ${operationId} does not exist.`, }); } catch (e) { this.logger.error( `Error while trying to fetch ${operation} data for operation id ${operationId}. Error message: ${e.message}. ${e.stack}`, ); return this.returnResponse(res, 400, { code: 400, message: `Unexpected error at getting results: ${e.message}`, }); } } } export default ResultController; ================================================ FILE: src/controllers/http-api/v0/update-http-api-controller-v0.js ================================================ import BaseController from '../base-http-api-controller.js'; import { ERROR_TYPE, OPERATION_ID_STATUS, OPERATION_STATUS, LOCAL_STORE_TYPES, } from '../../../constants/constants.js'; class UpdateController extends BaseController { constructor(ctx) { super(ctx); this.operationService = ctx.updateService; this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.repositoryModuleManager = ctx.repositoryModuleManager; } async handleRequest(req, res) { const { assertion, assertionId, blockchain, contract, tokenId } = req.body; this.logger.info( `Received asset with assertion id: ${assertionId}, blockchain: ${blockchain}, hub contract: ${contract}, token id: ${tokenId}`, ); const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.UPDATE.UPDATE_START, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_INIT_START, ); this.returnResponse(res, 202, { operationId, }); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.UPDATE.UPDATE_INIT_END, ); await this.repositoryModuleManager.createOperationRecord( this.operationService.getOperationName(), operationId, OPERATION_STATUS.IN_PROGRESS, ); try { await this.operationIdService.cacheOperationIdData(operationId, { public: { assertion, assertionId, }, blockchain, contract, tokenId, }); const commandSequence = ['updateValidateAssetCommand', 'networkUpdateCommand']; await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), delay: 0, period: 5000, retries: 3, data: { blockchain, contract, tokenId, assertionId, operationId, storeType: LOCAL_STORE_TYPES.TRIPLE, }, transactional: false, }); } catch (error) { this.logger.error( `Error while initializing update data: ${error.message}. ${error.stack}`, ); await this.operationService.markOperationAsFailed( operationId, blockchain, 'Unable to update data, Failed to process input data!', ERROR_TYPE.UPDATE.UPDATE_ROUTE_ERROR, ); } } } export default UpdateController; ================================================ FILE: src/controllers/http-api/v1/.gitkeep ================================================ ================================================ FILE: src/controllers/http-api/v1/ask-http-api-controller-v1.js ================================================ import { OPERATION_ID_STATUS, OPERATION_STATUS, ERROR_TYPE } from '../../../constants/constants.js'; import BaseController from '../base-http-api-controller.js'; class AskController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.operationService = ctx.askService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.ualService = ctx.ualService; this.validationService = ctx.validationService; this.blockchainModuleManager = ctx.blockchainModuleManager; } async handleRequest(req, res) { const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.ASK.ASK_START, ); await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.ASK.ASK_START, ); this.returnResponse(res, 202, { operationId, }); await this.repositoryModuleManager.createOperationRecord( this.operationService.getOperationName(), operationId, OPERATION_STATUS.IN_PROGRESS, ); const { ual, blockchain, minimumNumberOfNodeReplications } = req.body; try { this.logger.info(`Ask for ${ual} with operation id ${operationId} initiated.`); const commandSequence = ['askFindShardCommand', 'networkAskCommand']; const { contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); const datasetRoot = await this.blockchainModuleManager.getKnowledgeCollectionLatestMerkleRoot( blockchain, contract, knowledgeCollectionId, ); await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), delay: 0, data: { ual, operationId, blockchain, datasetRoot, minimumNumberOfNodeReplications, }, transactional: false, }); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.ASK.ASK_END, ); } catch (error) { this.logger.error(`Error while initializing ask: ${error.message}.`); await this.operationService.markOperationAsFailed( operationId, blockchain, 'Unable to check ask, Failed to process input data!', ERROR_TYPE.ASK.ASK_ERROR, ); } } } export default AskController; ================================================ FILE: src/controllers/http-api/v1/direct-query-http-api-controller-v1.js ================================================ import BaseController from '../base-http-api-controller.js'; import { OPERATION_ID_STATUS, TRIPLE_STORE_REPOSITORIES, QUERY_TYPES, } from '../../../constants/constants.js'; class DirectQueryController extends BaseController { constructor(ctx) { super(ctx); this.config = ctx.config; this.fileService = ctx.fileService; this.dataService = ctx.dataService; this.tripleStoreService = ctx.tripleStoreService; this.paranetService = ctx.paranetService; this.ualService = ctx.ualService; this.operationIdService = ctx.operationIdService; } async handleRequest(req, res) { const { type: queryType, paranetUAL } = req.body; let { query, repository } = req.body; let data; const operationId = await this.operationIdService.generateId(); try { this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.QUERY.QUERY_INIT_START, operationId, ); if (paranetUAL) { repository = this.paranetService.getParanetRepositoryName(paranetUAL); } else { let tripleStoreMigrationAlreadyExecuted = false; try { // TODO: If triple store is migrated we should catch this and not check every time tripleStoreMigrationAlreadyExecuted = (await this.fileService.readFile( '/root/ot-node/data/migrations/v8DataMigration', )) === 'MIGRATED'; } catch (e) { this.logger.warn(`No triple store migration file error: ${e}`); } repository = !tripleStoreMigrationAlreadyExecuted && repository ? [repository, TRIPLE_STORE_REPOSITORIES.DKG] : TRIPLE_STORE_REPOSITORIES.DKG; } const pattern = /SERVICE\s+<([^>]+)>/g; const matches = query.match(pattern); if (matches?.length > 0) { for (const match of matches) { const repositoryInOriginalQuery = match.split('<')[1].split('>')[0]; const repositoryName = this.validateRepositoryName(repositoryInOriginalQuery); const federatedQueryRepositoryEndpoint = this.tripleStoreService.getRepositorySparqlEndpoint(repositoryName); query = query.replace( repositoryInOriginalQuery, federatedQueryRepositoryEndpoint, ); } } this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.QUERY.QUERY_INIT_END, operationId, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.QUERY.QUERY_START, operationId, ); switch (queryType) { case QUERY_TYPES.CONSTRUCT: { if (Array.isArray(repository)) { const [dataV6, dataV8] = await Promise.all([ this.tripleStoreService.construct( query, repository[0], this.config.modules.tripleStore.timeout.query, ), this.tripleStoreService.construct( query, repository[1], this.config.modules.tripleStore.timeout.query, ), ]); data = this.dataService.removeDuplicateObjectsFromArray([ ...dataV6, ...dataV8, ]); } else { data = await this.tripleStoreService.construct( query, repository, this.config.modules.tripleStore.timeout.query, ); } break; } case QUERY_TYPES.SELECT: { if (Array.isArray(repository)) { const [dataV6, dataV8] = await Promise.all([ this.tripleStoreService.select( query, repository[0], this.config.modules.tripleStore.timeout.query, ), this.tripleStoreService.select( query, repository[1], this.config.modules.tripleStore.timeout.query, ), ]); data = this.dataService.removeDuplicateObjectsFromArray([ ...dataV6, ...dataV8, ]); } else { data = await this.tripleStoreService.select( query, repository, this.config.modules.tripleStore.timeout.query, ); } break; } default: this.returnResponse(res, 400, `Unknown query type ${queryType}`); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.QUERY.QUERY_FAILED, operationId, ); return; } } catch (e) { this.returnResponse(res, 500, e.message); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.QUERY.QUERY_FAILED, operationId, ); return; } this.returnResponse(res, 200, { data, }); this.operationIdService.emitChangeEvent(OPERATION_ID_STATUS.QUERY.QUERY_END, operationId); } validateRepositoryName(repository) { let isParanetRepoValid = false; if (this.ualService.isUAL(repository)) { const paranetRepoName = this.paranetService.getParanetRepositoryName(repository); isParanetRepoValid = this.config.assetSync?.syncParanets.includes(repository); if (isParanetRepoValid) { return paranetRepoName; } } const isTripleStoreRepoValid = Object.values(TRIPLE_STORE_REPOSITORIES).includes(repository); if (isTripleStoreRepoValid) { return repository; } if (!isParanetRepoValid && !isTripleStoreRepoValid) { throw new Error(`Query failed! Repository with name: ${repository} doesn't exist`); } } } export default DirectQueryController; ================================================ FILE: src/controllers/http-api/v1/finality-http-api-controller-v1.js ================================================ import BaseController from '../base-http-api-controller.js'; class FinalityController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.operationService = ctx.finalityService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.ualService = ctx.ualService; this.validationService = ctx.validationService; } async handleRequest(req, res) { const { ual } = req.query; const finality = await this.repositoryModuleManager.getFinalityAcksCount(ual || ''); if (typeof finality !== 'number') return this.returnResponse(res, 400, { message: 'Asset with provided UAL was not published to this node.', }); this.returnResponse(res, 200, { finality }); } } export default FinalityController; ================================================ FILE: src/controllers/http-api/v1/get-http-api-controller-v1.js ================================================ import { OPERATION_ID_STATUS, OPERATION_STATUS, ERROR_TYPE, TRIPLES_VISIBILITY, COMMAND_PRIORITY, } from '../../../constants/constants.js'; import BaseController from '../base-http-api-controller.js'; class GetController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.operationService = ctx.getService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.ualService = ctx.ualService; this.validationService = ctx.validationService; this.fileService = ctx.fileService; this.paranetService = ctx.paranetService; this.blockchainModuleManager = ctx.blockchainModuleManager; } async handleRequest(req, res) { let operationId; let blockchain; let contract; let knowledgeCollectionId; let knowledgeAssetId; try { operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.GET.GET_START, ); await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.GET.GET_INIT_START, ); this.returnResponse(res, 202, { operationId, }); await this.repositoryModuleManager.createOperationRecord( this.operationService.getOperationName(), operationId, OPERATION_STATUS.IN_PROGRESS, ); const { paranetUAL, includeMetadata, contentType } = req.body; const ual = req.body.id; ({ blockchain, contract, knowledgeCollectionId, knowledgeAssetId } = this.ualService.resolveUAL(ual)); contract = contract.toLowerCase(); let paranetNodesAccessPolicy; this.logger.info(`Get for ${ual} with operation id ${operationId} initiated.`); if (paranetUAL) { const { contract: paranetContract, knowledgeCollectionId: paranetKnowledgeCollectionId, knowledgeAssetId: paranetKnowledgeAssetId, } = this.ualService.resolveUAL(paranetUAL); const paranetId = this.paranetService.constructParanetId( paranetContract, paranetKnowledgeCollectionId, paranetKnowledgeAssetId, ); paranetNodesAccessPolicy = await this.blockchainModuleManager.getNodesAccessPolicy( blockchain, paranetId, ); } await this.commandExecutor.add({ name: 'getCommand', sequence: [], data: { ual, includeMetadata, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, operationId, paranetUAL, paranetNodesAccessPolicy, contentType: contentType ?? TRIPLES_VISIBILITY.ALL, }, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.GET.GET_INIT_END, ); } catch (error) { this.logger.error(`Error while initializing get data: ${error.message}.`); await this.operationService.markOperationAsFailed( operationId, blockchain, 'Unable to get data, Failed to process input data!', ERROR_TYPE.GET.GET_ROUTE_ERROR, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.GET.GET_FAILED, operationId, ); } } } export default GetController; ================================================ FILE: src/controllers/http-api/v1/info-http-api-controller-v1.js ================================================ import { createRequire } from 'module'; import BaseController from '../base-http-api-controller.js'; const require = createRequire(import.meta.url); const { version } = require('../../../../package.json'); class InfoController extends BaseController { handleRequest(_, res) { this.returnResponse(res, 200, { version, }); } } export default InfoController; ================================================ FILE: src/controllers/http-api/v1/local-store-http-api-controller-v1.js ================================================ import { kcTools } from 'assertion-tools'; import BaseController from '../base-http-api-controller.js'; import { PRIVATE_HASH_SUBJECT_PREFIX, TRIPLE_STORE_REPOSITORIES, } from '../../../constants/constants.js'; class LocalStoreController extends BaseController { constructor(ctx) { super(ctx); this.validationService = ctx.validationService; this.ualService = ctx.ualService; this.tripleStoreService = ctx.tripleStoreService; this.cryptoService = ctx.cryptoService; } async handleRequest(req, res) { const { dataset, blockchain, datasetRoot, UAL } = req.body; let contract; let knowledgeCollectionId; try { ({ contract, knowledgeCollectionId } = this.ualService.resolveUAL(UAL)); const { publicKnowledgeAssetsUALs, privateKnowledgeAssetsUALs } = this.getKAUALs( dataset, UAL, ); const alreadyInserted = await this.tripleStoreService.ask(` ASK { FILTER ( ${[...publicKnowledgeAssetsUALs, ...privateKnowledgeAssetsUALs] .map((ual) => `EXISTS { GRAPH <${ual}> { ?s ?p ?o } }`) .join(' && ')} ) } `); if (alreadyInserted) { return this.returnResponse(res, 200, { status: true, }); } } catch (error) { return this.returnResponse(res, 500, { status: false, error, }); } try { const validations = [ this.validationService.validateDatasetRoot(dataset.public, datasetRoot), this.validationService.validateDatasetRootOnBlockchain( datasetRoot, blockchain, contract, knowledgeCollectionId, ), ]; if (dataset?.private?.length) { validations.push( this.validationService.validatePrivateMerkleRoot( dataset.public, dataset.private, ), ); } await Promise.all(validations); } catch (error) { return this.returnResponse(res, 200, { status: false, error, }); } try { await this.tripleStoreService.insertKnowledgeCollection( TRIPLE_STORE_REPOSITORIES.DKG, UAL, dataset, ); return this.returnResponse(res, 200, { status: true, }); } catch (error) { return this.returnResponse(res, 200, { status: false, error, }); } } getKAUALs(dataset, UAL) { const privateHashTriples = []; const filteredPublic = []; let privateKnowledgeAssetsUALs = []; // Check if already inserted dataset.public.forEach((triple) => { if (triple.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { privateHashTriples.push(triple); } else { filteredPublic.push(triple); } }); const publicKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( filteredPublic, true, ); publicKnowledgeAssetsTriplesGrouped.push( ...kcTools.groupNquadsBySubject(privateHashTriples, true), ); let publicKnowledgeAssetsUALs = publicKnowledgeAssetsTriplesGrouped.map( (_, index) => `${UAL}/${index + 1}`, ); if (dataset.private?.length) { const privateKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( dataset.private, true, ); const publicSubjectMap = publicKnowledgeAssetsTriplesGrouped.reduce( (map, group, index) => { const [publicSubject] = group[0].split(' '); map.set(publicSubject, index); return map; }, new Map(), ); for (const privateTriple of privateKnowledgeAssetsTriplesGrouped) { const [privateSubject] = privateTriple[0].split(' '); if (publicSubjectMap.has(privateSubject)) { const ualIndex = publicSubjectMap.get(privateSubject); privateKnowledgeAssetsUALs.push(publicKnowledgeAssetsUALs[ualIndex]); } else { const privateSubjectHashed = `<${PRIVATE_HASH_SUBJECT_PREFIX}${this.cryptoService.sha256( privateSubject.slice(1, -1), )}>`; if (publicSubjectMap.has(privateSubjectHashed)) { const ualIndex = publicSubjectMap.get(privateSubjectHashed); privateKnowledgeAssetsUALs.push(publicKnowledgeAssetsUALs[ualIndex]); } } } } publicKnowledgeAssetsUALs = publicKnowledgeAssetsUALs.map((ual) => `${ual}/public`); privateKnowledgeAssetsUALs = privateKnowledgeAssetsUALs.map((ual) => `${ual}/private`); return { publicKnowledgeAssetsUALs, privateKnowledgeAssetsUALs }; } } export default LocalStoreController; ================================================ FILE: src/controllers/http-api/v1/publish-http-api-controller-v1.js ================================================ import BaseController from '../base-http-api-controller.js'; import { ERROR_TYPE, OPERATION_ID_STATUS, OPERATION_STATUS, LOCAL_STORE_TYPES, COMMAND_PRIORITY, PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS, } from '../../../constants/constants.js'; class PublishController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationService = ctx.publishService; this.operationIdService = ctx.operationIdService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.pendingStorageService = ctx.pendingStorageService; this.networkModuleManager = ctx.networkModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; } async handleRequest(req, res) { const { dataset, datasetRoot, blockchain, minimumNumberOfNodeReplications } = req.body; this.logger.info( `Received asset with dataset root: ${datasetRoot}, blockchain: ${blockchain}`, ); const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.PUBLISH.PUBLISH_START, blockchain, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_INIT_START, operationId, blockchain, ); this.returnResponse(res, 202, { operationId, }); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.PUBLISH.PUBLISH_INIT_END, ); await this.repositoryModuleManager.createOperationRecord( this.operationService.getOperationName(), operationId, OPERATION_STATUS.IN_PROGRESS, ); try { await this.operationIdService.cacheOperationIdDataToMemory(operationId, { dataset, datasetRoot, }); await this.operationIdService.cacheOperationIdDataToFile(operationId, { dataset, datasetRoot, }); let effectiveMinReplications = minimumNumberOfNodeReplications; let chainMinNumber = null; try { const chainMin = await this.blockchainModuleManager.getMinimumRequiredSignatures( blockchain, ); chainMinNumber = Number(chainMin); } catch (err) { this.logger.warn( `Failed to fetch on-chain minimumRequiredSignatures for ${blockchain}: ${err.message}`, ); } const userMinNumber = Number(effectiveMinReplications); const resolvedUserMin = !Number.isNaN(userMinNumber) && userMinNumber > 0 ? userMinNumber : PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS; if (!Number.isNaN(chainMinNumber) && chainMinNumber > 0) { effectiveMinReplications = Math.max(chainMinNumber, resolvedUserMin); } else { effectiveMinReplications = resolvedUserMin; } if (effectiveMinReplications === 0) { this.logger.error( `Effective minimum replications resolved to 0 for operationId: ${operationId}, blockchain: ${blockchain}. This should never happen.`, ); } const publisherNodePeerId = this.networkModuleManager.getPeerId().toB58String(); await this.pendingStorageService.cacheDataset( operationId, datasetRoot, dataset, publisherNodePeerId, ); const commandSequence = ['publishReplicationCommand']; await this.commandExecutor.add({ name: commandSequence[0], sequence: commandSequence.slice(1), data: { datasetRoot, blockchain, operationId, storeType: LOCAL_STORE_TYPES.TRIPLE, minimumNumberOfNodeReplications: effectiveMinReplications, }, transactional: false, priority: COMMAND_PRIORITY.HIGHEST, }); } catch (error) { this.logger.error( `Error while initializing publish data: ${error.message}. ${error.stack}`, ); await this.operationService.markOperationAsFailed( operationId, blockchain, 'Unable to publish data, Failed to process input data!', ERROR_TYPE.PUBLISH.PUBLISH_ROUTE_ERROR, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_FAILED, operationId, ); } } } export default PublishController; ================================================ FILE: src/controllers/http-api/v1/query-http-api-controller-v1.js ================================================ import BaseController from '../base-http-api-controller.js'; import { OPERATION_ID_STATUS, TRIPLE_STORE_REPOSITORIES } from '../../../constants/constants.js'; class QueryController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.fileService = ctx.fileService; } async handleRequest(req, res) { const { query, type: queryType, repository, paranetUAL } = req.body; const operationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.QUERY.QUERY_INIT_START, ); this.returnResponse(res, 202, { operationId, }); let tripleStoreMigrationAlreadyExecuted = false; try { tripleStoreMigrationAlreadyExecuted = (await this.fileService.readFile( '/root/ot-node/data/migrations/v8DataMigration', )) === 'MIGRATED'; } catch (e) { this.logger.warn(`No triple store migration file error: ${e}`); } await this.operationIdService.updateOperationIdStatus( operationId, null, OPERATION_ID_STATUS.QUERY.QUERY_INIT_END, ); await this.commandExecutor.add({ name: 'queryCommand', sequence: [], delay: 0, data: { query, queryType, repository: !tripleStoreMigrationAlreadyExecuted && repository ? [repository, TRIPLE_STORE_REPOSITORIES.DKG] : TRIPLE_STORE_REPOSITORIES.DKG, operationId, paranetUAL, }, transactional: false, }); } } export default QueryController; ================================================ FILE: src/controllers/http-api/v1/request-schema/.gitkeep ================================================ ================================================ FILE: src/controllers/http-api/v1/request-schema/ask-schema-v1.js ================================================ export default (argumentsObject) => ({ type: 'object', required: ['ual', 'blockchain', 'minimumNumberOfNodeReplications'], properties: { ual: { oneOf: [{ type: 'string' }, { type: 'array', items: { type: 'string' }, minItems: 1 }], }, blockchain: { enum: argumentsObject.blockchainImplementationNames, }, minimumNumberOfNodeReplications: { type: 'number', minimum: 0, }, batchSize: { type: 'number', minimum: 1, }, }, }); ================================================ FILE: src/controllers/http-api/v1/request-schema/direct-query-schema-v1.js ================================================ import { QUERY_TYPES } from '../../../../constants/constants.js'; export default () => ({ type: 'object', required: ['type', 'query'], properties: { type: { enum: [QUERY_TYPES.CONSTRUCT, QUERY_TYPES.SELECT], }, query: { type: 'string', }, }, }); ================================================ FILE: src/controllers/http-api/v1/request-schema/finality-schema-v1.js ================================================ export default () => ({ type: 'object', required: ['ual'], properties: { ual: { type: 'string', }, }, }); ================================================ FILE: src/controllers/http-api/v1/request-schema/get-schema-v1.js ================================================ export default () => ({ type: 'object', required: ['id'], properties: { id: { type: 'string', }, contentType: { type: 'string', }, includeMetadata: { type: 'boolean', }, paranetUAL: { type: ['string', 'null'], }, }, }); ================================================ FILE: src/controllers/http-api/v1/request-schema/local-store-schema-v1.js ================================================ export default (argumentsObject) => ({ type: 'object', required: ['datasetRoot', 'dataset', 'blockchain'], properties: { datasetRoot: { type: 'string', minLength: 66, maxLength: 66, }, dataset: { type: 'object', properties: { public: { type: 'array', items: { type: 'string', }, minItems: 1, }, private: { type: 'array', items: { type: 'string', }, }, }, required: ['public'], additionalProperties: false, }, blockchain: { enum: argumentsObject.blockchainImplementationNames, }, UAL: { type: 'string', }, }, }); ================================================ FILE: src/controllers/http-api/v1/request-schema/publish-schema-v1.js ================================================ export default (argumentsObject) => ({ type: 'object', required: ['datasetRoot', 'dataset', 'blockchain'], properties: { datasetRoot: { type: 'string', minLength: 66, maxLength: 66, }, dataset: { type: 'object', properties: { public: { type: 'array', items: { type: 'string', }, minItems: 1, }, private: { type: 'array', items: { type: 'string', }, }, }, required: ['public'], additionalProperties: false, }, blockchain: { enum: argumentsObject.blockchainImplementationNames, }, }, }); ================================================ FILE: src/controllers/http-api/v1/request-schema/query-schema-v1.js ================================================ import { QUERY_TYPES } from '../../../../constants/constants.js'; export default () => ({ type: 'object', required: ['type', 'query'], properties: { type: { enum: [QUERY_TYPES.CONSTRUCT, QUERY_TYPES.SELECT], }, query: { type: 'string', }, }, }); ================================================ FILE: src/controllers/http-api/v1/result-http-api-controller-v1.js ================================================ import { NETWORK_SIGNATURES_FOLDER, OPERATION_ID_STATUS, PUBLISHER_NODE_SIGNATURES_FOLDER, } from '../../../constants/constants.js'; import BaseController from '../base-http-api-controller.js'; class ResultController extends BaseController { constructor(ctx) { super(ctx); this.operationIdService = ctx.operationIdService; this.signatureService = ctx.signatureService; this.availableOperations = ['publish', 'get', 'query', 'update', 'ask', 'finality']; } async handleRequest(req, res) { if (!this.availableOperations.includes(req.params.operation)) { return this.returnResponse(res, 400, { code: 400, message: `Unsupported operation: ${req.params.operation}, available operations are: ${this.availableOperations}`, }); } const { operationId, operation } = req.params; if (!this.operationIdService.operationIdInRightFormat(operationId)) { return this.returnResponse(res, 400, { code: 400, message: `Operation id: ${operationId} is in wrong format`, }); } try { const handlerRecord = await this.operationIdService.getOperationIdRecord(operationId); if (handlerRecord) { const response = { status: handlerRecord.status, }; if (handlerRecord.status === OPERATION_ID_STATUS.FAILED) { response.data = JSON.parse(handlerRecord.data); } switch (operation) { case 'get': case 'query': case 'finality': if (handlerRecord.status === OPERATION_ID_STATUS.COMPLETED) { response.data = await this.operationIdService.getCachedOperationIdData( operationId, ); } break; case 'publish': case 'update': { const minAcksReached = handlerRecord.minAcksReached || false; response.data = { ...response.data, minAcksReached }; const shouldIncludeSignatures = minAcksReached || handlerRecord.status === OPERATION_ID_STATUS.COMPLETED; if (shouldIncludeSignatures) { try { const publisherNodeSignature = ( await this.signatureService.getSignaturesFromStorage( PUBLISHER_NODE_SIGNATURES_FOLDER, operationId, ) )[0]; const signatures = await this.signatureService.getSignaturesFromStorage( NETWORK_SIGNATURES_FOLDER, operationId, ); response.data = { ...response.data, minAcksReached: true, publisherNodeSignature, signatures, }; } catch (e) { this.logger.warn( `Failed to read signatures for operationId ${operationId}: ${e.message}`, ); } } break; } case 'ask': response.data = await this.operationIdService.getCachedOperationIdData( operationId, ); break; default: break; } return this.returnResponse(res, 200, response); } return this.returnResponse(res, 400, { code: 400, message: `Handler with id: ${operationId} does not exist.`, }); } catch (e) { this.logger.error( `Error while trying to fetch ${operation} data for operation id ${operationId}. Error message: ${e.message}. ${e.stack}`, ); return this.returnResponse(res, 400, { code: 400, message: `Unexpected error at getting results: ${e.message}`, }); } } } export default ResultController; ================================================ FILE: src/controllers/rpc/ask-rpc-controller.js ================================================ import { NETWORK_MESSAGE_TYPES } from '../../constants/constants.js'; import BaseController from './base-rpc-controller.js'; class AskController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationService = ctx.askService; } async v1_0_0HandleRequest(message, remotePeerId, protocol) { const { operationId, messageType } = message.header; const [handleRequestCommand] = this.getCommandSequence(protocol); let commandName; switch (messageType) { case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST: commandName = handleRequestCommand; break; default: throw Error('unknown messageType'); } await this.commandExecutor.add({ name: commandName, sequence: [], delay: 0, data: { remotePeerId, operationId, protocol, ual: message.data.ual, numberOfFoundNodes: message.data.numberOfFoundNodes, blockchain: message.data.blockchain, }, transactional: false, }); } } export default AskController; ================================================ FILE: src/controllers/rpc/base-rpc-controller.js ================================================ class BaseController { constructor(ctx) { this.logger = ctx.logger; this.protocolService = ctx.protocolService; } returnResponse(res, status, data) { res.status(status).send(data); } getCommandSequence(protocol) { return this.protocolService.getReceiverCommandSequence(protocol); } } export default BaseController; ================================================ FILE: src/controllers/rpc/batch-get-rpc-controller.js ================================================ import { NETWORK_MESSAGE_TYPES } from '../../constants/constants.js'; import BaseController from './base-rpc-controller.js'; class BatchGetRpcController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationService = ctx.batchGetService; } async v1_0_0HandleRequest(message, remotePeerId, protocol) { const { operationId, messageType } = message.header; const handleRequestCommand = 'v1_0_0HandleBatchGetRequestCommand'; let commandName; switch (messageType) { case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST: commandName = handleRequestCommand; break; default: throw Error('unknown messageType'); } await this.commandExecutor.add({ name: commandName, sequence: [], delay: 0, data: { remotePeerId, operationId, protocol, uals: message.data.uals, blockchain: message.data.blockchain, tokenIds: message.data.tokenIds, includeMetadata: message.data.includeMetadata, }, transactional: false, }); } } export default BatchGetRpcController; ================================================ FILE: src/controllers/rpc/finality-rpc-controller.js ================================================ import { NETWORK_MESSAGE_TYPES } from '../../constants/constants.js'; import BaseController from './base-rpc-controller.js'; class FinalityController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationService = ctx.finalityService; } async v1_0_0HandleRequest(message, remotePeerId, protocol) { const { operationId, messageType } = message.header; const handleRequestCommands = this.getCommandSequence(protocol); let commandName; switch (messageType) { case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST: [commandName] = handleRequestCommands; break; default: throw Error('unknown messageType'); } await this.commandExecutor.add({ name: commandName, sequence: [...handleRequestCommands.slice(1)], delay: 0, data: { remotePeerId, operationId, protocol, ual: message.data.ual, blockchain: message.data.blockchain, publishOperationId: message.data.publishOperationId, }, transactional: false, }); } getCommandSequence(protocol) { // TODO: Rework this to schedule different command for update return [...this.protocolService.getReceiverCommandSequence(protocol)]; } } export default FinalityController; ================================================ FILE: src/controllers/rpc/get-rpc-controller.js ================================================ import { DEFAULT_GET_STATE, NETWORK_MESSAGE_TYPES, TRIPLE_STORE_REPOSITORY, } from '../../constants/constants.js'; import BaseController from './base-rpc-controller.js'; class GetController extends BaseController { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; this.operationService = ctx.getService; } async v1_0_0HandleRequest(message, remotePeerId, protocol) { const { operationId, messageType } = message.header; const [handleRequestCommand] = this.getCommandSequence(protocol); let commandName; switch (messageType) { case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST: commandName = handleRequestCommand; break; default: throw Error('unknown messageType'); } await this.commandExecutor.add({ name: commandName, sequence: [], data: { remotePeerId, operationId, protocol, ual: message.data.ual, blockchain: message.data.blockchain, contract: message.data.contract, knowledgeCollectionId: message.data.knowledgeCollectionId, tokenIds: message.data.tokenIds, knowledgeAssetId: message.data.knowledgeAssetId, includeMetadata: message.data.includeMetadata, state: message.data.state ?? DEFAULT_GET_STATE, paranetUAL: message.data.paranetUAL, paranetId: message.data.paranetId, migrationFlag: message.data.migrationFlag, repository: message.data.repository ?? TRIPLE_STORE_REPOSITORY.DKG, }, transactional: false, }); } } export default GetController; ================================================ FILE: src/controllers/rpc/publish-rpc-controller.js ================================================ import BaseController from './base-rpc-controller.js'; import { NETWORK_MESSAGE_TYPES, COMMAND_PRIORITY } from '../../constants/constants.js'; class PublishController extends BaseController { constructor(ctx) { super(ctx); this.operationService = ctx.publishService; this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; } async v1_0_0HandleRequest(message, remotePeerId, protocol) { const { operationId, messageType } = message.header; const command = { sequence: [], transactional: false, data: {} }; const [handleRequestCommand] = this.getCommandSequence(protocol); if (messageType === NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST) { Object.assign(command, { name: handleRequestCommand, priority: COMMAND_PRIORITY.HIGHEST, }); await this.operationIdService.cacheOperationIdDataToMemory(operationId, { dataset: message.data.dataset, datasetRoot: message.data.datasetRoot, }); await this.operationIdService.cacheOperationIdDataToFile(operationId, { dataset: message.data.dataset, datasetRoot: message.data.datasetRoot, }); } else { throw new Error('Unknown message type'); } command.data = { ...command.data, remotePeerId, operationId, protocol, dataset: message.data.dataset, datasetRoot: message.data.datasetRoot, blockchain: message.data.blockchain, isOperationV0: message.data.isOperationV0, contract: message.data.contract, tokenId: message.data.tokenId, }; await this.commandExecutor.add(command); } } export default PublishController; ================================================ FILE: src/controllers/rpc/rpc-router.js ================================================ class RpcRouter { constructor(ctx) { this.networkModuleManager = ctx.networkModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.protocolService = ctx.protocolService; this.logger = ctx.logger; this.publishRpcController = ctx.publishRpcController; this.getRpcController = ctx.getRpcController; this.updateRpcController = ctx.updateRpcController; this.askRpcController = ctx.askRpcController; this.finalityRpcController = ctx.finalityRpcController; this.batchGetRpcController = ctx.batchGetRpcController; } initialize() { this.initializeListeners(); } initializeListeners() { const protocols = this.protocolService.getProtocols().flatMap((p) => p); for (const protocol of protocols) { const version = this.protocolService.toAwilixVersion(protocol); const operation = this.protocolService.toOperation(protocol); const handleRequest = `${version}HandleRequest`; const controller = `${operation}RpcController`; const blockchainImplementations = this.blockchainModuleManager.getImplementationNames(); this.networkModuleManager.handleMessage(protocol, (message, remotePeerId) => { const modifiedMessage = this.modifyMessage(message, blockchainImplementations); this[controller][handleRequest](modifiedMessage, remotePeerId, protocol); }); } } modifyMessage(message, blockchainImplementations) { const modifiedMessage = message; if (modifiedMessage.data.blockchain?.split(':').length === 1) { for (const implementation of blockchainImplementations) { if (implementation.split(':')[0] === modifiedMessage.data.blockchain) { modifiedMessage.data.blockchain = implementation; break; } } } return modifiedMessage; } } export default RpcRouter; ================================================ FILE: src/controllers/rpc/update-rpc-controller.js ================================================ import BaseController from './base-rpc-controller.js'; import { NETWORK_MESSAGE_TYPES } from '../../constants/constants.js'; class UpdateController extends BaseController { constructor(ctx) { super(ctx); this.operationService = ctx.updateService; this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; } async v1_0_0HandleRequest(message, remotePeerId, protocol) { const { operationId, messageType } = message.header; const command = { sequence: [], delay: 0, transactional: false, data: {} }; let dataSource; const [handleInitCommand, handleRequestCommand] = this.getCommandSequence(protocol); switch (messageType) { case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_INIT: dataSource = message.data; command.name = handleInitCommand; command.period = 5000; command.retries = 3; break; case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST: // eslint-disable-next-line no-case-declarations dataSource = await this.operationIdService.getCachedOperationIdData(operationId); await this.operationIdService.cacheOperationIdData(operationId, { assertionId: dataSource.assertionId, assertion: message.data.assertion, }); command.name = handleRequestCommand; break; default: throw Error('unknown message type'); } command.data = { ...command.data, remotePeerId, operationId, protocol, assertionId: dataSource.assertionId, blockchain: dataSource.blockchain, contract: dataSource.contract, tokenId: dataSource.tokenId, }; await this.commandExecutor.add(command); } } export default UpdateController; ================================================ FILE: src/logger/logger.js ================================================ import { pino } from 'pino'; import pretty from 'pino-pretty'; /** * Class for logging messages. */ class Logger { /** * Create a new logger. * @param {string} logLevel - The log level to use for the logger. */ constructor(logLevel = 'trace', pinoInstance = null) { this.logLevel = logLevel; this._timers = new Map(); if (!pinoInstance) this.initialize(logLevel); else this.pinoLogger = pinoInstance; } /** * Initialize the logger. * @param {string} logLevel - The log level to use for the logger. */ initialize(logLevel) { try { const stream = pretty({ colorize: true, level: this.logLevel, translateTime: 'yyyy-mm-dd HH:MM:ss', ignore: 'pid,hostname,Event_name,Operation_name,Id_operation', hideObject: true, messageFormat: (log, messageKey) => { const { commandId, commandName, operationId } = log; let context = ''; if (operationId) context += `{Operation ID: ${operationId}} `; if (commandName) context += `[${commandName}] `; if (commandId) context += `(Command ID: ${commandId}) `; return `${context} ${log[messageKey]}`; }, }); this.pinoLogger = pino( { customLevels: { emit: 15, api: 25, }, level: logLevel, }, stream, ); } catch (e) { // eslint-disable-next-line no-console console.error(`Failed to create logger. Error message: ${e.message}`); } } /** * Create a child logger with the given bindings. * @param {pino.Bindings} bindings - The bindings to use for the child logger. * @return {Logger} The child logger. */ child(bindings) { return new Logger(this.logLevel, this.pinoLogger.child(bindings, {})); } /** * Restart the logger. */ restart() { this.initialize(this.logLevel, true); } // =========================== // ===== TIMERS ======== // =========================== /** * Start a timer countdown. Equivalent to console.time(label). * @param {string} label - The label to use for the timer. */ startTimer(label) { // TODO: Maybe add dedicated level just for timers? // if (this.pinoLogger.levelVal > this.pinoLogger.levels.values.trace) // return; this._timers.set(label, process.hrtime.bigint()); } /** * End a timer countdown. Should be used only in trace level, equivalent to console.timeEnd(label). * @note Requires startTimer to be called first. * @param {string} label - The label to use for the timer. */ endTimer(label) { const start = this._timers.get(label); if (!start) return; this._timers.delete(label); const diffNs = process.hrtime.bigint() - start; // TODO: Fix precision on micro/nano seconds scale // eslint-disable-next-line no-undef const diffMs = Number(diffNs / BigInt(1e6)).toFixed(2); this.pinoLogger.trace(`${label} - ${diffMs}ms`); } // =========================== // ==== LOG LEVELS ====== // =========================== /** * Log a silent message. * @param {any} obj - The object to log. */ silent(obj) { this.pinoLogger.silent(obj); } /** * Log a fatal message. * @param {any} obj - The object to log. */ fatal(obj) { this.pinoLogger.fatal(obj); } /** * Log an error message. * @param {any} obj - The object to log. */ error(obj) { this.pinoLogger.error(obj); } /** * Log a warning message. * @param {any} obj - The object to log. */ warn(obj) { this.pinoLogger.warn(obj); } /** * Log an info message. * @param {any} obj - The object to log. */ info(obj) { this.pinoLogger.info(obj); } /** * Log a debug message. * @param {any} obj - The object to log. */ debug(obj) { this.pinoLogger.debug(obj); } /** * Log an emit message. * @param {any} obj - The object to log. */ emit(obj) { // TODO: Check if confused with node.js event emitter this.pinoLogger.emit(obj); } /** * Log a trace message. * @param {any} obj - The object to log. */ trace(obj) { this.pinoLogger.trace(obj); } /** * Log an API message. * @param {any} obj - The object to log. */ api(obj) { this.pinoLogger.api(obj); } /** * Close the logger. * @param {string} closingMessage - The message to log when closing the logger. */ closeLogger(closingMessage) { const finalLogger = pino.final(this.pinoLogger); finalLogger.info(closingMessage); } } export default Logger; ================================================ FILE: src/migration/base-migration.js ================================================ import path from 'path'; import FileService from '../service/file-service.js'; class BaseMigration { constructor(migrationName, logger, config) { if (!migrationName || migrationName === '') { throw new Error('Unable to initialize base migration: name not passed in constructor.'); } if (!logger) { throw new Error( 'Unable to initialize base migration: logger object not passed in constructor.', ); } if (!config) { throw new Error( 'Unable to initialize base migration: config object not passed in constructor.', ); } this.migrationName = migrationName; this.logger = logger; this.config = config; this.fileService = new FileService({ config: this.config, logger: this.logger }); } async migrationAlreadyExecuted(migrationName = null) { const migrationFilePath = path.join( this.fileService.getMigrationFolderPath(), migrationName ?? this.migrationName, ); if (await this.fileService.pathExists(migrationFilePath)) { return true; } return false; } async migrate(migrationPath = null) { this.logger.info(`Starting ${this.migrationName} migration.`); this.startedTimestamp = Date.now(); await this.executeMigration(); const migrationFolderPath = migrationPath || this.fileService.getMigrationFolderPath(); await this.fileService.writeContentsToFile( migrationFolderPath, this.migrationName, 'MIGRATED', ); this.logger.info( `${this.migrationName} migration completed. Lasted: ${ Date.now() - this.startedTimestamp } millisecond(s).`, ); } async getMigrationInfo(migrationName = null) { const migrationFolderPath = this.fileService.getMigrationFolderPath(); const migrationInfoFileName = `${migrationName ?? this.migrationName}_info`; const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); let migrationInfo = null; if (await this.fileService.pathExists(migrationInfoPath)) { migrationInfo = await this.fileService .readFile(migrationInfoPath, true) .catch(() => {}); } return migrationInfo; } async saveMigrationInfo(migrationInfo) { const migrationFolderPath = this.fileService.getMigrationFolderPath(); const migrationInfoFileName = `${this.migrationName}_info`; await this.fileService.writeContentsToFile( migrationFolderPath, migrationInfoFileName, JSON.stringify(migrationInfo), false, ); } async executeMigration() { throw Error('Execute migration method not implemented'); } } export default BaseMigration; ================================================ FILE: src/migration/migration-executor.js ================================================ import path from 'path'; import { NODE_ENVIRONMENTS } from '../constants/constants.js'; import TripleStoreUserConfigurationMigration from './triple-store-user-configuration-migration.js'; import RedisSetupMigration from './redis-setup-migration.js'; class MigrationExecutor { static async executeTripleStoreUserConfigurationMigration(container, logger, config) { if ( process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST || process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVNET ) return; const migration = new TripleStoreUserConfigurationMigration( 'tripleStoreUserConfigurationMigrationV8', logger, config, ); if (!(await migration.migrationAlreadyExecuted())) { try { await migration.migrate(); } catch (error) { logger.error( `Unable to execute triple store user configuration migration. Error: ${error.message}`, ); } } } static async executeRedisSetupMigration(container, logger, config) { if ( process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST || process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVNET ) return; const migration = new RedisSetupMigration('redisSetupMigration', logger, config); if (!(await migration.migrationAlreadyExecuted())) { try { await migration.migrate(); } catch (error) { logger.error(`Unable to execute redis setup migration. Error: ${error.message}`); } } } static exitNode(code = 0) { process.exit(code); } static async migrationAlreadyExecuted(migrationName, fileService) { const migrationFilePath = path.join(fileService.getMigrationFolderPath(), migrationName); if (await fileService.pathExists(migrationFilePath)) { return true; } return false; } } export default MigrationExecutor; ================================================ FILE: src/migration/redis-setup-migration.js ================================================ import { execSync } from 'child_process'; import BaseMigration from './base-migration.js'; import { NODE_ENVIRONMENTS } from '../constants/constants.js'; class RedisSetupMigration extends BaseMigration { async executeMigration() { if ( process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST ) { this.logger.info('Skipping Redis setup in development/test environment'); return; } if (this.isRedisInstalledAndRunning()) { this.logger.info('✅ Redis is already installed and running.'); // Check if configuration is correct if (this.isRedisConfiguredCorrectly()) { this.logger.info('✅ Redis is configured correctly. No changes needed.'); return; } this.logger.info('⚠️ Redis is installed but configuration needs updating.'); this.updateRedisConfiguration(); } else { this.logger.info('🔧 Installing Redis...'); this.installRedis(); } this.verifyRedisInstallation(); } installRedis() { this.run('sudo apt update', 'Updating package list'); this.run('sudo apt install -y redis-server', 'Installing Redis server'); // Backup original config before modifying this.run( 'sudo cp /etc/redis/redis.conf /etc/redis/redis.conf.backup', 'Backing up original redis.conf', ); this.updateRedisConfiguration(); this.run('sudo systemctl restart redis.service', 'Restarting Redis service'); this.run('sudo systemctl enable redis.service', 'Enabling Redis to start on boot'); this.run('sudo systemctl status redis.service --no-pager', 'Checking Redis service status'); } updateRedisConfiguration() { this.logger.info('🔧 Updating Redis configuration...'); // Ensure redis.conf uses systemd this.modifyRedisConf('supervised', 'supervised systemd', 'Enabling systemd supervision'); // Enable AOF persistence this.modifyRedisConf('appendonly', 'appendonly yes', 'Enabling AOF persistence'); this.modifyRedisConf( 'appendfsync', 'appendfsync everysec', 'Setting AOF fsync every second', ); // Enforce noeviction policy this.modifyRedisConf( 'maxmemory-policy', 'maxmemory-policy noeviction', 'Setting noeviction policy', ); // Restart Redis to apply configuration changes this.run( 'sudo systemctl restart redis.service', 'Restarting Redis service to apply configuration', ); } isRedisConfiguredCorrectly() { try { const configPath = '/etc/redis/redis.conf'; const configContent = execSync(`sudo cat ${configPath}`, { stdio: 'pipe' }).toString(); const checks = [ { pattern: /supervised\s+systemd/, name: 'systemd supervision' }, { pattern: /appendonly\s+yes/, name: 'AOF persistence' }, { pattern: /appendfsync\s+everysec/, name: 'AOF fsync every second' }, { pattern: /maxmemory-policy\s+noeviction/, name: 'noeviction policy' }, ]; let allCorrect = true; for (const check of checks) { if (!check.pattern.test(configContent)) { this.logger.warn(`⚠️ Configuration issue: ${check.name} not set correctly`); allCorrect = false; } } if (allCorrect) { this.logger.info('✅ All Redis configuration checks passed'); } else { this.logger.info('⚠️ Some Redis configuration settings need to be updated'); } return allCorrect; } catch (err) { this.logger.warn(`⚠️ Could not read Redis configuration: ${err.message}`); return false; } } isRedisInstalledAndRunning() { try { execSync('which redis-server', { stdio: 'ignore' }); const serviceStatus = execSync('systemctl is-active redis.service', { stdio: 'pipe' }) .toString() .trim(); const ping = execSync('redis-cli ping', { stdio: 'pipe' }).toString().trim(); return serviceStatus === 'active' && ping === 'PONG'; } catch { return false; } } verifyRedisInstallation() { try { const ping = execSync('redis-cli ping').toString().trim(); if (ping === 'PONG') { this.logger.info('🎉 Redis is installed and responding: PONG'); // Final configuration check if (this.isRedisConfiguredCorrectly()) { this.logger.info( '🎉 Redis setup completed successfully with correct configuration!', ); } else { this.logger.error('❌ Redis is running but configuration is still incorrect'); process.exit(1); } } else { this.logger.error('❌ Redis did not respond with PONG'); process.exit(1); } } catch (err) { this.logger.error('❌ Redis ping failed'); this.logger.error(err.message); process.exit(1); } } run(cmd, description) { this.logger.info(`🔧 ${description}...`); try { const output = execSync(cmd, { stdio: 'inherit' }); return output?.toString().trim(); } catch (err) { this.logger.error(`❌ Failed: ${description}`); this.logger.error(err.message); process.exit(1); } } modifyRedisConf(pattern, replacement, description) { const configPath = '/etc/redis/redis.conf'; // Use a simpler approach: comment out lines that contain the pattern (not commented) const commentCommand = `sudo sed -i '/^[[:space:]]*${pattern}/s/^/# /' ${configPath}`; this.run(commentCommand, `Commenting out existing ${description} settings`); // Step 2: Add the new setting at the end of the file this.run( `echo '${replacement}' | sudo tee -a ${configPath}`, `Adding ${replacement} to redis.conf`, ); // Step 3: Verify the change was made try { const grepCheck = `grep -q "^${replacement}$" ${configPath}`; execSync(grepCheck, { stdio: 'ignore' }); this.logger.info(`✅ Successfully updated: ${description}`); } catch { this.logger.error(`❌ Failed to verify: ${description}`); throw new Error(`Failed to update ${description}`); } } } export default RedisSetupMigration; ================================================ FILE: src/migration/triple-store-user-configuration-migration.js ================================================ import appRootPath from 'app-root-path'; import path from 'path'; import BaseMigration from './base-migration.js'; class TripleStoreUserConfigurationMigration extends BaseMigration { async executeMigration() { const configurationFolderPath = path.join(appRootPath.path, '..'); const configurationFilePath = path.join( configurationFolderPath, this.config.configFilename, ); const userConfiguration = await this.fileService.readFile(configurationFilePath, true); if ('tripleStore' in userConfiguration.modules) { const oldConfigTripleStore = userConfiguration.modules; for (const implementation in oldConfigTripleStore.tripleStore.implementation) { if (oldConfigTripleStore.tripleStore.implementation[implementation].enabled) { const { url, username, password } = oldConfigTripleStore.tripleStore.implementation[implementation].config .repositories.publicCurrent; oldConfigTripleStore.tripleStore.implementation[ implementation ].config.repositories.dkg = { url, name: 'dkg', username, password, }; } } await this.fileService.writeContentsToFile( configurationFolderPath, this.config.configFilename, JSON.stringify(userConfiguration, null, 4), ); } } } export default TripleStoreUserConfigurationMigration; ================================================ FILE: src/modules/auto-updater/auto-updater-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class AutoUpdaterModuleManager extends BaseModuleManager { getName() { return 'autoUpdater'; } /** * @typedef VersionResults * @param {Boolean} UpToDate - If the local version is the same as the remote version. * @param {String} currentVersion - The version of the local application. * @param {String} remoteVersion - The version of the application in the git repository. * * Checks the local version of the application against the remote repository. * @returns Promise{VersionResults} - An object with the results of the version comparison. */ async compareVersions() { if (this.initialized) { return this.getImplementation().module.compareVersions(); } } /** * Clones the git repository and installs the update over the local application. * A backup of the application is created before the update is installed. * If configured, a completion command will be executed and the process for the app will be stopped. */ async update() { if (this.initialized) { return this.getImplementation().module.update(); } } } export default AutoUpdaterModuleManager; ================================================ FILE: src/modules/auto-updater/implementation/ot-auto-updater.js ================================================ import path from 'path'; import fs from 'fs-extra'; import { exec } from 'child_process'; import https from 'https'; import appRootPath from 'app-root-path'; import semver from 'semver'; import axios from 'axios'; import unzipper from 'unzipper'; const REPOSITORY_URL = 'https://github.com/OriginTrail/dkg-engine'; const ARCHIVE_REPOSITORY_URL = 'github.com/OriginTrail/dkg-engine/archive/'; class OTAutoUpdater { async initialize(config, logger) { this.config = config; this.logger = logger; if (!this.config) throw Error('You must pass a config object to AutoUpdater.'); if (!this.config.branch) this.config.branch = 'master'; } async compareVersions() { try { this.logger.debug('AutoUpdater - Comparing versions...'); const currentVersion = await this.readAppVersion(appRootPath.path); const remoteVersion = await this.readRemoteVersion(); this.logger.debug(`AutoUpdater - Current version: ${currentVersion}`); this.logger.debug(`AutoUpdater - Remote Version: ${remoteVersion}`); if (currentVersion === remoteVersion) { return { upToDate: true, currentVersion, }; } return { upToDate: false, currentVersion, remoteVersion, }; } catch (e) { this.logger.error( `AutoUpdater - Error comparing local and remote versions. Error message: ${e.message}`, ); return { upToDate: false, currentVersion: 'Error', remoteVersion: 'Error', }; } } async update() { try { this.logger.debug(`AutoUpdater - Updating dkg-engine from ${REPOSITORY_URL}`); const currentDirectory = appRootPath.path; const rootPath = path.join(currentDirectory, '..'); const currentVersion = await this.readAppVersion(currentDirectory); const newVersion = await this.readRemoteVersion(); const updateDirectory = path.join(rootPath, newVersion); const zipArchiveDestination = `${updateDirectory}.zip`; const tmpExtractionPath = path.join(rootPath, 'TmpExtractionPath'); await this.downloadUpdate(zipArchiveDestination); await this.unzipFile(tmpExtractionPath, zipArchiveDestination); await this.moveAndCleanExtractedData(tmpExtractionPath, updateDirectory); await this.copyConfigFiles(currentDirectory, updateDirectory); await this.installDependencies(updateDirectory); const currentSymlinkFolder = path.join(rootPath, 'current'); if (await fs.pathExists(currentSymlinkFolder)) { await fs.remove(currentSymlinkFolder); } await fs.ensureSymlink(updateDirectory, currentSymlinkFolder); this.logger.debug('AutoUpdater - Finished installing updated version.'); await this.removeOldVersions(currentVersion, newVersion); return true; } catch (e) { this.logger.error(`AutoUpdater - Error updating application. Error message: ${e}`); return false; } } async removeOldVersions(currentVersion, newVersion) { try { const rootPath = path.join(appRootPath.path, '..'); const oldVersionsDirs = (await fs.promises.readdir(rootPath, { withFileTypes: true })) .filter((dirent) => dirent.isDirectory()) .map((dirent) => dirent.name) .filter( (name) => semver.valid(name) && name !== newVersion && name !== currentVersion, ); const deletePromises = oldVersionsDirs .map((dirName) => path.join(rootPath, dirName)) .map((fullPath) => fs.promises.rm(fullPath, { recursive: true, force: true })); await Promise.all(deletePromises); } catch (e) { throw Error('AutoUpdater - There was an error removing old versions'); } } /** * Copies user config files to destination directory */ async copyConfigFiles(source, destination) { this.logger.debug('AutoUpdater - Copying config files...'); this.logger.debug(`AutoUpdater - Destination: ${destination}`); await fs.ensureDir(destination); const envFilePath = path.join(source, '.env'); const newEnvFilePath = path.join(destination, '.env'); await fs.copy(envFilePath, newEnvFilePath); } /** * Reads the applications version from the package.json file. */ async readAppVersion(appPath) { const file = path.join(appPath, 'package.json'); this.logger.debug(`AutoUpdater - Reading app version from ${file}`); const appPackage = await fs.promises.readFile(file); return JSON.parse(appPackage).version; } /** * A promise wrapper for sending a get https requests. * @param {String} url - The Https address to request. * @param {String} options - The request options. */ promiseHttpsRequest(url, options) { return new Promise((resolve, reject) => { const req = https.request(url, options, (res) => { let body = ''; res.on('data', (data) => { body += data; }); res.on('end', () => { if (res.statusCode === 200) return resolve(body); this.logger.warn(`AutoUpdater - Bad Response ${res.statusCode}`); reject(res.statusCode); }); }); this.logger.debug(`AutoUpdater - Sending request to ${url}`); req.on('error', reject); req.end(); }); } /** * Reads the applications version from the git repository. */ async readRemoteVersion() { const options = {}; let url = `${REPOSITORY_URL}/${this.config.branch}/package.json`; if (url.includes('github')) url = url.replace('github.com', 'raw.githubusercontent.com'); this.logger.debug(`AutoUpdater - Reading remote version from ${url}`); try { const body = await this.promiseHttpsRequest(url, options); const remotePackage = JSON.parse(body); const { version } = remotePackage; return version; } catch (e) { throw Error( `This repository requires a token or does not exist. Error message: ${e.message}`, ); } } downloadUpdate(destination) { return new Promise((resolve, reject) => { const url = `https://${path.join(ARCHIVE_REPOSITORY_URL, this.config.branch)}.zip`; this.logger.debug(`AutoUpdater - Downloading dkg-engine .zip file from url: ${url}`); axios({ method: 'get', url, responseType: 'stream' }) .then((response) => { const fileStream = fs.createWriteStream(destination); response.data.pipe(fileStream); fileStream.on('finish', () => { fileStream.close(); // close() is async, call cb after close completes. resolve(); }); fileStream.on('error', (err) => { // Handle errors fs.unlinkSync(destination); reject(err); }); }) .catch((error) => { reject( Error( `AutoUpdater - Unable to download new version of dkg-engine. Error: ${error.message}`, ), ); }); }); } unzipFile(destination, source) { this.logger.debug(`AutoUpdater - Unzipping dkg-engine new version archive`); return new Promise((resolve, reject) => { const fileReadStream = fs .createReadStream(source) .pipe(unzipper.Extract({ path: destination })); fileReadStream.on('close', () => { this.logger.debug(`AutoUpdater - Unzip completed`); fs.removeSync(source); resolve(); }); fileReadStream.on('error', (err) => { reject(err); }); }); } async moveAndCleanExtractedData(extractedDataPath, destinationPath) { this.logger.debug(`AutoUpdater - Cleaning update destination directory`); const destinationDirFiles = await fs.readdir(extractedDataPath); if (destinationDirFiles.length !== 1) { await fs.remove(extractedDataPath); throw Error('Extracted archive for new dkg-engine version is not valid'); } const sourcePath = path.join(extractedDataPath, destinationDirFiles[0]); await fs.remove(destinationPath); await fs.move(sourcePath, destinationPath); await fs.remove(extractedDataPath); } /** * Runs npm install to update/install the application dependencies. */ installDependencies(destination) { return new Promise((resolve, reject) => { this.logger.debug( `AutoUpdater - Installing application dependencies in ${destination}`, ); const command = `cd ${destination} && npm ci --omit=dev --ignore-scripts`; const child = exec(command); let rejected = false; child.stdout.on('data', (data) => { this.logger.trace(`AutoUpdater - npm ci - ${data.replace(/\r?\n|\r/g, '')}`); }); child.stderr.on('data', (data) => { if (data.includes('ERROR')) { this.logger.trace(`Error message: ${data}`); // npm passes warnings as errors, only reject if "error" is included const errorData = data.replace(/\r?\n|\r/g, ''); this.logger.error( `AutoUpdater - Error installing dependencies. Error message: ${errorData}`, ); if (!rejected) { rejected = true; reject(errorData); } } }); child.stdout.on('end', () => { if (!rejected) { this.logger.debug(`AutoUpdater - Dependencies installed successfully`); resolve(); } }); }); } } export default OTAutoUpdater; ================================================ FILE: src/modules/base-module-manager.js ================================================ import ModuleConfigValidation from './module-config-validation.js'; import { REQUIRED_MODULES } from '../constants/constants.js'; class BaseModuleManager { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; this.moduleConfigValidation = new ModuleConfigValidation(ctx); } async initialize() { try { const moduleConfig = this.config.modules[this.getName()]; this.moduleConfigValidation.validateModule(this.getName(), moduleConfig); this.handlers = {}; for (const implementationName in moduleConfig.implementation) { if (!moduleConfig.implementation[implementationName].enabled) { // eslint-disable-next-line no-continue continue; } const implementationConfig = moduleConfig.implementation[implementationName]; if (!implementationConfig) { this.logger.warn( `${implementationName} module implementation configuration not defined.`, ); return false; } if (!implementationConfig.package) { this.logger.warn(`Package for ${this.getName()} module is not defined`); return false; } implementationConfig.config.appDataPath = this.config.appDataPath; // eslint-disable-next-line no-await-in-loop const ModuleClass = (await import(implementationConfig.package)).default; const module = new ModuleClass(); // eslint-disable-next-line no-await-in-loop await module.initialize(implementationConfig.config, this.logger); module.getImplementationName = () => implementationName; this.logger.info( `${this.getName()} module initialized with implementation: ${implementationName}`, ); this.handlers[implementationName] = { module, config: implementationConfig.config, }; } this.initialized = true; return true; } catch (error) { if (REQUIRED_MODULES.includes(this.getName())) { throw new Error( `${this.getName()} module is required but got error during initialization - ${ error.message }`, ); } this.logger.error(error.message); return false; } } getName() { throw new Error('Get name method not implemented in child class of base module interface.'); } getImplementation(name = null) { const keys = Object.keys(this.handlers); if (keys.length === 1 || !name) { return this.handlers[keys[0]]; } return this.handlers[name]; } getImplementationNames() { return Object.keys(this.handlers); } removeImplementation(name = null) { const keys = Object.keys(this.handlers); if (keys.length === 1 || !name) { delete this.handlers[keys[0]]; } delete this.handlers[name]; } getModuleConfiguration(name = null) { return this.getImplementation(name).config; } } export default BaseModuleManager; ================================================ FILE: src/modules/blockchain/blockchain-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class BlockchainModuleManager extends BaseModuleManager { getName() { return 'blockchain'; } callImplementationFunction(blockchain, functionName, args = []) { if (this.getImplementation(blockchain)) { return this.getImplementation(blockchain).module[functionName](...args); } } initializeTransactionQueues(blockchain, concurrency) { return this.callImplementationFunction(blockchain, 'getTotalTransactionQueueLength', [ concurrency, ]); } getTotalTransactionQueueLength(blockchain) { return this.callImplementationFunction(blockchain, 'getTotalTransactionQueueLength'); } async initializeContracts(blockchain) { return this.callImplementationFunction(blockchain, 'initializeContracts'); } initializeAssetStorageContract(blockchain, contractAddress) { return this.callImplementationFunction(blockchain, 'initializeAssetStorageContract', [ contractAddress, ]); } initializeContract(blockchain, contractName, contractAddress) { return this.callImplementationFunction(blockchain, 'initializeContract', [ contractName, contractAddress, ]); } getContractAddress(blockchain, contractName) { return this.callImplementationFunction(blockchain, 'getContractAddress', [contractName]); } setContractCallCache(blockchain, contractName, functionName, value) { return this.callImplementationFunction(blockchain, 'setContractCallCache', [ contractName, functionName, value, ]); } getPublicKeys(blockchain) { return this.callImplementationFunction(blockchain, 'getPublicKeys'); } getManagementKey(blockchain) { return this.callImplementationFunction(blockchain, 'getManagementKey'); } async isAssetStorageContract(blockchain, contractAddress) { return this.callImplementationFunction(blockchain, 'isAssetStorageContract', [ contractAddress, ]); } async getBlockNumber(blockchain) { return this.callImplementationFunction(blockchain, 'getBlockNumber'); } async getIdentityId(blockchain) { return this.callImplementationFunction(blockchain, 'getIdentityId'); } async identityIdExists(blockchain) { return this.callImplementationFunction(blockchain, 'identityIdExists'); } async createProfile(blockchain, peerId) { return this.callImplementationFunction(blockchain, 'createProfile', [peerId]); } async getGasPrice(blockchain) { return this.callImplementationFunction(blockchain, 'getGasPrice'); } async healthCheck(blockchain) { return this.callImplementationFunction(blockchain, 'healthCheck'); } async restartService(blockchain) { return this.callImplementationFunction(blockchain, 'restartService'); } async getKnowledgeCollectionMerkleRootByIndex( blockchain, assetStorageContractAddress, knowledgeCollectionId, index, ) { return this.callImplementationFunction(blockchain, 'getCollectionMerkleRootByIndex', [ assetStorageContractAddress, knowledgeCollectionId, index, ]); } async getKnowledgeCollectionLatestMerkleRoot( blockchain, assetStorageContractAddress, knowledgeCollectionId, ) { return this.callImplementationFunction( blockchain, 'getKnowledgeCollectionLatestMerkleRoot', [assetStorageContractAddress, knowledgeCollectionId], ); } async getLatestKnowledgeCollectionId(blockchain, assetStorageContractAddress) { return this.callImplementationFunction(blockchain, 'getLatestKnowledgeCollectionId', [ assetStorageContractAddress, ]); } getAssetStorageContractAddresses(blockchain) { return this.callImplementationFunction(blockchain, 'getAssetStorageContractAddresses'); } async getKnowledgeCollectionMerkleRoots( blockchain, assetStorageContractAddress, knowledgeCollectionId, ) { return this.callImplementationFunction(blockchain, 'getKnowledgeCollectionMerkleRoots', [ assetStorageContractAddress, knowledgeCollectionId, ]); } // async getKnowledgeAssetOwner(blockchain, assetContractAddress, tokenId) { // return this.callImplementationFunction(blockchain, 'getKnowledgeAssetOwner', [ // assetContractAddress, // tokenId, // ]); // } async getLatestMerkleRootPublisher( blockchain, assetStorageContractAddress, knowledgeCollectionId, ) { return this.callImplementationFunction(blockchain, 'getLatestMerkleRootPublisher', [ assetStorageContractAddress, knowledgeCollectionId, ]); } async getShardingTableHead(blockchain) { return this.callImplementationFunction(blockchain, 'getShardingTableHead'); } async getShardingTableLength(blockchain) { return this.callImplementationFunction(blockchain, 'getShardingTableLength'); } async getShardingTablePage(blockchain, startingIdentityId, nodesNum) { return this.callImplementationFunction(blockchain, 'getShardingTablePage', [ startingIdentityId, nodesNum, ]); } async getKnowledgeCollectionSize(blockchain, knowledgeCollectionId) { return this.callImplementationFunction(blockchain, 'getKnowledgeCollectionSize', [ knowledgeCollectionId, ]); } async getKnowledgeAssetsRange(blockchain, assetStorageContractAddress, knowledgeCollectionId) { return this.callImplementationFunction(blockchain, 'getKnowledgeAssetsRange', [ assetStorageContractAddress, knowledgeCollectionId, ]); } async getParanetKnowledgeCollectionCount(blockchain, paranetId) { return this.callImplementationFunction(blockchain, 'getParanetKnowledgeCollectionCount', [ paranetId, ]); } async getParanetKnowledgeCollectionLocatorsWithPagination( blockchain, paranetId, offset, limit, ) { return this.callImplementationFunction( blockchain, 'getParanetKnowledgeCollectionLocatorsWithPagination', [paranetId, offset, limit], ); } async getMinimumStake(blockchain) { return this.callImplementationFunction(blockchain, 'getMinimumStake'); } async getMaximumStake(blockchain) { return this.callImplementationFunction(blockchain, 'getMaximumStake'); } async getMinimumRequiredSignatures(blockchain) { return this.callImplementationFunction(blockchain, 'getMinimumRequiredSignatures'); } async getLatestBlock(blockchain) { return this.callImplementationFunction(blockchain, 'getLatestBlock'); } async getBlockchainTimestamp(blockchain) { return this.callImplementationFunction(blockchain, 'getBlockchainTimestamp'); } async getParanetMetadata(blockchain, paranetId) { return this.callImplementationFunction(blockchain, 'getParanetMetadata', [paranetId]); } async getParanetName(blockchain, paranetId) { return this.callImplementationFunction(blockchain, 'getParanetName', [paranetId]); } async getDescription(blockchain, paranetId) { return this.callImplementationFunction(blockchain, 'getDescription', [paranetId]); } async paranetExists(blockchain, paranetId) { return this.callImplementationFunction(blockchain, 'paranetExists', [paranetId]); } async isPermissionedNode(blockchain, paranetId, identityId) { return this.callImplementationFunction(blockchain, 'isPermissionedNode', [ paranetId, identityId, ]); } async getNodesAccessPolicy(blockchain, paranetId) { return this.callImplementationFunction(blockchain, 'getNodesAccessPolicy', [paranetId]); } async getPermissionedNodes(blockchain, paranetId) { return this.callImplementationFunction(blockchain, 'getPermissionedNodes', [paranetId]); } async getNodeId(blockchain, identityId) { return this.callImplementationFunction(blockchain, 'getNodeId', [identityId]); } async signMessage(blockchain, messageHash) { return this.callImplementationFunction(blockchain, 'signMessage', [messageHash]); } async getStakeWeightedAverageAsk(blockchain) { return this.callImplementationFunction(blockchain, 'getStakeWeightedAverageAsk', []); } async getTimeUntilNextEpoch(blockchain) { return this.callImplementationFunction(blockchain, 'getTimeUntilNextEpoch', []); } async getEpochLength(blockchain) { return this.callImplementationFunction(blockchain, 'getEpochLength', []); } async isKnowledgeCollectionRegistered(blockchain, paranetId, knowledgeCollectionId) { return this.callImplementationFunction(blockchain, 'isKnowledgeCollectionRegistered', [ paranetId, knowledgeCollectionId, ]); } async getActiveProofPeriodStatus(blockchain) { return this.callImplementationFunction(blockchain, 'getActiveProofPeriodStatus'); } async createChallenge(blockchain) { return this.callImplementationFunction(blockchain, 'createChallenge', []); } async getNodeChallenge(blockchain, nodeId) { return this.callImplementationFunction(blockchain, 'getNodeChallenge', [nodeId]); } async submitProof(blockchain, chunk, merkleProof) { return this.callImplementationFunction(blockchain, 'submitProof', [chunk, merkleProof]); } async getNodeEpochProofPeriodScore(blockchain, nodeId, epoch, proofPeriodStartBlock) { return this.callImplementationFunction(blockchain, 'getNodeEpochProofPeriodScore', [ nodeId, epoch, proofPeriodStartBlock, ]); } async getTransaction(blockchain, txHash) { return this.callImplementationFunction(blockchain, 'getTransaction', [txHash]); } async getBlockTimestamp(blockchain, blockNumber) { return this.callImplementationFunction(blockchain, 'getBlockTimestamp', [blockNumber]); } async getDelegators(blockchain, identityId) { return this.callImplementationFunction(blockchain, 'getDelegators', [identityId]); } async getLastClaimedEpoch(blockchain, identityId, address) { return this.callImplementationFunction(blockchain, 'getLastClaimedEpoch', [ identityId, address, ]); } async hasEverDelegated(blockchain, identityId, address) { return this.callImplementationFunction(blockchain, 'hasEverDelegated', [ identityId, address, ]); } async getCurrentEpoch(blockchain) { return this.callImplementationFunction(blockchain, 'getCurrentEpoch', []); } async batchClaimDelegatorRewards(blockchain, identityId, epochs, delegators) { return this.callImplementationFunction(blockchain, 'batchClaimDelegatorRewards', [ identityId, epochs, delegators, ]); } async getAssetStorageContractsAddress(blockchain) { return this.callImplementationFunction(blockchain, 'getAssetStorageContractsAddress'); } // SUPPORT FOR OLD CONTRACTS async getLatestAssertionId(blockchain, assetContractAddress, tokenId) { return this.callImplementationFunction(blockchain, 'getLatestAssertionId', [ assetContractAddress, tokenId, ]); } getImplementation(name = null) { const keys = Object.keys(this.handlers); if (!keys.includes(name)) { throw new Error(`Blockchain: ${name} implementation is not enabled.`); } return this.handlers[name]; } } export default BlockchainModuleManager; ================================================ FILE: src/modules/blockchain/implementation/base/base-service.js ================================================ import Web3Service from '../web3-service.js'; class BaseService extends Web3Service { constructor(ctx) { super(ctx); this.baseTokenTicker = 'ETH'; this.tracTicker = 'TRAC'; } async getGasPrice() { return this.provider.getGasPrice(); } } export default BaseService; ================================================ FILE: src/modules/blockchain/implementation/eth/eth-service.js ================================================ import Web3Service from '../web3-service.js'; class EthService extends Web3Service { constructor(ctx) { super(ctx); this.baseTokenTicker = 'ETH'; this.tracTicker = 'TRAC'; } } export default EthService; ================================================ FILE: src/modules/blockchain/implementation/gnosis/gnosis-service.js ================================================ import axios from 'axios'; import ethers from 'ethers'; import Web3Service from '../web3-service.js'; import { GNOSIS_DEFAULT_GAS_PRICE, NODE_ENVIRONMENTS } from '../../../../constants/constants.js'; class GnosisService extends Web3Service { constructor(ctx) { super(ctx); this.baseTokenTicker = 'GNO'; this.tracTicker = 'TRAC'; this.defaultGasPrice = ethers.utils.parseUnits( process.env.NODE_ENV === NODE_ENVIRONMENTS.MAINNET ? GNOSIS_DEFAULT_GAS_PRICE.MAINNET.toString() : GNOSIS_DEFAULT_GAS_PRICE.TESTNET.toString(), 'gwei', ); } async getGasPrice() { let gasPrice; try { const response = await axios.get(this.config.gasPriceOracleLink); if (response?.data?.average) { // returns gwei gasPrice = Number(response.data.average); this.logger.debug(`Gas price from Gnosis oracle link: ${gasPrice} gwei`); gasPrice = ethers.utils.parseUnits(gasPrice.toString(), 'gwei'); } else if (response?.data?.result) { // returns wei gasPrice = Number(response.data.result, 10); this.logger.debug(`Gas price from Gnosis oracle link: ${gasPrice} wei`); } else { this.logger.warn( `Gas price oracle: ${this.config.gasPriceOracleLink} returns gas price in unsupported format. Using default value: ${this.defaultGasPrice} Gwei.`, ); return this.defaultGasPrice; } } catch (error) { this.logger.warn( `Failed to fetch the gas price from the Gnosis: ${error}. Using default value: ${this.defaultGasPrice} Gwei.`, ); return this.defaultGasPrice; } if (gasPrice && gasPrice.gt && gasPrice.gt(this.defaultGasPrice)) { return gasPrice; } return this.defaultGasPrice; } buildTransactionGasParams(gasPrice) { const minPriorityFee = ethers.BigNumber.from(2_000_000_000); let maxPriorityFeePerGas = minPriorityFee; if (ethers.BigNumber.isBigNumber(gasPrice)) { const derived = gasPrice.div(5); if (derived.gt(minPriorityFee)) { maxPriorityFeePerGas = derived; } } const maxFeePerGas = ethers.BigNumber.isBigNumber(gasPrice) && gasPrice.gt(maxPriorityFeePerGas) ? gasPrice : maxPriorityFeePerGas.mul(2); return { maxFeePerGas, maxPriorityFeePerGas }; } async healthCheck() { try { const blockNumber = await this.getBlockNumber(); if (blockNumber) return true; } catch (e) { this.logger.error(`Error on checking Gnosis blockchain. ${e}`); return false; } return false; } } export default GnosisService; ================================================ FILE: src/modules/blockchain/implementation/hardhat/hardhat-service.js ================================================ import ethers from 'ethers'; import Web3Service from '../web3-service.js'; class HardhatService extends Web3Service { constructor(ctx) { super(ctx); this.baseTokenTicker = 'HARDHAT_TOKENS'; this.tracTicker = 'gTRAC'; } async getBlockchainTimestamp() { const latestBlock = await super.getLatestBlock(); return latestBlock.timestamp; } async providerReady() { return this.provider.ready; } async getGasPrice() { return ethers.utils.parseUnits('20', 'wei'); } } export default HardhatService; ================================================ FILE: src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js ================================================ import { ApiPromise, WsProvider, HttpProvider } from '@polkadot/api'; import { ethers } from 'ethers'; import { NEURO_DEFAULT_GAS_PRICE, NODE_ENVIRONMENTS } from '../../../../constants/constants.js'; import Web3Service from '../web3-service.js'; const NATIVE_TOKEN_DECIMALS = 12; class OtParachainService extends Web3Service { constructor(ctx) { super(ctx); this.baseTokenTicker = 'OTP'; this.tracTicker = 'TRAC'; } async initialize(config, logger) { this.config = config; this.logger = logger; this.rpcNumber = 0; await this.initializeParachainProvider(); // await this.checkEvmWallets(); await this.parachainProvider.disconnect(); await super.initialize(config, logger); } async checkEvmWallets() { this.invalidWallets = []; for (const wallet of this.config.operationalWallets) { // eslint-disable-next-line no-await-in-loop const walletMapped = await this.checkEvmAccountMapping(wallet.evmAddress); if (!walletMapped) { this.invalidWallets.push(wallet); } } if (this.invalidWallets.length === this.config.operationalWallets.length) { throw Error('Unable to find mappings for all operational wallets'); } this.invalidWallets.forEach((wallet) => this.logger.warn( `Unable to find account mapping for wallet: ${wallet.evmAddress}, wallet removed from the list`, ), ); const { evmManagementWalletPublicKey } = this.config; const managementWalletMapped = await this.checkEvmAccountMapping( evmManagementWalletPublicKey, ); if (!managementWalletMapped) { throw Error('Missing account mapping for management wallet'); } } async checkEvmAccountMapping(walletPublicKey) { const account = await this.queryParachainState('evmAccounts', 'accounts', [ walletPublicKey, ]); if (!account || account.toHex() === '0x') { return false; } return true; } async callParachainExtrinsic(keyring, extrinsic, method, args) { let result; while (!result) { try { // eslint-disable-next-line no-await-in-loop result = this.parachainProvider.tx[extrinsic][method](...args).signAndSend(keyring); return result; } catch (error) { // eslint-disable-next-line no-await-in-loop await this.handleParachainError(error, method); } } } async queryParachainState(state, method, args) { let result; while (!result) { try { // eslint-disable-next-line no-await-in-loop result = await this.parachainProvider.query[state][method](...args); return result; } catch (error) { // eslint-disable-next-line no-await-in-loop await this.handleParachainError(error, method); } } } async initializeParachainProvider() { let tries = 0; let isRpcConnected = false; while (!isRpcConnected) { if (tries >= this.config.rpcEndpoints.length) { throw Error( 'Blockchain initialisation failed, unable to initialize parachain provider!', ); } try { let provider; if (this.config.rpcEndpoints[this.rpcNumber].startsWith('ws')) { provider = new WsProvider(this.config.rpcEndpoints[this.rpcNumber]); } else { provider = new HttpProvider(this.config.rpcEndpoints[this.rpcNumber]); } // eslint-disable-next-line no-await-in-loop this.parachainProvider = await new ApiPromise({ provider }).isReadyOrError; isRpcConnected = true; } catch (e) { this.logger.warn( `Unable to create parachain provider for endpoint : ${ this.config.rpcEndpoints[this.rpcNumber] }. Error: ${e.message}`, ); tries += 1; this.rpcNumber = (this.rpcNumber + 1) % this.config.rpcEndpoints.length; } } } async getGasPrice() { if (this.config.gasPriceOracleLink) return super.getGasPrice(); try { return this.provider.getGasPrice(); } catch (error) { const defaultGasPrice = process.env.NODE_ENV === NODE_ENVIRONMENTS.MAINNET ? NEURO_DEFAULT_GAS_PRICE.MAINNET : NEURO_DEFAULT_GAS_PRICE.TESTNET; return ethers.utils.parseUnits(defaultGasPrice.toString(), 'wei'); } } async handleParachainError(error, method) { let isRpcError = false; try { await this.parachainProvider.rpc.net.listening(); } catch (rpcError) { isRpcError = true; this.logger.warn( `Unable to execute substrate method ${method} using blockchain rpc : ${ this.config.rpcEndpoints[this.rpcNumber] }.`, ); await this.restartParachainProvider(); } if (!isRpcError) throw error; } async getLatestTokenId(assetContractAddress) { return this.provider.getStorageAt(assetContractAddress.toString().toLowerCase(), 7); } async restartParachainProvider() { this.rpcNumber = (this.rpcNumber + 1) % this.config.rpcEndpoints.length; this.logger.warn( `There was an issue with current parachain provider. Connecting to ${ this.config.rpcEndpoints[this.rpcNumber] }`, ); await this.initializeParachainProvider(); } async getNativeTokenBalance(wallet) { const nativeBalance = await wallet.getBalance(); return nativeBalance / 10 ** NATIVE_TOKEN_DECIMALS; } getValidOperationalWallets() { const wallets = []; this.config.operationalWallets.forEach((wallet) => { if ( this.invalidWallets?.find( (invalidWallet) => invalidWallet.privateKey === wallet.privateKey, ) ) { this.logger.warn( `Skipping initialization of wallet. Wallet public key: ${wallet.evmAddress}`, ); } else { try { wallets.push(new ethers.Wallet(wallet.privateKey, this.provider)); } catch (error) { this.logger.warn( `Invalid evm private key, unable to create wallet instance. Wallet public key: ${wallet.evmAddress}. Error: ${error.message}`, ); } } }); return wallets; } } export default OtParachainService; ================================================ FILE: src/modules/blockchain/implementation/polygon/polygon-service.js ================================================ import Web3Service from '../web3-service.js'; class PolygonService extends Web3Service { constructor(ctx) { super(ctx); this.baseTokenTicker = 'MATIC'; this.tracTicker = 'mTRAC'; } } export default PolygonService; ================================================ FILE: src/modules/blockchain/implementation/web3-service-validator.js ================================================ class Web3ServiceValidator { static validateResult(functionName, contractName, result, logger) { if (Web3ServiceValidator[`${functionName}Validator`]) { logger.trace( `Calling web3 service validator for function name: ${functionName}, contract: ${contractName}`, ); return Web3ServiceValidator[`${functionName}Validator`](result); } return true; } } export default Web3ServiceValidator; ================================================ FILE: src/modules/blockchain/implementation/web3-service.js ================================================ /* eslint-disable no-await-in-loop */ import { ethers, BigNumber } from 'ethers'; import axios from 'axios'; import async from 'async'; import { setTimeout as sleep } from 'timers/promises'; import { SOLIDITY_ERROR_STRING_PREFIX, SOLIDITY_PANIC_CODE_PREFIX, SOLIDITY_PANIC_REASONS, ZERO_PREFIX, TRANSACTION_QUEUE_CONCURRENCY, TRANSACTION_POLLING_TIMEOUT_MILLIS, TRANSACTION_CONFIRMATIONS, WS_RPC_PROVIDER_PRIORITY, HTTP_RPC_PROVIDER_PRIORITY, FALLBACK_PROVIDER_QUORUM, RPC_PROVIDER_STALL_TIMEOUT, CACHED_FUNCTIONS, CACHE_DATA_TYPES, CONTRACTS, CONTRACT_FUNCTION_PRIORITY, TRANSACTION_PRIORITY, CONTRACT_FUNCTION_GAS_LIMIT_INCREASE_FACTORS, ABIs, EXPECTED_TRANSACTION_ERRORS, } from '../../../constants/constants.js'; import Web3ServiceValidator from './web3-service-validator.js'; class Web3Service { async initialize(config, logger) { this.config = config; this.logger = logger; this.contractCallCache = {}; await this.initializeWeb3(); this.initializeTransactionQueues(); await this.initializeContracts(); this.initializeProviderDebugging(); } initializeTransactionQueues(concurrency = TRANSACTION_QUEUE_CONCURRENCY) { this.transactionQueues = {}; for (const operationalWallet of this.operationalWallets) { const transactionQueue = async.priorityQueue((args, cb) => { const { contractInstance, functionName, transactionArgs, gasPrice } = args; this._executeContractFunction( contractInstance, functionName, transactionArgs, gasPrice, operationalWallet, ) .then((result) => { cb({ result }); }) .catch((error) => { cb({ error }); }); }, concurrency); this.transactionQueues[operationalWallet.address] = transactionQueue; } this.transactionQueueOrder = Object.keys(this.transactionQueues); } queueTransaction(contractInstance, functionName, transactionArgs, callback, gasPrice) { const selectedQueue = this.selectTransactionQueue(); const priority = CONTRACT_FUNCTION_PRIORITY[functionName] ?? TRANSACTION_PRIORITY.MEDIUM; this.logger.info(`Calling ${functionName} with priority: ${priority}`); selectedQueue.push( { contractInstance, functionName, transactionArgs, gasPrice, }, priority, callback, ); } removeTransactionQueue(walletAddress) { delete this.transactionQueues[walletAddress]; } getTotalTransactionQueueLength() { let totalLength = 0; Object.values(this.transactionQueues).forEach((queue) => { totalLength += queue.length(); }); return totalLength; } selectTransactionQueue() { const queues = Object.keys(this.transactionQueues).map((wallet) => ({ wallet, length: this.transactionQueues[wallet].length(), })); const minLength = Math.min(...queues.map((queue) => queue.length)); const shortestQueues = queues.filter((queue) => queue.length === minLength); if (shortestQueues.length === 1) { return this.transactionQueues[shortestQueues[0].wallet]; } const selectedQueueWallet = this.transactionQueueOrder.find((roundRobinNext) => shortestQueues.some((shortestQueue) => shortestQueue.wallet === roundRobinNext), ); this.transactionQueueOrder.push( this.transactionQueueOrder .splice(this.transactionQueueOrder.indexOf(selectedQueueWallet), 1) .pop(), ); return this.transactionQueues[selectedQueueWallet]; } getValidOperationalWallets() { const wallets = []; this.config.operationalWallets.forEach((wallet) => { try { wallets.push(new ethers.Wallet(wallet.privateKey, this.provider)); } catch (error) { this.logger.warn( `Invalid evm private key, unable to create wallet instance. Wallet public key: ${wallet.evmAddress}. Error: ${error.message}`, ); } }); return wallets; } getRandomOperationalWallet() { const randomIndex = Math.floor(Math.random() * this.operationalWallets.length); return this.operationalWallets[randomIndex]; } async initializeWeb3() { const providers = []; for (const rpcEndpoint of this.config.rpcEndpoints) { const isWebSocket = rpcEndpoint.startsWith('ws'); const Provider = isWebSocket ? ethers.providers.WebSocketProvider : ethers.providers.JsonRpcProvider; const priority = isWebSocket ? WS_RPC_PROVIDER_PRIORITY : HTTP_RPC_PROVIDER_PRIORITY; try { const provider = new Provider(rpcEndpoint); // eslint-disable-next-line no-await-in-loop await provider.getNetwork(); providers.push({ provider, priority, weight: 1, stallTimeout: RPC_PROVIDER_STALL_TIMEOUT, }); this.logger.debug( `Connected to the blockchain RPC: ${this.maskRpcUrl(rpcEndpoint)}.`, ); } catch (e) { this.logger.warn( `Unable to connect to the blockchain RPC: ${this.maskRpcUrl(rpcEndpoint)}.`, ); } } try { this.provider = new ethers.providers.FallbackProvider( providers, FALLBACK_PROVIDER_QUORUM, ); // eslint-disable-next-line no-await-in-loop await this.providerReady(); } catch (e) { throw new Error( `RPC Fallback Provider initialization failed. Fallback Provider quorum: ${FALLBACK_PROVIDER_QUORUM}. Error: ${e.message}.`, ); } this.operationalWallets = this.getValidOperationalWallets(); if (this.operationalWallets.length === 0) { throw Error( 'Unable to initialize web3 service, all operational wallets provided are invalid', ); } } async initializeContracts() { this.contracts = {}; this.contractAddresses = {}; this.logger.info( `Initializing contracts with hub contract address: ${this.config.hubContractAddress}`, ); this.contracts.Hub = new ethers.Contract( this.config.hubContractAddress, ABIs.Hub, this.operationalWallets[0], ); this.contractAddresses[this.config.hubContractAddress] = this.contracts.Hub; const contractsArray = await this.callContractFunction( this.contracts.Hub, 'getAllContracts', [], ); contractsArray.forEach(([contractName, contractAddress]) => { this.initializeContract(contractName, contractAddress); }); this.assetStorageContracts = {}; const assetStoragesArray = await this.callContractFunction( this.contracts.Hub, 'getAllAssetStorages', [], ); assetStoragesArray.forEach(([, assetStorageAddress]) => { this.initializeAssetStorageContract(assetStorageAddress); }); this.logger.info(`Contracts initialized`); await this.logBalances(); } initializeProviderDebugging() { this.provider.on('debug', (info) => { const { method } = info.request; if (['call', 'estimateGas'].includes(method)) { const contractInstance = this.contractAddresses[info.request.params.transaction.to]; const inputData = info.request.params.transaction.data; const decodedInputData = this._decodeInputData( inputData, contractInstance.interface, ); const functionFragment = contractInstance.interface.getFunction( inputData.slice(0, 10), ); const functionName = functionFragment.name; const inputs = functionFragment.inputs .map((input, i) => { const argName = input.name; const argValue = this._formatArgument(decodedInputData[i]); return `${argName}=${argValue}`; }) .join(', '); if (info.backend.error) { const decodedErrorData = this._decodeErrorData( info.backend.error, contractInstance.interface, ); this.logger.debug( `${functionName}(${inputs}) ${method} has failed; Error: ${decodedErrorData}; ` + `RPC: ${this.maskRpcUrl(info.backend.provider.connection.url)}.`, ); } else if (info.backend.result !== undefined) { let message = `${functionName}(${inputs}) ${method} has been successfully executed; `; if (info.backend.result !== null && method !== 'estimateGas') { try { const decodedResultData = this._decodeResultData( inputData.slice(0, 10), info.backend.result, contractInstance.interface, ); message += `Result: ${decodedResultData}; `; } catch (error) { this.logger.warn( `Unable to decode result data for. Message: ${message}`, ); } } message += `RPC: ${this.maskRpcUrl(info.backend.provider.connection.url)}.`; this.logger.debug(message); } } }); } maskRpcUrl(url) { if (url.includes('apiKey')) { return url.split('apiKey')[0]; } return url; } initializeAssetStorageContract(assetStorageAddress) { this.assetStorageContracts[assetStorageAddress.toLowerCase()] = new ethers.Contract( assetStorageAddress, ABIs.KnowledgeCollectionStorage, this.operationalWallets[0], ); this.contractAddresses[assetStorageAddress] = this.assetStorageContracts[assetStorageAddress.toLowerCase()]; } setContractCallCache(contractName, functionName, value) { if (CACHED_FUNCTIONS[contractName]?.[functionName]) { const type = CACHED_FUNCTIONS[contractName][functionName]; if (!this.contractCallCache[contractName]) { this.contractCallCache[contractName] = {}; } switch (type) { case CACHE_DATA_TYPES.NUMBER: this.contractCallCache[contractName][functionName] = Number(value); break; default: this.contractCallCache[contractName][functionName] = value; } } } getContractCallCache(contractName, functionName) { if ( CACHED_FUNCTIONS[contractName]?.[functionName] && this.contractCallCache[contractName]?.[functionName] ) { return this.contractCallCache[contractName][functionName]; } return null; } initializeContract(contractName, contractAddress) { if (ABIs[contractName] != null) { this.contracts[contractName] = new ethers.Contract( contractAddress, ABIs[contractName], this.operationalWallets[0], ); this.contractAddresses[contractAddress] = this.contracts[contractName]; } } getContractAddress(contractName) { const contract = this.contracts[contractName]; if (!contract) { return null; } return contract.address; } async providerReady() { return this.provider.getNetwork(); } getPublicKeys() { return this.operationalWallets.map((wallet) => wallet.address); } getManagementKey() { return this.config.evmManagementWalletPublicKey; } async logBalances() { for (const wallet of this.operationalWallets) { // eslint-disable-next-line no-await-in-loop const nativeBalance = await this.getNativeTokenBalance(wallet); // eslint-disable-next-line no-await-in-loop const tokenBalance = await this.getTokenBalance(wallet.address); this.logger.info( `Balance of ${wallet.address} is ${nativeBalance} ${this.baseTokenTicker} and ${tokenBalance} ${this.tracTicker}.`, ); } } async getNativeTokenBalance(wallet) { const nativeBalance = await wallet.getBalance(); return Number(ethers.utils.formatEther(nativeBalance)); } async getTokenBalance(publicKey) { const tokenBalance = await this.callContractFunction(this.contracts.Token, 'balanceOf', [ publicKey, ]); return Number(ethers.utils.formatEther(tokenBalance)); } async getBlockNumber() { const latestBlock = await this.provider.getBlock('latest'); return latestBlock.number; } async getIdentityId() { if (this.identityId) { return this.identityId; } const promises = this.operationalWallets.map((wallet) => this.callContractFunction( this.contracts.IdentityStorage, 'getIdentityId', [wallet.address], CONTRACTS.IDENTITY_STORAGE, ).then((identityId) => [wallet.address, Number(identityId)]), ); const results = await Promise.all(promises); this.identityId = 0; const walletWithIdentityZero = []; results.forEach(([publicKey, identityId]) => { this.logger.trace( `Identity id: ${identityId} found for wallet: ${publicKey} on blockchain: ${this.getBlockchainId()}`, ); if (identityId !== 0) { if (this.identityId !== identityId && this.identityId !== 0) { const index = this.operationalWallets.find( (wallet) => wallet.address === publicKey, ); this.operationalWallets.splice(index, 1); this.logger.warn( `Found invalid identity id. Identity id: ${identityId} found for wallet: ${publicKey}, expected identity id: ${ this.identityId } on blockchain: ${this.getBlockchainId()}. Operational wallet will not be used for transactions.`, ); this.removeTransactionQueue(publicKey); } else { this.identityId = identityId; } } else { walletWithIdentityZero.push(publicKey); } }); if (this.identityId !== 0) { walletWithIdentityZero.forEach((publicKey) => { const index = this.operationalWallets.find( (wallet) => wallet.address === publicKey, ); this.operationalWallets.splice(index, 1); this.logger.warn( `Operational wallet: ${publicKey} don't have profile connected to it, expected identity id: ${ this.identityId } on blockchain ${this.getBlockchainId()}`, ); }); } if (this.operationalWallets.length === 0) { throw new Error( `Unable to find valid operational wallets for blockchain implementation: ${this.getBlockchainId()}`, ); } return this.identityId; } async identityIdExists() { const identityId = await this.getIdentityId(); return !!identityId; } async createProfile(peerId) { if (!this.config.nodeName) { throw new Error( 'Missing nodeName in blockchain configuration. Please add it and start the node again.', ); } const maxNumberOfRetries = 3; let retryCount = 0; let profileCreated = false; const retryDelayInSec = 12; while (retryCount + 1 <= maxNumberOfRetries && !profileCreated) { try { // eslint-disable-next-line no-await-in-loop await this._executeContractFunction( this.contracts.Profile, 'createProfile', [ this.getManagementKey(), this.getPublicKeys().slice(1), this.config.nodeName, ethers.utils.hexlify(ethers.utils.toUtf8Bytes(peerId)), this.config.operatorFee, ], null, this.operationalWallets[0], ); this.logger.info( `Profile created with name: ${this.config.nodeName}, wallet: ${ this.operationalWallets[0].address }, on blockchain ${this.getBlockchainId()}`, ); profileCreated = true; } catch (error) { if (error.message.includes('Profile already exists')) { this.logger.info( `Skipping profile creation, already exists on blockchain ${this.getBlockchainId()}.`, ); profileCreated = true; } else if (retryCount + 1 < maxNumberOfRetries) { retryCount += 1; this.logger.warn( `Unable to create profile. Will retry in ${retryDelayInSec}s. Retries left: ${ maxNumberOfRetries - retryCount } on blockchain ${this.getBlockchainId()}. Error: ${error}`, ); // eslint-disable-next-line no-await-in-loop await sleep(retryDelayInSec * 1000); } else { throw error; } } } } async getGasPrice() { try { const response = await axios.get(this.config.gasPriceOracleLink); const gasPriceRounded = Math.round(response.data.standard.maxFee * 1e9); return gasPriceRounded; } catch (error) { return undefined; } } buildTransactionGasParams(gasPrice) { return { gasPrice }; } async callContractFunction(contractInstance, functionName, args, contractName = null) { const maxNumberOfRetries = 3; const retryDelayInSec = 12; let retryCount = 0; let result = this.getContractCallCache(contractName, functionName); try { if (!result) { while (retryCount < maxNumberOfRetries) { result = await contractInstance[functionName](...args); const resultIsValid = Web3ServiceValidator.validateResult( functionName, contractName, result, this.logger, ); if (resultIsValid) { this.setContractCallCache(contractName, functionName, result); return result; } if (retryCount === maxNumberOfRetries - 1) { return null; } await sleep(retryDelayInSec * 1000); retryCount += 1; } } } catch (error) { this._decodeContractCallError(contractInstance, functionName, error, args); } return result; } async _executeContractFunction( contractInstance, functionName, args, predefinedGasPrice, operationalWallet, ) { let result; let gasPrice = predefinedGasPrice ?? (await this.getGasPrice()); let gasLimit; let retryCount = 0; const maxRetries = 3; for (let estimateAttempt = 0; estimateAttempt < 3; estimateAttempt += 1) { try { /* eslint-disable no-await-in-loop */ gasLimit = await contractInstance.estimateGas[functionName](...args); break; } catch (error) { const errMsg = error.message?.toLowerCase() ?? ''; const isTransient = errMsg.includes(EXPECTED_TRANSACTION_ERRORS.EXECUTION_FAILED.toLowerCase()) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.FEE_TOO_LOW.toLowerCase()) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.SOCKET_HANG_UP) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.ECONNRESET) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.ECONNREFUSED) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.SERVER_ERROR) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.BAD_GATEWAY) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.SERVICE_UNAVAILABLE) || errMsg.includes(EXPECTED_TRANSACTION_ERRORS.EXPECT_BLOCK_NUMBER); if (isTransient && estimateAttempt < 2) { this.logger.warn( `Gas estimation for ${functionName} failed with transient error on ${this.getBlockchainId()}, ` + `retrying (${estimateAttempt + 1}/3): ${error.message}`, ); await new Promise((r) => { setTimeout(r, 2000); }); continue; } this._decodeEstimateGasError(contractInstance, functionName, error, args); } } gasLimit = gasLimit ?? ethers.utils.parseUnits('900', 'kwei'); const gasLimitMultiplier = CONTRACT_FUNCTION_GAS_LIMIT_INCREASE_FACTORS[functionName] ?? 1; gasLimit = gasLimit.mul(gasLimitMultiplier * 100).div(100); while (retryCount < maxRetries) { try { this.logger.debug( `Sending signed transaction ${functionName} to the blockchain ${this.getBlockchainId()}` + ` with gas limit: ${gasLimit.toString()} and gasPrice ${gasPrice.toString()}. ` + `Transaction queue length: ${this.getTotalTransactionQueueLength()}. Wallet used: ${ operationalWallet.address }${retryCount > 0 ? ` (retry ${retryCount})` : ''}`, ); const txOverrides = this.buildTransactionGasParams(gasPrice); txOverrides.gasLimit = gasLimit; const tx = await contractInstance .connect(operationalWallet) [functionName](...args, txOverrides); try { result = await this.provider.waitForTransaction( tx.hash, TRANSACTION_CONFIRMATIONS, TRANSACTION_POLLING_TIMEOUT_MILLIS, ); if (result.status === 0) { await this.provider.call(tx, tx.blockNumber); } } catch (error) { if ( error.message .toLowerCase() .includes(EXPECTED_TRANSACTION_ERRORS.TIMEOUT_EXCEEDED.toLowerCase()) ) { const existingReceipt = await this.provider.getTransactionReceipt(tx.hash); if (existingReceipt) { this.logger.info( `Transaction ${functionName} (${tx.hash}) confirmed despite timeout. Block: ${existingReceipt.blockNumber}`, ); if (existingReceipt.status === 0) { await this.provider.call(tx, existingReceipt.blockNumber); } return existingReceipt; } throw error; } this._decodeWaitForTxError(contractInstance, functionName, error, args); } return result; } catch (error) { const errorMessage = error.message.toLowerCase(); const isNonceError = errorMessage.includes( EXPECTED_TRANSACTION_ERRORS.NONCE_TOO_LOW.toLowerCase(), ) || errorMessage.includes( EXPECTED_TRANSACTION_ERRORS.REPLACEMENT_UNDERPRICED.toLowerCase(), ) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ALREADY_KNOWN.toLowerCase()); const isTimeoutError = errorMessage.includes( EXPECTED_TRANSACTION_ERRORS.TIMEOUT_EXCEEDED.toLowerCase(), ); const isExecutionError = errorMessage.includes( EXPECTED_TRANSACTION_ERRORS.EXECUTION_FAILED.toLowerCase(), ) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.FEE_TOO_LOW.toLowerCase()); const isNetworkError = errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.SOCKET_HANG_UP) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ECONNRESET) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ECONNREFUSED) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.SERVER_ERROR) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.BAD_GATEWAY) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.SERVICE_UNAVAILABLE) || errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.EXPECT_BLOCK_NUMBER); if (isNonceError || isTimeoutError || isExecutionError || isNetworkError) { retryCount += 1; if (retryCount < maxRetries) { const shouldBumpGas = isNonceError || isExecutionError; if (shouldBumpGas) { gasPrice = ethers.BigNumber.isBigNumber(gasPrice) ? gasPrice.mul(120).div(100) : Math.ceil(gasPrice * 1.2); } let errorType = 'Nonce'; if (isTimeoutError) errorType = 'Timeout'; else if (isExecutionError) errorType = 'Execution/fee'; else if (isNetworkError) errorType = 'Network'; this.logger.warn( `${errorType} error detected for ${functionName} on ${this.getBlockchainId()}. ` + `Retrying ${ shouldBumpGas ? `with increased gas price: ${gasPrice}` : 'with same gas price' } (retry ${retryCount}/${maxRetries})`, ); continue; } let errorType = 'nonce'; if (isTimeoutError) errorType = 'timeout'; else if (isExecutionError) errorType = 'execution/fee'; else if (isNetworkError) errorType = 'network'; this.logger.error( `Max retries (${maxRetries}) reached for ${errorType} error in ${functionName} on ${this.getBlockchainId()}. ` + `Final gas price: ${gasPrice}`, ); } throw error; } } } _decodeEstimateGasError(contractInstance, functionName, error, args) { try { const decodedErrorData = this._decodeErrorData(error, contractInstance.interface); if (error.transaction === undefined) { throw new Error( `Gas estimation for ${functionName} has failed, reason: ${decodedErrorData}`, ); } const functionFragment = contractInstance.interface.getFunction( error.transaction.data.slice(0, 10), ); const inputs = functionFragment.inputs .map((input, i) => { const argName = input.name; const argValue = this._formatArgument(args[i]); return `${argName}=${argValue}`; }) .join(', '); throw new Error( `Gas estimation for ${functionName}(${inputs}) has failed, reason: ${decodedErrorData}`, ); } catch (decodeError) { this.logger.warn(`Unable to decode estimate gas error: ${decodeError}`); throw error; } } _decodeWaitForTxError(contractInstance, functionName, error, args) { try { const decodedErrorData = this._decodeErrorData(error, contractInstance.interface); let sigHash; if (error.transaction) { sigHash = error.transaction.data.slice(0, 10); } else { sigHash = this._getFunctionSighash(contractInstance, functionName, args); } const functionFragment = contractInstance.interface.getFunction(sigHash); const inputs = functionFragment.inputs .map((input, i) => { const argName = input.name; const argValue = this._formatArgument(args[i]); return `${argName}=${argValue}`; }) .join(', '); throw new Error( `Transaction ${functionName}(${inputs}) has been reverted, reason: ${decodedErrorData}`, ); } catch (decodeError) { this.logger.warn(`Unable to decode wait for transaction error: ${decodeError}`); throw error; } } _decodeContractCallError(contractInstance, functionName, error, args) { try { const decodedErrorData = this._decodeErrorData(error, contractInstance.interface); const functionFragment = contractInstance.interface.getFunction( error.transaction.data.slice(0, 10), ); const inputs = functionFragment.inputs .map((input, i) => { const argName = input.name; const argValue = this._formatArgument(args[i]); return `${argName}=${argValue}`; }) .join(', '); throw new Error(`Call ${functionName}(${inputs}) failed, reason: ${decodedErrorData}`); } catch (decodeError) { this.logger.warn(`Unable to decode contract call error: ${decodeError}`); throw error; } } _getFunctionSighash(contractInstance, functionName, args) { const functions = Object.keys(contractInstance.interface.functions) .filter((key) => contractInstance.interface.functions[key].name === functionName) .map((key) => ({ signature: key, ...contractInstance.interface.functions[key] })); for (const func of functions) { try { // Checks if given arguments can be encoded with function ABI inputs // may be useful for overloaded functions as it would help to find // needed function fragment ethers.utils.defaultAbiCoder.encode(func.inputs, args); const sighash = ethers.utils.hexDataSlice( ethers.utils.keccak256(ethers.utils.toUtf8Bytes(func.signature)), 0, 4, ); return sighash; } catch (error) { continue; } } throw new Error('No matching function signature found'); } _getErrorData(error) { let nestedError = error; while (nestedError && nestedError.error) { nestedError = nestedError.error; } const errorData = nestedError.data; if (errorData === undefined) { throw error; } let returnData = typeof errorData === 'string' ? errorData : errorData.data; if (typeof returnData === 'object' && returnData.data) { returnData = returnData.data; } if (returnData === undefined || typeof returnData !== 'string') { throw error; } return returnData; } _decodeInputData(inputData, contractInterface) { if (inputData === ZERO_PREFIX) { return 'Empty input data.'; } return contractInterface.decodeFunctionData(inputData.slice(0, 10), inputData); } _decodeErrorData(evmError, contractInterface) { let errorData; try { errorData = this._getErrorData(evmError); } catch (error) { return error.message; } // Handle empty error data if (errorData === ZERO_PREFIX) { return 'Empty error data.'; } // Handle standard solidity string error if (errorData.startsWith(SOLIDITY_ERROR_STRING_PREFIX)) { const encodedReason = errorData.slice(SOLIDITY_ERROR_STRING_PREFIX.length); try { return ethers.utils.defaultAbiCoder.decode(['string'], `0x${encodedReason}`)[0]; } catch (error) { return error.message; } } // Handle solidity panic code if (errorData.startsWith(SOLIDITY_PANIC_CODE_PREFIX)) { const encodedReason = errorData.slice(SOLIDITY_PANIC_CODE_PREFIX.length); let code; try { [code] = ethers.utils.defaultAbiCoder.decode(['uint256'], `0x${encodedReason}`); } catch (error) { return error.message; } return SOLIDITY_PANIC_REASONS[code] ?? 'Unknown Solidity panic code.'; } // Try parsing a custom error using the contract ABI try { const decodedCustomError = contractInterface.parseError(errorData); const formattedArgs = decodedCustomError.errorFragment.inputs .map((input, i) => { const argName = input.name; const argValue = this._formatArgument(decodedCustomError.args[i]); return `${argName}=${argValue}`; }) .join(', '); return `custom error ${decodedCustomError.name}(${formattedArgs})`; } catch (error) { return `Failed to decode custom error data. Error: ${error}`; } } _decodeResultData(fragment, resultData, contractInterface) { if (resultData === ZERO_PREFIX) { return 'Empty input data.'; } return contractInterface.decodeFunctionResult(fragment, resultData); } _formatArgument(value) { if (value === null || value === undefined) { return 'null'; } if (typeof value === 'string') { return value; } if (typeof value === 'number' || BigNumber.isBigNumber(value)) { return value.toString(); } if (Array.isArray(value)) { return `[${value.map((v) => this._formatArgument(v)).join(', ')}]`; } if (typeof value === 'object') { const formattedEntries = Object.entries(value).map( ([k, v]) => `${k}: ${this._formatArgument(v)}`, ); return `{${formattedEntries.join(', ')}}`; } return value.toString(); } async isAssetStorageContract(contractAddress) { return this.callContractFunction(this.contracts.Hub, 'isAssetStorage(address)', [ contractAddress, ]); } async getKnowledgeCollectionMerkleRootByIndex( assetStorageContractAddress, knowledgeCollectionId, index, ) { const assetStorageContractInstance = this.assetStorageContracts[assetStorageContractAddress.toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); return this.callContractFunction(assetStorageContractInstance, 'getMerkleRootByIndex', [ knowledgeCollectionId, index, ]); } async getKnowledgeCollectionLatestMerkleRoot( assetStorageContractAddress, knowledgeCollectionId, ) { const assetStorageContractInstance = this.assetStorageContracts[assetStorageContractAddress.toString().toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); return this.callContractFunction(assetStorageContractInstance, 'getLatestMerkleRoot', [ knowledgeCollectionId, ]); } async getLatestKnowledgeCollectionId(assetStorageContractAddress) { const assetStorageContractInstance = this.assetStorageContracts[assetStorageContractAddress.toString().toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); const lastKnowledgeCollectionId = await this.callContractFunction( assetStorageContractInstance, 'getLatestKnowledgeCollectionId', [], ); return lastKnowledgeCollectionId; } getAssetStorageContractAddresses() { return Object.keys(this.assetStorageContracts); } async getKnowledgeCollectionMerkleRoots(assetStorageContractAddress, tokenId) { const assetStorageContractInstance = this.assetStorageContracts[assetStorageContractAddress.toString().toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); return this.callContractFunction(assetStorageContractInstance, 'getMerkleRoots', [tokenId]); } // async getKnowledgeAssetOwner(assetContractAddress, tokenId) { // const assetStorageContractInstance = // this.assetStorageContracts[assetContractAddress.toString().toLowerCase()]; // if (!assetStorageContractInstance) // throw new Error('Unknown asset storage contract address'); // return this.callContractFunction(assetStorageContractInstance, 'ownerOf', [tokenId]); // } async getLatestMerkleRootPublisher(assetStorageContractAddress, knowledgeCollectionId) { const assetStorageContractInstance = this.assetStorageContracts[assetStorageContractAddress.toString().toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); const knowledgeCollectionPublisher = await this.callContractFunction( assetStorageContractInstance, 'getLatestMerkleRootPublisher', [knowledgeCollectionId], ); return knowledgeCollectionPublisher; } async getKnowledgeCollectionSize(assetStorageContractAddress, knowledgeCollectionId) { const assetStorageContractInstance = this.assetStorageContracts[assetStorageContractAddress.toString().toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); const knowledgeCollectionSize = await this.callContractFunction( assetStorageContractInstance, 'getByteSize', [knowledgeCollectionId], ); return Number(knowledgeCollectionSize); } async getKnowledgeAssetsRange(assetStorageContractAddress, knowledgeCollectionId) { const assetStorageContractInstance = this.assetStorageContracts[assetStorageContractAddress.toString().toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); const knowledgeAssetsRange = await this.callContractFunction( assetStorageContractInstance, 'getKnowledgeAssetsRange', [knowledgeCollectionId], ); return { startTokenId: Number( knowledgeAssetsRange[0] .sub(BigNumber.from(knowledgeCollectionId - 1).mul('0x0f4240')) .toString(), ), endTokenId: Number( knowledgeAssetsRange[1] .sub(BigNumber.from(knowledgeCollectionId - 1).mul('0x0f4240')) .toString(), ), burned: knowledgeAssetsRange[2].map((burned) => Number( burned .sub(BigNumber.from(knowledgeCollectionId - 1).mul('0x0f4240')) .toString(), ), ), }; } async getMinimumStake() { const minimumStake = await this.callContractFunction( this.contracts.ParametersStorage, 'minimumStake', [], CONTRACTS.PARAMETERS_STORAGE, ); return Number(ethers.utils.formatEther(minimumStake)); } async getMaximumStake() { const maximumStake = await this.callContractFunction( this.contracts.ParametersStorage, 'maximumStake', [], CONTRACTS.PARAMETERS_STORAGE, ); return Number(ethers.utils.formatEther(maximumStake)); } async getMinimumRequiredSignatures() { return this.callContractFunction( this.contracts.ParametersStorage, 'minimumRequiredSignatures', [], CONTRACTS.PARAMETERS_STORAGE, ); } async getShardingTableHead() { return this.callContractFunction(this.contracts.ShardingTableStorage, 'head', []); } async getShardingTableLength() { const nodesCount = await this.callContractFunction( this.contracts.ShardingTableStorage, 'nodesCount', [], ); return Number(nodesCount); } async getShardingTablePage(startingIdentityId, nodesNum) { return this.callContractFunction( this.contracts.ShardingTable, 'getShardingTable(uint72,uint72)', [startingIdentityId, nodesNum], ); } getBlockchainId() { return this.getImplementationName(); } async healthCheck() { try { const gasPrice = await this.operationalWallets[0].getGasPrice(); if (gasPrice) return true; } catch (e) { this.logger.error(`Error on checking blockchain. ${e}`); return false; } return false; } async restartService() { await this.initializeWeb3(); await this.initializeContracts(); } async getBlockchainTimestamp() { return Math.floor(Date.now() / 1000); } async getLatestBlock() { const currentBlock = await this.provider.getBlockNumber(); const blockTimestamp = await this.provider.getBlock(currentBlock); return blockTimestamp; } async getParanetKnowledgeCollectionCount(paranetId) { return this.callContractFunction( this.contracts.ParanetsRegistry, 'getKnowledgeCollectionsCount', [paranetId], ); } async getParanetKnowledgeCollectionLocatorsWithPagination(paranetId, offset, limit) { return this.callContractFunction( this.contracts.Paranet, 'getKnowledgeCollectionLocatorsWithPagination', [paranetId, offset, limit], ); } async getParanetMetadata(paranetId) { return this.callContractFunction( this.contracts.ParanetsRegistry, 'getParanetMetadata', [paranetId], CONTRACTS.PARANETS_REGISTRY, ); } async getParanetName(paranetId) { return this.callContractFunction( this.contracts.ParanetsRegistry, 'getName', [paranetId], CONTRACTS.PARANETS_REGISTRY, ); } async getDescription(paranetId) { return this.callContractFunction( this.contracts.ParanetsRegistry, 'getDescription', [paranetId], CONTRACTS.PARANETS_REGISTRY, ); } async paranetExists(paranetId) { return this.callContractFunction( this.contracts.ParanetsRegistry, 'paranetExists', [paranetId], CONTRACTS.PARANETS_REGISTRY, ); } async isPermissionedNode(paranetId, identityId) { return this.callContractFunction(this.contracts.ParanetsRegistry, 'isPermissionedNode', [ paranetId, identityId, ]); } async getNodesAccessPolicy(paranetId) { return this.callContractFunction(this.contracts.ParanetsRegistry, 'getNodesAccessPolicy', [ paranetId, ]); } async getPermissionedNodes(paranetId) { return this.callContractFunction( this.contracts.ParanetsRegistry, 'getPermissionedNodes', [paranetId], CONTRACTS.PARANETS_REGISTRY, ); } async getNodeId(identityId) { return this.callContractFunction(this.contracts.ProfileStorage, 'getNodeId', [identityId]); } async signMessage(messageHash) { const wallet = this.getRandomOperationalWallet(); return wallet.signMessage(ethers.utils.arrayify(messageHash)); } async getStakeWeightedAverageAsk() { return this.callContractFunction( this.contracts.AskStorage, 'getStakeWeightedAverageAsk', [], ); } async getTimeUntilNextEpoch() { return this.callContractFunction(this.contracts.Chronos, 'timeUntilNextEpoch', []); } async getEpochLength() { return this.callContractFunction(this.contracts.Chronos, 'epochLength', []); } async isKnowledgeCollectionRegistered(paranetId, knowledgeCollectionId) { return this.callContractFunction( this.contracts.ParanetsRegistry, 'isKnowledgeCollectionRegistered', [paranetId, knowledgeCollectionId], ); } async getActiveProofPeriodStatus() { return this.callContractFunction( this.contracts.RandomSampling, 'getActiveProofPeriodStatus', [], ); } async createChallenge() { return new Promise((resolve) => { this.queueTransaction( this.contracts.RandomSampling, 'createChallenge', [], (result) => { if (result.error || result?.result?.status === 0) { resolve({ success: false, error: result?.error ?? 'Error message not found', }); } else { resolve({ success: true, result: result.result, }); } }, ); }); } async getNodeChallenge(nodeId) { return this.callContractFunction(this.contracts.RandomSamplingStorage, 'getNodeChallenge', [ nodeId, ]); } async submitProof(chunk, merkleProof) { return new Promise((resolve, reject) => { this.queueTransaction( this.contracts.RandomSampling, 'submitProof', [chunk, merkleProof], (result) => { if (result.error) { reject(result.error); } else { resolve({ success: true, result: result.result, }); } }, ); }); } async getNodeEpochProofPeriodScore(nodeId, epoch, proofPeriodStartBlock) { return this.callContractFunction( this.contracts.RandomSamplingStorage, 'getNodeEpochProofPeriodScore', [nodeId, epoch, proofPeriodStartBlock], ); } async getTransaction(txHash) { return this.provider.getTransaction(txHash); } async getBlockTimestamp(blockNumber) { const block = await this.provider.getBlock(blockNumber); return block.timestamp; } async getDelegators(identityId) { return this.callContractFunction(this.contracts.DelegatorsInfo, 'getDelegators', [ identityId, ]); } async hasEverDelegated(identityId, address) { return this.callContractFunction(this.contracts.DelegatorsInfo, 'hasEverDelegatedToNode', [ identityId, address, ]); } async getCurrentEpoch() { return this.callContractFunction(this.contracts.Chronos, 'getCurrentEpoch', []); } async getLastClaimedEpoch(identityId, address) { return this.callContractFunction(this.contracts.DelegatorsInfo, 'getLastClaimedEpoch', [ identityId, address, ]); } async batchClaimDelegatorRewards(identityId, epochs, delegators) { return new Promise((resolve, reject) => { this.queueTransaction( this.contracts.Staking, 'batchClaimDelegatorRewards', [identityId, epochs, delegators], (result) => { if (result.error) { reject(result.error); } else { resolve({ success: true, result: result.result, }); } }, ); }); } async getAssetStorageContractsAddress() { return Object.keys(this.assetStorageContracts); } // SUPPORT FOR OLD CONTRACTS async getLatestAssertionId(assetContractAddress, tokenId) { const assetStorageContractInstance = this.assetStorageContracts[assetContractAddress.toString().toLowerCase()]; if (!assetStorageContractInstance) throw new Error('Unknown asset storage contract address'); return this.callContractFunction(assetStorageContractInstance, 'getLatestAssertionId', [ tokenId, ]); } } export default Web3Service; ================================================ FILE: src/modules/blockchain-events/blockchain-events-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class BlockchainEventsModuleManager extends BaseModuleManager { getContractAddress(implementationName, blockchain, contractName) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getContractAddress( blockchain, contractName, ); } } updateContractAddress(implementationName, blockchain, contractName, contractAddress) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.updateContractAddress( blockchain, contractName, contractAddress, ); } } async getBlock(implementationName, blockchain, tag) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getBlock(blockchain, tag); } } async getPastEvents( implementationName, blockchain, contractNames, eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, ) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getPastEvents( blockchain, contractNames, eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, ); } } getName() { return 'blockchainEvents'; } } export default BlockchainEventsModuleManager; ================================================ FILE: src/modules/blockchain-events/implementation/blockchain-events-service.js ================================================ class BlockchainEventsService { async initialize(config, logger) { this.logger = logger; this.config = config; } getContractAddress() { throw Error('getContractAddress not implemented'); } updateContractAddress() { throw Error('updateContractAddress not implemented'); } async getBlock() { throw Error('getBlock not implemented'); } async getPastEvents() { throw Error('getPastEvents not implemented'); } } export default BlockchainEventsService; ================================================ FILE: src/modules/blockchain-events/implementation/ot-ethers/ot-ethers.js ================================================ /* eslint-disable no-await-in-loop */ import { ethers } from 'ethers'; import BlockchainEventsService from '../blockchain-events-service.js'; import { MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH, MAX_BLOCKCHAIN_EVENT_SYNC_OF_HISTORICAL_BLOCKS_IN_MILLS, NODE_ENVIRONMENTS, ABIs, MONITORED_CONTRACTS, } from '../../../../constants/constants.js'; class OtEthers extends BlockchainEventsService { async initialize(config, logger) { await super.initialize(config, logger); this.contractCallCache = {}; await this._initializeRpcProviders(); await this._initializeContracts(); } async _initializeRpcProviders() { this.providers = {}; for (const blockchain of this.config.blockchains) { const validProviders = []; for (const rpcEndpoint of this.config.rpcEndpoints[blockchain]) { try { const provider = new ethers.providers.JsonRpcProvider(rpcEndpoint); // eslint-disable-next-line no-await-in-loop await provider.getNetwork(); validProviders.push(provider); } catch (error) { this.logger.error( `Failed to initialize provider: ${rpcEndpoint}. Error: ${error.message}`, ); } } if (validProviders.length === 0) { throw new Error(`No valid providers found for blockchain: ${blockchain}`); } this.providers[blockchain] = validProviders; this.logger.info( `Initialized ${validProviders.length} valid providers for blockchain: ${blockchain}`, ); } } _getRandomProvider(blockchain) { const blockchainProviders = this.providers[blockchain]; if (!blockchainProviders || blockchainProviders.length === 0) { throw new Error(`No providers available for blockchain: ${blockchain}`); } const randomIndex = Math.floor(Math.random() * blockchainProviders.length); return blockchainProviders[randomIndex]; } _getShuffledProviders(blockchain) { const blockchainProviders = this.providers[blockchain]; if (!blockchainProviders || blockchainProviders.length === 0) { throw new Error(`No providers available for blockchain: ${blockchain}`); } const shuffled = [...blockchainProviders]; for (let i = shuffled.length - 1; i > 0; i -= 1) { const j = Math.floor(Math.random() * (i + 1)); [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; } return shuffled; } async _sendWithFailover(blockchain, method, params) { const providers = this._getShuffledProviders(blockchain); let lastError; for (const provider of providers) { try { return await provider.send(method, params); } catch (error) { lastError = error; this.logger.warn( `RPC provider failed for ${method} on ${blockchain}: ${error.message}. ` + `Trying next provider (${providers.indexOf(provider) + 1}/${ providers.length })...`, ); } } throw lastError; } async _initializeContracts() { this.contracts = {}; for (const blockchain of this.config.blockchains) { this.contracts[blockchain] = {}; this.logger.info( `Initializing contracts with hub contract address: ${this.config.hubContractAddress[blockchain]}`, ); this.contracts[blockchain].Hub = this.config.hubContractAddress[blockchain]; const provider = this._getRandomProvider(blockchain); const hubContract = new ethers.Contract( this.config.hubContractAddress[blockchain], ABIs.Hub, provider, ); const contractsAray = await hubContract.getAllContracts(); const assetStoragesArray = await hubContract.getAllAssetStorages(); const allContracts = [...contractsAray, ...assetStoragesArray]; for (const [contractName, contractAddress] of allContracts) { if (MONITORED_CONTRACTS.includes(contractName) && ABIs[contractName] != null) { this.contracts[blockchain][contractName] = contractAddress; } } } } getContractAddress(blockchain, contractName) { return this.contracts[blockchain][contractName]; } updateContractAddress(blockchain, contractName, contractAddress) { this.contracts[blockchain][contractName] = contractAddress; } async getBlock(blockchain, tag) { const provider = this._getRandomProvider(blockchain); return provider.getBlock(tag); } async getPastEvents(blockchain, contractNames, eventsToFilter, lastCheckedBlock, currentBlock) { const maxBlocksToSync = await this._getMaxNumberOfHistoricalBlocksForSync(blockchain); let fromBlock = currentBlock - lastCheckedBlock > maxBlocksToSync ? currentBlock : lastCheckedBlock + 1; const eventsMissed = currentBlock - lastCheckedBlock > maxBlocksToSync; if (eventsMissed) { return { events: [], lastCheckedBlock: currentBlock, eventsMissed, }; } const contractAddresses = []; const topics = []; const addressToContractNameMap = {}; for (const contractName of contractNames) { const contractAddress = this.contracts[blockchain][contractName]; if (!contractAddress) { continue; } const provider = this._getRandomProvider(blockchain); const contract = new ethers.Contract(contractAddress, ABIs[contractName], provider); const contractTopics = []; for (const filterName in contract.filters) { if (!eventsToFilter.includes(filterName)) { continue; } const filter = contract.filters[filterName]().topics[0]; contractTopics.push(filter); } if (contractTopics.length > 0) { contractAddresses.push(contract.address); topics.push(...contractTopics); addressToContractNameMap[contract.address.toLowerCase()] = contractName; } } const events = []; let toBlock = currentBlock; try { while (fromBlock <= currentBlock) { toBlock = Math.min( fromBlock + MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH - 1, currentBlock, ); const fromBlockParam = ethers.BigNumber.from(fromBlock) .toHexString() .replace(/^0x0+/, '0x'); const toBlockParam = ethers.BigNumber.from(toBlock) .toHexString() .replace(/^0x0+/, '0x'); const newLogs = await this._sendWithFailover(blockchain, 'eth_getLogs', [ { address: contractAddresses, fromBlock: fromBlockParam, toBlock: toBlockParam, topics: [topics], }, ]); for (const log of newLogs) { const contractName = addressToContractNameMap[log.address]; const contractInterface = new ethers.utils.Interface(ABIs[contractName]); try { const parsedLog = contractInterface.parseLog(log); events.push({ blockchain, contract: contractName, contractAddress: log.address, event: parsedLog.name, data: JSON.stringify( Object.fromEntries( Object.entries(parsedLog.args).map(([k, v]) => [ k, ethers.BigNumber.isBigNumber(v) ? v.toString() : v, ]), ), ), blockNumber: parseInt(log.blockNumber, 16), transactionIndex: parseInt(log.transactionIndex, 16), logIndex: parseInt(log.logIndex, 16), txHash: log.transactionHash, }); } catch (error) { this.logger.warn( `Failed to parse log for contract: ${contractName}. Error: ${error.message}`, ); } } fromBlock = toBlock + 1; } } catch (error) { this.logger.warn( `Unable to process block range from: ${fromBlock} to: ${toBlock} on blockchain: ${blockchain}. Error: ${error.message}`, ); } return { events, eventsMissed, }; } async _getMaxNumberOfHistoricalBlocksForSync(blockchain) { if (!this.maxNumberOfHistoricalBlocksForSync) { if ( [NODE_ENVIRONMENTS.DEVELOPMENT, NODE_ENVIRONMENTS.TEST].includes( process.env.NODE_ENV, ) ) { this.maxNumberOfHistoricalBlocksForSync = Infinity; } else { const blockTimeMillis = await this._getBlockTimeMillis(blockchain); this.maxNumberOfHistoricalBlocksForSync = Math.round( // 60 * 60 * 1000 = 1 hour MAX_BLOCKCHAIN_EVENT_SYNC_OF_HISTORICAL_BLOCKS_IN_MILLS / blockTimeMillis, ); } } return this.maxNumberOfHistoricalBlocksForSync; } async _getBlockTimeMillis(blockchain, blockRange = 1000) { const latestBlock = await this.getBlock(blockchain); const olderBlock = await this.getBlock(blockchain, latestBlock.number - blockRange); const timeDiffMillis = (latestBlock.timestamp - olderBlock.timestamp) * 1000; return timeDiffMillis / blockRange; } } export default OtEthers; ================================================ FILE: src/modules/http-client/http-client-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class HttpClientModuleManager extends BaseModuleManager { constructor(ctx) { super(ctx); this.authService = ctx.authService; } getName() { return 'httpClient'; } get(route, callback, options = {}) { if (this.initialized) { return this.getImplementation().module.get(route, callback, options); } } post(route, callback, options = {}) { if (this.initialized) { return this.getImplementation().module.post(route, callback, options); } } use(route, callback, options = {}) { if (this.initialized) { return this.getImplementation().module.use(route, callback, options); } } createRouterInstance() { if (this.initialized) { return this.getImplementation().module.createRouterInstance(); } } sendResponse(res, status, returnObject) { if (this.initialized) { return this.getImplementation().module.sendResponse(res, status, returnObject); } } async listen() { if (this.initialized) { return this.getImplementation().module.listen(); } } selectMiddlewares(options) { if (this.initialized) { return this.getImplementation().module.selectMiddlewares(options); } } initializeBeforeMiddlewares(blockchainImpelemntations) { if (this.initialized) { return this.getImplementation().module.initializeBeforeMiddlewares( this.authService, blockchainImpelemntations, ); } } initializeAfterMiddlewares() { if (this.initialized) { return this.getImplementation().module.initializeAfterMiddlewares(this.authService); } } } export default HttpClientModuleManager; ================================================ FILE: src/modules/http-client/implementation/express-http-client.js ================================================ import express from 'express'; import https from 'https'; import fs from 'fs-extra'; import fileUpload from 'express-fileupload'; import cors from 'cors'; import requestValidationMiddleware from './middleware/request-validation-middleware.js'; import rateLimiterMiddleware from './middleware/rate-limiter-middleware.js'; import authenticationMiddleware from './middleware/authentication-middleware.js'; import authorizationMiddleware from './middleware/authorization-middleware.js'; import blockchainIdMiddleware from './middleware/blockchain-id-midleware.js'; import { BYTES_IN_MEGABYTE, MAX_FILE_SIZE } from '../../../constants/constants.js'; class ExpressHttpClient { async initialize(config, logger) { this.config = config; this.logger = logger; this.app = express(); } get(route, callback, options) { this.app.get(route, ...this.selectMiddlewares(options), callback); } post(route, callback, options) { this.app.post(route, ...this.selectMiddlewares(options), callback); } use(route, callback, options) { this.app.use(route, ...this.selectMiddlewares(options), callback); } createRouterInstance() { return express.Router(); } sendResponse(res, status, returnObject) { res.status(status); res.send(returnObject); } async listen() { if (this.config.useSsl) { const [key, cert] = await Promise.all([ fs.promises.readFile(this.config.sslKeyPath), fs.promises.readFile(this.config.sslCertificatePath), ]); this.httpsServer = https.createServer( { key, cert, }, this.app, ); this.httpsServer.listen(this.config.port); } else { this.app.listen(this.config.port); } this.logger.info(`Node listening on port: ${this.config.port}`); } selectMiddlewares(options) { const middlewares = []; if (options.rateLimit) middlewares.push(rateLimiterMiddleware(this.config.rateLimiter)); if (options.requestSchema) middlewares.push(requestValidationMiddleware(options.requestSchema)); return middlewares; } initializeBeforeMiddlewares(authService, blockchainImplementations) { this._initializeCorsMiddleware(); this.app.use(authenticationMiddleware(authService)); this.app.use(authorizationMiddleware(authService)); this._initializeBaseMiddlewares(); this.app.use(blockchainIdMiddleware(blockchainImplementations)); } initializeAfterMiddlewares() { // placeholder method for after middlewares } _initializeCorsMiddleware() { const corsOptions = {}; if (this.config.auth?.cors?.allowedOrigin) { corsOptions.origin = this.config.auth.cors.allowedOrigin; } this.app.use(cors(corsOptions)); } _initializeBaseMiddlewares() { this.app.use( fileUpload({ createParentPath: true, }), ); this.app.use(express.json({ limit: `${MAX_FILE_SIZE / BYTES_IN_MEGABYTE}mb` })); this.app.use((req, res, next) => { this.logger.api(`${req.method}: ${req.url} request received`); return next(); }); } } export default ExpressHttpClient; ================================================ FILE: src/modules/http-client/implementation/middleware/authentication-middleware.js ================================================ const parseIp = (req) => { let xForwardedFor; let socketRemoteAddress; if (req.socket) { socketRemoteAddress = req.socket.remoteAddress; } return xForwardedFor || socketRemoteAddress; }; export default (authService) => async (req, res, next) => { // eslint-disable-next-line no-useless-escape const match = req.path.match(/^\/(?:v[0-9]+\/)?([^\/\?]+)/); if (!match) return res.status(404).send('Not found.'); const operation = match[0].substring(1).toUpperCase(); if (authService.isPublicOperation(operation)) { return next(); } const ip = parseIp(req); const token = req.headers.authorization && req.headers.authorization.startsWith('Bearer ') && req.headers.authorization.split(' ')[1]; const isAuthenticated = await authService.authenticate(ip, token); if (!isAuthenticated) { return res.status(401).send('Unauthenticated.'); } next(); }; ================================================ FILE: src/modules/http-client/implementation/middleware/authorization-middleware.js ================================================ const getToken = (req) => { if (req.headers.authorization && req.headers.authorization.startsWith('Bearer ')) { return req.headers.authorization.split(' ')[1]; } }; export default (authService) => async (req, res, next) => { // eslint-disable-next-line no-useless-escape const match = req.path.match(/^\/(?:v[0-9]+\/)?([^\/\?]+)/); if (!match) return res.status(404).send('Not found.'); const operation = match[0].substring(1).toUpperCase(); if (authService.isPublicOperation(operation)) { return next(); } const token = getToken(req); const isAuthorized = await authService.isAuthorized(token, operation); if (!isAuthorized) { return res.status(403).send('Forbidden.'); } next(); }; ================================================ FILE: src/modules/http-client/implementation/middleware/blockchain-id-midleware.js ================================================ function addBlockchainId(blockchain, blockchainImplementations) { let updatedBlockchain = blockchain; if (blockchain?.split(':').length === 1) { for (const implementation of blockchainImplementations) { if (implementation.split(':')[0] === blockchain) { updatedBlockchain = implementation; break; } } } return updatedBlockchain; } export default function blockchainIdMiddleware(blockchainImplementations) { return (req, res, next) => { if (req.method === 'GET') req.query.blockchain = addBlockchainId(req.query.blockchain, blockchainImplementations); else if (Array.isArray(req.body)) { for (const element of req.body) { element.blockchain = addBlockchainId(element.blockchain, blockchainImplementations); } } else { req.body.blockchain = addBlockchainId(req.body.blockchain, blockchainImplementations); } next(); }; } ================================================ FILE: src/modules/http-client/implementation/middleware/rate-limiter-middleware.js ================================================ import rateLimiter from 'express-rate-limit'; export default (config) => rateLimiter({ windowMs: config.timeWindowSeconds * 1000, max: config.maxRequests, message: `Too many requests sent, maximum number of requests per minute is ${config.maxRequests}`, standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers legacyHeaders: false, // Disable the `X-RateLimit-*` headers }); ================================================ FILE: src/modules/http-client/implementation/middleware/request-validation-middleware.js ================================================ import { Validator } from 'jsonschema'; const v = new Validator(); function preValidateProperty(object, key, schema, options, ctx) { const value = object[key]; if (typeof value === 'undefined') return; // Test if the schema declares a type, but the type keyword fails validation if ( schema.type && v.attributes.type.call(v, value, schema, options, ctx.makeChild(schema, key)) ) { // If the type is "number" but the instance is not a number, cast it if (schema.type === 'number' && typeof value !== 'number') { // eslint-disable-next-line no-param-reassign object[key] = parseFloat(value); } } } export default function requestValidationMiddleware(requestSchema) { return (req, res, next) => { let result; if (req.method === 'GET') result = v.validate(req.query, requestSchema, { preValidateProperty }); else if (req.get('Content-Type') !== 'application/json') { res.status(401).send('Invalid header format'); return; } else result = v.validate(req.body, requestSchema); if (result.errors.length > 0) { res.status(400).json({ status: 'FAILED', errors: result.errors.map((e) => e.message), }); } else { next(); } }; } ================================================ FILE: src/modules/module-config-validation.js ================================================ import { REQUIRED_MODULES, TRIPLE_STORE_REPOSITORIES } from '../constants/constants.js'; class ModuleConfigValidation { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; } validateModule(name, config) { this.validateRequiredModule(name, config); const capitalizedName = name.charAt(0).toUpperCase() + name.slice(1); if (typeof this[`validate${capitalizedName}`] === 'function') { this[`validate${capitalizedName}`](config); } else { throw new Error(`Missing validation for ${capitalizedName}`); } } validateAutoUpdater() { return true; } validateBlockchain() { return true; } validateHttpClient() { return true; } validateNetwork() { return true; } validateRepository() { return true; } validateBlockchainEvents(config) { const occurences = {}; for (const implementation of Object.values(config.implementation)) { // eslint-disable-next-line no-continue if (!implementation.enabled) { continue; } if (implementation.config.blockchains.length === 0) { throw new Error( 'Blockchains must be specified in the blockchain events service config.', ); } if ( implementation.config.blockchains.length > Object.keys(implementation.config.rpcEndpoints).length ) { throw new Error('Missing RPC edpoints in the blockchain events service config.'); } if ( implementation.config.blockchains.length > Object.keys(implementation.config.hubContractAddress).length ) { throw new Error('Missing hub addresses in the blockchain events service config.'); } for (const blockchain of implementation.config.blockchains) { if (!occurences[blockchain]) { occurences[blockchain] = 0; } occurences[blockchain] += 1; if (occurences[blockchain] > 1) { throw new Error( `Exactly one blockchain events service for blockchain ${blockchain} needs to be defined.`, ); } if ( !implementation.config.rpcEndpoints[blockchain] || implementation.config.rpcEndpoints[blockchain].length === 0 ) { throw new Error( `RPC endpoint is not defined for blockchain: ${blockchain} in the blockchain events service config.`, ); } if (!implementation.config.hubContractAddress[blockchain]) { throw new Error( `Hub contract address is not defined for blockchain: ${blockchain} in the blockchain events service config.`, ); } } } } validateTripleStore(config) { const occurences = {}; for (const implementation of Object.values(config.implementation)) { // eslint-disable-next-line no-continue if (!implementation.enabled) { continue; } for (const repository in implementation.config.repositories) { if (!occurences[repository]) { occurences[repository] = 0; } occurences[repository] += 1; } } for (const repository of Object.values(TRIPLE_STORE_REPOSITORIES)) { if (occurences[repository] !== 1) { throw new Error( `Exactly one config for repository ${repository} needs to be defined.`, ); } } } validateValidation() { return true; } validateRequiredModule(moduleName, moduleConfig) { if ( !moduleConfig?.enabled || !Object.values(moduleConfig.implementation).filter( (implementationConfig) => implementationConfig.enabled, ).length ) { const message = `${moduleName} module not defined or enabled in configuration`; if (REQUIRED_MODULES.includes(moduleName)) { throw new Error(`${message} but it's required!`); } this.logger.warn(message); } } validateTelemetry() { return true; } } export default ModuleConfigValidation; ================================================ FILE: src/modules/network/implementation/libp2p-service.js ================================================ import appRootPath from 'app-root-path'; import libp2p from 'libp2p'; import KadDHT from 'libp2p-kad-dht'; import { join } from 'path'; import Bootstrap, { tag } from 'libp2p-bootstrap'; import { NOISE } from 'libp2p-noise'; import MPLEX from 'libp2p-mplex'; import TCP from 'libp2p-tcp'; import pipe from 'it-pipe'; import map from 'it-map'; import { encode, decode } from 'it-length-prefixed'; import { create as _create, createFromPrivKey, createFromB58String } from 'peer-id'; import { InMemoryRateLimiter } from 'rolling-rate-limiter'; import toobusy from 'toobusy-js'; import { mkdir, writeFile, readFile, stat } from 'fs/promises'; import ip from 'ip'; import { TimeoutController } from 'timeout-abort-controller'; import { NETWORK_API_RATE_LIMIT, NETWORK_API_SPAM_DETECTION, NETWORK_MESSAGE_TYPES, NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES, LIBP2P_KEY_DIRECTORY, LIBP2P_KEY_FILENAME, NODE_ENVIRONMENTS, BYTES_IN_MEGABYTE, } from '../../../constants/constants.js'; const devEnvironment = process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST; const initializationObject = { addresses: { listen: ['/ip4/0.0.0.0/tcp/9000'], }, modules: { transport: [TCP], streamMuxer: [MPLEX], connEncryption: [NOISE], dht: KadDHT, }, }; class Libp2pService { async initialize(config, logger) { this.config = config; this.logger = logger; initializationObject.peerRouting = this.config.peerRouting; const externalIp = ip.isV4Format(this.config.nat.externalIp) && ip.isPublic(this.config.nat.externalIp) ? this.config.nat.externalIp : undefined; if (this.config.nat.externalIp != null && externalIp == null) { this.logger.warn( `Invalid external ip defined in configuration: ${this.config.nat.externalIp}. External ip must be in V4 format, and public.`, ); } initializationObject.config = { dht: { enabled: true, ...this.config.dht, }, nat: { ...this.config.nat, externalIp, }, }; initializationObject.dialer = this.config.connectionManager; if (this.config.bootstrap.length > 0) { initializationObject.modules.peerDiscovery = [Bootstrap]; initializationObject.config.peerDiscovery = { autoDial: true, [tag]: { enabled: true, list: this.config.bootstrap, }, }; } initializationObject.addresses = { listen: [`/ip4/0.0.0.0/tcp/${this.config.port}`], announce: externalIp ? [`/ip4/${externalIp}/tcp/${this.config.port}`] : [], }; let id; if (!this.config.peerId) { if (!devEnvironment || !this.config.privateKey) { this.config.privateKey = await this.readPrivateKeyFromFile(); } if (!this.config.privateKey) { id = await _create({ bits: 1024, keyType: 'RSA' }); this.config.privateKey = id.toJSON().privKey; await this.savePrivateKeyInFile(this.config.privateKey); } else { id = await createFromPrivKey(this.config.privateKey); } this.config.peerId = id; } initializationObject.peerId = this.config.peerId; this._initializeRateLimiters(); this.sessions = {}; this.node = await libp2p.create(initializationObject); const peerId = this.node.peerId.toB58String(); this.config.id = peerId; } async start() { await this.node.start(); const port = parseInt(this.node.multiaddrs.toString().split('/')[4], 10); this.logger.info(`Network ID is ${this.config.id}, connection port is ${port}`); } async onPeerConnected(listener) { this.node.connectionManager.on('peer:connect', listener); } async savePrivateKeyInFile(privateKey) { const { fullPath, directoryPath } = this.getKeyPath(); await mkdir(directoryPath, { recursive: true }); await writeFile(fullPath, privateKey); } getKeyPath() { let directoryPath; if (!devEnvironment) { directoryPath = join( appRootPath.path, '..', this.config.appDataPath, LIBP2P_KEY_DIRECTORY, ); } else { directoryPath = join(appRootPath.path, this.config.appDataPath, LIBP2P_KEY_DIRECTORY); } const fullPath = join(directoryPath, LIBP2P_KEY_FILENAME); return { fullPath, directoryPath }; } async readPrivateKeyFromFile() { const keyPath = this.getKeyPath(); if (await this.fileExists(keyPath.fullPath)) { const key = (await readFile(keyPath.fullPath)).toString(); return key; } } async fileExists(filePath) { try { await stat(filePath); return true; } catch (e) { return false; } } _initializeRateLimiters() { const basicRateLimiter = new InMemoryRateLimiter({ interval: NETWORK_API_RATE_LIMIT.TIME_WINDOW_MILLS, maxInInterval: NETWORK_API_RATE_LIMIT.MAX_NUMBER, }); const spamDetection = new InMemoryRateLimiter({ interval: NETWORK_API_SPAM_DETECTION.TIME_WINDOW_MILLS, maxInInterval: NETWORK_API_SPAM_DETECTION.MAX_NUMBER, }); this.rateLimiter = { basicRateLimiter, spamDetection, }; this.blackList = {}; } getMultiaddrs() { return this.node.multiaddrs; } getProtocols(peerIdObject) { return this.node.peerStore.protoBook.get(peerIdObject); } getAddresses(peerIdObject) { return this.node.peerStore.addressBook.get(peerIdObject); } getPeers() { return this.node.connectionManager.connections; } getPeerId() { return this.node.peerId; } handleMessage(protocol, handler) { this.logger.info(`Enabling network protocol: ${protocol}`); this.node.handle(protocol, async (handlerProps) => { const { stream } = handlerProps; const peerIdString = handlerProps.connection.remotePeer.toB58String(); const { message, valid, busy } = await this._readMessageFromStream( stream, this.isRequestValid.bind(this), peerIdString, ); this.updateSessionStream(message.header.operationId, peerIdString, stream); if (!valid) { await this.sendMessageResponse( protocol, peerIdString, NETWORK_MESSAGE_TYPES.RESPONSES.NACK, message.header.operationId, { errorMessage: 'Invalid request message' }, ); this.removeCachedSession(message.header.operationId, peerIdString); } else if (busy) { await this.sendMessageResponse( protocol, peerIdString, NETWORK_MESSAGE_TYPES.RESPONSES.BUSY, message.header.operationId, {}, ); this.removeCachedSession(message.header.operationId, peerIdString); } else { this.logger.debug( `Receiving message from ${peerIdString} to ${this.config.id}: protocol: ${protocol}, messageType: ${message.header.messageType};`, ); await handler(message, peerIdString); } }); } updateSessionStream(operationId, peerIdString, stream) { this.logger.trace( `Storing new session stream for remotePeerId: ${peerIdString} with operation id: ${operationId}`, ); if (!this.sessions[peerIdString]) { this.sessions[peerIdString] = { [operationId]: { stream, }, }; } else if (!this.sessions[peerIdString][operationId]) { this.sessions[peerIdString][operationId] = { stream, }; } else { this.sessions[peerIdString][operationId] = { stream, }; } } getSessionStream(operationId, peerIdString) { if (this.sessions[peerIdString] && this.sessions[peerIdString][operationId]) { this.logger.trace( `Session found remotePeerId: ${peerIdString}, operation id: ${operationId}`, ); return this.sessions[peerIdString][operationId].stream; } return null; } createStreamMessage(message, operationId, messageType) { return { header: { messageType, operationId, }, data: message, }; } async sendMessage(protocol, peerIdString, messageType, operationId, message, timeout) { const nackMessage = { header: { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK }, data: { errorMessage: '', }, }; const peerIdObject = createFromB58String(peerIdString); const publicIp = (this.getAddresses(peerIdObject) ?? []) .map((addr) => addr.multiaddr) .filter((addr) => addr.isThinWaistAddress()) .map((addr) => addr.toString().split('/')) .filter((splittedAddr) => !ip.isPrivate(splittedAddr[2]))[0]?.[2]; this.logger.trace( `Dialing remotePeerId: ${peerIdString} with public ip: ${publicIp}: protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}`, ); let dialResult; let dialStart; let dialEnd; try { dialStart = Date.now(); dialResult = await this.node.dialProtocol(peerIdObject, protocol); dialEnd = Date.now(); } catch (error) { dialEnd = Date.now(); nackMessage.data.errorMessage = `Unable to dial peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ dialEnd - dialStart } ms. Error: ${error.message}`; return nackMessage; } this.logger.trace( `Created stream for peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ dialEnd - dialStart } ms.`, ); const { stream } = dialResult; this.updateSessionStream(operationId, peerIdString, stream); const streamMessage = this.createStreamMessage(message, operationId, messageType); this.logger.trace( `Sending message to ${peerIdString}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}`, ); let sendMessageStart; let sendMessageEnd; try { sendMessageStart = Date.now(); await this._sendMessageToStream(stream, streamMessage); sendMessageEnd = Date.now(); } catch (error) { sendMessageEnd = Date.now(); nackMessage.data.errorMessage = `Unable to send message to peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}, execution time: ${ sendMessageEnd - sendMessageStart } ms. Error: ${error.message}`; return nackMessage; } let readResponseStart; let readResponseEnd; let response; const abortSignalEventListener = async () => { stream.abort(); response = null; }; const timeoutController = new TimeoutController(timeout); try { readResponseStart = Date.now(); timeoutController.signal.addEventListener('abort', abortSignalEventListener, { once: true, }); response = await this._readMessageFromStream( stream, this.isResponseValid.bind(this), peerIdString, ); if (timeoutController.signal.aborted) { throw Error('Message timed out!'); } timeoutController.signal.removeEventListener('abort', abortSignalEventListener); timeoutController.clear(); readResponseEnd = Date.now(); } catch (error) { timeoutController.signal.removeEventListener('abort', abortSignalEventListener); timeoutController.clear(); readResponseEnd = Date.now(); nackMessage.data.errorMessage = `Unable to read response from peer ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, execution time: ${ readResponseEnd - readResponseStart } ms. Error: ${error.message}`; return nackMessage; } this.logger.trace( `Receiving response from ${peerIdString}. protocol: ${protocol}, messageType: ${ response.message?.header?.messageType }, operationId: ${operationId}, execution time: ${ readResponseEnd - readResponseStart } ms.`, ); if (!response.valid) { nackMessage.data.errorMessage = 'Invalid response'; return nackMessage; } return response.message; } async sendMessageResponse(protocol, peerIdString, messageType, operationId, message) { this.logger.debug( `Sending response from ${this.config.id} to ${peerIdString}: protocol: ${protocol}, messageType: ${messageType};`, ); const stream = this.getSessionStream(operationId, peerIdString); if (!stream) { throw Error(`Unable to find opened stream for remotePeerId: ${peerIdString}`); } const response = this.createStreamMessage(message, operationId, messageType); await this._sendMessageToStream(stream, response); } async _sendMessageToStream(stream, message) { const stringifiedHeader = JSON.stringify(message.header); const stringifiedData = JSON.stringify(message.data); const chunks = [stringifiedHeader]; const chunkSize = BYTES_IN_MEGABYTE; // 1 MB // split data into 1 MB chunks for (let i = 0; i < stringifiedData.length; i += chunkSize) { chunks.push(stringifiedData.slice(i, i + chunkSize)); } await pipe( chunks, // turn strings into buffers (source) => map(source, (string) => Buffer.from(string)), // Encode with length prefix (so receiving side knows how much data is coming) encode(), // Write to the stream (the sink) stream.sink, ); } async _readMessageFromStream(stream, isMessageValid, peerIdString) { return pipe( // Read from the stream (the source) stream.source, // Decode length-prefixed data decode(), // Turn buffers into strings (source) => map(source, (buf) => buf.toString()), // Sink function (source) => this.readMessageSink(source, isMessageValid, peerIdString), ); } async readMessageSink(source, isMessageValid, peerIdString) { const message = { header: { operationId: '' }, data: {} }; // we expect first buffer to be header const stringifiedHeader = (await source.next()).value; if (!stringifiedHeader?.length) { return { message, valid: false, busy: false }; } try { message.header = JSON.parse(stringifiedHeader); } catch (error) { // Return the same format as invalid request case return { message, valid: false, busy: false }; } // validate request / response if (!(await isMessageValid(message.header, peerIdString))) { return { message, valid: false }; } // business check if PROTOCOL_INIT message if ( message.header.messageType === NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_INIT && this.isBusy() ) { return { message, valid: true, busy: true }; } let stringifiedData = ''; // read data the data try { for await (const chunk of source) { stringifiedData += chunk; } message.data = JSON.parse(stringifiedData); } catch (error) { // If data parsing fails, return invalid message response return { message, valid: false, busy: false }; } return { message, valid: true, busy: false }; } async isRequestValid(header, peerIdString) { // filter spam requests if (await this.limitRequest(header, peerIdString)) return false; // header well formed if ( !header.operationId || !header.messageType || !Object.keys(NETWORK_MESSAGE_TYPES.REQUESTS).includes(header.messageType) ) return false; if (header.messageType === NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_INIT) { return true; } return this.sessionExists(peerIdString, header.operationId); } sessionExists() { return true; } async isResponseValid() { return true; } healthCheck() { // TODO: broadcast ping or sent msg to yourself const connectedNodes = this.node.connectionManager.size; if (connectedNodes > 0) return true; return false; } async limitRequest(header, peerIdString) { // if (header.sessionId && this.sessions.receiver[header.sessionId]) return false; if (this.blackList[peerIdString]) { const remainingMinutes = Math.floor( NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES - (Date.now() - this.blackList[peerIdString]) / (1000 * 60), ); if (remainingMinutes > 0) { this.logger.debug( `Blocking request from ${peerIdString}. Node is blacklisted for ${remainingMinutes} minutes.`, ); return true; } delete this.blackList[peerIdString]; } if (await this.rateLimiter.spamDetection.limit(peerIdString)) { this.blackList[peerIdString] = Date.now(); this.logger.debug( `Blocking request from ${peerIdString}. Spammer detected and blacklisted for ${NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES} minutes.`, ); return true; } if (await this.rateLimiter.basicRateLimiter.limit(peerIdString)) { this.logger.debug( `Blocking request from ${peerIdString}. Max number of requests exceeded.`, ); return true; } return false; } isBusy() { const distinctOperations = new Set(); for (const peerId in this.sessions) { for (const operationId in Object.keys(this.sessions[peerId])) { distinctOperations.add(operationId); } } return toobusy(); // || distinctOperations.size > constants.MAX_OPEN_SESSIONS; } getPrivateKey() { return this.config.privateKey; } getName() { return 'Libp2p'; } async findPeer(peerId) { return this.node.peerRouting.findPeer(createFromB58String(peerId)); } async dial(peerId) { return this.node.dial(createFromB58String(peerId)); } async getPeerInfo(peerId) { return this.node.peerStore.get(createFromB58String(peerId)); } removeCachedSession(operationId, peerIdString) { if (this.sessions[peerIdString]?.[operationId]?.stream) { this.sessions[peerIdString][operationId].stream.close(); delete this.sessions[peerIdString][operationId]; this.logger.trace( `Removed session for remotePeerId: ${peerIdString}, operationId: ${operationId}.`, ); } } } export default Libp2pService; ================================================ FILE: src/modules/network/network-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class NetworkModuleManager extends BaseModuleManager { getName() { return 'network'; } async start() { if (this.initialized) { return this.getImplementation().module.start(); } } async onPeerConnected(listener) { if (this.initialized) { return this.getImplementation().module.onPeerConnected(listener); } } getMultiaddrs() { if (this.initialized) { return this.getImplementation().module.getMultiaddrs(); } } getPeers() { if (this.initialized) { return this.getImplementation().module.getPeers(); } } async sendMessage(protocol, remotePeerId, messageType, operationId, message, timeout) { if (this.initialized) { return this.getImplementation().module.sendMessage( protocol, remotePeerId, messageType, operationId, message, timeout, ); } } async sendMessageResponse(protocol, remotePeerId, messageType, operationId, message) { if (this.initialized) { return this.getImplementation().module.sendMessageResponse( protocol, remotePeerId, messageType, operationId, message, ); } } handleMessage(protocol, handler, options) { if (this.initialized) { this.getImplementation().module.handleMessage(protocol, handler, options); } } getPeerId() { if (this.initialized) { return this.getImplementation().module.getPeerId(); } } async healthCheck() { if (this.initialized) { return this.getImplementation().module.healthCheck(); } } async findPeer(peerId) { if (this.initialized) { return this.getImplementation().module.findPeer(peerId); } } async dial(peerId) { if (this.initialized) { return this.getImplementation().module.dial(peerId); } } async getPeerInfo(peerId) { if (this.initialized) { return this.getImplementation().module.getPeerInfo(peerId); } } removeCachedSession(operationId, remotePeerId) { if (this.initialized) { this.getImplementation().module.removeCachedSession(operationId, remotePeerId); } } } export default NetworkModuleManager; ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20211117005500-create-commands.js ================================================ export const up = async ({ context: { queryInterface, Sequelize } }) => { await queryInterface.createTable('commands', { id: { allowNull: false, primaryKey: true, type: Sequelize.STRING, }, name: { type: Sequelize.STRING, allowNull: false, }, data: { type: Sequelize.JSON, allowNull: false, }, sequence: { type: Sequelize.JSON, allowNull: true, }, ready_at: { type: Sequelize.INTEGER, allowNull: false, }, delay: { type: Sequelize.INTEGER, allowNull: false, }, started_at: { type: Sequelize.INTEGER, allowNull: true, }, deadline_at: { type: Sequelize.INTEGER, allowNull: true, }, period: { type: Sequelize.INTEGER, allowNull: true, }, status: { type: Sequelize.STRING, allowNull: false, }, message: { type: Sequelize.TEXT('long'), allowNull: true, }, parent_id: { type: Sequelize.STRING, allowNull: true, }, retries: { type: Sequelize.INTEGER, allowNull: true, }, transactional: { type: Sequelize.BOOLEAN, allowNull: false, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); }; export const down = async ({ context: { queryInterface } }) => { await queryInterface.dropTable('commands'); }; ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20211117005504-create-operation_ids.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { return queryInterface.createTable('operation_ids', { operation_id: { allowNull: false, primaryKey: true, type: Sequelize.STRING, }, data: { allowNull: true, type: Sequelize.TEXT, }, status: { allowNull: false, type: Sequelize.STRING, }, timestamp: { type: Sequelize.BIGINT, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { return queryInterface.dropTable('operation_ids'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220620100000-create-publish.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('publish', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('publish'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220620100005-create-publish-response.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('publish_response', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, keyword: { allowNull: false, type: Sequelize.STRING, }, status: { allowNull: false, type: Sequelize.STRING, }, message: { allowNull: true, type: Sequelize.TEXT, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('publish_response'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220623125000-create-get.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('get', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('get'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220623125001-create-get-response.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('get_response', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, keyword: { allowNull: false, type: Sequelize.STRING, }, status: { allowNull: false, type: Sequelize.STRING, }, message: { allowNull: true, type: Sequelize.TEXT, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('get_response'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220624020509-create-event.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('event', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, name: { allowNull: false, type: Sequelize.STRING, }, timestamp: { allowNull: false, type: Sequelize.STRING, }, value1: { allowNull: true, type: Sequelize.STRING, }, value2: { allowNull: true, type: Sequelize.STRING, }, value3: { allowNull: true, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('event'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220624103229-create-ability.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('ability', { id: { allowNull: false, autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, name: { type: Sequelize.STRING, unique: true, }, created_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('ability'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220624103610-create-role.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('role', { id: { allowNull: false, autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, name: { type: Sequelize.STRING, unique: true, }, created_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('role'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220624103615-create-user.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('user', { id: { allowNull: false, autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, name: { type: Sequelize.STRING, unique: true, }, role_id: { type: Sequelize.INTEGER, references: { model: 'role', key: 'id', }, }, created_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('user'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220624103658-create-token.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('token', { id: { allowNull: false, primaryKey: true, type: Sequelize.STRING, }, name: { allowNull: false, type: Sequelize.STRING, unique: true, }, user_id: { type: Sequelize.INTEGER, references: { model: 'user', key: 'id', }, }, expires_at: { type: Sequelize.DATE, allowNull: true, }, revoked: { type: Sequelize.BOOLEAN, defaultValue: false, }, created_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('token'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220624113659-create-role-ability.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('role_ability', { id: { allowNull: false, autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, ability_id: { type: Sequelize.INTEGER, references: { model: 'ability', key: 'id', }, }, role_id: { type: Sequelize.INTEGER, references: { model: 'role', key: 'id', }, }, created_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: true, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('role_ability'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20220628113824-add-predefined-auth-entities.js ================================================ const routes = [ 'PUBLISH', 'PROVISION', 'UPDATE', 'GET', 'SEARCH', 'SEARCH_ASSERTION', 'QUERY', 'PROOFS', 'OPERATION_RESULT', 'INFO', ]; export async function up({ context: { queryInterface } }) { const transaction = await queryInterface.sequelize.transaction(); try { await queryInterface.bulkInsert( 'ability', routes.map((r) => ({ name: r })), { transaction, }, ); const [abilities] = await queryInterface.sequelize.query('SELECT id from ability', { transaction, }); await queryInterface.bulkInsert( 'role', [ { name: 'ADMIN', }, ], { transaction, }, ); const [[role]] = await queryInterface.sequelize.query( "SELECT id from role where name='ADMIN'", { transaction, }, ); const roleAbilities = abilities.map((e) => ({ ability_id: e.id, role_id: role.id, })); await queryInterface.bulkInsert('role_ability', roleAbilities, { transaction }); await queryInterface.bulkInsert( 'user', [ { name: 'node-runner', role_id: role.id, }, ], { transaction }, ); transaction.commit(); } catch (e) { transaction.rollback(); throw e; } } export async function down({ context: { queryInterface } }) { queryInterface.sequelize.query('TRUNCATE TABLE role_ability;'); queryInterface.sequelize.query('TRUNCATE TABLE role;'); queryInterface.sequelize.query('TRUNCATE TABLE ability;'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20221025120253-create-blockchain-event.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('blockchain_event', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, contract: { type: Sequelize.STRING, allowNull: false, }, blockchain_id: { allowNull: false, type: Sequelize.STRING, }, event: { allowNull: false, type: Sequelize.STRING, }, data: { allowNull: false, type: Sequelize.TEXT('long'), }, block: { allowNull: false, type: Sequelize.INTEGER, }, processed: { allowNull: false, type: Sequelize.BOOLEAN, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('blockchain_event'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20221025212800-create-shard.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('shard', { peer_id: { type: Sequelize.STRING, primaryKey: true, }, blockchain_id: { type: Sequelize.STRING, primaryKey: true, }, ask: { type: Sequelize.INTEGER, allowNull: false, }, stake: { type: Sequelize.INTEGER, allowNull: false, }, last_seen: { type: Sequelize.DATE, allowNull: false, defaultValue: new Date(0), }, last_dialed: { type: Sequelize.DATE, allowNull: false, defaultValue: new Date(0), }, sha256: { type: Sequelize.STRING, allowNull: false, }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('shard'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20221028125900-create-blockchain.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('blockchain', { blockchain_id: { type: Sequelize.STRING, primaryKey: true, }, contract: { type: Sequelize.STRING, primaryKey: true, }, last_checked_block: { type: Sequelize.BIGINT, allowNull: false, defaultValue: -1, }, last_checked_timestamp: { type: Sequelize.BIGINT, allowNull: false, defaultValue: 0, }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('blockchain'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20221114115524-update-publish-add-agreement-data.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.addColumn('publish', 'agreementId', { type: Sequelize.STRING, }); await queryInterface.addColumn('publish', 'agreementStatus', { type: Sequelize.STRING, }); } export async function down({ context: { queryInterface } }) { await queryInterface.removeColumn('publish', 'agreementId'); await queryInterface.removeColumn('publish', 'agreementStatus'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20221206183634-update-shard-types.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('shard', 'ask', { type: Sequelize.STRING, }); await queryInterface.changeColumn('shard', 'stake', { type: Sequelize.STRING, }); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('shard', 'ask', { type: Sequelize.INTEGER, }); await queryInterface.changeColumn('shard', 'ask', { type: Sequelize.INTEGER, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20221214110050-update-commands-types.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('commands', 'ready_at', { type: Sequelize.BIGINT, }); await queryInterface.changeColumn('commands', 'delay', { type: Sequelize.BIGINT, }); await queryInterface.changeColumn('commands', 'started_at', { type: Sequelize.BIGINT, }); await queryInterface.changeColumn('commands', 'deadline_at', { type: Sequelize.BIGINT, }); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('commands', 'ready_at', { type: Sequelize.INTEGER, }); await queryInterface.changeColumn('commands', 'delay', { type: Sequelize.INTEGER, }); await queryInterface.changeColumn('commands', 'started_at', { type: Sequelize.INTEGER, }); await queryInterface.changeColumn('commands', 'deadline_at', { type: Sequelize.INTEGER, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20221215130500-update-event-types.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('event', 'value1', { type: Sequelize.TEXT, }); await queryInterface.changeColumn('event', 'value2', { type: Sequelize.TEXT, }); await queryInterface.changeColumn('event', 'value3', { type: Sequelize.TEXT, }); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('event', 'value1', { type: Sequelize.STRING, }); await queryInterface.changeColumn('event', 'value2', { type: Sequelize.STRING, }); await queryInterface.changeColumn('event', 'value3', { type: Sequelize.STRING, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20230216112400-add-abilities.js ================================================ const newRoutes = ['BID-SUGGESTION', 'LOCAL-STORE']; async function getRoleAbilities(names, queryInterface, transaction) { const [abilities] = await queryInterface.sequelize.query( `SELECT id from ability where name IN (${names.map((name) => `'${name}'`).join(', ')})`, { transaction, }, ); const [[role]] = await queryInterface.sequelize.query( "SELECT id from role where name='ADMIN'", { transaction, }, ); return abilities.map((ability) => ({ ability_id: ability.id, role_id: role.id, })); } async function removeAbilities(names, queryInterface, transaction) { await queryInterface.bulkDelete( 'role_ability', await getRoleAbilities(names, queryInterface, transaction), { transaction }, ); await queryInterface.bulkDelete( 'ability', names.map((name) => ({ name })), { transaction }, ); } async function addAbilities(names, queryInterface, transaction) { await queryInterface.bulkInsert( 'ability', names.map((name) => ({ name })), { transaction }, ); await queryInterface.bulkInsert( 'role_ability', await getRoleAbilities(names, queryInterface, transaction), { transaction }, ); } export async function up({ context: { queryInterface } }) { const transaction = await queryInterface.sequelize.transaction(); try { await addAbilities(newRoutes, queryInterface, transaction); transaction.commit(); } catch (e) { transaction.rollback(); throw e; } } export async function down({ context: { queryInterface } }) { const transaction = await queryInterface.sequelize.transaction(); try { await removeAbilities(newRoutes, queryInterface, transaction); transaction.commit(); } catch (e) { transaction.rollback(); throw e; } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20230227094500-create-update.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('update', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('update'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20230303131200-update-publish-remove-agreement-data.js ================================================ const columns = ['agreementId', 'agreementStatus']; export async function up({ context: { queryInterface } }, logger) { const transaction = await queryInterface.sequelize.transaction(); try { await Promise.all( columns.map((column) => queryInterface.removeColumn('publish', column, { transaction }).catch((error) => { logger.warn(`Error removing column: ${column}: ${error.message}`); }), ), ); await transaction.commit(); } catch (error) { await transaction.rollback(); throw error; } } export async function down({ context: { queryInterface, Sequelize } }) { const transaction = await queryInterface.sequelize.transaction(); try { await Promise.all( columns.map((column) => queryInterface.addColumn( 'publish', column, { type: Sequelize.STRING, }, { transaction }, ), ), ); await transaction.commit(); } catch (error) { await transaction.rollback(); throw error; } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20230303131400-create-update-response.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('update_response', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, keyword: { allowNull: false, type: Sequelize.STRING, }, status: { allowNull: false, type: Sequelize.STRING, }, message: { allowNull: true, type: Sequelize.TEXT, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('update_response'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20230413194400-update-command-period-type.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('commands', 'period', { type: Sequelize.BIGINT, }); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('commands', 'period', { type: Sequelize.BIGINT, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20230419140000-create-service-agreements.js ================================================ export const up = async ({ context: { queryInterface, Sequelize } }) => { await queryInterface.createTable('service_agreement', { blockchain_id: { type: Sequelize.STRING, allowNull: false, }, asset_storage_contract_address: { type: Sequelize.STRING(42), allowNull: false, }, token_id: { type: Sequelize.INTEGER.UNSIGNED, allowNull: false, }, agreement_id: { type: Sequelize.STRING, primaryKey: true, }, start_time: { type: Sequelize.INTEGER.UNSIGNED, allowNull: false, }, epochs_number: { type: Sequelize.SMALLINT.UNSIGNED, allowNull: false, }, epoch_length: { type: Sequelize.INTEGER.UNSIGNED, allowNull: false, }, score_function_id: { type: Sequelize.TINYINT.UNSIGNED, allowNull: false, }, state_index: { type: Sequelize.SMALLINT.UNSIGNED, allowNull: false, }, assertion_id: { type: Sequelize.STRING, allowNull: false, }, hash_function_id: { type: Sequelize.TINYINT.UNSIGNED, allowNull: false, }, keyword: { type: Sequelize.STRING, allowNull: false, }, proof_window_offset_perc: { type: Sequelize.TINYINT.UNSIGNED, allowNull: false, }, last_commit_epoch: { type: Sequelize.SMALLINT.UNSIGNED, }, last_proof_epoch: { type: Sequelize.SMALLINT.UNSIGNED, }, }); }; export const down = async ({ context: { queryInterface } }) => { await queryInterface.dropTable('service_agreement'); }; ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20230502110300-add-blockchain-event-index.js ================================================ export async function up({ context: { queryInterface } }) { await queryInterface.addIndex('blockchain_event', ['processed'], { name: 'idx_blockchain_event_processed', }); } export async function down({ context: { queryInterface } }) { await queryInterface.removeIndex('blockchain_event', 'idx_blockchain_event_processed'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20231201140100-event-add-blockchain-id.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.addColumn('event', 'blockchain_id', { type: Sequelize.STRING, }); } export async function down({ context: { queryInterface } }) { await queryInterface.removeColumn('event', 'blockchain_id'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20231221131300-update-abilities.js ================================================ const newRoutes = [ 'V0/PUBLISH', 'V0/UPDATE', 'V0/GET', 'V0/QUERY', 'V0/OPERATION_RESULT', 'V0/INFO', 'V0/BID-SUGGESTION', 'V0/LOCAL-STORE', ]; const outdatedRoutes = ['PROVISION', 'SEARCH', 'SEARCH_ASSERTION', 'PROOFS']; async function getAbilityIds(names, queryInterface, transaction) { const [abilities] = await queryInterface.sequelize.query( `SELECT id FROM ability WHERE name IN (${names.map((name) => `'${name}'`).join(', ')})`, { transaction }, ); return abilities.map((ability) => ability.id); } async function getRoleIds(queryInterface, transaction) { const [roles] = await queryInterface.sequelize.query( 'SELECT id FROM role WHERE name IS NOT NULL;', { transaction, }, ); return roles.map((role) => role.id); } async function getRoleAbilities(names, queryInterface, transaction) { const abilityIds = await getAbilityIds(names, queryInterface, transaction); const roleIds = await getRoleIds(queryInterface, transaction); return roleIds.flatMap((roleId) => abilityIds.map((abilityId) => ({ ability_id: abilityId, role_id: roleId, })), ); } async function removeAbilities(names, queryInterface, transaction) { const roleIds = await getRoleIds(queryInterface, transaction); const abilityIds = await getAbilityIds(names, queryInterface, transaction); await queryInterface.bulkDelete( 'role_ability', { role_id: roleIds, ability_id: abilityIds, }, { transaction }, ); await queryInterface.bulkDelete('ability', { id: abilityIds }, { transaction }); } async function addAbilities(names, queryInterface, transaction) { await queryInterface.bulkInsert( 'ability', names.map((name) => ({ name })), { transaction }, ); await queryInterface.bulkInsert( 'role_ability', await getRoleAbilities(names, queryInterface, transaction), { transaction }, ); } export async function up({ context: { queryInterface } }) { const transaction = await queryInterface.sequelize.transaction(); try { await addAbilities(newRoutes, queryInterface, transaction); await removeAbilities(outdatedRoutes, queryInterface, transaction); transaction.commit(); } catch (e) { transaction.rollback(); throw e; } } export async function down({ context: { queryInterface } }) { const transaction = await queryInterface.sequelize.transaction(); try { await removeAbilities(newRoutes, queryInterface, transaction); await addAbilities(outdatedRoutes, queryInterface, transaction); transaction.commit(); } catch (e) { transaction.rollback(); throw e; } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20233010122500-update-blockchain-id.js ================================================ const CHAIN_IDS = { development: 31337, test: 31337, devnet: 2160, testnet: 20430, mainnet: 2043, }; const chainId = CHAIN_IDS[process.env.NODE_ENV]; export async function up({ context: { queryInterface } }) { await queryInterface.sequelize.query(` update shard set blockchain_id='otp:${chainId}' `); await queryInterface.sequelize.query(` update service_agreement set blockchain_id='otp:${chainId}' `); await queryInterface.sequelize.query(` update blockchain_event set blockchain_id='otp:${chainId}' `); await queryInterface.sequelize.query(` update blockchain set blockchain_id='otp:${chainId}' `); } export async function down({ context: { queryInterface } }) { await queryInterface.sequelize.query(` update shard set blockchain_id='otp' `); await queryInterface.sequelize.query(` update service_agreement set blockchain_id='otp' `); await queryInterface.sequelize.query(` update blockchain_event set blockchain_id='otp' `); await queryInterface.sequelize.query(` update blockchain set blockchain_id='otp' `); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20233011121700-remove-blockchain-info.js ================================================ const chiadoBlockchainId = 'gnosis:10200'; export async function up({ context: { queryInterface } }) { await queryInterface.sequelize.query(` delete from blockchain where blockchain_id='${chiadoBlockchainId}' `); } // eslint-disable-next-line no-empty-function export async function down() {} ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { const tableInfo = await queryInterface.describeTable('shard'); if (!tableInfo.sha256_blob) { await queryInterface.addColumn('shard', 'sha256_blob', { type: Sequelize.BLOB, }); } } export async function down({ context: { queryInterface } }) { await queryInterface.removeColumn('shard', 'sha256_blob'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240201100000-remove-sha256Blob.js ================================================ export async function up({ context: { queryInterface } }) { const tableInfo = await queryInterface.describeTable('shard'); if (tableInfo.sha256_blob) { await queryInterface.removeColumn('shard', 'sha256_blob'); } } export async function down({ context: { queryInterface, Sequelize } }) { const tableInfo = await queryInterface.describeTable('shard'); if (!tableInfo.sha256_blob) { await queryInterface.addColumn('shard', 'sha256_blob', { type: Sequelize.BLOB, }); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240221162000-add-service-agreement-data-source.js ================================================ import { SERVICE_AGREEMENT_SOURCES } from '../../../../../constants/constants.js'; export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.addColumn('service_agreement', 'data_source', { type: Sequelize.ENUM(...Object.values(SERVICE_AGREEMENT_SOURCES)), }); } export async function down({ context: { queryInterface } }) { await queryInterface.removeColumn('service_agreement', 'data_source'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240429083058-create-paranet.js ================================================ export const up = async ({ context: { queryInterface, Sequelize } }) => { await queryInterface.createTable('paranet', { id: { allowNull: false, autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, name: { type: Sequelize.STRING, }, blockchain_id: { type: Sequelize.STRING, primaryKey: true, }, description: { type: Sequelize.STRING, }, paranet_id: { type: Sequelize.STRING, }, ka_count: { type: Sequelize.INTEGER, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); }; export const down = async ({ context: { queryInterface } }) => { await queryInterface.dropTable('Paranet'); }; ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240529070000-create-missed-paranet-asset.js ================================================ export const up = async ({ context: { queryInterface, Sequelize } }) => { await queryInterface.createTable('missed_paranet_asset', { id: { autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, blockchain_id: { allowNull: false, type: Sequelize.STRING, }, ual: { allowNull: false, type: Sequelize.STRING, }, paranet_ual: { allowNull: false, type: Sequelize.STRING, }, knowledge_asset_id: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); }; export const down = async ({ context: { queryInterface } }) => { await queryInterface.dropTable('missed_paranet_asset'); }; ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240923195000-create-publish-paranet.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('publish_paranet', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('publish_paranet'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240924161700-create-paranet-synced-asset.js ================================================ export const up = async ({ context: { queryInterface, Sequelize } }) => { await queryInterface.createTable('paranet_synced_asset', { id: { autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, blockchain_id: { allowNull: false, type: Sequelize.STRING, }, ual: { allowNull: false, type: Sequelize.STRING, }, paranet_ual: { allowNull: false, type: Sequelize.STRING, }, public_assertion_id: { allowNull: true, type: Sequelize.STRING, }, private_assertion_id: { allowNull: false, type: Sequelize.STRING, }, sender: { allowNull: false, type: Sequelize.STRING, }, transaction_hash: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); const [triggerInsertExists] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS trigger_exists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'before_insert_paranet_synced_asset'; `); if (triggerInsertExists[0].trigger_exists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER before_insert_paranet_synced_asset BEFORE INSERT ON paranet_synced_asset FOR EACH ROW BEGIN SET NEW.created_at = NOW(); END; `); } const [triggerUpdateExists] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS trigger_exists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'before_update_paranet_synced_asset'; `); if (triggerUpdateExists[0].trigger_exists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER before_update_paranet_synced_asset BEFORE UPDATE ON paranet_synced_asset FOR EACH ROW BEGIN SET NEW.updated_at = NOW(); END; `); } const indexes = [ { name: 'idx_paranet_ual_created_at', columns: '(paranet_ual, created_at)' }, { name: 'idx_sender', columns: '(sender)' }, { name: 'idx_paranet_ual_unique', columns: '(paranet_ual)' }, ]; for (const index of indexes) { // eslint-disable-next-line no-await-in-loop const [indexExists] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS index_exists FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = 'paranet_synced_asset' AND index_name = '${index.name}'; `); if (indexExists[0].index_exists === 0) { // eslint-disable-next-line no-await-in-loop await queryInterface.sequelize.query(` CREATE INDEX ${index.name} ON paranet_synced_asset ${index.columns}; `); } } }; export const down = async ({ context: { queryInterface } }) => { await queryInterface.dropTable('paranet_synced_asset'); await queryInterface.sequelize.query(` DROP TRIGGER IF EXISTS before_insert_paranet_synced_asset; `); await queryInterface.sequelize.query(` DROP TRIGGER IF EXISTS before_update_paranet_synced_asset; `); }; ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240924205500-create-publish-paranet-response.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('publish_paranet_response', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, keyword: { allowNull: false, type: Sequelize.STRING, }, status: { allowNull: false, type: Sequelize.STRING, }, message: { allowNull: true, type: Sequelize.TEXT, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('publish_paranet_response'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240927110000-change-paranet-synced-asset-nullable-assertions.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('paranet_synced_asset', 'public_assertion_id', { type: Sequelize.STRING, allowNull: false, }); await queryInterface.changeColumn('paranet_synced_asset', 'private_assertion_id', { type: Sequelize.STRING, allowNull: true, }); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('paranet_synced_asset', 'public_assertion_id', { type: Sequelize.STRING, allowNull: true, }); await queryInterface.changeColumn('paranet_synced_asset', 'private_assertion_id', { type: Sequelize.STRING, allowNull: false, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20240930113000-add-error-message.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.addColumn('missed_paranet_asset', 'error_message', { type: Sequelize.TEXT, allowNull: true, }); } export async function down({ context: { queryInterface } }) { await queryInterface.removeColumn('missed_paranet_asset', 'error_message'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241011112100-remove-knowledge-asset-id.js ================================================ export async function up({ context: { queryInterface } }) { await queryInterface.removeColumn('missed_paranet_asset', 'knowledge_asset_id'); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.addColumn('missed_paranet_asset', 'knowledge_asset_id', { type: Sequelize.STRING, allowNull: false, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241014164500-paranet-synced-asset-optional-fileds.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('paranet_synced_asset', 'sender', { type: Sequelize.STRING, allowNull: true, }); await queryInterface.changeColumn('paranet_synced_asset', 'transaction_hash', { type: Sequelize.STRING, allowNull: true, }); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('paranet_synced_asset', 'sender', { type: Sequelize.STRING, allowNull: false, }); await queryInterface.changeColumn('paranet_synced_asset', 'transaction_hash', { type: Sequelize.STRING, allowNull: false, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241023170300-add-synced-data-source.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.addColumn('paranet_synced_asset', 'data_source', { type: Sequelize.TEXT, allowNull: true, }); } export async function down({ context: { queryInterface } }) { await queryInterface.removeColumn('paranet_synced_asset', 'data_source'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241105150000-change-data-source-col-type-in-paranet-synced-asset.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('paranet_synced_asset', 'data_source', { type: Sequelize.ENUM('sync', 'local_store'), allowNull: true, }); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.changeColumn('paranet_synced_asset', 'data_source', { type: Sequelize.TEXT, allowNull: true, }); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241105160000-add-indexes-to-tables.js ================================================ export async function up({ context: { queryInterface } }) { const indexes = [ { table: 'shard', column: ['blockchain_id'], name: 'shard_blockchain_id_index' }, { table: 'shard', column: ['last_dialed'], name: 'last_dialed_index' }, { table: 'paranet_synced_asset', column: ['ual'], name: 'paranet_synced_asset_ual_index' }, { table: 'paranet_synced_asset', column: ['paranet_ual', 'data_source'], name: 'paranet_ual_data_source_index', }, { table: 'paranet', column: ['blockchain_id', 'paranet_id'], name: 'blockchain_id_paranet_id_index', }, { table: 'missed_paranet_asset', column: ['paranet_ual'], name: 'paranet_ual_index' }, { table: 'missed_paranet_asset', column: ['ual'], name: 'missed_paranet_asset_ual_index' }, { table: 'event', column: ['name', 'timestamp'], name: 'name_timestamp_index' }, { table: 'event', column: ['operation_id'], name: 'event_operation_id_index' }, { table: 'commands', column: ['name', 'status'], name: 'name_status_index' }, { table: 'commands', column: ['status', 'started_at'], name: 'status_started_at_index' }, { table: 'get', column: ['operation_id'], name: 'get_operation_id_index' }, { table: 'publish', column: ['operation_id'], name: 'publish_operation_id_index' }, { table: 'update', column: ['operation_id'], name: 'update_operation_id_index' }, { table: 'publish_paranet', column: ['operation_id'], name: 'publish_paranet_operation_id_index', }, { table: 'get', column: ['created_at'], name: 'get_created_at_index' }, { table: 'publish', column: ['created_at'], name: 'publish_created_at_index' }, { table: 'update', column: ['created_at'], name: 'update_created_at_index' }, { table: 'publish_paranet', column: ['created_at'], name: 'publish_paranet_created_at_index', }, { table: 'get_response', column: ['operation_id'], name: 'get_response_operation_id_index', }, { table: 'publish_response', column: ['operation_id'], name: 'operation_id_index' }, { table: 'update_response', column: ['operation_id'], name: 'update_response_operation_id_index', }, { table: 'publish_paranet_response', column: ['operation_id'], name: 'publish_paranet_response_operation_id_index', }, { table: 'get_response', column: ['created_at'], name: 'get_response_created_at_index' }, { table: 'publish_response', column: ['created_at'], name: 'publish_response_created_at_index', }, { table: 'update_response', column: ['created_at'], name: 'update_response_created_at_index', }, { table: 'publish_paranet_response', column: ['created_at'], name: 'publish_paranet_response_created_at_index', }, { table: 'blockchain', column: ['contract'], name: 'contract_index' }, ]; for (const index of indexes) { const { table, column, name } = index; // eslint-disable-next-line no-await-in-loop const [indexExists] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS index_exists FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = '${table}' AND index_name = '${name}'; `); if (indexExists[0].index_exists === 0) { // eslint-disable-next-line no-await-in-loop await queryInterface.sequelize.query(` CREATE INDEX \`${name}\` ON \`${table}\` (${column.map((col) => `\`${col}\``).join(', ')}); `); } } } export async function down({ context: { queryInterface } }) { const indexes = [ { table: 'shard', name: 'shard_blockchain_id_index' }, { table: 'shard', name: 'last_dialed_index' }, { table: 'paranet_synced_asset', name: 'paranet_synced_asset_ual_index' }, { table: 'paranet_synced_asset', name: 'paranet_ual_data_source_index' }, { table: 'paranet', name: 'blockchain_id_paranet_id_index' }, { table: 'missed_paranet_asset', name: 'paranet_ual_index' }, { table: 'missed_paranet_asset', name: 'missed_paranet_asset_ual_index' }, { table: 'event', name: 'name_timestamp_index' }, { table: 'event', name: 'event_operation_id_index' }, { table: 'commands', name: 'name_status_index' }, { table: 'commands', name: 'status_started_at_index' }, { table: 'get', name: 'get_operation_id_index' }, { table: 'publish', name: 'publish_operation_id_index' }, { table: 'update', name: 'update_operation_id_index' }, { table: 'publish_paranet', name: 'publish_paranet_operation_id_index' }, { table: 'get', name: 'get_created_at_index' }, { table: 'publish', name: 'publish_created_at_index' }, { table: 'update', name: 'update_created_at_index' }, { table: 'publish_paranet', name: 'publish_paranet_created_at_index' }, { table: 'get_response', name: 'get_response_operation_id_index' }, { table: 'publish_response', name: 'publish_response_operation_id_index' }, { table: 'update_response', name: 'update_response_operation_id_index' }, { table: 'publish_paranet_response', name: 'publish_paranet_response_operation_id_index', }, { table: 'get_response', name: 'get_response_created_at_index' }, { table: 'publish_response', name: 'publish_response_created_at_index' }, { table: 'update_response', name: 'update_response_created_at_index' }, { table: 'publish_paranet_response', name: 'publish_paranet_response_created_at_index', }, { table: 'blockchain', name: 'contract_index' }, ]; for (const { table, name } of indexes) { // eslint-disable-next-line no-await-in-loop const [indexExists] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS index_exists FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = '${table}' AND index_name = '${name}'; `); if (indexExists[0].index_exists > 0) { // eslint-disable-next-line no-await-in-loop await queryInterface.removeIndex(table, name); } } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241125151200-rename-keyword-column-to-datasetroot-in-responses.js ================================================ export async function up({ context: { queryInterface } }) { // Helper function to check if a column exists async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('publish_response', 'keyword')) { await queryInterface.renameColumn('publish_response', 'keyword', 'dataset_root'); } if (await columnExists('get_response', 'keyword')) { await queryInterface.renameColumn('get_response', 'keyword', 'dataset_root'); } } export async function down({ context: { queryInterface } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('publish_response', 'dataset_root')) { await queryInterface.renameColumn('publish_response', 'dataset_root', 'keyword'); } if (await columnExists('get_response', 'dataset_root')) { await queryInterface.renameColumn('get_response', 'dataset_root', 'keyword'); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241126114400-add-commands-priority.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (!(await columnExists('commands', 'priority'))) { await queryInterface.addColumn('commands', 'priority', { type: Sequelize.BIGINT, }); } } export async function down({ context: { queryInterface } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('commands', 'priority')) { await queryInterface.removeColumn('commands', 'priority'); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241129120000-add-commands-is_blocking.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (!(await columnExists('commands', 'is_blocking'))) { await queryInterface.addColumn('commands', 'is_blocking', { type: Sequelize.BOOLEAN, }); } } export async function down({ context: { queryInterface } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('commands', 'is_blocking')) { await queryInterface.removeColumn('commands', 'is_blocking'); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241129125800-remove-datasetroot-response-table.js ================================================ export async function up({ context: { queryInterface } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('publish_response', 'dataset_root')) { await queryInterface.removeColumn('publish_response', 'dataset_root'); } if (await columnExists('get_response', 'dataset_root')) { await queryInterface.removeColumn('get_response', 'dataset_root'); } } export async function down({ context: { queryInterface, Sequelize } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (!(await columnExists('publish_response', 'dataset_root'))) { await queryInterface.addColumn('publish_response', 'dataset_root', { type: Sequelize.STRING, allowNull: false, }); } if (!(await columnExists('get_response', 'dataset_root'))) { await queryInterface.addColumn('get_response', 'dataset_root', { type: Sequelize.STRING, allowNull: false, }); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241201152000-update-blockchain-events.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { // Helper function to check if a column exists async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('blockchain_event', 'blockchain_id')) { await queryInterface.renameColumn('blockchain_event', 'blockchain_id', 'blockchain'); } if (await columnExists('blockchain_event', 'block')) { await queryInterface.changeColumn('blockchain_event', 'block', { type: Sequelize.BIGINT, }); await queryInterface.renameColumn('blockchain_event', 'block', 'block_number'); } if (!(await columnExists('blockchain_event', 'transaction_index'))) { await queryInterface.addColumn('blockchain_event', 'transaction_index', { type: Sequelize.BIGINT, }); } if (!(await columnExists('blockchain_event', 'log_index'))) { await queryInterface.addColumn('blockchain_event', 'log_index', { type: Sequelize.BIGINT, }); } if (!(await columnExists('blockchain_event', 'contract_address'))) { await queryInterface.addColumn('blockchain_event', 'contract_address', { type: Sequelize.STRING, }); } } export async function down({ context: { queryInterface, Sequelize } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('blockchain_event', 'block_number')) { await queryInterface.renameColumn('blockchain_event', 'block_number', 'block'); } if (await columnExists('blockchain_event', 'block')) { await queryInterface.changeColumn('blockchain_event', 'block', { type: Sequelize.INTEGER, }); } if (await columnExists('blockchain_event', 'blockchain')) { await queryInterface.renameColumn('blockchain_event', 'blockchain', 'blockchain_id'); } if (await columnExists('blockchain_event', 'transaction_index')) { await queryInterface.removeColumn('blockchain_event', 'transaction_index'); } if (await columnExists('blockchain_event', 'log_index')) { await queryInterface.removeColumn('blockchain_event', 'log_index'); } if (await columnExists('blockchain_event', 'contract_address')) { await queryInterface.removeColumn('blockchain_event', 'contract_address'); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241202214500-update-blockchain-table.js ================================================ export async function up({ context: { queryInterface } }) { const tableInfo = await queryInterface.describeTable('blockchain'); if (tableInfo.blockchain_id) { await queryInterface.renameColumn('blockchain', 'blockchain_id', 'blockchain'); } await queryInterface.sequelize.query(` DELETE t1 FROM blockchain t1 JOIN blockchain t2 ON t1.blockchain = t2.blockchain AND ( t1.last_checked_block > t2.last_checked_block OR (t1.last_checked_block = t2.last_checked_block AND t1.last_checked_timestamp > t2.last_checked_timestamp) ); `); await queryInterface.sequelize.query(` ALTER TABLE blockchain DROP PRIMARY KEY, ADD PRIMARY KEY (blockchain); `); await queryInterface.removeColumn('blockchain', 'contract'); } export async function down({ context: { queryInterface, Sequelize } }) { await queryInterface.renameColumn('blockchain', 'blockchain', 'blockchain_id'); await queryInterface.addColumn('blockchain', 'contract', { type: Sequelize.STRING, allowNull: false, }); await queryInterface.sequelize.query(` ALTER TABLE blockchain DROP PRIMARY KEY, ADD PRIMARY KEY (blockchain_id, contract); `); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241203125000-create-finality.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('finality', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('finality'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241203125001-create-finality-response.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('finality_response', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, message: { allowNull: true, type: Sequelize.TEXT, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('finality_response'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241211204400-rename-ask.js ================================================ export async function up({ context: { queryInterface } }) { async function tableExists(table) { const [results] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS table_exists FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = '${table}'; `); return results[0].table_exists > 0; } if (await tableExists('finality')) { await queryInterface.renameTable('finality', 'ask'); } if (await tableExists('finality_response')) { await queryInterface.renameTable('finality_response', 'ask_response'); } } export async function down({ context: { queryInterface } }) { async function tableExists(table) { const [results] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS table_exists FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = '${table}'; `); return results[0].table_exists > 0; } if (await tableExists('ask')) { await queryInterface.renameTable('ask', 'finality'); } if (await tableExists('ask_response')) { await queryInterface.renameTable('ask_response', 'finality_response'); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241211205400-create-finality-response.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('finality_response', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, message: { allowNull: true, type: Sequelize.TEXT, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('finality_response'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241211205400-create-finality-status.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('finality_status', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.STRING, allowNull: false, }, ual: { type: Sequelize.STRING, }, peer_id: { type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); await queryInterface.addConstraint('finality_status', { fields: ['ual', 'peer_id'], type: 'unique', }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('finality_status'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241211205400-create-finality.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('finality', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('finality'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241212122200-add-min-acks-reached-column.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.addColumn('operation_ids', 'min_acks_reached', { type: Sequelize.BOOLEAN, }); } export async function down({ context: { queryInterface } }) { await queryInterface.removeColumn('operation_ids', 'min_acks_reached'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241215122200-create-paranet-kc.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('paranet_kc', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, blockchain_id: { type: Sequelize.STRING, allowNull: false, }, ual: { type: Sequelize.STRING, allowNull: false, }, paranet_ual: { type: Sequelize.STRING, allowNull: false, }, error_message: { type: Sequelize.TEXT, allowNull: true, }, is_synced: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false, }, retries: { allowNull: false, type: Sequelize.INTEGER, defaultValue: 0, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); const [[{ constraintExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS constraintExists FROM information_schema.TABLE_CONSTRAINTS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'paranet_kc' AND CONSTRAINT_NAME = 'paranet_kc_ual_paranet_ual_uk'; `); if (!constraintExists) { await queryInterface.addConstraint('paranet_kc', { fields: ['ual', 'paranet_ual'], type: 'unique', name: 'paranet_kc_ual_paranet_ual_uk', // Keep the default or a custom name }); } const [[{ indexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'paranet_kc' AND INDEX_NAME = 'idx_paranet_kc_sync_batch'; `); if (!indexExists) { await queryInterface.addIndex( 'paranet_kc', ['paranet_ual', 'is_synced', 'retries', 'updated_at'], { name: 'idx_paranet_kc_sync_batch' }, ); } const [[{ triggerInsertExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerInsertExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_insert_paranet_kc'; `); if (triggerInsertExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_insert_paranet_kc BEFORE INSERT ON paranet_kc FOR EACH ROW BEGIN SET NEW.created_at = NOW(); END; `); } const [[{ triggerUpdateExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerUpdateExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_update_paranet_kc'; `); if (triggerUpdateExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_update_paranet_kc BEFORE UPDATE ON paranet_kc FOR EACH ROW BEGIN SET NEW.updated_at = NOW(); END; `); } } export async function down({ context: { queryInterface } }) { const [[{ indexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'paranet_kc' AND INDEX_NAME = 'idx_paranet_kc_sync_batch'; `); if (indexExists) { await queryInterface.removeIndex('paranet_kc', 'idx_paranet_kc_sync_batch'); } await queryInterface.dropTable('paranet_kc'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20241226151800-prune-commands.js ================================================ export async function up({ context: { queryInterface } }) { await queryInterface.sequelize.query('TRUNCATE TABLE commands;'); } export async function down() { // No need to do anything in the down method for truncation } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20250401123500-truncate-commands-table.js ================================================ export async function up({ context: { queryInterface } }) { await queryInterface.sequelize.query('TRUNCATE TABLE commands;'); } export async function down() { // No need to do anything in the down method for truncation } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20250401155600-create-random-sampling-chanalage.js ================================================ export const up = async ({ context: { queryInterface, Sequelize } }) => { await queryInterface.createTable('random_sampling_challenge', { id: { autoIncrement: true, primaryKey: true, type: Sequelize.INTEGER, }, blockchain_id: { allowNull: false, type: Sequelize.STRING, }, // start_date: { // allowNull: false, // type: Sequelize.DATE, // }, // end_date: { // allowNull: false, // type: Sequelize.DATE, // }, contract_address: { allowNull: false, type: Sequelize.STRING, }, knowledge_collection_id: { allowNull: false, type: Sequelize.INTEGER, }, chunk_number: { allowNull: false, type: Sequelize.INTEGER, }, active_proof_period_start_block: { allowNull: false, type: Sequelize.BIGINT, }, epoch: { allowNull: false, type: Sequelize.INTEGER, }, sent_successfully: { allowNull: false, type: Sequelize.BOOLEAN, defaultValue: false, }, finalized: { allowNull: false, type: Sequelize.BOOLEAN, defaultValue: false, }, score: { allowNull: false, type: Sequelize.BIGINT, defaultValue: 0, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); const [[{ indexExists: randomSamplingBlockchainIdEpochSentSuccessfullyIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'random_sampling_challenge' AND INDEX_NAME = 'idx_rs_challenge_status'; `); if (!randomSamplingBlockchainIdEpochSentSuccessfullyIndexExists) { await queryInterface.addIndex( 'random_sampling_challenge', ['blockchain_id', 'epoch', 'sent_successfully', 'updated_at'], { name: 'idx_rs_challenge_status' }, ); } const [[{ triggerInsertExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerInsertExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_insert_random_sampling_challenge'; `); if (triggerInsertExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_insert_random_sampling_challenge BEFORE INSERT ON random_sampling_challenge FOR EACH ROW BEGIN SET NEW.created_at = NOW(); END; `); } const [[{ triggerUpdateExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerUpdateExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_update_random_sampling_challenge'; `); if (triggerUpdateExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_update_random_sampling_challenge BEFORE UPDATE ON random_sampling_challenge FOR EACH ROW BEGIN SET NEW.updated_at = NOW(); END; `); } }; export const down = async ({ context: { queryInterface } }) => { const [[{ indexExists: randomSamplingBlockchainIdEpochSentSuccessfullyIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'random_sampling_challenge' AND INDEX_NAME = 'idx_rs_challenge_status'; `); if (randomSamplingBlockchainIdEpochSentSuccessfullyIndexExists) { await queryInterface.removeIndex('random_sampling_challenge', 'idx_rs_challenge_status'); } await queryInterface.dropTable('random_sampling_challenge'); }; ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20250408164300-create-triples-inserted-count-table.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('triples_insert_count', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, count: { type: Sequelize.BIGINT, allowNull: false, defaultValue: 0, }, }); } export async function down() { // No need to do anything in the down method for truncation } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20250422150500-add-tx-hash-blockchain-event.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (!(await columnExists('blockchain_event', 'tx_hash'))) { await queryInterface.addColumn('blockchain_event', 'tx_hash', { type: Sequelize.STRING, }); } } export async function down({ context: { queryInterface } }) { async function columnExists(table, column) { const tableDescription = await queryInterface.describeTable(table); return Object.prototype.hasOwnProperty.call(tableDescription, column); } if (await columnExists('blockchain_event', 'tx_hash')) { await queryInterface.removeColumn('blockchain_event', 'tx_hash'); } } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20250509142900-create-batch-get.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('batch_get', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, operation_id: { type: Sequelize.UUID, allowNull: false, }, status: { allowNull: false, type: Sequelize.STRING, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } export async function down({ context: { queryInterface } }) { await queryInterface.dropTable('batch_get'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20250509142901-create-latest-synced-kc.js ================================================ export async function up({ context: { queryInterface, Sequelize } }) { await queryInterface.createTable('latest_synced_kc', { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, blockchain: { type: Sequelize.STRING, allowNull: false, }, contract_address: { type: Sequelize.STRING, allowNull: false, }, latest_synced_kc: { type: Sequelize.INTEGER, allowNull: false, defaultValue: 0, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); const [[{ triggerInsertExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerInsertExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_insert_latest_synced_kc'; `); if (triggerInsertExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_insert_latest_synced_kc BEFORE INSERT ON latest_synced_kc FOR EACH ROW BEGIN SET NEW.created_at = NOW(); END; `); } const [[{ triggerUpdateExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerUpdateExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_update_latest_synced_kc'; `); if (triggerUpdateExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_update_latest_synced_kc BEFORE UPDATE ON latest_synced_kc FOR EACH ROW BEGIN SET NEW.updated_at = NOW(); END; `); } const [[{ blockchainContractAddressIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'latest_synced_kc' AND INDEX_NAME = 'idx_latest_synced_kc_blockchain_contract_address'; `); if (!blockchainContractAddressIndexExists) { await queryInterface.addIndex('latest_synced_kc', ['blockchain', 'contract_address'], { unique: true, name: 'idx_latest_synced_kc_blockchain_contract_address', }); } const [[{ blockchainIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'latest_synced_kc' AND INDEX_NAME = 'idx_latest_synced_kc_blockchain'; `); if (!blockchainIndexExists) { await queryInterface.addIndex('latest_synced_kc', ['blockchain'], { name: 'idx_latest_synced_kc_blockchain', }); } } export async function down({ context: { queryInterface } }) { const [[{ blockchainContractAddressIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'latest_synced_kc' AND INDEX_NAME = 'idx_latest_synced_kc_blockchain_contract_address'; `); if (blockchainContractAddressIndexExists) { await queryInterface.removeIndex( 'latest_synced_kc', 'idx_latest_synced_kc_blockchain_contract_address', ); } const [[{ blockchainIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'latest_synced_kc' AND INDEX_NAME = 'idx_latest_synced_kc_blockchain'; `); if (blockchainIndexExists) { await queryInterface.removeIndex('latest_synced_kc', 'idx_latest_synced_kc_blockchain'); } await queryInterface.dropTable('latest_synced_kc'); } ================================================ FILE: src/modules/repository/implementation/sequelize/migrations/20250509142902-create-sync-missed-kc.js ================================================ /* eslint-disable no-await-in-loop */ import { NODE_ENVIRONMENTS } from '../../../../../constants/constants.js'; export async function up({ context: { queryInterface, Sequelize } }) { const nodeEnv = process.env.NODE_ENV; let blockchains = []; if (nodeEnv === NODE_ENVIRONMENTS.DEVELOPMENT || nodeEnv === NODE_ENVIRONMENTS.TEST) { blockchains = ['hardhat1', 'hardhat2']; } else if (nodeEnv === NODE_ENVIRONMENTS.TESTNET || nodeEnv === NODE_ENVIRONMENTS.MAINNET) { blockchains = ['otp', 'gnosis', 'base']; } else { throw new Error(`Invalid node environment: ${nodeEnv}`); } for (const blockchain of blockchains) { // Check if table exists const [[{ tableExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS tableExists FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = '${blockchain}_sync_missed_kc'; `); if (tableExists === 0) { await queryInterface.createTable(`${blockchain}_sync_missed_kc`, { id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, }, kc_id: { type: Sequelize.INTEGER, allowNull: false, }, contract_address: { type: Sequelize.STRING, allowNull: false, }, synced: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false, }, sync_error: { type: Sequelize.STRING, allowNull: true, }, retry_count: { type: Sequelize.INTEGER, allowNull: false, defaultValue: 0, }, created_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, updated_at: { allowNull: false, type: Sequelize.DATE, defaultValue: Sequelize.literal('NOW()'), }, }); } const [[{ triggerInsertExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerInsertExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_insert_${blockchain}_sync_missed_kc'; `); if (triggerInsertExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_insert_${blockchain}_sync_missed_kc BEFORE INSERT ON ${blockchain}_sync_missed_kc FOR EACH ROW BEGIN SET NEW.created_at = NOW(); END; `); } const [[{ triggerUpdateExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS triggerUpdateExists FROM information_schema.triggers WHERE trigger_schema = DATABASE() AND trigger_name = 'after_update_${blockchain}_sync_missed_kc'; `); if (triggerUpdateExists === 0) { await queryInterface.sequelize.query(` CREATE TRIGGER after_update_${blockchain}_sync_missed_kc BEFORE UPDATE ON ${blockchain}_sync_missed_kc FOR EACH ROW BEGIN SET NEW.updated_at = NOW(); END; `); } const [[{ contractKcIdIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${blockchain}_sync_missed_kc' AND INDEX_NAME = 'idx_${blockchain}_sync_missed_kc_contract_kc_id'; `); if (!contractKcIdIndexExists) { await queryInterface.addIndex( `${blockchain}_sync_missed_kc`, ['contract_address', 'kc_id'], { name: `idx_${blockchain}_sync_missed_kc_contract_kc_id` }, ); } const [[{ retryIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${blockchain}_sync_missed_kc' AND INDEX_NAME = 'idx_${blockchain}_sync_missed_kc_retry_index'; `); if (!retryIndexExists) { await queryInterface.addIndex( `${blockchain}_sync_missed_kc`, ['contract_address', 'synced', 'updated_at', 'retry_count'], { name: `idx_${blockchain}_sync_missed_kc_retry_index` }, ); } } } export async function down({ context: { queryInterface } }) { const nodeEnv = process.env.NODE_ENV; let blockchains = []; if (nodeEnv === NODE_ENVIRONMENTS.DEVELOPMENT || nodeEnv === NODE_ENVIRONMENTS.TEST) { blockchains = ['hardhat1:31337', 'hardhat2:31337']; } else if (nodeEnv === NODE_ENVIRONMENTS.TESTNET || nodeEnv === NODE_ENVIRONMENTS.MAINNET) { blockchains = ['otp', 'gnosis', 'base']; } else { throw new Error(`Invalid node environment: ${nodeEnv}`); } for (const blockchain of blockchains) { const [[{ contractKcIdIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${blockchain}_sync_missed_kc' AND INDEX_NAME = 'idx_${blockchain}_sync_missed_kc_contract_kc_id'; `); if (contractKcIdIndexExists) { await queryInterface.removeIndex( `${blockchain}_sync_missed_kc`, `idx_${blockchain}_sync_missed_kc_contract_kc_id`, ); } const [[{ retryIndexExists }]] = await queryInterface.sequelize.query(` SELECT COUNT(*) AS indexExists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${blockchain}_sync_missed_kc' AND INDEX_NAME = 'idx_${blockchain}_sync_missed_kc_retry_index'; `); if (retryIndexExists) { await queryInterface.removeIndex( `${blockchain}_sync_missed_kc`, `idx_${blockchain}_sync_missed_kc_retry_index`, ); } await queryInterface.dropTable(`${blockchain}_sync_missed_kc`); } } ================================================ FILE: src/modules/repository/implementation/sequelize/models/ability.js ================================================ export default (sequelize, DataTypes) => { const ability = sequelize.define( 'ability', { name: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); ability.associate = () => { // define association here }; return ability; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/ask-response.js ================================================ export default (sequelize, DataTypes) => { const askResponse = sequelize.define( 'ask_response', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, message: DataTypes.TEXT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); askResponse.associate = () => { // associations can be defined here }; return askResponse; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/ask.js ================================================ export default (sequelize, DataTypes) => { const ask = sequelize.define( 'ask', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); ask.associate = () => { // associations can be defined here }; return ask; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/base-sync-missed-kc.js ================================================ export default (sequelize, DataTypes) => { const baseSyncMissedKc = sequelize.define( 'base_sync_missed_kc', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, kcId: { type: DataTypes.INTEGER, allowNull: false, field: 'kc_id', }, contractAddress: { type: DataTypes.STRING, allowNull: false, field: 'contract_address', }, synced: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false, }, syncError: { type: DataTypes.STRING, allowNull: true, field: 'sync_error', }, retryCount: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'retry_count', }, createdAt: { type: DataTypes.DATE, field: 'created_at', }, updatedAt: { type: DataTypes.DATE, field: 'updated_at', }, }, { underscored: true }, ); baseSyncMissedKc.associate = () => { // associations can be defined here }; return baseSyncMissedKc; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/batch-get.js ================================================ export default (sequelize, DataTypes) => { const batchGet = sequelize.define( 'batch_get', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); batchGet.associate = () => { // associations can be defined here }; return batchGet; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/blockchain-event.js ================================================ export default (sequelize, DataTypes) => { const event = sequelize.define( 'blockchain_event', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, contract: DataTypes.STRING, contractAddress: DataTypes.STRING, blockchain: DataTypes.STRING, event: DataTypes.STRING, data: DataTypes.TEXT, blockNumber: DataTypes.BIGINT, transactionIndex: DataTypes.BIGINT, txHash: DataTypes.STRING, logIndex: DataTypes.BIGINT, processed: DataTypes.BOOLEAN, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); event.associate = () => { // associations can be defined here }; return event; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/blockchain.js ================================================ export default (sequelize, DataTypes) => { const blockchain = sequelize.define( 'blockchain', { blockchain: { type: DataTypes.STRING, primaryKey: true, }, lastCheckedBlock: DataTypes.BIGINT, lastCheckedTimestamp: DataTypes.BIGINT, }, { underscored: true }, ); blockchain.associate = () => { // associations can be defined here }; return blockchain; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/commands.js ================================================ import { Model } from 'sequelize'; import { v4 as uuidv4 } from 'uuid'; export default (sequelize, DataTypes) => { class commands extends Model { static associate(models) { commands._models = models; // define association here } } commands.init( { id: { type: DataTypes.UUID, primaryKey: true, defaultValue: () => uuidv4(), }, name: DataTypes.STRING, data: DataTypes.JSON, priority: DataTypes.BIGINT, isBlocking: DataTypes.BOOLEAN, sequence: DataTypes.JSON, readyAt: DataTypes.BIGINT, delay: DataTypes.BIGINT, startedAt: DataTypes.BIGINT, deadlineAt: DataTypes.BIGINT, period: DataTypes.BIGINT, status: DataTypes.STRING, message: DataTypes.TEXT, parentId: DataTypes.UUID, transactional: DataTypes.BOOLEAN, retries: { type: DataTypes.INTEGER, defaultValue: 0, }, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { sequelize, modelName: 'commands', timestamps: false, underscored: true, }, ); return commands; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/event.js ================================================ export default (sequelize, DataTypes) => { const event = sequelize.define( 'event', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, blockchainId: DataTypes.STRING, name: DataTypes.STRING, timestamp: DataTypes.STRING, value1: DataTypes.TEXT, value2: DataTypes.TEXT, value3: DataTypes.TEXT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); event.associate = () => { // associations can be defined here }; return event; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/finality-response.js ================================================ export default (sequelize, DataTypes) => { const finalityResponse = sequelize.define( 'finality_response', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, message: DataTypes.TEXT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); finalityResponse.associate = () => { // associations can be defined here }; return finalityResponse; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/finality-status.js ================================================ export default (sequelize, DataTypes) => { const finalityStatus = sequelize.define( 'finality_status', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.STRING, ual: DataTypes.STRING, peerId: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); finalityStatus.associate = () => { // associations can be defined here }; return finalityStatus; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/finality.js ================================================ export default (sequelize, DataTypes) => { const finality = sequelize.define( 'finality', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); finality.associate = () => { // associations can be defined here }; return finality; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/get-response.js ================================================ export default (sequelize, DataTypes) => { const getResponse = sequelize.define( 'get_response', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, message: DataTypes.TEXT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); getResponse.associate = () => { // associations can be defined here }; return getResponse; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/get.js ================================================ export default (sequelize, DataTypes) => { const get = sequelize.define( 'get', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); get.associate = () => { // associations can be defined here }; return get; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/gnosis-sync-missed-kc.js ================================================ export default (sequelize, DataTypes) => { const gnosisSyncMissedKc = sequelize.define( 'gnosis_sync_missed_kc', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, kcId: { type: DataTypes.INTEGER, allowNull: false, field: 'kc_id', }, contractAddress: { type: DataTypes.STRING, allowNull: false, field: 'contract_address', }, synced: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false, }, syncError: { type: DataTypes.STRING, allowNull: true, field: 'sync_error', }, retryCount: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'retry_count', }, createdAt: { type: DataTypes.DATE, field: 'created_at', }, updatedAt: { type: DataTypes.DATE, field: 'updated_at', }, }, { underscored: true }, ); gnosisSyncMissedKc.associate = () => { // associations can be defined here }; return gnosisSyncMissedKc; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/hardhat1-sync-missed-kc.js ================================================ export default (sequelize, DataTypes) => { const hardhat1SyncMissedKc = sequelize.define( 'hardhat1_sync_missed_kc', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, kcId: { type: DataTypes.INTEGER, allowNull: false, field: 'kc_id', }, contractAddress: { type: DataTypes.STRING, allowNull: false, field: 'contract_address', }, synced: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false, }, syncError: { type: DataTypes.STRING, allowNull: true, field: 'sync_error', }, retryCount: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'retry_count', }, createdAt: { type: DataTypes.DATE, field: 'created_at', }, updatedAt: { type: DataTypes.DATE, field: 'updated_at', }, }, { underscored: true }, ); hardhat1SyncMissedKc.associate = () => { // associations can be defined here }; return hardhat1SyncMissedKc; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/hardhat2-sync-missed-kc.js ================================================ export default (sequelize, DataTypes) => { const hardhat2SyncMissedKc = sequelize.define( 'hardhat2_sync_missed_kc', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, kcId: { type: DataTypes.INTEGER, allowNull: false, field: 'kc_id', }, contractAddress: { type: DataTypes.STRING, allowNull: false, field: 'contract_address', }, synced: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false, }, syncError: { type: DataTypes.STRING, allowNull: true, field: 'sync_error', }, retryCount: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'retry_count', }, createdAt: { type: DataTypes.DATE, field: 'created_at', }, updatedAt: { type: DataTypes.DATE, field: 'updated_at', }, }, { underscored: true }, ); hardhat2SyncMissedKc.associate = () => { // associations can be defined here }; return hardhat2SyncMissedKc; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/latest-synced-kc.js ================================================ export default (sequelize, DataTypes) => { const latestSyncedKc = sequelize.define( 'latest_synced_kc', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, blockchain: { type: DataTypes.STRING, allowNull: false, }, contractAddress: { type: DataTypes.STRING, allowNull: false, field: 'contract_address', }, latestSyncedKc: { type: DataTypes.INTEGER, allowNull: false, field: 'latest_synced_kc', }, createdAt: { type: DataTypes.DATE, field: 'created_at', }, updatedAt: { type: DataTypes.DATE, field: 'updated_at', }, }, { underscored: true }, ); latestSyncedKc.associate = () => { // associations can be defined here }; return latestSyncedKc; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/missed-paranet-asset.js ================================================ // NOT USED ANYMORE export default (sequelize, DataTypes) => { const blockchain = sequelize.define( 'missed_paranet_asset', { id: { autoIncrement: true, primaryKey: true, type: DataTypes.INTEGER, }, blockchainId: { allowNull: false, type: DataTypes.STRING, }, ual: { allowNull: false, type: DataTypes.STRING, }, paranetUal: { allowNull: false, type: DataTypes.STRING, }, errorMessage: { allowNull: true, type: DataTypes.TEXT, }, createdAt: { type: DataTypes.DATE, }, updatedAt: { type: DataTypes.DATE, }, }, { underscored: true }, ); blockchain.associate = () => { // associations can be defined here }; return blockchain; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/operation_ids.js ================================================ import { v4 as uuidv4 } from 'uuid'; export default (sequelize, DataTypes) => { const operationIds = sequelize.define( 'operation_ids', { operationId: { type: DataTypes.UUID, primaryKey: true, defaultValue: () => uuidv4(), }, data: DataTypes.TEXT, status: DataTypes.STRING, minAcksReached: DataTypes.BOOLEAN, timestamp: { type: DataTypes.BIGINT, defaultValue: () => Date.now(), }, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); operationIds.associate = () => { // associations can be defined here }; return operationIds; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/otp-sync-missed-kc.js ================================================ export default (sequelize, DataTypes) => { const otpSyncMissedKc = sequelize.define( 'otp_sync_missed_kc', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, kcId: { type: DataTypes.INTEGER, allowNull: false, field: 'kc_id', }, contractAddress: { type: DataTypes.STRING, allowNull: false, field: 'contract_address', }, synced: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false, }, syncError: { type: DataTypes.STRING, allowNull: true, field: 'sync_error', }, retryCount: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'retry_count', }, createdAt: { type: DataTypes.DATE, field: 'created_at', }, updatedAt: { type: DataTypes.DATE, field: 'updated_at', }, }, { underscored: true }, ); otpSyncMissedKc.associate = () => { // associations can be defined here }; return otpSyncMissedKc; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/paranet-kc.js ================================================ export default (sequelize, DataTypes) => { const paranetKC = sequelize.define( 'paranet_kc', { id: { autoIncrement: true, primaryKey: true, type: DataTypes.INTEGER, }, blockchainId: { allowNull: false, type: DataTypes.STRING, }, ual: { allowNull: false, type: DataTypes.STRING, }, paranetUal: { allowNull: false, type: DataTypes.STRING, }, errorMessage: { allowNull: true, type: DataTypes.TEXT, }, isSynced: { allowNull: false, type: DataTypes.BOOLEAN, defaultValue: false, }, retries: { allowNull: false, type: DataTypes.INTEGER, defaultValue: 0, }, createdAt: { type: DataTypes.DATE, }, updatedAt: { type: DataTypes.DATE, }, }, { underscored: true, indexes: [ { unique: true, fields: ['ual', 'paranetUal'], // Composite unique constraint on `ual` and `paranetUal` }, { fields: ['paranetUal', 'isSynced', 'retries', 'updatedAt'], }, ], }, ); paranetKC.associate = () => { // Define associations here if needed }; return paranetKC; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/paranet-synced-asset.js ================================================ import { PARANET_SYNC_SOURCES } from '../../../../../constants/constants.js'; // NOT USED ANYMORE export default (sequelize, DataTypes) => { const blockchain = sequelize.define( 'paranet_synced_asset', { id: { autoIncrement: true, primaryKey: true, type: DataTypes.INTEGER, }, blockchainId: { allowNull: false, type: DataTypes.STRING, }, ual: { allowNull: false, type: DataTypes.STRING, }, paranetUal: { allowNull: false, type: DataTypes.STRING, }, publicAssertionId: { allowNull: true, type: DataTypes.STRING, }, privateAssertionId: { allowNull: true, type: DataTypes.STRING, }, sender: { allowNull: true, type: DataTypes.STRING, }, transactionHash: { allowNull: true, type: DataTypes.STRING, }, dataSource: { allowNull: true, type: DataTypes.ENUM(...Object.values(PARANET_SYNC_SOURCES)), }, createdAt: { type: DataTypes.DATE, }, updatedAt: { type: DataTypes.DATE, }, }, { underscored: true }, ); blockchain.associate = () => { // associations can be defined here }; return blockchain; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/paranet.js ================================================ export default (sequelize, DataTypes) => { const paranet = sequelize.define( 'paranet', { id: { autoIncrement: true, primaryKey: true, type: DataTypes.INTEGER, }, name: { type: DataTypes.STRING, }, description: { type: DataTypes.STRING, }, paranetId: { type: DataTypes.STRING, }, kaCount: { type: DataTypes.INTEGER, }, blockchainId: { type: DataTypes.STRING, }, createdAt: { type: DataTypes.DATE, }, updatedAt: { type: DataTypes.DATE, }, }, { underscored: true }, ); paranet.associate = () => { // associations can be defined here }; return paranet; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/publish-paranet-response.js ================================================ export default (sequelize, DataTypes) => { const publishParanetResponse = sequelize.define( 'publish_paranet_response', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, message: DataTypes.TEXT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); publishParanetResponse.associate = () => { // associations can be defined here }; return publishParanetResponse; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/publish-paranet.js ================================================ export default (sequelize, DataTypes) => { const publishParanet = sequelize.define( 'publish_paranet', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); publishParanet.associate = () => { // associations can be defined here }; return publishParanet; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/publish-response.js ================================================ export default (sequelize, DataTypes) => { const publishResponse = sequelize.define( 'publish_response', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, message: DataTypes.TEXT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); publishResponse.associate = () => { // associations can be defined here }; return publishResponse; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/publish.js ================================================ export default (sequelize, DataTypes) => { const publish = sequelize.define( 'publish', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); publish.associate = () => { // associations can be defined here }; return publish; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/random-sampling-challenge.js ================================================ export default (sequelize, DataTypes) => { const randomSamplingChallenge = sequelize.define( 'random_sampling_challenge', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, blockchainId: DataTypes.STRING, // startDate: DataTypes.DATE, // endDate: DataTypes.DATE, contractAddress: DataTypes.STRING, knowledgeCollectionId: DataTypes.INTEGER, chunkNumber: DataTypes.INTEGER, epoch: DataTypes.INTEGER, activeProofPeriodStartBlock: DataTypes.BIGINT, finalized: DataTypes.BOOLEAN, sentSuccessfully: DataTypes.BOOLEAN, score: DataTypes.BIGINT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); randomSamplingChallenge.associate = () => { // associations can be defined here }; return randomSamplingChallenge; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/role-ability.js ================================================ export default (sequelize, DataTypes) => { const roleAbility = sequelize.define( 'role_ability', { createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); roleAbility.associate = (models) => { roleAbility.hasOne(models.ability, { as: 'ability' }); roleAbility.hasOne(models.role, { as: 'role' }); }; return roleAbility; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/role.js ================================================ export default (sequelize, DataTypes) => { const role = sequelize.define( 'role', { name: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); role.associate = (models) => { role.belongsToMany(models.ability, { as: 'abilities', foreignKey: 'ability_id', through: models.role_ability, }); }; return role; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/shard.js ================================================ export default (sequelize, DataTypes) => { const shard = sequelize.define( 'shard', { peerId: { type: DataTypes.STRING, primaryKey: true }, blockchainId: { type: DataTypes.STRING, primaryKey: true }, ask: { type: DataTypes.INTEGER, allowNull: false, }, stake: { type: DataTypes.INTEGER, allowNull: false, }, lastSeen: { type: DataTypes.DATE, allowNull: false, defaultValue: new Date(0), }, lastDialed: { type: DataTypes.DATE, allowNull: false, defaultValue: new Date(0), }, sha256: { type: DataTypes.STRING, allowNull: false, }, }, { underscored: true }, ); shard.associate = () => { // associations can be defined here }; return shard; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/token.js ================================================ export default (sequelize, DataTypes) => { const token = sequelize.define( 'token', { id: { type: DataTypes.STRING, primaryKey: true }, revoked: DataTypes.BOOLEAN, userId: DataTypes.INTEGER, name: { type: DataTypes.STRING, }, expiresAt: DataTypes.DATE, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); token.associate = (models) => { token.belongsTo(models.user, { as: 'user' }); }; return token; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/triples-inserted-count.js ================================================ export default (sequelize, DataTypes) => { const TriplesInsertCount = sequelize.define( 'triples_insert_count', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, count: { type: DataTypes.BIGINT, allowNull: false, defaultValue: 0, }, }, { timestamps: false, freezeTableName: true, }, ); return TriplesInsertCount; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/update-response.js ================================================ export default (sequelize, DataTypes) => { const updateResponse = sequelize.define( 'update_response', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, message: DataTypes.TEXT, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); updateResponse.associate = () => { // associations can be defined here }; return updateResponse; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/update.js ================================================ export default (sequelize, DataTypes) => { const update = sequelize.define( 'update', { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, }, operationId: DataTypes.UUID, status: DataTypes.STRING, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); update.associate = () => { // associations can be defined here }; return update; }; ================================================ FILE: src/modules/repository/implementation/sequelize/models/user.js ================================================ export default (sequelize, DataTypes) => { const user = sequelize.define( 'user', { name: { type: DataTypes.STRING, unique: true, }, createdAt: DataTypes.DATE, updatedAt: DataTypes.DATE, }, { underscored: true }, ); user.associate = (models) => { user.hasMany(models.token, { as: 'tokens' }); user.hasOne(models.role, { as: 'role' }); }; return user; }; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js ================================================ import Sequelize from 'sequelize'; class BlockchainEventRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.blockchain_event; } async insertBlockchainEvents(events, options) { const chunkSize = 10000; let insertedEvents = []; for (let i = 0; i < events.length; i += chunkSize) { const chunk = events.slice(i, i + chunkSize); // eslint-disable-next-line no-await-in-loop const insertedChunk = await this.model.bulkCreate( chunk.map((event) => ({ blockchain: event.blockchain, contract: event.contract, contractAddress: event.contractAddress, event: event.event, data: event.data, blockNumber: event.blockNumber, transactionIndex: event.transactionIndex, logIndex: event.logIndex, processed: false, txHash: event.txHash, })), { ignoreDuplicates: true, ...options, }, ); insertedEvents = insertedEvents.concat(insertedChunk.map((event) => event.dataValues)); } return insertedEvents; } async getAllUnprocessedBlockchainEvents(blockchain, eventNames, options) { return this.model.findAll({ where: { blockchain, processed: false, event: { [Sequelize.Op.in]: eventNames }, }, order: [ ['blockNumber', 'asc'], ['transactionIndex', 'asc'], ['logIndex', 'asc'], ], ...options, }); } async markAllBlockchainEventsAsProcessed(blockchain, options) { return this.model.update( { processed: true }, { where: { blockchain, processed: false }, ...options, }, ); } async removeEvents(ids, options) { await this.model.destroy({ where: { id: { [Sequelize.Op.in]: ids }, }, ...options, }); } async removeContractEventsAfterBlock( blockchain, contract, contractAddress, blockNumber, transactionIndex, options, ) { return this.model.destroy({ where: { blockchain, contract, contractAddress, [Sequelize.Op.or]: [ // Events in blocks after the given blockNumber { blockNumber: { [Sequelize.Op.gt]: blockNumber } }, // Events in the same blockNumber but with a higher transactionIndex { blockNumber, transactionIndex: { [Sequelize.Op.gt]: transactionIndex }, }, ], }, ...options, }); } async findAndRemoveProcessedEvents(timestamp, limit, options) { return this.model.destroy({ where: { processed: true, createdAt: { [Sequelize.Op.lte]: timestamp }, }, limit, ...options, }); } async findProcessedEvents(timestamp, limit, options) { return this.model.findAll({ where: { processed: true, createdAt: { [Sequelize.Op.lte]: timestamp }, }, order: [['createdAt', 'asc']], raw: true, limit, ...options, }); } } export default BlockchainEventRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/blockchain-missed-kc-repository.js ================================================ import Sequelize from 'sequelize'; import { NODE_ENVIRONMENTS } from '../../../../../constants/constants.js'; class BlockchainMissedKcRepository { constructor(models) { const nodeEnv = process.env.NODE_ENV; if (nodeEnv === NODE_ENVIRONMENTS.DEVELOPMENT || nodeEnv === NODE_ENVIRONMENTS.TEST) { this.models = { hardhat1: models.hardhat1_sync_missed_kc, hardhat2: models.hardhat2_sync_missed_kc, }; } else if (nodeEnv === NODE_ENVIRONMENTS.TESTNET || nodeEnv === NODE_ENVIRONMENTS.MAINNET) { this.models = { otp: models.otp_sync_missed_kc, gnosis: models.gnosis_sync_missed_kc, base: models.base_sync_missed_kc, }; } else { throw new Error(`Invalid node environment: ${nodeEnv}`); } } async insertMissedKc(blockchain, records, error, options) { const blockchainName = blockchain.split(':')[0]; const model = this.models[blockchainName]; const query = ` INSERT INTO ${blockchainName}_sync_missed_kc (kc_id, contract_address, sync_error) VALUES ${records .map((record) => `('${record.kcId}', '${record.contractAddress}', '${error}')`) .join(',')} `; return model.sequelize.query(query, { type: model.sequelize.QueryTypes.INSERT, ...options, }); } async getMissedKcForRetry(blockchain, contractAddress, limit, options) { const blockchainName = blockchain.split(':')[0]; const model = this.models[blockchainName]; return model.findAll({ where: { contract_address: contractAddress, synced: false, [Sequelize.Op.and]: [ Sequelize.literal(` NOW() >= LEAST( DATE_ADD(updated_at, INTERVAL POW(2, retry_count) MINUTE), DATE_ADD(updated_at, INTERVAL 7 DAY) ) `), ], }, limit, ...options, }); } async incrementRetryCount(blockchain, records, options) { const blockchainName = blockchain.split(':')[0]; const kcIds = [...new Set(records.map((r) => r.kcId))]; const contractAddresses = [...new Set(records.map((r) => r.contractAddress))]; const model = this.models[blockchainName]; const query = ` UPDATE ${blockchainName}_sync_missed_kc SET retry_count = retry_count + 1 WHERE kc_id IN (:kcIds) AND contract_address IN (:contractAddresses) `; return model.sequelize.query(query, { replacements: { kcIds, contractAddresses, blockchainId: blockchain, }, type: model.sequelize.QueryTypes.UPDATE, ...options, }); } async setSyncedToTrue(blockchain, records, options) { const blockchainName = blockchain.split(':')[0]; const kcIds = [...new Set(records.map((r) => r.kcId))]; const contractAddresses = [...new Set(records.map((r) => r.contractAddress))]; const model = this.models[blockchainName]; const query = ` UPDATE ${blockchainName}_sync_missed_kc SET synced = true WHERE kc_id IN (:kcIds) AND contract_address IN (:contractAddresses) `; return model.sequelize.query(query, { replacements: { kcIds, contractAddresses, blockchainId: blockchain, }, type: model.sequelize.QueryTypes.UPDATE, ...options, }); } async getMissedKcForRetryCount(blockchain, contractAddress, options) { const blockchainName = blockchain.split(':')[0]; const model = this.models[blockchainName]; return model.count({ where: { contract_address: contractAddress, synced: false }, ...options, }); } } export default BlockchainMissedKcRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js ================================================ class BlockchainRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.blockchain; } async getLastCheckedBlock(blockchain, options) { return this.model.findOne({ where: { blockchain }, ...options, }); } async updateLastCheckedBlock(blockchain, currentBlock, timestamp, options) { return this.model.upsert( { blockchain, lastCheckedBlock: currentBlock, lastCheckedTimestamp: timestamp, }, options, ); } } export default BlockchainRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/command-repository.js ================================================ import Sequelize from 'sequelize'; import { COMMAND_STATUS } from '../../../../../constants/constants.js'; class CommandRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.commands; } async updateCommand(update, options) { await this.model.update(update, options); } async destroyCommand(name, options) { await this.model.destroy({ where: { name: { [Sequelize.Op.eq]: name }, }, ...options, }); } async createCommand(command, options) { return this.model.create(command, options); } async getCommandsWithStatus(statusArray, excludeNameArray, options) { return this.model.findAll({ where: { status: { [Sequelize.Op.in]: statusArray, }, name: { [Sequelize.Op.notIn]: excludeNameArray }, }, ...options, }); } async getCommandWithId(id, options) { return this.model.findOne({ where: { id, }, ...options, }); } async removeCommands(ids, options) { await this.model.destroy({ where: { id: { [Sequelize.Op.in]: ids }, }, ...options, }); } async findFinalizedCommands(timestamp, limit, options) { return this.model.findAll({ where: { status: { [Sequelize.Op.in]: [ COMMAND_STATUS.COMPLETED, COMMAND_STATUS.FAILED, COMMAND_STATUS.EXPIRED, COMMAND_STATUS.UNKNOWN, ], }, startedAt: { [Sequelize.Op.lte]: timestamp }, }, order: [['startedAt', 'asc']], raw: true, limit, ...options, }); } async findAndRemoveFinalizedCommands(timestamp, limit, options) { return this.model.destroy({ where: { [Sequelize.Op.or]: [ { status: { [Sequelize.Op.in]: [ COMMAND_STATUS.COMPLETED, COMMAND_STATUS.FAILED, COMMAND_STATUS.EXPIRED, COMMAND_STATUS.UNKNOWN, ], }, }, { startedAt: { [Sequelize.Op.lte]: timestamp }, }, ], }, limit, ...options, }); } async findUnfinalizedCommandsByName(name, options) { return this.model.findAll({ where: { name, status: { [Sequelize.Op.notIn]: [ COMMAND_STATUS.COMPLETED, COMMAND_STATUS.FAILED, COMMAND_STATUS.EXPIRED, COMMAND_STATUS.UNKNOWN, ], }, }, raw: true, ...options, }); } } export default CommandRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/event-repository.js ================================================ import Sequelize from 'sequelize'; import { OPERATION_ID_STATUS, HIGH_TRAFFIC_OPERATIONS_NUMBER_PER_HOUR, SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES, } from '../../../../../constants/constants.js'; class EventRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.event; } async createEventRecord( operationId, blockchainId, name, timestamp, value1, value2, value3, options, ) { return this.model.create( { operationId, blockchainId, name, timestamp, value1, value2, value3, }, options, ); } async getUnpublishedEvents(options) { // events without COMPLETE/FAILED status which are older than 30min // are also considered finished const minutes = 5; let operationIds = await this.model.findAll({ raw: true, attributes: [ Sequelize.fn('DISTINCT', Sequelize.col('operation_id')), Sequelize.col('timestamp'), ], where: { [Sequelize.Op.or]: { name: { [Sequelize.Op.in]: [ OPERATION_ID_STATUS.COMPLETED, OPERATION_ID_STATUS.FAILED, ], }, timestamp: { [Sequelize.Op.lt]: Sequelize.literal( `(UNIX_TIMESTAMP()*1000 - 1000*60*${minutes})`, ), }, }, }, order: [['timestamp', 'asc']], limit: Math.floor(HIGH_TRAFFIC_OPERATIONS_NUMBER_PER_HOUR / 60) * SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES, ...options, }); operationIds = operationIds.map((e) => e.operation_id); return this.model.findAll({ where: { operationId: { [Sequelize.Op.in]: operationIds, }, }, ...options, }); } async destroyEvents(ids, options) { await this.model.destroy({ where: { id: { [Sequelize.Op.in]: ids, }, }, ...options, }); } } export default EventRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/finality-status-repository.js ================================================ class FinalityStatusRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.finality_status; } async getFinalityAcksCount(ual, options) { return this.model.count({ where: { ual }, ...options, }); } async saveFinalityAck(operationId, ual, peerId, options) { return this.model.upsert({ operationId, ual, peerId }, options); } async getPublishOperationIdByUal(ual, options) { const record = await this.model.findOne({ where: { ual }, attributes: ['operationId'], ...options, }); return record?.operationId ?? null; } } export default FinalityStatusRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/inserted-triples-repository.js ================================================ import { Sequelize } from 'sequelize'; class TriplesInsertCountRepository { constructor(models) { this.model = models.triples_insert_count; } async getCount() { const record = await this.model.findOne(); return record?.count || 0; } async increment(by = 1, options = {}) { const [record] = await this.model.findOrCreate({ where: {}, defaults: { count: 0 }, ...options, }); await this.model.update( { count: Sequelize.literal(`count + ${by}`), }, { where: { id: record.id }, ...options, }, ); } } export default TriplesInsertCountRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/latest-synced-kc-repository.js ================================================ class LatestSyncedKcRepository { constructor(ctx) { this.model = ctx.latest_synced_kc; } async getKCStorageContracts(blockchainId) { return this.model.findAll({ attributes: ['contract_address'], where: { blockchain: blockchainId }, }); } getSyncRecordForBlockchain(blockchainId) { return this.model.findAll({ where: { blockchain: blockchainId }, }); } async addSyncContracts(blockchainId, contracts) { const query = ` INSERT INTO latest_synced_kc (blockchain, contract_address) VALUES ${contracts.map((contract) => `('${blockchainId}', '${contract}')`).join(',')} `; return this.model.sequelize.query(query, { type: this.model.sequelize.QueryTypes.INSERT }); } async updateLatestSyncedKc(blockchainId, contractAddress, latestSyncedKc, options) { return this.model.update( { latestSyncedKc }, { where: { blockchain: blockchainId, contractAddress }, ...options }, ); } } export default LatestSyncedKcRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/missed-paranet-asset-repository.js ================================================ import Sequelize from 'sequelize'; class MissedParanetAssetRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.missed_paranet_asset; } async createMissedParanetAssetRecord(missedParanetAsset, options) { return this.model.create(missedParanetAsset, options); } async getMissedParanetAssetsRecordsWithRetryCount( paranetUal, retryCountLimit, retryDelayInMs, limit, options, ) { const now = new Date(); const delayDate = new Date(now.getTime() - retryDelayInMs); const queryOptions = { attributes: [ 'blockchainId', 'ual', 'paranetUal', [Sequelize.fn('MAX', Sequelize.col('created_at')), 'latestCreatedAt'], [Sequelize.fn('COUNT', Sequelize.col('ual')), 'retryCount'], ], where: { paranetUal, }, group: ['ual', 'blockchainId', 'paranetUal'], having: Sequelize.and( Sequelize.literal(`COUNT(ual) < ${retryCountLimit}`), Sequelize.literal(`MAX(created_at) <= '${delayDate.toISOString()}'`), ), ...options, }; if (limit !== null) { queryOptions.limit = limit; } return this.model.findAll(queryOptions); } async missedParanetAssetRecordExists(ual, options) { const missedParanetAssetRecord = await this.model.findOne({ where: { ual }, ...options, }); return !!missedParanetAssetRecord; } async removeMissedParanetAssetRecordsByUAL(ual, options) { await this.model.destroy({ where: { ual, }, ...options, }); } async getCountOfMissedAssetsOfParanet(paranetUal, options) { const records = await this.model.findAll({ attributes: ['paranet_ual', 'ual'], where: { paranetUal, }, group: ['paranet_ual', 'ual'], ...options, }); return records.length; } async getFilteredCountOfMissedAssetsOfParanet( paranetUal, retryCountLimit, retryDelayInMs, options, ) { const now = new Date(); const delayDate = new Date(now.getTime() - retryDelayInMs); const records = await this.model.findAll({ attributes: [ [Sequelize.fn('MAX', Sequelize.col('created_at')), 'latestCreatedAt'], [Sequelize.fn('COUNT', Sequelize.col('ual')), 'retryCount'], ], where: { paranetUal, }, group: ['paranet_ual', 'ual'], having: { retryCount: { [Sequelize.Op.lt]: retryCountLimit, }, latestCreatedAt: { [Sequelize.Op.lte]: delayDate, }, }, ...options, }); return records.length; } } export default MissedParanetAssetRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js ================================================ import Sequelize from 'sequelize'; class OperationIdRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.operation_ids; } async createOperationIdRecord(handlerData, options) { return this.model.create(handlerData, options); } async getOperationIdRecord(operationId, options) { return this.model.findOne({ where: { operationId, }, ...options, }); } async updateOperationIdRecord(data, operationId, options) { await this.model.update(data, { where: { operationId, }, ...options, }); } async removeOperationIdRecord(timeToBeDeleted, statuses, options) { await this.model.destroy({ where: { timestamp: { [Sequelize.Op.lt]: timeToBeDeleted }, status: { [Sequelize.Op.in]: statuses }, }, ...options, }); } async updateMinAcksReached(operationId, minAcksReached, options) { await this.model.update( { minAcksReached }, { where: { operationId, }, ...options, }, ); } } export default OperationIdRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/operation-repository.js ================================================ import { Sequelize } from 'sequelize'; class OperationRepository { constructor(models) { this.sequelize = models.sequelize; this.models = { get: models.get, publish: models.publish, update: models.update, publish_paranet: models.publish_paranet, ask: models.ask, finality: models.finality, batch_get: models.batch_get, }; } async createOperationRecord(operation, operationId, status, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[operationModel].create( { operationId, status, }, options, ); } async findAndRemoveProcessedOperationRecords(operation, timestamp, limit, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[`${operationModel}`].destroy({ where: { createdAt: { [Sequelize.Op.lte]: timestamp }, }, limit, ...options, }); } async removeOperationRecords(operation, ids, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[operationModel].destroy({ where: { id: { [Sequelize.Op.in]: ids }, }, ...options, }); } async findProcessedOperations(operation, timestamp, limit, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[`${operationModel}`].findAll({ where: { createdAt: { [Sequelize.Op.lte]: timestamp }, }, order: [['createdAt', 'asc']], raw: true, limit, ...options, }); } async getOperationStatus(operation, operationId, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[operationModel].findOne({ attributes: ['status'], where: { operationId, }, ...options, }); } async updateOperationStatus(operation, operationId, status, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); await this.models[operationModel].update( { status }, { where: { operationId, }, ...options, }, ); } } export default OperationRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/operation-response.js ================================================ import Sequelize from 'sequelize'; class OperationResponseRepository { constructor(models) { this.sequelize = models.sequelize; this.models = { get_response: models.get_response, publish_response: models.publish_response, update_response: models.update_response, publish_paranet_response: models.publish_paranet_response, ask_response: models.ask_response, finality_response: models.finality_response, }; } async createOperationResponseRecord(status, operation, operationId, message, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); await this.models[`${operationModel}_response`].create( { status, message, operationId, }, options, ); } async getOperationResponsesStatuses(operation, operationId, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[`${operationModel}_response`].findAll({ attributes: ['status'], where: { operationId, }, ...options, }); } async findProcessedOperationResponse(timestamp, limit, operation, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[`${operationModel}_response`].findAll({ where: { createdAt: { [Sequelize.Op.lte]: timestamp }, }, order: [['createdAt', 'asc']], raw: true, limit, ...options, }); } async findAndRemoveProcessedOperationResponse(operation, timestamp, limit, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); return this.models[`${operationModel}_response`].destroy({ where: { createdAt: { [Sequelize.Op.lte]: timestamp }, }, limit, ...options, }); } async removeOperationResponse(ids, operation, options) { const operationModel = operation.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); await this.models[`${operationModel}_response`].destroy({ where: { id: { [Sequelize.Op.in]: ids }, }, ...options, }); } } export default OperationResponseRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/paranet-kc-repository.js ================================================ import Sequelize from 'sequelize'; class ParanetKcRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.paranet_kc; } async createParanetKcRecords(paranetUal, blockchainId, uals, options = {}) { return this.model.bulkCreate( uals.map((ual) => ({ paranetUal, blockchainId, ual, isSynced: false })), options, ); } async getCount(paranetUal, options = {}) { return this.model.count({ where: { paranetUal, }, ...options, }); } async getCountSynced(paranetUal, options = {}) { return this.model.count({ where: { paranetUal, isSynced: true, }, ...options, }); } async getCountUnsynced(paranetUal, options = {}) { return this.model.count({ where: { paranetUal, isSynced: false, }, ...options, }); } async getSyncBatch(paranetUal, maxRetries, delayInMs, limit = null, options = {}) { const queryOptions = { where: { paranetUal, isSynced: false, [Sequelize.Op.and]: [ { retries: { [Sequelize.Op.lt]: maxRetries } }, { [Sequelize.Op.or]: [ { retries: 0 }, { updatedAt: { [Sequelize.Op.lte]: new Date(Date.now() - delayInMs), }, }, ], }, ], }, order: [['retries', 'DESC']], ...options, }; if (limit !== null) { queryOptions.limit = limit; } return this.model.findAll(queryOptions); } async incrementRetries(paranetUal, ual, errorMessage = null, options = {}) { const [affectedRows] = await this.model.update( { retries: Sequelize.literal('retries + 1'), errorMessage, }, { where: { ual, paranetUal, }, ...options, }, ); return affectedRows; } async markAsSynced(paranetUal, ual, options = {}) { const [affectedRows] = await this.model.update( { isSynced: true }, { where: { ual, paranetUal, }, ...options, }, ); return affectedRows; } } export default ParanetKcRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/paranet-repository.js ================================================ import { Sequelize } from 'sequelize'; class ParanetRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.paranet; } async createParanetRecord(name, description, paranetId, blockchainId, options) { return this.model.create( { name, description, paranetId, kaCount: 0, blockchainId, }, { ignoreDuplicates: true, ...options, }, ); } async getParanet(paranetId, blockchainId, options) { return this.model.findOne({ where: { paranetId, blockchainId, }, ...options, }); } async addToParanetKaCount(paranetId, blockchainId, kaCount, options) { return this.model.update( { kaCount: Sequelize.literal(`ka_count + ${kaCount}`), }, { where: { paranetId, blockchainId, }, ...options, }, ); } async paranetExists(paranetId, blockchainId, options) { const paranetRecord = await this.model.findOne({ where: { paranetId, blockchainId, }, ...options, }); return !!paranetRecord; } async getParanetKnowledgeAssetsCount(paranetId, blockchainId, options) { return this.model.findAll({ attributes: ['ka_count'], where: { paranetId, blockchainId, }, ...options, }); } async incrementParanetKaCount(paranetId, blockchainId, options) { return this.model.update( { kaCount: Sequelize.literal(`ka_count + 1`), }, { where: { paranetId, blockchainId, }, ...options, }, ); } async getParanetsBlockchains(options) { return this.model.findAll({ attributes: [ [Sequelize.fn('DISTINCT', Sequelize.col('blockchain_id')), 'blockchain_id'], ], ...options, }); } } export default ParanetRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/paranet-synced-asset-repository.js ================================================ // DEPRECATED class ParanetSyncedAssetRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.paranet_synced_asset; } async createParanetSyncedAssetRecord( blockchainId, ual, paranetUal, publicAssertionId, privateAssertionId, sender, transactionHash, dataSource, options, ) { return this.model.create( { blockchainId, ual, paranetUal, publicAssertionId, privateAssertionId, sender, transactionHash, dataSource, }, options, ); } async getParanetSyncedAssetRecordByUAL(ual, options) { return this.model.findOne({ where: { ual }, ...options, }); } async getParanetSyncedAssetRecordsCountByDataSource(paranetUal, dataSource, options) { return this.model.count({ where: { paranetUal, dataSource, }, ...options, }); } async paranetSyncedAssetRecordExists(ual, options) { const paranetSyncedAssetRecord = await this.getParanetSyncedAssetRecordByUAL(ual, options); return !!paranetSyncedAssetRecord; } } export default ParanetSyncedAssetRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/random-sampling-challenge-repository.js ================================================ class RandomSamplingChallengeRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.random_sampling_challenge; } async createRandomSamplingChallengeRecord(randomSamplingChallenge, options) { return this.model.create(randomSamplingChallenge, options); } async updateRandomSamplingChallengeRecord(randomSamplingChallenge, options) { return this.model.update(randomSamplingChallenge, options); } async setCompletedAndScoreRandomSamplingChallengeRecord( randomSamplingChallengeId, completed, score, options, ) { return this.model.update( { sentSuccessfully: completed, score }, { where: { id: randomSamplingChallengeId }, ...options }, ); } async setCompletedAndFinalizedRandomSamplingChallengeRecord( randomSamplingChallengeId, completed, finalized, options, ) { return this.model.update( { completed, finalized }, { where: { id: randomSamplingChallengeId }, ...options }, ); } async getLatestRandomSamplingChallengeRecordForBlockchainId(blockchainId) { return this.model.findOne({ where: { blockchainId, }, order: [['createdAt', 'DESC']], }); } async deleteRandomSamplingChallengeRecord(id, options = {}) { return this.model.destroy({ where: { id }, ...options, }); } async deleteRandomSamplingChallengeForBlockchainIdEpoch(blockchainId, epoch) { return this.model.destroy({ where: { blockchainId, epoch, }, }); } } export default RandomSamplingChallengeRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/shard-repository.js ================================================ import Sequelize from 'sequelize'; class ShardRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.shard; } async createManyPeerRecords(peerRecords, options) { return this.model.bulkCreate(peerRecords, { validate: true, updateOnDuplicate: ['ask', 'stake', 'sha256'], ...options, }); } async removeShardingTablePeerRecords(blockchainId, options) { return this.model.destroy({ where: { blockchainId }, ...options, }); } async createPeerRecord(peerId, blockchainId, ask, stake, lastSeen, sha256, options) { return this.model.create( { peerId, blockchainId, ask, stake, lastSeen, sha256, }, { ignoreDuplicates: true, ...options, }, ); } async getAllPeerRecords(blockchainId, filterInactive, options) { const where = { blockchainId }; if (filterInactive) { where.lastSeen = { [Sequelize.Op.eq]: Sequelize.col('last_dialed') }; } return this.model.findAll({ where, attributes: [ 'peerId', 'blockchainId', 'ask', 'stake', 'lastSeen', 'lastDialed', 'sha256', ], order: [['sha256', 'asc']], ...options, }); } async getPeerRecordsByIds(blockchainId, peerIds, options) { return this.model.findAll({ where: { blockchainId, peerId: { [Sequelize.Op.in]: peerIds, }, }, ...options, }); } async getPeerRecord(peerId, blockchainId, options) { return this.model.findOne({ where: { blockchainId, peerId, }, ...options, }); } async getPeersCount(blockchainId, options) { return this.model.count({ where: { blockchainId, }, ...options, }); } async getPeersToDial(limit, dialFrequencyMillis, options) { const result = await this.model.findAll({ attributes: ['peer_id'], where: { lastDialed: { [Sequelize.Op.lt]: new Date(Date.now() - dialFrequencyMillis), }, }, order: [['last_dialed', 'asc']], group: ['peer_id', 'last_dialed'], limit, raw: true, ...options, }); return (result ?? []).map((record) => ({ peerId: record.peer_id })); } async updatePeerAsk(peerId, blockchainId, ask, options) { return this.model.update( { ask }, { where: { peerId, blockchainId, }, ...options, }, ); } async updatePeerStake(peerId, blockchainId, stake, options) { return this.model.update( { stake }, { where: { peerId, blockchainId, }, ...options, }, ); } async updatePeerRecordLastDialed(peerId, timestamp, options) { return this.model.update( { lastDialed: timestamp, }, { where: { peerId }, ...options, }, ); } async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp, options) { return this.model.update( { lastDialed: timestamp, lastSeen: timestamp, }, { where: { peerId }, ...options, }, ); } async removePeerRecord(blockchainId, peerId, options) { await this.model.destroy({ where: { blockchainId, peerId, }, ...options, }); } async cleanShardingTable(blockchainId, options) { await this.model.destroy({ where: blockchainId ? { blockchainId } : {}, ...options, }); } async isNodePartOfShard(blockchainId, peerId, options) { const nodeIsPartOfShard = await this.model.findOne({ where: { blockchainId, peerId }, ...options, }); return !!nodeIsPartOfShard; } } export default ShardRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/token-repository.js ================================================ import Sequelize from 'sequelize'; class TokenRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.token; } async saveToken(tokenId, userId, tokenName, expiresAt, options) { return this.model.create( { id: tokenId, userId, expiresAt, name: tokenName, }, options, ); } async isTokenRevoked(tokenId, options) { const token = await this.model.findByPk(tokenId, options); return token && token.revoked; } async getTokenAbilities(tokenId, options) { const abilities = await this.sequelize.query( `SELECT a.name FROM token t INNER JOIN user u ON t.user_id = u.id INNER JOIN role r ON u.role_id = u.id INNER JOIN role_ability ra on r.id = ra.role_id INNER JOIN ability a on ra.ability_id = a.id WHERE t.id=$tokenId;`, { bind: { tokenId }, type: Sequelize.QueryTypes.SELECT, ...options, }, ); return abilities.map((e) => e.name); } } export default TokenRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/repositories/user-repository.js ================================================ class UserRepository { constructor(models) { this.sequelize = models.sequelize; this.model = models.user; } async getUser(username, options) { return this.model.findOne({ where: { name: username, }, ...options, }); } } export default UserRepository; ================================================ FILE: src/modules/repository/implementation/sequelize/sequelize-migrator.js ================================================ import { createRequire } from 'module'; import { Umzug, SequelizeStorage } from 'umzug'; import { Sequelize } from 'sequelize'; import path from 'path'; const require = createRequire(import.meta.url); function createMigrator(sequelize, config, logger) { return new Umzug({ migrations: { glob: [ 'migrations/*.{js,cjs,mjs}', { cwd: path.dirname(import.meta.url.replace('file://', '')) }, ], resolve: (params) => { if (params.path.endsWith('.mjs') || params.path.endsWith('.js')) { const getModule = () => import(`file:///${params.path.replace(/\\/g, '/')}`); return { name: params.name, path: params.path, up: async (upParams) => (await getModule()).up(upParams, logger), down: async (downParams) => (await getModule()).down(downParams, logger), }; } return { name: params.name, path: params.path, // eslint-disable-next-line import/no-dynamic-require ...require(params.path), }; }, }, context: { queryInterface: sequelize.getQueryInterface(), Sequelize }, storage: new SequelizeStorage({ sequelize, tableName: 'sequelize_meta' }), logger: config.logging ? config.logger : { info: () => {} }, }); } export default createMigrator; ================================================ FILE: src/modules/repository/implementation/sequelize/sequelize-repository.js ================================================ import mysql from 'mysql2'; import path from 'path'; import fs from 'fs'; import Sequelize from 'sequelize'; import { fileURLToPath } from 'url'; import createMigrator from './sequelize-migrator.js'; import BlockchainEventRepository from './repositories/blockchain-event-repository.js'; import BlockchainRepository from './repositories/blockchain-repository.js'; import CommandRepository from './repositories/command-repository.js'; import EventRepository from './repositories/event-repository.js'; import ParanetRepository from './repositories/paranet-repository.js'; import ParanetKcRepository from './repositories/paranet-kc-repository.js'; import OperationIdRepository from './repositories/operation-id-repository.js'; import OperationRepository from './repositories/operation-repository.js'; import OperationResponseRepository from './repositories/operation-response.js'; import ShardRepository from './repositories/shard-repository.js'; import TokenRepository from './repositories/token-repository.js'; import UserRepository from './repositories/user-repository.js'; // import MissedParanetAssetRepository from './repositories/missed-paranet-asset-repository.js'; // import ParanetSyncedAssetRepository from './repositories/paranet-synced-asset-repository.js'; import TriplesInsertCountRepository from './repositories/inserted-triples-repository.js'; import FinalityStatusRepository from './repositories/finality-status-repository.js'; import RandomSamplingChallengeRepository from './repositories/random-sampling-challenge-repository.js'; import LatestSyncedKcRepository from './repositories/latest-synced-kc-repository.js'; import BlockchainMissedKcRepository from './repositories/blockchain-missed-kc-repository.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); class SequelizeRepository { async initialize(config, logger) { this.config = config; this.logger = logger; this.setEnvParameters(); await this.createDatabaseIfNotExists(); this.initializeSequelize(); await this.runMigrations(); await this.loadModels(); this.repositories = { blockchain_event: new BlockchainEventRepository(this.models), blockchain: new BlockchainRepository(this.models), command: new CommandRepository(this.models), event: new EventRepository(this.models), paranet: new ParanetRepository(this.models), // paranet_synced_asset: new ParanetSyncedAssetRepository(this.models), // missed_paranet_asset: new MissedParanetAssetRepository(this.models), paranet_kc: new ParanetKcRepository(this.models), operation_id: new OperationIdRepository(this.models), operation: new OperationRepository(this.models), operation_response: new OperationResponseRepository(this.models), shard: new ShardRepository(this.models), token: new TokenRepository(this.models), user: new UserRepository(this.models), finality_status: new FinalityStatusRepository(this.models), random_sampling_challenge: new RandomSamplingChallengeRepository(this.models), inserted_triples: new TriplesInsertCountRepository(this.models), latest_synced_kc: new LatestSyncedKcRepository(this.models), blockchain_missed_kc: new BlockchainMissedKcRepository(this.models), }; } initializeSequelize() { this.config.define = { timestamps: false, freezeTableName: true, }; const sequelize = new Sequelize( process.env.SEQUELIZE_REPOSITORY_DATABASE, process.env.SEQUELIZE_REPOSITORY_USER, process.env.SEQUELIZE_REPOSITORY_PASSWORD, this.config, ); this.models = { sequelize, Sequelize }; } setEnvParameters() { process.env.SEQUELIZE_REPOSITORY_USER = this.config.user; process.env.SEQUELIZE_REPOSITORY_PASSWORD = process.env.REPOSITORY_PASSWORD ?? this.config.password; process.env.SEQUELIZE_REPOSITORY_DATABASE = this.config.database; process.env.SEQUELIZE_REPOSITORY_HOST = this.config.host; process.env.SEQUELIZE_REPOSITORY_PORT = this.config.port; process.env.SEQUELIZE_REPOSITORY_DIALECT = this.config.dialect; } async createDatabaseIfNotExists() { const connection = mysql.createConnection({ host: process.env.SEQUELIZE_REPOSITORY_HOST, port: process.env.SEQUELIZE_REPOSITORY_PORT, user: process.env.SEQUELIZE_REPOSITORY_USER, password: process.env.SEQUELIZE_REPOSITORY_PASSWORD, }); await connection .promise() .query(`CREATE DATABASE IF NOT EXISTS \`${this.config.database}\`;`); connection.destroy(); } async dropDatabase() { const connection = mysql.createConnection({ host: process.env.SEQUELIZE_REPOSITORY_HOST, port: process.env.SEQUELIZE_REPOSITORY_PORT, user: process.env.SEQUELIZE_REPOSITORY_USER, password: process.env.SEQUELIZE_REPOSITORY_PASSWORD, }); await connection.promise().query(`DROP DATABASE IF EXISTS \`${this.config.database}\`;`); connection.destroy(); } async runMigrations() { const migrator = createMigrator(this.models.sequelize, this.config, this.logger); try { await migrator.up(); } catch (error) { this.logger.error(`Failed to execute ${migrator.name} migration: ${error.message}.`); await migrator.down(); throw error; } } async loadModels() { const modelsDirectory = path.join(__dirname, 'models'); // disable automatic timestamps const files = (await fs.promises.readdir(modelsDirectory)).filter( (file) => file.indexOf('.') !== 0 && file.slice(-3) === '.js', ); for (const file of files) { // eslint-disable-next-line no-await-in-loop const { default: f } = await import(`./models/${file}`); const model = f(this.models.sequelize, Sequelize.DataTypes); this.models[model.name] = model; } Object.keys(this.models).forEach((modelName) => { if (this.models[modelName].associate) { this.models[modelName].associate(this.models); } }); } async transaction(execFn) { if (execFn) { return this.models.sequelize.transaction(async (t) => execFn(t)); } return this.models.sequelize.transaction(); } getRepository(repositoryName) { return this.repositories[repositoryName]; } async query(query, options) { return this.models.sequelize.query(query, options); } async destroyAllRecords(table, options) { return this.models[table].destroy({ where: {}, ...options }); } } export default SequelizeRepository; ================================================ FILE: src/modules/repository/repository-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class RepositoryModuleManager extends BaseModuleManager { getName() { return 'repository'; } getRepository(repoName) { if (!this.initialized) { throw new Error('RepositoryModuleManager not initialized'); } return this.getImplementation().module.getRepository(repoName); } async transaction(execFn) { if (this.initialized) { return this.getImplementation().module.transaction(execFn); } } async dropDatabase() { if (this.initialized) { return this.getImplementation().module.dropDatabase(); } } async query(query, options = {}) { if (this.initialized) { return this.getImplementation().module.query(query, options); } } async destroyAllRecords(table, options = {}) { if (this.initialized) { return this.getImplementation().module.destroyAllRecords(table, options); } } async updateCommand(update, options = {}) { return this.getRepository('command').updateCommand(update, options); } async destroyCommand(name, options = {}) { return this.getRepository('command').destroyCommand(name, options); } async createCommand(command, options = {}) { return this.getRepository('command').createCommand(command, options); } async getCommandsWithStatus(statusArray, excludeNameArray = [], options = {}) { return this.getRepository('command').getCommandsWithStatus( statusArray, excludeNameArray, options, ); } async getCommandWithId(id, options = {}) { return this.getRepository('command').getCommandWithId(id, options); } async removeCommands(ids, options = {}) { return this.getRepository('command').removeCommands(ids, options); } async findFinalizedCommands(timestamp, limit, options = {}) { return this.getRepository('command').findFinalizedCommands(timestamp, limit, options); } async findAndRemoveFinalizedCommands(timestamp, limit, options = {}) { return this.getRepository('command').findAndRemoveFinalizedCommands( timestamp, limit, options, ); } async findUnfinalizedCommandsByName(limit, options = {}) { return this.getRepository('command').findUnfinalizedCommandsByName(limit, options); } async createOperationIdRecord(handlerData, options = {}) { return this.getRepository('operation_id').createOperationIdRecord(handlerData, options); } async updateOperationIdRecord(data, operationId, options = {}) { return this.getRepository('operation_id').updateOperationIdRecord( data, operationId, options, ); } async getOperationIdRecord(operationId, options = {}) { return this.getRepository('operation_id').getOperationIdRecord(operationId, options); } async removeOperationIdRecord(timeToBeDeleted, statuses, options = {}) { return this.getRepository('operation_id').removeOperationIdRecord( timeToBeDeleted, statuses, options, ); } async updateMinAcksReached(operationId, minAcksReached, options = {}) { return this.getRepository('operation_id').updateMinAcksReached( operationId, minAcksReached, options, ); } async createOperationRecord(operation, operationId, status, options = {}) { return this.getRepository('operation').createOperationRecord( operation, operationId, status, options, ); } async removeOperationRecords(operation, ids, options = {}) { return this.getRepository('operation').removeOperationRecords(operation, ids, options); } async findProcessedOperations(operation, timestamp, limit, options = {}) { return this.getRepository('operation').findProcessedOperations( operation, timestamp, limit, options, ); } async findAndRemoveProcessedOperationRecords(operation, timestamp, limit, options = {}) { return this.getRepository('operation').findAndRemoveProcessedOperationRecords( operation, timestamp, limit, options, ); } async getOperationStatus(operation, operationId, options = {}) { return this.getRepository('operation').getOperationStatus(operation, operationId, options); } async updateOperationStatus(operation, operationId, status, options = {}) { return this.getRepository('operation').updateOperationStatus( operation, operationId, status, options, ); } async createOperationResponseRecord(status, operation, operationId, errorMessage, options) { return this.getRepository('operation_response').createOperationResponseRecord( status, operation, operationId, errorMessage, options, ); } async getOperationResponsesStatuses(operation, operationId, options = {}) { return this.getRepository('operation_response').getOperationResponsesStatuses( operation, operationId, options, ); } async findProcessedOperationResponse(timestamp, limit, operation, options = {}) { return this.getRepository('operation_response').findProcessedOperationResponse( timestamp, limit, operation, options, ); } async findAndRemoveProcessedOperationResponse(operation, timestamp, limit, options = {}) { return this.getRepository('operation_response').findAndRemoveProcessedOperationResponse( operation, timestamp, limit, options, ); } async removeOperationResponse(ids, operation, options = {}) { return this.getRepository('operation_response').removeOperationResponse( ids, operation, options, ); } async createManyPeerRecords(peers, options = {}) { return this.getRepository('shard').createManyPeerRecords(peers, options); } async removeShardingTablePeerRecords(blockchain, options = {}) { return this.getRepository('shard').removeShardingTablePeerRecords(blockchain, options); } async createPeerRecord(peerId, blockchain, ask, stake, lastSeen, sha256, options = {}) { return this.getRepository('shard').createPeerRecord( peerId, blockchain, ask, stake, lastSeen, sha256, options, ); } async getPeerRecord(peerId, blockchain, options = {}) { return this.getRepository('shard').getPeerRecord(peerId, blockchain, options); } async getAllPeerRecords(blockchain, filterInactive = false, options = {}) { return this.getRepository('shard').getAllPeerRecords(blockchain, filterInactive, options); } async getPeerRecordsByIds(blockchain, peerIds, options = {}) { return this.getRepository('shard').getPeerRecordsByIds(blockchain, peerIds, options); } async getPeersCount(blockchain, options = {}) { return this.getRepository('shard').getPeersCount(blockchain, options); } async getPeersToDial(limit, dialFrequencyMillis, options = {}) { return this.getRepository('shard').getPeersToDial(limit, dialFrequencyMillis, options); } async removePeerRecord(blockchain, peerId, options = {}) { return this.getRepository('shard').removePeerRecord(blockchain, peerId, options); } async updatePeerRecordLastDialed(peerId, timestamp, options = {}) { return this.getRepository('shard').updatePeerRecordLastDialed(peerId, timestamp, options); } async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp, options = {}) { return this.getRepository('shard').updatePeerRecordLastSeenAndLastDialed( peerId, timestamp, options, ); } async updatePeerAsk(peerId, blockchainId, ask, options = {}) { return this.getRepository('shard').updatePeerAsk(peerId, blockchainId, ask, options); } async updatePeerStake(peerId, blockchainId, stake, options = {}) { return this.getRepository('shard').updatePeerStake(peerId, blockchainId, stake, options); } async getNeighbourhood(assertionId, r2, options = {}) { return this.getRepository('shard').getNeighbourhood(assertionId, r2, options); } async cleanShardingTable(blockchainId, options = {}) { return this.getRepository('shard').cleanShardingTable(blockchainId, options); } async isNodePartOfShard(blockchainId, peerId, options = {}) { return this.getRepository('shard').isNodePartOfShard(blockchainId, peerId, options); } async destroyEvents(ids, options = {}) { return this.getRepository('event').destroyEvents(ids, options); } async getUser(username, options = {}) { return this.getRepository('user').getUser(username, options); } async saveToken(tokenId, userId, tokenName, expiresAt, options = {}) { return this.getRepository('token').saveToken( tokenId, userId, tokenName, expiresAt, options, ); } async isTokenRevoked(tokenId, options = {}) { return this.getRepository('token').isTokenRevoked(tokenId, options); } async getTokenAbilities(tokenId, options = {}) { return this.getRepository('token').getTokenAbilities(tokenId, options); } async insertBlockchainEvents(events, options = {}) { return this.getRepository('blockchain_event').insertBlockchainEvents(events, options); } async getAllUnprocessedBlockchainEvents(blockchain, eventNames, options = {}) { return this.getRepository('blockchain_event').getAllUnprocessedBlockchainEvents( blockchain, eventNames, options, ); } async markAllBlockchainEventsAsProcessed(blockchain, options = {}) { return this.getRepository('blockchain_event').markAllBlockchainEventsAsProcessed( blockchain, options, ); } async removeEvents(ids, options = {}) { return this.getRepository('blockchain_event').removeEvents(ids, options); } async removeContractEventsAfterBlock( blockchain, contract, contractAddress, blockNumber, transactionIndex, options = {}, ) { return this.getRepository('blockchain_event').removeContractEventsAfterBlock( blockchain, contract, contractAddress, blockNumber, transactionIndex, options, ); } async findProcessedEvents(timestamp, limit, options = {}) { return this.getRepository('blockchain_event').findProcessedEvents( timestamp, limit, options, ); } async findAndRemoveProcessedEvents(timestamp, limit, options = {}) { return this.getRepository('blockchain_event').findAndRemoveProcessedEvents( timestamp, limit, options, ); } async getLastCheckedBlock(blockchain, options = {}) { return this.getRepository('blockchain').getLastCheckedBlock(blockchain, options); } async updateLastCheckedBlock(blockchain, currentBlock, timestamp, options = {}) { return this.getRepository('blockchain').updateLastCheckedBlock( blockchain, currentBlock, timestamp, options, ); } async addToParanetKaCount(paranetId, blockchainId, kaCount, options = {}) { return this.getRepository('paranet').addToParanetKaCount( paranetId, blockchainId, kaCount, options, ); } async createParanetRecord(name, description, paranetId, blockchainId, options = {}) { this.getRepository('paranet').createParanetRecord( name, description, paranetId, blockchainId, options, ); } async paranetExists(paranetId, blockchainId, options = {}) { return this.getRepository('paranet').paranetExists(paranetId, blockchainId, options); } async getParanet(paranetId, blockchainId, options = {}) { return this.getRepository('paranet').getParanet(paranetId, blockchainId, options); } async getParanetKnowledgeAssetsCount(paranetId, blockchainId, options = {}) { return this.getRepository('paranet').getParanetKnowledgeAssetsCount( paranetId, blockchainId, options, ); } async getParanetsBlockchains(options = {}) { return this.getRepository('paranet').getParanetsBlockchains(options); } async incrementParanetKaCount(paranetId, blockchainId, options = {}) { return this.getRepository('paranet').incrementParanetKaCount( paranetId, blockchainId, options, ); } async createParanetKcRecords(paranetUal, blockchainId, uals, options = {}) { return this.getRepository('paranet_kc').createParanetKcRecords( paranetUal, blockchainId, uals, options, ); } async getParanetKcCount(paranetUal, options = {}) { return this.getRepository('paranet_kc').getCount(paranetUal, options); } async getParanetKcSyncedCount(paranetUal, options = {}) { return this.getRepository('paranet_kc').getCountSynced(paranetUal, options); } async getParanetKcUnsyncedCount(paranetUal, options = {}) { return this.getRepository('paranet_kc').getCountUnsynced(paranetUal, options); } async getParanetKcSyncBatch(paranetUal, retriesMax, retryDelayMs, limit = null, options = {}) { return this.getRepository('paranet_kc').getSyncBatch( paranetUal, retriesMax, retryDelayMs, limit, options, ); } async paranetKcIncrementRetries(paranetUal, ual, errorMessage = null, options = {}) { return this.getRepository('paranet_kc').incrementRetries( paranetUal, ual, errorMessage, options, ); } async paranetKcMarkAsSynced(paranetUal, ual, options = {}) { return this.getRepository('paranet_kc').markAsSynced(paranetUal, ual, options); } async getFinalityAcksCount(ual, options = {}) { return this.getRepository('finality_status').getFinalityAcksCount(ual, options); } async getPublishOperationIdByUal(ual, options = {}) { return this.getRepository('finality_status').getPublishOperationIdByUal(ual, options); } async getLatestRandomSamplingChallengeRecordForBlockchainId(blockchainId, limit = 1) { return this.getRepository( 'random_sampling_challenge', ).getLatestRandomSamplingChallengeRecordForBlockchainId(blockchainId, limit); } async createRandomSamplingChallengeRecord(randomSamplingChallenge, options) { return this.getRepository('random_sampling_challenge').createRandomSamplingChallengeRecord( randomSamplingChallenge, options, ); } async updateRandomSamplingChallengeRecord(randomSamplingChallenge, options) { return this.getRepository('random_sampling_challenge').updateRandomSamplingChallengeRecord( randomSamplingChallenge, options, ); } async deleteRandomSamplingChallengeRecord(randomSamplingChallengeId, options) { return this.getRepository('random_sampling_challenge').deleteRandomSamplingChallengeRecord( randomSamplingChallengeId, options, ); } async setCompletedAndScoreRandomSamplingChallengeRecord( randomSamplingChallengeId, completed, score, options, ) { return this.getRepository( 'random_sampling_challenge', ).setCompletedAndScoreRandomSamplingChallengeRecord( randomSamplingChallengeId, completed, score, options, ); } async setCompletedAndFinalizedRandomSamplingChallengeRecord( randomSamplingChallengeId, completed, finalized, options, ) { return this.getRepository( 'random_sampling_challenge', ).setCompletedAndFinalizedRandomSamplingChallengeRecord( randomSamplingChallengeId, completed, finalized, options, ); } async saveFinalityAck(publishOperationId, ual, peerId, options = {}) { return this.getRepository('finality_status').saveFinalityAck( publishOperationId, ual, peerId, options, ); } async incrementInsertedTriples(count) { return this.getRepository('inserted_triples').increment(count); } async getKCStorageContracts(blockchainId) { return this.getRepository('latest_synced_kc').getKCStorageContracts(blockchainId); } async getSyncRecordForBlockchain(blockchainId) { return this.getRepository('latest_synced_kc').getSyncRecordForBlockchain(blockchainId); } async addSyncContracts(blockchainId, contracts) { return this.getRepository('latest_synced_kc').addSyncContracts(blockchainId, contracts); } async insertMissedKc(blockchainId, records, error, options = {}) { return this.getRepository('blockchain_missed_kc').insertMissedKc( blockchainId, records, error, options, ); } async getMissedKcForRetry(blockchain, contractAddress, limit, options) { return this.getRepository('blockchain_missed_kc').getMissedKcForRetry( blockchain, contractAddress, limit, options, ); } async updateLatestSyncedKc(blockchainId, contractAddress, latestSyncedKc, options = {}) { return this.getRepository('latest_synced_kc').updateLatestSyncedKc( blockchainId, contractAddress, latestSyncedKc, options, ); } async incrementRetryCount(blockchain, records, options) { return this.getRepository('blockchain_missed_kc').incrementRetryCount( blockchain, records, options, ); } async setSyncedToTrue(blockchain, records, options) { return this.getRepository('blockchain_missed_kc').setSyncedToTrue( blockchain, records, options, ); } async getMissedKcForRetryCount(blockchain, contractAddress, options) { return this.getRepository('blockchain_missed_kc').getMissedKcForRetryCount( blockchain, contractAddress, options, ); } } export default RepositoryModuleManager; ================================================ FILE: src/modules/telemetry/implementation/quest-telemetry.js ================================================ import { Sender } from '@questdb/nodejs-client'; class QuestTelemetry { async initialize(config, logger) { this.config = config; this.logger = logger; this.localSender = Sender.fromConfig(this.config.localEndpoint); if (this.config.sendToSignalingService) { this.signalingServiceSender = Sender.fromConfig(this.config.signalingServiceEndpoint); } } listenOnEvents(eventEmitter, onEventReceived) { return eventEmitter.on('operation_status_changed', onEventReceived); } async sendTelemetryData( operationId, timestamp, blockchainId = '', name = '', value1 = null, value2 = null, value3 = null, ) { try { const table = this.localSender.table('event'); table.symbol('operationId', operationId || 'NULL'); table.symbol('blockchainId', blockchainId || 'NULL'); table.symbol('name', name || 'NULL'); if (value1 !== null) table.symbol('value1', value1); if (value2 !== null) table.symbol('value2', value2); if (value3 !== null) table.symbol('value3', value3); table.timestampColumn('timestamp', timestamp * 1000); await table.at(Date.now(), 'ms'); await this.localSender.flush(); await this.localSender.close(); // this.logger.info('Event telemetry successfully sent to local QuestDB'); } catch (err) { this.logger.error(`Error sending telemetry to local QuestDB: ${err.message}`); } } } export default QuestTelemetry; ================================================ FILE: src/modules/telemetry/telemetry-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class TelemetryModuleManager extends BaseModuleManager { constructor(ctx) { super(ctx); this.eventEmitter = ctx.eventEmitter; } getName() { return 'telemetry'; } async initialize() { await super.initialize(); this.listenOnEvents((eventData) => { this.sendTelemetryData( eventData.operationId, eventData.timestamp, eventData.blockchainId, eventData.lastEvent, eventData.value1, eventData.value2, eventData.value3, ); }); } listenOnEvents(onEventReceived) { if (this.config.modules.telemetry.enabled && this.initialized) { return this.getImplementation().module.listenOnEvents( this.eventEmitter, onEventReceived, ); } } async sendTelemetryData(operationId, timestamp, blockchainId, name, value1, value2, value3) { if (this.config.modules.telemetry.enabled && this.initialized) { return this.getImplementation().module.sendTelemetryData( operationId, timestamp, blockchainId, name, value1, value2, value3, ); } } } export default TelemetryModuleManager; ================================================ FILE: src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js ================================================ import axios from 'axios'; import OtTripleStore from '../ot-triple-store.js'; import { MEDIA_TYPES } from '../../../../constants/constants.js'; class OtBlazegraph extends OtTripleStore { async initialize(config, logger) { await super.initialize(config, logger); // this regex will match \Uxxxxxxxx but will exclude cases where there is a double slash before U (\\U) this.unicodeRegex = /(? { await this.createRepository(repository); }), ); } async createRepository(repository) { const { url, name } = this.repositories[repository]; if (!(await this.repositoryExists(repository))) { await axios.post( `${url}/blazegraph/namespace`, `com.bigdata.rdf.sail.truthMaintenance=false\n` + `com.bigdata.namespace.${name}.spo.com.bigdata.btree.BTree.branchingFactor=1024\n` + `com.bigdata.rdf.store.AbstractTripleStore.textIndex=false\n` + `com.bigdata.rdf.store.AbstractTripleStore.justify=false\n` + `com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=false\n` + `com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms\n` + `com.bigdata.rdf.sail.namespace=${name}\n` + `com.bigdata.rdf.store.AbstractTripleStore.quads=true\n` + `com.bigdata.namespace.${name}.lex.com.bigdata.btree.BTree.branchingFactor=400\n` + `com.bigdata.rdf.store.AbstractTripleStore.geoSpatial=false\n` + `com.bigdata.journal.Journal.groupCommit=false\n` + `com.bigdata.rdf.sail.isolatableIndices=false\n` + `com.bigdata.rdf.store.AbstractTripleStore.enableRawRecordsSupport=false\n` + `com.bigdata.rdf.store.AbstractTripleStore.Options.inlineTextLiterals=true\n` + `com.bigdata.rdf.store.AbstractTripleStore.Options.maxInlineTextLength=128\n` + `com.bigdata.rdf.store.AbstractTripleStore.Options.blobsThreshold=256\n`, { headers: { 'Content-Type': 'text/plain', }, }, ); } } initializeSparqlEndpoints(repository) { const { url, name } = this.repositories[repository]; this.repositories[repository].sparqlEndpoint = `${url}/blazegraph/namespace/${name}/sparql`; this.repositories[ repository ].sparqlEndpointUpdate = `${url}/blazegraph/namespace/${name}/sparql`; } getRepositoryUrl(repository) { return this.repositories[repository].url; } hasUnicodeCodePoints(input) { return this.unicodeRegex.test(input); } decodeUnicodeCodePoints(input) { const decodedString = input.replace(this.unicodeRegex, (match, hex) => { const codePoint = parseInt(hex, 16); return String.fromCodePoint(codePoint); }); return decodedString; } utfConverter(input) { return Buffer.from(input, 'utf8').toString(); } async construct(repository, query, timeout) { return this._executeQuery(repository, query, MEDIA_TYPES.N_QUADS, timeout); } async select(repository, query, timeout) { const result = await this._executeQuery(repository, query, MEDIA_TYPES.JSON, timeout); return result ? JSON.parse(result) : []; } async ask(repository, query, timeout = 10000) { const result = await this._executeQuery(repository, query, MEDIA_TYPES.JSON, timeout); return result ? JSON.parse(result).boolean : false; } async _executeQuery(repository, query, mediaType, timeout) { const result = await axios.post(this.repositories[repository].sparqlEndpoint, query, { headers: { 'Content-Type': 'application/sparql-query', 'X-BIGDATA-MAX-QUERY-MILLIS': timeout, Accept: mediaType, }, }); let response; if (mediaType === MEDIA_TYPES.JSON) { // Check if this is an ASK query by looking for the boolean property if (result.data.boolean !== undefined) { // This is an ASK query response response = JSON.stringify(result.data); } else { // This is a SELECT query response const { bindings } = result.data.results; let output = '[\n'; bindings.forEach((binding, bindingIndex) => { let string = ' {\n'; const keys = Object.keys(binding); keys.forEach((key, index) => { let value = ''; const entry = binding[key]; if (entry.datatype) { // e.g., "\"6900000\"^^http://www.w3.org/2001/XMLSchema#integer" const literal = `"${entry.value}"^^${entry.datatype}`; value = JSON.stringify(literal); } else if (entry['xml:lang']) { // e.g., "\"text here\"@en" const literal = `"${entry.value}"@${entry['xml:lang']}`; value = JSON.stringify(literal); } else if (entry.type === 'uri') { // URIs should be escaped and quoted directly value = JSON.stringify(entry.value); } else { // For plain literals, wrap in quotes and stringify const literal = `"${entry.value}"`; value = JSON.stringify(literal); } const isLast = index === keys.length - 1; string += ` "${key}": ${value}${isLast ? '' : ','}\n`; }); const isLastBinding = bindingIndex === bindings.length - 1; string += ` }${isLastBinding ? '\n' : ',\n'}`; output += string; }); output += ']'; response = output; } } else { response = result.data; } // Handle Blazegraph special characters corruption if (this.hasUnicodeCodePoints(response)) { response = this.decodeUnicodeCodePoints(response); } response = this.utfConverter(response); return response; } async healthCheck(repository) { try { const response = await axios.get( `${this.repositories[repository].url}/blazegraph/status`, {}, ); if (response.data !== null) { return true; } return false; } catch (e) { return false; } } async queryVoid(repository, query, timeout) { try { return await axios.post(this.repositories[repository].sparqlEndpoint, query, { headers: { 'Content-Type': 'application/sparql-update; charset=UTF-8', 'X-BIGDATA-MAX-QUERY-MILLIS': timeout, }, }); } catch (error) { const status = error?.response?.status; const dataSnippet = typeof error?.response?.data === 'string' ? error.response.data.slice(0, 200) : ''; this.logger.error( `[OtBlazegraph.queryVoid] Update failed for ${repository} (status: ${status}): ${ error.message }${dataSnippet ? ` | data: ${dataSnippet}` : ''}`, ); throw error; } } async deleteRepository(repository) { const { url, name } = this.repositories[repository]; this.logger.info( `Deleting ${this.getName()} triple store repository: ${repository} with name: ${name}`, ); if (await this.repositoryExists(repository)) { await axios .delete(`${url}/blazegraph/namespace/${name}`, {}) .catch((e) => this.logger.error( `Error while deleting ${this.getName()} triple store repository: ${repository} with name: ${name}. Error: ${ e.message }`, ), ); } } async repositoryExists(repository) { const { url, name } = this.repositories[repository]; try { await axios.get(`${url}/blazegraph/namespace/${name}/properties`, { params: { 'describe-each-named-graph': 'false', }, headers: { Accept: 'application/ld+json', }, }); return true; } catch (error) { if (error.response && error.response.status === 404) { // Expected error: GraphDB is up but has not created node0 repository // dkg-engine will create repo in initialization return false; } this.logger.error( `Error while getting ${this.getName()} repositories. Error: ${error.message}`, ); return false; } } getName() { return 'OtBlazegraph'; } } export default OtBlazegraph; ================================================ FILE: src/modules/triple-store/implementation/ot-fuseki/ot-fuseki.js ================================================ import axios from 'axios'; import OtTripleStore from '../ot-triple-store.js'; class OtFuseki extends OtTripleStore { async initialize(config, logger) { await super.initialize(config, logger); await Promise.all( Object.keys(this.repositories).map(async (repository) => { await this.createRepository(repository); }), ); } async createRepository(repository) { const { url, name } = this.repositories[repository]; if (!(await this.repositoryExists(repository))) { await axios.post( `${url}/$/datasets?dbName=${name}&dbType=tdb`, {}, { headers: { 'Content-Type': 'text/plain', }, }, ); } } initializeSparqlEndpoints(repository) { const { url, name } = this.repositories[repository]; this.repositories[repository].sparqlEndpoint = `${url}/${name}/sparql`; this.repositories[repository].sparqlEndpointUpdate = `${url}/${name}/update`; } async healthCheck(repository) { try { const response = await axios.get(`${this.repositories[repository].url}/$/ping`, {}); if (response.data !== null) { return true; } return false; } catch (e) { return false; } } async deleteRepository(repository) { const { url, name } = this.repositories[repository]; this.logger.info( `Deleting ${this.getName()} triple store repository: ${repository} with name: ${name}`, ); if (await this.repositoryExists(repository)) { await axios .delete(`${url}/$/datasets/${name}`, {}) .catch((e) => this.logger.error( `Error while deleting ${this.getName()} triple store repository: ${repository} with name: ${name}. Error: ${ e.message }`, ), ); } } async repositoryExists(repository) { const { url, name } = this.repositories[repository]; try { const response = await axios.get(`${url}/$/datasets`); return response.data.datasets.filter((dataset) => dataset['ds.name'] === `/${name}`) .length; } catch (error) { this.logger.error( `Error while getting ${this.getName()} repositories. Error: ${error.message}`, ); return false; } } getName() { return 'OtFuseki'; } } export default OtFuseki; ================================================ FILE: src/modules/triple-store/implementation/ot-graphdb/ot-graphdb.js ================================================ import graphdb from 'graphdb'; import axios from 'axios'; import OtTripleStore from '../ot-triple-store.js'; const { server, repository: repo, http } = graphdb; class OtGraphdb extends OtTripleStore { async initialize(config, logger) { await super.initialize(config, logger); await Promise.all( Object.keys(this.repositories).map(async (repository) => { await this.createRepository(repository); }), ); } async createRepository(repository) { const { url, name } = this.repositories[repository]; const serverConfig = new server.ServerClientConfig(url) .setTimeout(40000) .setHeaders({ Accept: http.RDFMimeType.N_QUADS, }) .setKeepAlive(true); const s = new server.GraphDBServerClient(serverConfig); // eslint-disable-next-line no-await-in-loop const exists = await s.hasRepository(name); if (!exists) { try { // eslint-disable-next-line no-await-in-loop await s.createRepository( new repo.RepositoryConfig( name, '', new Map(), '', 'Repo title', repo.RepositoryType.FREE, ), ); } catch (e) { // eslint-disable-next-line no-await-in-loop await s.createRepository( new repo.RepositoryConfig( name, '', {}, 'graphdb:SailRepository', 'Repo title', 'graphdb', ), ); } } } initializeSparqlEndpoints(repository) { const { url, name } = this.repositories[repository]; this.repositories[repository].sparqlEndpoint = `${url}/repositories/${name}`; this.repositories[ repository ].sparqlEndpointUpdate = `${url}/repositories/${name}/statements`; } async healthCheck(repository) { const { url, username, password } = this.repositories[repository]; try { const response = await axios.get( `${url}/repositories/${repository}/health`, {}, { auth: { username, password, }, }, ); if (response.data.status === 'green') { return true; } return false; } catch (e) { if (e.response && e.response.status === 404) { // Expected error: GraphDB is up but has not created node0 repository // dkg-engine will create repo in initialization return true; } return false; } } async deleteRepository(repository) { const { url, name } = this.repositories[repository]; this.logger.info( `Deleting ${this.getName()} triple store repository: ${repository} with name: ${name}`, ); const serverConfig = new server.ServerClientConfig(url) .setTimeout(40000) .setHeaders({ Accept: http.RDFMimeType.N_QUADS, }) .setKeepAlive(true); const s = new server.GraphDBServerClient(serverConfig); s.deleteRepository(name).catch((e) => this.logger.warn( `Error while deleting ${this.getName()} triple store repository: ${repository} with name: ${name}. Error: ${ e.message }`, ), ); } async repositoryExists(repository) { const { url, name } = this.repositories[repository]; const serverConfig = new server.ServerClientConfig(url) .setTimeout(40000) .setHeaders({ Accept: http.RDFMimeType.N_QUADS, }) .setKeepAlive(true); const s = new server.GraphDBServerClient(serverConfig); return s.hasRepository(name); } getName() { return 'GraphDB'; } async queryVoid(repository, query) { const endpoint = `${this.repositories[repository].url}/repositories/${repository}/statements`; try { await axios.post(endpoint, query, { headers: { 'Content-Type': 'application/sparql-update', }, }); return true; } catch (error) { console.error(`SPARQL update failed: ${error.message}`); throw error; } } async ask(repository, query) { const endpoint = `${this.repositories[repository].url}/repositories/${repository}`; try { const response = await axios.post(endpoint, query, { headers: { 'Content-Type': 'application/sparql-query', Accept: 'application/json', }, }); return response.data.boolean; // true or false } catch (error) { console.error(`ASK query failed: ${error.message}`); throw error; } } async construct(repository, query, accept = 'application/n-triples') { const endpoint = `${this.repositories[repository].url}/repositories/${repository}`; try { const response = await axios.post(endpoint, query, { headers: { 'Content-Type': 'application/sparql-query', Accept: accept, }, }); return response.data; } catch (error) { console.error(`SPARQL query failed: ${error.message}`); throw error; } } async select(repository, query) { // todo: add media type once bug is fixed // no media type is passed because of comunica bug // https://github.com/comunica/comunica/issues/1034 const result = await this._executeQuery(repository, query); return result ?? []; } async _executeQuery(repository, query, mediaType, accept = 'application/sparql-results+json') { const endpoint = `${this.repositories[repository].url}/repositories/${repository}`; try { const response = await axios.post(endpoint, query, { headers: { 'Content-Type': 'application/sparql-query', Accept: accept, }, }); const result = []; for (const elem of response.data.results.bindings) { const obj = {}; Object.keys(elem).forEach((key) => { obj[key] = elem[key].value; }); result.push(obj); } return result; } catch (error) { console.error(`SPARQL query failed: ${error.message}`); throw error; } } } export default OtGraphdb; ================================================ FILE: src/modules/triple-store/implementation/ot-neptune/ot-neptune.js ================================================ import axios from 'axios'; import OtTripleStore from '../ot-triple-store.js'; class OtNeptune extends OtTripleStore { async initialize(config, logger) { await super.initialize(config, logger); } /* eslint-disable-next-line no-unused-vars */ async createRepository(repository) { /* eslint-disable-next-line no-empty-function */ } initializeSparqlEndpoints(repository) { /* eslint-disable-next-line no-unused-vars */ const { url, name } = this.repositories[repository]; this.repositories[repository].sparqlEndpoint = `${url}/sparql`; this.repositories[repository].sparqlEndpointUpdate = `${url}/sparql`; } /* eslint-disable-next-line no-unused-vars */ async deleteRepository(repository) { /* eslint-disable-next-line no-empty-function */ } async healthCheck(repository) { try { const response = await axios.get(`${this.repositories[repository].url}/status`); if (response.data && response.data.status === 'healthy') { return true; } return false; } catch (e) { return false; } } getName() { return 'OtNeptune'; } } export default OtNeptune; ================================================ FILE: src/modules/triple-store/implementation/ot-triple-store.js ================================================ import { QueryEngine as Engine } from '@comunica/query-sparql'; import axios from 'axios'; import { setTimeout } from 'timers/promises'; import { SCHEMA_CONTEXT, TRIPLE_STORE_CONNECT_MAX_RETRIES, TRIPLE_STORE_CONNECT_RETRY_FREQUENCY, MEDIA_TYPES, UAL_PREDICATE, BASE_NAMED_GRAPHS, TRIPLE_ANNOTATION_LABEL_PREDICATE, TRIPLES_VISIBILITY, DKG_PREDICATE, HAS_KNOWLEDGE_ASSET_SUFFIX, HAS_NAMED_GRAPH_SUFFIX, DKG_METADATA_PREDICATES, } from '../../../constants/constants.js'; class OtTripleStore { async initialize(config, logger) { this.logger = logger; this.repositories = config.repositories; this.initializeRepositories(); this.initializeContexts(); await this.ensureConnections(); this.queryEngine = new Engine(); } initializeRepositories() { for (const repository of Object.keys(this.repositories)) { this.initializeSparqlEndpoints(repository); } } async initializeParanetRepository(repository) { const publicCurrent = 'publicCurrent'; this.repositories[repository] = { url: this.repositories[publicCurrent].url, name: repository, username: this.repositories[publicCurrent].username, password: this.repositories[publicCurrent].password, }; this.initializeSparqlEndpoints(repository); this.initializeContexts(); await this.ensureConnections(); await this.createRepository(repository); } repositoryInitilized(repository) { return Boolean(this.repositories && this.repositories[repository]); } async createRepository() { throw Error('CreateRepository not implemented'); } initializeSparqlEndpoints() { throw Error('initializeSparqlEndpoints not implemented'); } async deleteRepository() { throw Error('deleteRepository not implemented'); } initializeContexts() { for (const repository in this.repositories) { const sources = [ { type: 'sparql', value: this.repositories[repository].sparqlEndpoint, }, ]; this.repositories[repository].updateContext = { sources, destination: { type: 'sparql', value: this.repositories[repository].sparqlEndpointUpdate, }, httpTimeout: 60_000, httpBodyTimeout: true, }; this.repositories[repository].queryContext = { sources, httpTimeout: 60_000, httpBodyTimeout: true, }; } } async ensureConnections() { const ensureConnectionPromises = Object.keys(this.repositories).map(async (repository) => { let ready = await this.healthCheck(repository); let retries = 0; while (!ready && retries < TRIPLE_STORE_CONNECT_MAX_RETRIES) { retries += 1; this.logger.warn( `Cannot connect to Triple store (${this.getName()}), repository: ${repository}, located at: ${ this.repositories[repository].url } retry number: ${retries}/${TRIPLE_STORE_CONNECT_MAX_RETRIES}. Retrying in ${TRIPLE_STORE_CONNECT_RETRY_FREQUENCY} seconds.`, ); /* eslint-disable no-await-in-loop */ await setTimeout(TRIPLE_STORE_CONNECT_RETRY_FREQUENCY * 1000); ready = await this.healthCheck(repository); } if (retries === TRIPLE_STORE_CONNECT_MAX_RETRIES) { this.logger.error( `Triple Store (${this.getName()}) not available, max retries reached.`, ); process.exit(1); } }); await Promise.all(ensureConnectionPromises); } async insertAssertionBatch( repository, insertMap, metadata, createdMetadata, currentNamedGraphTriples, timeout, ) { const graphsForDataInsert = []; for (const [ual, triples] of Object.entries(insertMap)) { const graph = ` GRAPH <${ual}> { ${triples.join('\n')} } `; graphsForDataInsert.push(graph); } const metadataGraphForInsert = ` GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { ${Object.values(metadata) .map((triples) => triples.join('\n')) .join('\n')} ${createdMetadata.join('\n')} } `; const currentNamedGraphInsert = ` GRAPH <${BASE_NAMED_GRAPHS.CURRENT}> { ${currentNamedGraphTriples.join('\n')} } `; const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> INSERT DATA { ${graphsForDataInsert.join('\n')} ${metadataGraphForInsert} ${currentNamedGraphInsert} } `; await this.queryVoid(repository, query, timeout); } async deleteUniqueKnowledgeCollectionTriplesFromUnifiedGraph(repository, namedGraph, ual) { const query = ` DELETE { GRAPH <${namedGraph}> { ?s ?p ?o . << ?s ?p ?o >> ?annotationPredicate ?annotationValue . } } WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} ?annotationValue . } FILTER(STRSTARTS(STR(?annotationValue), "${ual}/")) { SELECT ?s ?p ?o (COUNT(?annotationValue) AS ?annotationCount) WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} ?annotationValue . } } GROUP BY ?s ?p ?o HAVING(?annotationCount = 1) } } `; await this.queryVoid(repository, query); } async getKnowledgeCollectionFromUnifiedGraph(repository, namedGraph, ual, sort) { const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o . } WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} ?ual . FILTER(STRSTARTS(STR(?ual), "${ual}/")) } } ${sort ? 'ORDER BY ?s' : ''} `; return this.construct(repository, query); } async getKnowledgeCollectionPublicFromUnifiedGraph(repository, namedGraph, ual, sort) { const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} ?ual . FILTER(STRSTARTS(STR(?ual), "${ual}/")) FILTER NOT EXISTS { << ?s ?p ?o >> ${TRIPLE_ANNOTATION_LABEL_PREDICATE} "private" . } } } ${sort ? 'ORDER BY ?s' : ''} `; return this.construct(repository, query); } async knowledgeCollectionExistsInUnifiedGraph(repository, namedGraph, ual) { const query = ` ASK WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} ?ual FILTER(STRSTARTS(STR(?ual), "${ual}/")) } } `; return this.ask(repository, query); } async deleteUniqueKnowledgeAssetTriplesFromUnifiedGraph(repository, namedGraph, ual) { const query = ` DELETE { GRAPH <${namedGraph}> { ?s ?p ?o . << ?s ?p ?o >> ?annotationPredicate ?annotationValue . } } WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} <${ual}> . } { SELECT ?s ?p ?o (COUNT(?annotationValue) AS ?annotationCount) WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} ?annotationValue . } } GROUP BY ?s ?p ?o HAVING(?annotationCount = 1) } } `; await this.queryVoid(repository, query); } async getKnowledgeAssetFromUnifiedGraph(repository, namedGraph, ual) { const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o . } WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} <${ual}> . } } `; return this.construct(repository, query); } async getKnowledgeAssetPublicFromUnifiedGraph(repository, namedGraph, ual) { const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} <${ual}> . FILTER NOT EXISTS { << ?s ?p ?o >> ${TRIPLE_ANNOTATION_LABEL_PREDICATE} "private" . } } } `; return this.construct(repository, query); } async knowledgeAssetExistsInUnifiedGraph(repository, namedGraph, ual) { const query = ` ASK WHERE { GRAPH <${namedGraph}> { << ?s ?p ?o >> ${UAL_PREDICATE} <${ual}> } } `; return this.ask(repository, query); } async createKnowledgeCollectionNamedGraphs( repository, uals, assetsNQuads, visibility, timeout, retries = 5, retryDelay = 10, ) { const graphInserts = uals .map( (ual, index) => ` GRAPH <${ual}/${visibility}> { ${assetsNQuads[index].join('\n')} } `, ) .join('\n'); const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> INSERT DATA { ${graphInserts} } `; let attempts = 0; let success = false; while (attempts < retries && !success) { try { await this.queryVoid(repository, query, timeout); success = true; } catch (error) { attempts += 1; if (attempts <= retries) { this.logger.warn( `Batch insert failed for ${uals[0] .split('/') .slice(0, -1) .join( '/', )} graphs. Attempt ${attempts}/${retries}. Retrying in ${retryDelay}ms.`, ); await setTimeout(retryDelay); } else { throw new Error( `Failed to perform batch insert after ${retries} attempts. Error: ${error.message}`, ); } } } } async createParanetKnoledgeCollectionConnection( repository, kcUAL, paranetUAL, contentType, timeout, ) { const getNamedGraphsQuery = ` PREFIX dkg: SELECT ?g WHERE { GRAPH { <${kcUAL}> dkg:hasNamedGraph ?g . } } `; let metadataConnections = await this.select(repository, getNamedGraphsQuery); if (contentType === 'public') { metadataConnections = metadataConnections.filter((row) => !row.g.includes('/private')); } const paranetConnectionTriples = metadataConnections .map( (row) => ` <${paranetUAL}> <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${row.g}> .`, ) .join('\n'); const query = ` INSERT DATA { GRAPH <${paranetUAL}> { ${paranetConnectionTriples} } } `; await this.queryVoid(repository, query, timeout); } async insertMetadataTriples(repository, kcUAL, kaUALs, visibility, timeout) { const currentTriples = kaUALs .map( (ual) => ` <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${ual}/${visibility}> .`, ) .join('\n'); const connectionTriples = kaUALs .map((ual) => { const graphWithVisibility = `${ual}/${visibility}`; return [ `<${kcUAL}> <${DKG_PREDICATE}${HAS_KNOWLEDGE_ASSET_SUFFIX}> <${ual}> .`, `<${kcUAL}> <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${graphWithVisibility}> .`, ].join('\n'); }) .join('\n'); const query = ` INSERT DATA { GRAPH <${BASE_NAMED_GRAPHS.CURRENT}> { ${currentTriples} } GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { ${connectionTriples} } } `; await this.queryVoid(repository, query, timeout); } async deleteKnowledgeCollectionNamedGraphs(repository, namedGraphs) { if (!namedGraphs || namedGraphs.length === 0) return; const query = `${namedGraphs.map((graph) => `DROP GRAPH <${graph}>`).join(';\n')};`; await this.queryVoid(repository, query); } async getKnowledgeCollectionNamedGraphsOld(repository, ual, tokenIds, visibility, timeout) { const namedGraphs = Array.from( { length: tokenIds.endTokenId - tokenIds.startTokenId + 1 }, (_, i) => tokenIds.startTokenId + i, ) .filter((id) => !tokenIds.burned.includes(id)) .map((id) => `${ual}/${id}`); const assertion = {}; if (visibility === TRIPLES_VISIBILITY.PUBLIC || visibility === TRIPLES_VISIBILITY.ALL) { const query = ` PREFIX schema: CONSTRUCT { ?s ?p ?o . } WHERE { GRAPH ?g { ?s ?p ?o . } VALUES ?g { ${namedGraphs .map((graph) => `<${graph}/${TRIPLES_VISIBILITY.PUBLIC}>`) .join('\n')} } }`; assertion.public = await this.construct(repository, query, timeout); } if (visibility === TRIPLES_VISIBILITY.PRIVATE || visibility === TRIPLES_VISIBILITY.ALL) { const query = ` PREFIX schema: CONSTRUCT { ?s ?p ?o . } WHERE { GRAPH ?g { ?s ?p ?o . } VALUES ?g { ${namedGraphs .map((graph) => `<${graph}/${TRIPLES_VISIBILITY.PRIVATE}>`) .join('\n')} } }`; assertion.private = await this.construct(repository, query, timeout); } return assertion; } async getKnowledgeCollectionNamedGraphsOldInBatch( repository, ualTokenIds, visibility, timeout, ) { const kaUALs = Array.from(Object.entries(ualTokenIds)).flatMap(([ual, tokenIds]) => { const arr = Array.from( { length: tokenIds.endTokenId - tokenIds.startTokenId + 1 }, (_, i) => tokenIds.startTokenId + i, ); if ( visibility === TRIPLES_VISIBILITY.PUBLIC || visibility === TRIPLES_VISIBILITY.PRIVATE ) { return arr .filter((id) => !tokenIds.burned.includes(id)) .map((id) => `<${ual}/${id}/${visibility}>`); } // visibility === TRIPLES_VISIBILITY.ALL; // It should add both public and private suffixes return arr .filter((id) => !tokenIds.burned.includes(id)) .flatMap((id) => [ `<${ual}/${id}/${TRIPLES_VISIBILITY.PUBLIC}>`, `<${ual}/${id}/${TRIPLES_VISIBILITY.PRIVATE}>`, ]); }); const query = ` SELECT ?g ?s ?p ?o WHERE { VALUES ?g { ${kaUALs.join('\n')} } GRAPH ?g { ?s ?p ?o } } `; const result = await axios.post(this.repositories[repository].sparqlEndpoint, query, { headers: { 'Content-Type': 'application/sparql-query', Accept: 'text/tab-separated-values', 'X-BIGDATA-MAX-QUERY-MILLIS': timeout, }, }); return result.data; } async checkIfKnowledgeAssetExists(repository, kaUAL, timeout = 10000) { const query = ` ASK { GRAPH <${kaUAL}> { ?s ?p ?o } }`; try { return this.ask(repository, query, timeout); } catch (error) { this.logger.error(`Error checking if knowledge asset exists: ${error}`); return false; } } async getKnowledgeCollectionNamedGraphs( repository, ual, knowledgeAssetId, visibility, timeout, ) { const assertion = {}; let publicPrivateMetadataConnections = null; const getNamedGraphsQuery = ` PREFIX dkg: SELECT ?g WHERE { GRAPH { <${ual}> dkg:hasNamedGraph ?g . } } `; const getConstructQuery = (graphList) => ` PREFIX schema: CONSTRUCT { ?s ?p ?o . } WHERE { GRAPH ?g { ?s ?p ?o . } VALUES ?g { ${graphList.map((g) => `<${g}>`).join('\n')} } } `; const buildSingleGraph = async (visibilityType) => { const graph = `${ual}/${knowledgeAssetId}/${visibilityType}`; return getConstructQuery([graph]); }; const buildAllGraphs = async (filter) => { if (!publicPrivateMetadataConnections) { publicPrivateMetadataConnections = await this.select( repository, getNamedGraphsQuery, timeout, ); } return publicPrivateMetadataConnections .map((row) => row.g) .filter((graph) => graph.includes(filter)); }; if (visibility === TRIPLES_VISIBILITY.PUBLIC || visibility === TRIPLES_VISIBILITY.ALL) { if (knowledgeAssetId) { const singleGraph = await buildSingleGraph(TRIPLES_VISIBILITY.PUBLIC); assertion.public = await this.construct(repository, singleGraph, timeout); } else { const publicGraphs = await buildAllGraphs('/public'); assertion.public = publicGraphs.length ? await this.construct(repository, getConstructQuery(publicGraphs), timeout) : ''; } } if (visibility === TRIPLES_VISIBILITY.PRIVATE || visibility === TRIPLES_VISIBILITY.ALL) { if (knowledgeAssetId) { const singleGraph = await buildSingleGraph(TRIPLES_VISIBILITY.PRIVATE); assertion.private = await this.construct(repository, singleGraph, timeout); } else { const privateGraphs = await buildAllGraphs('/private'); assertion.private = privateGraphs.length ? await this.construct(repository, getConstructQuery(privateGraphs), timeout) : ''; } } return assertion; } async getMetadataInBatch(repository, uals) { const query = ` CONSTRUCT { ?ual ?p ?o } WHERE { VALUES ?ual { ${uals.map((ual) => `<${ual}>`).join('\n')} } GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { ?ual ?p ?o } } `; return this.construct(repository, query); } async knowledgeCollectionNamedGraphsExist(repository, ual) { const query = ` ASK { GRAPH <${ual}/1/public> { ?s ?p ?o } } `; return this.ask(repository, query); } async deleteKnowledgeAssetNamedGraph(repository, ual) { const query = ` DROP GRAPH <${ual}> `; await this.queryVoid(repository, query); } async getKnowledgeAssetNamedGraph(repository, ual, visibility, timeout) { let whereClause; const nquads = {}; switch (visibility) { case TRIPLES_VISIBILITY.PUBLIC: case TRIPLES_VISIBILITY.PRIVATE: { whereClause = ` WHERE { GRAPH <${ual}/${visibility}> { ?s ?p ?o . } } `; const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } ${whereClause} `; nquads[visibility] = await this.construct(repository, query, timeout); nquads[visibility] = nquads[visibility].split('\n'); break; } case TRIPLES_VISIBILITY.ALL: { const publicWhereClause = ` WHERE { GRAPH <${ual}/${TRIPLES_VISIBILITY.PUBLIC}> { ?s ?p ?o . } } `; const privateWhereClause = ` WHERE { GRAPH <${ual}/${TRIPLES_VISIBILITY.PRIVATE}> { ?s ?p ?o . } } `; const publicQuery = ` PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } ${publicWhereClause} `; const privateQuery = ` PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } ${privateWhereClause} `; nquads[TRIPLES_VISIBILITY.PUBLIC] = await this.construct( repository, publicQuery, timeout, ); nquads[TRIPLES_VISIBILITY.PRIVATE] = await this.construct( repository, privateQuery, timeout, ); nquads[TRIPLES_VISIBILITY.PUBLIC] = nquads[TRIPLES_VISIBILITY.PUBLIC] .split('\n') .slice(0, -1); nquads[TRIPLES_VISIBILITY.PRIVATE] = nquads[TRIPLES_VISIBILITY.PRIVATE] .split('\n') .slice(0, -1); break; } default: throw new Error(`Unsupported visibility: ${visibility}`); } return nquads; } async knowledgeAssetNamedGraphExists(repository, name) { const query = ` ASK { GRAPH <${name}> { ?s ?p ?o } } `; return this.ask(repository, query); } async insertKnowledgeCollectionMetadata(repository, metadataNQuads, timeout) { const query = ` PREFIX schema: <${SCHEMA_CONTEXT}> INSERT DATA { GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { ${metadataNQuads} } } `; await this.queryVoid(repository, query, timeout); } async deleteKnowledgeCollectionMetadata(repository, uals) { const cleanedUals = [...new Set(uals.map((ual) => ual.replace(/\/(public|private)$/, '')))]; const kcUAL = cleanedUals[0].split('/').slice(0, -1).join('/'); let query = `${cleanedUals .map( (ual) => `DELETE WHERE { GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { <${ual}> ?p ?o . } }`, ) .join(';\n')};`; query += `DELETE WHERE { GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { <${kcUAL}> ?p ?o . } }`; await this.queryVoid(repository, query); } async deletePublishTimestampMetadata(repository, ual) { const query = ` DELETE WHERE { GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { <${ual}> <${DKG_METADATA_PREDICATES.PUBLISH_TIME}> ?o . } } `; await this.queryVoid(repository, query); } async getKnowledgeCollectionMetadata(repository, ual, timeout) { const query = ` CONSTRUCT { <${ual}> ?p ?o . } WHERE { GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { <${ual}> ?p ?o . } } `; return this.construct(repository, query, timeout); } async getKnowledgeAssetMetadata(repository, ual, timeout) { const query = ` CONSTRUCT { <${ual}> ?p ?o . } WHERE { GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { <${ual}> ?p ?o . } } `; return this.construct(repository, query, timeout); } async knowledgeCollectionMetadataExists(repository, ual) { const query = ` ASK { GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { ?ual ?p ?o FILTER(STRSTARTS(STR(?ual), "${ual}/")) } } `; return this.ask(repository, query); } async findAllNamedGraphsByUAL(repository, ual) { const query = ` SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o } FILTER(STRSTARTS(STR(?g), "${ual}")) }`; this.select(repository, query); } async findAllSubjectsWithGraphNames(repository, ual) { const query = ` SELECT DISTINCT ?s ?g WHERE { GRAPH ?g { ?s ?p ?o } FILTER(STRSTARTS(STR(?g), "${ual}")) }`; this.select(repository, query); } async getLatestAssertionId(repository, ual) { const query = `SELECT ?assertionId WHERE { GRAPH { <${ual}> ?p ?assertionId } }`; const data = await this.select(repository, query); const fullAssertionId = data?.[0]?.assertionId; const latestAssertionId = fullAssertionId?.replace('assertion:', ''); return latestAssertionId; } async construct(repository, query) { return this._executeQuery(repository, query, MEDIA_TYPES.N_QUADS); } async select(repository, query) { // todo: add media type once bug is fixed // no media type is passed because of comunica bug // https://github.com/comunica/comunica/issues/1034 const result = await this._executeQuery(repository, query); return result ? JSON.parse(result) : []; } async queryVoid(repository, query) { return this.queryEngine.queryVoid(query, this.repositories[repository].updateContext); } async ask(repository, query) { return this.queryEngine.queryBoolean(query, this.repositories[repository].queryContext); } async healthCheck() { return true; } async _executeQuery(repository, query, mediaType) { const result = await this.queryEngine.query( query, this.repositories[repository].queryContext, ); const { data } = await this.queryEngine.resultToString(result, mediaType); let response = ''; for await (const chunk of data) { response += chunk; } return response; } async reinitialize() { const ready = await this.healthCheck(); if (!ready) { this.logger.warn( `Cannot connect to Triple store (${this.getName()}), check if your triple store is running.`, ); } else { this.implementation.initialize(this.logger); } } // OLD REPOSITORIES SUPPORT cleanEscapeCharacter(query) { return query.replace(/['|[\]\\]/g, '\\$&'); } async getV6Assertion(repository, assertionId) { if (!assertionId) return ''; const escapedGraphName = this.cleanEscapeCharacter(assertionId); const query = `PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } WHERE { { GRAPH { ?s ?p ?o . } } }`; return this.construct(repository, query); } } export default OtTripleStore; ================================================ FILE: src/modules/triple-store/triple-store-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class TripleStoreModuleManager extends BaseModuleManager { initializeParanetRepository(repository) { return this.getImplementation().module.initializeParanetRepository(repository); } repositoryInitilized(repository) { return this.getImplementation().module.repositoryInitilized(repository); } getRepositoryUrl(implementationName, repository) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getRepositoryUrl(repository); } } async deleteUniqueKnowledgeCollectionTriplesFromUnifiedGraph( implementationName, repository, namedGraph, ual, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.deleteUniqueKnowledgeCollectionTriplesFromUnifiedGraph( repository, namedGraph, ual, ); } } async getKnowledgeCollectionFromUnifiedGraph( implementationName, repository, namedGraph, ual, sort, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeCollectionFromUnifiedGraph(repository, namedGraph, ual, sort); } } async getKnowledgeCollectionPublicFromUnifiedGraph( implementationName, repository, namedGraph, ual, sort, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeCollectionPublicFromUnifiedGraph( repository, namedGraph, ual, sort, ); } } async knowledgeCollectionExistsInUnifiedGraph(implementationName, repository, namedGraph, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.knowledgeCollectionExistsInUnifiedGraph(repository, namedGraph, ual); } } async deleteUniqueKnowledgeAssetTriplesFromUnifiedGraph( implementationName, repository, namedGraph, ual, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.deleteUniqueKnowledgeAssetTriplesFromUnifiedGraph(repository, namedGraph, ual); } } async getKnowledgeAssetFromUnifiedGraph(implementationName, repository, namedGraph, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeAssetFromUnifiedGraph(repository, namedGraph, ual); } } async getKnowledgeAssetPublicFromUnifiedGraph(implementationName, repository, namedGraph, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeAssetPublicFromUnifiedGraph(repository, namedGraph, ual); } } async knowledgeAssetExistsInUnifiedGraph(implementationName, repository, namedGraph, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.knowledgeAssetExistsInUnifiedGraph(repository, namedGraph, ual); } } async createKnowledgeCollectionNamedGraphs( implementationName, repository, uals, assetsNQuads, visibility, timeout, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.createKnowledgeCollectionNamedGraphs( repository, uals, assetsNQuads, visibility, timeout, ); } } async createParanetKnoledgeCollectionConnection( implementationName, repository, knowledgeCollectionUal, paranetUAL, contentType, timeout, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.createParanetKnoledgeCollectionConnection( repository, knowledgeCollectionUal, paranetUAL, contentType, timeout, ); } } async insertMetadataTriples(implementationName, repository, kcUal, uals, visibility, timeout) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.insertMetadataTriples( repository, kcUal, uals, visibility, timeout, ); } } async deleteKnowledgeCollectionNamedGraphs(implementationName, repository, namedGraphs) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.deleteKnowledgeCollectionNamedGraphs(repository, namedGraphs); } } async getKnowledgeCollectionNamedGraphs( implementationName, repository, ual, knowledgeAssetId, visibility, timeout, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeCollectionNamedGraphs( repository, ual, knowledgeAssetId, visibility, timeout, ); } } async getKnowledgeCollectionNamedGraphsInBatch( implementationName, repository, uals, visibility, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeCollectionNamedGraphsInBatch(repository, uals, visibility); } } async getKnowledgeCollectionNamedGraphsOld( implementationName, repository, ual, tokenIds, visibility, timeout, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeCollectionNamedGraphsOld( repository, ual, tokenIds, visibility, timeout, ); } } async checkIfKnowledgeAssetExists(implementationName, repository, kaUAL) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.checkIfKnowledgeAssetExists( repository, kaUAL, ); } } async getKnowledgeCollectionNamedGraphsOldInBatch( implementationName, repository, tokenIds, visibility, timeout, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.getKnowledgeCollectionNamedGraphsOldInBatch( repository, tokenIds, visibility, timeout, ); } } async getMetadataInBatch(implementationName, repository, uals) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getMetadataInBatch( repository, uals, ); } } async knowledgeCollectionNamedGraphsExist(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.knowledgeCollectionNamedGraphsExist(repository, ual); } } async deleteKnowledgeAssetNamedGraph(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.deleteKnowledgeAssetNamedGraph( repository, ual, ); } } async getKnowledgeAssetNamedGraph(implementationName, repository, ual, visibility, timeout) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getKnowledgeAssetNamedGraph( repository, ual, visibility, timeout, ); } } async knowledgeAssetNamedGraphExists(implementationName, repository, name) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.knowledgeAssetNamedGraphExists( repository, name, ); } } async insertKnowledgeCollectionMetadata( implementationName, repository, metadataNQuads, timeout, ) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.insertKnowledgeCollectionMetadata(repository, metadataNQuads, timeout); } } async deleteKnowledgeCollectionMetadata(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.deleteKnowledgeCollectionMetadata(repository, ual); } } async deletePublishTimestampMetadata(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.deletePublishTimestampMetadata( repository, ual, ); } } async getKnowledgeCollectionMetadata(implementationName, repository, ual, timeout) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getKnowledgeCollectionMetadata( repository, ual, timeout, ); } } async getKnowledgeAssetMetadata(implementationName, repository, ual, timeout) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getKnowledgeAssetMetadata( repository, ual, timeout, ); } } async knowledgeCollectionMetadataExists(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation( implementationName, ).module.knowledgeCollectionMetadataExists(repository, ual); } } async getLatestAssertionId(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getLatestAssertionId( repository, ual, ); } } async construct(implementationName, repository, query, timeout) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.construct( repository, query, timeout, ); } } async select(implementationName, repository, query, timeout) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.select( repository, query, timeout, ); } } async queryVoid(implementationName, repository, query) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.queryVoid(repository, query); } } async deleteRepository(implementationName, repository) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.deleteRepository(repository); } } async findAllNamedGraphsByUAL(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.findAllNamedGraphsByUAL( repository, ual, ); } } async findAllSubjectsWithGraphNames(implementationName, repository, ual) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.findAllSubjectsWithGraphNames( implementationName, repository, ual, ); } } async ask(implementationName, repository, query) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.ask(repository, query); } } async insertAssertionBatch( implementationName, repository, insertMap, metadata, createdMetadata, currentNamedGraphTriples, timeout, ) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.insertAssertionBatch( repository, insertMap, metadata, createdMetadata, currentNamedGraphTriples, timeout, ); } } getName() { return 'tripleStore'; } // OLD REPOSITORIES SUPPORT async getV6Assertion(implementationName, repository, assertionId) { if (this.getImplementation(implementationName)) { return this.getImplementation(implementationName).module.getV6Assertion( repository, assertionId, ); } } } export default TripleStoreModuleManager; ================================================ FILE: src/modules/validation/implementation/merkle-validation.js ================================================ import { kcTools } from 'assertion-tools'; class MerkleValidation { async initialize(config, logger) { this.config = config; this.logger = logger; } async calculateRoot(assertion) { return kcTools.calculateMerkleRoot(assertion); } } export default MerkleValidation; ================================================ FILE: src/modules/validation/validation-module-manager.js ================================================ import BaseModuleManager from '../base-module-manager.js'; class ValidationModuleManager extends BaseModuleManager { getName() { return 'validation'; } async calculateRoot(assertion) { if (this.initialized) { if (!assertion) { throw new Error('Calculation failed: Assertion cannot be null or undefined.'); } return this.getImplementation().module.calculateRoot(assertion); } throw new Error('Validation module is not initialized.'); } } export default ValidationModuleManager; ================================================ FILE: src/service/ask-service.js ================================================ import OperationService from './operation-service.js'; import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, ERROR_TYPE, OPERATIONS, OPERATION_REQUEST_STATUS, ASK_BATCH_SIZE, } from '../constants/constants.js'; class AskService extends OperationService { constructor(ctx) { super(ctx); this.operationName = OPERATIONS.ASK; this.networkProtocols = NETWORK_PROTOCOLS.ASK; this.errorType = ERROR_TYPE.ASK.ASK_ERROR; this.completedStatuses = [ OPERATION_ID_STATUS.ASK.ASK_FETCH_FROM_NODES_END, OPERATION_ID_STATUS.ASK.ASK_END, OPERATION_ID_STATUS.COMPLETED, ]; } async processResponse(command, responseStatus, responseData) { const { operationId, blockchain, numberOfFoundNodes, leftoverNodes, batchSize } = command.data; const responseStatusesFromDB = await this.getResponsesStatuses( responseStatus, responseData.errorMessage, operationId, ); const { completedNumber, failedNumber } = responseStatusesFromDB[operationId]; const totalResponses = completedNumber + failedNumber; const isAllNodesResponded = numberOfFoundNodes === totalResponses; const isBatchCompleted = totalResponses % batchSize === 0; const minimumNumberOfNodeReplications = command.data.minimumNumberOfNodeReplications ?? numberOfFoundNodes; this.logger.debug( `Processing ${ this.operationName } response with status: ${responseStatus} for operationId: ${operationId}. Total number of nodes: ${numberOfFoundNodes}, number of nodes in batch: ${Math.min( numberOfFoundNodes, batchSize, )} number of leftover nodes: ${ leftoverNodes.length }, number of responses: ${totalResponses}, Completed: ${completedNumber}, Failed: ${failedNumber}`, ); if (responseData.errorMessage) { this.logger.trace( `Error message for operation id: ${operationId} : ${responseData.errorMessage}`, ); } if ( responseStatus === OPERATION_REQUEST_STATUS.COMPLETED && completedNumber === minimumNumberOfNodeReplications ) { await this.markOperationAsCompleted( operationId, blockchain, { completedNodes: completedNumber, allNodesReplicatedData: true, }, [...this.completedStatuses], ); this.logResponsesSummary(completedNumber, failedNumber); } else if ( completedNumber < minimumNumberOfNodeReplications && (isAllNodesResponded || isBatchCompleted) ) { const potentialCompletedNumber = completedNumber + leftoverNodes.length; await this.operationIdService.cacheOperationIdDataToFile(operationId, { completedNodes: completedNumber, allNodesReplicatedData: false, }); // Still possible to meet minimumNumberOfNodeReplications, schedule leftover nodes if ( leftoverNodes.length > 0 && potentialCompletedNumber >= minimumNumberOfNodeReplications ) { await this.scheduleOperationForLeftoverNodes(command.data, leftoverNodes); } else { // Not enough potential responses to meet minimumNumberOfNodeReplications, or no leftover nodes await this.markOperationAsFailed( operationId, blockchain, `Unable to replicate data on the network!`, this.errorType, ); this.logResponsesSummary(completedNumber, failedNumber); } } } getBatchSize(batchSize = null) { return batchSize ?? ASK_BATCH_SIZE; } } export default AskService; ================================================ FILE: src/service/auth-service.js ================================================ import ipLib from 'ip'; import jwtUtil from './util/jwt-util.js'; class AuthService { constructor(ctx) { this._authConfig = ctx.config.auth; this._repository = ctx.repositoryModuleManager; this._logger = ctx.logger; } /** * Authenticate users based on provided ip and token * @param ip * @param token * @returns {boolean} */ async authenticate(ip, token) { const isWhitelisted = this._isIpWhitelisted(ip); const isTokenValid = await this._isTokenValid(token); const tokenAuthEnabled = this._authConfig.tokenBasedAuthEnabled; const ipAuthEnabled = this._authConfig.ipBasedAuthEnabled; const requiresBoth = this._authConfig.bothIpAndTokenAuthRequired; let isAuthenticated = false; if (tokenAuthEnabled && ipAuthEnabled) { isAuthenticated = requiresBoth ? isWhitelisted && isTokenValid : isWhitelisted || isTokenValid; } else { isAuthenticated = isWhitelisted && isTokenValid; } if (!isAuthenticated) { this._logMessage('Received unauthenticated request.'); } return isAuthenticated; } /** * Checks whether user whose token is provided has abilities for system operation * @param token * @param systemOperation * @returns {Promise} */ async isAuthorized(token, systemOperation) { if (!this._authConfig.tokenBasedAuthEnabled) { return true; } /* If IP is whitelisted and both IP and Token Auth is NOT required pass authorization check. Authentication middleware checks if IP is white listed before authorization middleware. */ if (!(await this._isTokenValid(token))) { if ( !this._authConfig.bothIpAndTokenAuthRequired && this._authConfig.ipBasedAuthEnabled ) { return true; } return false; } const tokenId = jwtUtil.getPayload(token).jti; const abilities = await this._repository.getTokenAbilities(tokenId); const isAuthorized = abilities.includes(systemOperation); const logMessage = isAuthorized ? `Token ${tokenId} is successfully authenticated and authorized.` : `Received unauthorized request.`; this._logMessage(logMessage); return isAuthorized; } /** * Determines whether operation is listed in config.auth.publicOperations * @param operationName * @returns {boolean} */ isPublicOperation(operationName) { if (!Array.isArray(this._authConfig.publicOperations)) { return false; } return this._authConfig.publicOperations.some( (publicOperation) => publicOperation === `V0/${operationName}` || publicOperation === operationName, ); } /** * Validates token structure and revoked status * If ot-node is configured not to do a token based auth, it will return true * @param token * @returns {boolean} * @private */ async _isTokenValid(token) { if (!this._authConfig.tokenBasedAuthEnabled) { return true; } if (!token) { return false; } if (!jwtUtil.validateJWT(token)) { return false; } const isRevoked = await this._isTokenRevoked(token); return isRevoked !== null && !isRevoked; } /** * Checks whether provided ip is whitelisted in config * Returns false if ip based auth is disabled * @param reqIp * @returns {boolean} * @private */ _isIpWhitelisted(reqIp) { if (!this._authConfig.ipBasedAuthEnabled) { return true; } for (const whitelistedIp of this._authConfig.ipWhitelist) { let isEqual = false; try { isEqual = ipLib.isEqual(reqIp, whitelistedIp); } catch (e) { // if ip is not valid IP isEqual should remain false } if (isEqual) { return true; } } return false; } /** * Checks whether provided token is revoked * Returns false if token based auth is disabled * @param token * @returns {Promise|boolean} * @private */ _isTokenRevoked(token) { if (!this._authConfig.tokenBasedAuthEnabled) { return false; } const tokenId = jwtUtil.getPayload(token).jti; return this._repository.isTokenRevoked(tokenId); } /** * Logs message if loggingEnabled is set to true * @param message * @private */ _logMessage(message) { if (this._authConfig.loggingEnabled) { this._logger.info(`[AUTH] ${message}`); } } } export default AuthService; ================================================ FILE: src/service/batch-get-service.js ================================================ import OperationService from './operation-service.js'; import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, ERROR_TYPE, OPERATIONS, } from '../constants/constants.js'; class BatchGetService extends OperationService { constructor(ctx) { super(ctx); this.operationName = OPERATIONS.BATCH_GET; this.networkProtocols = NETWORK_PROTOCOLS.BATCH_GET; this.errorType = ERROR_TYPE.BATCH_GET.BATCH_GET_ERROR; this.completedStatuses = [ OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_FETCH_FROM_NODES_END, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_END, OPERATION_ID_STATUS.COMPLETED, ]; } } export default BatchGetService; ================================================ FILE: src/service/blockchain-events-service.js ================================================ class BlockchainEventsService { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; this.blockchainEventsModuleManager = ctx.blockchainEventsModuleManager; } initializeBlockchainEventsServices() { this.blockchainEventsServicesImplementations = {}; for (const implementationName of this.blockchainEventsModuleManager.getImplementationNames()) { for (const blockchain in this.blockchainEventsModuleManager.getImplementation( implementationName, ).module.blockchains) { this.blockchainEventsServicesImplementations[blockchain] = implementationName; } } } getContractAddress(blockchain, contractName) { return this.blockchainEventsModuleManager.getContractAddress( this.blockchainEventsServicesImplementations[blockchain], blockchain, contractName, ); } updateContractAddress(blockchain, contractName, contractAddress) { return this.blockchainEventsModuleManager.updateContractAddress( this.blockchainEventsServicesImplementations[blockchain], blockchain, contractName, contractAddress, ); } async getBlock(blockchain, tag = 'latest') { return this.blockchainEventsModuleManager.getBlock( this.blockchainEventsServicesImplementations[blockchain], blockchain, tag, ); } async getPastEvents( blockchain, contractNames, eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, ) { return this.blockchainEventsModuleManager.getPastEvents( this.blockchainEventsServicesImplementations[blockchain], blockchain, contractNames, eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, ); } } export default BlockchainEventsService; ================================================ FILE: src/service/claim-rewards-service.js ================================================ import { CLAIM_REWARDS_BATCH_SIZE, CLAIM_REWARDS_INTERVAL } from '../constants/constants.js'; class ClaimRewardsService { constructor(ctx) { this.ctx = ctx; this.logger = ctx.logger; this.ualService = ctx.ualService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.tripleStoreService = ctx.tripleStoreService; this.validationService = ctx.validationService; this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; } async initialize() { this.logger.info('[CLAIM] Initializing ClaimRewardsService'); const promises = []; for (const blockchainId of this.blockchainModuleManager.getImplementationNames()) { this.logger.info( `[CLAIM] Initializing claim rewards service for blockchain ${blockchainId}`, ); promises.push(this.claimRewardsMechanism(blockchainId)); } await Promise.all(promises); this.logger.info('[CLAIM] ClaimRewardsService initialization completed'); } async claimRewardsMechanism(blockchainId) { this.logger.debug( `[CLAIM] Setting up claim rewards mechanism for blockchain ${blockchainId}`, ); // Flag to track if mechanism is running let isRunning = false; // Set up interval const interval = setInterval(async () => { // Skip if already running if (isRunning) { this.logger.debug( `[CLAIM] Claim rewards mechanism for ${blockchainId} still running, skipping this interval`, ); return; } try { isRunning = true; this.logger.debug( `[CLAIM] Starting claim rewards cycle for blockchain ${blockchainId}`, ); // Proofing logic await this.claimRewards(blockchainId); this.logger.debug( `[CLAIM] Completed claim rewards cycle for blockchain ${blockchainId}`, ); } catch (error) { this.logger.error( `[CLAIM] Error in claim rewards mechanism for ${blockchainId}: ${error.message}, stack: ${error.stack}`, ); } finally { isRunning = false; } }, CLAIM_REWARDS_INTERVAL); // Store interval reference for cleanup this[`${blockchainId}Interval`] = interval; this.logger.info( `[CLAIM] Claim rewards mechanism initialized for blockchain ${blockchainId}`, ); // Run immediately on startup try { isRunning = true; this.logger.debug( `[CLAIM] Running initial claim rewards cycle for blockchain ${blockchainId}`, ); await this.claimRewards(blockchainId); } catch (error) { this.logger.error( `[CLAIM] Error in initial claim rewards run for ${blockchainId}: ${error.message}, stack: ${error.stack}`, ); // this.operationIdService.emitChangeEvent( // 'CLAIM_REWARDS_ERROR', // this.generateOperationId(blockchainId, 0, 0), // blockchainId, // error.message, // error.stack, // ); } finally { isRunning = false; } } async claimRewards(blockchainId) { const identityId = await this.blockchainModuleManager.getIdentityId(blockchainId); const nodeDelegatorAddresses = await this.blockchainModuleManager.getDelegators( blockchainId, identityId, ); const lastClaimedEpochAddressesMap = {}; await Promise.all( nodeDelegatorAddresses.map(async (delegatorAddress) => { const lastClaimedEpoch = await this.blockchainModuleManager.getLastClaimedEpoch( blockchainId, identityId, delegatorAddress, ); if (!lastClaimedEpochAddressesMap[`${lastClaimedEpoch}`]) { lastClaimedEpochAddressesMap[`${lastClaimedEpoch}`] = []; } lastClaimedEpochAddressesMap[`${lastClaimedEpoch}`].push(delegatorAddress); }), ); const currentEpoch = Number( (await this.blockchainModuleManager.getCurrentEpoch(blockchainId)).toString(), ); if (lastClaimedEpochAddressesMap['0'] && lastClaimedEpochAddressesMap['0'].length > 0) { // This means delegator never claimed for the node, but is in the list of delegators // This means node never claimed and delegated before introduction of random sampling // If he staked or claimed before the value would have been set correctly const delegatorAddresses = lastClaimedEpochAddressesMap['0']; await Promise.all( delegatorAddresses.map(async (delegatorAddress) => { const hasEverDelegated = await this.blockchainModuleManager.hasEverDelegated( blockchainId, identityId, delegatorAddress, ); // TODO: How will this impact mainnet where this function landed at same time as proofing if (!hasEverDelegated) { if (lastClaimedEpochAddressesMap[`${currentEpoch - 1}`]) { lastClaimedEpochAddressesMap[`${currentEpoch - 1}`].push( ...delegatorAddresses, ); } else { lastClaimedEpochAddressesMap[`${currentEpoch - 1}`] = delegatorAddresses; } } }), ); } if (lastClaimedEpochAddressesMap[`0`]) { delete lastClaimedEpochAddressesMap[`0`]; } const sortedEpochs = Object.keys(lastClaimedEpochAddressesMap) .map(Number) // convert keys to numbers .sort((a, b) => a - b); // sort numerically ascending for (let i = 0; i < sortedEpochs.length; i += 1) { const epoch = sortedEpochs[i]; const delegatorAddresses = lastClaimedEpochAddressesMap[epoch.toString()]; if (epoch + 1 !== currentEpoch) { for (let j = 0; j < delegatorAddresses.length; j += CLAIM_REWARDS_BATCH_SIZE) { const batch = delegatorAddresses.slice(j, j + CLAIM_REWARDS_BATCH_SIZE); try { const batchClaimed = // eslint-disable-next-line no-await-in-loop await this.blockchainModuleManager.batchClaimDelegatorRewards( blockchainId, identityId, [epoch + 1], batch, ); if (batchClaimed.success) { this.logger.info( `[CLAIM] Claimed rewards for batch ${batch} in epoch ${ epoch + 1 } on ${blockchainId}`, ); // If there are more epochs for this batch move them to next batch if (lastClaimedEpochAddressesMap[`${epoch + 1}`]) { lastClaimedEpochAddressesMap[`${epoch + 1}`].push(...batch); } else { lastClaimedEpochAddressesMap[`${epoch + 1}`] = batch; // lastClaimedEpochAddressesMap[`${epoch + 1}`] didn't exist before so we need to also update sortedEpochs // splice handles if i + 1 === sortedEpochs.length sortedEpochs.splice(i + 1, 0, epoch + 1); } } else { this.logger.error( `[CLAIM] Error claiming rewards for batch ${batch} in epoch ${ epoch + 1 } on ${blockchainId}`, batchClaimed.error, ); } } catch (error) { this.logger.error( `[CLAIM] Error claiming rewards for batch ${batch} in epoch ${ epoch + 1 } on ${blockchainId}`, error, ); } } } } } } export default ClaimRewardsService; ================================================ FILE: src/service/crypto-service.js ================================================ import ethers from 'ethers'; class CryptoService { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; } toBigNumber(value) { return ethers.BigNumber.from(value); } keccak256(data) { if (!ethers.utils.isBytesLike(data)) { const bytesLikeData = ethers.utils.toUtf8Bytes(data); return ethers.utils.keccak256(bytesLikeData); } return ethers.utils.keccak256(data); } sha256(data) { if (!ethers.utils.isBytesLike(data)) { const bytesLikeData = ethers.utils.toUtf8Bytes(data); return ethers.utils.sha256(bytesLikeData); } return ethers.utils.sha256(data); } encodePacked(types, values) { return ethers.utils.solidityPack(types, values); } keccak256EncodePacked(types, values) { return ethers.utils.solidityKeccak256(types, values); } sha256EncodePacked(types, values) { return ethers.utils.soliditySha256(types, values); } convertUint8ArrayToHex(uint8Array) { return ethers.utils.hexlify(uint8Array); } convertAsciiToHex(string) { return this.convertUint8ArrayToHex(ethers.utils.toUtf8Bytes(string)); } convertHexToAscii(hexString) { return ethers.utils.toUtf8String(hexString); } convertBytesToUint8Array(bytesLikeData) { return ethers.utils.arrayify(bytesLikeData); } convertToWei(value, fromUnit = 'ether') { return ethers.utils.parseUnits(value.toString(), fromUnit); } convertFromWei(value, toUnit = 'ether') { return ethers.utils.formatUnits(value, toUnit); } splitSignature(flatSignature) { return ethers.utils.splitSignature(flatSignature); } } export default CryptoService; ================================================ FILE: src/service/dependency-injection.js ================================================ import awilix from 'awilix'; class DependencyInjection { static async initialize() { const container = awilix.createContainer({ injectionMode: awilix.InjectionMode.PROXY, }); await container.loadModules( [ 'src/controllers/**/*.js', 'src/service/*.js', 'src/commands/**/**/**/*.js', 'src/commands/*.js', 'src/modules/base-module-manager.js', 'src/modules/**/*module-manager.js', ], { esModules: true, formatName: 'camelCase', resolverOptions: { lifetime: awilix.Lifetime.SINGLETON, register: awilix.asClass, }, }, ); return container; } static registerValue(container, valueName, value) { container.register({ [valueName]: awilix.asValue(value), }); } } export default DependencyInjection; ================================================ FILE: src/service/file-service.js ================================================ import os from 'os'; import path from 'path'; import { mkdir, writeFile, readFile, unlink, stat, readdir, rm, appendFile, chmod, } from 'fs/promises'; import appRootPath from 'app-root-path'; import { BLS_KEY_DIRECTORY, BLS_KEY_FILENAME, MIGRATION_FOLDER, NODE_ENVIRONMENTS, } from '../constants/constants.js'; class FileService { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; } getFileExtension(filePath) { return path.extname(filePath).toLowerCase(); } /** * Write contents to file * @param directory * @param filename * @param data * @returns {Promise} */ async writeContentsToFile(directory, filename, data, log = true, flag = 'w') { if (log) { this.logger.debug(`Saving file with name: ${filename} in the directory: ${directory}`); } await mkdir(directory, { recursive: true }); const fullpath = path.join(directory, filename); await writeFile(fullpath, data, { flag }); return fullpath; } async appendContentsToFile(directory, filename, data, log = true) { if (log) { this.logger.debug(`Saving file with name: ${filename} in the directory: ${directory}`); } await mkdir(directory, { recursive: true }); const fullPath = path.join(directory, filename); await appendFile(fullPath, data); return fullPath; } async readDirectory(dirPath) { this.logger.debug(`Reading folder at path: ${dirPath}`); try { return readdir(dirPath); } catch (error) { if (error.code === 'ENOENT') { throw Error(`Folder not found at path: ${dirPath}`); } throw error; } } async stat(filePath) { return stat(filePath); } async pathExists(fileOrDirPath) { try { await stat(fileOrDirPath); return true; } catch (error) { if (error.code === 'ENOENT') { return false; } throw error; } } async readFile(filePath, convertToJSON = false) { this.logger.debug(`Reading file: ${filePath}, converting to json: ${convertToJSON}`); try { const data = await readFile(filePath); return convertToJSON ? JSON.parse(data) : data.toString(); } catch (error) { if (error.code === 'ENOENT') { throw Error(`File not found at path: ${filePath}`); } throw error; } } async removeFile(filePath) { this.logger.trace(`Removing file at path: ${filePath}`); try { await unlink(filePath); return true; } catch (error) { if (error.code === 'ENOENT') { this.logger.debug(`File not found at path: ${filePath}`); return false; } throw error; } } async removeFolder(folderPath) { this.logger.debug(`Removing folder at path: ${folderPath}`); try { await rm(folderPath, { recursive: true }); return true; } catch (error) { if (error.code === 'ENOENT') { this.logger.debug(`Folder not found at path: ${folderPath}`); return false; } throw error; } } getBinariesFolderPath() { return path.join(appRootPath.path, 'bin'); } getBinaryPath(binary) { let binaryName = binary; if (process.platform === 'win32') { binaryName += '.exe'; } return path.join(this.getBinariesFolderPath(), process.platform, process.arch, binaryName); } async makeBinaryExecutable(binary) { const binaryPath = this.getBinaryPath(binary); if (os.platform() !== 'win32') { await chmod(binaryPath, '755', (err) => { if (err) { throw err; } this.logger.debug(`Permissions for binary ${binaryPath} have been set to 755.`); }); } } getBLSSecretKeyFolderPath() { return path.join(this.getDataFolderPath(), BLS_KEY_DIRECTORY); } getBLSSecretKeyPath() { return path.join(this.getBLSSecretKeyFolderPath(), BLS_KEY_FILENAME); } getDataFolderPath() { if ( process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVNET || process.env.NODE_ENV === NODE_ENVIRONMENTS.TESTNET || process.env.NODE_ENV === NODE_ENVIRONMENTS.MAINNET ) { return path.join(appRootPath.path, '..', this.config.appDataPath); } return path.join(appRootPath.path, this.config.appDataPath); } getUpdateFilePath() { return path.join(this.getDataFolderPath(), 'UPDATED'); } getMigrationFolderPath() { return path.join(this.getDataFolderPath(), MIGRATION_FOLDER); } getOperationIdCachePath() { return path.join(this.getDataFolderPath(), 'operation_id_cache'); } getOperationIdDocumentPath(operationId) { return path.join(this.getOperationIdCachePath(), operationId); } getPendingStorageCachePath() { return path.join(this.getDataFolderPath(), 'pending_storage_cache'); } getPendingStorageDocumentPath(operationId) { return path.join(this.getPendingStorageCachePath(), operationId); } getSignatureStorageCachePath() { return path.join(this.getDataFolderPath(), 'signature_storage_cache'); } getSignatureStorageFolderPath(folderName) { return path.join(this.getSignatureStorageCachePath(), folderName); } getSignatureStorageDocumentPath(folderName, operationId) { return path.join(this.getSignatureStorageFolderPath(folderName), operationId); } getParentDirectory(filePath) { return path.dirname(filePath); } } export default FileService; ================================================ FILE: src/service/finality-service.js ================================================ import OperationService from './operation-service.js'; import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, ERROR_TYPE, OPERATIONS, OPERATION_REQUEST_STATUS, FINALITY_BATCH_SIZE, FINALITY_MIN_NUM_OF_NODE_REPLICATIONS, } from '../constants/constants.js'; class FinalityService extends OperationService { constructor(ctx) { super(ctx); this.operationName = OPERATIONS.FINALITY; this.networkProtocols = NETWORK_PROTOCOLS.FINALITY; this.errorType = ERROR_TYPE.FINALITY.FINALITY_ERROR; this.completedStatuses = [ OPERATION_ID_STATUS.PUBLISH_FINALIZATION.PUBLISH_FINALIZATION_END, OPERATION_ID_STATUS.COMPLETED, ]; this.ualService = ctx.ualService; this.tripleStoreService = ctx.tripleStoreService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.paranetService = ctx.paranetService; } async processResponse(operationId, blockchain, responseStatus, responseData) { const responseStatusesFromDB = await this.getResponsesStatuses( responseStatus, responseData.errorMessage, operationId, ); const { completedNumber, failedNumber } = responseStatusesFromDB[operationId]; this.logger.debug( `Processing ${this.operationName} response with status: ${responseStatus} for operationId: ${operationId}. ` + `Completed: ${completedNumber}, Failed: ${failedNumber}`, ); if (responseData.errorMessage) { this.logger.trace( `Error message for operation id: ${operationId} : ${responseData.errorMessage}`, ); } if (responseStatus === OPERATION_REQUEST_STATUS.COMPLETED) { await this.markOperationAsCompleted( operationId, blockchain, { completedNodes: 1, allNodesReplicatedData: true, }, [...this.completedStatuses], ); this.logResponsesSummary(completedNumber, failedNumber); } else { await this.markOperationAsFailed( operationId, blockchain, `Unable to send ACK for finalization!`, this.errorType, ); this.logResponsesSummary(completedNumber, failedNumber); } } getBatchSize(batchSize = null) { return batchSize ?? FINALITY_BATCH_SIZE; } getMinAckResponses(minimumNumberOfNodeReplications = null) { return minimumNumberOfNodeReplications ?? FINALITY_MIN_NUM_OF_NODE_REPLICATIONS; } } export default FinalityService; ================================================ FILE: src/service/get-service.js ================================================ import OperationService from './operation-service.js'; import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, ERROR_TYPE, OPERATIONS, OPERATION_REQUEST_STATUS, GET_BATCH_SIZE, GET_MIN_NUM_OF_NODE_REPLICATIONS, } from '../constants/constants.js'; class GetService extends OperationService { constructor(ctx) { super(ctx); this.operationName = OPERATIONS.GET; this.networkProtocols = NETWORK_PROTOCOLS.GET; this.errorType = ERROR_TYPE.GET.GET_ERROR; this.completedStatuses = [ OPERATION_ID_STATUS.GET.GET_FETCH_FROM_NODES_END, OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED, ]; } async processResponse(command, responseStatus, responseData) { const { operationId, blockchain, numberOfFoundNodes, leftoverNodes, batchSize, minAckResponses, assertionId, } = command.data; const responseStatusesFromDB = await this.getResponsesStatuses( responseStatus, responseData.errorMessage, operationId, ); const { completedNumber, failedNumber } = responseStatusesFromDB[operationId]; const totalResponses = completedNumber + failedNumber; const isAllNodesResponded = numberOfFoundNodes === totalResponses; const isBatchCompleted = totalResponses % batchSize === 0; this.logger.debug( `Processing ${ this.operationName } response with status: ${responseStatus} for operationId: ${operationId}. Total number of nodes: ${numberOfFoundNodes}, number of nodes in batch: ${Math.min( numberOfFoundNodes, batchSize, )} number of leftover nodes: ${ leftoverNodes.length }, number of responses: ${totalResponses}, Completed: ${completedNumber}, Failed: ${failedNumber}`, ); if (responseData.errorMessage) { this.logger.trace( `Error message for operation id: ${operationId} : ${responseData.errorMessage}`, ); } if ( responseStatus === OPERATION_REQUEST_STATUS.COMPLETED && completedNumber === minAckResponses ) { await this.markOperationAsCompleted(operationId, blockchain, responseData, [ ...this.completedStatuses, ]); this.logResponsesSummary(completedNumber, failedNumber); } else if (completedNumber < minAckResponses && (isAllNodesResponded || isBatchCompleted)) { const potentialCompletedNumber = completedNumber + leftoverNodes.length; // Still possible to meet minAckResponses, schedule leftover nodes if (leftoverNodes.length > 0 && potentialCompletedNumber >= minAckResponses) { await this.scheduleOperationForLeftoverNodes(command.data, leftoverNodes); } else { // Not enough potential responses to meet minAckResponses, or no leftover nodes this.markOperationAsFailed( operationId, blockchain, `Unable to find assertion ${assertionId} on the network!`, this.errorType, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.GET.GET_FAILED, operationId, ); this.logResponsesSummary(completedNumber, failedNumber); } } } getBatchSize(batchSize = null) { return batchSize ?? GET_BATCH_SIZE; } getMinAckResponses(minimumNumberOfNodeReplications = null) { return minimumNumberOfNodeReplications ?? GET_MIN_NUM_OF_NODE_REPLICATIONS; } } export default GetService; ================================================ FILE: src/service/json-schema-service.js ================================================ import path from 'path'; import appRootPath from 'app-root-path'; class JsonSchemaService { constructor(ctx) { this.blockchainModuleManager = ctx.blockchainModuleManager; } async loadSchema(version, schemaName, argumentsObject = {}) { const schemaPath = path.resolve( appRootPath.path, `src/controllers/http-api/${version}/request-schema/${schemaName}-schema-${version}.js`, ); const schemaModule = await import(schemaPath); const schemaFunction = schemaModule.default; if (schemaFunction.length !== 0) { return schemaFunction(argumentsObject); } return schemaFunction(); } async bidSuggestionSchema(version) { const schemaArgs = {}; switch (version) { case 'v0': case 'v1': schemaArgs.blockchainImplementationNames = this.blockchainModuleManager.getImplementationNames(); break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'bid-suggestion', schemaArgs); } async publishSchema(version) { const schemaArgs = {}; switch (version) { case 'v0': case 'v1': schemaArgs.blockchainImplementationNames = this.blockchainModuleManager.getImplementationNames(); break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'publish', schemaArgs); } async updateSchema(version) { const schemaArgs = {}; switch (version) { case 'v0': case 'v1': schemaArgs.blockchainImplementationNames = this.blockchainModuleManager.getImplementationNames(); break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'update', schemaArgs); } async getSchema(version) { const schemaArgs = {}; switch (version) { case 'v0': case 'v1': break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'get', schemaArgs); } async querySchema(version) { const schemaArgs = {}; switch (version) { case 'v0': case 'v1': break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'query', schemaArgs); } async localStoreSchema(version) { const schemaArgs = {}; switch (version) { case 'v0': case 'v1': schemaArgs.blockchainImplementationNames = this.blockchainModuleManager.getImplementationNames(); break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'local-store', schemaArgs); } async finalitySchema(version) { const schemaArgs = {}; switch (version) { case 'v1': schemaArgs.blockchainImplementationNames = this.blockchainModuleManager.getImplementationNames(); break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'finality', schemaArgs); } async askSchema(version) { const schemaArgs = {}; switch (version) { case 'v1': schemaArgs.blockchainImplementationNames = this.blockchainModuleManager.getImplementationNames(); break; default: throw Error(`HTTP API version: ${version} isn't supported.`); } return this.loadSchema(version, 'ask', schemaArgs); } } export default JsonSchemaService; ================================================ FILE: src/service/messaging-service.js ================================================ import { NETWORK_MESSAGE_TYPES, OPERATION_REQUEST_STATUS } from '../constants/constants.js'; class MessagingService { constructor(ctx) { this.networkModuleManager = ctx.networkModuleManager; } async sendProtocolMessage(node, operationId, message, messageType, timeout) { const response = await this.networkModuleManager.sendMessage( node.protocol, node.id, messageType, operationId, message, timeout, ); this.networkModuleManager.removeCachedSession(operationId, node.id); return response; } async handleProtocolResponse(response, operationService, blockchain, operationId) { switch (response.header.messageType) { case NETWORK_MESSAGE_TYPES.RESPONSES.BUSY: return this.handleBusyResponse(); case NETWORK_MESSAGE_TYPES.RESPONSES.NACK: return this.handleNackResponse( operationService, blockchain, operationId, response.data, ); case NETWORK_MESSAGE_TYPES.RESPONSES.ACK: return this.handleAckResponse( operationService, blockchain, operationId, response.data, ); default: return this.handleUnknownResponse(operationService, blockchain, operationId); } } async handleBusyResponse() { return { retry: true }; } async handleAckResponse(operationService, blockchain, operationId, responseData) { await operationService.processResponse( operationId, blockchain, OPERATION_REQUEST_STATUS.COMPLETED, responseData, ); return { success: true }; } async handleNackResponse(operationService, blockchain, operationId, responseData) { await operationService.processResponse( operationId, blockchain, OPERATION_REQUEST_STATUS.FAILED, { errorMessage: `Received NACK response. Error: ${responseData.errorMessage}`, }, ); return { failed: true }; } async handleUnknownResponse(operationService, blockchain, operationId) { await operationService.processResponse( operationId, blockchain, OPERATION_REQUEST_STATUS.FAILED, { errorMessage: `Received unknown message type during`, // TODO: Add command name }, ); return { failed: true }; } } export default MessagingService; ================================================ FILE: src/service/operation-id-service.js ================================================ import { validate, v4 as uuidv4 } from 'uuid'; import path from 'path'; class OperationIdService { constructor(ctx) { this.logger = ctx.logger; this.fileService = ctx.fileService; this.repositoryModuleManager = ctx.repositoryModuleManager; this.eventEmitter = ctx.eventEmitter; this.memoryCachedHandlersData = {}; } generateId() { return uuidv4(); } async generateOperationId(status, blockchain, previousOperationId = null) { const operationIdObject = await this.repositoryModuleManager.createOperationIdRecord({ status, }); const { operationId } = operationIdObject; this.emitChangeEvent(status, operationId, blockchain, previousOperationId); this.logger.debug(`Generated operation id for request ${operationId}`); return operationId; } async getOperationIdRecord(operationId) { const operationIdRecord = await this.repositoryModuleManager.getOperationIdRecord( operationId, ); return operationIdRecord; } operationIdInRightFormat(operationId) { return validate(operationId); } async updateOperationIdStatusWithValues( operationId, blockchain, status, value1 = null, value2 = null, value3 = null, timestamp = Date.now(), ) { const response = { status, timestamp, }; this.emitChangeEvent(status, operationId, blockchain, value1, value2, value3, timestamp); await this.repositoryModuleManager.updateOperationIdRecord(response, operationId); } async updateOperationIdStatus( operationId, blockchain, status, errorMessage = null, errorType = null, ) { const response = { status, }; if (errorMessage !== null) { this.logger.debug(`Marking operation id ${operationId} as failed`); response.data = JSON.stringify({ errorMessage, errorType }); await this.removeOperationIdCache(operationId); } if (errorType) { this.emitChangeEvent(errorType, operationId, blockchain, errorMessage, errorType); } else { this.emitChangeEvent(status, operationId, blockchain, errorMessage, errorType); } await this.repositoryModuleManager.updateOperationIdRecord(response, operationId); } emitChangeEvent( status, operationId, blockchainId = null, value1 = null, value2 = null, value3 = null, timestamp = Date.now(), ) { const eventName = 'operation_status_changed'; const eventData = { lastEvent: status, operationId, blockchainId, timestamp, value1, value2, value3, }; this.eventEmitter.emit(eventName, eventData); } async cacheOperationIdDataToMemory(operationId, data) { this.logger.debug(`Caching data for operation id: ${operationId} in memory`); this.memoryCachedHandlersData[operationId] = { data, timestamp: Date.now() }; } async cacheOperationIdDataToFile(operationId, data) { this.logger.debug(`Caching data for operation id: ${operationId} in file`); const operationIdCachePath = this.fileService.getOperationIdCachePath(); await this.fileService.writeContentsToFile( operationIdCachePath, operationId, JSON.stringify(data), ); } async getCachedOperationIdData(operationId) { if (this.memoryCachedHandlersData[operationId]) { this.logger.debug(`Reading operation id: ${operationId} cached data from memory`); return this.memoryCachedHandlersData[operationId].data; } this.logger.debug( `Didn't manage to get cached ${operationId} data from memory, trying file`, ); const documentPath = this.fileService.getOperationIdDocumentPath(operationId); let data; if (await this.fileService.pathExists(documentPath)) { data = await this.fileService.readFile(documentPath, true); } return data; } async removeOperationIdCache(operationId) { this.logger.debug(`Removing operation id: ${operationId} cached data`); const operationIdCachePath = this.fileService.getOperationIdDocumentPath(operationId); await this.fileService.removeFile(operationIdCachePath); this.removeOperationIdMemoryCache(operationId); } removeOperationIdMemoryCache(operationId) { this.logger.debug(`Removing operation id: ${operationId} cached data from memory`); delete this.memoryCachedHandlersData[operationId]; } getOperationIdMemoryCacheSizeBytes() { let total = 0; for (const operationId in this.memoryCachedHandlersData) { const { data } = this.memoryCachedHandlersData[operationId]; total += Buffer.from(JSON.stringify(data)).byteLength; } return total; } async getOperationIdFileCacheSizeBytes() { const cacheFolderPath = this.fileService.getOperationIdCachePath(); const cacheFolderExists = await this.fileService.pathExists(cacheFolderPath); if (!cacheFolderExists) return 0; const fileList = await this.fileService.readDirectory(cacheFolderPath); const sizeResults = await Promise.allSettled( fileList.map((fileName) => this.fileService .stat(path.join(cacheFolderPath, fileName)) .then((stats) => stats.size), ), ); return sizeResults .filter((res) => res.status === 'fulfilled') .reduce((acc, res) => acc + res.value, 0); } async removeExpiredOperationIdMemoryCache(expiredTimeout) { const now = Date.now(); let deleted = 0; for (const operationId in this.memoryCachedHandlersData) { const { data, timestamp } = this.memoryCachedHandlersData[operationId]; if (timestamp + expiredTimeout < now) { delete this.memoryCachedHandlersData[operationId]; deleted += Buffer.from(JSON.stringify(data)).byteLength; } } return deleted; } async removeExpiredOperationIdFileCache(expiredTimeout, batchSize) { const cacheFolderPath = this.fileService.getOperationIdCachePath(); const cacheFolderExists = await this.fileService.pathExists(cacheFolderPath); if (!cacheFolderExists) { return; } const fileList = await this.fileService.readDirectory(cacheFolderPath); const now = new Date(); const deleteFile = async (fileName) => { const filePath = path.join(cacheFolderPath, fileName); const createdDate = (await this.fileService.stat(filePath)).mtime; if (createdDate.getTime() + expiredTimeout < now.getTime()) { await this.fileService.removeFile(filePath); return true; } return false; }; let totalDeleted = 0; for (let i = 0; i < fileList.length; i += batchSize) { const batch = fileList.slice(i, i + batchSize); // eslint-disable-next-line no-await-in-loop const deletionResults = await Promise.allSettled(batch.map(deleteFile)); totalDeleted += deletionResults.filter( (result) => result.status === 'fulfilled' && result.value, ).length; } return totalDeleted; } } export default OperationIdService; ================================================ FILE: src/service/operation-service.js ================================================ import { Mutex } from 'async-mutex'; import { OPERATION_ID_STATUS, OPERATION_REQUEST_STATUS, OPERATION_STATUS, } from '../constants/constants.js'; const MUTEX_TTL_MS = 5 * 60 * 1000; const MUTEX_SWEEP_INTERVAL_MS = 5 * 60 * 1000; class OperationService { constructor(ctx) { this.logger = ctx.logger; this.repositoryModuleManager = ctx.repositoryModuleManager; this.operationIdService = ctx.operationIdService; this.commandExecutor = ctx.commandExecutor; this._operationMutexes = new Map(); this._terminalOperations = new Map(); this._sweepInterval = setInterval(() => this._sweepStaleMutexes(), MUTEX_SWEEP_INTERVAL_MS); if (this._sweepInterval.unref) { this._sweepInterval.unref(); } } _getOperationMutex(operationId) { if (!this._operationMutexes.has(operationId)) { this._operationMutexes.set(operationId, new Mutex()); } return this._operationMutexes.get(operationId); } _markOperationTerminal(operationId) { this._terminalOperations.set(operationId, Date.now()); } _isOperationTerminal(operationId) { return this._terminalOperations.has(operationId); } _sweepStaleMutexes() { const now = Date.now(); for (const [operationId, terminatedAt] of this._terminalOperations) { if (now - terminatedAt >= MUTEX_TTL_MS) { this._operationMutexes.delete(operationId); this._terminalOperations.delete(operationId); } } } getOperationName() { return this.operationName; } getNetworkProtocols() { return this.networkProtocols; } async getOperationStatus(operationId) { return this.repositoryModuleManager.getOperationStatus( this.getOperationName(), operationId, ); } async getResponsesStatuses(responseStatus, errorMessage, operationId) { let responses = []; const self = this; const mutex = this._getOperationMutex(operationId); await mutex.runExclusive(async () => { if (self._isOperationTerminal(operationId)) { self.logger.debug(`Skipping late response for terminal operation ${operationId}`); return; } await self.repositoryModuleManager.createOperationResponseRecord( responseStatus, this.operationName, operationId, errorMessage, ); responses = await self.repositoryModuleManager.getOperationResponsesStatuses( this.operationName, operationId, ); }); const operationIdStatuses = {}; operationIdStatuses[operationId] = { failedNumber: 0, completedNumber: 0 }; for (const response of responses) { if (response.status === OPERATION_REQUEST_STATUS.FAILED) { operationIdStatuses[operationId].failedNumber += 1; } else { operationIdStatuses[operationId].completedNumber += 1; } } return operationIdStatuses; } async markOperationAsCompleted( operationId, blockchain, responseData, endStatuses, options = {}, ) { this._markOperationTerminal(operationId); const { reuseExistingCache = false } = options; this.logger.info(`Finalizing ${this.operationName} for operationId: ${operationId}`); await this.repositoryModuleManager.updateOperationStatus( this.operationName, operationId, OPERATION_STATUS.COMPLETED, ); if (responseData === null) { await this.operationIdService.removeOperationIdCache(operationId); } else { await this.operationIdService.cacheOperationIdDataToMemory(operationId, responseData); if (!reuseExistingCache) { await this.operationIdService.cacheOperationIdDataToFile(operationId, responseData); } } for (let i = 0; i < endStatuses.length; i += 1) { const status = endStatuses[i]; const response = { status, }; this.operationIdService.emitChangeEvent(status, operationId, blockchain); if (i === endStatuses.length - 1) { // eslint-disable-next-line no-await-in-loop await this.repositoryModuleManager.updateOperationIdRecord(response, operationId); } } } async markOperationAsFailed(operationId, blockchain, message, errorType) { this._markOperationTerminal(operationId); this.logger.info(`${this.operationName} for operationId: ${operationId} failed.`); await this.operationIdService.removeOperationIdCache(operationId); await this.repositoryModuleManager.updateOperationStatus( this.operationName, operationId, OPERATION_STATUS.FAILED, ); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.FAILED, message, errorType, ); } async scheduleOperationForLeftoverNodes(commandData, leftoverNodes) { await this.commandExecutor.add({ name: `${this.operationName}ScheduleMessagesCommand`, delay: 0, data: { ...commandData, leftoverNodes }, transactional: false, }); } logResponsesSummary(completedNumber, failedNumber) { this.logger.info( `Total number of responses: ${ failedNumber + completedNumber }, failed: ${failedNumber}, completed: ${completedNumber}`, ); } getBatchSize() { throw Error('getBatchSize not implemented'); } getMinAckResponses() { throw Error('getMinAckResponses not implemented'); } } export default OperationService; ================================================ FILE: src/service/paranet-service.js ================================================ class ParanetService { constructor(ctx) { this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.ualService = ctx.ualService; this.cryptoService = ctx.cryptoService; } async initializeParanetRecord(blockchain, paranetId) { const paranetName = await this.blockchainModuleManager.getParanetName( blockchain, paranetId, ); const paranetDescription = await this.blockchainModuleManager.getDescription( blockchain, paranetId, ); if (!(await this.repositoryModuleManager.paranetExists(paranetId, blockchain))) { await this.repositoryModuleManager.createParanetRecord( paranetName, paranetDescription, paranetId, blockchain, ); } } constructParanetId(contract, knowledgeCollectionId, knowledgeAssetId) { return this.cryptoService.keccak256EncodePacked( ['address', 'uint256', 'uint256'], [contract, knowledgeCollectionId, knowledgeAssetId], ); } constructKnowledgeAssetId(contract, tokenId) { return this.cryptoService.keccak256EncodePacked( ['address', 'uint256'], [contract, tokenId], ); } getParanetRepositoryName(paranetUAL) { if (this.ualService.isUAL(paranetUAL)) { // Replace : and / with - return `paranet-${paranetUAL.replace(/[/:]/g, '-').toLowerCase()}`; } throw new Error( `Unable to get Paranet repository name. Paranet id doesn't have UAL format: ${paranetUAL}`, ); } getParanetIdFromUAL(paranetUAL) { const { contract, knowledgeCollectionId, knowledgeAssetId } = this.ualService.resolveUAL(paranetUAL); return this.constructParanetId(contract, knowledgeCollectionId, knowledgeAssetId); } } export default ParanetService; ================================================ FILE: src/service/pending-storage-service.js ================================================ import path from 'path'; import { NETWORK_SIGNATURES_FOLDER, PUBLISHER_NODE_SIGNATURES_FOLDER, } from '../constants/constants.js'; class PendingStorageService { constructor(ctx) { this.logger = ctx.logger; this.fileService = ctx.fileService; this.repositoryModuleManager = ctx.repositoryModuleManager; // this is not used this.tripleStoreService = ctx.tripleStoreService; // this is not used this._merkleRootIndex = new Map(); } async cacheDataset(operationId, datasetRoot, dataset, remotePeerId) { this.logger.debug( `Caching ${datasetRoot} dataset root, operation id: ${operationId} in file in pending storage`, ); this._merkleRootIndex.set(datasetRoot, operationId); await this.fileService.writeContentsToFile( this.fileService.getPendingStorageCachePath(), operationId, JSON.stringify({ merkleRoot: datasetRoot, assertion: dataset, remotePeerId, }), ); } getOperationIdByMerkleRoot(merkleRoot) { return this._merkleRootIndex.get(merkleRoot) ?? null; } async getCachedDataset(operationId) { this.logger.debug(`Retrieving cached dataset for ${operationId} from pending storage`); const filePath = this.fileService.getPendingStorageDocumentPath(operationId); try { const fileContents = await this.fileService.readFile(filePath, true); return fileContents.assertion; } catch (error) { this.logger.error( `Failed to retrieve or parse cached dataset for ${operationId}: ${error.message}`, ); throw error; } } async removeExpiredFileCache(expirationTimeMillis, maxRemovalCount) { this.logger.debug( `Cleaning up expired files older than ${expirationTimeMillis} milliseconds. Max removal: ${maxRemovalCount}`, ); const now = Date.now(); let removedCount = 0; try { // Define the paths to the directories we want to clean const storagePaths = [ this.fileService.getPendingStorageCachePath(), this.fileService.getSignatureStorageFolderPath(NETWORK_SIGNATURES_FOLDER), this.fileService.getSignatureStorageFolderPath(PUBLISHER_NODE_SIGNATURES_FOLDER), ]; const filesToDelete = []; // Function to collect files from the provided base path const collectFiles = async (basePath) => { if (!(await this.fileService.pathExists(basePath))) { this.logger.warn(`Storage path does not exist: ${basePath}`); return; } const files = await this.fileService.readDirectory(basePath); // Add all files found in the directory to the filesToDelete array files.forEach((file) => { filesToDelete.push({ file, basePath }); }); }; // Collect files from both storage paths for (const basePath of storagePaths) { // eslint-disable-next-line no-await-in-loop await collectFiles(basePath); } // Function to delete an expired file const deleteFile = async ({ file, basePath }) => { const filePath = path.join(basePath, file); this.logger.debug(`Attempting to delete file: ${filePath}`); try { const fileStats = await this.fileService.stat(filePath); this.logger.debug(`File stats for ${filePath}: ${JSON.stringify(fileStats)}`); const createdDate = fileStats.mtime; if (createdDate.getTime() + expirationTimeMillis < now) { this._removeMerkleRootIndexEntry(file); await this.fileService.removeFile(filePath); this.logger.debug(`Deleted expired file: ${filePath}`); return true; } } catch (fileError) { this.logger.warn(`Failed to process file ${filePath}: ${fileError.message}`); } return false; }; // Process files in batches for (let i = 0; i < filesToDelete.length; i += maxRemovalCount) { const batch = filesToDelete.slice(i, i + maxRemovalCount); // eslint-disable-next-line no-await-in-loop const deletionResults = await Promise.allSettled(batch.map(deleteFile)); removedCount += deletionResults.filter( (result) => result.status === 'fulfilled' && result.value, ).length; if (removedCount >= maxRemovalCount) { this.logger.debug(`Reached max removal count: ${maxRemovalCount}`); return removedCount; } } } catch (error) { this.logger.error(`Error during file cleanup: ${error.message}`); throw error; } this.logger.debug(`Total files removed: ${removedCount}`); return removedCount; } async getCachedAssertion(repository, blockchain, contract, tokenId, assertionId, operationId) { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.debug( `Reading cached assertion for ual: ${ual}, assertion id: ${assertionId}, operation id: ${operationId} from file in ${repository} pending storage`, ); try { const documentPath = await this.fileService.getPendingStorageDocumentPath(operationId); const data = await this.fileService.readFile(documentPath, true); return data; } catch (error) { this.logger.debug( `Assertion not found in ${repository} pending storage. Error message: ${error.message}, ${error.stackTrace}`, ); return null; } } async removeCachedAssertion(repository, blockchain, contract, tokenId, operationId) { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.debug( `Removing cached assertion for ual: ${ual} operation id: ${operationId} from file in ${repository} pending storage`, ); this._removeMerkleRootIndexEntry(operationId); const pendingAssertionPath = await this.fileService.getPendingStorageDocumentPath( operationId, ); await this.fileService.removeFile(pendingAssertionPath); const pendingStorageFolderPath = this.fileService.getParentDirectory(pendingAssertionPath); try { const otherPendingAssertions = await this.fileService.readDirectory( pendingStorageFolderPath, ); if (otherPendingAssertions.length === 0) { await this.fileService.removeFolder(pendingStorageFolderPath); } } catch (error) { this.logger.debug( `Assertions folder not found in ${repository} pending storage. ` + `Error message: ${error.message}, ${error.stackTrace}`, ); } } _removeMerkleRootIndexEntry(operationId) { for (const [root, opId] of this._merkleRootIndex) { if (opId === operationId) { this._merkleRootIndex.delete(root); break; } } } async getPendingState(operationId) { return this.fileService.getPendingStorageLatestDocument(operationId); } } export default PendingStorageService; ================================================ FILE: src/service/proofing-service.js ================================================ import { kcTools } from 'assertion-tools'; import { setTimeout } from 'timers/promises'; import { PROOFING_INTERVAL, REORG_PROOFING_BUFFER, PRIVATE_HASH_SUBJECT_PREFIX, CHUNK_SIZE, OPERATION_ID_STATUS, TRIPLES_VISIBILITY, PROOFING_MAX_ATTEMPTS, } from '../constants/constants.js'; class ProofingService { constructor(ctx) { this.ctx = ctx; this.logger = ctx.logger; this.ualService = ctx.ualService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.networkModuleManager = ctx.networkModuleManager; this.tripleStoreService = ctx.tripleStoreService; this.validationService = ctx.validationService; this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; } async initialize() { this.logger.info('[PROOFING] Initializing ProofingService'); const promises = []; for (const blockchainId of this.blockchainModuleManager.getImplementationNames()) { this.logger.info( `[PROOFING] Initializing proofing service for blockchain ${blockchainId}`, ); promises.push(this.proofingMechanism(blockchainId)); } await Promise.all(promises); this.logger.info('[PROOFING] ProofingService initialization completed'); } async proofingMechanism(blockchainId) { this.logger.debug( `[PROOFING] Setting up proofing mechanism for blockchain ${blockchainId}`, ); // Flag to track if mechanism is running let isRunning = false; // Set up interval const interval = setInterval(async () => { // Skip if already running if (isRunning) { this.logger.debug( `[PROOFING] Proofing mechanism for ${blockchainId} still running, skipping this interval`, ); return; } try { isRunning = true; this.logger.debug( `[PROOFING] Starting proofing cycle for blockchain ${blockchainId}`, ); // Proofing logic await this.runProofing(blockchainId); this.logger.debug( `[PROOFING] Completed proofing cycle for blockchain ${blockchainId}`, ); } catch (error) { this.logger.error( `[PROOFING] Error in proofing mechanism for ${blockchainId}: ${error.message}, stack: ${error.stack}`, ); } finally { isRunning = false; } }, PROOFING_INTERVAL); // Store interval reference for cleanup this[`${blockchainId}Interval`] = interval; this.logger.info( `[PROOFING] Proofing mechanism initialized for blockchain ${blockchainId}`, ); } async runProofing(blockchainId) { this.logger.debug(`[PROOFING] Running proofing mechanism for ${blockchainId}`); const peerId = this.networkModuleManager.getPeerId().toB58String(); const isNodePartOfShard = await this.repositoryModuleManager.isNodePartOfShard( blockchainId, peerId, ); if (!isNodePartOfShard) { this.logger.debug( `[PROOFING] Skipping proofing. Node is not part of shard for blockchain: ${blockchainId}, peerId: ${peerId}`, ); return; } const identityId = await this.blockchainModuleManager.getIdentityId(blockchainId); // Check what is current proof period {isValid, activeProofPeriodStartBlock} const activeProofPeriodStatus = await this.blockchainModuleManager.getActiveProofPeriodStatus(blockchainId); const latestChallenge = await this.repositoryModuleManager.getLatestRandomSamplingChallengeRecordForBlockchainId( blockchainId, ); this.logger.debug( `[PROOFING] Checking proof period validity: isValid=${activeProofPeriodStatus.isValid}, activeProofPeriodStartBlock=${activeProofPeriodStatus.activeProofPeriodStartBlock}, latestChallengeBlock=${latestChallenge?.activeProofPeriodStartBlock}, sentSuccessfully=${latestChallenge?.sentSuccessfully}, blockchainId=${blockchainId}`, ); if ( activeProofPeriodStatus.isValid && latestChallenge?.activeProofPeriodStartBlock === activeProofPeriodStatus.activeProofPeriodStartBlock.toNumber() ) { if (latestChallenge.sentSuccessfully) { if (!latestChallenge.finalized) { this.logger.debug( `[PROOFING] Processing non-finalized challenge for blockchain: ${blockchainId}`, ); // We have latest challenge and we sent valid proof // Check onchain if it has score const score = await this.blockchainModuleManager.getNodeEpochProofPeriodScore( blockchainId, identityId, latestChallenge.epoch, latestChallenge.activeProofPeriodStartBlock, ); this.logger.debug( `[PROOFING] Retrieved node score for blockchain: ${blockchainId}, identityId: ${identityId}, score: ${score.toString()}`, ); // If score is greater than 0 than proof was sent and was valid // Ensure no reorgs happened by checking if it has score and enough time has passed and if possible mark it as finalized if (score.gt(0)) { // Sent more than minute ago check onchain confirm it finalized and it's good if ( latestChallenge.updatedAt.getTime() + REORG_PROOFING_BUFFER <= Date.now() ) { this.logger.info( `[PROOFING] Finalizing challenge for blockchainId: ${blockchainId}, challengeId: ${latestChallenge.id}`, ); latestChallenge.finalized = true; await this.repositoryModuleManager.setCompletedAndFinalizedRandomSamplingChallengeRecord( latestChallenge.id, true, true, ); this.operationIdService.emitChangeEvent( 'PROOF_CHALANGE_FINALIZED', this.generateOperationId( blockchainId, latestChallenge.epoch, latestChallenge.activeProofPeriodStartBlock, ), blockchainId, latestChallenge.epoch, latestChallenge.activeProofPeriodStartBlock, ); } else { this.logger.info( `[PROOFING] Waiting for reorg buffer to pass before finalizing for blockchain: ${blockchainId}, challengeId: ${latestChallenge.id}`, ); } } else { this.logger.warn( `[PROOFING] Zero score detected, resetting challenge status for blockchain: ${blockchainId}, challengeId: ${latestChallenge.id}`, ); latestChallenge.sentSuccessfully = false; latestChallenge.finalized = false; await this.repositoryModuleManager.setCompletedAndFinalizedRandomSamplingChallengeRecord( latestChallenge.id, latestChallenge.sentSuccessfully, latestChallenge.finalized, ); await this.prepareAndSendProof(blockchainId, identityId); } } } else { const ual = this.ualService.deriveUAL( blockchainId, latestChallenge.contractAddress, latestChallenge.knowledgeCollectionId, ); const data = await this.fetchAndProcessAssertion(blockchainId, ual); this.operationIdService.emitChangeEvent( 'PROOF_ASSERTION_FETCHED', this.generateOperationId( blockchainId, latestChallenge.epoch, latestChallenge.activeProofPeriodStartBlock, ), blockchainId, latestChallenge.epoch, latestChallenge.activeProofPeriodStartBlock, ); if (data.public.length === 0) { this.logger.warn( `[PROOFING] No assertions found for blockchain: ${blockchainId}, challengeId: ${latestChallenge.id}, ual: ${ual}`, ); return; } const proof = await this.calculateAndSubmitProof( data, latestChallenge, blockchainId, ); this.logger.info( `[PROOFING] Proof calculated and submitted successfully for blockchain: ${blockchainId}, challengeId: ${latestChallenge.id}`, ); return proof; } // If finalized is do nothing, wait for next proof } else { this.logger.info(`[PROOFING] Preparing new proof for blockchain: ${blockchainId}`); // Node needs to get new challenge or Node sent wrong proof await this.prepareAndSendProof(blockchainId, identityId); } } async prepareAndSendProof(blockchainId, identityId) { this.logger.debug(`[PROOFING] Starting proof preparation for blockchain: ${blockchainId}`); try { const newChallenge = await this.getAndPersistNewChallenge(blockchainId, identityId); const ual = this.ualService.deriveUAL( blockchainId, newChallenge.contractAddress, newChallenge.knowledgeCollectionId, ); this.logger.debug( `[PROOFING] New challenge created: challengeId=${newChallenge.id}, epoch=${newChallenge.epoch}, contractAddress=${newChallenge.contractAddress}, knowledgeCollectionId=${newChallenge.knowledgeCollectionId}`, ); const data = await this.fetchAndProcessAssertion(blockchainId, ual); this.operationIdService.emitChangeEvent( 'PROOF_ASSERTION_FETCHED', this.generateOperationId( blockchainId, newChallenge.epoch, newChallenge.activeProofPeriodStartBlock, ), blockchainId, newChallenge.epoch, newChallenge.activeProofPeriodStartBlock, ); if (data.public.length === 0) { throw new Error( `[PROOFING] No assertions found for blockchain: ${blockchainId}, ual: ${ual}`, ); } const proof = await this.calculateAndSubmitProof(data, newChallenge, blockchainId); this.logger.info( `[PROOFING] Proof calculated and submitted successfully for blockchain: ${blockchainId}, challengeId: ${newChallenge.id}`, ); return proof; } catch (error) { this.logger.error( `[PROOFING] Failed to prepare and send proof for blockchain: ${blockchainId}. Error: ${error.message}, stack: ${error.stack}`, ); throw error; } } async getAndPersistNewChallenge(blockchainId, identityId) { // Node has challenge for previous period need to get new one // Get new challenge const createChallengeResult = await this.blockchainModuleManager.createChallenge( blockchainId, ); if ( !createChallengeResult.success && !createChallengeResult?.error?.message?.includes( 'An unsolved challenge already exists for this node in the current proof period', ) ) { // Throw an error only if it's not the expected "already exists" error throw new Error(createChallengeResult.error); } const newChallenge = await this.blockchainModuleManager.getNodeChallenge( blockchainId, identityId, ); if (createChallengeResult.success) { // Only emit the event if a new challenge was actually generated this.operationIdService.emitChangeEvent( 'PROOF_NEW_CHALANGE_GENERATED', this.generateOperationId( blockchainId, newChallenge.epoch.toNumber(), newChallenge.activeProofPeriodStartBlock.toNumber(), ), blockchainId, newChallenge.epoch.toNumber(), newChallenge.activeProofPeriodStartBlock.toNumber(), ); } const newChallengeRecord = { blockchainId, epoch: newChallenge.epoch.toNumber(), activeProofPeriodStartBlock: newChallenge.activeProofPeriodStartBlock.toNumber(), contractAddress: newChallenge.knowledgeCollectionStorageContract.toLowerCase(), knowledgeCollectionId: newChallenge.knowledgeCollectionId.toNumber(), chunkNumber: newChallenge.chunkId.toNumber(), sentSuccessfully: false, finalized: false, }; const newRecord = await this.repositoryModuleManager.createRandomSamplingChallengeRecord( newChallengeRecord, ); this.operationIdService.emitChangeEvent( 'PROOF_NEW_CHALANGE_PERSISTED', this.generateOperationId( blockchainId, newChallenge.epoch.toNumber(), newChallenge.activeProofPeriodStartBlock.toNumber(), ), blockchainId, newChallenge.epoch.toNumber(), newChallenge.activeProofPeriodStartBlock.toNumber(), ); return newRecord; } async fetchAndProcessAssertion(blockchainId, ual) { let attempt = 0; let getResult; const getOperationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.GET.GET_START, ); this.operationIdService.emitChangeEvent( 'PROOFING_GET_STARTED', getOperationId, blockchainId, ); this.logger.debug( `[PROOFING] Proofing GET started for blockchain: ${blockchainId}, operationId: ${getOperationId}`, ); const { contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); await this.commandExecutor.add({ name: 'getCommand', sequence: [], delay: 0, data: { operationId: getOperationId, blockchain: blockchainId, contract, knowledgeCollectionId, state: 0, ual, contentType: TRIPLES_VISIBILITY.PUBLIC, }, transactional: false, }); do { // eslint-disable-next-line no-await-in-loop await setTimeout(500); // eslint-disable-next-line no-await-in-loop getResult = await this.operationIdService.getOperationIdRecord(getOperationId); attempt += 1; } while ( attempt < PROOFING_MAX_ATTEMPTS && getResult?.status !== OPERATION_ID_STATUS.FAILED && getResult?.status !== OPERATION_ID_STATUS.COMPLETED ); if (getResult?.status !== OPERATION_ID_STATUS.COMPLETED) { // We need to stop here and retry later throw new Error( `[PROOFING] Unable to Proofing GET Knowledge Collection for proof Id: ${knowledgeCollectionId}, for contract: ${contract}, blockchain: ${blockchainId}, GET result: ${JSON.stringify( getResult, )}`, ); } const { assertion } = await this.operationIdService.getCachedOperationIdData( getOperationId, ); this.logger.debug( `[PROOFING] Proofing GET: ${assertion.public.length} nquads found for asset with ual: ${ual}`, ); return assertion; } async calculateAndSubmitProof(data, challenge, blockchainId) { const publicAssertion = data.public; const filteredPublic = []; const privateHashTriples = []; publicAssertion.forEach((triple) => { if (triple.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { privateHashTriples.push(triple); } else { filteredPublic.push(triple); } }); let publicKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( filteredPublic, true, ); publicKnowledgeAssetsTriplesGrouped.push( ...kcTools.groupNquadsBySubject(privateHashTriples, true), ); publicKnowledgeAssetsTriplesGrouped = publicKnowledgeAssetsTriplesGrouped .map((t) => t.sort()) .flat(); // Calculate proof const proof = kcTools.calculateMerkleProof( publicKnowledgeAssetsTriplesGrouped, CHUNK_SIZE, challenge.chunkNumber, ); // Submit proof // How to validate result? (we do it in next iteration) const chunks = kcTools.splitIntoChunks(publicKnowledgeAssetsTriplesGrouped); const chunk = chunks[challenge.chunkNumber]; await this.blockchainModuleManager.submitProof(blockchainId, chunk, proof.proof); this.operationIdService.emitChangeEvent( 'PROOF_SUBMITTED', this.generateOperationId( blockchainId, challenge.epoch, challenge.activeProofPeriodStartBlock, ), blockchainId, null, null, ); const score = await this.blockchainModuleManager.getNodeEpochProofPeriodScore( blockchainId, await this.blockchainModuleManager.getIdentityId(blockchainId), challenge.epoch, challenge.activeProofPeriodStartBlock, ); if (score.gt(0)) { // Move score persistence to finalization await this.repositoryModuleManager.setCompletedAndScoreRandomSamplingChallengeRecord( challenge.id, true, BigInt(score.toString()), // eslint-disable-line no-undef ); this.operationIdService.emitChangeEvent( 'PROOF_SUBMITTED_SUCCESSFULLY', this.generateOperationId( blockchainId, challenge.epoch, challenge.activeProofPeriodStartBlock, ), blockchainId, null, null, ); } return proof; } generateOperationId(blockchainId, epoch, activeProofPeriodStartBlock) { return `${blockchainId}-${epoch}-${activeProofPeriodStartBlock}`; } // Add cleanup method to stop intervals cleanup() { this.logger.info('[PROOFING] Starting ProofingService cleanup'); for (const blockchainId of this.blockchainModuleManager.getImplementationNames()) { const intervalKey = `${blockchainId}Interval`; if (this[intervalKey]) { this.logger.debug(`Clearing interval for blockchain ${blockchainId}`); clearInterval(this[intervalKey]); this[intervalKey] = null; } } this.logger.info('[PROOFING] ProofingService cleanup completed'); } } export default ProofingService; ================================================ FILE: src/service/protocol-service.js ================================================ import { NETWORK_PROTOCOLS } from '../constants/constants.js'; class ProtocolService { constructor(ctx) { this.logger = ctx.logger; } toAwilixVersion(protocol) { const { version } = this.resolveProtocol(protocol); return `v${version.split('.').join('_')}`; } resolveProtocol(protocol) { const [, name, version] = protocol.split('/'); return { name, version }; } getProtocols() { return Object.values(NETWORK_PROTOCOLS); } toOperation(protocol) { const { name } = this.resolveProtocol(protocol); switch (name) { case 'store': return 'publish'; case 'batch-get': return 'batchGet'; default: return name; } } getReceiverCommandSequence(protocol) { const version = this.toAwilixVersion(protocol); const { name } = this.resolveProtocol(protocol); const capitalizedOperation = name.charAt(0).toUpperCase() + name.slice(1); const prefix = `${version}Handle${capitalizedOperation}`; return [`${prefix}RequestCommand`]; } getSenderCommandSequence(protocol) { const version = this.toAwilixVersion(protocol); const operation = this.toOperation(protocol); const capitalizedOperation = operation.charAt(0).toUpperCase() + operation.slice(1); const prefix = `${version}${capitalizedOperation}`; return [`${prefix}RequestCommand`]; } } export default ProtocolService; ================================================ FILE: src/service/publish-service.js ================================================ import OperationService from './operation-service.js'; import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, ERROR_TYPE, OPERATIONS, PUBLISH_BATCH_SIZE, PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS, } from '../constants/constants.js'; class PublishService extends OperationService { constructor(ctx) { super(ctx); this.repositoryModuleManager = ctx.repositoryModuleManager; this.operationName = OPERATIONS.PUBLISH; this.networkProtocols = NETWORK_PROTOCOLS.STORE; this.errorType = ERROR_TYPE.PUBLISH.PUBLISH_ERROR; this.completedStatuses = [ OPERATION_ID_STATUS.PUBLISH.PUBLISH_REPLICATE_END, OPERATION_ID_STATUS.PUBLISH.PUBLISH_END, OPERATION_ID_STATUS.COMPLETED, ]; } async processResponse(command, responseStatus, responseData, errorMessage = null) { const { operationId, blockchain, numberOfFoundNodes, batchSize, minAckResponses, datasetRoot, } = command.data; const datasetRootStatus = await this.getResponsesStatuses( responseStatus, errorMessage, operationId, ); const { completedNumber, failedNumber } = datasetRootStatus[operationId]; const totalResponses = completedNumber + failedNumber; this.logger.debug( `Processing ${ this.operationName } response with status: ${responseStatus} for operationId: ${operationId}, dataset root: ${datasetRoot}. Total number of nodes: ${numberOfFoundNodes}, number of nodes in batch: ${Math.min( numberOfFoundNodes, batchSize, )}, number of responses: ${totalResponses}, Completed: ${completedNumber}, Failed: ${failedNumber}, minimum replication factor: ${minAckResponses}`, ); if (responseData.errorMessage) { this.logger.trace( `Error message for operation id: ${operationId}, dataset root: ${datasetRoot} : ${responseData.errorMessage}`, ); } // 1. Check minimum replication reached // if (completedNumber === minAckResponses) { // this.logger.debug( // `Minimum replication ${minAckResponses} reached for operationId: ${operationId}, dataset root: ${datasetRoot}`, // ); // await this.repositoryModuleManager.updateMinAcksReached(operationId, true); // } // 2. Check if all responses have been received // 2.1 If minimum replication is reached, mark the operation as completed const record = await this.operationIdService.getOperationIdRecord(operationId); if (record?.minAcksReached) return; if (completedNumber >= minAckResponses) { this.logger.info( `[PUBLISH] Minimum replication reached for operationId: ${operationId}, ` + `datasetRoot: ${datasetRoot}, completed: ${completedNumber}/${minAckResponses}`, ); await this.repositoryModuleManager.updateMinAcksReached(operationId, true); const cachedData = (await this.operationIdService.getCachedOperationIdData(operationId)) || null; await this.markOperationAsCompleted( operationId, blockchain, cachedData, this.completedStatuses, { reuseExistingCache: true }, ); this.logResponsesSummary(completedNumber, failedNumber); } // 2.2 Otherwise, mark as failed else if (totalResponses === numberOfFoundNodes) { this.logger.warn( `[PUBLISH] Failed for operationId: ${operationId}, ` + `only ${completedNumber}/${minAckResponses} nodes responded successfully`, ); await this.markOperationAsFailed( operationId, blockchain, 'Not replicated to enough nodes!', this.errorType, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.PUBLISH.PUBLISH_FAILED, operationId, ); this.logResponsesSummary(completedNumber, failedNumber); } // else { // // 3. Not all responses have arrived yet. // const potentialCompletedNumber = completedNumber + leftoverNodes.length; // const canStillReachMinReplication = potentialCompletedNumber >= minAckResponses; // const canScheduleBatch = (totalResponses - 1) % batchSize === 0; // // 3.1 Check if minimum replication can still be achieve by scheduling leftover nodes // // (and it's at the end of a batch) // // if (leftoverNodes.length > 0 && canStillReachMinReplication && canScheduleBatch) { // // await this.scheduleOperationForLeftoverNodes(command.data, leftoverNodes); // // } // // 3.2 If minimum replication cannot be reached and it's end of a batch, mark as failed // if (!canStillReachMinReplication && canScheduleBatch) { // await this.markOperationAsFailed( // operationId, // blockchain, // 'Not replicated to enough nodes!', // this.errorType, // ); // this.logResponsesSummary(completedNumber, failedNumber); // } // } } getBatchSize(batchSize = null) { return batchSize ?? PUBLISH_BATCH_SIZE; } getMinAckResponses(minimumNumberOfNodeReplications = null) { return minimumNumberOfNodeReplications ?? PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS; } } export default PublishService; ================================================ FILE: src/service/sharding-table-service.js ================================================ import { BYTES_IN_KILOBYTE, PEER_RECORD_UPDATE_DELAY } from '../constants/constants.js'; class ShardingTableService { constructor(ctx) { this.logger = ctx.logger; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.networkModuleManager = ctx.networkModuleManager; this.cryptoService = ctx.cryptoService; this.memoryCachedPeerIds = {}; } async initialize() { await this.networkModuleManager.onPeerConnected((connection) => { this.updatePeerRecordLastSeenAndLastDialed(connection.remotePeer.toB58String()).catch( (error) => { this.logger.warn(`Unable to update connected peer, error: ${error.message}`); }, ); }); } async pullBlockchainShardingTable(blockchainId, transaction = null) { const options = transaction ? { transaction } : {}; this.logger.debug( `Removing nodes from local sharding table for blockchain ${blockchainId}.`, ); await this.repositoryModuleManager.removeShardingTablePeerRecords(blockchainId, options); const shardingTableLength = await this.blockchainModuleManager.getShardingTableLength( blockchainId, ); let startingIdentityId = await this.blockchainModuleManager.getShardingTableHead( blockchainId, ); const pageSize = 10; const shardingTable = []; this.logger.debug( `Started pulling ${shardingTableLength} nodes from blockchain sharding table.`, ); let sliceIndex = 0; while (shardingTable.length < shardingTableLength) { // eslint-disable-next-line no-await-in-loop const nodes = await this.blockchainModuleManager.getShardingTablePage( blockchainId, startingIdentityId, pageSize, ); shardingTable.push(...nodes.slice(sliceIndex).filter((node) => node.nodeId !== '0x')); sliceIndex = 1; startingIdentityId = nodes[nodes.length - 1].identityId; } this.logger.debug( `Finished pulling ${shardingTable.length} nodes from blockchain sharding table.`, ); const newPeerRecords = await Promise.all( shardingTable.map(async (peer) => { const nodeId = this.cryptoService.convertHexToAscii(peer.nodeId); const sha256 = await this.cryptoService.sha256(nodeId); return { peerId: nodeId, blockchainId, ask: this.cryptoService.convertFromWei(peer.ask, 'ether'), stake: this.cryptoService.convertFromWei(peer.stake, 'ether'), sha256, }; }), ); await this.repositoryModuleManager.createManyPeerRecords(newPeerRecords, options); } async findShard(blockchainId, filterInactive = false) { let peers = await this.repositoryModuleManager.getAllPeerRecords( blockchainId, filterInactive, ); peers = peers.map((peer, index) => ({ ...peer.dataValues, index })); return peers; } async isNodePartOfShard(blockchainId, peerId) { return this.repositoryModuleManager.isNodePartOfShard(blockchainId, peerId); } // TODO: Remove this calculateBidSuggestion(askOffset, sorted, blockchainId, kbSize, epochsNumber, r0) { const effectiveAskOffset = Math.min(askOffset, sorted.length - 1); const { ask } = sorted[effectiveAskOffset]; const bidSuggestion = this.cryptoService .convertToWei(ask) .mul(kbSize) .mul(epochsNumber) .mul(r0) .div(BYTES_IN_KILOBYTE); return bidSuggestion.toString(); } async findEligibleNodes(neighbourhood, bid, r1, r0) { return neighbourhood.filter((node) => node.ask <= bid / r0).slice(0, r1); } async dial(peerId) { try { const { addresses } = await this.findPeerAddressAndProtocols(peerId); if (addresses.length) { if (peerId !== this.networkModuleManager.getPeerId().toB58String()) { this.logger.trace(`Dialing peer ${peerId}.`); await this.networkModuleManager.dial(peerId); } await this.updatePeerRecordLastSeenAndLastDialed(peerId); } else { await this.updatePeerRecordLastDialed(peerId); } } catch (error) { this.logger.trace(`Unable to dial peer ${peerId}. Error: ${error.message}`); await this.updatePeerRecordLastDialed(peerId); } } async updatePeerRecordLastSeenAndLastDialed(peerId) { const now = Date.now(); const timestampThreshold = now - PEER_RECORD_UPDATE_DELAY; if (!this.memoryCachedPeerIds[peerId]) { this.memoryCachedPeerIds[peerId] = { lastDialed: 0, lastSeen: 0, }; } if ( this.memoryCachedPeerIds[peerId].lastSeen < timestampThreshold || this.memoryCachedPeerIds[peerId].lastDialed < timestampThreshold ) { const [rowsUpdated] = await this.repositoryModuleManager.updatePeerRecordLastSeenAndLastDialed( peerId, now, ); if (rowsUpdated) { this.memoryCachedPeerIds[peerId].lastDialed = now; this.memoryCachedPeerIds[peerId].lastSeen = now; } } } async updatePeerRecordLastDialed(peerId) { const now = Date.now(); const timestampThreshold = now - PEER_RECORD_UPDATE_DELAY; if (!this.memoryCachedPeerIds[peerId]) { this.memoryCachedPeerIds[peerId] = { lastDialed: 0, lastSeen: 0, }; } if (this.memoryCachedPeerIds[peerId].lastDialed < timestampThreshold) { const [rowsUpdated] = await this.repositoryModuleManager.updatePeerRecordLastDialed( peerId, now, ); if (rowsUpdated) { this.memoryCachedPeerIds[peerId].lastDialed = now; } } } async findPeerAddressAndProtocols(peerId) { this.logger.trace(`Searching for peer ${peerId} multiaddresses in peer store.`); let peerInfo = await this.networkModuleManager.getPeerInfo(peerId); if ( !peerInfo?.addresses?.length && peerId !== this.networkModuleManager.getPeerId().toB58String() ) { try { this.logger.trace(`Searching for peer ${peerId} multiaddresses on the network.`); await this.networkModuleManager.findPeer(peerId); peerInfo = await this.networkModuleManager.getPeerInfo(peerId); } catch (error) { this.logger.trace(`Unable to find peer ${peerId}. Error: ${error.message}`); } } return { id: peerId, addresses: peerInfo?.addresses ?? [], protocols: peerInfo?.protocols ?? [], }; } } export default ShardingTableService; ================================================ FILE: src/service/signature-service.js ================================================ class SignatureService { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; this.cryptoService = ctx.cryptoService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.fileService = ctx.fileService; } async signMessage(blockchain, messageHash) { const flatSignature = await this.blockchainModuleManager.signMessage( blockchain, messageHash, ); const { v, r, s, _vs } = this.cryptoService.splitSignature(flatSignature); return { v, r, s, vs: _vs }; } async addSignatureToStorage(folderName, operationId, identityId, v, r, s, vs) { await this.fileService.appendContentsToFile( this.fileService.getSignatureStorageFolderPath(folderName), operationId, `${JSON.stringify({ identityId, v, r, s, vs })}\n`, ); } async getSignaturesFromStorage(folderName, operationId) { const signatureStorageFile = this.fileService.getSignatureStorageDocumentPath( folderName, operationId, ); const rawSignatures = await this.fileService.readFile(signatureStorageFile); const signaturesArray = []; for (const line of rawSignatures.split('\n')) { const trimmedLine = line.trim(); if (trimmedLine) { signaturesArray.push(JSON.parse(trimmedLine)); } } return signaturesArray; } } export default SignatureService; ================================================ FILE: src/service/sync-service.js ================================================ import { v4 as uuidv4 } from 'uuid'; import { setTimeout } from 'timers/promises'; import { SYNC_INTERVAL, OPERATION_ID_STATUS, DKG_METADATA_PREDICATES, TRIPLE_STORE_REPOSITORY, BATCH_GET_UAL_MAX_LIMIT, SYNC_BATCH_GET_MAX_ATTEMPTS, SYNC_BATCH_GET_WAIT_TIME, } from '../constants/constants.js'; class SyncService { // TODO: Send getter for Neuroweb fixed on last finalised block, there should be ethers flag constructor(ctx) { this.ctx = ctx; this.syncConfig = ctx.config.assetSync.syncDKG; this.logger = ctx.logger; this.ualService = ctx.ualService; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.tripleStoreService = ctx.tripleStoreService; this.validationService = ctx.validationService; this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.syncStatus = {}; this.registeredIntervals = []; } async initialize() { if (!this.syncConfig.enabled) { this.logger.info('[DKG SYNC] SyncService disabled'); return; } this.logger.info('[DKG SYNC] Initializing SyncService'); this.syncBatchSize = this.syncConfig.syncBatchSize; const blockchainIds = this.blockchainModuleManager.getImplementationNames(); const promises = await Promise.all( blockchainIds.map(async (blockchainId) => { this.logger.info( `[DKG SYNC] Initializing sync service for blockchain ${blockchainId}`, ); // Check if operationalDB has all contract present in hub const contracts = await this.blockchainModuleManager.getAssetStorageContractsAddress( blockchainId, ); const dbContracts = await this.repositoryModuleManager.getKCStorageContracts( blockchainId, ); const missingContracts = contracts.filter( (contract) => !dbContracts.some( (dbContract) => dbContract.toJSON().contract_address === contract, ), ); if (missingContracts.length > 0) { this.logger.info( `[DKG SYNC] Adding missing contracts for blockchain ${blockchainId}: ${missingContracts.join( ', ', )}`, ); await this.repositoryModuleManager.addSyncContracts( blockchainId, missingContracts, ); } return this.syncMechanism(blockchainId); }), ); await Promise.all(promises); this.logger.info('[DKG SYNC] SyncService initialization completed'); } // Weirdly named, why not start mechanism? async syncMechanism(blockchainId) { this.logger.debug(`[DKG SYNC] Setting up sync mechanism for blockchain ${blockchainId}`); // Set up intervals let isMissedRunning = false; const intervalMissed = setInterval(async () => { if (isMissedRunning) { this.logger.debug( `[DKG SYNC] Sync missed KC mechanism for ${blockchainId} still running, skipping this interval`, ); return; } try { isMissedRunning = true; this.logger.debug( `[DKG SYNC] Starting sync missed KC cycle for blockchain ${blockchainId}`, ); const syncRecords = ( await this.repositoryModuleManager.getSyncRecordForBlockchain(blockchainId) ).map((syncRecord) => syncRecord.toJSON()); // Run missed KC sync for each contract in parallel await Promise.all( syncRecords.map((record) => this.runSyncMissed(blockchainId, record.contractAddress), ), ); this.logger.debug( `[DKG SYNC] Completed sync missed KC cycle for blockchain ${blockchainId}`, ); } catch (error) { this.logger.error( `[DKG SYNC] Error in sync missed KC mechanism for ${blockchainId}: ${error.message}, stack: ${error.stack}`, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.SYNC.SYNC_MISSED_FAILED, uuidv4(), blockchainId, error.message, error.stack, ); } finally { isMissedRunning = false; } }, SYNC_INTERVAL); let isNewRunning = false; const intervalNew = setInterval(async () => { if (isNewRunning) { this.logger.debug( `[DKG SYNC] Sync new KC mechanism for ${blockchainId} still running, skipping this interval`, ); return; } try { isNewRunning = true; this.logger.debug( `[DKG SYNC] Starting sync new KC cycle for blockchain ${blockchainId}`, ); const syncRecords = ( await this.repositoryModuleManager.getSyncRecordForBlockchain(blockchainId) ).map((syncRecord) => syncRecord.toJSON()); await this.runSyncNewKc(blockchainId, syncRecords); this.logger.debug( `[DKG SYNC] Completed sync new KC cycle for blockchain ${blockchainId}`, ); } catch (error) { this.logger.error( `[DKG SYNC] Error in sync new KC mechanism for ${blockchainId}: ${error.message}, stack: ${error.stack}`, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.SYNC.SYNC_NEW_FAILED, uuidv4(), blockchainId, error.message, error.stack, ); } finally { isNewRunning = false; } }, SYNC_INTERVAL); // Register intervals for the cleanup this.registeredIntervals.push(intervalMissed); this.registeredIntervals.push(intervalNew); // this[`${blockchainId}Interval`] = interval; this.logger.info(`[DKG SYNC] Sync mechanism initialized for blockchain ${blockchainId}`); } async runSyncNewKc(blockchainId, syncRecords) { const syncOperationId = uuidv4(); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.SYNC.SYNC_NEW_START, syncOperationId, blockchainId, ); const latestKnowledgeCollectionIds = {}; const knowledgeCollectionResults = await Promise.all( syncRecords.map(async (syncRecord) => { const latestKnowledgeCollectionId = await this.blockchainModuleManager.getLatestKnowledgeCollectionId( blockchainId, syncRecord.contractAddress, ); return { contractAddress: syncRecord.contractAddress, latestKnowledgeCollectionId, latestSyncedKc: syncRecord.latestSyncedKc, }; }), ); // Filter out null results and build the latestKnowledgeCollectionIds object knowledgeCollectionResults.forEach((result) => { if (result !== null) { latestKnowledgeCollectionIds[result.contractAddress] = { latestKnowledgeCollectionId: result.latestKnowledgeCollectionId, latestSyncedKc: result.latestSyncedKc, }; } }); if (this.syncStatus && this.syncStatus[blockchainId]) { const totallatestKnowledgeCollectionId = Object.values( this.syncStatus[blockchainId], ).reduce((acc, curr) => acc + curr.latestKnowledgeCollectionId, 0); const totalLatestSyncedKc = Object.values(this.syncStatus[blockchainId]).reduce( (acc, curr) => acc + curr.latestSyncedKc, 0, ); const totalMissedKc = Object.values(this.syncStatus[blockchainId]).reduce( (acc, curr) => acc + curr.missedKc, 0, ); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.SYNC.SYNC_PROGRESS_STATUS, syncOperationId, blockchainId, totalLatestSyncedKc, totalMissedKc, totallatestKnowledgeCollectionId, ); const totalMissedKcChecked = !Number.isFinite(totalMissedKc) || Number.isNaN(totalMissedKc) ? 0 : totalMissedKc; const syncPrecentage = (100 * (totalLatestSyncedKc - totalMissedKcChecked)) / totallatestKnowledgeCollectionId; this.logger.info( `[DKG SYNC] DKG Sync for blockchain ${blockchainId} Status: ${syncPrecentage}%`, ); } const contractPromises = Object.entries(latestKnowledgeCollectionIds).map( async ([contractAddress, syncObject]) => { await this.syncNewKc(blockchainId, contractAddress, syncObject); }, ); await Promise.all(contractPromises); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.SYNC.SYNC_NEW_END, syncOperationId, blockchainId, ); } async runSyncMissed(blockchainId, contractAddress) { const syncOperationId = uuidv4(); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.SYNC.SYNC_MISSED_START, syncOperationId, blockchainId, ); await this.syncMissedKc(blockchainId, contractAddress); this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.SYNC.SYNC_MISSED_END, syncOperationId, blockchainId, ); } // TODO: Add syncOperationId with additional events to syncNewKc async syncNewKc(blockchainId, contractAddress, syncObject) { const uals = []; const { latestSyncedKc } = syncObject; const latestKnowledgeCollectionId = syncObject.latestKnowledgeCollectionId.toNumber(); if (!this.syncStatus[blockchainId]) { this.syncStatus[blockchainId] = {}; } if (!this.syncStatus[blockchainId][contractAddress]) { this.syncStatus[blockchainId][contractAddress] = {}; } this.syncStatus[blockchainId][contractAddress].latestSyncedKc = latestSyncedKc; this.syncStatus[blockchainId][contractAddress].latestKnowledgeCollectionId = latestKnowledgeCollectionId; // Calculate upper bound const maxId = Math.min( latestKnowledgeCollectionId, latestSyncedKc + this.syncBatchSize, latestSyncedKc + BATCH_GET_UAL_MAX_LIMIT, ); // Generate UALs from (latestSyncedKc + 1) to maxId for (let id = latestSyncedKc + 1; id <= maxId; id += 1) { const ual = this.ualService.deriveUAL(blockchainId, contractAddress, id); uals.push(ual); } if (uals.length === 0) { this.logger.info(`[DKG SYNC] No UALs to sync for blockchain ${blockchainId}`); return; } const { batchGetResult, batchGetOperationId } = await this.callBatchGet(uals, blockchainId); if (batchGetResult?.status !== OPERATION_ID_STATUS.COMPLETED) { throw new Error( `[DKG SYNC] Unable to Batch GET Knowledge Collection for blockchain: ${blockchainId}, GET result: ${JSON.stringify( batchGetResult, )}`, ); } let insertFailed = false; const data = await this.operationIdService.getCachedOperationIdData(batchGetOperationId); if (Object.values(data.remote).length > 0) { // Update metadata timestamps const updatedMetadata = { ...data.metadata }; Object.entries(updatedMetadata).forEach(([ual, triples]) => { if (Array.isArray(triples)) { updatedMetadata[ual] = triples.map((triple) => { if (triple.includes(DKG_METADATA_PREDICATES.PUBLISH_TIME)) { const splitTriple = triple.split(' '); return `${splitTriple[0]} ${ splitTriple[1] } "${new Date().toISOString()}"^^ .`; } return triple; }); } else { updatedMetadata[ual] = []; } }); data.metadata = updatedMetadata; try { await this.tripleStoreService.insertKnowledgeCollectionBatch( TRIPLE_STORE_REPOSITORY.DKG, data, ); } catch (error) { this.logger.error( `[SYNC] Unable to insert Knowledge Collections for blockchain: ${blockchainId}, error: ${error.message}`, ); insertFailed = true; } } const missingUals = uals.filter((ual) => { const isInLocal = data.local.includes(ual); const hasPublic = data.remote[ual]?.public?.length > 0; // Insert failed, so if it's not in local, it's a missing UAL if (insertFailed) { return !isInLocal; } // If it's not in local and has no public data, it's a missing UAL return !isInLocal && !hasPublic; }); const insertRecords = missingUals.map((ual) => { const { knowledgeCollectionId, contract } = this.ualService.resolveUAL(ual); return { kcId: knowledgeCollectionId, contractAddress: contract, }; }); const transaction = await this.repositoryModuleManager.transaction(); try { if (insertRecords.length > 0) { const error = 'KC not found on network'; await this.repositoryModuleManager.insertMissedKc( blockchainId, insertRecords, error, { transaction }, ); } await this.repositoryModuleManager.updateLatestSyncedKc( blockchainId, contractAddress, latestSyncedKc + uals.length, { transaction }, ); await transaction.commit(); } catch (error) { await transaction.rollback(); throw error; } } // TODO: Add syncOperationId with additional events to syncMissedKc async syncMissedKc(blockchainId, contract) { const missedKcForRetry = await this.repositoryModuleManager.getMissedKcForRetry( blockchainId, contract, this.syncBatchSize > BATCH_GET_UAL_MAX_LIMIT ? BATCH_GET_UAL_MAX_LIMIT : this.syncBatchSize, ); const missedKcForRetryCount = await this.repositoryModuleManager.getMissedKcForRetryCount( blockchainId, contract, ); if (!this.syncStatus[blockchainId]) { this.syncStatus[blockchainId] = {}; } if (!this.syncStatus[blockchainId][contract]) { this.syncStatus[blockchainId][contract] = {}; } this.syncStatus[blockchainId][contract].missedKc = missedKcForRetryCount; if (missedKcForRetry.length === 0) { this.logger.info(`[SYNC] No missed KC for retry for blockchain ${blockchainId}`); return; } // Contracut uals from object const missedUals = missedKcForRetry.map((missedKc) => { const missedKcJson = missedKc.toJSON(); return this.ualService.deriveUAL( blockchainId, missedKcJson.contractAddress, missedKcJson.kcId, ); }); // Call batch get const { batchGetResult, batchGetOperationId } = await this.callBatchGet( missedUals, blockchainId, ); if (batchGetResult?.status !== OPERATION_ID_STATUS.COMPLETED) { throw new Error( `[SYNC] Unable to Batch GET Knowledge Collection for blockchain: ${blockchainId}, GET result: ${JSON.stringify( batchGetResult, )}`, ); } // Insert let insertFailed = false; const data = await this.operationIdService.getCachedOperationIdData(batchGetOperationId); if (Object.values(data.remote).length > 0) { // Update metadata timestamps const updatedMetadata = { ...data.metadata }; Object.entries(updatedMetadata).forEach(([ual, triples]) => { if (Array.isArray(triples)) { updatedMetadata[ual] = triples.map((triple) => { if (triple.includes(DKG_METADATA_PREDICATES.PUBLISH_TIME)) { const splitTriple = triple.split(' '); return `${splitTriple[0]} ${ splitTriple[1] } "${new Date().toISOString()}"^^ .`; } return triple; }); } else { updatedMetadata[ual] = []; } }); data.metadata = updatedMetadata; try { await this.tripleStoreService.insertKnowledgeCollectionBatch('dkg', data); } catch (error) { this.logger.error( `[DKG SYNC] Unable to insert Knowledge Collection for blockchain: ${blockchainId}`, ); insertFailed = true; } } const missingUals = []; const syncedUals = []; missedUals.forEach((ual) => { const isLocal = data.local.includes(ual); const hasRemoteData = data.remote[ual]?.public?.length > 0; // If insert failed, and KC not locally present, add it to missed UALs if (insertFailed) { if (!isLocal) { missingUals.push(ual); } else { syncedUals.push(ual); } } // If insert was successful, and KC is locally present or fetched from remote node, add it to synced UALs else if (isLocal || hasRemoteData) { syncedUals.push(ual); } else { missingUals.push(ual); } }); const recordsToUpdateForRetry = missingUals.map((ual) => { const { knowledgeCollectionId, contract: ualContract } = this.ualService.resolveUAL(ual); return { kcId: knowledgeCollectionId, contractAddress: ualContract, }; }); const recordsToUpdateForSuccess = syncedUals.map((ual) => { const { knowledgeCollectionId, contract: ualContract } = this.ualService.resolveUAL(ual); return { kcId: knowledgeCollectionId, contractAddress: ualContract, }; }); const transaction = await this.repositoryModuleManager.transaction(); try { if (recordsToUpdateForRetry.length > 0) { await this.repositoryModuleManager.incrementRetryCount( blockchainId, recordsToUpdateForRetry, { transaction }, ); } if (recordsToUpdateForSuccess.length > 0) { await this.repositoryModuleManager.setSyncedToTrue( blockchainId, recordsToUpdateForSuccess, { transaction }, ); } await transaction.commit(); } catch (error) { await transaction.rollback(); throw error; } } async callBatchGet(uals, blockchainId) { const batchGetOperationId = await this.operationIdService.generateOperationId( OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_INIT, blockchainId, ); await this.commandExecutor.add({ name: 'batchGetCommand', sequence: [], delay: 0, data: { operationId: batchGetOperationId, uals, blockchain: blockchainId, includeMetadata: true, contentType: 'all', }, transactional: false, }); let batchGetResult; let attempts = 0; // Poll for result while (attempts < SYNC_BATCH_GET_MAX_ATTEMPTS) { // eslint-disable-next-line no-await-in-loop await setTimeout(SYNC_BATCH_GET_WAIT_TIME); // eslint-disable-next-line no-await-in-loop batchGetResult = await this.operationIdService.getOperationIdRecord( batchGetOperationId, ); if ( batchGetResult?.status === OPERATION_ID_STATUS.FAILED || batchGetResult?.status === OPERATION_ID_STATUS.COMPLETED ) { break; } attempts += 1; } return { batchGetResult, batchGetOperationId }; } // Add cleanup method to stop intervals cleanup() { for (const interval of this.registeredIntervals) { clearInterval(interval); } } } export default SyncService; ================================================ FILE: src/service/triple-store-service.js ================================================ /* eslint-disable no-await-in-loop */ import { setTimeout } from 'timers/promises'; import { kcTools } from 'assertion-tools'; import { BASE_NAMED_GRAPHS, TRIPLE_STORE_REPOSITORY, TRIPLES_VISIBILITY, PRIVATE_HASH_SUBJECT_PREFIX, DKG_PREDICATE, HAS_KNOWLEDGE_ASSET_SUFFIX, HAS_NAMED_GRAPH_SUFFIX, DKG_METADATA_PREDICATES, SCHEMA_CONTEXT, MAX_TOKEN_ID_PER_GET_PAGE, } from '../constants/constants.js'; class TripleStoreService { constructor(ctx) { this.config = ctx.config; this.logger = ctx.config.logging.enableExperimentalScopes ? ctx.logger.child({ scope: 'TripleStoreService', }) : ctx.logger; this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; this.operationIdService = ctx.operationIdService; this.ualService = ctx.ualService; this.dataService = ctx.dataService; this.paranetService = ctx.paranetService; this.cryptoService = ctx.cryptoService; } initializeRepositories() { this.repositoryImplementations = {}; for (const implementationName of this.tripleStoreModuleManager.getImplementationNames()) { for (const repository in this.tripleStoreModuleManager.getImplementation( implementationName, ).module.repositories) { this.repositoryImplementations[repository] = implementationName; } } } async insertKnowledgeCollection( repository, knowledgeCollectionUAL, triples, metadata, retries = 5, retryDelay = 50, paranetUAL = '', contentType = '', ) { this.logger.info( `Inserting Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` + `to the Triple Store's ${repository} repository.`, ); const publicAssertion = triples.public ?? triples; const filteredPublic = []; const privateHashTriples = []; const tripleSet = new Set(); let totalNumberOfTriplesInserted = triples?.public ? triples.public.length + (triples.private?.length ?? 0) : triples?.length ?? 0; publicAssertion.forEach((triple) => { if (triple.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { privateHashTriples.push(triple); } else { filteredPublic.push(triple); } }); const publicKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( filteredPublic, true, ); publicKnowledgeAssetsTriplesGrouped.push( ...kcTools.groupNquadsBySubject(privateHashTriples, true), ); const publicKnowledgeAssetsUALs = publicKnowledgeAssetsTriplesGrouped.map( (_, index) => `${knowledgeCollectionUAL}/${index + 1}`, ); const allPossibleNamedGraphs = []; let privateGraphsInsert = ''; let currentPrivateMetadataTriples = ''; let connectionPrivateMetadataTriples = ''; const publicGraphsInsert = publicKnowledgeAssetsUALs .map( (ual, index) => ` GRAPH <${ual}/${TRIPLES_VISIBILITY.PUBLIC}> { ${publicKnowledgeAssetsTriplesGrouped[index].join('\n')} } `, ) .join('\n'); const currentPublicMetadataTriples = publicKnowledgeAssetsUALs .map( (ual) => ` <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${ual}/${TRIPLES_VISIBILITY.PUBLIC}> .`, ) .join('\n'); const connectionPublicMetadataTriples = publicKnowledgeAssetsUALs .map((ual) => { const graphWithVisibility = `${ual}/${TRIPLES_VISIBILITY.PUBLIC}`; return [ `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_KNOWLEDGE_ASSET_SUFFIX}> <${ual}> .`, `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${graphWithVisibility}> .`, ].join('\n'); }) .join('\n'); // current metadata triple relates to which named graph that represents Knowledge Asset hold the lates(current) data // so for each Knowledge Asset there will be one current metadata triple // in this case there are publicKnowledgeAssetsUALs.length number of named graphs created so for each there will be one current metadata triple totalNumberOfTriplesInserted += publicKnowledgeAssetsUALs.length; publicKnowledgeAssetsUALs.forEach((ual) => { const graphWithVisibility = `${ual}/public`; tripleSet.add( `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_KNOWLEDGE_ASSET_SUFFIX}> <${ual}> .`, ); tripleSet.add( `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${graphWithVisibility}> .`, ); }); this.logger.info( `Adding metadata triples for public asets for Knowledge Collection: ${knowledgeCollectionUAL}`, ); allPossibleNamedGraphs.push(...publicKnowledgeAssetsUALs.map((ual) => `${ual}/public`)); if (triples.private?.length) { const privateKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( triples.private, true, ); const privateKnowledgeAssetsUALs = []; const publicSubjectMap = publicKnowledgeAssetsTriplesGrouped.reduce( (map, group, index) => { const [publicSubject] = group[0].split(' '); map.set(publicSubject, index); return map; }, new Map(), ); for (const privateTriple of privateKnowledgeAssetsTriplesGrouped) { const [privateSubject] = privateTriple[0].split(' '); if (publicSubjectMap.has(privateSubject)) { const ualIndex = publicSubjectMap.get(privateSubject); privateKnowledgeAssetsUALs.push(publicKnowledgeAssetsUALs[ualIndex]); } else { const privateSubjectHashed = `<${PRIVATE_HASH_SUBJECT_PREFIX}${this.cryptoService.sha256( privateSubject.slice(1, -1), )}>`; if (publicSubjectMap.has(privateSubjectHashed)) { const ualIndex = publicSubjectMap.get(privateSubjectHashed); privateKnowledgeAssetsUALs.push(publicKnowledgeAssetsUALs[ualIndex]); } } } privateGraphsInsert = privateKnowledgeAssetsUALs .map( (ual, index) => ` GRAPH <${ual}/${TRIPLES_VISIBILITY.PRIVATE}> { ${privateKnowledgeAssetsTriplesGrouped[index].join('\n')} } `, ) .join('\n'); currentPrivateMetadataTriples = privateKnowledgeAssetsUALs .map( (ual) => ` <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${ual}/${TRIPLES_VISIBILITY.PRIVATE}> .`, ) .join('\n'); connectionPrivateMetadataTriples = privateKnowledgeAssetsUALs .map((ual) => { const graphWithVisibility = `${ual}/${TRIPLES_VISIBILITY.PRIVATE}`; return [ `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_KNOWLEDGE_ASSET_SUFFIX}> <${ual}> .`, `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${graphWithVisibility}> .`, ].join('\n'); }) .join('\n'); // current metadata triple relates to which named graph that represents Knowledge Asset hold the lates(current) data // so for each Knowledge Asset there will be one current metadata triple // in this case there are privateKnowledgeAssetsUALs.length number of named graphs created so for each there will be one current metadata triple totalNumberOfTriplesInserted += privateKnowledgeAssetsUALs.length; privateKnowledgeAssetsUALs.forEach((ual) => { const graphWithVisibility = `${ual}/private`; tripleSet.add( `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_KNOWLEDGE_ASSET_SUFFIX}> <${ual}> .`, ); tripleSet.add( `<${knowledgeCollectionUAL}> <${DKG_PREDICATE}${HAS_NAMED_GRAPH_SUFFIX}> <${graphWithVisibility}> .`, ); }); this.logger.info( `Adding metadata triples for private asets for Knowledge Collection: ${knowledgeCollectionUAL}`, ); allPossibleNamedGraphs.push( ...privateKnowledgeAssetsUALs.map((ual) => `${ual}/private`), ); } // TODO: add new metadata triples and move to function insertMetadataTriples let metadataTriples = publicKnowledgeAssetsUALs .map( (publicKnowledgeAssetUAL) => `<${publicKnowledgeAssetUAL}> "${publicKnowledgeAssetUAL}:0" .`, ) .join('\n'); metadataTriples += `\n<${knowledgeCollectionUAL}> <${DKG_METADATA_PREDICATES.PUBLISHED_BY}> .` + `\n<${knowledgeCollectionUAL}> <${DKG_METADATA_PREDICATES.PUBLISHED_AT_BLOCK}> "${metadata.blockNumber}" .` + `\n<${knowledgeCollectionUAL}> <${DKG_METADATA_PREDICATES.PUBLISH_TX}> "${metadata.txHash}" .` + `\n<${knowledgeCollectionUAL}> <${ DKG_METADATA_PREDICATES.PUBLISH_TIME }> "${new Date().toISOString()}"^^ .` + `\n<${knowledgeCollectionUAL}> <${DKG_METADATA_PREDICATES.BLOCK_TIME}> "${new Date( metadata.blockTimestamp * 1000, ).toISOString()}"^^ .`; // totalNumberOfTriplesInserted += publicKnowledgeAssetsUALs.length + 5; // one metadata triple for each public KA const insertQuery = ` PREFIX schema: <${SCHEMA_CONTEXT}> INSERT DATA { ${publicGraphsInsert} ${privateGraphsInsert} GRAPH <${BASE_NAMED_GRAPHS.CURRENT}> { ${currentPublicMetadataTriples} ${currentPrivateMetadataTriples} } GRAPH <${BASE_NAMED_GRAPHS.METADATA}> { ${connectionPublicMetadataTriples} ${connectionPrivateMetadataTriples} ${metadataTriples} } } `; const uniqueTripleCount = tripleSet.size; totalNumberOfTriplesInserted += uniqueTripleCount; let attempts = 0; let success = false; while (attempts < retries && !success) { try { await this.tripleStoreModuleManager.queryVoid( this.repositoryImplementations[repository], repository, insertQuery, this.config.modules.tripleStore.timeout.insert, ); if (paranetUAL) { await this.tripleStoreModuleManager.createParanetKnoledgeCollectionConnection( this.repositoryImplementations[repository], repository, knowledgeCollectionUAL, paranetUAL, contentType, this.config.modules.tripleStore.timeout.insert, ); totalNumberOfTriplesInserted += allPossibleNamedGraphs.length; // one triple will be created for each Knowledge Asset inserted into paranet this.logger.info(`Adding connection triples for paranet: ${paranetUAL}`); } success = true; this.logger.info( `Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` + `has been successfully inserted to the Triple Store's ${repository} repository.`, ); } catch (error) { this.logger.error( `Error during insertion of the Knowledge Collection to the Triple Store's ${repository} repository. ` + `UAL: ${knowledgeCollectionUAL}. Error: ${error.message}`, ); attempts += 1; if (attempts < retries) { this.logger.info( `Retrying insertion of the Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` + `to the Triple Store's ${repository} repository. Attempt ${ attempts + 1 } of ${retries} after delay of ${retryDelay} ms.`, ); await setTimeout(retryDelay); } else { this.logger.error( `Max retries reached for the insertion of the Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` + `to the Triple Store's ${repository} repository. Rolling back data.`, ); this.logger.info( `Rolling back Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` + `from the Triple Store's ${repository} repository Named Graphs.`, ); await Promise.all([ this.tripleStoreModuleManager.deleteKnowledgeCollectionNamedGraphs( this.repositoryImplementations[repository], repository, allPossibleNamedGraphs, ), this.tripleStoreModuleManager.deleteKnowledgeCollectionMetadata( this.repositoryImplementations[repository], repository, allPossibleNamedGraphs, ), ]); throw new Error( `Failed to store Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` + `to the Triple Store's ${repository} repository after maximum retries. Error ${error}`, ); } } } return totalNumberOfTriplesInserted; } async insertKnowledgeCollectionBatch(repository, KCs) { // this.logger.info( // `Inserting Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` + // `to the Triple Store's ${repository} repository.`, // ); // This metadata is not validated const { remote, metadata } = KCs; const insert = {}; const createdMetadata = []; const currentNamedGraphTriples = []; // remote { ual: { public: [triples], private: [triples] } } for (const ual of Object.keys(remote)) { const triples = remote[ual].public; const filteredPublic = []; const privateHashTriples = []; triples.forEach((triple) => { if (triple.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { privateHashTriples.push(triple); } else { filteredPublic.push(triple); } }); const publicKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( filteredPublic, true, ); publicKnowledgeAssetsTriplesGrouped.push( ...kcTools.groupNquadsBySubject(privateHashTriples, true), ); const publicKnowledgeAssetsUALs = publicKnowledgeAssetsTriplesGrouped.map( (_, index) => `${ual}/${index + 1}`, ); for (const [index, kaUAL] of publicKnowledgeAssetsUALs.entries()) { insert[`${kaUAL}/public`] = publicKnowledgeAssetsTriplesGrouped[index]; createdMetadata.push(`<${kaUAL}> "${kaUAL}:0" .`); currentNamedGraphTriples.push( ` <${kaUAL}/public> .`, ); createdMetadata.push( `<${ual}> <${kaUAL}> .`, ); } } await this.tripleStoreModuleManager.insertAssertionBatch( TRIPLE_STORE_REPOSITORY.DKG, repository, insert, metadata, createdMetadata, currentNamedGraphTriples, this.config.modules.tripleStore.timeout.insert, ); } async deletePublishTimestampMetadata(repository, ual) { await this.tripleStoreModuleManager.deletePublishTimestampMetadata( this.repositoryImplementations[repository], repository, ual, ); } async checkIfKnowledgeCollectionExistsInUnifiedGraph( ual, repository = TRIPLE_STORE_REPOSITORY.DKG, ) { const knowledgeCollectionExists = await this.tripleStoreModuleManager.knowledgeCollectionExistsInUnifiedGraph( this.repositoryImplementations[repository], repository, BASE_NAMED_GRAPHS.UNIFIED, ual, ); return knowledgeCollectionExists; } async getAssertion( blockchain, contract, knowledgeCollectionId, knowledgeAssetId, tokenIds, migrationFlag, visibility = TRIPLES_VISIBILITY.PUBLIC, repository = TRIPLE_STORE_REPOSITORY.DKG, operationId = undefined, ) { // TODO: Use stateId let ual = `did:dkg:${blockchain}/${contract}/${knowledgeCollectionId}`; // Performance instrumentation (enable only if operationId is supplied) const logTime = operationId !== undefined; const startTimer = (label) => { if (logTime) this.logger.startTimer(label); }; const endTimer = (label) => { if (logTime) this.logger.endTimer(label); }; const totalLabel = `[TripleStoreService.getAssertion TOTAL] ${operationId} ${ual}`; startTimer(totalLabel); let nquads = {}; if (typeof knowledgeAssetId === 'number') { ual = `${ual}/${knowledgeAssetId}`; const singleLabel = `[TripleStoreService.getAssertion SINGLE] ${operationId} ${ual}`; startTimer(singleLabel); this.logger.debug(`Getting Assertion with the UAL: ${ual}.`); nquads = await this.tripleStoreModuleManager.getKnowledgeAssetNamedGraph( this.repositoryImplementations[repository], repository, // TODO: Add state with implemented update `${ual}`, visibility, this.config.modules.tripleStore.timeout.get, ); endTimer(singleLabel); } else { this.logger.debug(`Getting Assertion with the UAL: ${ual}.`); const existsLabel = `[TripleStoreService.getAssertion EXISTS_CHECK] ${operationId} ${ual}`; startTimer(existsLabel); // first check if the knowledge collection exists in triple store using ASK const firstKAInCollection = `${ual}/${tokenIds.startTokenId}/${TRIPLES_VISIBILITY.PUBLIC}`; const lastKAInCollection = `${ual}/${tokenIds.endTokenId}/${TRIPLES_VISIBILITY.PUBLIC}`; const firstKAExists = this.tripleStoreModuleManager.checkIfKnowledgeAssetExists( this.repositoryImplementations[repository], repository, firstKAInCollection, this.config.modules.tripleStore.timeout.ask, ); const lastKAExists = this.tripleStoreModuleManager.checkIfKnowledgeAssetExists( this.repositoryImplementations[repository], repository, lastKAInCollection, this.config.modules.tripleStore.timeout.ask, ); const [firstKAResult, lastKAResult] = await Promise.all([firstKAExists, lastKAExists]); endTimer(existsLabel); if (!(firstKAResult && lastKAResult)) { this.logger.warn( `Knowledge Collection with the UAL: ${ual} does not exist in the Triple Store's ${repository} repository.`, ); endTimer(totalLabel); return { public: [], private: [] }; } // tokenIds are used to construct named graphs // do pagination through tokenIds const collectionLabel = `[TripleStoreService.getAssertion COLLECTION] ${operationId} ${ual}`; startTimer(collectionLabel); if (visibility === TRIPLES_VISIBILITY.PUBLIC || visibility === TRIPLES_VISIBILITY.ALL) { nquads.public = []; } if ( visibility === TRIPLES_VISIBILITY.PRIVATE || visibility === TRIPLES_VISIBILITY.ALL ) { nquads.private = []; } const maxTokenId = tokenIds.endTokenId; for (let i = 0; i <= tokenIds.endTokenId; i += MAX_TOKEN_ID_PER_GET_PAGE) { const paginationNquads = await this.tripleStoreModuleManager.getKnowledgeCollectionNamedGraphsOld( this.repositoryImplementations[repository], repository, ual, { startTokenId: i + 1, endTokenId: Math.min(i + MAX_TOKEN_ID_PER_GET_PAGE, maxTokenId), burned: tokenIds.burned, }, visibility, this.config.modules.tripleStore.timeout.get, ); if (paginationNquads?.public) { nquads.public.push( ...paginationNquads.public.split('\n').filter((line) => line !== ''), ); } if (paginationNquads?.private) { nquads.private.push( ...paginationNquads.private.split('\n').filter((line) => line !== ''), ); } } endTimer(collectionLabel); } const numberOfnquads = (nquads?.public?.length ?? 0) + (nquads?.private?.length ?? 0); this.logger.debug( `Assertion: ${ual} ${ numberOfnquads ? '' : 'is not' } found in the Triple Store's ${repository} repository.`, ); if (nquads.length) { this.logger.debug( `Number of n-quads retrieved from the Triple Store's ${repository} repository: ${numberOfnquads}.`, ); } endTimer(totalLabel); return nquads; } async getAssertionsInBatch( repository, uals, ualTokenIds, visibility = 'public', operationId = undefined, ) { // Conditional performance logging const logTime = operationId !== undefined; const startTimer = (label) => { if (logTime) this.logger.startTimer(label); }; const endTimer = (label) => { if (logTime) this.logger.endTimer(label); }; const totalLabel = `[TripleStoreService.getAssertionsInBatch TOTAL] ${operationId} ${uals.length}`; startTimer(totalLabel); const results = await Promise.all( uals.map(async (ual) => { const { blockchain, contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); return this.getAssertion( blockchain, contract, knowledgeCollectionId, null, ualTokenIds[ual], false, visibility, repository, operationId, ); }), ); const result = {}; for (const [index, ual] of uals.entries()) { result[ual] = results[index]; } endTimer(totalLabel); return result; } async getV6Assertion(repository, assertionId) { this.logger.debug( `Getting Assertion with the ID: ${assertionId} from the Triple Store's ${repository} repository.`, ); const nquads = await this.tripleStoreModuleManager.getV6Assertion( this.repositoryImplementations[repository], repository, assertionId, ); this.logger.debug( `Assertion: ${assertionId} ${ nquads.length ? '' : 'is not' } found in the Triple Store's ${repository} repository.`, ); if (nquads.length) { this.logger.debug( `Number of n-quads retrieved from the Triple Store's ${repository} repository: ${nquads.length}.`, ); } return nquads; } async checkIfKnowledgeAssetExists(repository, kaUAL) { const knowledgeAssetExists = await this.tripleStoreModuleManager.checkIfKnowledgeAssetExists( this.repositoryImplementations[repository], repository, kaUAL, ); return knowledgeAssetExists; } async getAssertionMetadata( blockchain, contract, knowledgeCollectionId, repository = TRIPLE_STORE_REPOSITORY.DKG, ) { const ual = `did:dkg:${blockchain}/${contract}/${knowledgeCollectionId}`; this.logger.debug(`Getting Assertion Metadata with the UAL: ${ual}.`); let nquads = await this.tripleStoreModuleManager.getKnowledgeCollectionMetadata( this.repositoryImplementations[repository], repository, ual, this.config.modules.tripleStore.timeout.get, ); nquads = nquads.split('\n').filter((line) => line !== ''); this.logger.debug( `Knowledge Asset Metadata: ${ual} ${ nquads.length ? '' : 'is not' } found in the Triple Store's ${repository} repository.`, ); if (nquads.length) { this.logger.debug( `Number of n-quads retrieved from the Triple Store's ${repository} repository: ${nquads.length}.`, ); } return nquads; } async getAssertionMetadataBatch(uals) { const metadataTriples = await this.tripleStoreModuleManager.getMetadataInBatch( this.repositoryImplementations[TRIPLE_STORE_REPOSITORY.DKG], TRIPLE_STORE_REPOSITORY.DKG, uals, ); const metadata = {}; for (const line of metadataTriples.split('\n').filter((result) => result !== '')) { const splitLine = line.split(' '); const ual = splitLine[0].replace(/[<>]/g, ''); if (!metadata[ual]) { metadata[ual] = [line]; } else { metadata[ual].push(line); } } return metadata; } async getLatestAssertionId(repository, ual) { const nquads = await this.tripleStoreModuleManager.getLatestAssertionId( this.repositoryImplementations[repository], repository, ual, ); return nquads; } async construct(query, repository = TRIPLE_STORE_REPOSITORY.DKG, timeout = 60000) { return this.tripleStoreModuleManager.construct( this.repositoryImplementations[repository] ?? this.repositoryImplementations[TRIPLE_STORE_REPOSITORY.DKG], repository, query, timeout, ); } async getKnowledgeAssetNamedGraph(repository, ual, visibility, timeout) { return this.tripleStoreModuleManager.getKnowledgeAssetNamedGraph( this.repositoryImplementations[repository], repository, ual, visibility, timeout, ); } async select(query, repository = TRIPLE_STORE_REPOSITORY.DKG, timeout = 60000) { return this.tripleStoreModuleManager.select( this.repositoryImplementations[repository] ?? this.repositoryImplementations[TRIPLE_STORE_REPOSITORY.DKG], repository, query, timeout, ); } async ask(query, repository = TRIPLE_STORE_REPOSITORY.DKG) { return this.tripleStoreModuleManager.ask( this.repositoryImplementations[repository] ?? this.repositoryImplementations[TRIPLE_STORE_REPOSITORY.DKG], repository, query, ); } getRepositorySparqlEndpoint(repository) { const implementationName = this.repositoryImplementations[repository]; const endpoint = this.tripleStoreModuleManager.getImplementation(implementationName).module.repositories[ repository ].sparqlEndpoint; return endpoint; } } export default TripleStoreService; ================================================ FILE: src/service/ual-service.js ================================================ class UALService { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; this.blockchainModuleManager = ctx.blockchainModuleManager; this.cryptoService = ctx.cryptoService; } deriveUAL(blockchain, contract, knowledgeCollectionId, knowledgeAssetId) { const ual = `did:dkg:${blockchain.toLowerCase()}/${contract.toLowerCase()}/${knowledgeCollectionId}`; return knowledgeAssetId ? `${ual}/${knowledgeAssetId}` : ual; } // did:dkg:otp:2043/0x123231/5 isUAL(ual) { if (!ual.startsWith('did:dkg:')) return false; const parts = ual.replace('did:', '').replace('dkg:', '').split('/'); parts.push(...parts.pop().split(':')); if (parts.length === 4) { return ( this.isContract(parts[1]) && !Number.isNaN(Number(parts[2])) && !Number.isNaN(Number(parts[3])) ); } if (parts.length === 3) { // eslint-disable-next-line no-restricted-globals return this.isContract(parts[1]) && !Number.isNaN(Number(parts[2])); } if (parts.length === 2) { const parts2 = parts[0].split(':'); // eslint-disable-next-line no-restricted-globals if (parts2.length === 3) { return ( parts2.length === 2 && this.isContract(parts2[2]) && !Number.isNaN(Number(parts[1])) ); } return ( parts2.length === 2 && this.isContract(parts2[1]) && !Number.isNaN(Number(parts[1])) ); } } resolveUAL(ual) { const parts = ual.replace('did:', '').replace('dkg:', '').split('/'); // TODO: Resolve UAL with state // parts.push(...parts.pop().split(':')); if (parts.length === 4) { const contract = parts[1]; if (!this.isContract(contract)) { throw new Error(`Invalid contract format: ${contract}`); } let blockchainName = parts[0]; if (blockchainName.split(':').length === 1) { for (const implementation of this.blockchainModuleManager.getImplementationNames()) { if (implementation.split(':')[0] === blockchainName) { blockchainName = implementation; break; } } } return { blockchain: blockchainName, contract, knowledgeCollectionId: Number(parts[2]), knowledgeAssetId: Number(parts[3]), }; } if (parts.length === 3) { const contract = parts[1]; if (!this.isContract(contract)) { throw new Error(`Invalid contract format: ${contract}`); } let blockchainName = parts[0]; if (blockchainName.split(':').length === 1) { for (const implementation of this.blockchainModuleManager.getImplementationNames()) { if (implementation.split(':')[0] === blockchainName) { blockchainName = implementation; break; } } } return { blockchain: blockchainName, contract, knowledgeCollectionId: Number(parts[2]), }; } if (parts.length === 2) { const parts2 = parts[0].split(':'); if (parts2.length === 3) { const contract = parts2[2]; if (!this.isContract(contract)) { throw new Error(`Invalid contract format: ${contract}`); } return { blockchain: parts2[0] + parts2[1], contract, knowledgeCollectionId: Number(parts[1]), }; } if (parts2.length === 2) { let blockchainWithId; for (const implementation of this.blockchainModuleManager.getImplementationNames()) { if (implementation.split(':')[0] === blockchainWithId) { blockchainWithId = implementation; break; } } const contract = parts2[1]; if (!this.isContract(contract)) { throw new Error(`Invalid contract format: ${contract}`); } return { blockchain: blockchainWithId, contract, knowledgeCollectionId: Number(parts[1]), }; } } throw new Error(`UAL doesn't have correct format: ${ual}`); } isContract(contract) { const contractRegex = /^0x[a-fA-F0-9]{40}$/; return contractRegex.test(contract); } // TODO: Do we need still need this async calculateLocationKeyword( blockchain, contract, knowledgeCollectionId, assertionId = null, ) { const firstAssertionId = assertionId ?? (await this.blockchainModuleManager.getKnowledgeCollectionMerkleRootByIndex( blockchain, contract, knowledgeCollectionId, 0, )); return this.cryptoService.encodePacked( ['address', 'bytes32'], [contract, firstAssertionId], ); } getUalWithoutChainId(ual, blockchain) { const blockchainParts = blockchain.split(':'); if (ual.includes(blockchain)) { return ual.replace(blockchain, blockchainParts[0]); } return ual; } } export default UALService; ================================================ FILE: src/service/update-service.js ================================================ import OperationService from './operation-service.js'; import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, ERROR_TYPE, OPERATIONS, } from '../constants/constants.js'; class UpdateService extends OperationService { constructor(ctx) { super(ctx); this.repositoryModuleManager = ctx.repositoryModuleManager; this.operationName = OPERATIONS.UPDATE; this.networkProtocols = NETWORK_PROTOCOLS.UPDATE; this.errorType = ERROR_TYPE.UPDATE.UPDATE_ERROR; this.completedStatuses = [ OPERATION_ID_STATUS.UPDATE.UPDATE_REPLICATE_END, OPERATION_ID_STATUS.UPDATE.UPDATE_END, OPERATION_ID_STATUS.COMPLETED, ]; } async processResponse(command, responseStatus, responseData, errorMessage = null) { const { operationId, blockchain, numberOfFoundNodes, leftoverNodes, batchSize, minAckResponses, datasetRoot, } = command.data; const datasetRootStatus = await this.getResponsesStatuses( responseStatus, errorMessage, operationId, ); const { completedNumber, failedNumber } = datasetRootStatus[operationId]; const totalResponses = completedNumber + failedNumber; this.logger.debug( `Processing ${ this.operationName } response with status: ${responseStatus} for operationId: ${operationId}. Total number of nodes: ${numberOfFoundNodes}, number of nodes in batch: ${Math.min( numberOfFoundNodes, batchSize, )} number of leftover nodes: ${ leftoverNodes.length }, number of responses: ${totalResponses}, Completed: ${completedNumber}, Failed: ${failedNumber}, minimum replication factor: ${minAckResponses}`, ); if (responseData.errorMessage) { this.logger.trace( `Error message for operation id: ${operationId} : ${responseData.errorMessage}`, ); } // Minimum replication reached, mark in the operational DB if (completedNumber === minAckResponses) { this.logger.debug( `Minimum replication ${minAckResponses} reached for operationId: ${operationId}, dataset root: ${datasetRoot}`, ); await this.repositoryModuleManager.updateMinAcksReached(operationId, true); } // All requests sent, minimum replication reached, mark as completed if (leftoverNodes.length === 0 && completedNumber >= minAckResponses) { await this.markOperationAsCompleted( operationId, blockchain, null, this.completedStatuses, ); this.logResponsesSummary(completedNumber, failedNumber); } // All requests sent, minimum replication not reached, mark as failed if (leftoverNodes.length === 0 && completedNumber < minAckResponses) { this.markOperationAsFailed( operationId, blockchain, 'Not replicated to enough nodes!', this.errorType, ); this.logResponsesSummary(completedNumber, failedNumber); } // Not all requests sent, still possible to reach minimum replication, // schedule requests for leftover nodes const potentialCompletedNumber = completedNumber + leftoverNodes.length; if (leftoverNodes.length > 0 && potentialCompletedNumber >= minAckResponses) { await this.scheduleOperationForLeftoverNodes(command.data, leftoverNodes); } } } export default UpdateService; ================================================ FILE: src/service/util/jwt-util.js ================================================ import 'dotenv/config'; import jwt from 'jsonwebtoken'; import { validate } from 'uuid'; class JwtUtil { constructor() { this._secret = process.env.JWT_SECRET; } /** * Generates new JWT token * @param uuid uuid from token table * @param expiresIn optional parameter. accepts values for ms package (https://www.npmjs.com/package/ms) * @returns {string|null} */ generateJWT(uuid, expiresIn = null) { if (!validate(uuid)) { return null; } const options = { jwtid: uuid, }; if (expiresIn) { options.expiresIn = expiresIn; } return jwt.sign({}, this._secret, options); } /** * Validates JWT token * @param {string} token * @returns {boolean} */ validateJWT(token) { try { jwt.verify(token, this._secret); } catch (e) { return false; } return true; } /** * Returns JWT payload * @param {string} token * @returns {*} */ getPayload(token) { return jwt.decode(token); } /** * Decodes token * @param token * @returns {{payload: any, signature: *, header: *}|*} */ decode(token) { return jwt.decode(token, { complete: true }); } } const jwtUtil = new JwtUtil(); export default jwtUtil; ================================================ FILE: src/service/util/string-util.js ================================================ class StringUtil { toCamelCase(str) { return str.replace(/[-_]+(.)/g, (_, group) => group.toUpperCase()); } capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); } } const stringUtil = new StringUtil(); export default stringUtil; ================================================ FILE: src/service/validation-service.js ================================================ import { kcTools } from 'assertion-tools'; import { ZERO_ADDRESS, PRIVATE_ASSERTION_PREDICATE, ZERO_BYTES32, PRIVATE_HASH_SUBJECT_PREFIX, } from '../constants/constants.js'; class ValidationService { constructor(ctx) { this.logger = ctx.logger; this.config = ctx.config; this.validationModuleManager = ctx.validationModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; } async validateUal(blockchain, contract, tokenId) { this.logger.info( `Validating UAL: did:dkg:${blockchain.toLowerCase()}/${contract.toLowerCase()}/${tokenId}`, ); let isValid = true; try { const result = await this.blockchainModuleManager.getLatestMerkleRootPublisher( blockchain, contract, tokenId, ); if (!result || result === ZERO_ADDRESS) { isValid = false; } } catch (err) { isValid = false; } return isValid; } async validateUalV6(blockchain, contract, tokenId) { this.logger.info( `Validating UAL: did:dkg:${blockchain.toLowerCase()}/${contract.toLowerCase()}/${tokenId}`, ); let isValid = true; try { const result = await this.blockchainModuleManager.getLatestAssertionId( blockchain, contract, tokenId, ); if (!result || result === ZERO_BYTES32) { isValid = false; } } catch (err) { isValid = false; } return isValid; } async validateAssertion(assertionId, blockchain, assertion) { this.logger.info(`Validating assertionId: ${assertionId}`); await this.validateDatasetRoot(assertion, assertionId); this.logger.info(`Assertion integrity validated! AssertionId: ${assertionId}`); } async validateDatasetRootOnBlockchain( knowledgeCollectionMerkleRoot, blockchain, assetStorageContractAddress, knowledgeCollectionId, ) { const blockchainAssertionRoot = await this.blockchainModuleManager.getKnowledgeCollectionLatestMerkleRoot( blockchain, assetStorageContractAddress, knowledgeCollectionId, ); if (knowledgeCollectionMerkleRoot !== blockchainAssertionRoot) { throw new Error( `Merkle Root validation failed. Merkle Root on chain: ${blockchainAssertionRoot}; Calculated Merkle Root: ${knowledgeCollectionMerkleRoot}`, ); } } // Used to validate assertion node received through network get async validateDatasetOnBlockchain( assertion, blockchain, assetStorageContractAddress, knowledgeCollectionId, ) { const knowledgeCollectionMerkleRoot = await this.validationModuleManager.calculateRoot( assertion, ); await this.validateDatasetRootOnBlockchain( knowledgeCollectionMerkleRoot, blockchain, assetStorageContractAddress, knowledgeCollectionId, ); } async validateDatasetRoot(dataset, datasetRoot) { const calculatedDatasetRoot = await this.validationModuleManager.calculateRoot(dataset); if (datasetRoot !== calculatedDatasetRoot) { throw new Error( `Merkle Root validation failed. Received Merkle Root: ${datasetRoot}; Calculated Merkle Root: ${calculatedDatasetRoot}`, ); } } async validatePrivateMerkleRoot(publicAssertion, privateAssertion) { const privateAssertionTriple = publicAssertion.find((triple) => triple.includes(PRIVATE_ASSERTION_PREDICATE), ); if (privateAssertionTriple) { const privateAssertionRoot = privateAssertionTriple.split(' ')[2].replace(/['"]/g, ''); // Is this cause of the problem, maybe do it in same was as on client const privateAssertionSorted = privateAssertion.sort(); await this.validateDatasetRoot(privateAssertionSorted, privateAssertionRoot); } } async validateGetResponse( assertion, blockchain, contract, knowledgeCollectionId, knowledgeAssetId, ) { if (assertion.public) { // We can only validate whole collection not particular KA if (knowledgeAssetId) { const publicAssertion = assertion?.public; const filteredPublic = []; const privateHashTriples = []; publicAssertion.forEach((triple) => { if (triple.startsWith(`<${PRIVATE_HASH_SUBJECT_PREFIX}`)) { privateHashTriples.push(triple); } else { filteredPublic.push(triple); } }); const publicKnowledgeAssetsTriplesGrouped = kcTools.groupNquadsBySubject( filteredPublic, true, ); publicKnowledgeAssetsTriplesGrouped.push( ...kcTools.groupNquadsBySubject(privateHashTriples, true), ); try { await this.validateDatasetOnBlockchain( publicKnowledgeAssetsTriplesGrouped.map((t) => t.sort()).flat(), blockchain, contract, knowledgeCollectionId, ); if (assertion?.private?.length) await this.validatePrivateMerkleRoot(assertion.public, assertion.private); } catch (e) { return false; } } return true; } return false; } } export default ValidationService; ================================================ FILE: test/assertions/assertions.js ================================================ const context = { xsd: 'http://www.w3.org/2001/XMLSchema#', testProperty: 'http://example.com' }; function createTestGraph(id, type, values) { return { '@context': context, '@graph': values.map((value, i) => ({ '@id': id + i, testProperty: { '@type': type, '@value': value }, })), }; } // XSD:DECIMAL let id = 'test:decimal'; let type = 'xsd:decimal'; let values = [100, 100.0, '100', '100.0']; const decimal = createTestGraph(id, type, values); // XSD:DATETIME id = 'test:dateTime'; type = 'xsd:dateTime'; values = [ "2022-08-20'T'13:20:10*633+0000", '2022 Mar 03 05:12:41.211 PDT', 'Jan 21 18:20:11 +0000 2022', '19/Apr/2022:06:36:15 -0700', 'Dec 2, 2022 2:39:58 AM', 'Jun 09 2022 15:28:14', 'Apr 20 00:00:35 2010', 'Sep 28 19:00:00 +0000', 'Mar 16 08:12:04', '2022-10-14T22:11:20+0000', "2022-07-01T14:59:55.711'+0000'", '2022-07-01T14:59:55.711Z', '2022-08-19 12:17:55 -0400', '2022-08-19 12:17:55-0400', '2022-06-26 02:31:29,573', '2022/04/12*19:37:50', '2022 Apr 13 22:08:13.211*PDT', '2022 Mar 10 01:44:20.392', '2022-03-10 14:30:12,655+0000', '2022-02-27 15:35:20.311', '2022-03-12 13:11:34.222-0700', "2022-07-22'T'16:28:55.444", "2022-09-08'T'03:13:10", "2022-03-12'T'17:56:22'-0700'", "2022-11-22'T'10:10:15.455", "2022-02-11'T'18:31:44", '2022-10-30*02:47:33:899', '2022-07-04*13:23:55', '22-02-11 16:47:35,985 +0000', '22-06-26 02:31:29,573', '22-04-19 12:00:17', '06/01/22 04:11:05', '220423 11:42:35', '20220423 11:42:35.173', '08/10/22*13:33:56', '11/22/2022*05:13:11', '05/09/2022*08:22:14*612', '04/23/22 04:34:22 +0000', '10/03/2022 07:29:46 -0700', '11:42:35', '11:42:35.173', '11:42:35,173', '23/Apr 11:42:35,173', '23/Apr/2022:11:42:35', '23/Apr/2022 11:42:35', '23-Apr-2022 11:42:35', '23-Apr-2022 11:42:35.883', '23 Apr 2022 11:42:35', '23 Apr 2022 10:32:35*311', '0423_11:42:35', '0423_11:42:35.883', '8/5/2022 3:31:18 AM:234', '9/28/2022 2:23:15 PM', ]; const dateTime = createTestGraph(id, type, values); export default { decimal, dateTime, }; ================================================ FILE: test/bdd/features/bid-suggestion.feature ================================================ @ignore Feature: Bid suggestion tests # @ignore: dkg.js SDK removed network.getBidSuggestion() and assertion.getSizeInBytes() # in v8. Re-enable once the SDK exposes a bid-suggestion API again. Background: Setup local blockchain, bootstraps and nodes Given the blockchains are set up And 1 bootstrap is running @bid-suggestion Scenario: Get bid suggestion with a valid assertion And I setup 2 additional nodes And I wait for 15 seconds When I call Get Bid Suggestion on the node 1 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 Then I call Info route on the node 1 ================================================ FILE: test/bdd/features/get-errors.feature ================================================ Feature: Get errors test Background: Setup local blockchain, bootstraps and nodes Given the blockchains are set up And 1 bootstrap is running @ignore Scenario: Getting non-existent UAL # @ignore: A validly-formatted but non-existent UAL causes the node's get # operation to stay IN_PROGRESS indefinitely while it searches the network. # The operation never reaches a terminal status, so polling times out. And I setup 1 additional node And I wait for 15 seconds When I call Get directly on the node 1 with nonExistentUAL on blockchain hardhat1:31337 And I wait for latest Get to finalize Then Latest Get operation finished with status: FAILED @get-error Scenario: Getting invalid UAL And I setup 1 additional node And I wait for 15 seconds When I call Get directly on the node 1 with invalidUAL on blockchain hardhat1:31337 And I wait for latest Get to finalize Then Latest Get operation finished with status: GetRouteError ================================================ FILE: test/bdd/features/publish-errors.feature ================================================ Feature: Publish errors test Background: Setup local blockchain, bootstraps and nodes Given the blockchains are set up And 1 bootstrap is running @publish-error Scenario: Publish a knowledge asset directly on the node with invalid request And I setup 1 additional node And I wait for 15 seconds When I call Publish directly on the node 1 with validPublishRequestBody And I wait for latest Publish to finalize Then Latest Publish operation finished with status: HTTP_404 ================================================ FILE: test/bdd/features/publish.feature ================================================ Feature: Publish related tests Background: Setup local blockchain, bootstraps and nodes Given the blockchains are set up And 1 bootstrap is running @smoke @publish Scenario: Publishing a valid assertion And I setup 1 additional node And I wait for nodes to sync and mark active When I call Publish on the node 1 with validAssertion on blockchain hardhat1:31337 And I wait for latest Publish to finalize Then Latest Publish operation finished with status: COMPLETED @publish @get Scenario: Publish and retrieve a knowledge asset And I setup 1 additional node And I wait for nodes to sync and mark active When I call Publish on the node 1 with validAssertion on blockchain hardhat1:31337 And I wait for latest Publish to finalize Then Latest Publish operation finished with status: COMPLETED And I wait for 10 seconds When I get operation result from node 1 for latest published assertion And I wait for latest resolve to finalize Then Latest Get operation finished with status: COMPLETED ================================================ FILE: test/bdd/features/smoke.feature ================================================ Feature: Smoke tests — node health and basic operation Background: Setup local blockchain, bootstraps and nodes Given the blockchains are set up And 1 bootstrap is running @smoke Scenario: Nodes start up and respond to the info route And I setup 2 additional nodes And I wait for 5 seconds Then Node 1 responds to info route And Node 2 responds to info route ================================================ FILE: test/bdd/features/update-errors.feature ================================================ Feature: Update errors test Background: Setup local blockchain, bootstraps and nodes Given the blockchains are set up And 1 bootstrap is running @update-error Scenario: Update knowledge asset that was not previously published And I setup 1 additional node And I wait for 15 seconds When I call Update directly on the node 1 with validUpdateRequestBody And I wait for latest Update to finalize Then Latest Update operation finished with status: HTTP_404 ================================================ FILE: test/bdd/run-bdd.sh ================================================ #!/usr/bin/env bash set -euo pipefail BLAZEGRAPH_JAR="${BLAZEGRAPH_JAR:-$HOME/blazegraph/blazegraph.jar}" BLAZEGRAPH_PORT=9999 BLAZEGRAPH_PID="" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" cleanup() { echo "" echo "Cleaning up..." if [[ -n "$BLAZEGRAPH_PID" ]] && kill -0 "$BLAZEGRAPH_PID" 2>/dev/null; then echo " Stopping Blazegraph (PID $BLAZEGRAPH_PID)" kill "$BLAZEGRAPH_PID" 2>/dev/null || true wait "$BLAZEGRAPH_PID" 2>/dev/null || true fi echo "Cleanup complete" } trap cleanup EXIT # -- Preflight checks --------------------------------------------------------- echo "Checking prerequisites..." if ! command -v java &>/dev/null; then echo "ERROR: Java not found. Install a JDK (e.g. brew install --cask zulu17)." exit 1 fi if [[ ! -f "$BLAZEGRAPH_JAR" ]]; then echo "ERROR: Blazegraph jar not found at $BLAZEGRAPH_JAR" echo " Set BLAZEGRAPH_JAR env var to the correct path." exit 1 fi if ! mysql -u root -e "SELECT 1" &>/dev/null; then echo "ERROR: MySQL is not running or root access failed." echo " Start it with: brew services start mysql@8.0" exit 1 fi echo " OK: MySQL is reachable" if ! redis-cli ping &>/dev/null; then echo "ERROR: Redis is not running." echo " Start it with: brew services start redis" exit 1 fi echo " OK: Redis is reachable" # -- Start Blazegraph --------------------------------------------------------- BLAZEGRAPH_ALREADY_RUNNING=false if curl -sf "http://localhost:${BLAZEGRAPH_PORT}/blazegraph/status" &>/dev/null; then echo " OK: Blazegraph already running on port ${BLAZEGRAPH_PORT}" BLAZEGRAPH_ALREADY_RUNNING=true else echo " Starting Blazegraph on port ${BLAZEGRAPH_PORT}..." BLAZEGRAPH_DATA_DIR="/tmp/blazegraph-bdd-data" mkdir -p "$BLAZEGRAPH_DATA_DIR" cd "$BLAZEGRAPH_DATA_DIR" java -server -Xmx4g "-Djetty.port=${BLAZEGRAPH_PORT}" \ -jar "$BLAZEGRAPH_JAR" &>/tmp/blazegraph-bdd.log & BLAZEGRAPH_PID=$! cd "$PROJECT_ROOT" for i in $(seq 1 30); do if curl -sf "http://localhost:${BLAZEGRAPH_PORT}/blazegraph/status" &>/dev/null; then echo " OK: Blazegraph started (PID $BLAZEGRAPH_PID)" break fi if ! kill -0 "$BLAZEGRAPH_PID" 2>/dev/null; then echo "ERROR: Blazegraph process died. Check /tmp/blazegraph-bdd.log" exit 1 fi sleep 2 done if ! curl -sf "http://localhost:${BLAZEGRAPH_PORT}/blazegraph/status" &>/dev/null; then echo "ERROR: Blazegraph did not start within 60 seconds." exit 1 fi fi # -- Run BDD tests ------------------------------------------------------------ echo "" echo "Running BDD tests..." echo "" cd "$PROJECT_ROOT" TEST_EXIT=0 REPOSITORY_PASSWORD="${REPOSITORY_PASSWORD:-}" npx cucumber-js \ --config cucumber.js \ --tags "not @ignore" \ --format progress \ --format-options '{"colorsEnabled": true}' \ test/bdd/ \ --import test/bdd/steps/ \ --exit \ "$@" || TEST_EXIT=$? echo "" if [[ $TEST_EXIT -eq 0 ]]; then echo "All BDD tests passed!" else echo "Some BDD tests failed (exit code $TEST_EXIT)" fi # If we started Blazegraph, stop it (handled by trap). If it was already # running, leave BLAZEGRAPH_PID empty so the trap doesn't kill it. if $BLAZEGRAPH_ALREADY_RUNNING; then BLAZEGRAPH_PID="" fi exit $TEST_EXIT ================================================ FILE: test/bdd/steps/api/bid-suggestion.mjs ================================================ import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { readFile } from 'fs/promises'; const assertions = JSON.parse(await readFile('test/bdd/steps/api/datasets/assertions.json')); When( /^I call Get Bid Suggestion on node (\d+) using parameters ([^"]*), hashFunctionId (\d+), scoreFunctionId (\d+), within blockchain ([^"]*)/, { timeout: 300000 }, async function getBidSuggestionWithHashAndScore( node, assertionName, hashFunctionId, scoreFunctionId, blockchain, ) { this.logger.log( `I call get bid suggestion route on the node ${node} on blockchain ${blockchain} with hashFunctionId ${hashFunctionId} and scoreFunctionId ${scoreFunctionId}`, ); expect( !!this.state.localBlockchains[blockchain], `Blockchain with name ${blockchain} not found`, ).to.be.equal(true); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, ).to.be.equal(true); expect( Number.isInteger(hashFunctionId), `hashFunctionId value: ${hashFunctionId} is not an integer!`, ).to.be.equal(true); expect( Number.isInteger(scoreFunctionId), `scoreFunctionId value: ${scoreFunctionId} is not an integer!`, ).to.be.equal(true); const assertion = assertions[assertionName]; const publicAssertionId = await this.state.nodes[node - 1].client .getPublicAssertionId(assertion) .catch((error) => { assert.fail(`Error while trying to get public assertion id. ${error}`); }); const sizeInBytes = Buffer.byteLength(JSON.stringify(assertion)); const options = { ...this.state.nodes[node - 1].clientBlockchainOptions[blockchain], hashFunctionId: hashFunctionId, scoreFunctionId: scoreFunctionId, }; let getBidSuggestionError; const result = await this.state.nodes[node - 1].client .getBidSuggestion(publicAssertionId, sizeInBytes, options) .catch((error) => { getBidSuggestionError = error; assert.fail(`Error while trying to get bid suggestion. ${error}`); }); this.state.latestBidSuggestionResult = { nodeId: node - 1, publicAssertionId, sizeInBytes, assertion: assertions[assertionName], result, getBidSuggestionError, }; }, ); When( /^I call Get Bid Suggestion on the node (\d+) with ([^"]*) on blockchain ([^"]*)/, { timeout: 300000 }, async function getBidSuggestion(node, assertionName, blockchain) { this.logger.log( `I call get bid suggestion route on the node ${node} on blockchain ${blockchain}`, ); expect( !!this.state.localBlockchains[blockchain], `Blockchain with name ${blockchain} not found`, ).to.be.equal(true); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, ).to.be.equal(true); const assertion = assertions[assertionName]; const publicAssertionId = await this.state.nodes[node - 1].client .getPublicAssertionId(assertion) .catch((error) => { assert.fail(`Error while trying to get public assertion id. ${error}`); }); const sizeInBytes = Buffer.byteLength(JSON.stringify(assertion)); const options = this.state.nodes[node - 1].clientBlockchainOptions[blockchain]; let getBidSuggestionError; const result = await this.state.nodes[node - 1].client .getBidSuggestion(publicAssertionId, sizeInBytes, options) .catch((error) => { getBidSuggestionError = error; assert.fail(`Error while trying to get bid suggestion. ${error}`); }); this.state.latestBidSuggestionResult = { nodeId: node - 1, publicAssertionId, sizeInBytes, assertion: assertions[assertionName], result, getBidSuggestionError, }; }, ); ================================================ FILE: test/bdd/steps/api/get.mjs ================================================ import { Then, When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { readFile } from 'fs/promises'; import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests.json')); const httpApiHelper = new HttpApiHelper(); When( /^I call Get directly on the node (\d+) with ([^"]*) on blockchain ([^"]*)/, { timeout: 30000 }, async function getFromNode(node, requestName, blockchain) { this.logger.log(`I call get directly on the node ${node} on blockchain ${blockchain}`); expect( !!this.state.localBlockchains[blockchain], `Blockchain with name ${blockchain} not found`, ).to.be.equal(true); expect( !!requests[requestName], `Request body with name: ${requestName} not found!`, ).to.be.equal(true); const requestBody = JSON.parse(JSON.stringify(requests[requestName])); requestBody.id = requestBody.id.replace('blockchain', blockchain); try { const result = await httpApiHelper.get( this.state.nodes[node - 1].nodeRpcUrl, requestBody, ); const { operationId } = result.data; this.state.latestGetData = { nodeId: node - 1, operationId, }; } catch (error) { this.state.latestError = error; } }, ); Then(/^It should fail with status code (\d+)/, function checkLatestError(expectedStatusCode) { const expectedStatusCodeInt = parseInt(expectedStatusCode, 10); assert(this.state.latestError, 'No error occurred'); assert(this.state.latestError.statusCode, 'No status code in error'); assert( this.state.latestError.statusCode === expectedStatusCodeInt, `Expected request to fail with status code ${expectedStatusCodeInt}, but it failed with another code.`, ); }); When('I wait for latest Get to finalize', { timeout: 120000 }, async function getFinalize() { this.logger.log('I wait for latest get to finalize'); expect( !!this.state.latestGetData, 'Latest get data is undefined. Get was not started.', ).to.be.equal(true); const { nodeId, operationId } = this.state.latestGetData; this.logger.log(`Polling get result for operation id: ${operationId} on node: ${nodeId}`); const result = await httpApiHelper.pollOperationResult( this.state.nodes[nodeId].nodeRpcUrl, 'get', operationId, { intervalMs: 4000, maxRetries: 25 }, ); this.logger.log(`Get operation status: ${result.data.status}`); this.state.latestGetData.result = result; this.state.latestGetData.status = result.data.status; this.state.latestGetData.errorType = result.data.data?.errorType; }); ================================================ FILE: test/bdd/steps/api/info.mjs ================================================ import { When, Then } from '@cucumber/cucumber'; import assert from 'assert'; When(/^I call Info route on the node (\d+)/, { timeout: 120000 }, async function infoRouteCall(node) { this.logger.log(`I call info route on the node ${node}`); this.state.latestInfoData = await this.state.nodes[node - 1].client.info(); }); Then(/^The node version should start with number (\d+)/, function checkNodeVersion(number) { assert.ok(this.state.latestInfoData, 'No info response recorded — call the info route first'); assert.equal( this.state.latestInfoData.version.startsWith(number), true, `Expected version to start with ${number}, got: ${this.state.latestInfoData.version}`, ); }); ================================================ FILE: test/bdd/steps/api/publish.mjs ================================================ import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { readFile } from 'fs/promises'; import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; const assertions = JSON.parse(await readFile('test/bdd/steps/api/datasets/assertions.json')); const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests.json')); const httpApiHelper = new HttpApiHelper(); When( /^I call Publish on the node (\d+) with ([^"]*) on blockchain ([^"]*)/, { timeout: 120000 }, async function publish(node, assertionName, blockchain) { this.logger.log(`I call publish route on the node ${node} on blockchain ${blockchain}`); expect( !!this.state.localBlockchains[blockchain], `Blockchain with name ${blockchain} not found`, ).to.be.equal(true); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, ).to.be.equal(true); const assertion = assertions[assertionName]; const options = this.state.nodes[node - 1].clientBlockchainOptions[blockchain]; const result = await this.state.nodes[node - 1].client .publish(assertion, options) .catch((error) => { assert.fail(`Error while trying to publish assertion. ${error}`); }); const publishOp = result.operation?.publish ?? {}; this.state.latestPublishData = { nodeId: node - 1, UAL: result.UAL, operationId: publishOp.operationId, assertion: assertions[assertionName], status: publishOp.status || 'PENDING', errorType: publishOp.errorType, result, }; }, ); When( /^I call Publish directly on the node (\d+) with ([^"]*)/, { timeout: 70000 }, async function publishDirect(node, requestName) { this.logger.log(`I call publish on the node ${node} directly`); expect( !!requests[requestName], `Request body with name: ${requestName} not found!`, ).to.be.equal(true); const requestBody = requests[requestName]; try { const result = await httpApiHelper.publish( this.state.nodes[node - 1].nodeRpcUrl, requestBody, ); const { operationId } = result.data; this.state.latestPublishData = { nodeId: node - 1, operationId, }; } catch (error) { this.state.latestPublishData = { nodeId: node - 1, status: 'FAILED', errorType: error.statusCode ? `HTTP_${error.statusCode}` : 'FAILED', }; } }, ); When('I wait for latest Publish to finalize', { timeout: 120000 }, async function publishFinalize() { this.logger.log('I wait for latest publish to finalize'); expect( !!this.state.latestPublishData, 'Latest publish data is undefined. Publish was not started.', ).to.be.equal(true); const { nodeId, operationId, status } = this.state.latestPublishData; if (!operationId) { this.logger.log(`No operationId to poll, using existing status: ${status}`); return; } this.logger.log(`Polling publish result for operation id: ${operationId} on node: ${nodeId}`); const result = await httpApiHelper.pollOperationResult( this.state.nodes[nodeId].nodeRpcUrl, 'publish', operationId, { intervalMs: 5000, maxRetries: 20 }, ); this.logger.log(`Publish operation status: ${result.data.status}`); this.state.latestPublishData.result = result; this.state.latestPublishData.status = result.data.status; this.state.latestPublishData.errorType = result.data.data?.errorType; }); ================================================ FILE: test/bdd/steps/api/resolve.mjs ================================================ import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; const httpApiHelper = new HttpApiHelper(); When( /^I get operation result from node (\d+) for latest published assertion/, { timeout: 120000 }, async function resolveCall(node) { this.logger.log('I call get result for the latest operation'); expect( !!this.state.latestPublishData, 'Latest publish data is undefined. Publish is not finalized.', ).to.be.equal(true); try { const result = await this.state.nodes[node - 1].client .get(this.state.latestPublishData.UAL) .catch((error) => { assert.fail(`Error while trying to resolve assertion. ${error}`); }); const getOp = result.operation?.get ?? result.operation ?? {}; const hasData = !!(result.assertion || result.public || result.data); // The SDK's asset.get() completes the full get flow internally. // If it returned with an errorType, the operation failed. // If it returned assertion data OR has no operationId to poll, // the operation completed successfully inside the SDK. let resolvedStatus = getOp.status || 'PENDING'; if (getOp.errorType) { resolvedStatus = 'FAILED'; } else if (hasData || !getOp.operationId) { resolvedStatus = 'COMPLETED'; } this.state.latestGetData = { nodeId: node - 1, operationId: getOp.operationId, result, status: resolvedStatus, errorType: getOp.errorType, }; } catch (e) { this.logger.log(`Error while getting operation result: ${e}`); this.state.latestGetData = { nodeId: node - 1, status: 'FAILED', }; } }, ); When( 'I wait for latest resolve to finalize', { timeout: 120000 }, async function resolveFinalizeCall() { this.logger.log('I wait for latest resolve to finalize'); expect( !!this.state.latestGetData, 'Latest resolve data is undefined. Resolve is not started.', ).to.be.equal(true); const { nodeId, operationId, status } = this.state.latestGetData; if (!operationId || (status && ['COMPLETED', 'FAILED'].includes(status))) { this.logger.log( `Resolve already finalized (status: ${status}, operationId: ${operationId})`, ); return; } this.logger.log( `Polling resolve result for operation id: ${operationId} on node: ${nodeId}`, ); const result = await httpApiHelper.pollOperationResult( this.state.nodes[nodeId].nodeRpcUrl, 'get', operationId, { intervalMs: 4000, maxRetries: 25 }, ); this.logger.log(`Resolve operation status: ${result.data.status}`); this.state.latestGetData.result = result; this.state.latestGetData.status = result.data.status; this.state.latestGetData.errorType = result.data.data?.errorType; }, ); ================================================ FILE: test/bdd/steps/api/update.mjs ================================================ import { When } from '@cucumber/cucumber'; import { expect } from 'chai'; import { readFile } from 'fs/promises'; import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests.json')); const httpApiHelper = new HttpApiHelper(); When( /^I call Update directly on the node (\d+) with ([^"]*)/, { timeout: 70000 }, async function updateDirect(node, requestName) { this.logger.log(`I call update on the node ${node} directly`); expect( !!requests[requestName], `Request body with name: ${requestName} not found!`, ).to.be.equal(true); const requestBody = requests[requestName]; try { const result = await httpApiHelper.update( this.state.nodes[node - 1].nodeRpcUrl, requestBody, ); const { operationId } = result.data; this.state.latestUpdateData = { nodeId: node - 1, operationId, }; } catch (error) { this.state.latestUpdateData = { nodeId: node - 1, status: 'FAILED', errorType: error.statusCode ? `HTTP_${error.statusCode}` : 'FAILED', }; } }, ); When('I wait for latest Update to finalize', { timeout: 120000 }, async function updateFinalize() { this.logger.log('I wait for latest update to finalize'); expect( !!this.state.latestUpdateData, 'Latest update data is undefined. Update was not started.', ).to.be.equal(true); const { nodeId, operationId, status } = this.state.latestUpdateData; if (!operationId) { this.logger.log(`No operationId to poll, using existing status: ${status}`); return; } this.logger.log(`Polling update result for operation id: ${operationId} on node: ${nodeId}`); const result = await httpApiHelper.pollOperationResult( this.state.nodes[nodeId].nodeRpcUrl, 'update', operationId, { intervalMs: 5000, maxRetries: 20 }, ); this.logger.log(`Update operation status: ${result.data.status}`); this.state.latestUpdateData.result = result; this.state.latestUpdateData.status = result.data.status; this.state.latestUpdateData.errorType = result.data.data?.errorType; }); ================================================ FILE: test/bdd/steps/blockchain.mjs ================================================ import { Given } from '@cucumber/cucumber'; import fs from 'fs'; import LocalBlockchain from './lib/local-blockchain.mjs'; const BLOCKCHAIN_CONFIGS = [ { name: 'hardhat1:31337', port: 8545 }, { name: 'hardhat2:31337', port: 9545 }, ]; Given(/^the blockchains are set up$/, { timeout: 240_000 }, async function blockchainSetup() { await Promise.all( BLOCKCHAIN_CONFIGS.map(({ name, port }) => { this.logger.log(`Starting local blockchain ${name} on port: ${port}`); const blockchainConsole = new console.Console( fs.createWriteStream( `${this.state.scenarioLogDir}/blockchain-${name.replace(':', '-')}.log`, ), ); const localBlockchain = new LocalBlockchain(); this.state.localBlockchains[name] = localBlockchain; return localBlockchain.initialize(port, blockchainConsole); }), ); // The on-chain default minimumRequiredSignatures is 3, which requires 3 nodes in the // shard before a publish can succeed. Lower it to 2 so our small BDD network (1 bootstrap // + 2 regular nodes) can publish without running into "Unable to find enough nodes". // Lower the on-chain minimumRequiredSignatures for our small BDD network. // The ShardingTableCheckCommand syncs the on-chain sharding table into each node's // local DB every 10 seconds, so nodes may not see each other's profiles yet when a // publish arrives. Setting this to 1 ensures the publishing node itself (always in // its own shard) satisfies the requirement. for (const blockchain of Object.values(this.state.localBlockchains)) { await blockchain.setParametersStorageParams({ minimumRequiredSignatures: 1 }); } }); ================================================ FILE: test/bdd/steps/common.mjs ================================================ import { execSync } from 'child_process'; import { Given, Then } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import fs from 'fs'; import path from 'path'; import { setTimeout as sleep } from 'timers/promises'; import mysql from 'mysql2'; import DkgClientHelper from '../../utilities/dkg-client-helper.mjs'; import StepsUtils, { BOOTSTRAP_NETWORK_PORT, BOOTSTRAP_RPC_PORT, } from '../../utilities/steps-utils.mjs'; import FileService from '../../../src/service/file-service.js'; const stepsUtils = new StepsUtils(); Given( /^I setup (\d+)[ additional]* node[s]*$/, { timeout: 60000 }, async function nodeSetup(nodeCount) { this.logger.log(`I setup ${nodeCount} node${nodeCount !== 1 ? 's' : ''}`); const currentNumberOfNodes = Object.keys(this.state.nodes).length; await Promise.all( Array.from({ length: nodeCount }, (_, i) => { const nodeIndex = currentNumberOfNodes + i; // wallets[0] is reserved for the bootstrap node; regular nodes start from index 1 const walletIndex = nodeIndex + 1; const blockchains = Object.entries(this.state.localBlockchains).map( ([blockchainId, blockchain]) => { const wallets = blockchain.getWallets(); return { blockchainId, operationalWallet: wallets[walletIndex], managementWallet: wallets[walletIndex + Math.floor(wallets.length / 2)], port: blockchain.port, }; }, ); const rpcPort = 8901 + nodeIndex; const networkPort = 9001 + nodeIndex; const nodeName = `origintrail-test-${nodeIndex}`; const nodeConfiguration = stepsUtils.createNodeConfiguration( blockchains, nodeIndex, nodeName, rpcPort, networkPort, false, this.state.bootstrapPeerMultiaddr, ); // Remove stale data from any interrupted prior run so the node starts clean fs.rmSync(path.join(process.cwd(), nodeConfiguration.appDataPath), { recursive: true, force: true, }); const forkedNode = stepsUtils.forkNode(nodeConfiguration); // Track immediately so the After hook can kill it even if the step times out // before the process sends STARTED. this.state.pendingProcesses.push(forkedNode); const logFileStream = fs.createWriteStream( `${this.state.scenarioLogDir}/${nodeName}.log`, ); forkedNode.stdout.setEncoding('utf8'); forkedNode.stdout.on('data', (data) => logFileStream.write(data)); forkedNode.stderr.setEncoding('utf8'); forkedNode.stderr.on('data', (data) => logFileStream.write(`[stderr] ${data}`)); return new Promise((resolve, reject) => { let settled = false; const done = (fn, ...args) => { if (!settled) { settled = true; fn(...args); } }; const removePending = () => { const idx = this.state.pendingProcesses.indexOf(forkedNode); if (idx !== -1) this.state.pendingProcesses.splice(idx, 1); }; forkedNode.on('error', (err) => { removePending(); done(reject, err); }); forkedNode.on('exit', (code, signal) => { removePending(); done( reject, new Error( `Node ${nodeIndex} process exited with code=${code} signal=${signal} before sending STARTED`, ), ); }); forkedNode.on('message', (response) => { if (response.error) { // Process reported an error - keep in pendingProcesses for cleanup done( reject, new Error( `Error initializing node ${nodeIndex}: ${response.error}`, ), ); return; } try { const [[firstBlockchainId, firstBlockchain]] = Object.entries( this.state.localBlockchains, ); const firstWallets = firstBlockchain.getWallets(); const client = new DkgClientHelper({ endpoint: 'http://localhost', port: rpcPort, blockchain: { name: firstBlockchainId, publicKey: firstWallets[walletIndex].address, privateKey: firstWallets[walletIndex].privateKey, rpc: `http://localhost:${firstBlockchain.port}`, hubContract: '0x5FbDB2315678afecb367f032d93F642f64180aa3', }, maxNumberOfRetries: 20, frequency: 5, contentType: 'all', }); const clientBlockchainOptions = {}; Object.entries(this.state.localBlockchains).forEach( ([blockchainId, blockchain]) => { const wallets = blockchain.getWallets(); clientBlockchainOptions[blockchainId] = { blockchain: { name: blockchainId, publicKey: wallets[walletIndex].address, privateKey: wallets[walletIndex].privateKey, rpc: `http://localhost:${blockchain.port}`, hubContract: '0x5FbDB2315678afecb367f032d93F642f64180aa3', }, }; }, ); this.state.nodes[nodeIndex] = { client, forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, fileService: new FileService({ config: nodeConfiguration, logger: this.logger, }), clientBlockchainOptions, }; // Registration succeeded — safe to remove from pending tracking removePending(); done(resolve); } catch (err) { // Registration failed — keep in pendingProcesses so After hook can kill it done(reject, err); } }); }); }), ); }, ); Given( /^(\d+) bootstrap is running$/, { timeout: 60000 }, async function bootstrapRunning(nodeCount) { expect(this.state.bootstraps).to.have.length(0); expect(nodeCount).to.be.equal(1); // only one supported currently this.logger.log('Initializing bootstrap node'); const portOffset = Math.floor(Math.random() * 1000); const rpcPort = BOOTSTRAP_RPC_PORT + portOffset; const networkPort = BOOTSTRAP_NETWORK_PORT + portOffset; for (const port of [rpcPort, networkPort]) { try { execSync(`npx kill-port --port ${port}`, { stdio: 'ignore' }); } catch { // Port may already be free } } this.state.bootstrapPeerMultiaddr = `/ip4/127.0.0.1/tcp/${networkPort}/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj`; const blockchains = Object.entries(this.state.localBlockchains).map( ([blockchainId, blockchain]) => ({ blockchainId, operationalWallet: blockchain.getWallets()[0], managementWallet: blockchain.getWallets()[Math.floor(blockchain.getWallets().length / 2)], port: blockchain.port, }), ); const nodeName = 'origintrail-test-bootstrap'; const nodeConfiguration = stepsUtils.createNodeConfiguration( blockchains, 0, // bootstrap always uses wallet index 0 nodeName, rpcPort, networkPort, true, // bootstrap=true: fixed libp2p key, isolated DB/data paths ); this.state.bootstrapRpcPort = rpcPort; // Clear any stale data from a previously failed run before starting fs.rmSync(path.join(process.cwd(), nodeConfiguration.appDataPath), { recursive: true, force: true, }); const forkedNode = stepsUtils.forkNode(nodeConfiguration); // Track immediately so the After hook can kill it even if the step times out // before the process sends STARTED. this.state.pendingProcesses.push(forkedNode); const logFileStream = fs.createWriteStream( `${this.state.scenarioLogDir}/${nodeName}.log`, ); forkedNode.stdout.setEncoding('utf8'); forkedNode.stdout.on('data', (data) => logFileStream.write(data)); forkedNode.stderr.setEncoding('utf8'); forkedNode.stderr.on('data', (data) => logFileStream.write(`[stderr] ${data}`)); await new Promise((resolve, reject) => { let settled = false; const done = (fn, ...args) => { if (!settled) { settled = true; fn(...args); } }; const removePending = () => { const idx = this.state.pendingProcesses.indexOf(forkedNode); if (idx !== -1) this.state.pendingProcesses.splice(idx, 1); }; forkedNode.on('error', (err) => { removePending(); done(reject, err); }); forkedNode.on('exit', (code, signal) => { removePending(); done( reject, new Error( `Bootstrap process exited with code=${code} signal=${signal} before sending STARTED`, ), ); }); forkedNode.on('message', (response) => { if (response.error) { // Process reported an error — keep in pendingProcesses for cleanup done( reject, new Error(`Error initializing bootstrap node: ${response.error}`), ); return; } try { const [[firstBlockchainId, firstBlockchain]] = Object.entries( this.state.localBlockchains, ); const client = new DkgClientHelper({ endpoint: 'http://localhost', port: rpcPort, blockchain: { name: firstBlockchainId, publicKey: firstBlockchain.getWallets()[0].address, privateKey: firstBlockchain.getWallets()[0].privateKey, rpc: `http://localhost:${firstBlockchain.port}`, hubContract: '0x5FbDB2315678afecb367f032d93F642f64180aa3', }, useSSL: false, timeout: 25, loglevel: 'trace', }); this.state.bootstraps.push({ client, forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, fileService: new FileService({ config: nodeConfiguration, logger: this.logger, }), }); // Registration succeeded — safe to remove from pending tracking removePending(); done(resolve); } catch (err) { // Registration failed — keep in pendingProcesses so After hook can kill it done(reject, err); } }); }); }, ); Then( /Latest (Get|Publish|Update) operation finished with status: (\S+)$/, { timeout: 120000 }, async function latestOperationFinished(operationName, status) { this.logger.log(`Latest ${operationName} operation finished with status: ${status}`); const operationData = `latest${operationName}Data`; expect( !!this.state[operationData], `Latest ${operationName} result is undefined. ${operationData} result not started.`, ).to.be.equal(true); expect( !!(this.state[operationData].result || this.state[operationData].status), `Latest ${operationName} has no result or status. ${operationData} is not finished.`, ).to.be.equal(true); expect( this.state[operationData].errorType ?? this.state[operationData].status, `${operationData} result status validation failed`, ).to.be.equal(status); }, ); Given(/^I wait for (\d+) seconds$/, { timeout: 100000 }, async function waitFor(seconds) { this.logger.log(`I wait for ${seconds} seconds`); await sleep(seconds * 1000); }); /** * Deterministic wait for the sharding table to be populated and peers marked active. * * The publish pipeline needs shard records to exist before it can find replication peers. * ShardingTableCheckCommand creates them every ~10 s, but only when the on-chain count * differs from the local count. This step polls until all expected records are present, * then stamps them with the current time so DialPeersCommand doesn't needlessly re-dial * healthy peers whose lastDialed is still the epoch default. */ Given( /^I wait for nodes to sync and mark active$/, { timeout: 30000 }, async function waitForSyncAndActivate() { const expectedPeerCount = this.state.bootstraps.length + Object.keys(this.state.nodes).length; const allNodes = [...this.state.bootstraps, ...Object.values(this.state.nodes)]; const dbNames = allNodes.map((n) => n.configuration.operationalDatabase.databaseName); const con = mysql.createConnection({ host: 'localhost', user: 'root', password: process.env.REPOSITORY_PASSWORD, }); // Poll until shard records appear in every node's DB const maxAttempts = 12; for (let attempt = 1; attempt <= maxAttempts; attempt++) { let allSynced = true; for (const db of dbNames) { try { // eslint-disable-next-line no-await-in-loop const [rows] = await con .promise() .query(`SELECT COUNT(*) AS cnt FROM \`${db}\`.shard`); if (rows[0].cnt < expectedPeerCount) { allSynced = false; break; } } catch { allSynced = false; break; } } if (allSynced) { this.logger.log( `Sharding table synced after ${attempt * 2}s (${expectedPeerCount} peers)`, ); break; } if (attempt === maxAttempts) { this.logger.log( 'Warning: sharding table may not have fully synced within the timeout', ); } // eslint-disable-next-line no-await-in-loop await sleep(2000); } // Stamp fresh records with current time so that: // 1. filterInactive (WHERE last_seen = last_dialed) keeps passing // 2. DialPeersCommand doesn't waste cycles re-dialing perfectly healthy peers // whose lastDialed is still the epoch default (Date(0)) for (const db of dbNames) { try { // eslint-disable-next-line no-await-in-loop await con .promise() .query(`UPDATE \`${db}\`.shard SET last_seen = NOW(), last_dialed = NOW()`); } catch (e) { this.logger.log(`Warning: could not update shard in ${db}: ${e.message}`); } } con.end(); }, ); Given(/^Node (\d+) responds to info route$/, { timeout: 30000 }, async function (nodeNumber) { const nodeIndex = parseInt(nodeNumber, 10) - 1; const MAX_RETRIES = 10; let response; for (let i = 0; i < MAX_RETRIES; i += 1) { try { // eslint-disable-next-line no-await-in-loop response = await this.state.nodes[nodeIndex].client.info(); break; } catch { // eslint-disable-next-line no-await-in-loop await sleep(2000); } } this.logger.log(`Node ${nodeNumber} info response: ${JSON.stringify(response)}`); assert.ok(response && response.version, 'Expected node info to contain "version" field'); }); ================================================ FILE: test/bdd/steps/hooks.mjs ================================================ import 'dotenv/config'; import { execSync } from 'child_process'; import { setTimeout } from 'timers/promises'; import { Before, BeforeAll, After, AfterAll } from '@cucumber/cucumber'; import slugify from 'slugify'; import fs from 'fs'; import mysql from 'mysql2'; import { NODE_ENVIRONMENTS } from '../../../src/constants/constants.js'; import TripleStoreModuleManager from '../../../src/modules/triple-store/triple-store-module-manager.js'; /** Delay after killing node processes so the OS releases ports before the next scenario/retry. */ const PORT_RELEASE_DELAY_MS = 2500; process.env.NODE_ENV = NODE_ENVIRONMENTS.TEST; BeforeAll(() => {}); Before(async function beforeMethod(testCase) { this.logger = console; this.logger.log('\n🟡 Starting scenario:', testCase.pickle.name); this.state = { localBlockchains: {}, nodes: {}, bootstraps: [], pendingProcesses: [], }; // Flush Redis to remove stale BullMQ queues/jobs from prior scenarios. // Each node uses a per-node queue name (command-executor-node0, etc.); without // flushing, old job schedulers and pending jobs survive across scenarios. try { execSync('redis-cli FLUSHALL', { stdio: 'ignore' }); } catch { // Non-fatal: Redis may not have stale data } // Drop stale databases from prior crashed runs so nodes start clean on first attempt try { const con = mysql.createConnection({ host: 'localhost', user: 'root', password: process.env.REPOSITORY_PASSWORD, }); const staleDbNames = [ 'operationaldbbootstrap', ...Array.from({ length: 10 }, (_, i) => `operationaldbnode${i}`), ]; for (const db of staleDbNames) { await con.promise().query(`DROP DATABASE IF EXISTS \`${db}\`;`); } con.end(); } catch { // Non-fatal: node will attempt to create the DB itself } let logDir = process.env.CUCUMBER_ARTIFACTS_DIR || '.'; logDir += `/test/bdd/log/${slugify(testCase.pickle.name)}`; fs.mkdirSync(logDir, { recursive: true }); this.state.scenarioLogDir = logDir; this.logger.log('📁 Scenario logs:', logDir); }); After({ timeout: 60000 }, async function afterMethod(testCase) { const tripleStoreConfiguration = []; const databaseNames = []; const promises = []; // SIGKILL all node processes so they are terminated immediately without waiting for // async cleanup that could hang (e.g. trying to close blockchain connections to an // already-stopped Hardhat instance). This guarantees all ports are released before // the next scenario (or retry) starts. for (const proc of this.state.pendingProcesses) { proc.kill('SIGKILL'); } const allNodes = [...Object.values(this.state.nodes), ...this.state.bootstraps]; for (const node of allNodes) { node.forkedNode.kill('SIGKILL'); const tripleStoreModuleConfig = node.configuration.modules.tripleStore; const OT_BLAZEGRAPH_PACKAGE = './triple-store/implementation/ot-blazegraph/ot-blazegraph.js'; const enabledTripleStore = { enabled: true, implementation: {}, }; for (const [implName, implConfig] of Object.entries( tripleStoreModuleConfig.implementation || {}, )) { enabledTripleStore.implementation[implName] = { ...implConfig, enabled: true, package: implConfig.package || OT_BLAZEGRAPH_PACKAGE, }; } tripleStoreConfiguration.push({ appDataPath: node.configuration.appDataPath, modules: { tripleStore: enabledTripleStore }, }); databaseNames.push(node.configuration.operationalDatabase.databaseName); promises.push(node.fileService.removeFolder(node.fileService.getDataFolderPath())); } await setTimeout(PORT_RELEASE_DELAY_MS); for (const [blockchainId, blockchain] of Object.entries(this.state.localBlockchains)) { this.logger.log(`🛑 Stopping local blockchain ${blockchainId}`); promises.push(blockchain.stop()); } this.logger.log('🧹 Cleaning up repositories and databases...'); const con = mysql.createConnection({ host: 'localhost', user: 'root', password: process.env.REPOSITORY_PASSWORD, }); for (const db of databaseNames) { const sql = `DROP DATABASE IF EXISTS \`${db}\`;`; promises.push(con.promise().query(sql)); } for (const tsConfig of tripleStoreConfiguration) { promises.push( (async () => { const tripleStoreModuleManager = new TripleStoreModuleManager({ config: tsConfig, logger: this.logger, }); await tripleStoreModuleManager.initialize(); for (const impl of tripleStoreModuleManager.getImplementationNames()) { const { config: implConfig } = tripleStoreModuleManager.getImplementation(impl); if (!implConfig?.repositories) continue; for (const repo of Object.keys(implConfig.repositories)) { this.logger.log('🗑 Removing triple store repository:', repo); await tripleStoreModuleManager.deleteRepository(impl, repo); } } })(), ); } await Promise.all(promises); con.end(); this.logger.log('\n✅ Completed scenario:', testCase.pickle.name); this.logger.log( `📄 Location: ${testCase.gherkinDocument.uri}:${testCase.gherkinDocument.feature.location.line}`, ); this.logger.log(`🟢 Status: ${testCase.result.status}`); const durationMs = testCase.result.duration ? (Number(testCase.result.duration.seconds) || 0) * 1000 + (Number(testCase.result.duration.nanos) || 0) / 1e6 : 0; this.logger.log(`⏱ Duration: ${Math.round(durationMs)} ms\n`); }); AfterAll(async () => {}); process.on('unhandledRejection', (reason) => { console.error('Unhandled rejection in test runner:', reason); process.abort(); }); ================================================ FILE: test/bdd/steps/lib/local-blockchain.mjs ================================================ import { ethers } from 'ethers'; import { readFile } from 'fs/promises'; import { exec, execSync } from 'child_process'; const Hub = JSON.parse((await readFile('node_modules/dkg-evm-module/abi/Hub.json')).toString()); const ParametersStorage = JSON.parse( (await readFile('node_modules/dkg-evm-module/abi/ParametersStorage.json')).toString(), ); const hubContractAddress = '0x5FbDB2315678afecb367f032d93F642f64180aa3'; /** * LocalBlockchain wraps a local Hardhat node process for BDD testing. * * Starts a Hardhat chain via `npm run start:local_blockchain -- `, * connects an ethers provider, loads predefined test wallets, and exposes * helpers to mutate on-chain ParametersStorage values during scenarios. * * Basic usage: * const localBlockchain = new LocalBlockchain(); * await localBlockchain.initialize(8545, console); * // use localBlockchain.getWallets(), setR0(), setR1(), etc. * await localBlockchain.stop(); */ class LocalBlockchain { async initialize(port, _console = console, version = '') { this.port = port; this.startBlockchainProcess = exec( `npm run start:local_blockchain${version} -- ${port}`, ); this.startBlockchainProcess.stdout.on('data', (data) => { _console.log(data); }); this.provider = new ethers.providers.JsonRpcProvider(`http://localhost:${port}`); const [privateKeysFile, publicKeysFile] = await Promise.all([ readFile('test/bdd/steps/api/datasets/privateKeys.json'), readFile('test/bdd/steps/api/datasets/publicKeys.json'), ]); const privateKeys = JSON.parse(privateKeysFile.toString()); const publicKeys = JSON.parse(publicKeysFile.toString()); this.wallets = privateKeys.map((privateKey, index) => ({ address: publicKeys[index], privateKey, })); const wallet = new ethers.Wallet(this.wallets[0].privateKey, this.provider); this.hubContract = new ethers.Contract(hubContractAddress, Hub, wallet); this.ParametersStorageInterface = new ethers.utils.Interface(ParametersStorage); // provider.ready resolves when the JSON-RPC port is open, which happens before Hardhat // finishes deploying contracts. Poll the hub contract until it actually responds so that // the step only completes once the full on-chain environment is ready. await this.provider.ready; await this._waitForContracts(port, _console); } async _waitForContracts(port, _console) { const MAX_ATTEMPTS = 60; const INTERVAL_MS = 5000; for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) { try { // eslint-disable-next-line no-await-in-loop await this.hubContract.getContractAddress('ParametersStorage'); _console.log(`Contracts deployed and ready on port ${port}`); return; } catch { _console.log( `Waiting for contracts on port ${port} (attempt ${attempt + 1}/${MAX_ATTEMPTS})…`, ); // eslint-disable-next-line no-await-in-loop await new Promise((r) => setTimeout(r, INTERVAL_MS)); } } throw new Error( `Hub contract on port ${port} did not become ready after ${MAX_ATTEMPTS * (INTERVAL_MS / 1000)}s`, ); } async stop() { const commandLog = execSync(`npm run kill:local_blockchain -- ${this.port}`); console.log(`Killing hardhat process: ${commandLog.toString()}`); this.startBlockchainProcess.kill(); } getWallets() { return this.wallets; } async setParametersStorageParams(params) { const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); for (const parameter of Object.keys(params)) { const blockchainMethodName = `set${ parameter.charAt(0).toUpperCase() + parameter.slice(1) }`; console.log(`Setting ${parameter} in parameters storage to: ${params[parameter]}`); const encodedData = this.ParametersStorageInterface.encodeFunctionData( blockchainMethodName, [params[parameter]], ); // eslint-disable-next-line no-await-in-loop await this.hubContract.forwardCall(parametersStorageAddress, encodedData); } } async setR0(r0) { console.log(`Setting R0 in parameters storage to: ${r0}`); const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR0', [r0]); const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); await this.hubContract.forwardCall(parametersStorageAddress, encodedData); } async setR1(r1) { console.log(`Setting R1 in parameters storage to: ${r1}`); const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR1', [r1]); const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); await this.hubContract.forwardCall(parametersStorageAddress, encodedData); } async setFinalizationCommitsNumber(commitsNumber) { console.log(`Setting finalizationCommitsNumber in parameters storage to: ${commitsNumber}`); const encodedData = this.ParametersStorageInterface.encodeFunctionData( 'setFinalizationCommitsNumber', [commitsNumber], ); const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); await this.hubContract.forwardCall(parametersStorageAddress, encodedData); } } export default LocalBlockchain; ================================================ FILE: test/bdd/steps/lib/ot-node-process.mjs ================================================ import { setTimeout } from 'timers/promises'; import OTNode from '../../../../ot-node.js'; import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; const httpApiHelper = new HttpApiHelper(); // In small BDD test networks (3 nodes), libp2p's KadDHT periodically performs // peer lookups that fail because the routing table is empty/sparse. These // surface as unhandled promise rejections which, in Node.js >= 15, terminate // the process. Catching them here keeps the test nodes alive. process.on('unhandledRejection', (reason) => { const msg = reason instanceof Error ? reason.message : String(reason); const code = reason?.code; if (code === 'ERR_LOOKUP_FAILED' || code === 'NOT_FOUND' || code === 'NO_ROUTERS_AVAILABLE') { // Expected in small test networks — suppress silently. return; } console.error(`[test-node] Unhandled rejection: ${msg}`); }); process.on('message', async (data) => { const config = JSON.parse(data); try { process.env.OPERATIONAL_DB_NAME = config.operationalDatabase.databaseName; // OTNode constructor reads configjson[NODE_ENV] as the default config base. // We must keep NODE_ENV='test' during construction so the 'test' defaults // (e.g. tripleStore.ot-blazegraph.enabled=true) are used. const newNode = new OTNode(config); // Switch to 'development' AFTER config is built but BEFORE start() so the // CommandExecutor creates per-node BullMQ queues (command-executor-{nodeName}) // instead of a shared 'command-executor' queue that causes job stealing. process.env.NODE_ENV = 'development'; await newNode.start(); const nodeHostname = `http://localhost:${config.rpcPort}`; const MAX_HTTP_POLL_ATTEMPTS = 30; let started = false; for (let attempt = 0; attempt < MAX_HTTP_POLL_ATTEMPTS; attempt += 1) { try { // eslint-disable-next-line no-await-in-loop await httpApiHelper.info(nodeHostname); started = true; break; } catch { // eslint-disable-next-line no-await-in-loop await setTimeout(1000); } } if (!started) { throw new Error( `Node HTTP API on port ${config.rpcPort} did not become ready after ${MAX_HTTP_POLL_ATTEMPTS}s`, ); } process.send({ status: 'STARTED' }); } catch (error) { process.send({ error: error.message }); } }); ================================================ FILE: test/modules/telemetry/config.json ================================================ { "modules": { "telemetry": { "enabled": true, "implementation": { "ot-telemetry": { "enabled": true, "package": "./telemetry/implementation/ot-telemetry.js", "config": { "sendTelemetryData": false, "signalingServerUrl": "null" } } } } } } ================================================ FILE: test/modules/telemetry/telemetry.js ================================================ import { readFile } from 'fs/promises'; import { describe, it, before } from 'mocha'; import { expect, assert } from 'chai'; import Logger from '../../../src/logger/logger.js'; import TelemetryModuleManager from '../../../src/modules/telemetry/telemetry-module-manager.js'; let logger; let telemetryModuleManager; const config = JSON.parse(await readFile('./test/modules/telemetry/config.json')); describe('Telemetry module', () => { before('Initialize logger', () => { logger = new Logger('trace'); logger.info = () => {}; }); describe('Handle received events', () => { it('should call onEventReceived when event is emitted', async () => { const eventEmitter = { eventListeners: {}, on(eventName, callback) { if (!this.eventListeners[eventName]) { this.eventListeners[eventName] = []; } this.eventListeners[eventName].push(callback); }, emit(eventName, ...args) { if (this.eventListeners[eventName]) { this.eventListeners[eventName].forEach((callback) => callback(...args)); } }, }; let callbackCalled = false; function onEventReceived() { callbackCalled = true; } telemetryModuleManager = new TelemetryModuleManager({ config, logger, eventEmitter }); await telemetryModuleManager.initialize(); telemetryModuleManager.listenOnEvents(onEventReceived); eventEmitter.emit('operation_status_changed'); assert(expect(callbackCalled).to.be.true); }); }); }); ================================================ FILE: test/unit/api/http-api-router.test.js ================================================ import { beforeEach, describe, it } from 'mocha'; import { expect } from 'chai'; import sinon from 'sinon'; import HttpApiRouter from '../../../src/controllers/http-api/http-api-router.js'; import JsonSchemaServiceMock from '../mock/json-schema-service-mock.js'; import HttpClientModuleManagerMock from '../mock/http-client-module-manager-mock.js'; import { HTTP_API_ROUTES } from '../../../src/constants/constants.js'; describe('HTTP API Router test', async () => { let httpApiRouter; const controllerMocks = {}; beforeEach(() => { // Mock Controllers Object.keys(HTTP_API_ROUTES).forEach((version) => { Object.keys(HTTP_API_ROUTES[version]).forEach((operation) => { const versionedController = `${operation}HttpApiController${ version.charAt(1).toUpperCase() + version.slice(2) }`; controllerMocks[versionedController] = { handleRequest: sinon.stub() }; }); }); // Mock context const ctx = { httpClientModuleManager: new HttpClientModuleManagerMock(), jsonSchemaService: new JsonSchemaServiceMock(), ...controllerMocks, }; // Initialize HttpApiRouter with mocks httpApiRouter = new HttpApiRouter(ctx); }); it('Router has all defined routes', async () => { // Extract unique HTTP methods present across all versions const httpMethods = new Set(); Object.values(HTTP_API_ROUTES).forEach((routes) => { Object.values(routes).forEach((route) => { httpMethods.add(route.method); }); }); // Create spies for each extracted HTTP method on each router instance and httpClientModuleManager const spies = {}; Object.keys(HTTP_API_ROUTES).forEach((version) => { spies[version] = {}; Array.from(httpMethods).forEach((method) => { spies[version][method] = sinon.spy(httpApiRouter.routers[version], method); }); }); const httpClientModuleManagerUseSpy = sinon.spy( httpApiRouter.httpClientModuleManager, 'use', ); // Initialize the routes await httpApiRouter.initialize(); // Validate each route Object.entries(HTTP_API_ROUTES).forEach(([version, routes]) => { expect(httpClientModuleManagerUseSpy.calledWith(`/${version}`)).to.equal(true); Object.values(routes).forEach((routeDetails) => { const { method, path } = routeDetails; expect(spies[version][method].calledWith(path)).to.equal(true); }); }); expect(httpClientModuleManagerUseSpy.calledWith('/latest')).to.equal(true); expect(httpClientModuleManagerUseSpy.calledWith('/')).to.equal(true); // Restore all spies Object.values(spies).forEach((versionSpies) => { Object.values(versionSpies).forEach((spy) => { spy.restore(); }); }); httpClientModuleManagerUseSpy.restore(); }); }); ================================================ FILE: test/unit/commands/operation-id-cleaner-command.test.js ================================================ import { describe, it, beforeEach, afterEach } from 'mocha'; import { expect } from 'chai'; import sinon from 'sinon'; import OperationIdCleanerCommand from '../../../src/commands/cleaners/operation-id-cleaner-command.js'; import { OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS, OPERATION_ID_STATUS, } from '../../../src/constants/constants.js'; describe('OperationIdCleanerCommand', () => { let clock; let operationIdService; let repositoryModuleManager; let logger; let command; beforeEach(() => { clock = sinon.useFakeTimers(new Date('2023-01-01T00:00:00Z').getTime()); operationIdService = { getOperationIdMemoryCacheSizeBytes: sinon.stub().returns(1024), getOperationIdFileCacheSizeBytes: sinon.stub().resolves(2048), removeExpiredOperationIdMemoryCache: sinon.stub().resolves(512), removeExpiredOperationIdFileCache: sinon.stub().resolves(3), }; repositoryModuleManager = { removeOperationIdRecord: sinon.stub().resolves(), }; logger = { debug: sinon.spy(), info: sinon.spy(), warn: sinon.spy(), error: sinon.spy(), }; command = new OperationIdCleanerCommand({ logger, repositoryModuleManager, operationIdService, fileService: {}, }); }); afterEach(() => { clock.restore(); }); it('cleans memory with 1h TTL and files with 24h TTL while reporting footprint', async () => { await command.execute(); expect(operationIdService.getOperationIdMemoryCacheSizeBytes.calledOnce).to.be.true; expect(operationIdService.getOperationIdFileCacheSizeBytes.calledOnce).to.be.true; expect( repositoryModuleManager.removeOperationIdRecord.calledWith( Date.now() - OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, [OPERATION_ID_STATUS.COMPLETED, OPERATION_ID_STATUS.FAILED], ), ).to.be.true; expect( operationIdService.removeExpiredOperationIdMemoryCache.calledWith( OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS, ), ).to.be.true; expect( operationIdService.removeExpiredOperationIdFileCache.calledWith( OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, ), ).to.be.true; expect(logger.debug.called).to.be.true; }); it('handles missing memory cache gracefully', async () => { operationIdService.getOperationIdMemoryCacheSizeBytes.throws(new Error('no memory cache')); await command.execute(); expect( repositoryModuleManager.removeOperationIdRecord.calledWith( Date.now() - OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, [OPERATION_ID_STATUS.COMPLETED, OPERATION_ID_STATUS.FAILED], ), ).to.be.true; expect( operationIdService.removeExpiredOperationIdFileCache.calledWith( OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, ), ).to.be.true; }); }); ================================================ FILE: test/unit/controllers/publish-http-api-controller-v1.test.js ================================================ import { describe, it } from 'mocha'; import { expect } from 'chai'; import PublishController from '../../../src/controllers/http-api/v1/publish-http-api-controller-v1.js'; import { PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS } from '../../../src/constants/constants.js'; const createRes = () => { const res = { statusCode: null, body: null, status(code) { this.statusCode = code; return this; }, json(payload) { this.body = payload; return this; }, send(payload) { this.body = payload; return this; }, }; return res; }; describe('publish-http-api-controller-v1', () => { const baseCtx = () => { const addedCommands = []; return { commandExecutor: { add: async (cmd) => { addedCommands.push(cmd); }, _added: addedCommands, }, publishService: { getOperationName: () => 'publish', }, operationIdService: { generateOperationId: async () => 'op-id-123', emitChangeEvent: () => {}, updateOperationIdStatus: async () => {}, cacheOperationIdDataToMemory: async () => {}, cacheOperationIdDataToFile: async () => {}, }, repositoryModuleManager: { createOperationRecord: async () => {}, }, pendingStorageService: { cacheDataset: async () => {}, }, networkModuleManager: { getPeerId: () => ({ toB58String: () => 'peer-self' }), }, blockchainModuleManager: { getMinimumRequiredSignatures: async () => PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS, }, logger: { info: () => {}, warn: () => {}, error: () => {}, }, }; }; it('clamps minimumNumberOfNodeReplications to on-chain minimum', async () => { const ctx = baseCtx(); ctx.blockchainModuleManager.getMinimumRequiredSignatures = async () => 5; // on-chain min const controller = new PublishController(ctx); const req = { body: { dataset: { public: {} }, datasetRoot: '0xroot', blockchain: 'hardhat', minimumNumberOfNodeReplications: 2, // below chain min }, }; const res = createRes(); await controller.handleRequest(req, res); expect(res.statusCode).to.equal(202); const added = ctx.commandExecutor._added[0]; expect(added.data.minimumNumberOfNodeReplications).to.equal(5); }); it('allows higher user override than on-chain minimum', async () => { const ctx = baseCtx(); ctx.blockchainModuleManager.getMinimumRequiredSignatures = async () => 3; // on-chain min const controller = new PublishController(ctx); const req = { body: { dataset: { public: {} }, datasetRoot: '0xroot', blockchain: 'hardhat', minimumNumberOfNodeReplications: 7, // above chain min }, }; const res = createRes(); await controller.handleRequest(req, res); expect(res.statusCode).to.equal(202); const added = ctx.commandExecutor._added[0]; expect(added.data.minimumNumberOfNodeReplications).to.equal(7); }); it('falls back to on-chain minimum when user value is zero or invalid', async () => { const ctx = baseCtx(); ctx.blockchainModuleManager.getMinimumRequiredSignatures = async () => 4; // on-chain min const controller = new PublishController(ctx); const req = { body: { dataset: { public: {} }, datasetRoot: '0xroot', blockchain: 'hardhat', minimumNumberOfNodeReplications: 0, // invalid/zero }, }; const res = createRes(); await controller.handleRequest(req, res); expect(res.statusCode).to.equal(202); const added = ctx.commandExecutor._added[0]; expect(added.data.minimumNumberOfNodeReplications).to.equal(4); }); }); ================================================ FILE: test/unit/middleware/authentication-middleware.test.js ================================================ import sinon from 'sinon'; import { describe, it, afterEach } from 'mocha'; import { expect } from 'chai'; import authenticationMiddleware from '../../../src/modules/http-client/implementation/middleware/authentication-middleware.js'; import AuthService from '../../../src/service/auth-service.js'; describe('authentication middleware test', async () => { const sandbox = sinon.createSandbox(); const getAuthService = (options) => sandbox.createStubInstance(AuthService, { authenticate: options.isAuthenticated, isPublicOperation: options.isPublicOperation, }); afterEach(() => { sandbox.restore(); }); it('calls next if isPublic evaluated to true', async () => { const middleware = authenticationMiddleware( getAuthService({ isPublicOperation: true, }), ); const req = { headers: { authorization: 'Bearer token' }, path: '/publish' }; const spySend = sandbox.spy(); const spyStatus = sandbox.spy(() => ({ send: spySend })); const nextSpy = sandbox.spy(); await middleware(req, { status: spyStatus }, nextSpy); expect(nextSpy.calledOnce).to.be.true; expect(spyStatus.notCalled).to.be.true; expect(spySend.notCalled).to.be.true; }); it('calls next if isAuthenticated is evaluated as true', async () => { const middleware = authenticationMiddleware( getAuthService({ isPublicOperation: false, isAuthenticated: true, }), ); const req = { headers: { authorization: 'Bearer token' }, path: '/publish' }; const spySend = sandbox.spy(); const spyStatus = sandbox.spy(() => ({ send: spySend })); const nextSpy = sandbox.spy(); await middleware(req, { status: spyStatus }, nextSpy); expect(nextSpy.calledOnce).to.be.true; expect(spyStatus.notCalled).to.be.true; expect(spySend.notCalled).to.be.true; }); it('returns 401 if isAuthenticated is evaluated as false', async () => { const middleware = authenticationMiddleware( getAuthService({ isPublicOperation: false, isAuthenticated: false, }), ); const req = { headers: { authorization: 'Bearer token' }, path: '/publish' }; const spySend = sandbox.spy(); const spyStatus = sandbox.spy(() => ({ send: spySend })); const spyNext = sandbox.spy(); await middleware(req, { status: spyStatus }, spyNext); const [statusCode] = spyStatus.args[0]; expect(statusCode).to.be.eq(401); expect(spyStatus.calledOnce).to.be.true; expect(spySend.calledOnce).to.be.true; expect(spyNext.notCalled).to.be.true; }); }); ================================================ FILE: test/unit/middleware/authorization-middleware.test.js ================================================ import sinon from 'sinon'; import { describe, it, afterEach } from 'mocha'; import { expect } from 'chai'; import authorizationMiddleware from '../../../src/modules/http-client/implementation/middleware/authorization-middleware.js'; import AuthService from '../../../src/service/auth-service.js'; describe('authentication middleware test', async () => { const sandbox = sinon.createSandbox(); const getAuthService = (options) => sandbox.createStubInstance(AuthService, { authenticate: options.isAuthenticated, isAuthorized: options.isAuthorized, isPublicOperation: options.isPublicOperation, }); afterEach(() => { sandbox.restore(); }); it('calls next if isPublicOperation is resolved to true', async () => { const middleware = authorizationMiddleware( getAuthService({ isPublicOperation: true, }), ); const req = { headers: { authorization: 'Bearer token' }, path: '/publish' }; const spySend = sandbox.spy(); const spyStatus = sandbox.spy(() => ({ send: spySend })); const nextSpy = sandbox.spy(); await middleware(req, { status: spyStatus }, nextSpy); expect(nextSpy.calledOnce).to.be.true; expect(spyStatus.notCalled).to.be.true; expect(spySend.notCalled).to.be.true; }); it('calls next if isAuthenticated is evaluated as true', async () => { const middleware = authorizationMiddleware( getAuthService({ isPublicOperation: true, }), ); const req = { headers: { authorization: 'Bearer token' }, path: '/publish' }; const spySend = sandbox.spy(); const spyStatus = sandbox.spy(() => ({ send: spySend })); const nextSpy = sandbox.spy(); await middleware(req, { status: spyStatus }, nextSpy); expect(nextSpy.calledOnce).to.be.true; expect(spyStatus.notCalled).to.be.true; expect(spySend.notCalled).to.be.true; }); it('returns 403 if isAuthenticated is evaluated as false', async () => { const middleware = authorizationMiddleware( getAuthService({ isPublicOperation: false, isAuthenticated: false, }), ); const req = { headers: { authorization: 'Bearer token' }, path: '/publish' }; const spySend = sandbox.spy(); const spyStatus = sandbox.spy(() => ({ send: spySend })); const spyNext = sandbox.spy(); await middleware(req, { status: spyStatus }, spyNext); const [statusCode] = spyStatus.args[0]; expect(statusCode).to.be.eq(403); expect(spyStatus.calledOnce).to.be.true; expect(spySend.calledOnce).to.be.true; expect(spyNext.notCalled).to.be.true; }); }); ================================================ FILE: test/unit/mock/blockchain-module-manager-mock.js ================================================ import { ethers } from 'ethers'; class BlockchainModuleManagerMock { getR2() { return 20; } getR1() { return 8; } getR0() { return 3; } encodePacked(blockchain, types, values) { return ethers.utils.solidityPack(types, values); } convertBytesToUint8Array(blockchain, bytesLikeData) { return ethers.utils.arrayify(bytesLikeData); } convertToWei(blockchainId, value) { return ethers.utils.parseUnits(value.toString(), 'ether'); } toBigNumber(blockchain, value) { return ethers.BigNumber.from(value); } getAssertionSize(blockchain, assertionId) { return 246; } getAssertionTriplesNumber(blockchain, assertionId) { return undefined; } getAssertionChunksNumber(blockchain, assertionId) { return undefined; } } export default BlockchainModuleManagerMock; ================================================ FILE: test/unit/mock/command-executor-mock.js ================================================ class CommandExecutorMock { add(addCommand) {} } export default CommandExecutorMock; ================================================ FILE: test/unit/mock/event-emitter-mock.js ================================================ class EventEmitterMock {} export default EventEmitterMock; ================================================ FILE: test/unit/mock/http-client-module-manager-mock.js ================================================ import express from 'express'; class HttpClientModuleManagerMock { createRouterInstance() { return express.Router(); } initializeBeforeMiddlewares() {} async listen() {} use(path, callback) {} selectMiddlewares(options) { return []; } initializeAfterMiddlewares() {} } export default HttpClientModuleManagerMock; ================================================ FILE: test/unit/mock/json-schema-service-mock.js ================================================ class JsonSchemaServiceMock {} export default JsonSchemaServiceMock; ================================================ FILE: test/unit/mock/network-module-manager-mock.js ================================================ class NetworkModuleManagerMock { getPeerId() { return { toB58String: () => 'myPeerId', }; } } export default NetworkModuleManagerMock; ================================================ FILE: test/unit/mock/operation-id-service-mock.js ================================================ class OperationIdServiceMock { constructor(ctx) { this.repositoryModuleManager = ctx.repositoryModuleManager; } cacheOperationIdDataToFile(operationId, data) {} cacheOperationIdDataToMemory(operationId, data) {} async updateOperationIdStatus( operationId, blockchain, status, errorMessage = null, errorType = null, ) { await this.repositoryModuleManager.updateOperationIdRecord( { status, timestamp: new Date().toISOString(), }, operationId, ); } } export default OperationIdServiceMock; ================================================ FILE: test/unit/mock/repository-module-manager-mock.js ================================================ class RepositoryModuleManagerMock { responseStatuses = [ { id: 1, operationId: 'f6354c2c-d460-11ed-afa1-0242ac120002', keyword: 'origintrail', status: 'COMPLETED', message: 'message', createdAt: '1970-01-01 00:00:00', updatedAt: '1970-01-01 00:00:00', }, ]; getAllPeerRecords() { return [ { peerId: 'QmcJY13uLyt2VQ6QiVNcYiWaxdfaHWHj3T7G472uaHPBf7', blockchainId: 'ganache', ask: '0.2824612246520951', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0x6e08776479a010d563855dbc371a66f692d3edcbcf2b02c30f9879ebe02244e8', }, { peerId: 'Qmcxo88zf5zEvyBLYTrtfG8nGJQW6zHpf58b5MUcjoYVqL', blockchainId: 'ganache', ask: '0.11680988694381877', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0x113d3da32b0e0b7031d188736792bbea0baf7911acb905511ac7dda2be9a6f55', }, { peerId: 'QmQeNwBzgeMQxquQEDXvBHqXBHNBEvKHtyHURg4QvnoLrD', blockchainId: 'ganache', ask: '0.25255488168658036', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0xba14ac66ab5be40bf458bad9b4e9f10a9d06375b233e91a6ce3c2d4cbf9deea5', }, { peerId: 'QmU4ty8X8L4Xk6cbDCoyJUhgeBNLDo3HprTGEhNd9CtiT7', blockchainId: 'ganache', ask: '0.25263875217271087', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0x5b3fdb88b3270a99cc89d28e0a4504d28789e5f8ca53080aa7608db48546d56b', }, { peerId: 'QmWmgmMCQQ1awraTeQqwsbWgqtR3ZMuX7NhbHyiftuAspb', blockchainId: 'ganache', ask: '0.2429885059428509', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0x820a8e38cb792b89c8b69eb9c192faf3def6175c97c4c0f17708161bcb9c5028', }, { peerId: 'QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj', blockchainId: 'ganache', ask: '0.210617584797714', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0xf764186e9b675f3fd00af72026cf075d05ce8fc951ba089351d645b363acd3d3', }, { peerId: 'QmXgeHgBVbd7iyTp8PapUAyeKciqbsXTEvsakCjW7wZRqT', blockchainId: 'ganache', ask: '0.2290449496761527', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0xaaeed7b766483aef7cf2d07325f336b3e703e2b7573e540ca8c6e2aab34265c3', }, { peerId: 'QmYys42KLmGEE9hEmJCVCe3SR3G9zf4epoAwDUK7pVUP6S', blockchainId: 'ganache', ask: '0.1637075464317365', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0xc3bb7b5433ebe62ff9e98c6d439223d07d44e16e7d5e210e727823f87c0ef24b', }, { peerId: 'QmZi2nDhZJfa1Z5iXjvxQ1BigpR8TdTQ3gWQDGecn34e9x', blockchainId: 'ganache', ask: '0.10242295311162795', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0x510ca60cdd7b33bf8d978576981ae7f9caaf5f133ddd40693d8ce007614c0a09', }, { peerId: 'QmZueq5jip24v5dbCSBGt8v16hPjUN1CXRb3zGaxH1jfHM', blockchainId: 'ganache', ask: '0.23374911902136858', stake: '50000.0', lastSeen: '1970-01-01 00:00:00', lastDialed: '1970-01-01 00:00:00', sha256: '0x7b4f717bd647104a72c7f1fce4600366982f36ebb1cef41540a5541c8e8ca1dd', }, ]; } getAllResponseStatuses() { return this.responseStatuses; } async getOperationResponsesStatuses(operation, operationId) { return this.responseStatuses.filter((rs) => rs.operationId === operationId); } async updateOperationIdRecord(data, operationId) { this.responseStatuses = this.responseStatuses.map((rs) => rs.operationId === operationId ? { ...rs, status: data.status, updatedAt: data.timestamp } : rs, ); } async updateOperationStatus(operation, operationId, status) { this.responseStatuses = this.responseStatuses.map((rs) => rs.operationId === operationId ? { ...rs, status, updatedAt: new Date().toISOString() } : rs, ); } async createOperationResponseRecord(status, operation, operationId, errorMessage) { this.responseStatuses = [ ...this.responseStatuses, { id: this.responseStatuses[this.responseStatuses.length - 1].id + 1, status, operationId, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }, ]; } } export default RepositoryModuleManagerMock; ================================================ FILE: test/unit/mock/validation-module-manager-mock.js ================================================ import { ethers } from 'ethers'; class ValidationModuleManagerMock { callHashFunction(data) { const bytesLikeData = ethers.utils.toUtf8Bytes(data); return ethers.utils.sha256(bytesLikeData); } getHashFunctionName() { return 'sha256'; } calculateRoot(assertion) { return '0xde58cc52a5ce3a04ae7a05a13176226447ac02489252e4d37a72cbe0aea46b42'; } } export default ValidationModuleManagerMock; ================================================ FILE: test/unit/modules/repository/config.json ================================================ { "modules": { "repository": { "enabled": true, "implementation": { "sequelize-repository": { "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb-test", "user": "root", "password": "", "port": "3306", "host": "localhost", "dialect": "mysql", "logging": false } } } } } } ================================================ FILE: test/unit/modules/repository/repository.test.js ================================================ import { utils } from 'ethers'; import { describe, it, before, beforeEach, afterEach, after } from 'mocha'; import { expect, assert } from 'chai'; import { readFile } from 'fs/promises'; import Logger from '../../../../src/logger/logger.js'; import RepositoryModuleManager from '../../../../src/modules/repository/repository-module-manager.js'; let logger; let repositoryModuleManager; const config = JSON.parse(await readFile('./test/unit/modules/repository/config.json')); const blockchain = 'hardhat'; const createAgreement = ({ blockchainId = blockchain, assetStorageContractAddress = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', tokenId, id = null, startTime, epochsNumber = 2, epochLength = 100, scoreFunctionId = 1, proofWindowOffsetPerc = 66, hashFunctionId = 1, keyword = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca0768e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', assertionId = '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', stateIndex = 1, lastCommitEpoch = null, lastProofEpoch = null, }) => { const agreementId = id ?? utils.sha256( utils.toUtf8Bytes( utils.solidityPack( ['address', 'uint256', 'bytes'], [assetStorageContractAddress, tokenId, keyword], ), ), ); return { blockchainId, assetStorageContractAddress, tokenId, agreementId, startTime, epochsNumber, epochLength, scoreFunctionId, proofWindowOffsetPerc, hashFunctionId, keyword, assertionId, stateIndex, lastCommitEpoch, lastProofEpoch, }; }; describe('Repository module', () => { before('Initialize repository module manager', async function initializeRepository() { this.timeout(30_000); logger = new Logger('trace'); logger.info = () => {}; repositoryModuleManager = new RepositoryModuleManager({ config, logger }); await repositoryModuleManager.initialize(); await repositoryModuleManager.destroyAllRecords('service_agreement'); }); afterEach('Destroy all records', async function destroyAllRecords() { this.timeout(30_000); await repositoryModuleManager.destroyAllRecords('service_agreement'); }); after(async function dropDatabase() { this.timeout(30_000); await repositoryModuleManager.dropDatabase(); }); describe('Empty repository', () => { it('returns empty list if no service agreements', async () => { const eligibleAgreements = await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( Date.now(), blockchain, 25, ); assert(expect(eligibleAgreements).to.exist); expect(eligibleAgreements).to.be.instanceOf(Array); expect(eligibleAgreements).to.have.length(0); }); }); describe('Insert and update service agreement', () => { const agreement = { blockchainId: blockchain, assetStorageContractAddress: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', tokenId: 0, agreementId: '0x44cf660357e2d7462c25fd8e50b68abe332d7a70b07a76e92f628846ea585881', startTime: 1683032289, epochsNumber: 2, epochLength: 360, scoreFunctionId: 1, proofWindowOffsetPerc: 66, hashFunctionId: 1, keyword: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca0768e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', assertionId: '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', stateIndex: 1, }; it('inserts service agreement', async () => { const inserted = await repositoryModuleManager.updateServiceAgreementRecord( agreement.blockchainId, agreement.assetStorageContractAddress, agreement.tokenId, agreement.agreementId, agreement.startTime, agreement.epochsNumber, agreement.epochLength, agreement.scoreFunctionId, agreement.proofWindowOffsetPerc, agreement.hashFunctionId, agreement.keyword, agreement.assertionId, agreement.stateIndex, agreement.lastCommitEpoch, agreement.lastProofEpoch, ); const row = inserted[0]?.dataValues; assert(expect(row).to.exist); expect(row.blockchainId).to.equal(agreement.blockchainId); expect(row.assetStorageContractAddress).to.equal(agreement.assetStorageContractAddress); expect(row.tokenId).to.equal(agreement.tokenId); expect(row.agreementId).to.equal(agreement.agreementId); expect(row.startTime).to.equal(agreement.startTime); expect(row.epochsNumber).to.equal(agreement.epochsNumber); expect(row.epochLength).to.equal(agreement.epochLength); expect(row.scoreFunctionId).to.equal(agreement.scoreFunctionId); expect(row.proofWindowOffsetPerc).to.equal(agreement.proofWindowOffsetPerc); expect(row.hashFunctionId).to.equal(agreement.hashFunctionId); expect(row.keyword).to.equal(agreement.keyword); expect(row.assertionId).to.equal(agreement.assertionId); expect(row.stateIndex).to.equal(agreement.stateIndex); assert(expect(row.lastCommitEpoch).to.not.exist); assert(expect(row.lastProofEpoch).to.not.exist); }); }); describe('Eligible service agreements', () => { const agreements = [ createAgreement({ tokenId: 0, startTime: 0 }), createAgreement({ tokenId: 1, startTime: 15, lastCommitEpoch: 0, }), createAgreement({ tokenId: 2, startTime: 25 }), createAgreement({ tokenId: 3, startTime: 25, lastCommitEpoch: 0, lastProofEpoch: 0, }), createAgreement({ tokenId: 4, startTime: 49 }), ]; beforeEach(async () => { await Promise.all( agreements.map((agreement) => repositoryModuleManager.updateServiceAgreementRecord( agreement.blockchainId, agreement.assetStorageContractAddress, agreement.tokenId, agreement.agreementId, agreement.startTime, agreement.epochsNumber, agreement.epochLength, agreement.scoreFunctionId, agreement.proofWindowOffsetPerc, agreement.hashFunctionId, agreement.keyword, agreement.assertionId, agreement.stateIndex, agreement.lastCommitEpoch, agreement.lastProofEpoch, ), ), ); }); describe('getEligibleAgreementsForSubmitCommit returns correct agreements', () => { const testEligibleAgreementsForSubmitCommit = (currentTimestamp, commitWindowDurationPerc, expectedAgreements) => async () => { const eligibleAgreements = await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( currentTimestamp, blockchain, commitWindowDurationPerc, ); assert(expect(eligibleAgreements).to.exist); expect(eligibleAgreements).to.be.instanceOf(Array); expect(eligibleAgreements).to.have.length(expectedAgreements.length); expect(eligibleAgreements).to.have.deep.members(expectedAgreements); // ensure order is correct for (let i = 0; i < eligibleAgreements.length; i += 1) { assert.strictEqual( eligibleAgreements[i].timeLeftInSubmitCommitWindow, expectedAgreements[i].timeLeftInSubmitCommitWindow, ); } }; it( 'returns two eligible service agreements at timestamp 49', testEligibleAgreementsForSubmitCommit(49, 25, [ { ...agreements[2], currentEpoch: 0, timeLeftInSubmitCommitWindow: 1 }, { ...agreements[4], currentEpoch: 0, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns one eligible service agreement at timestamp 51', testEligibleAgreementsForSubmitCommit(51, 25, [ { ...agreements[4], currentEpoch: 0, timeLeftInSubmitCommitWindow: 23 }, ]), ); it( 'returns no eligible service agreement at timestamp 74', testEligibleAgreementsForSubmitCommit(74, 25, []), ); it( 'returns no eligible service agreements at timestamp 75', testEligibleAgreementsForSubmitCommit(75, 25, []), ); it( 'returns one eligible service agreements at timestamp 100', testEligibleAgreementsForSubmitCommit(100, 25, [ { ...agreements[0], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns two eligible service agreements at timestamp 124', testEligibleAgreementsForSubmitCommit(124, 25, [ { ...agreements[0], currentEpoch: 1, timeLeftInSubmitCommitWindow: 1 }, { ...agreements[1], currentEpoch: 1, timeLeftInSubmitCommitWindow: 16 }, ]), ); it( 'returns three eligible service agreements at timestamp 125', testEligibleAgreementsForSubmitCommit(125, 25, [ { ...agreements[1], currentEpoch: 1, timeLeftInSubmitCommitWindow: 15 }, { ...agreements[2], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, { ...agreements[3], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns three eligible service agreements at timestamp 126', testEligibleAgreementsForSubmitCommit(126, 25, [ { ...agreements[1], currentEpoch: 1, timeLeftInSubmitCommitWindow: 14 }, { ...agreements[2], currentEpoch: 1, timeLeftInSubmitCommitWindow: 24 }, { ...agreements[3], currentEpoch: 1, timeLeftInSubmitCommitWindow: 24 }, ]), ); it( 'returns three eligible service agreements at timestamp 149', testEligibleAgreementsForSubmitCommit(149, 25, [ { ...agreements[2], currentEpoch: 1, timeLeftInSubmitCommitWindow: 1 }, { ...agreements[3], currentEpoch: 1, timeLeftInSubmitCommitWindow: 1 }, { ...agreements[4], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns one eligible service agreements at timestamp 151', testEligibleAgreementsForSubmitCommit(151, 25, [ { ...agreements[4], currentEpoch: 1, timeLeftInSubmitCommitWindow: 23 }, ]), ); it( 'returns no eligible service agreements at timestamp 175', testEligibleAgreementsForSubmitCommit(175, 25, []), ); it( 'returns no eligible service agreements at timestamp 225', testEligibleAgreementsForSubmitCommit(225, 25, []), ); }); describe('getEligibleAgreementsForSubmitProof returns correct agreements', () => { const testEligibleAgreementsForSubmitProof = (currentTimestamp, proofWindowDurationPerc, expectedAgreements) => async () => { const eligibleAgreements = await repositoryModuleManager.getEligibleAgreementsForSubmitProof( currentTimestamp, blockchain, proofWindowDurationPerc, ); assert(expect(eligibleAgreements).to.exist); expect(eligibleAgreements).to.be.instanceOf(Array); expect(eligibleAgreements).to.have.length(expectedAgreements.length); expect(eligibleAgreements).to.have.deep.members(expectedAgreements); // ensure order is correct for (let i = 0; i < eligibleAgreements.length; i += 1) { assert.strictEqual( eligibleAgreements[i].timeLeftInSubmitProofWindow, expectedAgreements[i].timeLeftInSubmitProofWindow, ); } }; it( 'returns no eligible service agreement at timestamp 49', testEligibleAgreementsForSubmitProof(49, 33, []), ); it( 'returns no eligible service agreement at timestamp 67', testEligibleAgreementsForSubmitProof(67, 33, []), ); it( 'returns no eligible service agreement at timestamp 80', testEligibleAgreementsForSubmitProof(80, 33, []), ); it( 'returns one eligible service agreements at timestamp 81', testEligibleAgreementsForSubmitProof(81, 33, [ { ...agreements[1], currentEpoch: 0, timeLeftInSubmitProofWindow: 33 }, ]), ); it( 'returns one eligible service agreements at timestamp 92', testEligibleAgreementsForSubmitProof(92, 33, [ { ...agreements[1], currentEpoch: 0, timeLeftInSubmitProofWindow: 22 }, ]), ); it( 'returns one eligible service agreements at timestamp 113', testEligibleAgreementsForSubmitProof(113, 33, [ { ...agreements[1], currentEpoch: 0, timeLeftInSubmitProofWindow: 1 }, ]), ); it( 'returns no eligible service agreements at timestamp 114', testEligibleAgreementsForSubmitProof(114, 33, []), ); it( 'returns no eligible service agreements at timestamp 167', testEligibleAgreementsForSubmitProof(167, 33, []), ); it( 'returns no eligible service agreements at timestamp 181', testEligibleAgreementsForSubmitProof(181, 33, []), ); it( 'returns no eligible service agreements at timestamp 192', testEligibleAgreementsForSubmitProof(192, 33, []), ); it( 'returns no eligible service agreements at timestamp 199', testEligibleAgreementsForSubmitProof(199, 33, []), ); it( 'returns no eligible service agreements at timestamp 200', testEligibleAgreementsForSubmitProof(200, 33, []), ); }); }); async function insertLoadTestAgreements(numAgreements) { let agreements = []; for (let tokenId = 0; tokenId < numAgreements; tokenId += 1) { agreements.push( createAgreement({ tokenId, startTime: Math.floor(Math.random() * 101), lastCommitEpoch: [null, 0][Math.floor(Math.random() * 3)], lastProofEpoch: [null, 0][Math.floor(Math.random() * 3)], }), ); if (agreements.length % 100_000 === 0) { // eslint-disable-next-line no-await-in-loop await repositoryModuleManager.bulkCreateServiceAgreementRecords(agreements); agreements = []; } } if (agreements.length) { await repositoryModuleManager.bulkCreateServiceAgreementRecords(agreements); } } describe.skip('test load', () => { describe('100_000 rows', () => { beforeEach(async function t() { this.timeout(0); await insertLoadTestAgreements(100_000); }); it('getEligibleAgreementsForSubmitCommit returns agreements in less than 100 ms', async () => { const start = performance.now(); await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( 100, blockchain, 25, ); const end = performance.now(); const duration = end - start; expect(duration).to.be.lessThan(100); }); it('getEligibleAgreementsForSubmitProof returns agreements in less than 100 ms', async () => { const start = performance.now(); await repositoryModuleManager.getEligibleAgreementsForSubmitProof( 100, blockchain, 33, ); const end = performance.now(); const duration = end - start; expect(duration).to.be.lessThan(100); }); }); describe('1_000_000 rows', () => { beforeEach(async function t() { this.timeout(0); await insertLoadTestAgreements(1_000_000); }); it('getEligibleAgreementsForSubmitCommit returns agreements in less than 1000 ms', async () => { const start = performance.now(); await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( 100, blockchain, 25, ); const end = performance.now(); const duration = end - start; expect(duration).to.be.lessThan(1000); }); it('getEligibleAgreementsForSubmitProof returns agreements in less than 1000 ms', async () => { const start = performance.now(); await repositoryModuleManager.getEligibleAgreementsForSubmitProof( 100, blockchain, 33, ); const end = performance.now(); const duration = end - start; expect(duration).to.be.lessThan(1000); }); }); }); }); ================================================ FILE: test/unit/modules/triple-store/config.json ================================================ { "modules": { "tripleStore": { "enabled": true, "implementation": { "ot-blazegraph": { "enabled": true, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", "config": { "repositories": { "privateCurrent": { "url": "http://localhost:9999", "name": "triple-store-test-private-current", "username": "admin", "password": "" }, "privateHistory": { "url": "http://localhost:9999", "name": "triple-store-test-private-history", "username": "admin", "password": "" }, "publicCurrent": { "url": "http://localhost:9999", "name": "triple-store-test-public-current", "username": "admin", "password": "" }, "publicHistory": { "url": "http://localhost:9999", "name": "triple-store-test-public-history", "username": "admin", "password": "" } } } } } } } } ================================================ FILE: test/unit/modules/triple-store/triple-store.test.js ================================================ /* import { describe, it, before, beforeEach } from 'mocha'; import chai from 'chai'; import { readFile } from 'fs/promises'; import { formatAssertion, calculateRoot } from 'assertion-tools'; import { TRIPLE_STORE_REPOSITORIES } from '../../../src/constants/constants.js'; import Logger from '../../../src/logger/logger.js'; import TripleStoreModuleManager from '../../../src/modules/triple-store/triple-store-module-manager.js'; import DataService from '../../../src/service/data-service.js'; import assertions from '../../assertions/assertions.js'; const { assert } = chai; let logger; let tripleStoreModuleManager; let dataService; const config = JSON.parse(await readFile('./test/modules/triple-store/config.json')); const implementationName = 'ot-blazegraph'; async function _insertAndGet(content) { const assertion = await formatAssertion(content); const assertionId = calculateRoot(assertion); await tripleStoreModuleManager.insertKnowledgeAssets( implementationName, TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, assertionId, assertion.join('\n'), ); const nquads = await tripleStoreModuleManager.getKnowledgeCollection( implementationName, TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, assertionId, ); const retrievedAssertion = await dataService.toNQuads(nquads, 'application/n-quads'); const retrievedAssertionId = calculateRoot(retrievedAssertion); assert.deepEqual(retrievedAssertion, assertion, `assertions are not equal`); assert.equal(retrievedAssertionId, assertionId, `assertion ids are not equal`); } describe('Triple store module', () => { before('Initialize logger', () => { logger = new Logger('trace'); logger.info = () => {}; }); beforeEach('Initialize triple store module manager', async () => { tripleStoreModuleManager = new TripleStoreModuleManager({ config, logger, }); await tripleStoreModuleManager.initialize(); const implementation = tripleStoreModuleManager.getImplementation(implementationName); await Promise.all( Object.keys(implementation.config.repositories).map((repository) => implementation.module.deleteRepository(repository), ), ); await tripleStoreModuleManager.initialize(); }); before('Initialize data service', async () => { dataService = new DataService({ logger, }); }); describe('Insert and get return same assertions:', async () => { for (const assertionName in assertions) { it(`${assertionName}`, () => _insertAndGet(assertions[assertionName])); } }); }); */ ================================================ FILE: test/unit/modules/validation/config.json ================================================ { "modules":{ "validation":{ "enabled":true, "implementation":{ "merkle-validation":{ "enabled":true, "package":"./validation/implementation/merkle-validation.js", "config":{} } } } } } ================================================ FILE: test/unit/modules/validation/validation-module-manager.test.js ================================================ import { describe, it, beforeEach } from 'mocha'; import { expect, assert } from 'chai'; import { readFile } from 'fs/promises'; import { calculateRoot } from 'assertion-tools'; import ValidationModuleManager from '../../../../src/modules/validation/validation-module-manager.js'; import Logger from '../../../../src/logger/logger.js'; let validationManager; const config = JSON.parse(await readFile('./test/unit/modules/validation/config.json', 'utf-8')); const assertion = [ { '@context': 'https://schema.org', '@id': 'https://tesla.modelX/2321', '@type': 'Car', name: 'Tesla Model X', brand: { '@type': 'Brand', name: 'Tesla', }, model: 'Model X', manufacturer: { '@type': 'Organization', name: 'Tesla, Inc.', }, fuelType: 'Electric', }, ]; const invalidValues = [null, undefined]; const hashFunctionId = 1; const keyword = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca0768e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3'; describe.only('Validation module manager', async () => { beforeEach('initialize validation module manage', async () => { validationManager = new ValidationModuleManager({ config, logger: new Logger(), }); validationManager.initialized = true; expect(await validationManager.initialize()).to.be.true; }); it('validates module name is as expected', async () => { const moduleName = await validationManager.getName(); expect(moduleName).to.equal('validation'); }); it('validate successful root hash calculation, expect to be matched', async () => { const expectedRootHash = await calculateRoot(assertion); const calculatedRootHash = await validationManager.calculateRoot(assertion); assert(expect(calculatedRootHash).to.exist); expect(calculatedRootHash).to.equal(expectedRootHash); }); it('root hash cannot be calculated without initialization', async () => { validationManager.initialized = false; try { await validationManager.calculateRoot(assertion); } catch (error) { expect(error.message).to.equal('Validation module is not initialized.'); } }); it('root hash calculation failed when assertion is null or undefined', async () => { invalidValues.forEach((value) => { expect(() => validationManager.calculateRoot(value)).to.throw( Error, 'Calculation failed: Assertion cannot be null or undefined.', ); }); }); it('successful getting merkle proof hash', async () => { const calculatedMerkleHash = await validationManager.getMerkleProof(assertion, 0); assert(expect(calculatedMerkleHash).to.exist); expect(calculatedMerkleHash).to.be.instanceof(Object); expect(calculatedMerkleHash).to.haveOwnProperty('leaf').and.to.be.a('string'); expect(calculatedMerkleHash).to.haveOwnProperty('proof').and.to.be.a('array'); }); it('merkle prof hash cannot be calculated without initialization', async () => { validationManager.initialized = false; try { await validationManager.getMerkleProof(assertion, 0); } catch (error) { expect(error.message).to.equal('Validation module is not initialized.'); } }); it('failed merkle proof hash calculation when assertion is null or undefined', async () => { for (const value of invalidValues) { // eslint-disable-next-line no-await-in-loop expect(await validationManager.getMerkleProof(value, 0)).to.be.rejectedWith( Error, 'Get merkle proof failed: Assertion cannot be null or undefined.', ); } }); it('validate getting function name', async () => { const getFnHashName = validationManager.getHashFunctionName(hashFunctionId); assert(expect(getFnHashName).to.exist); expect(getFnHashName).to.equal('sha256'); }); it('failed getting function name without initialization', async () => { validationManager.initialized = false; try { validationManager.getHashFunctionName(hashFunctionId); } catch (error) { expect(error.message).to.equal('Validation module is not initialized.'); } }); it('validate successful calling function name', async () => { const callFunction = await validationManager.callHashFunction(hashFunctionId, keyword); assert(expect(callFunction).to.exist); expect(callFunction).to.be.a('string'); expect(callFunction).to.equal( '0x5fe7425e0d956e2cafeac276c3ee8e70f377b2bd14790bc6d4777c3e7ba63b46', ); }); it('unsuccessful calling function name without initialization', async () => { validationManager.initialized = false; try { await validationManager.callHashFunction(hashFunctionId, keyword); } catch (error) { expect(error.message).to.equal('Validation module is not initialized.'); } }); it('failed function name initialization when function id is null or undefined', async () => { async function testInvalidValues() { for (const value of invalidValues) { try { // eslint-disable-next-line no-await-in-loop await validationManager.getMerkleProof(value, 0); } catch (error) { expect(error.message).to.equal( 'Get merkle proof failed: Assertion cannot be null or undefined.', ); } } } await testInvalidValues(); }); it('failed function name initialization when data is null or undefined', async () => { async function testInvalidValues() { for (const value of invalidValues) { try { // eslint-disable-next-line no-await-in-loop await validationManager.callHashFunction(value, 0); } catch (error) { expect(error.message).to.equal( 'Calling hash fn failed: Values cannot be null or undefined.', ); } } } await testInvalidValues(); }); }); ================================================ FILE: test/unit/service/auth-service.test.js ================================================ import 'dotenv/config'; import { expect } from 'chai'; import { describe, it, afterEach } from 'mocha'; import { v4 as uuid } from 'uuid'; import sinon from 'sinon'; import AuthService from '../../../src/service/auth-service.js'; import jwtUtil from '../../../src/service/util/jwt-util.js'; import RepositoryModuleManager from '../../../src/modules/repository/repository-module-manager.js'; const whitelistedIps = [ '::1', '127.0.0.1', '54.31.28.8', 'a3c6:3c39:492c:831b:d1a1:7944:b984:f32a', ]; const invalidIps = ['247.8.32.50', null, undefined, {}, [], true, false, 123, '123...', NaN]; const invalidTokens = ['token', 12345, {}, [], undefined, null, true, false, '123.321.32', NaN]; const tests = [ { ipAuthEnabled: false, tokenAuthEnabled: false, ipValid: false, tokenValid: false, tokenRevoked: false, tokenExpired: false, expected: true, }, { ipAuthEnabled: true, tokenAuthEnabled: false, ipValid: false, tokenValid: false, tokenRevoked: false, tokenExpired: false, expected: false, }, { ipAuthEnabled: true, tokenAuthEnabled: false, ipValid: true, tokenValid: false, tokenRevoked: false, tokenExpired: false, expected: true, }, { ipAuthEnabled: false, tokenAuthEnabled: true, ipValid: false, tokenValid: false, tokenRevoked: false, tokenExpired: false, expected: false, }, { ipAuthEnabled: false, tokenAuthEnabled: true, ipValid: true, tokenValid: false, tokenRevoked: false, tokenExpired: false, expected: false, }, { ipAuthEnabled: false, tokenAuthEnabled: true, ipValid: false, tokenValid: true, tokenRevoked: false, tokenExpired: false, expected: true, }, { ipAuthEnabled: false, tokenAuthEnabled: true, ipValid: false, tokenValid: true, tokenRevoked: true, tokenExpired: false, expected: false, }, { ipAuthEnabled: false, tokenAuthEnabled: true, ipValid: false, tokenValid: true, tokenRevoked: false, tokenExpired: true, expected: false, }, { ipAuthEnabled: false, tokenAuthEnabled: true, ipValid: false, tokenValid: true, tokenRevoked: true, tokenExpired: true, expected: false, }, { ipAuthEnabled: true, tokenAuthEnabled: true, ipValid: false, tokenValid: false, tokenRevoked: false, tokenExpired: false, expected: false, }, { ipAuthEnabled: true, tokenAuthEnabled: true, ipValid: true, tokenValid: false, tokenRevoked: false, tokenExpired: false, expected: false, }, { ipAuthEnabled: true, tokenAuthEnabled: true, ipValid: true, tokenValid: true, tokenRevoked: false, tokenExpired: false, expected: true, }, { ipAuthEnabled: true, tokenAuthEnabled: true, ipValid: true, tokenValid: true, tokenRevoked: true, tokenExpired: false, expected: false, }, { ipAuthEnabled: true, tokenAuthEnabled: true, ipValid: true, tokenValid: true, tokenRevoked: false, tokenExpired: true, expected: false, }, { ipAuthEnabled: true, tokenAuthEnabled: true, ipValid: true, tokenValid: true, tokenRevoked: true, tokenExpired: true, expected: false, }, ]; const configObj = { auth: { ipWhitelist: whitelistedIps, publicOperations: ['QUERY'], }, }; const getConfig = (ipAuthEnabled, tokenAuthEnabled) => { const configClone = JSON.parse(JSON.stringify(configObj)); configClone.auth.ipBasedAuthEnabled = ipAuthEnabled; configClone.auth.tokenBasedAuthEnabled = tokenAuthEnabled; return configClone; }; const getRepository = (isTokenRevoked, tokenAbilitiesValid) => sinon.createStubInstance(RepositoryModuleManager, { isTokenRevoked, getTokenAbilities: tokenAbilitiesValid ? ['QUERY', 'PUBLISH', 'SEARCH'] : [], }); const getIps = (isValid) => { if (isValid) { return whitelistedIps; } return invalidIps; }; const getTokens = (isValid, isExpired) => { if (isValid) { if (isExpired) { return [jwtUtil.generateJWT(uuid(), '-2d')]; } return [jwtUtil.generateJWT(uuid())]; } return invalidTokens; }; describe('authenticate()', async () => { afterEach(() => { sinon.restore(); }); for (const t of tests) { let testText = ''; for (const field in t) { if (field === 'expected') { testText += `${field.toUpperCase()}: ${t[field]}`; } else { testText += `${field}: ${t[field]} | `; } } it(testText, async () => { const config = getConfig(t.ipAuthEnabled, t.tokenAuthEnabled); const repositoryModuleManager = getRepository(t.tokenRevoked); const ips = getIps(t.ipValid); const tokens = getTokens(t.tokenValid, t.tokenExpired); const authService = new AuthService({ config, repositoryModuleManager }); for (const ip of ips) { for (const token of tokens) { // eslint-disable-next-line no-await-in-loop const isAuthenticated = await authService.authenticate(ip, token); expect(isAuthenticated).to.be.equal(t.expected); } } }); } it('returns false if token is valid but is not found in the database', async () => { const config = getConfig(false, true); const repositoryModuleManager = getRepository(null, true); const [token] = getTokens(true); const authService = new AuthService({ config, repositoryModuleManager }); const isAuthenticated = await authService.authenticate('', token); expect(isAuthenticated).to.be.false; }); }); describe('isAuthorized()', async () => { afterEach(() => { sinon.restore(); }); it('returns true if tokenBasedAuthentication is disabled', async () => { const config = getConfig(false, false); const authService = new AuthService({ config }); const isAuthorized = await authService.isAuthorized(null, null); expect(isAuthorized).to.be.equal(true); }); it('returns true if user has ability to perform an action', async () => { const config = getConfig(false, true); const repositoryModuleManager = getRepository(false, true); const jwt = jwtUtil.generateJWT(uuid()); const authService = new AuthService({ config, repositoryModuleManager }); const isAuthorized = await authService.isAuthorized(jwt, 'QUERY'); expect(isAuthorized).to.be.equal(true); }); it("returns false if user doesn't have ability to perform an action", async () => { const config = getConfig(false, true); const jwt = jwtUtil.generateJWT(uuid()); const authService = new AuthService({ config, repositoryModuleManager: getRepository(false, true), }); const isAuthorized = await authService.isAuthorized(jwt, 'OPERATION'); expect(isAuthorized).to.be.equal(false); }); it('returns false if user roles are not found', async () => { const config = getConfig(false, true); const jwt = jwtUtil.generateJWT(uuid()); const authService = new AuthService({ config, repositoryModuleManager: getRepository(false, false), }); const isAuthorized = await authService.isAuthorized(jwt, 'PUBLISH'); expect(isAuthorized).to.be.equal(false); }); }); describe('isPublicOperation()', async () => { afterEach(() => { sinon.restore(); }); it('returns true if route is public', async () => { const config = getConfig(false, false); const authService = new AuthService({ config }); const isPublic = authService.isPublicOperation('QUERY'); expect(isPublic).to.be.equal(true); }); it('returns false if route is not public', async () => { const config = getConfig(false, false, true); const authService = new AuthService({ config }); const isPublic = authService.isPublicOperation('PUBLISH'); expect(isPublic).to.be.equal(false); }); it('returns false if public routes are not defined', async () => { const config = getConfig(false, false, true); config.auth.publicOperations = undefined; const authService = new AuthService({ config }); const isPublic = authService.isPublicOperation('PUBLISH'); expect(isPublic).to.be.equal(false); }); }); ================================================ FILE: test/unit/service/get-service.test.js ================================================ import { beforeEach, afterEach, describe, it } from 'mocha'; import { expect } from 'chai'; import sinon from 'sinon'; import { OPERATION_REQUEST_STATUS } from '../../../src/constants/constants.js'; import RepositoryModuleManagerMock from '../mock/repository-module-manager-mock.js'; import ValidationModuleManagerMock from '../mock/validation-module-manager-mock.js'; import BlockchainModuleManagerMock from '../mock/blockchain-module-manager-mock.js'; import OperationIdServiceMock from '../mock/operation-id-service-mock.js'; import CommandExecutorMock from '../mock/command-executor-mock.js'; import GetService from '../../../src/service/get-service.js'; import Logger from '../../../src/logger/logger.js'; let getService; let cacheOperationIdDataSpy; let commandExecutorAddSpy; describe('Get service test', async () => { beforeEach(() => { const repositoryModuleManagerMock = new RepositoryModuleManagerMock(); getService = new GetService({ repositoryModuleManager: repositoryModuleManagerMock, operationIdService: new OperationIdServiceMock({ repositoryModuleManager: repositoryModuleManagerMock, }), commandExecutor: new CommandExecutorMock(), validationModuleManager: new ValidationModuleManagerMock(), blockchainModuleManager: new BlockchainModuleManagerMock(), logger: new Logger(), }); cacheOperationIdDataSpy = sinon.spy(getService.operationIdService, 'cacheOperationIdData'); commandExecutorAddSpy = sinon.spy(getService.commandExecutor, 'add'); }); afterEach(() => { cacheOperationIdDataSpy.restore(); commandExecutorAddSpy.restore(); }); it('Completed get completes with low ACK ask', async () => { await getService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [], keyword: 'origintrail', batchSize: 10, minAckResponses: 1, }, }, OPERATION_REQUEST_STATUS.COMPLETED, { nquads: ' .', }, ); const returnedResponses = getService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( cacheOperationIdDataSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', { assertion: ' .', }), ).to.be.true; expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.COMPLETED, ).to.be.true; }); it('Completed get leads to scheduling operation for leftover nodes and status stays same', async () => { await getService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [1, 2, 3, 4], keyword: 'origintrail', batchSize: 10, minAckResponses: 12, }, }, OPERATION_REQUEST_STATUS.COMPLETED, {}, ); const returnedResponses = getService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( commandExecutorAddSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', [1, 2, 3, 4]), ); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.COMPLETED, ).to.be.true; }); }); ================================================ FILE: test/unit/service/operation-id-service-cache.test.js ================================================ import { describe, it, beforeEach, afterEach } from 'mocha'; import { expect } from 'chai'; import fs from 'fs/promises'; import path from 'path'; import os from 'os'; import OperationIdService from '../../../src/service/operation-id-service.js'; describe('OperationIdService file cache cleanup', () => { let tmpDir; let service; beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'opid-cache-')); const now = Date.now(); // Older than TTL (2 hours) const oldFile = path.join(tmpDir, 'old.json'); await fs.writeFile(oldFile, '{}'); await fs.utimes( oldFile, new Date(now - 2 * 60 * 60 * 1000), new Date(now - 2 * 60 * 60 * 1000), ); // Newer than TTL (10 minutes) const newFile = path.join(tmpDir, 'new.json'); await fs.writeFile(newFile, '{}'); await fs.utimes(newFile, new Date(now - 10 * 60 * 1000), new Date(now - 10 * 60 * 1000)); const fileService = { getOperationIdCachePath: () => tmpDir, async pathExists(p) { try { await fs.stat(p); return true; } catch { return false; } }, readDirectory: (p) => fs.readdir(p), stat: (p) => fs.stat(p), removeFile: (p) => fs.rm(p, { force: true }), }; service = new OperationIdService({ logger: { debug: () => {}, warn: () => {}, error: () => {} }, fileService, repositoryModuleManager: {}, eventEmitter: { emit: () => {} }, }); }); afterEach(async () => { await fs.rm(tmpDir, { recursive: true, force: true }); }); it('removes only files older than TTL', async () => { const deleted = await service.removeExpiredOperationIdFileCache(60 * 60 * 1000, 10); const remainingFiles = await fs.readdir(tmpDir); expect(deleted).to.equal(1); expect(remainingFiles).to.deep.equal(['new.json']); }); }); ================================================ FILE: test/unit/service/operation-service.test.js ================================================ import { beforeEach, afterEach, describe, it } from 'mocha'; import { expect } from 'chai'; import sinon from 'sinon'; import { OPERATION_REQUEST_STATUS } from '../../../src/constants/constants.js'; import RepositoryModuleManagerMock from '../mock/repository-module-manager-mock.js'; import ValidationModuleManagerMock from '../mock/validation-module-manager-mock.js'; import BlockchainModuleManagerMock from '../mock/blockchain-module-manager-mock.js'; import OperationIdServiceMock from '../mock/operation-id-service-mock.js'; import CommandExecutorMock from '../mock/command-executor-mock.js'; import PublishService from '../../../src/service/publish-service.js'; import Logger from '../../../src/logger/logger.js'; let publishService; let cacheOperationIdDataSpy; let commandExecutorAddSpy; describe('Operation service test', async () => { beforeEach(() => { const repositoryModuleManagerMock = new RepositoryModuleManagerMock(); publishService = new PublishService({ repositoryModuleManager: repositoryModuleManagerMock, operationIdService: new OperationIdServiceMock({ repositoryModuleManager: repositoryModuleManagerMock, }), commandExecutor: new CommandExecutorMock(), validationModuleManager: new ValidationModuleManagerMock(), blockchainModuleManager: new BlockchainModuleManagerMock(), logger: new Logger(), }); cacheOperationIdDataSpy = sinon.spy( publishService.operationIdService, 'cacheOperationIdData', ); commandExecutorAddSpy = sinon.spy(publishService.commandExecutor, 'add'); }); afterEach(() => { cacheOperationIdDataSpy.restore(); commandExecutorAddSpy.restore(); }); it('Creates a response record and returns status for each keyword', async () => { await publishService.getResponsesStatuses( OPERATION_REQUEST_STATUS.FAILED, null, '5195d01a-b437-4aae-b388-a77b9fa715f1', 'origintrail', ); const returnedResponses = await publishService.getResponsesStatuses( OPERATION_REQUEST_STATUS.COMPLETED, null, '5195d01a-b437-4aae-b388-a77b9fa715f1', 'origintrail', ); // Do two calls to make sure the state has persisted after the first one expect(returnedResponses).to.deep.equal({ origintrail: { failedNumber: 1, completedNumber: 1 }, }); }); }); ================================================ FILE: test/unit/service/publish-service.test.js ================================================ import { beforeEach, afterEach, describe, it } from 'mocha'; import { expect } from 'chai'; import sinon from 'sinon'; import { OPERATION_REQUEST_STATUS } from '../../../src/constants/constants.js'; import RepositoryModuleManagerMock from '../mock/repository-module-manager-mock.js'; import ValidationModuleManagerMock from '../mock/validation-module-manager-mock.js'; import BlockchainModuleManagerMock from '../mock/blockchain-module-manager-mock.js'; import OperationIdServiceMock from '../mock/operation-id-service-mock.js'; import CommandExecutorMock from '../mock/command-executor-mock.js'; import PublishService from '../../../src/service/publish-service.js'; import Logger from '../../../src/logger/logger.js'; let publishService; let cacheOperationIdDataSpy; let commandExecutorAddSpy; describe('Publish service test', async () => { beforeEach(() => { const repositoryModuleManagerMock = new RepositoryModuleManagerMock(); publishService = new PublishService({ repositoryModuleManager: repositoryModuleManagerMock, operationIdService: new OperationIdServiceMock({ repositoryModuleManager: repositoryModuleManagerMock, }), commandExecutor: new CommandExecutorMock(), validationModuleManager: new ValidationModuleManagerMock(), blockchainModuleManager: new BlockchainModuleManagerMock(), logger: new Logger(), }); cacheOperationIdDataSpy = sinon.spy( publishService.operationIdService, 'cacheOperationIdData', ); commandExecutorAddSpy = sinon.spy(publishService.commandExecutor, 'add'); }); afterEach(() => { cacheOperationIdDataSpy.restore(); commandExecutorAddSpy.restore(); }); it('Completed publish completes with low ACK ask', async () => { await publishService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [], keyword: 'origintrail', batchSize: 10, minAckResponses: 1, }, }, OPERATION_REQUEST_STATUS.COMPLETED, {}, ); const returnedResponses = publishService.repositoryModuleManager.getAllResponseStatuses(); expect(cacheOperationIdDataSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', {})).to.be .false; expect(returnedResponses.length).to.be.equal(2); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.COMPLETED, ).to.be.true; }); it('Completed publish fails with high ACK ask', async () => { await publishService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [], keyword: 'origintrail', batchSize: 10, minAckResponses: 12, }, }, OPERATION_REQUEST_STATUS.COMPLETED, {}, ); const returnedResponses = publishService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.FAILED, ).to.be.true; }); it('Failed publish fails with low ACK ask', async () => { await publishService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [], keyword: 'origintrail', batchSize: 10, minAckResponses: 1, }, }, OPERATION_REQUEST_STATUS.FAILED, {}, ); const returnedResponses = publishService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.FAILED, ).to.be.true; }); it('Completed publish leads to scheduling operation for leftover nodes and status stays same', async () => { await publishService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [1, 2, 3, 4], keyword: 'origintrail', batchSize: 10, minAckResponses: 12, }, }, OPERATION_REQUEST_STATUS.COMPLETED, {}, ); const returnedResponses = publishService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( commandExecutorAddSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', [1, 2, 3, 4]), ); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.COMPLETED, ).to.be.true; }); }); ================================================ FILE: test/unit/service/sharding-table-service.test.js ================================================ import { beforeEach, describe, it } from 'mocha'; import { expect } from 'chai'; import ShardingTableService from '../../../src/service/sharding-table-service.js'; import BlockchainModuleManagerMock from '../mock/blockchain-module-manager-mock.js'; import RepositoryModuleManagerMock from '../mock/repository-module-manager-mock.js'; import NetworkModuleManagerMock from '../mock/network-module-manager-mock.js'; import ValidationModuleManagerMock from '../mock/validation-module-manager-mock.js'; import EventEmitterMock from '../mock/event-emitter-mock.js'; import { BYTES_IN_KILOBYTE } from '../../../src/constants/constants.js'; let shardingTableService; describe('Sharding table service test', async () => { beforeEach(() => { shardingTableService = new ShardingTableService({ blockchainModuleManager: new BlockchainModuleManagerMock(), repositoryModuleManager: new RepositoryModuleManagerMock(), networkModuleManager: new NetworkModuleManagerMock(), validationModuleManager: new ValidationModuleManagerMock(), eventEmitter: new EventEmitterMock(), }); }); it('Get bid suggestion, returns bid suggestion successfully', async () => { const epochsNumber = 5; const assertionSize = BYTES_IN_KILOBYTE; const contentAssetStorageAddress = '0xABd59A9aa71847F499d624c492d3903dA953d67a'; const firstAssertionId = '0xb44062de45333119471934bc0340c05ff09c0b463392384bc2030cd0a20c334b'; const hashFunctionId = 1; const bidSuggestions = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, assertionSize, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); expect(bidSuggestions).to.be.equal('3788323225298705400'); }); it('Get bid suggestion, returns valid value for assertion size 1b and ask 1 wei', async () => { const epochsNumber = 5; const contentAssetStorageAddress = '0xABd59A9aa71847F499d624c492d3903dA953d67a'; const firstAssertionId = '0xb44062de45333119471934bc0340c05ff09c0b463392384bc2030cd0a20c334b'; const hashFunctionId = 1; const askInWei = '0.000000000000000001'; const peers = shardingTableService.repositoryModuleManager.getAllPeerRecords(); shardingTableService.repositoryModuleManager.getAllPeerRecords = () => { peers.forEach((peer) => { // eslint-disable-next-line no-param-reassign peer.ask = askInWei; }); return peers; }; const bidSuggestion1B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, 1, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); expect(bidSuggestion1B).to.be.equal('15'); const bidSuggestion10B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, 10, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); expect(bidSuggestion10B).to.be.equal('15'); const bidSuggestion1024B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, 1024, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); expect(bidSuggestion1024B).to.be.equal('15'); const bidSuggestion2048B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, 2048, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); expect(bidSuggestion2048B).to.be.equal('30'); }); }); ================================================ FILE: test/unit/service/update-service.test.js ================================================ import { beforeEach, afterEach, describe, it } from 'mocha'; import { expect } from 'chai'; import sinon from 'sinon'; import { OPERATION_REQUEST_STATUS } from '../../../src/constants/constants.js'; import RepositoryModuleManagerMock from '../mock/repository-module-manager-mock.js'; import ValidationModuleManagerMock from '../mock/validation-module-manager-mock.js'; import BlockchainModuleManagerMock from '../mock/blockchain-module-manager-mock.js'; import OperationIdServiceMock from '../mock/operation-id-service-mock.js'; import CommandExecutorMock from '../mock/command-executor-mock.js'; import UpdateService from '../../../src/service/update-service.js'; import Logger from '../../../src/logger/logger.js'; let updateService; let cacheOperationIdDataSpy; let commandExecutorAddSpy; describe('Update service test', async () => { beforeEach(() => { const repositoryModuleManagerMock = new RepositoryModuleManagerMock(); updateService = new UpdateService({ repositoryModuleManager: repositoryModuleManagerMock, operationIdService: new OperationIdServiceMock({ repositoryModuleManager: repositoryModuleManagerMock, }), commandExecutor: new CommandExecutorMock(), validationModuleManager: new ValidationModuleManagerMock(), blockchainModuleManager: new BlockchainModuleManagerMock(), logger: new Logger(), }); cacheOperationIdDataSpy = sinon.spy( updateService.operationIdService, 'cacheOperationIdData', ); commandExecutorAddSpy = sinon.spy(updateService.commandExecutor, 'add'); }); afterEach(() => { cacheOperationIdDataSpy.restore(); commandExecutorAddSpy.restore(); }); it('Completed update completes with low ACK ask', async () => { await updateService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [], keyword: 'origintrail', batchSize: 10, minAckResponses: 1, }, }, OPERATION_REQUEST_STATUS.COMPLETED, {}, ); const returnedResponses = updateService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect(cacheOperationIdDataSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', {})).to.be .false; expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.COMPLETED, ).to.be.true; }); it('Completed update fails with high ACK ask', async () => { await updateService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [], keyword: 'origintrail', batchSize: 10, minAckResponses: 12, }, }, OPERATION_REQUEST_STATUS.COMPLETED, {}, ); const returnedResponses = updateService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.FAILED, ).to.be.true; }); it('Failed update fails with low ACK ask', async () => { await updateService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [], keyword: 'origintrail', batchSize: 10, minAckResponses: 1, }, }, OPERATION_REQUEST_STATUS.FAILED, {}, ); const returnedResponses = updateService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.FAILED, ).to.be.true; }); it('Completed update leads to scheduling operation for leftover nodes and status stays same', async () => { await updateService.processResponse( { data: { operationId: '5195d01a-b437-4aae-b388-a77b9fa715f1', blockchain: 'hardhat', numberOfFoundNodes: 1, leftoverNodes: [1, 2, 3, 4], keyword: 'origintrail', batchSize: 10, minAckResponses: 12, }, }, OPERATION_REQUEST_STATUS.COMPLETED, {}, ); const returnedResponses = updateService.repositoryModuleManager.getAllResponseStatuses(); expect(returnedResponses.length).to.be.equal(2); expect( commandExecutorAddSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', [1, 2, 3, 4]), ); expect( returnedResponses[returnedResponses.length - 1].status === OPERATION_REQUEST_STATUS.COMPLETED, ).to.be.true; }); }); ================================================ FILE: test/unit/service/util/jwt-util.test.js ================================================ import 'dotenv/config'; import { v4 as uuid } from 'uuid'; import { expect } from 'chai'; import { describe, it } from 'mocha'; import jwtUtil from '../../../../src/service/util/jwt-util.js'; const getPayload = (token) => { const b64Payload = token.split('.')[1]; return JSON.parse(Buffer.from(b64Payload, 'base64').toString()); }; const nonJwts = [ '123', 214214124124, 'header.payload.signature', null, undefined, true, false, {}, [], ]; describe('Auth JWT generation', async () => { it('generates JWT token with tokenId payload', () => { const tokenId = uuid(); const token = jwtUtil.generateJWT(tokenId); expect(token).not.be.null; expect(getPayload(token).jti).to.be.equal(tokenId); }); it('generates null if invalid tokenId is provided', () => { const nonUuids = [true, false, undefined, null, {}, [], 'string', 123]; for (const val of nonUuids) { const token = jwtUtil.generateJWT(val); expect(token).to.be.null; } }); it('generates payload without expiration date if expiresIn argument is not provided', () => { const token = jwtUtil.generateJWT(uuid()); expect(getPayload(token).exp).to.be.undefined; }); it('generates payload with expiration date if expiresIn argument is provided', () => { const token = jwtUtil.generateJWT(uuid(), '2d'); expect(getPayload(token).exp).to.be.ok; }); }); describe('JWT token validation', async () => { it('returns true if JWT is valid', async () => { const token = jwtUtil.generateJWT(uuid()); expect(jwtUtil.validateJWT(token)).to.be.true; }); it('returns true if JWT is expired', async () => { const token = jwtUtil.generateJWT(uuid(), '-2d'); expect(jwtUtil.validateJWT(token)).to.be.false; }); it('returns false if JWT is not valid', async () => { const token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c'; expect(jwtUtil.validateJWT(token)).to.be.false; }); it('returns false if non JWT value is passed', async () => { for (const val of nonJwts) { expect(jwtUtil.validateJWT(val)).to.be.false; } }); }); describe('JWT payload extracting', async () => { it('returns JWT payload if valid token is provided', async () => { const token = jwtUtil.generateJWT(uuid()); expect(jwtUtil.getPayload(token)).to.be.ok; }); it('returns null if invalid token is provided', async () => { for (const val of nonJwts) { expect(jwtUtil.getPayload(val)).to.be.null; } }); }); describe('JWT decoding', async () => { it('returns decoded JWT (header, payload, signature) if valid token is provided', async () => { const token = jwtUtil.generateJWT(uuid()); expect(jwtUtil.decode(token).header).to.be.ok; expect(jwtUtil.decode(token).payload).to.be.ok; expect(jwtUtil.decode(token).signature).to.be.ok; }); it('returns null if invalid token is provided', async () => { for (const val of nonJwts) { expect(jwtUtil.decode(val)).to.be.null; } }); }); ================================================ FILE: test/unit/service/validation-service.test.js ================================================ import { beforeEach, describe, it } from 'mocha'; import { expect } from 'chai'; import ValidationModuleManagerMock from '../mock/validation-module-manager-mock.js'; import BlockchainModuleManagerMock from '../mock/blockchain-module-manager-mock.js'; import ValidationService from '../../../src/service/validation-service.js'; import Logger from '../../../src/logger/logger.js'; let validationService; describe('Validation service test', async () => { beforeEach(() => { validationService = new ValidationService({ validationModuleManager: new ValidationModuleManagerMock(), blockchainModuleManager: new BlockchainModuleManagerMock(), logger: new Logger(), config: { maximumAssertionSizeInKb: 2500, }, }); }); it('Validates assertion correctly', async () => { let errorThrown = false; try { await validationService.validateAssertion( '0xde58cc52a5ce3a04ae7a05a13176226447ac02489252e4d37a72cbe0aea46b42', 'hardhat', { '@context': 'https://schema.org', '@id': 'https://tesla.modelX/2321', '@type': 'Car', name: 'Tesla Model X', brand: { '@type': 'Brand', name: 'Tesla', }, model: 'Model X', manufacturer: { '@type': 'Organization', name: 'Tesla, Inc.', }, fuelType: 'Electric', }, ); } catch (error) { errorThrown = true; } expect(errorThrown).to.be.false; }); it('Tries to validate assertion but fails due to assertion size mismatch', async () => { // todo after corrective component is implemented, update this logic // let errorThrown = false; // try { // await validationService.validateAssertion( // '0xde58cc52a5ce3a04ae7a05a13176226447ac02489252e4d37a72cbe0aea46b42', // 'hardhat', // { // '@context': 'https://schema.org', // '@id': 'https://tesla.modelX/2321', // '@type': 'Car', // name: 'Tesla Model X', // }, // ); // } catch (error) { // errorThrown = true; // } // expect(errorThrown).to.be.true; }); it('Tries to validate assertion but fails due to triple number mismatch', async () => { validationService.blockchainModuleManager.getAssertionTriplesNumber = ( blockchain, assertionId, ) => 5; // Will lead to mismatch with assertion calculated value let errorThrown = false; try { await validationService.validateAssertion( '0xde58cc52a5ce3a04ae7a05a13176226447ac02489252e4d37a72cbe0aea46b42', 'hardhat', { '@context': 'https://schema.org', '@id': 'https://tesla.modelX/2321', '@type': 'Car', name: 'Tesla Model X', brand: { '@type': 'Brand', name: 'Tesla', }, model: 'Model X', manufacturer: { '@type': 'Organization', name: 'Tesla, Inc.', }, fuelType: 'Electric', }, ); } catch (error) { errorThrown = true; } expect(errorThrown).to.be.true; }); it('Tries to validate assertion but fails due to chunk number mismatch', async () => { validationService.blockchainModuleManager.getAssertionChunksNumber = ( blockchain, assertionId, ) => 5; // Will lead to mismatch with assertion chunk number calculated value let errorThrown = false; try { await validationService.validateAssertion( '0xde58cc52a5ce3a04ae7a05a13176226447ac02489252e4d37a72cbe0aea46b42', 'hardhat', { '@context': 'https://schema.org', '@id': 'https://tesla.modelX/2321', '@type': 'Car', name: 'Tesla Model X', brand: { '@type': 'Brand', name: 'Tesla', }, model: 'Model X', manufacturer: { '@type': 'Organization', name: 'Tesla, Inc.', }, fuelType: 'Electric', }, ); } catch (error) { errorThrown = true; } expect(errorThrown).to.be.true; }); it('Tries to validate assertion but fails due to validation manager returning wrong assertion id', async () => { // todo after corrective component is implemented, update this logic // Will lead to mismatch with passed assertion id // validationService.validationModuleManager.calculateRoot = (assertion) => ''; // // let errorThrown = false; // try { // await validationService.validateAssertion( // '0xde58cc52a5ce3a04ae7a05a13176226447ac02489252e4d37a72cbe0aea46b42', // 'hardhat', // { // '@context': 'https://schema.org', // '@id': 'https://tesla.modelX/2321', // '@type': 'Car', // name: 'Tesla Model X', // brand: { // '@type': 'Brand', // name: 'Tesla', // }, // model: 'Model X', // manufacturer: { // '@type': 'Organization', // name: 'Tesla, Inc.', // }, // fuelType: 'Electric', // }, // ); // } catch (error) { // errorThrown = true; // } // expect(errorThrown).to.be.true; }); }); ================================================ FILE: test/unit/sparlql-query-service.test.js ================================================ // /* eslint-disable */ // import Blazegraph from '../../src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js'; // import GraphDB from '../../src/modules/triple-store/implementation/ot-graphdb/ot-graphdb.js'; // import Fuseki from '../../src/modules/triple-store/implementation/ot-fuseki/ot-fuseki.js'; // const { // describe, // it, // before, // after, // } = require('mocha'); // import chai from 'chai'; // const { // assert, // expect, // } = chai; // import Logger from '../../src/logger/logger'; // import fs from 'fs'; // let sparqlService = null; // let logger = null; // chai.use(require('chai-as-promised')); // this.makeId = function (length) { // let result = ''; // const characters = 'abcdefghijklmnopqrstuvwxyz0123456789'; // const charactersLength = characters.length; // for (let i = 0; i < length; i++) { // result += characters.charAt(Math.floor(Math.random() * // charactersLength)); // } // return result; // }; // describe('Sparql module', () => { // before('Initialize Logger', async () => { // logger = new Logger('trace', false); // }); // before('Init Sparql Module', async () => { // const configFile = JSON.parse(fs.readFileSync('.origintrail_noderc.tests')); // const config = configFile.graphDatabase; // assert.isNotNull(config.url); // assert.isNotEmpty(config.url); // assert.isNotNull(config.name); // assert.isNotEmpty(config.name); // switch (config.implementation) { // case 'Blazegraph': // sparqlService = new Blazegraph(config); // break; // case 'GraphDB': // sparqlService = new GraphDB(config); // break; // case 'Fuseki': // sparqlService = new Fuseki(config); // break; // default: // throw Error('Unknown graph database implementation') // } // await sparqlService.initialize(logger); // }); // it('Check for cleanup', async () => { // // Success // expect(sparqlService.cleanEscapeCharacter('keywordabc')) // .to // .equal('keywordabc'); // // Fail // expect(sparqlService.cleanEscapeCharacter('keywordabc\'')) // .to // .equal('keywordabc\\\''); // }); // it('Check limit creation', async () => { // // Success // expect(() => sparqlService.createLimitQuery({ limit: 'abc' })) // .to // .throw(Error); // expect(() => sparqlService.createLimitQuery({ limit: Math.random() })) // .to // .throw(Error); // expect(sparqlService.createLimitQuery({})) // .to // .equal(''); // // var randomnumber = Math.floor(Math.random() * (maximum - minimum + 1)) + minimum; // // eslint-disable-next-line no-bitwise // const random = (Math.random() * (99999999 - 1 + 1)) << 0; // const negativeRandom = random * -1; // expect(sparqlService.createLimitQuery({ limit: random })) // .to // .equal(`LIMIT ${random}`); // expect(() => sparqlService.createLimitQuery({ limit: negativeRandom })) // .to // .throw(Error); // }); // it('Check FindAssertionsByKeyword Errorhandling', async () => { // await expect(sparqlService.findAssertionsByKeyword('abc', { limit: 'aaaa' }, false)) // .to // .be // .rejectedWith(Error); // await expect(sparqlService.findAssertionsByKeyword('abc', { limit: '90' }, 'test')) // .to // .be // .rejectedWith(Error); // await expect(sparqlService.findAssertionsByKeyword('abc', { // limit: '90', // prefix: 'test', // })) // .to // .be // .rejectedWith(Error); // }); // it('Check FindAssertionsByKeyword functionality', async () => { // let id = this.makeId(65); // const addTriple = await sparqlService.insert(` schema:hasKeywords "${id}" `, `did:dkg:${id}`); // // This can also be mocked if necessary // const test = await sparqlService.findAssertionsByKeyword(id, { // limit: 5, // prefix: true, // }, true); // expect(test) // .to // .be // .not // .empty; // const testTwo = await sparqlService.findAssertionsByKeyword(id, { // limit: 5, // prefix: false, // }, true); // // eslint-disable-next-line no-unused-expressions // expect(testTwo) // .to // .be // .not // .empty; // }) // .timeout(600000); // it('Check createFilterParameter', async () => { // expect(sparqlService.createFilterParameter('', '')) // .to // .equal(''); // expect(sparqlService.createFilterParameter('\'', '')) // .to // .equal(''); // expect(sparqlService.createFilterParameter('\'', sparqlService.filtertype.KEYWORD)) // .to // .equal('FILTER (lcase(?keyword) = \'\\\'\')'); // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.KEYWORD)) // .to // .equal('FILTER (lcase(?keyword) = \'abcd\')'); // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.KEYWORDPREFIX)) // .to // .equal('FILTER contains(lcase(?keyword),\'abcd\')'); // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.TYPES)) // .to // .equal('FILTER (?type IN (abcd))'); // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.ISSUERS)) // .to // .equal('FILTER (?issuer IN (abcd))'); // }); // it('Check FindAssetsByKeyword functionality', async () => { // //Add new entry, so we can check if we find it really // let id = this.makeId(65); // let triples = ` // schema:hasKeywords "${id}" . // schema:hasTimestamp "2022-04-18T06:48:05.123Z" . // schema:hasUALs "${id}" . // schema:hasIssuer "${id}" . // schema:hasType "${id}" . // `; // const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); // expect(addTriple) // .to // .be // .true; // const testContains = await sparqlService.findAssetsByKeyword(id.substring(1, 20), { // limit: 5, // prefix: true, // }, true); // // eslint-disable-next-line no-unused-expressions // expect(testContains) // .to // .be // .not // .empty; // const testExact = await sparqlService.findAssetsByKeyword(id, { // limit: 5, // prefix: true, // }, true); // // eslint-disable-next-line no-unused-expressions // expect(testExact) // .to // .be // .not // .empty; // }) // .timeout(600000); // it('Check resolve functionality', async () => { // // This can also be mocked if necessary // const test = await sparqlService.resolve('0e62550721611b96321c7459e7790498240431025e46fce9cd99f2ea9763ffb0'); // // eslint-disable-next-line no-unused-expressions // expect(test) // .to // .be // .not // .null; // }) // .timeout(600000); // it('Check insert functionality', async () => { // // This can also be mocked if necessary // let id = this.makeId(65); // const test = await sparqlService.insert(` schema:hasKeywords "${id}" `, `did:dkg:${id}`); // // eslint-disable-next-line no-unused-expressions // expect(test) // .to // .be // .true; // }) // .timeout(600000); // it('Check assertions By Asset functionality', async () => { // // This can also be mocked if necessary // let id = this.makeId(65); // let triples = ` // schema:hasKeywords "${id}" . // schema:hasTimestamp "2022-04-18T06:48:05.123Z" . // schema:hasUALs "${id}" . // schema:hasIssuer "${id}" . // schema:hasType "${id}" . // `; // const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); // expect(addTriple) // .to // .be // .true; // const testExact = await sparqlService.assertionsByAsset(id, { // limit: 5, // prefix: true, // }, true); // // eslint-disable-next-line no-unused-expressions // expect(testExact) // .to // .be // .not // .empty; // }) // .timeout(600000); // it('Check find Assertions functionality', async () => { // // This can also be mocked if necessary // let id = this.makeId(65); // let triples = ` // schema:hasKeywords "${id}" . // schema:hasTimestamp "2022-04-18T06:48:05.123Z" . // schema:hasUALs "${id}" . // schema:hasIssuer "${id}" . // schema:hasType "${id}" . // `; // const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); // expect(addTriple) // .to // .be // .true; // const testExact = await sparqlService.findAssertions(triples); // // eslint-disable-next-line no-unused-expressions // expect(testExact) // .to // .be // .not // .empty; // }) // .timeout(600000); // }); ================================================ FILE: test/utilities/dkg-client-helper.mjs ================================================ import DKG from 'dkg.js'; import { CONTENT_ASSET_HASH_FUNCTION_ID } from '../../src/constants/constants.js'; class DkgClientHelper { constructor(config) { this.client = new DKG(config); } async info() { return this.client.node.info(); } async publish(data, userOptions = {}) { const defaultOptions = { visibility: 'public', epochsNum: 5, hashFunctionId: CONTENT_ASSET_HASH_FUNCTION_ID, minimumNumberOfNodeReplications: 1, minimumNumberOfFinalizationConfirmations: 0, }; const options = { ...defaultOptions, ...userOptions }; return this.client.asset.create(data, options); } async get(ual, state, userOptions = {}) { const defaultOptions = { state, validate: true, }; const options = { ...defaultOptions, ...userOptions }; return this.client.asset.get(ual, options); } async query(query) { return this.client.query(query); } } export default DkgClientHelper; ================================================ FILE: test/utilities/http-api-helper.mjs ================================================ import { setTimeout } from 'timers/promises'; import axios from 'axios'; const TERMINAL_STATUSES = ['COMPLETED', 'FAILED']; class HttpApiHelper { async info(nodeRpcUrl) { return this._sendRequest('get', `${nodeRpcUrl}/info`); } async get(nodeRpcUrl, requestBody) { return this._sendRequest('post', `${nodeRpcUrl}/get`, requestBody); } async getOperationResult(nodeRpcUrl, operationName, operationId) { return this._sendRequest('get', `${nodeRpcUrl}/${operationName}/${operationId}`); } async publish(nodeRpcUrl, requestBody) { return this._sendRequest('post', `${nodeRpcUrl}/publish`, requestBody); } async update(nodeRpcUrl, requestBody) { return this._sendRequest('post', `${nodeRpcUrl}/update`, requestBody); } /** * Polls an operation until it reaches a terminal status (COMPLETED or FAILED). * @param {string} nodeRpcUrl * @param {string} operationName e.g. 'publish', 'get', 'update' * @param {string} operationId * @param {object} [options] * @param {number} [options.intervalMs=5000] delay between retries * @param {number} [options.maxRetries=5] * @returns {Promise} the final operation result response */ async pollOperationResult(nodeRpcUrl, operationName, operationId, { intervalMs = 5000, maxRetries = 5 } = {}) { for (let attempt = 0; attempt < maxRetries; attempt += 1) { // eslint-disable-next-line no-await-in-loop const result = await this.getOperationResult(nodeRpcUrl, operationName, operationId); if (TERMINAL_STATUSES.includes(result.data.status)) { return result; } if (attempt < maxRetries - 1) { // eslint-disable-next-line no-await-in-loop await setTimeout(intervalMs); } } throw new Error( `Operation ${operationName}/${operationId} did not reach a terminal status after ${maxRetries} attempts`, ); } async _sendRequest(method, url, data) { return axios({ method, url, ...data && { data }, }).catch((error) => { const errorWithStatus = new Error(error.message); if (error.response) { errorWithStatus.statusCode = error.response.status; } throw errorWithStatus; }); } } export default HttpApiHelper; ================================================ FILE: test/utilities/steps-utils.mjs ================================================ import { fork } from 'child_process'; const otNodeProcessPath = './test/bdd/steps/lib/ot-node-process.mjs'; /** * Fixed libp2p private key for the bootstrap node. * Produces a deterministic PeerID (QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj) that matches * the bootstrap peer address baked into the default network config so regular nodes can find it. */ const BOOTSTRAP_LIBP2P_PRIVATE_KEY = 'CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ='; // Port 9000 is PHP-FPM's default port and is commonly occupied on developer machines. // Use high-numbered ports that are unlikely to conflict with system services or retries. const BOOTSTRAP_NETWORK_PORT = 19000; const BOOTSTRAP_RPC_PORT = 18900; /** * Loopback multiaddr for the bootstrap node. Regular nodes dial this on startup for DHT seeding. * PeerID corresponds to BOOTSTRAP_LIBP2P_PRIVATE_KEY. Uses 127.0.0.1 — the default config uses * 0.0.0.0 which is not a valid dial address and was causing silent connection failures. */ const BOOTSTRAP_PEER_MULTIADDR = `/ip4/127.0.0.1/tcp/${BOOTSTRAP_NETWORK_PORT}/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj`; class StepsUtils { forkNode(nodeConfiguration) { const forkedNode = fork(otNodeProcessPath, [], { silent: true }); forkedNode.send(JSON.stringify(nodeConfiguration)); return forkedNode; } /** * Builds a full node configuration object for BDD test scenarios. * * @param {Array<{blockchainId: string, port: number, operationalWallet: object, managementWallet: object}>} blockchains * @param {number} nodeIndex - Zero-based index; drives unique DB names, ports, and triple-store repos * @param {string} nodeName * @param {number} rpcPort - HTTP API port * @param {number} networkPort - libp2p P2P port * @param {boolean} [bootstrap=false] - When true, uses the fixed libp2p key (known PeerID), * empty bootstrap list, and isolated DB/data paths * @param {string} [bootstrapPeerMultiaddr] - For regular nodes, the bootstrap peer multiaddr to dial. * If omitted, BOOTSTRAP_PEER_MULTIADDR is used. * @returns {object} Node configuration */ createNodeConfiguration( blockchains, nodeIndex, nodeName, rpcPort, networkPort, bootstrap = false, bootstrapPeerMultiaddr = BOOTSTRAP_PEER_MULTIADDR, ) { let config = { modules: { blockchain: { implementation: {}, }, network: { implementation: { 'libp2p-service': { config: { port: networkPort, privateKey: bootstrap ? BOOTSTRAP_LIBP2P_PRIVATE_KEY : undefined, bootstrap: bootstrap ? [] : [bootstrapPeerMultiaddr], peerRouting: { refreshManager: { enabled: false, }, }, }, }, }, }, repository: { implementation: { 'sequelize-repository': { config: { database: bootstrap ? 'operationaldbbootstrap' : `operationaldbnode${nodeIndex}`, }, }, }, }, tripleStore: { implementation: { 'ot-blazegraph': { config: { repositories: { dkg: { url: 'http://localhost:9999', name: bootstrap ? 'dkg-bootstrap' : `dkg-${nodeIndex}`, username: 'admin', password: '', }, privateCurrent: { url: 'http://localhost:9999', name: bootstrap ? 'private-current-bootstrap' : `private-current-${nodeIndex}`, username: 'admin', password: '', }, publicCurrent: { url: 'http://localhost:9999', name: bootstrap ? 'public-current-bootstrap' : `public-current-${nodeIndex}`, username: 'admin', password: '', }, }, }, }, }, }, validation: { enabled: true, implementation: { 'merkle-validation': { enabled: true, package: './validation/implementation/merkle-validation.js', }, }, }, httpClient: { implementation: { 'express-http-client': { config: { port: rpcPort, }, }, }, }, }, auth: { ipBasedAuthEnabled: false, }, operationalDatabase: { databaseName: bootstrap ? 'operationaldbbootstrap' : `operationaldbnode${nodeIndex}`, }, rpcPort, appDataPath: bootstrap ? 'test-data-bootstrap' : `test-data${nodeIndex}`, graphDatabase: { name: nodeName, }, }; for (const blockchain of blockchains) { config.modules.blockchain.implementation[blockchain.blockchainId] = { enabled: true, package: './blockchain/implementation/hardhat/hardhat-service.js', config: { hubContractAddress: '0x5FbDB2315678afecb367f032d93F642f64180aa3', rpcEndpoints: [`http://localhost:${blockchain.port}`], initialStakeAmount: 50000, initialAskAmount: 0.2, operationalWallets: [{ privateKey: blockchain.operationalWallet.privateKey, evmAddress: blockchain.operationalWallet.address, }], evmManagementWalletPublicKey: blockchain.managementWallet.address, evmManagementWalletPrivateKey: blockchain.managementWallet.privateKey, nodeName: bootstrap ? 'bootstrap' : `node${nodeIndex}`, }, }; } return config; } } export { BOOTSTRAP_NETWORK_PORT, BOOTSTRAP_RPC_PORT, BOOTSTRAP_PEER_MULTIADDR }; export default StepsUtils; ================================================ FILE: test/utilities/utilities.js ================================================ import TripleStoreModuleManager from '../../src/modules/triple-store/triple-store-module-manager.js'; class Utilities { static unpackRawTableToArray(rawTable) { return rawTable.rawTable[0]; } /** * Unpacks cucumber dictionary into simple dictionary * @param rawTable */ static unpackRawTable(rawTable) { const parse = (val) => { if (!Number.isNaN(Number(val))) { return Number(val); } if (val.toLowerCase() === 'true' || val.toLowerCase() === 'false') { return Boolean(val); } return val; }; const unpacked = {}; if (rawTable) { for (const row of rawTable.rawTable) { let value; if (row.length > 2) { value = []; for (let index = 1; index < row.length; index += 1) { if (!row[index] != null && row[index] !== '') { value.push(parse(row[index])); } } } else { value = parse(row[1]); } const keyParts = row[0].split('.'); if (keyParts.length === 1) { unpacked[keyParts[0]] = value; } else { let current = unpacked; for (let j = 0; j < keyParts.length - 1; j += 1) { if (!current[keyParts[j]]) { current[keyParts[j]] = {}; } current = current[keyParts[j]]; } current[keyParts[keyParts.length - 1]] = value; } } } return unpacked; } static async deleteTripleStoreRepositories(config, logger) { const tripleStoreModuleManager = new TripleStoreModuleManager({ config, logger }); await tripleStoreModuleManager.initialize(); for (const implementationName of tripleStoreModuleManager.getImplementationNames()) { // eslint-disable-next-line no-shadow const { module, config } = tripleStoreModuleManager.getImplementation(implementationName); // eslint-disable-next-line no-await-in-loop await Promise.all( Object.keys(config.repositories).map((repository) => module.deleteRepository(repository), ), ); } } } export default Utilities; ================================================ FILE: tools/local-network-setup/.origintrail_noderc_template.json ================================================ { "logLevel": "trace", "modules": { "httpClient": { "enabled": true, "implementation": { "express-http-client": { "package": "./http-client/implementation/express-http-client.js", "config": {} } } }, "repository": { "enabled": true, "implementation": { "sequelize-repository": { "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": {} } } }, "tripleStore": { "enabled": true, "implementation": { "ot-blazegraph": { "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", "config": { "repositories": { "dkg": { "url": "http://localhost:9999", "name": "dkg", "username": "admin", "password": "" }, "privateCurrent": { "url": "http://localhost:9999", "name": "private-current", "username": "admin", "password": "" }, "publicCurrent": { "url": "http://localhost:9999", "name": "public-current", "username": "admin", "password": "" } } } }, "ot-fuseki": { "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", "config": { "repositories": { "dkg": { "url": "http://localhost:3030", "name": "dkg", "username": "admin", "password": "" } } } }, "ot-graphdb": { "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", "config": { "repositories": { "dkg": { "url": "http://localhost:7200", "name": "dkg", "username": "admin", "password": "" } } } } } }, "network": { "enabled": true, "implementation": { "libp2p-service": { "package": "./network/implementation/libp2p-service.js", "config": { "port": 9001, "bootstrap": [ "/ip4/0.0.0.0/tcp/9100/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" ] } } } }, "blockchain": { "implementation": { "hardhat1:31337": { "enabled": true, "package": "./blockchain/implementation/hardhat/hardhat-service.js", "config": { "operationalWallets": [ { "evmAddress": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", "privateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e" } ], "rpcEndpoints": [] } }, "hardhat2:31337": { "package": "./blockchain/implementation/hardhat/hardhat-service.js", "config": { "operationalWallets": [ { "evmAddress": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", "privateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e" } ], "rpcEndpoints": [] } } } }, "blockchainEvents": { "enabled": true, "implementation": { "ot-ethers": { "enabled": true, "package": "./blockchain-events/implementation/ot-ethers/ot-ethers.js", "config": { "blockchains": ["hardhat1:31337", "hardhat2:31337"], "rpcEndpoints": { "hardhat1:31337": ["http://localhost:8545"], "hardhat2:31337": ["http://localhost:9545"] }, "hubContractAddress": { "hardhat1:31337": "0x5FbDB2315678afecb367f032d93F642f64180aa3", "hardhat2:31337": "0x5FbDB2315678afecb367f032d93F642f64180aa3" } } } } } }, "auth": { "ipWhitelist": ["::1", "127.0.0.1"] } } ================================================ FILE: tools/local-network-setup/README.md ================================================ # DKG local network setup tool This tool will help you set up a local DKG V8 network running with the Hardhat blockchain. It is useful for development and testing purposes and is used internally by the OriginTrail core developers.
**Note: This tool is an internal tool used by the OriginTrail team and thus is developed for our workflow, meaning that it currently only supports MacOS and Linux**, but we encourage you to adapt it for your workflow as well. # Prerequisites - An installed and running triplestore (graph database) - We recommend testing with Blazegraph. In order to download Blazegraph, please visit their official [website](https://blazegraph.com/). Alternatively other triple stores can be used (GraphBD or and other RDF native graph databases) - An installed and running MySQL server - You should have installed npm and Node.js (v20) # Setup instructions In order to run the local network you fist need to clone the "dkg-engine" repository.
## 1. CLONE DKG-ENGINE REPOSITORY & INSTALL DEPENDENCIES After cloning the **dkg-engine** repository, please checkout to "v8/develop" branch and install dependencies by running: ```bash git clone https://github.com/OriginTrail/dkg-engine.git && cd dkg-engine/ && npm install && cd .. ```
### 2.2 Create the .env file inside the "dkg-engine" directory: ```bash nano .env ``` and paste the following content inside (save and close): ```bash NODE_ENV=development RPC_ENDPOINT_BC1=http://localhost:8545 RPC_ENDPOINT_BC2=http://localhost:9545 ``` **Note:** The private key above is used ONLY for convenience and SHOULD be changed to a secure key when used in production.
## 3. START THE LOCAL NETWORK ## Specifying the number of nodes You can specify to run anywhere between one and twenty nodes with the `--nodes` parameter. **Note:** All nodes assume MySQL username root and no password. To change the MySQL login information update the .origintrail_noderc template file sequelize-repository config object with your username and password
The first node will be named `bootstrap`, while subsequent nodes will be named `dh1, dh2, ...`.
### MacOS ```bash bash ./tools/local-network-setup/setup-macos-environment.sh --nodes=12 ``` ### Linux ```bash ./tools/local-network-setup/setup-linux-environment.sh --nodes=12 ``` **Note:** With the above commands, we will start two hardhat instances, deploy contracts, deploy a 12 node network (1 bootstrap and 11 subsequent nodes)
## Specifying the blockchain network You can specify the blockchain network you want to connect to with `--network` parameter. Available networks: - hardhat - default network # Contribution OriginTrail is an open source project. We happily invite you to join us in our mission of building decentralized knowledge graph - we're excited for your contributions! Join us in discord to meet the dev community ### Useful links [OriginTrail website](https://origintrail.io) [OriginTrail documentation page](http://docs.origintrail.io) [OriginTrail Discord Group](https://discordapp.com/invite/FCgYk2S) [OriginTrail Telegram Group](https://t.me/origintrail) [OriginTrail Twitter](https://twitter.com/origin_trail) ================================================ FILE: tools/local-network-setup/generate-config-files.js ================================================ /* eslint-disable */ import 'dotenv/config'; import mysql from 'mysql2'; import path from 'path'; import fs from 'fs-extra'; import TripleStoreModuleManager from '../../src/modules/triple-store/triple-store-module-manager.js'; import Logger from '../../src/logger/logger.js'; import { unlink } from 'fs/promises'; const { readFile, writeFile, stat } = fs; const generalConfig = JSON.parse(await readFile('./config/config.json')); const templatePath = path.join('./tools/local-network-setup/.origintrail_noderc_template.json'); const privateKeysFile = await readFile('test/bdd/steps/api/datasets/privateKeys.json'); const publicKeysFile = await readFile('test/bdd/steps/api/datasets/publicKeys.json'); const privateKeys = JSON.parse(privateKeysFile.toString()); const publicKeys = JSON.parse(publicKeysFile.toString()); // todo update this logic const privateKeysManagementWalletFile = await readFile( 'test/bdd/steps/api/datasets/privateKeys-management-wallets.json', ); const publicKeysManagementWalletFile = await readFile( 'test/bdd/steps/api/datasets/publicKeys-management-wallets.json', ); const privateKeysManagementWallet = JSON.parse(privateKeysManagementWalletFile.toString()); const publicKeysManagementWallet = JSON.parse(publicKeysManagementWalletFile.toString()); const logger = new Logger(generalConfig.development.logLevel); const numberOfNodes = parseInt(process.argv[2], 10); const blockchain = process.argv[3]; const tripleStoreImplementation = process.argv[4]; const hubContractAddress = process.argv[5]; const libp2pBootstrapPrivateKey = 'CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ='; logger.info(`Generating config for ${numberOfNodes} node(s)`); /******************************** CONFIG GENERATION *********************************/ const promises = []; for (let i = 0; i < numberOfNodes; i += 1) { promises.push(generateNodeConfig(i)); } await Promise.all(promises); /******************************** FUNCTIONS DEFINITIONS *********************************/ async function generateNodeConfig(nodeIndex) { const configPath = path.join( `./tools/local-network-setup/.node${nodeIndex}_origintrail_noderc.json`, ); if (await fileExists(configPath)) { await removeFile(configPath); } const template = JSON.parse(await readFile(templatePath)); logger.info(`Configuring node ${nodeIndex}`); template.modules.tripleStore = generateTripleStoreConfig( template.modules.tripleStore, nodeIndex, ); template.modules.blockchain = generateBlockchainConfig(template.modules.blockchain, nodeIndex); template.modules.httpClient = generateHttpClientConfig(template.modules.httpClient, nodeIndex); template.modules.network = generateNetworkConfig(template.modules.network, nodeIndex); template.modules.repository = generateRepositoryConfig(template.modules.repository, nodeIndex); template.appDataPath = `data${nodeIndex}`; template.logLevel = process.env.LOG_LEVEL ?? template.logLevel; await writeFile(configPath, JSON.stringify(template, null, 4)); const config = JSON.parse(await readFile(configPath)); await Promise.all([ dropDatabase( `operationaldb${nodeIndex}`, generalConfig.development.modules.repository.implementation['sequelize-repository'] .config, ), deleteTripleStoreRepositories(config), deleteDataFolder(config), ]); } function generateTripleStoreConfig(templateTripleStoreConfig, nodeIndex) { const tripleStoreConfig = JSON.parse(JSON.stringify(templateTripleStoreConfig)); for (const implementationName in tripleStoreConfig.implementation) { for (const [repository, config] of Object.entries( tripleStoreConfig.implementation[implementationName].config.repositories, )) { tripleStoreConfig.implementation[implementationName].config.repositories[ repository ].name = `${config.name}-${nodeIndex}`; } tripleStoreConfig.implementation[implementationName].enabled = implementationName === tripleStoreImplementation ? true : false; } return tripleStoreConfig; } function generateBlockchainConfig(templateBlockchainConfig, nodeIndex) { const blockchainConfig = JSON.parse(JSON.stringify(templateBlockchainConfig)); // console.log('************************'); // console.log(publicKeys[nodeIndex]); // console.log('************************'); blockchainConfig.implementation['hardhat1:31337'].config = { ...blockchainConfig.implementation['hardhat1:31337'].config, hubContractAddress, rpcEndpoints: [process.env.RPC_ENDPOINT_BC1], operationalWallets: [ { evmAddress: publicKeys[nodeIndex + 1], privateKey: privateKeys[nodeIndex + 1], }, ], evmManagementWalletPublicKey: publicKeysManagementWallet[nodeIndex + 1], evmManagementWalletPrivateKey: privateKeysManagementWallet[nodeIndex + 1], nodeName: `LocalNode${nodeIndex + 1}`, }; // TODO: Don't use string blockchainConfig.implementation['hardhat2:31337'].config = { ...blockchainConfig.implementation['hardhat2:31337'].config, hubContractAddress, rpcEndpoints: [process.env.RPC_ENDPOINT_BC2], operationalWallets: [ { evmAddress: publicKeys[nodeIndex + 1], privateKey: privateKeys[nodeIndex + 1], }, ], evmManagementWalletPublicKey: publicKeysManagementWallet[nodeIndex + 1], evmManagementWalletPrivateKey: privateKeysManagementWallet[nodeIndex + 1], sharesTokenName: `LocalNode${nodeIndex + 1}`, sharesTokenSymbol: `LN${nodeIndex + 1}`, }; // Used for testing, add a few more wallets to later nodes if (nodeIndex == 3) { blockchainConfig.implementation['hardhat1:31337'].config.operationalWallets.push({ evmAddress: publicKeys[publicKeys.length - 1 - 1], privateKey: privateKeys[privateKeys.length - 1 - 1], }); blockchainConfig.implementation['hardhat2:31337'].config.operationalWallets.push({ evmAddress: publicKeys[publicKeys.length - 1 - 2], privateKey: privateKeys[privateKeys.length - 1 - 2], }); } if (nodeIndex == 4) { blockchainConfig.implementation['hardhat1:31337'].config.operationalWallets.push({ evmAddress: publicKeys[publicKeys.length - 1 - 3], privateKey: privateKeys[privateKeys.length - 1 - 3], }); blockchainConfig.implementation['hardhat2:31337'].config.operationalWallets.push({ evmAddress: publicKeys[publicKeys.length - 1 - 4], privateKey: privateKeys[privateKeys.length - 1 - 4], }); } return blockchainConfig; } function generateHttpClientConfig(templateHttpClientConfig, nodeIndex) { const httpClientConfig = JSON.parse(JSON.stringify(templateHttpClientConfig)); httpClientConfig.implementation['express-http-client'].config.port = 8900 + nodeIndex; return httpClientConfig; } function generateNetworkConfig(templateNetworkConfig, nodeIndex) { const networkConfig = JSON.parse(JSON.stringify(templateNetworkConfig)); networkConfig.implementation['libp2p-service'].config.port = 9100 + nodeIndex; if (nodeIndex == 0) { networkConfig.implementation['libp2p-service'].config.privateKey = libp2pBootstrapPrivateKey; } return networkConfig; } function generateRepositoryConfig(templateRepositoryConfig, nodeIndex) { const repositoryConfig = JSON.parse(JSON.stringify(templateRepositoryConfig)); repositoryConfig.implementation[ 'sequelize-repository' ].config.database = `operationaldb${nodeIndex}`; return repositoryConfig; } async function dropDatabase(name, config) { logger.info(`Dropping database: ${name}`); const password = process.env.REPOSITORY_PASSWORD ?? config.password; const connection = mysql.createConnection({ database: name, user: config.user, host: config.host, password, }); try { await connection.promise().query(`DROP DATABASE IF EXISTS ${name};`); } catch (e) { logger.warn(`Error while dropping database. Error: ${e.message}`); } connection.destroy(); } async function deleteTripleStoreRepositories(config) { const tripleStoreModuleManager = new TripleStoreModuleManager({ config, logger }); await tripleStoreModuleManager.initialize(); for (const implementationName of tripleStoreModuleManager.getImplementationNames()) { const { module, config } = tripleStoreModuleManager.getImplementation(implementationName); await Promise.all( Object.keys(config.repositories).map((repository) => module.deleteRepository(repository), ), ); } } async function fileExists(filePath) { try { await stat(filePath); return true; } catch (e) { return false; } } async function removeFile(filePath) { await unlink(filePath); } async function deleteDataFolder(config) { if (await fileExists(config.appDataPath)) { logger.trace(`Removing file on path: ${config.appDataPath}`); await fs.rm(config.appDataPath, { recursive: true, force: true }); } } ================================================ FILE: tools/local-network-setup/run-local-blockchain.js ================================================ import LocalBlockchain from '../../test/bdd/steps/lib/local-blockchain.mjs'; const port = parseInt(process.argv[2], 10); const version = process.argv.length > 3 ? process.argv[3] : ''; const localBlockchain = new LocalBlockchain(); await localBlockchain.initialize(port, console, version); ================================================ FILE: tools/local-network-setup/setup-linux-environment.sh ================================================ #!/bin/bash pathToOtNode=$(pwd) numberOfNodes=12 network="hardhat1:31337" tripleStore="ot-blazegraph" availableNetworks=("hardhat1:31337") export $(xargs < $pathToOtNode/.env) export ACCESS_KEY=$RPC_ENDPOINT # Check for script arguments while [ $# -gt 0 ]; do case "$1" in # Override number of nodes if the argument is specified --nodes=*) numberOfNodes="${1#*=}" if [[ $numberOfNodes -le 0 ]] then echo Cannot run 0 nodes exit 1 fi ;; # Print script usage if --help is given --help) echo "Use --nodes= to specify the number of nodes to generate" echo "Use --network= to specify the network to connect to. Available networks: hardhat." exit 0 ;; --network=*) network="${1#*=}" if [[ ! " ${availableNetworks[@]} " =~ " ${network} " ]] then echo Invalid network parameter. Available networks: hardhat exit 1 fi ;; --tripleStore=*) tripleStore="${1#*=}" ;; *) printf "***************************\n" printf "* Error: Invalid argument.*\n" printf "***************************\n" exit 1 esac shift done if [[ $network == hardhat1:31337 ]] then echo ================================ echo ====== Starting hardhat1 ====== echo ================================ sh -c "cd $pathToOtNode && node tools/local-network-setup/run-local-blockchain.js 8545 " & echo Waiting for hardhat to start and contracts deployment while ! nc -z localhost 8545; do sleep 1 done echo Hardhat started. echo ================================ echo ====== Starting hardhat 2 ====== echo ================================ sh -c "cd $pathToOtNode && node tools/local-network-setup/run-local-blockchain.js 9545 " & echo Waiting for hardhat to start and contracts deployment while ! nc -z localhost 9545; do sleep 1 done echo Hardhat started. fi echo ================================ echo ====== Generating configs ====== echo ================================ node $pathToOtNode/tools/local-network-setup/generate-config-files.js $numberOfNodes $network $tripleStore $hubContractAddress sleep 5 echo ================================ echo ======== Starting nodes ======== echo ================================ startNode() { echo Starting node $1 sh -c "cd $pathToOtNode && node index.js ./tools/local-network-setup/.node$1_origintrail_noderc.json" & } i=0 while [[ $i -lt $numberOfNodes ]] do startNode $i ((i = i + 1)) done wait # Close started background processes when done (https://stackoverflow.com/a/2173421) trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT ================================================ FILE: tools/local-network-setup/setup-macos-environment.sh ================================================ #!/bin/bash # Source bash profile to ensure PATH includes Homebrew source ~/.bash_profile 2>/dev/null || true pathToOtNode=$(pwd) numberOfNodes=12 network="hardhat1:31337" tripleStore="ot-blazegraph" availableNetworks=("hardhat1:31337") export $(xargs < $pathToOtNode/.env) export ACCESS_KEY=$RPC_ENDPOINT # Check for script arguments while [ $# -gt 0 ]; do case "$1" in # Override number of nodes if the argument is specified --nodes=*) numberOfNodes="${1#*=}" if [[ $numberOfNodes -le 0 ]] then echo Cannot run 0 nodes exit 1 fi ;; # Print script usage if --help is given --help) echo "Use --nodes= to specify the number of nodes to generate" echo "Use --network= to specify the network to connect to. Available networks: hardhat, rinkeby. Default: hardhat" exit 0 ;; --network=*) network="${1#*=}" if [[ ! " ${availableNetworks[@]} " =~ " ${network} " ]] then echo Invalid network parameter. Available networks: hardhat exit 1 fi ;; --tripleStore=*) tripleStore="${1#*=}" ;; *) printf "***************************\n" printf "* Error: Invalid argument.*\n" printf "***************************\n" exit 1 esac shift done if [[ $network == hardhat1:31337 ]] then echo ================================ echo ====== Starting hardhat1 ====== echo ================================ osascript -e "tell app \"Terminal\" do script \"cd $pathToOtNode node tools/local-network-setup/run-local-blockchain.js 8545 \" end tell" echo Waiting for hardhat to start and contracts deployment while ! nc -z localhost 8545; do sleep 1 done echo Hardhat started. echo ================================ echo ====== Starting hardhat 2 ====== echo ================================ osascript -e "tell app \"Terminal\" do script \"cd $pathToOtNode node tools/local-network-setup/run-local-blockchain.js 9545 \" end tell" echo Waiting for hardhat to start and contracts deployment while ! nc -z localhost 9545; do sleep 1 done echo Hardhat started. fi echo ================================ echo ====== Generating configs ====== echo ================================ node $pathToOtNode/tools/local-network-setup/generate-config-files.js $numberOfNodes $network $tripleStore $hubContractAddress sleep 30 echo ================================ echo ====== Clearing Redis ========== echo ================================ # Clear all Redis data if redis-cli is available if command -v redis-cli &> /dev/null; then redis-cli FLUSHALL echo Redis cleared. elif [ -f "/opt/homebrew/bin/redis-cli" ]; then /opt/homebrew/bin/redis-cli FLUSHALL echo Redis cleared. else echo "redis-cli not found. Skipping Redis cleanup." echo "To install Redis on macOS, run: brew install redis" fi echo ================================ echo ======== Starting nodes ======== echo ================================ startNode() { echo Starting node $1 osascript -e "tell app \"Terminal\" do script \"cd $pathToOtNode node index.js ./tools/local-network-setup/.node$1_origintrail_noderc.json\" end tell" } i=0 while [[ $i -lt $numberOfNodes ]] do startNode $i ((i = i + 1)) done ================================================ FILE: tools/ot-parachain-account-mapping/create-account-mapping-signature.js ================================================ /* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable no-console */ import { Wallet } from '@ethersproject/wallet'; import { joinSignature } from '@ethersproject/bytes'; import { _TypedDataEncoder } from '@ethersproject/hash'; import { u8aToHex } from '@polkadot/util'; import { decodeAddress } from '@polkadot/util-crypto'; if (!process.argv[2]) { console.log('Missing argument PRIVATE_ETH_KEY'); console.log( 'Usage: npm run create-account-mapping-signature PRIVATE_ETH_KEY SUBSTRATE_PUBLIC_KEY', ); process.exit(1); } if (!process.argv[3]) { console.log('Missing argument SUBSTRATE_PUBLIC_KEY'); console.log( 'Usage: npm run create-account-mapping-signature PRIVATE_ETH_KEY SUBSTRATE_PUBLIC_KEY', ); process.exit(1); } const PRIVATE_ETH_KEY = process.argv[2]; const PUBLIC_SUBSTRATE_ADDRESS = process.argv[3]; const HEX_SUBSTRATE_ADDRESS = u8aToHex(decodeAddress(PUBLIC_SUBSTRATE_ADDRESS)); // Usage // node create-signature.js private_eth_key(with 0x) substrate_public_key async function sign() { const payload = { types: { EIP712Domain: [ { name: 'name', type: 'string', }, { name: 'version', type: 'string', }, { name: 'chainId', type: 'uint256', }, { name: 'salt', type: 'bytes32', }, ], Transaction: [ { name: 'substrateAddress', type: 'bytes', }, ], }, primaryType: 'Transaction', domain: { name: 'OTP EVM claim', version: '1', chainId: '2160', salt: '0x0542e99b538e30d713d3e020f18fa6717eb2c5452bd358e0dd791628260a36f0', }, message: { substrateAddress: `${HEX_SUBSTRATE_ADDRESS}`, }, }; const wallet = new Wallet(`${PRIVATE_ETH_KEY}`); const digest = _TypedDataEncoder.hash( payload.domain, { Transaction: payload.types.Transaction, }, payload.message, ); const signature = joinSignature(wallet._signingKey().signDigest(digest)); console.log('Paste the signature to polkadot.js api evmAddress claimAccount interface:'); console.log('==== Signature ===='); console.log(signature); console.log('==================='); } sign(); ================================================ FILE: tools/substrate-accounts-mapping/README.md ================================================ # Substrate accounts mapping tool This tool: - generates one shared management wallet (pair of substrate and eth addresses) - generates NUMBER_OF_ACCOUNTS (default 32) operational wallets - sends OTP to all substrate accounts - performs mapping between substrate and eth pairs - sends TRAC to all eth wallets - confirms that TRAC is received ## How to run Inside the .env file is stored substrate and eth pair for distribution account that will send OTP and TRAC to newly generated wallets. Example of env: ``` SUBSTRATE_ACCOUNT_PUBLIC_KEY="gJn..." SUBSTRATE_ACCOUNT_PRIVATE_KEY="URI FORMAT OF KEY" EVM_ACCOUNT_PUBLIC_KEY="0xPublicKey" EVM_ACCOUNT_PRIVATE_KEY="0xPrivateKey" ``` Run the script: ```bash node accounts-mapping.js ``` Result will be stored in `wallets.json` in this format: ```json [ { "evmOperationalWalletPublicKey": "", "evmOperationalWalletPrivateKey": "", "substrateOperationalWalletPublicKey": "", "substrateOperationalWalletPrivateKey": "", "evmManagementWalletPublicKey": "", "evmManagementWalletPrivateKey": "", "substrateManagementWalletPublicKey": "", "substrateManagementWalletPrivateKey": "" } ] ``` ## How to modify To change number of generated accounts, amount of OTP or TRAC to be sent, or other parameters, following variables should be modified: ```js const NUMBER_OF_ACCOUNTS = 32; const OTP_AMOUNT = 50 * 1e12; // 50 OTP const TRACE_AMOUNT = '0.000000001'; const GAS_PRICE = 20; const GAS_LIMIT = 60000; // Estimation is 45260 ``` Script by default script is created to be used for OriginTrail Parachain Mainnet, by modification of following variables it can be used for other parachains: ```js const TOKEN_ADDRESS = '0xffffffff00000000000000000000000000000001'; const HTTPS_ENDPOINT = 'https://astrosat-parachain-rpc.origin-trail.network'; const OTP_CHAIN_ID = '2043'; const OTP_GENESIS_HASH = '0xe7e0962324a3b86c83404dbea483f25fb5dab4c224791c81b756cfc948006174'; ``` ================================================ FILE: tools/substrate-accounts-mapping/accounts-mapping.js ================================================ /* eslint-disable no-await-in-loop */ /* eslint-disable no-console */ /* eslint-disable object-shorthand */ /* eslint-disable lines-between-class-members */ require('dotenv').config({ path: `${__dirname}/../../.env` }); const { setTimeout } = require('timers/promises'); const appRootPath = require('app-root-path'); const { ethers } = require('ethers'); const path = require('path'); const fs = require('fs'); const { ApiPromise, HttpProvider } = require('@polkadot/api'); const { Keyring } = require('@polkadot/keyring'); const { mnemonicGenerate, mnemonicToMiniSecret, decodeAddress } = require('@polkadot/util-crypto'); const { u8aToHex } = require('@polkadot/util'); const { Wallet } = require('@ethersproject/wallet'); const { joinSignature } = require('@ethersproject/bytes'); const { _TypedDataEncoder } = require('@ethersproject/hash'); const ERC20Token = require('dkg-evm-module/abi/Token.json'); const WALLETS_PATH = path.join(appRootPath.path, 'tools/substrate-accounts-mapping/wallets.json'); const otpAccountWithTokens = { accountPublicKey: process.env.SUBSTRATE_ACCOUNT_PUBLIC_KEY, accountPrivateKey: process.env.SUBSTRATE_ACCOUNT_PRIVATE_KEY, }; const evmAccountWithTokens = { publicKey: process.env.EVM_ACCOUNT_PUBLIC_KEY, privateKey: process.env.EVM_ACCOUNT_PRIVATE_KEY, }; const TOKEN_ADDRESS = '0xffffffff00000000000000000000000000000001'; const HTTPS_ENDPOINT = 'https://astrosat-parachain-rpc.origin-trail.network'; const NUMBER_OF_ACCOUNTS = 32; const OTP_AMOUNT = 50 * 1e12; // 50 OTP <--- Check this! const OTP_CHAIN_ID = '2043'; const OTP_GENESIS_HASH = '0xe7e0962324a3b86c83404dbea483f25fb5dab4c224791c81b756cfc948006174'; const GAS_PRICE = 20; const GAS_LIMIT = 60000; // Estimation is 45260 const TRACE_AMOUNT = '0.000000001'; // <--- Check this! class AccountsMapping { async initialize() { // Initialise the provider to connect to the local node const provider = new HttpProvider(HTTPS_ENDPOINT); // eslint-disable-next-line no-await-in-loop this.parachainProvider = await new ApiPromise({ provider }).isReady; this.ethersProvider = new ethers.providers.JsonRpcProvider(HTTPS_ENDPOINT); this.evmWallet = new ethers.Wallet(evmAccountWithTokens.privateKey, this.ethersProvider); this.initialized = true; this.tokenContract = new ethers.Contract(TOKEN_ADDRESS, ERC20Token.abi, this.evmWallet); } async mapAccounts() { if (!this.initialized) { await this.initialize(); } const currentWallets = []; console.log(`Generating, mapping and funding management wallet`); const { evmPublicKey: evmManagementWalletPublicKey, evmPrivateKey: evmManagementWalletPrivateKey, substratePublicKey: substrateManagementWalletPublicKey, substratePrivateKey: substrateManagementWalletPrivateKey, } = await this.generateWallets(); // Fund management wallet await this.fundAccountsWithOtp(substrateManagementWalletPublicKey); // Generate and fund all other wallets for (let i = 0; i < NUMBER_OF_ACCOUNTS; i += 1) { console.log(`Generating and funding with OTP wallet #${i + 1}`); const { evmPublicKey: evmOperationalWalletPublicKey, evmPrivateKey: evmOperationalWalletPrivateKey, substratePublicKey: substrateOperationalWalletPublicKey, substratePrivateKey: substrateOperationalWalletPrivateKey, } = await this.generateWallets(); await this.fundAccountsWithOtp(substrateOperationalWalletPublicKey); // Store new wallets currentWallets.push({ evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey, substrateOperationalWalletPublicKey, substrateOperationalWalletPrivateKey, evmManagementWalletPublicKey, evmManagementWalletPrivateKey, substrateManagementWalletPublicKey, substrateManagementWalletPrivateKey, }); await fs.promises.writeFile(WALLETS_PATH, JSON.stringify(currentWallets, null, 4)); } console.log('Waiting 35s for funding TXs to get into block!'); await this.sleepForMilliseconds(35 * 1000); console.log(`${NUMBER_OF_ACCOUNTS} wallets are generated and funded with OTP!`); console.log(`Executing mapping!`); // Map the management wallet await this.mapWallet( evmManagementWalletPublicKey, evmManagementWalletPrivateKey, substrateManagementWalletPublicKey, substrateManagementWalletPrivateKey, ); // Map all operational wallets for (const wallet of currentWallets) { await this.mapWallet( wallet.evmOperationalWalletPublicKey, wallet.evmOperationalWalletPrivateKey, wallet.substrateOperationalWalletPublicKey, wallet.substrateOperationalWalletPrivateKey, ); } console.log('Waiting 35s for mapping TXs to get into block!'); await this.sleepForMilliseconds(35 * 1000); console.log(`${NUMBER_OF_ACCOUNTS} wallets mapped!`); console.log(`Funding wallets with TRAC!`); let nonce = await this.evmWallet.getTransactionCount(); // Fund management wallet with TRACE this.fundAccountsWithTrac(evmManagementWalletPublicKey, nonce); // Fund rest of wallets for (const wallet of currentWallets) { if (await this.accountMapped(wallet.evmOperationalWalletPublicKey)) { nonce += 1; this.fundAccountsWithTrac(wallet.evmOperationalWalletPublicKey, nonce); } else { console.log(`Mapping failed or not finished for account: ${wallet}`); } } console.log('Waiting for Trac TXs to get into block!'); await this.sleepForMilliseconds(35 * 1000); console.log(`${NUMBER_OF_ACCOUNTS} wallets funded with TRAC!`); // Check the balance of new accounts for (const wallet of currentWallets) { const tokenBalance = await this.tokenContract.methods .balanceOf(wallet.evmOperationalWalletPublicKey) .call(); console.log( `New balance of ${wallet.evmOperationalWalletPublicKey} is ${tokenBalance} TRAC`, ); } } async generateWallets() { const { evmPublicKey, evmPrivateKey } = await this.generateEVMAccount(); const { substratePublicKey, substratePrivateKey } = await this.generateSubstrateAccount(); return { evmPublicKey, evmPrivateKey, substratePublicKey, substratePrivateKey, }; } async generateSubstrateAccount() { const keyring = new Keyring({ type: 'sr25519' }); keyring.setSS58Format(101); const mnemonic = mnemonicGenerate(); const mnemonicMini = mnemonicToMiniSecret(mnemonic); const substratePrivateKey = u8aToHex(mnemonicMini); const substratePublicKey = keyring.createFromUri(substratePrivateKey).address; return { substratePublicKey, substratePrivateKey, }; } async generateEVMAccount() { const { address, privateKey } = await ethers.Wallet.createRandom(); return { evmPublicKey: address, evmPrivateKey: privateKey }; } async mapWallet(evmPublicKey, evmPrivateKey, substratePublicKey, substratePrivateKey) { const signature = await this.sign(substratePublicKey, evmPrivateKey); const keyring = new Keyring({ type: 'sr25519' }); keyring.setSS58Format(101); const result = await this.callParachainExtrinsic( keyring.addFromSeed(substratePrivateKey), 'evmAccounts', 'claimAccount', [evmPublicKey, signature], ); if (result.toHex() === '0x') throw Error('Unable to create account mapping for otp'); console.log(result.toString()); console.log(`Account mapped for evm public key: ${evmPublicKey}`); } async fundAccountsWithOtp(substratePublicKey) { const keyring = new Keyring({ type: 'sr25519' }); keyring.setSS58Format(101); const uriKeyring = keyring.addFromSeed(otpAccountWithTokens.accountPrivateKey); return this.callParachainExtrinsic(uriKeyring, 'balances', 'transfer', [ substratePublicKey, OTP_AMOUNT, ]); } async fundAccountsWithTrac(evmWallet, nonce) { this.tokenContract.transfer(evmWallet, ethers.utils.parseEther(TRACE_AMOUNT), { gasPrice: GAS_PRICE, gasLimit: GAS_LIMIT, nonce: nonce, }); } async accountMapped(wallet) { const result = await this.queryParachainState('evmAccounts', 'accounts', [wallet]); return result && result.toHex() !== '0x'; } async callParachainExtrinsic(keyring, extrinsic, method, args) { // console.log(`Calling parachain extrinsic : ${extrinsic}, method: ${method}`); return this.parachainProvider.tx[extrinsic][method](...args).signAndSend(keyring, { nonce: -1, }); } async queryParachainState(state, method, args) { return this.parachainProvider.query[state][method](...args); } async sleepForMilliseconds(milliseconds) { await setTimeout(milliseconds); } async sign(publicAccountKey, privateEthKey) { const hexPubKey = u8aToHex(decodeAddress(publicAccountKey)); console.log(`Hex account pub: ${hexPubKey}`); const payload = { types: { EIP712Domain: [ { name: 'name', type: 'string', }, { name: 'version', type: 'string', }, { name: 'chainId', type: 'uint256', }, { name: 'salt', type: 'bytes32', }, ], Transaction: [ { name: 'substrateAddress', type: 'bytes', }, ], }, primaryType: 'Transaction', domain: { name: 'OTP EVM claim', version: '1', chainId: OTP_CHAIN_ID, salt: OTP_GENESIS_HASH, }, message: { substrateAddress: hexPubKey, }, }; const wallet = new Wallet(privateEthKey); const digest = _TypedDataEncoder.hash( payload.domain, { Transaction: payload.types.Transaction, }, payload.message, ); const signature = joinSignature(wallet._signingKey().signDigest(digest)); return signature; } } const am = new AccountsMapping(); am.mapAccounts(); ================================================ FILE: tools/token-generation.js ================================================ /* eslint no-console: 0 */ import 'dotenv/config'; import ms from 'ms'; import DeepExtend from 'deep-extend'; import rc from 'rc'; import fs from 'fs-extra'; import { v4 as uuid } from 'uuid'; import { createRequire } from 'module'; import Logger from '../src/logger/logger.js'; import RepositoryModuleManager from '../src/modules/repository/repository-module-manager.js'; import jwtUtil from '../src/service/util/jwt-util.js'; const require = createRequire(import.meta.url); const configjson = require('../config/config.json'); const pjson = require('../package.json'); const getLogger = () => new Logger('silent', false); let repository; const getConfig = () => { let config; let userConfig; if (process.env.USER_CONFIG_PATH) { const configurationFilename = process.env.USER_CONFIG_PATH; const pathSplit = configurationFilename.split('/'); userConfig = JSON.parse(fs.readFileSync(configurationFilename)); userConfig.configFilename = pathSplit[pathSplit.length - 1]; } const defaultConfig = JSON.parse(JSON.stringify(configjson[process.env.NODE_ENV])); if (userConfig) { config = DeepExtend(defaultConfig, userConfig); } else { config = rc(pjson.name, defaultConfig); } if (!config.configFilename) { // set default user configuration filename config.configFilename = '.origintrail_noderc'; } return config; }; const loadRepository = async () => { repository = new RepositoryModuleManager({ logger: getLogger(), config: getConfig() }); await repository.initialize(); }; /** * Returns argument from argv * @param argName * @returns {string|null} */ const getArg = (argName) => { const args = process.argv; const arg = args.find((a) => a.startsWith(argName)); if (!arg) { return null; } const argSplit = arg.split('='); if (!arg || argSplit.length < 2 || !argSplit[1]) { return null; } return argSplit[1]; }; /** * Returns user's name from arguments * @returns {string} */ const getUserFromArgs = () => { const arg = getArg('--user'); if (!arg) { return 'node-runner'; } return arg; }; /** * Returns expiresAt from arguments * If no expiresAt is provided, null is returned * Expressed in seconds or a string describing a time span zeit/ms * @returns {string|null} */ const getExpiresInArg = () => { const arg = getArg('--expiresIn'); if (!arg) { return null; } if (!ms(arg)) { console.log('\x1b[31m[ERROR]\x1b[0m Invalid value for expiresIn argument'); process.exit(1); } return arg; }; /** * Returns expiresAt from arguments * If no expiresAt is provided, null is returned * Expressed in seconds or a string describing a time span zeit/ms * @returns {string|null} */ const getTokenName = () => { const arg = getArg('--tokenName'); if (!arg) { console.log('\x1b[31m[ERROR]\x1b[0m Missing mandatory tokenName argument.'); process.exit(1); } return arg; }; const saveTokenData = async (tokenId, userId, tokenName, expiresIn) => { let expiresAt = null; if (expiresIn) { const time = new Date().getTime() + ms(expiresIn); expiresAt = new Date(time); } await repository.saveToken(tokenId, userId, tokenName, expiresAt); }; const printMessage = (token, hasExpiryDate) => { console.log('\x1b[32mAccess token successfully created.\x1b[0m '); if (!hasExpiryDate) { console.log('\x1b[33m[WARNING] Created token has no expiry date. \x1b[0m '); } console.log(token); console.log( '\x1b[32mMake sure to copy your personal access token now. You won’t be able to see it again!\x1b[0m ', ); }; const getUserId = async (username) => { const user = await repository.getUser(username); if (!user) { console.log(`\x1b[31m[ERROR]\x1b[0m User ${username} doesn't exist.`); process.exit(1); } return user.id; }; const generateToken = async () => { const username = getUserFromArgs(); const expiresIn = getExpiresInArg(); const tokenName = getTokenName(); await loadRepository(); const userId = await getUserId(username); const tokenId = uuid(); await saveTokenData(tokenId, userId, tokenName, expiresIn); const token = jwtUtil.generateJWT(tokenId, expiresIn); printMessage(token, expiresIn); process.exit(0); }; generateToken(); ================================================ FILE: v8-data-migration/abi/ContentAssetStorage.json ================================================ [ { "inputs": [ { "internalType": "address", "name": "hubAddress", "type": "address" } ], "stateMutability": "nonpayable", "type": "constructor" }, { "anonymous": false, "inputs": [ { "indexed": true, "internalType": "address", "name": "owner", "type": "address" }, { "indexed": true, "internalType": "address", "name": "approved", "type": "address" }, { "indexed": true, "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "Approval", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": true, "internalType": "address", "name": "owner", "type": "address" }, { "indexed": true, "internalType": "address", "name": "operator", "type": "address" }, { "indexed": false, "internalType": "bool", "name": "approved", "type": "bool" } ], "name": "ApprovalForAll", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": true, "internalType": "address", "name": "from", "type": "address" }, { "indexed": true, "internalType": "address", "name": "to", "type": "address" }, { "indexed": true, "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "Transfer", "type": "event" }, { "inputs": [ { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "approve", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "bytes32", "name": "assetAssertionId", "type": "bytes32" } ], "name": "assertionExists", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "owner", "type": "address" } ], "name": "balanceOf", "outputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "burn", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" }, { "internalType": "uint256", "name": "index", "type": "uint256" } ], "name": "deleteAssertionIssuer", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "deleteAsset", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [], "name": "generateTokenId", "outputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getApproved", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "uint256", "name": "index", "type": "uint256" } ], "name": "getAssertionIdByIndex", "outputs": [ { "internalType": "bytes32", "name": "", "type": "bytes32" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getAssertionIds", "outputs": [ { "internalType": "bytes32[]", "name": "", "type": "bytes32[]" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getAssertionIdsLength", "outputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" }, { "internalType": "uint256", "name": "assertionIndex", "type": "uint256" } ], "name": "getAssertionIssuer", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getAsset", "outputs": [ { "components": [ { "internalType": "bool", "name": "immutable_", "type": "bool" }, { "internalType": "bytes32[]", "name": "assertionIds", "type": "bytes32[]" } ], "internalType": "struct ContentAssetStructs.Asset", "name": "", "type": "tuple" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getLatestAssertionId", "outputs": [ { "internalType": "bytes32", "name": "", "type": "bytes32" } ], "stateMutability": "view", "type": "function" }, { "inputs": [], "name": "hub", "outputs": [ { "internalType": "contract Hub", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "owner", "type": "address" }, { "internalType": "address", "name": "operator", "type": "address" } ], "name": "isApprovedForAll", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "isMutable", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "bytes32", "name": "", "type": "bytes32" } ], "name": "issuers", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "mint", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [], "name": "name", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "ownerOf", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" } ], "name": "pushAssertionId", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "from", "type": "address" }, { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "safeTransferFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "from", "type": "address" }, { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes", "name": "data", "type": "bytes" } ], "name": "safeTransferFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "operator", "type": "address" }, { "internalType": "bool", "name": "approved", "type": "bool" } ], "name": "setApprovalForAll", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" }, { "internalType": "address", "name": "issuer", "type": "address" } ], "name": "setAssertionIssuer", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bool", "name": "immutable_", "type": "bool" } ], "name": "setMutability", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "bytes4", "name": "interfaceId", "type": "bytes4" } ], "name": "supportsInterface", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [], "name": "symbol", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "tokenURI", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "from", "type": "address" }, { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "transferFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [], "name": "version", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "pure", "type": "function" } ] ================================================ FILE: v8-data-migration/abi/ContentAssetStorageV2.json ================================================ [ { "inputs": [ { "internalType": "address", "name": "hubAddress", "type": "address" } ], "stateMutability": "nonpayable", "type": "constructor" }, { "inputs": [], "name": "NoMintedAssets", "type": "error" }, { "anonymous": false, "inputs": [ { "indexed": true, "internalType": "address", "name": "owner", "type": "address" }, { "indexed": true, "internalType": "address", "name": "approved", "type": "address" }, { "indexed": true, "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "Approval", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": true, "internalType": "address", "name": "owner", "type": "address" }, { "indexed": true, "internalType": "address", "name": "operator", "type": "address" }, { "indexed": false, "internalType": "bool", "name": "approved", "type": "bool" } ], "name": "ApprovalForAll", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "uint256", "name": "_fromTokenId", "type": "uint256" }, { "indexed": false, "internalType": "uint256", "name": "_toTokenId", "type": "uint256" } ], "name": "BatchMetadataUpdate", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "uint256", "name": "_tokenId", "type": "uint256" } ], "name": "MetadataUpdate", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": true, "internalType": "address", "name": "from", "type": "address" }, { "indexed": true, "internalType": "address", "name": "to", "type": "address" }, { "indexed": true, "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "Transfer", "type": "event" }, { "inputs": [ { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "approve", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "bytes32", "name": "assetAssertionId", "type": "bytes32" } ], "name": "assertionExists", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "owner", "type": "address" } ], "name": "balanceOf", "outputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "burn", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" }, { "internalType": "uint256", "name": "index", "type": "uint256" } ], "name": "deleteAssertionIssuer", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "deleteAsset", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [], "name": "generateTokenId", "outputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getApproved", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "uint256", "name": "index", "type": "uint256" } ], "name": "getAssertionIdByIndex", "outputs": [ { "internalType": "bytes32", "name": "", "type": "bytes32" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getAssertionIds", "outputs": [ { "internalType": "bytes32[]", "name": "", "type": "bytes32[]" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getAssertionIdsLength", "outputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" }, { "internalType": "uint256", "name": "assertionIndex", "type": "uint256" } ], "name": "getAssertionIssuer", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getAsset", "outputs": [ { "components": [ { "internalType": "bool", "name": "immutable_", "type": "bool" }, { "internalType": "bytes32[]", "name": "assertionIds", "type": "bytes32[]" } ], "internalType": "struct ContentAssetStructs.Asset", "name": "", "type": "tuple" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "getLatestAssertionId", "outputs": [ { "internalType": "bytes32", "name": "", "type": "bytes32" } ], "stateMutability": "view", "type": "function" }, { "inputs": [], "name": "hub", "outputs": [ { "internalType": "contract Hub", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "owner", "type": "address" }, { "internalType": "address", "name": "operator", "type": "address" } ], "name": "isApprovedForAll", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "isMutable", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "bytes32", "name": "", "type": "bytes32" } ], "name": "issuers", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [], "name": "lastTokenId", "outputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "mint", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [], "name": "name", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "ownerOf", "outputs": [ { "internalType": "address", "name": "", "type": "address" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" } ], "name": "pushAssertionId", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "from", "type": "address" }, { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "safeTransferFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "from", "type": "address" }, { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes", "name": "data", "type": "bytes" } ], "name": "safeTransferFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "operator", "type": "address" }, { "internalType": "bool", "name": "approved", "type": "bool" } ], "name": "setApprovalForAll", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bytes32", "name": "assertionId", "type": "bytes32" }, { "internalType": "address", "name": "issuer", "type": "address" } ], "name": "setAssertionIssuer", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "string", "name": "baseURI", "type": "string" } ], "name": "setBaseURI", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" }, { "internalType": "bool", "name": "immutable_", "type": "bool" } ], "name": "setMutability", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [ { "internalType": "bytes4", "name": "interfaceId", "type": "bytes4" } ], "name": "supportsInterface", "outputs": [ { "internalType": "bool", "name": "", "type": "bool" } ], "stateMutability": "view", "type": "function" }, { "inputs": [], "name": "symbol", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "view", "type": "function" }, { "inputs": [], "name": "tokenBaseURI", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "tokenURI", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "view", "type": "function" }, { "inputs": [ { "internalType": "address", "name": "from", "type": "address" }, { "internalType": "address", "name": "to", "type": "address" }, { "internalType": "uint256", "name": "tokenId", "type": "uint256" } ], "name": "transferFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function" }, { "inputs": [], "name": "version", "outputs": [ { "internalType": "string", "name": "", "type": "string" } ], "stateMutability": "pure", "type": "function" } ] ================================================ FILE: v8-data-migration/blockchain-utils.js ================================================ import { ethers } from 'ethers'; import { BLOCKCHAINS, ABIs, CONTENT_ASSET_STORAGE_CONTRACT } from './constants.js'; import { validateProvider, validateStorageContractAddress, validateStorageContractName, validateStorageContractAbi, validateBlockchainDetails, } from './validation.js'; import logger from './logger.js'; function maskRpcUrl(url) { // Validation if (!url || typeof url !== 'string') { throw new Error(`URL is not defined or it is not a string. URL: ${url}`); } if (url.includes('apiKey')) { return url.split('apiKey')[0]; } return url; } // Initialize rpc export async function initializeRpc(rpcEndpoint) { // Validation if (!rpcEndpoint || typeof rpcEndpoint !== 'string') { logger.error( `RPC endpoint is not defined or it is not a string. RPC endpoint: ${rpcEndpoint}`, ); process.exit(1); } // initialize all possible providers const Provider = ethers.providers.JsonRpcProvider; try { const provider = new Provider(rpcEndpoint); // eslint-disable-next-line no-await-in-loop await provider.getNetwork(); logger.info(`Connected to the blockchain RPC: ${maskRpcUrl(rpcEndpoint)}.`); return provider; } catch (e) { logger.error(`Unable to connect to the blockchain RPC: ${maskRpcUrl(rpcEndpoint)}.`); process.exit(1); } } export async function getStorageContractAndAddress( provider, storageContractAddress, storageContractName, storageContractAbi, ) { // Validation validateProvider(provider); validateStorageContractAddress(storageContractAddress); validateStorageContractName(storageContractName); validateStorageContractAbi(storageContractAbi); logger.info( `Initializing asset contract: ${storageContractName} with address: ${storageContractAddress}`, ); // initialize asset contract const storageContract = new ethers.Contract( storageContractAddress, storageContractAbi, provider, ); logger.info( `Contract ${storageContractName} initialized with address: ${storageContractAddress}`, ); return storageContract; } export async function getContentAssetStorageContract(provider, blockchainDetails) { // Validation validateProvider(provider); validateBlockchainDetails(blockchainDetails); const contentAssetStorageContarct = blockchainDetails.NAME === BLOCKCHAINS.NEUROWEB_TESTNET.NAME || blockchainDetails.NAME === BLOCKCHAINS.NEUROWEB_MAINNET.NAME ? ABIs.ContentAssetStorage : ABIs.ContentAssetStorageV2; const storageContract = await getStorageContractAndAddress( provider, blockchainDetails.CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS, CONTENT_ASSET_STORAGE_CONTRACT, contentAssetStorageContarct, ); return storageContract; } ================================================ FILE: v8-data-migration/constants.js ================================================ import { createRequire } from 'module'; // Triple store constants export const SCHEMA_CONTEXT = 'http://schema.org/'; export const METADATA_NAMED_GRAPH = 'metadata:graph'; export const PRIVATE_ASSERTION_ONTOLOGY = ''; export const TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; export const TRIPLE_STORE_CONNECT_RETRY_FREQUENCY = 10; export const N_QUADS = 'application/n-quads'; export const OT_BLAZEGRAPH = 'ot-blazegraph'; export const OT_FUSEKI = 'ot-fuseki'; export const OT_GRAPHDB = 'ot-graphdb'; export const PRIVATE_CURRENT = 'privateCurrent'; export const PUBLIC_CURRENT = 'publicCurrent'; export const DKG_REPOSITORY = 'dkg'; export const VISIBILITY = { PUBLIC: 'public', PRIVATE: 'private', }; export const BATCH_SIZE = 50; export const MAIN_DIR = '/root'; export const DEFAULT_CONFIG_PATH = `${MAIN_DIR}/ot-node/current/config/config.json`; export const NODERC_CONFIG_PATH = `${MAIN_DIR}/ot-node/.origintrail_noderc`; export const DATA_MIGRATION_DIR = `${MAIN_DIR}/ot-node/data/data-migration`; export const LOG_DIR = `${MAIN_DIR}/ot-node/data/data-migration/logs`; export const ENV_PATH = `${MAIN_DIR}/ot-node/current/.env`; export const MIGRATION_DIR = `${MAIN_DIR}/ot-node/data/migrations/`; export const MIGRATION_PROGRESS_FILE = 'v8DataMigration'; export const DB_URLS = { testnet: 'https://hosting.origin-trail.network/csv/testnet.db', mainnet: 'https://hosting.origin-trail.network/csv/mainnet.db', }; const require = createRequire(import.meta.url); export const ABIs = { ContentAssetStorageV2: require('./abi/ContentAssetStorageV2.json'), ContentAssetStorage: require('./abi/ContentAssetStorage.json'), }; export const BLOCKCHAINS = { BASE_DEVNET: { ID: 'base:84532', ENV: 'devnet', NAME: 'base_devnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0xbe08a25dcf2b68af88501611e5456571f50327b4', }, BASE_TESTNET: { ID: 'base:84532', ENV: 'testnet', NAME: 'base_testnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0x9e3071dc0730cb6dd0ce42969396d716ea33e7e1', }, BASE_MAINNET: { ID: 'base:8453', ENV: 'mainnet', NAME: 'base_mainnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0x3bdfa81079b2ba53a25a6641608e5e1e6c464597', }, GNOSIS_DEVNET: { ID: 'gnosis:10200', ENV: 'devnet', NAME: 'gnosis_devnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0x3db64dd0ac054610d1e2af9cca0fbcb1a7f4c2d8', }, GNOSIS_TESTNET: { ID: 'gnosis:10200', ENV: 'testnet', NAME: 'gnosis_testnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0xea3423e02c8d231532dab1bce5d034f3737b3638', }, GNOSIS_MAINNET: { ID: 'gnosis:100', ENV: 'mainnet', NAME: 'gnosis_mainnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0xf81a8c0008de2dcdb73366cf78f2b178616d11dd', }, NEUROWEB_TESTNET: { ID: 'otp:20430', ENV: 'testnet', NAME: 'neuroweb_testnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0x1a061136ed9f5ed69395f18961a0a535ef4b3e5f', }, NEUROWEB_MAINNET: { ID: 'otp:2043', ENV: 'mainnet', NAME: 'neuroweb_mainnet', CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS: '0x5cac41237127f94c2d21dae0b14bfefa99880630', }, }; export const CONTENT_ASSET_STORAGE_CONTRACT = 'ContentAssetStorage'; ================================================ FILE: v8-data-migration/logger.js ================================================ import pino from 'pino'; import fs from 'fs'; import { LOG_DIR } from './constants.js'; // Ensure logs directory exists if (!fs.existsSync(LOG_DIR)) { fs.mkdirSync(LOG_DIR, { recursive: true }); if (!fs.existsSync(LOG_DIR)) { throw new Error( `Something went wrong. Directory: ${LOG_DIR} does not exist after creation.`, ); } } const timers = new Map(); const baseLogger = pino({ transport: { targets: [ { target: 'pino-pretty', level: 'info', options: { colorize: true, translateTime: 'yyyy-mm-dd HH:MM:ss', }, }, { target: 'pino-pretty', level: 'info', options: { destination: `${LOG_DIR}/migration.log`, colorize: false, translateTime: 'yyyy-mm-dd HH:MM:ss', }, }, ], }, }); // Create enhanced logger with proper method binding const logger = { // Bind all methods from the base logger info: baseLogger.info.bind(baseLogger), error: baseLogger.error.bind(baseLogger), warn: baseLogger.warn.bind(baseLogger), debug: baseLogger.debug.bind(baseLogger), // Add our custom timing methods time(label) { timers.set(label, performance.now()); }, timeEnd(label) { const start = timers.get(label); if (!start) { this.warn(`Timer '${label}' does not exist`); return; } const duration = (performance.now() - start).toFixed(3); timers.delete(label); this.info(`${label}: ${duration}ms`); return duration; }, }; export default logger; ================================================ FILE: v8-data-migration/run-data-migration.sh ================================================ MAIN_DIR=/root cd $MAIN_DIR/ot-node/current/v8-data-migration/ && npm rebuild sqlite3 && nohup node v8-data-migration.js > $MAIN_DIR/ot-node/data/nohup.out 2>&1 & ================================================ FILE: v8-data-migration/sqlite-utils.js ================================================ import sqlite3 from 'sqlite3'; import { open } from 'sqlite'; import path from 'path'; import { DATA_MIGRATION_DIR } from './constants.js'; import logger from './logger.js'; export class SqliteDatabase { constructor() { this.db = null; } async initialize() { if (this.db) { return; } const dbPath = path.join(DATA_MIGRATION_DIR, `${process.env.NODE_ENV}.db`); this.db = await open({ filename: dbPath, driver: sqlite3.Database, }); if (!this.db) { logger.error('Failed to initialize SQLite database'); process.exit(1); } } async checkIntegrity() { this._validateConnection(); try { const result = await this.db.get('PRAGMA integrity_check;'); if (result.integrity_check === 'ok') { logger.info('Database integrity check passed.'); return true; } logger.error('Database integrity check failed:', result.integrity_check); return false; } catch (error) { logger.error('Error during integrity check:', error.message); return false; } } async getTableExists(blockchainName) { this._validateConnection(); this._validateBlockchainName(blockchainName); const tableExists = await this.db.get( `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, [blockchainName], ); return tableExists; } async getBatchOfUnprocessedTokenIds(blockchainName, batchSize) { this._validateConnection(); this._validateBlockchainName(blockchainName); const rows = await this.db.all( ` SELECT token_id, ual, assertion_id FROM ${blockchainName} WHERE processed = 0 LIMIT ? `, batchSize, ); const batchData = {}; rows.forEach((row) => { batchData[row.token_id] = { ual: row.ual, assertionId: row.assertion_id, processed: 'false', }; }); return batchData; } async markRowsAsProcessed(blockchainName, tokenIds) { this._validateConnection(); this._validateBlockchainName(blockchainName); const placeholders = tokenIds.map(() => '?').join(','); await this.db.run( ` UPDATE ${blockchainName} SET processed = 1 WHERE token_id IN (${placeholders}) `, tokenIds, ); } async getHighestTokenId(blockchainName) { this._validateConnection(); this._validateBlockchainName(blockchainName); const result = await this.db.get(`SELECT MAX(token_id) as max_id FROM ${blockchainName}`); return result.max_id; } async getUnprocessedCount(blockchainName) { this._validateConnection(); this._validateBlockchainName(blockchainName); const result = await this.db.get(` SELECT COUNT(*) as count FROM ${blockchainName} WHERE processed = 0 `); return result.count; } async insertAssertion(blockchainName, tokenId, ual, assertionId) { this._validateConnection(); this._validateBlockchainName(blockchainName); try { await this.db.run( `INSERT OR IGNORE INTO ${blockchainName} (token_id, ual, assertion_id, processed) VALUES (?, ?, ?, 1)`, [tokenId, ual, assertionId], ); return true; } catch (error) { logger.error(`Error inserting assertion into ${blockchainName} table:`, error.message); return false; } } async close() { if (this.db) { await this.db.close(); this.db = null; } } _validateConnection() { if (!this.db) { logger.error('Database not initialized. Call initialize() first.'); process.exit(1); } } _validateBlockchainName(blockchainName) { if (!blockchainName) { logger.error('Blockchain name is required'); process.exit(1); } } } // Export a single instance const sqliteDb = new SqliteDatabase(); export default sqliteDb; ================================================ FILE: v8-data-migration/triple-store-utils.js ================================================ import { setTimeout } from 'timers/promises'; import axios from 'axios'; import { OT_BLAZEGRAPH, OT_FUSEKI, OT_GRAPHDB, N_QUADS, SCHEMA_CONTEXT, PRIVATE_ASSERTION_ONTOLOGY, PUBLIC_CURRENT, PRIVATE_CURRENT, DKG_REPOSITORY, VISIBILITY, METADATA_NAMED_GRAPH, TRIPLE_STORE_CONNECT_MAX_RETRIES, TRIPLE_STORE_CONNECT_RETRY_FREQUENCY, } from './constants.js'; import { validateTripleStoreConfig, validateTripleStoreRepositories, validateTripleStoreImplementation, validateRepository, validateQuery, validateAssertionId, validateTokenId, validateAssertion, validateUal, } from './validation.js'; import logger from './logger.js'; export function getTripleStoreData(tripleStoreConfig) { // Validation validateTripleStoreConfig(tripleStoreConfig); let tripleStoreImplementation; const tripleStoreRepositories = {}; for (const [implementationName, implementationDetails] of Object.entries( tripleStoreConfig.implementation, )) { if (implementationDetails.enabled) { tripleStoreImplementation = implementationName; for (const [repository, repositoryDetails] of Object.entries( implementationDetails.config.repositories, )) { if ( repository === PRIVATE_CURRENT || repository === PUBLIC_CURRENT || repository === DKG_REPOSITORY ) { tripleStoreRepositories[repository] = repositoryDetails; } } break; } } return { tripleStoreImplementation, tripleStoreRepositories }; } // Initialize sparql endpoints function initializeSparqlEndpoints(tripleStoreRepositories, repository, tripleStoreImplementation) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateTripleStoreImplementation(tripleStoreImplementation); const { url, name } = tripleStoreRepositories[repository]; const updatedTripleStoreRepositories = tripleStoreRepositories; switch (tripleStoreImplementation) { case OT_BLAZEGRAPH: updatedTripleStoreRepositories[ repository ].sparqlEndpoint = `${url}/blazegraph/namespace/${name}/sparql`; updatedTripleStoreRepositories[ repository ].sparqlEndpointUpdate = `${url}/blazegraph/namespace/${name}/sparql`; break; case OT_FUSEKI: updatedTripleStoreRepositories[repository].sparqlEndpoint = `${url}/${name}/sparql`; updatedTripleStoreRepositories[ repository ].sparqlEndpointUpdate = `${url}/${name}/update`; break; case OT_GRAPHDB: updatedTripleStoreRepositories[ repository ].sparqlEndpoint = `${url}/repositories/${name}`; updatedTripleStoreRepositories[ repository ].sparqlEndpointUpdate = `${url}/repositories/${name}/statements`; break; default: throw new Error('Invalid triple store name in initializeSparqlEndpoints'); } return updatedTripleStoreRepositories; } export function initializeRepositories(tripleStoreRepositories, tripleStoreImplementation) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateTripleStoreImplementation(tripleStoreImplementation); let updatedTripleStoreRepositories = tripleStoreRepositories; for (const repository in tripleStoreRepositories) { logger.info(`Initializing a triple store repository: ${repository}`); updatedTripleStoreRepositories = initializeSparqlEndpoints( updatedTripleStoreRepositories, repository, tripleStoreImplementation, ); } return updatedTripleStoreRepositories; } export function initializeContexts(tripleStoreRepositories) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); const updatedTripleStoreRepositories = tripleStoreRepositories; for (const repository in updatedTripleStoreRepositories) { const sources = [ { type: 'sparql', value: updatedTripleStoreRepositories[repository].sparqlEndpoint, }, ]; updatedTripleStoreRepositories[repository].updateContext = { sources, destination: { type: 'sparql', value: updatedTripleStoreRepositories[repository].sparqlEndpointUpdate, }, }; updatedTripleStoreRepositories[repository].queryContext = { sources, }; } return updatedTripleStoreRepositories; } export async function healthCheck(tripleStoreRepositories, repository, tripleStoreImplementation) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateTripleStoreImplementation(tripleStoreImplementation); switch (tripleStoreImplementation) { case OT_BLAZEGRAPH: { try { const response = await axios.get( `${tripleStoreRepositories[repository].url}/blazegraph/status`, {}, ); if (response.data !== null) { return true; } return false; } catch (e) { logger.error(`Health check failed for repository ${repository}:`, e); return false; } } case OT_FUSEKI: { try { const response = await axios.get( `${tripleStoreRepositories[repository].url}/$/ping`, {}, ); if (response.data !== null) { return true; } return false; } catch (e) { logger.error(`Health check failed for repository ${repository}:`, e); return false; } } case OT_GRAPHDB: { const { url, username, password } = tripleStoreRepositories[repository]; try { const response = await axios.get( `${url}/repositories/${repository}/health`, {}, { auth: { username, password, }, }, ); if (response.data.status === 'green') { return true; } return false; } catch (e) { if (e.response && e.response.status === 404) { // Expected error: GraphDB is up but has not created node0 repository // dkg-engine will create repo in initialization return true; } logger.error(`Health check failed for repository ${repository}:`, e); return false; } } default: throw new Error('Invalid triple store name in healthCheck'); } } export async function ensureConnections(tripleStoreRepositories, tripleStoreImplementation) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateTripleStoreImplementation(tripleStoreImplementation); const ensureConnectionPromises = Object.keys(tripleStoreRepositories).map( async (repository) => { let ready = await healthCheck( tripleStoreRepositories, repository, tripleStoreImplementation, ); let retries = 0; while (!ready && retries < TRIPLE_STORE_CONNECT_MAX_RETRIES) { retries += 1; logger.warn( `Cannot connect to Triple store repository: ${repository}, located at: ${tripleStoreRepositories[repository].url} retry number: ${retries}/${TRIPLE_STORE_CONNECT_MAX_RETRIES}. Retrying in ${TRIPLE_STORE_CONNECT_RETRY_FREQUENCY} seconds.`, ); /* eslint-disable no-await-in-loop */ await setTimeout(TRIPLE_STORE_CONNECT_RETRY_FREQUENCY * 1000); ready = await healthCheck( tripleStoreRepositories, repository, tripleStoreImplementation, ); } if (retries === TRIPLE_STORE_CONNECT_MAX_RETRIES) { logger.error( `Triple Store repository: ${repository} not available, max retries reached.`, ); process.exit(1); } }, ); await Promise.all(ensureConnectionPromises); } // blazegraph only function hasUnicodeCodePoints(input) { const unicodeRegex = /(? { const codePoint = parseInt(hex, 16); return String.fromCodePoint(codePoint); }); return decodedString; } export async function _executeQuery( tripleStoreRepositories, repository, tripleStoreImplementation, query, mediaType, ) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateTripleStoreImplementation(tripleStoreImplementation); validateQuery(query); if (!mediaType) { logger.error(`[VALIDATION ERROR] Media type is not defined. Media type: ${mediaType}`); process.exit(1); } const response = await axios.post( tripleStoreRepositories[repository].sparqlEndpoint, new URLSearchParams({ query, }), { headers: { Accept: mediaType, }, }, ); let { data } = response; if (tripleStoreImplementation === OT_BLAZEGRAPH) { // Handle Blazegraph special characters corruption if (hasUnicodeCodePoints(data)) { data = decodeUnicodeCodePoints(data); } } return data; } export async function construct( tripleStoreRepositories, repository, tripleStoreImplementation, query, ) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateTripleStoreImplementation(tripleStoreImplementation); validateQuery(query); return _executeQuery( tripleStoreRepositories, repository, tripleStoreImplementation, query, N_QUADS, ); } function cleanEscapeCharacter(query) { return query.replace(/['|[\]\\]/g, '\\$&'); } export async function getAssertion( tripleStoreRepositories, repository, tripleStoreImplementation, assertionId, ) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateTripleStoreImplementation(tripleStoreImplementation); validateAssertionId(assertionId); const escapedGraphName = cleanEscapeCharacter(assertionId); const query = `PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } WHERE { { GRAPH { ?s ?p ?o . } } }`; return construct(tripleStoreRepositories, repository, tripleStoreImplementation, query); } export function extractPrivateAssertionId(publicAssertion) { // Validation validateAssertion(publicAssertion); const split = publicAssertion.split(PRIVATE_ASSERTION_ONTOLOGY); if (split.length <= 1 || !split[1].includes('"') || !split[1].includes('0x')) { return null; } const input = split[1]; const openingQuoteIndex = input.indexOf('"') + 1; const closingQuoteIndex = input.indexOf('"', openingQuoteIndex); const privateAssertionId = input.substring(openingQuoteIndex, closingQuoteIndex).trim(); if (!privateAssertionId || !privateAssertionId.includes('0x')) { return null; } return privateAssertionId; } export async function getAssertionFromV6TripleStore( tripleStoreRepositories, tripleStoreImplementation, tokenId, ualAssertionIdData, ) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateTripleStoreImplementation(tripleStoreImplementation); validateTokenId(tokenId); if ( !ualAssertionIdData || typeof ualAssertionIdData !== 'object' || !ualAssertionIdData.assertionId || !ualAssertionIdData.ual ) { logger.error( `[VALIDATION ERROR] Ual assertion ID data is not properly defined or it is not an object. Ual assertion ID data: ${ualAssertionIdData}`, ); process.exit(1); } const { assertionId, ual } = ualAssertionIdData; let success = false; let publicAssertion = null; let privateAssertion = null; try { // First try to fetch public data from private current repository publicAssertion = await getAssertion( tripleStoreRepositories, PRIVATE_CURRENT, tripleStoreImplementation, assertionId, ); // Check if public assertion is found in private current repository if (publicAssertion) { success = true; // Check if assertion contains a private assertion if (publicAssertion.includes(PRIVATE_ASSERTION_ONTOLOGY)) { // Extract the private assertionId from the publicAssertion if it exists const privateAssertionId = extractPrivateAssertionId(publicAssertion); if (!privateAssertionId) { logger.warn( `There was a problem while extracting the private assertionId from public assertion: ${publicAssertion}. Extracted privateAssertionId: ${privateAssertionId}`, ); success = false; return { tokenId, ual, publicAssertion, privateAssertion, success }; } privateAssertion = await getAssertion( tripleStoreRepositories, PRIVATE_CURRENT, tripleStoreImplementation, privateAssertionId, ); // If private assertionId exists but assertion could not be fetched if (!privateAssertion) { logger.warn( `Private assertion with id ${privateAssertionId} could not be fetched from ${PRIVATE_CURRENT} repository even though it should exist`, ); } } } else { publicAssertion = await getAssertion( tripleStoreRepositories, PUBLIC_CURRENT, tripleStoreImplementation, assertionId, ); success = true; } } catch (e) { logger.error( `Error fetching assertion from triple store for tokenId: ${tokenId}, assertionId: ${assertionId}, error: ${e}`, ); success = false; } return { tokenId, ual, publicAssertion, privateAssertion, success, assertionId, }; } export function processContent(str) { return str .split('\n') .map((line) => line.trim()) .filter((line) => line !== ''); } async function ask(tripleStoreRepositories, repository, query) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateQuery(query); try { const response = await axios.post( tripleStoreRepositories[repository].sparqlEndpoint, new URLSearchParams({ query, }), { headers: { Accept: 'application/json', }, }, ); return response.data.boolean; } catch (e) { logger.error( `Error while doing ASK query: ${query} in repository: ${repository}. Error: ${e.message}`, ); return false; } } export async function getKnowledgeCollectionNamedGraphsExist( tokenId, tripleStoreRepositories, knowledgeAssetUal, privateAssertion, ) { const askQueries = []; askQueries.push(` FILTER EXISTS { GRAPH <${knowledgeAssetUal}/${VISIBILITY.PUBLIC}> { ?s ?p ?o } } `); if (privateAssertion) { askQueries.push(` FILTER EXISTS { GRAPH <${knowledgeAssetUal}/${VISIBILITY.PRIVATE}> { ?s ?p ?o } } `); } askQueries.push(` FILTER EXISTS { GRAPH <${METADATA_NAMED_GRAPH}> { <${knowledgeAssetUal}> ?p ?o . } } `); const combinedQuery = ` ASK { ${askQueries.join('\n')} } `; const exists = await ask(tripleStoreRepositories, DKG_REPOSITORY, combinedQuery); return { tokenId, exists }; } export async function queryVoid(tripleStoreRepositories, repository, query) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateQuery(query); await axios.post(tripleStoreRepositories[repository].sparqlEndpointUpdate, query, { headers: { 'Content-Type': 'application/sparql-update', }, }); } function hasSpecialCharactersInIRI(assertion) { const lines = assertion.split('\n'); // {, }, |, ^, `, and \ without u or U // eslint-disable-next-line no-useless-escape const iriPattern = /<[^>]*(?:[\s{}\|^`]|\\(?![uU]))[^>]*>/; return lines.some((line) => { // Split quad into subject, predicate, object (ignore graph if present) const parts = line.trim().split(' '); // Check each part only if it starts with < and ends with > return parts.some((part) => { if (part.startsWith('<') && part.endsWith('>')) { return iriPattern.test(part); } return false; }); }); } export async function insertAssertionsIntoV8UnifiedRepository( v6Assertions, tripleStoreRepositories, ) { // Insert into new repository const successfullyProcessed = []; const assertionsToCheck = []; const insertQueries = []; for (const assertion of v6Assertions) { const { tokenId, ual, publicAssertion, privateAssertion } = assertion; // Assertion with assertionId does not exist in triple store. Continue if (!publicAssertion) { successfullyProcessed.push(tokenId); continue; } if (hasSpecialCharactersInIRI(publicAssertion)) { logger.warn( `Public assertion with tokenId: ${tokenId} contains illegal characters in IRI. Skipping... Public assertion: ${publicAssertion}`, ); successfullyProcessed.push(tokenId); continue; } const knowledgeAssetUal = `${ual.toLowerCase()}/1`; const publicNQuads = processContent(publicAssertion); insertQueries.push(` GRAPH <${knowledgeAssetUal}/${VISIBILITY.PUBLIC}> { ${publicNQuads.join('\n')} } `); if (privateAssertion) { const privateNQuads = processContent(privateAssertion); insertQueries.push(` GRAPH <${knowledgeAssetUal}/${VISIBILITY.PRIVATE}> { ${privateNQuads.join('\n')} } `); } const metadataNQuads = `<${knowledgeAssetUal}> "${knowledgeAssetUal}:0" .`; insertQueries.push(` GRAPH <${METADATA_NAMED_GRAPH}> { ${metadataNQuads} } `); assertionsToCheck.push(assertion); } if (insertQueries.length > 0) { const combinedQuery = ` PREFIX schema: <${SCHEMA_CONTEXT}> INSERT DATA { ${insertQueries.join('\n')} } `; logger.time(`INSERTING ${assertionsToCheck.length} ASSERTIONS INTO V8 TRIPLE STORE`); await queryVoid(tripleStoreRepositories, DKG_REPOSITORY, combinedQuery); logger.timeEnd(`INSERTING ${assertionsToCheck.length} ASSERTIONS INTO V8 TRIPLE STORE`); } return { successfullyProcessed, assertionsToCheck }; } export async function knowledgeCollectionNamedGraphExists( tripleStoreRepositories, repository, ual, visibility, ) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateUal(ual); if (!visibility || !Object.values(VISIBILITY).includes(visibility)) { throw new Error( `Visibility is not defined or it is not a valid visibility. Visibility: ${visibility}`, ); } const query = ` ASK { GRAPH ?g { ?s ?p ?o } FILTER(STRSTARTS(STR(?g), "${ual}/${visibility}")) } `; return ask(tripleStoreRepositories, repository, query); } export async function knowledgeAssetMetadataExists(tripleStoreRepositories, repository, ual) { // Validation validateTripleStoreRepositories(tripleStoreRepositories); validateRepository(repository); validateUal(ual); const query = ` ASK { GRAPH <${METADATA_NAMED_GRAPH}> { <${ual}> ?p ?o . } } `; return ask(tripleStoreRepositories, repository, query); } ================================================ FILE: v8-data-migration/v8-data-migration-utils.js ================================================ import fs from 'fs'; import path from 'path'; import { NODERC_CONFIG_PATH, MIGRATION_PROGRESS_FILE, DEFAULT_CONFIG_PATH, MIGRATION_DIR, } from './constants.js'; import { validateConfig } from './validation.js'; import logger from './logger.js'; export function initializeConfig() { const configPath = path.resolve(NODERC_CONFIG_PATH); const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); validateConfig(config); return config; } export function initializeDefaultConfig() { const configPath = path.resolve(DEFAULT_CONFIG_PATH); const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); validateConfig(config); return config; } export function ensureDirectoryExists(dirPath) { if (!fs.existsSync(dirPath)) { fs.mkdirSync(dirPath, { recursive: true }); logger.info(`Created directory: ${dirPath}`); if (!fs.existsSync(dirPath)) { logger.error( `Something went wrong. Directory: ${dirPath} does not exist after creation.`, ); process.exit(1); } } } export function ensureMigrationProgressFileExists() { ensureDirectoryExists(MIGRATION_DIR); const migrationProgressFilePath = path.join(MIGRATION_DIR, MIGRATION_PROGRESS_FILE); if (!fs.existsSync(migrationProgressFilePath)) { fs.writeFileSync(migrationProgressFilePath, ''); logger.info(`Created migration progress file: ${migrationProgressFilePath}`); if (!fs.existsSync(migrationProgressFilePath)) { throw new Error( `Something went wrong. Progress file: ${migrationProgressFilePath} does not exist after creation.`, ); } } else { logger.info(`Migration progress file already exists: ${migrationProgressFilePath}.`); logger.info('Checking if migration is already successful...'); const fileContent = fs.readFileSync(migrationProgressFilePath, 'utf8'); if (fileContent === 'MIGRATED') { logger.info('Migration is already successful. Exiting...'); process.exit(0); } } } export function markMigrationAsSuccessfull() { // Construct the full path to the migration progress file const migrationProgressFilePath = path.join(MIGRATION_DIR, MIGRATION_PROGRESS_FILE); // open file const file = fs.openSync(migrationProgressFilePath, 'w'); // write MIGRATED fs.writeSync(file, 'MIGRATED'); // close file fs.closeSync(file); } export function deleteFile(filePath) { if (fs.existsSync(filePath)) { fs.unlinkSync(filePath); logger.info(`Deleted file: ${filePath}`); if (fs.existsSync(filePath)) { logger.error(`File: ${filePath} still exists after deletion.`); process.exit(1); } } else { logger.info(`Did not delete file: ${filePath} because it does not exist.`); } } ================================================ FILE: v8-data-migration/v8-data-migration.js ================================================ import path from 'path'; import fs from 'fs'; import { createRequire } from 'module'; import dotenv from 'dotenv'; import axios from 'axios'; import { BATCH_SIZE, ENV_PATH, BLOCKCHAINS, DATA_MIGRATION_DIR, DB_URLS } from './constants.js'; import { initializeConfig, initializeDefaultConfig, ensureDirectoryExists, ensureMigrationProgressFileExists, markMigrationAsSuccessfull, deleteFile, } from './v8-data-migration-utils.js'; import { getAssertionFromV6TripleStore, insertAssertionsIntoV8UnifiedRepository, getTripleStoreData, initializeRepositories, initializeContexts, ensureConnections, getKnowledgeCollectionNamedGraphsExist, } from './triple-store-utils.js'; import { getContentAssetStorageContract, initializeRpc } from './blockchain-utils.js'; import { validateBlockchainName, validateBlockchainDetails, validateTokenId, validateTripleStoreRepositories, validateTripleStoreImplementation, validateBatchData, } from './validation.js'; import sqliteDb from './sqlite-utils.js'; import logger from './logger.js'; dotenv.config({ path: ENV_PATH, override: true }); const require = createRequire(import.meta.url); const { setTimeout } = require('timers/promises'); const successfulInsertsSet = new Set(); const totalInsertsSet = new Set(); async function processAndInsertNewerAssertions( blockchainDetails, blockchainName, highestTokenId, tripleStoreRepositories, tripleStoreImplementation, rpcEndpoints, ) { // Validation validateBlockchainName(blockchainName); validateBlockchainDetails(blockchainDetails); validateTokenId(highestTokenId); validateTripleStoreRepositories(tripleStoreRepositories); validateTripleStoreImplementation(tripleStoreImplementation); const provider = await initializeRpc(rpcEndpoints[0]); const storageContract = await getContentAssetStorageContract(provider, blockchainDetails); let assertionExists = true; let newTokenId = highestTokenId; /* eslint-disable no-await-in-loop */ while (assertionExists) { // increase the tokenId by 1 newTokenId += 1; logger.info(`Fetching assertion for tokenId: ${newTokenId}`); // construct new ual const newUal = `did:dkg:${blockchainDetails.ID}/${blockchainDetails.CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS}/${newTokenId}`; const assertionIds = await storageContract.getAssertionIds(newTokenId); if (assertionIds.length === 0) { logger.info( `You have processed all assertions on ${blockchainName}. Moving to the next blockchain...`, ); assertionExists = false; break; } // Get the latest assertionId const assertionId = assertionIds[assertionIds.length - 1]; const assertion = await getAssertionFromV6TripleStore( tripleStoreRepositories, tripleStoreImplementation, newTokenId, { assertionId, ual: newUal, }, ); if (!assertion.success) { logger.error( `Assertion with assertionId ${assertionId} exists in V6 triple store but could not be fetched. Retrying...`, ); newTokenId -= 1; continue; } const { successfullyProcessed, assertionsToCheck } = await insertAssertionsIntoV8UnifiedRepository([assertion], tripleStoreRepositories); if (successfullyProcessed.length > 0) { logger.info( `Assertion with assertionId ${assertionId} does not exist in V6 triple store.`, ); } if (assertionsToCheck.length > 0) { const { tokenId, ual, privateAssertion } = assertionsToCheck[0]; const knowledgeAssetUal = `${ual.toLowerCase()}/1`; logger.time(`GETTING KNOWLEDGE COLLECTION NAMED GRAPHS EXIST FOR 1 ASSERTION`); // eslint-disable-next-line no-await-in-loop const { exists } = await getKnowledgeCollectionNamedGraphsExist( tokenId, tripleStoreRepositories, knowledgeAssetUal, privateAssertion, ); logger.timeEnd(`GETTING KNOWLEDGE COLLECTION NAMED GRAPHS EXIST FOR 1 ASSERTION`); if (!exists) { logger.error( `Assertion with assertionId ${assertionId} was inserted but its KA named graph does not exist. Retrying...`, ); newTokenId -= 1; continue; } logger.info( `Successfully inserted public/private assertions into V8 triple store for tokenId: ${newTokenId}`, ); successfulInsertsSet.add(assertionId); totalInsertsSet.add(newTokenId); } const inserted = await sqliteDb.insertAssertion( blockchainName, newTokenId, newUal, assertionId, ); if (!inserted) { logger.error( `Assertion with assertionId ${assertionId} could not be inserted. Retrying...`, ); newTokenId -= 1; continue; } logger.info( `Assertion with tokenId ${newTokenId} inserted into db and marked as processed.`, ); } } async function processAndInsertAssertions( v6Assertions, tripleStoreRepositories, tripleStoreImplementation, ) { // Validation if (!v6Assertions || !Array.isArray(v6Assertions)) { throw new Error( `v6Assertions is not defined or it is not an array. V6 assertions: ${v6Assertions}`, ); } validateTripleStoreRepositories(tripleStoreRepositories); validateTripleStoreImplementation(tripleStoreImplementation); const { successfullyProcessed, assertionsToCheck } = await insertAssertionsIntoV8UnifiedRepository(v6Assertions, tripleStoreRepositories); logger.info( `Number of assertions that do not exist in V6 triple store: ${successfullyProcessed.length}`, ); logger.info(`Verifying V8 triple store insertions for ${assertionsToCheck.length} assertions`); const promises = []; for (const assertion of assertionsToCheck) { const { tokenId, ual, privateAssertion } = assertion; const knowledgeAssetUal = `${ual.toLowerCase()}/1`; promises.push( getKnowledgeCollectionNamedGraphsExist( tokenId, tripleStoreRepositories, knowledgeAssetUal, privateAssertion, ), ); } logger.time( `GETTING KNOWLEDGE COLLECTION NAMED GRAPHS EXIST FOR ${promises.length} ASSERTIONS`, ); const results = await Promise.all(promises); logger.timeEnd( `GETTING KNOWLEDGE COLLECTION NAMED GRAPHS EXIST FOR ${promises.length} ASSERTIONS`, ); const successfulInserts = results .filter((result) => result.exists) .map((result) => result.tokenId); // Find the assertion associated with the tokenId successfulInserts.forEach((tokenId) => { const assertionInserted = assertionsToCheck.find( (assertion) => assertion.tokenId === tokenId, ); if (assertionInserted) { successfulInsertsSet.add(assertionInserted.assertionId); } }); successfulInserts.forEach((tokenId) => { totalInsertsSet.add(tokenId); }); logger.info(`Number of successfully inserted assertions: ${successfulInserts.length}`); successfullyProcessed.push(...successfulInserts); logger.info(`Successfully processed assertions: ${successfullyProcessed.length}`); return successfullyProcessed; } async function getAssertionsInBatch( batchKeys, batchData, tripleStoreRepositories, tripleStoreImplementation, ) { // Validation if (!batchKeys || !Array.isArray(batchKeys)) { logger.error(`Batch keys is not defined or it is not an array. Batch keys: ${batchKeys}`); process.exit(1); } validateBatchData(batchData); validateTripleStoreRepositories(tripleStoreRepositories); validateTripleStoreImplementation(tripleStoreImplementation); const batchPromises = []; for (const tokenId of batchKeys) { batchPromises.push( getAssertionFromV6TripleStore( tripleStoreRepositories, tripleStoreImplementation, tokenId, batchData[tokenId], ), ); } const batchResults = await Promise.all(batchPromises); // Get all successful assertions const v6Assertions = batchResults.filter((result) => result.success); return v6Assertions; } async function downloadDb(dbFilePath) { logger.time(`Database file downloading time`); const maxAttempts = 3; for (let i = 0; i < maxAttempts; i += 1) { // Fetch the db file from the remote server logger.info( `Fetching ${process.env.NODE_ENV}.db file from ${DB_URLS[process.env.NODE_ENV]}. Try ${ i + 1 } of 3. This may take a while...`, ); try { const writer = fs.createWriteStream(dbFilePath); const response = await axios({ url: DB_URLS[process.env.NODE_ENV], method: 'GET', responseType: 'stream', }); // Pipe the response stream to the file response.data.pipe(writer); await new Promise((resolve, reject) => { let downloadComplete = false; response.data.on('end', () => { downloadComplete = true; }); writer.on('finish', resolve); writer.on('error', (err) => reject(new Error(`Write stream error: ${err.message}`)), ); response.data.on('error', (err) => reject(new Error(`Download stream error: ${err.message}`)), ); response.data.on('close', () => { if (!downloadComplete) { reject(new Error('Download stream closed before completing')); } }); }); if (fs.existsSync(dbFilePath)) { logger.info(`DB file downloaded successfully`); break; } logger.error(`DB file for ${process.env.NODE_ENV} is not present after download.`); } catch (error) { logger.error(`Error downloading DB file: ${error.message}`); } logger.info('Deleting downloaded db file to prevent data corruption'); deleteFile(dbFilePath); if (i === maxAttempts - 1) { logger.error('Max db download attempts reached. Terminating process...'); process.exit(1); } logger.info(`Retrying db download...`); } logger.timeEnd(`Database file downloading time`); } async function main() { ensureMigrationProgressFileExists(); // Make sure data/data-migration directory exists ensureDirectoryExists(DATA_MIGRATION_DIR); // initialize noderc config const config = initializeConfig(); // initialize default config const defaultConfig = initializeDefaultConfig(); // Initialize blockchain config const blockchainConfig = config.modules.blockchain; if (!blockchainConfig || !blockchainConfig.implementation) { logger.error('Invalid configuration for blockchain.'); process.exit(1); } logger.info('TRIPLE STORE INITIALIZATION START'); // Initialize triple store config const tripleStoreConfig = config.modules.tripleStore; if (!tripleStoreConfig || !tripleStoreConfig.implementation) { logger.error('Invalid configuration for triple store.'); process.exit(1); } const tripleStoreData = getTripleStoreData(tripleStoreConfig); // eslint-disable-next-line prefer-destructuring const tripleStoreImplementation = tripleStoreData.tripleStoreImplementation; // eslint-disable-next-line prefer-destructuring let tripleStoreRepositories = tripleStoreData.tripleStoreRepositories; if (Object.keys(tripleStoreRepositories).length !== 3) { logger.error( `Triple store repositories are not initialized correctly. Expected 3 repositories, got: ${ Object.keys(tripleStoreRepositories).length }`, ); process.exit(1); } // Initialize repositories tripleStoreRepositories = initializeRepositories( tripleStoreRepositories, tripleStoreImplementation, ); // Initialize contexts tripleStoreRepositories = initializeContexts(tripleStoreRepositories); // Ensure connections await ensureConnections(tripleStoreRepositories, tripleStoreImplementation); const maxAttempts = 2; for (let i = 0; i < maxAttempts; i += 1) { // Check if db exists and if it doesn't download it to the relevant directory const dbFilePath = path.join(DATA_MIGRATION_DIR, `${process.env.NODE_ENV}.db`); if (!fs.existsSync(dbFilePath)) { logger.info( `DB file for ${process.env.NODE_ENV} does not exist in ${DATA_MIGRATION_DIR}. Downloading it...`, ); await downloadDb(dbFilePath); } logger.info('Initializing SQLite database'); await sqliteDb.initialize(); // Check if db is corrupted and handle accordingly const integrityCheck = await sqliteDb.checkIntegrity(); if (!integrityCheck) { await sqliteDb.close(); logger.info('Db integrity check failed. Deleting corrupt db file.'); deleteFile(dbFilePath); if (i === maxAttempts - 1) { logger.error('Db integrity check failed. Terminating process...'); process.exit(1); } logger.info(`Retrying db download and integrity check...`); continue; } break; } try { // make sure blockchains are always migrated in this order - base, gnosis, neuroweb const sortedBlockchains = Object.keys(blockchainConfig.implementation).sort(); // Iterate through all chains for (const blockchain of sortedBlockchains) { logger.time(`PROCESSING TIME FOR ${blockchain}`); let processed = 0; const blockchainImplementation = blockchainConfig.implementation[blockchain]; if (!blockchainImplementation.enabled) { logger.info(`Blockchain ${blockchain} is not enabled. Skipping...`); continue; } const rpcEndpoints = blockchainImplementation?.config?.rpcEndpoints ? blockchainImplementation.config.rpcEndpoints : defaultConfig[process.env.NODE_ENV].modules.blockchain.implementation[blockchain] .config.rpcEndpoints; if (!Array.isArray(rpcEndpoints) || rpcEndpoints.length === 0) { logger.error(`RPC endpoints are not defined for blockchain ${blockchain}.`); process.exit(1); } let blockchainName; let blockchainDetails; for (const [, details] of Object.entries(BLOCKCHAINS)) { if (details.ID === blockchain && details.ENV === process.env.NODE_ENV) { blockchainName = details.NAME; blockchainDetails = details; break; } } if (!blockchainName) { logger.error( `Blockchain ${blockchain} not found. Make sure you have the correct blockchain ID and correct NODE_ENV in .env file.`, ); process.exit(1); } const tableExists = await sqliteDb.getTableExists(blockchainName); if (!tableExists) { logger.error(`Required table "${blockchainName}" does not exist in the database`); process.exit(1); } const highestTokenId = await sqliteDb.getHighestTokenId(blockchainName); if (!highestTokenId) { logger.error( `Something went wrong. Could not fetch highest tokenId for ${blockchainName}.`, ); process.exit(1); } logger.info(`Total amount of tokenIds: ${highestTokenId}`); const tokenIdsToProcessCount = await sqliteDb.getUnprocessedCount(blockchainName); logger.info(`Amount of tokenIds left to process: ${tokenIdsToProcessCount}`); // Process tokens in batches while (true) { logger.time('BATCH PROCESSING TIME'); const batchData = await sqliteDb.getBatchOfUnprocessedTokenIds( blockchainName, BATCH_SIZE, ); const batchKeys = Object.keys(batchData); if (batchKeys.length === 0) { logger.info('No more unprocessed tokenIds found. Moving on...'); logger.timeEnd('BATCH PROCESSING TIME'); break; } logger.info(`Processing batch: ${batchKeys}`); try { logger.time('FETCHING V6 ASSERTIONS'); const v6Assertions = await getAssertionsInBatch( batchKeys, batchData, tripleStoreRepositories, tripleStoreImplementation, ); logger.timeEnd('FETCHING V6 ASSERTIONS'); if (v6Assertions.length === 0) { throw new Error( `Something went wrong. Could not get any V6 assertions in batch ${batchKeys}`, ); } logger.info(`Number of V6 assertions to process: ${v6Assertions.length}`); const successfullyProcessed = await processAndInsertAssertions( v6Assertions, tripleStoreRepositories, tripleStoreImplementation, ); if (successfullyProcessed.length === 0) { throw new Error( `Could not insert any assertions out of ${v6Assertions.length}`, ); } logger.info( `Successfully processed/inserted assertions: ${successfullyProcessed.length}. Marking rows as processed in db...`, ); await sqliteDb.markRowsAsProcessed(blockchainName, successfullyProcessed); processed += successfullyProcessed.length; logger.info( `[PROGRESS] for ${blockchainName}: ${( (processed / tokenIdsToProcessCount) * 100 ).toFixed(2)}%. Total processed: ${processed}/${tokenIdsToProcessCount}`, ); // Pause for 500ms to deload the triple store await setTimeout(500); } catch (error) { logger.error(`Error processing batch: ${error}. Pausing for 5 seconds...`); await setTimeout(5000); } } logger.timeEnd(`PROCESSING TIME FOR ${blockchain}`); logger.time(`PROCESS AND INSERT NEWER ASSERTIONS FOR ${blockchainName}`); // If newer (unprocessed) assertions exist on-chain, fetch them and insert them into the V8 triple store repository // eslint-disable-next-line no-await-in-loop await processAndInsertNewerAssertions( blockchainDetails, blockchainName, highestTokenId, tripleStoreRepositories, tripleStoreImplementation, rpcEndpoints, ); logger.timeEnd(`PROCESS AND INSERT NEWER ASSERTIONS FOR ${blockchainName}`); logger.info( `Total amount of unique successfully inserted assertions for ${blockchainName}: ${successfulInsertsSet.size}`, ); logger.info( `Total amount of assertions inserted for ${blockchainName}: ${totalInsertsSet.size}`, ); } } finally { // Close database connection after all blockchains are processed await sqliteDb.close(); } markMigrationAsSuccessfull(); logger.info( `Total amount of unique successfully inserted assertions: ${successfulInsertsSet.size}`, ); logger.info(`Total amount of assertions inserted: ${totalInsertsSet.size}`); } main(); ================================================ FILE: v8-data-migration/validation.js ================================================ import logger from './logger.js'; export function validateConfig(config) { if (!config || typeof config !== 'object') { logger.error( `[VALIDATION ERROR] Config is not defined or it is not an object. Config: ${config}`, ); process.exit(1); } } export function validateBlockchainName(blockchainName) { if (!blockchainName || typeof blockchainName !== 'string') { logger.error( `[VALIDATION ERROR] Blockchain name is defined or it is not a string. Blockchain name: ${blockchainName}`, ); process.exit(1); } } export function validateBlockchainDetails(blockchainDetails) { if ( !blockchainDetails || typeof blockchainDetails !== 'object' || !Object.keys(blockchainDetails).includes('ID') || !Object.keys(blockchainDetails).includes('ENV') || !Object.keys(blockchainDetails).includes('NAME') || !Object.keys(blockchainDetails).includes('CONTENT_ASSET_STORAGE_CONTRACT_ADDRESS') ) { logger.error( `[VALIDATION ERROR] Blockchain details is defined or it is not an object. Blockchain details: ${blockchainDetails}`, ); process.exit(1); } } export function validateTokenId(tokenId) { if (typeof tokenId !== 'string' && typeof tokenId !== 'number') { logger.error( `[VALIDATION ERROR] Token ID is not a string or number. Token ID: ${tokenId}. Type: ${typeof tokenId}`, ); process.exit(1); } } export function validateUal(ual) { if (!ual.startsWith('did:dkg:') || typeof ual !== 'string') { logger.error(`[VALIDATION ERROR] UAL is not a valid UAL. UAL: ${ual}`); process.exit(1); } } export function validateTripleStoreRepositories(tripleStoreRepositories) { if (!tripleStoreRepositories || typeof tripleStoreRepositories !== 'object') { logger.error( `[VALIDATION ERROR] Triple store repositories is not defined or it is not an object. Triple store repositories: ${tripleStoreRepositories}`, ); process.exit(1); } } export function validateTripleStoreImplementation(tripleStoreImplementation) { if (!tripleStoreImplementation || typeof tripleStoreImplementation !== 'string') { logger.error( `[VALIDATION ERROR] Triple store implementation is not defined or it is not a string. Triple store implementation: ${tripleStoreImplementation}`, ); process.exit(1); } } export function validateTripleStoreConfig(tripleStoreConfig) { if (!tripleStoreConfig || typeof tripleStoreConfig !== 'object') { logger.error( `[VALIDATION ERROR] Triple store config is not defined or it is not an object. Triple store config: ${tripleStoreConfig}`, ); process.exit(1); } } export function validateRepository(repository) { if (!repository || typeof repository !== 'string') { logger.error( `[VALIDATION ERROR] Repository is not defined or it is not a string. Repository: ${repository}`, ); process.exit(1); } } export function validateQuery(query) { if (!query || typeof query !== 'string') { logger.error( `[VALIDATION ERROR] Query is not defined or it is not a string. Query: ${query}`, ); process.exit(1); } } export function validateAssertionId(assertionId) { if (!assertionId || typeof assertionId !== 'string') { logger.error( `[VALIDATION ERROR] Assertion ID is not defined or it is not a string. Assertion ID: ${assertionId}`, ); process.exit(1); } } export function validateAssertion(assertion) { if (!assertion || typeof assertion !== 'string') { logger.error( `[VALIDATION ERROR] Assertion is not defined or it is not a string. Assertion: ${assertion}`, ); process.exit(1); } } // BLOCKCHAIN export function validateProvider(provider) { if (!provider || typeof provider !== 'object') { logger.error( `[VALIDATION ERROR] Provider is not defined or it is not an object. Provider: ${provider}`, ); process.exit(1); } } export function validateStorageContractAddress(storageContractAddress) { if (!storageContractAddress || typeof storageContractAddress !== 'string') { logger.error( `[VALIDATION ERROR] Storage contract address is not defined or it is not a string. Storage contract address: ${storageContractAddress}`, ); process.exit(1); } } export function validateStorageContractName(storageContractName) { if (!storageContractName || typeof storageContractName !== 'string') { logger.error( `[VALIDATION ERROR] Storage contract name is not defined or it is not a string. Storage contract name: ${storageContractName}`, ); process.exit(1); } } export function validateStorageContractAbi(storageContractAbi) { if (!storageContractAbi || typeof storageContractAbi !== 'object') { logger.error( `[VALIDATION ERROR] Storage contract ABI is not defined or it is not an object. Storage contract ABI: ${storageContractAbi}`, ); process.exit(1); } } export function validateBatchData(batchData) { if (!batchData || typeof batchData !== 'object') { logger.error( `[VALIDATION ERROR] Batch data is not defined or it is not an object. Batch data: ${batchData}`, ); process.exit(1); } }