Repository: oceanprotocol/ocean.py Branch: main Commit: c7316103cdbc Files: 195 Total size: 662.9 KB Directory structure: gitextract_72imfimc/ ├── .bumpversion.cfg ├── .codeclimate.yml ├── .copyright.tmpl ├── .coveragerc ├── .docignore ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── black.yml │ ├── build.yml │ ├── libcheck.yml │ ├── pytest.yml │ └── release.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .prospector.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── READMEs/ │ ├── c2d-flow-more-examples.md │ ├── c2d-flow.md │ ├── custody-light-flow.md │ ├── developers.md │ ├── df.md │ ├── gas-strategy-remote.md │ ├── install.md │ ├── key-value-private.md │ ├── key-value-public.md │ ├── main-flow.md │ ├── parameters.md │ ├── predict-eth.md │ ├── profile-nfts-flow.md │ ├── publish-flow-credentials.md │ ├── publish-flow-graphql.md │ ├── publish-flow-onchain.md │ ├── publish-flow-restapi.md │ ├── release-process.md │ ├── search-and-filter-assets.md │ ├── services.md │ ├── setup-local.md │ ├── setup-remote.md │ └── using-clef.md ├── bumpversion.sh ├── conftest.py ├── conftest_ganache.py ├── ocean_lib/ │ ├── __init__.py │ ├── agreements/ │ │ ├── __init__.py │ │ ├── consumable.py │ │ └── service_types.py │ ├── aquarius/ │ │ ├── __init__.py │ │ ├── aquarius.py │ │ └── test/ │ │ ├── __init__.py │ │ ├── conftest.py │ │ └── test_aquarius.py │ ├── assets/ │ │ ├── __init__.py │ │ ├── asset_downloader.py │ │ ├── credentials.py │ │ ├── ddo.py │ │ └── test/ │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_asset_downloader.py │ │ └── test_ddo.py │ ├── data_provider/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── data_encryptor.py │ │ ├── data_service_provider.py │ │ ├── fileinfo_provider.py │ │ └── test/ │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_base.py │ │ └── test_data_service_provider.py │ ├── example_config.py │ ├── exceptions.py │ ├── http_requests/ │ │ ├── __init__.py │ │ └── requests_session.py │ ├── models/ │ │ ├── __init__.py │ │ ├── compute_input.py │ │ ├── data_nft.py │ │ ├── data_nft_factory.py │ │ ├── datatoken1.py │ │ ├── datatoken2.py │ │ ├── datatoken_base.py │ │ ├── df/ │ │ │ ├── df_rewards.py │ │ │ ├── df_strategy_v1.py │ │ │ └── test/ │ │ │ ├── conftest.py │ │ │ ├── test_df_rewards.py │ │ │ └── test_df_strategy_v1.py │ │ ├── dispenser.py │ │ ├── erc721_token_factory_base.py │ │ ├── factory_router.py │ │ ├── fixed_rate_exchange.py │ │ ├── test/ │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── test_data_nft.py │ │ │ ├── test_data_nft_factory.py │ │ │ ├── test_datatoken.py │ │ │ ├── test_datatoken_order_both_templates.py │ │ │ ├── test_dispenser.py │ │ │ ├── test_exchange_fees.py │ │ │ ├── test_exchange_main.py │ │ │ ├── test_factory_router.py │ │ │ └── test_fake_ocean.py │ │ └── ve/ │ │ ├── smart_wallet_checker.py │ │ ├── test/ │ │ │ ├── conftest.py │ │ │ ├── test_smart_wallet_checker.py │ │ │ ├── test_ve_allocate.py │ │ │ ├── test_ve_delegation.py │ │ │ ├── test_ve_fee_distributor.py │ │ │ ├── test_ve_fee_estimate.py │ │ │ └── test_ve_ocean.py │ │ ├── ve_allocate.py │ │ ├── ve_delegation.py │ │ ├── ve_fee_distributor.py │ │ ├── ve_fee_estimate.py │ │ └── ve_ocean.py │ ├── ocean/ │ │ ├── __init__.py │ │ ├── crypto.py │ │ ├── mint_fake_ocean.py │ │ ├── ocean.py │ │ ├── ocean_assets.py │ │ ├── ocean_compute.py │ │ ├── test/ │ │ │ ├── conftest.py │ │ │ ├── test_crypto.py │ │ │ ├── test_ocean.py │ │ │ ├── test_ocean_assets.py │ │ │ └── test_util.py │ │ └── util.py │ ├── services/ │ │ ├── consumer_parameters.py │ │ ├── service.py │ │ └── test/ │ │ ├── conftest.py │ │ ├── test_consumer_parameters.py │ │ └── test_service.py │ ├── structures/ │ │ ├── abi_tuples.py │ │ ├── algorithm_metadata.py │ │ ├── file_objects.py │ │ └── test/ │ │ ├── test_algorithm_metadata.py │ │ └── test_file_objects.py │ ├── test/ │ │ ├── __init__.py │ │ ├── test_config.py │ │ └── test_example_config.py │ └── web3_internal/ │ ├── __init__.py │ ├── clef.py │ ├── constants.py │ ├── contract_base.py │ ├── contract_utils.py │ ├── http_provider.py │ ├── request.py │ ├── test/ │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_contract_base.py │ │ ├── test_contract_utils.py │ │ └── test_wallet.py │ └── utils.py ├── pyproject.toml ├── pytest.ini ├── requirements_dev.txt ├── setup.cfg ├── setup.py └── tests/ ├── __init__.py ├── flows/ │ ├── __init__.py │ ├── conftest.py │ └── test_start_reuse_order_fees.py ├── generated-readmes/ │ └── __init__.py ├── integration/ │ ├── ganache/ │ │ ├── conftest.py │ │ ├── test_compute_flow.py │ │ ├── test_consume_flow.py │ │ ├── test_disconnecting_components.py │ │ ├── test_graphql.py │ │ ├── test_market_flow.py │ │ └── test_onchain.py │ └── remote/ │ ├── __init__.py │ ├── test_mumbai_main.py │ ├── test_mumbai_readme.py │ ├── test_polygon.py │ └── util.py ├── readmes/ │ ├── conftest.py │ └── test_readmes.py └── resources/ ├── __init__.py ├── ddo/ │ ├── ddo_algorithm.json │ ├── ddo_algorithm2.json │ ├── ddo_sa_sample.json │ ├── ddo_sa_sample_disabled.json │ ├── ddo_sa_sample_with_credentials.json │ ├── ddo_sample_algorithm.json │ ├── ddo_v4_sample.json │ ├── ddo_v4_with_compute_service.json │ ├── ddo_v4_with_compute_service2.json │ ├── ddo_with_compute_service.json │ └── valid_metadata.json ├── ddo_helpers.py ├── helper_functions.py ├── keys/ │ ├── key_file_1.json │ └── key_file_2.json ├── mocks/ │ ├── __init__.py │ ├── data_provider_mock.py │ └── http_client_mock.py └── test/ └── test_helper_functions.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .bumpversion.cfg ================================================ [bumpversion] current_version = 3.1.2 commit = True tag = True [bumpversion:file:setup.py] search = version='{current_version}' replace = version='{new_version}' [bumpversion:file:ocean_lib/__init__.py] search = __version__ = '{current_version}' replace = __version__ = '{new_version}' ================================================ FILE: .codeclimate.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## version: "2" checks: argument-count: enabled: false file-lines: enabled: false complex-logic: enabled: false method-complexity: enabled: false method-count: enabled: false method-lines: enabled: false ================================================ FILE: .copyright.tmpl ================================================ Copyright 2023 Ocean Protocol Foundation SPDX-License-Identifier: Apache-2.0 ================================================ FILE: .coveragerc ================================================ [run] omit = *test* ================================================ FILE: .docignore ================================================ **test** CHANGELOG.md ================================================ FILE: .github/dependabot.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: pip directory: "/" schedule: interval: weekly time: '03:00' timezone: Europe/Berlin ================================================ FILE: .github/workflows/black.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## name: black on: [push] jobs: black: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: psf/black@stable with: options: "--check --verbose" version: "22.10.0" ================================================ FILE: .github/workflows/build.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## name: Ocean.py multiple OS on: push: branches: - main tags: - '**' pull_request: branches: - '**' jobs: build: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] version: ['3.8', '3.10', '3.11'] steps: - name: Setup Ocean.py uses: actions/checkout@v3 - name: Set up Python ${{ matrix.version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.version }} - name: Install pypa/build run: >- python -m pip install build --user - name: Build a binary wheel and a source tarball run: >- python -m build --sdist --wheel --outdir dist/ - name: Install Mac OS specific dependencies if: ${{ matrix.os == 'macos-latest' }} run: | brew install autoconf automake libtool pkg-config - name: Install dependencies working-directory: ${{ github.workspace }} # vyper is grounded here until it declares explicit support for Python 3.11 run: | python -m pip install --upgrade pip pip install vyper==0.3.7 --ignore-requires-python pip install -r requirements_dev.txt ================================================ FILE: .github/workflows/libcheck.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## name: Ocean.py library check on: schedule: - cron: '30 5 * * 2' workflow_dispatch: jobs: build: environment: CC_REPORTER_ID runs-on: ubuntu-latest steps: - name: Setup Ocean.py uses: actions/checkout@v2 - name: Set up Python 3.8 uses: actions/setup-python@v2 with: python-version: '3.8' - uses: actions/checkout@v2 name: Checkout Barge with: repository: "oceanprotocol/barge" path: 'barge' - name: Run Barge working-directory: ${{ github.workspace }}/barge env: GANACHE_FORK: london run: | bash -x start_ocean.sh --no-dashboard 2>&1 --with-provider2 --with-c2d > start_ocean.log & for i in $(seq 1 108); do sleep 5 [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" -a -f "$HOME/.ocean/ocean-c2d/ready" ] && break done ls -la "$HOME/.ocean/ocean-contracts/artifacts/" - name: Install dependencies working-directory: ${{ github.workspace }} run: | python -m pip install --upgrade pip pip install ocean-lib pip install mkcodes pytest matplotlib - name: Delete default runner images run: | docker image rm node:14 docker image rm node:14-alpine docker image rm node:16 docker image rm node:16-alpine docker image rm node:18 docker image rm node:18-alpine docker image rm buildpack-deps:buster docker image rm buildpack-deps:bullseye docker image rm debian:10 docker image rm debian:11 docker image rm moby/buildkit:latest - name: Generate and test readmes working-directory: ${{ github.workspace }} env: TEST_PRIVATE_KEY1: "0x8467415bb2ba7c91084d932276214b11a3dd9bdb2930fefa194b666dd8020b99" TEST_PRIVATE_KEY2: "0x1d751ded5a32226054cd2e71261039b65afb9ee1c746d055dd699b1150a5befc" TEST_PRIVATE_KEY3: "0x732fbb7c355aa8898f4cff92fa7a6a947339eaf026a08a51f171199e35a18ae0" ADDRESS_FILE: "~/.ocean/ocean-contracts/artifacts/address.json" OCEAN_NETWORK_URL: "http://127.0.0.1:8545" OCEAN_CONFIG_FILE: "config.ini" FACTORY_DEPLOYER_PRIVATE_KEY: "0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58" MUMBAI_RPC_URL: ${{ secrets.MUMBAI_RPC_URL }} run: | mkcodes --github --output tests/generated-readmes/test_{name}.{ext} READMEs pytest tests/readmes/test_readmes.py - name: Slack notify via webhook uses: up9cloud/action-notify@master if: cancelled() == false env: GITHUB_JOB_STATUS: ${{ job.status }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} ================================================ FILE: .github/workflows/pytest.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## name: Ocean.py tests on: - push - pull_request env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: build: environment: CC_REPORTER_ID runs-on: ubuntu-latest steps: - name: Setup Ocean.py uses: actions/checkout@v2 - name: Set up Python 3.8 uses: actions/setup-python@v2 with: python-version: '3.8' - uses: actions/checkout@v2 name: Checkout Barge with: repository: "oceanprotocol/barge" path: 'barge' - name: Run Barge working-directory: ${{ github.workspace }}/barge env: GANACHE_FORK: london run: | bash -x start_ocean.sh --no-dashboard 2>&1 --with-provider2 --with-thegraph --with-c2d --skip-subgraph-deploy > start_ocean.log & - name: Install dependencies working-directory: ${{ github.workspace }} run: | python -m pip install --upgrade pip pip install -r requirements_dev.txt - name: Delete default runner images run: | docker image rm node:16 docker image rm node:16-alpine docker image rm node:18 docker image rm node:18-alpine docker image rm debian:10 docker image rm debian:11 docker image rm moby/buildkit:latest - name: Wait for contracts deployment working-directory: ${{ github.workspace }}/barge run: | for i in $(seq 1 250); do sleep 5 [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" -a -f "$HOME/.ocean/ocean-c2d/ready" ] && break done - name: "Read address.json contents" working-directory: ${{ github.workspace }} run: cat "$HOME/.ocean/ocean-contracts/artifacts/address.json" - name: Test with pytest run: | mkcodes --github --output tests/generated-readmes/test_{name}.{ext} READMEs coverage run --source ocean_lib -m pytest coverage report coverage xml env: REMOTE_TEST_PRIVATE_KEY1: ${{secrets.REMOTE_TEST_PRIVATE_KEY1}} REMOTE_TEST_PRIVATE_KEY2: ${{secrets.REMOTE_TEST_PRIVATE_KEY2}} MUMBAI_RPC_URL: ${{secrets.MUMBAI_RPC_URL}} - name: docker logs run: docker logs ocean_aquarius_1 && docker logs ocean_provider_1 if: ${{ failure() }} - name: Publish code coverage uses: paambaati/codeclimate-action@v2.7.5 env: CC_TEST_REPORTER_ID: ${{secrets.CC_TEST_REPORTER_ID}} ================================================ FILE: .github/workflows/release.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## name: Ocean.py release on: release: types: [created] jobs: build-n-publish: runs-on: ubuntu-latest steps: - name: Setup Ocean.py uses: actions/checkout@v2 - name: Set up Python 3.8 uses: actions/setup-python@v2 with: python-version: '3.8' - name: Install pypa/build run: >- python -m pip install build --user - name: Build a binary wheel and a source tarball run: >- python -m build --sdist --wheel --outdir dist/ - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ verbose: true password: ${{ secrets.PYPI_PASSWORD }} ================================================ FILE: .gitignore ================================================ .history __pycache__ .pytest_cache/ *.pyc *~ .eggs/ artifacts/address.json /venv/ ocean_lib.egg-info/ /build/ /contracts/ /interfaces/ /reports/ /tests/generated-readmes/test* .coverage .tox/ consume-downloads/ coverage.xml setup-local.sh ================================================ FILE: .pre-commit-config.yaml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## repos: - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: - id: isort - repo: https://github.com/psf/black rev: 'refs/tags/22.6.0:refs/tags/22.6.0' hooks: - id: black - repo: https://github.com/pycqa/flake8 rev: 6.0.0 hooks: - id: flake8 - repo: https://github.com/johann-petrak/licenseheaders.git rev: v0.8.8 hooks: - id: licenseheaders args: ["-t", ".copyright.tmpl", "-f"] ================================================ FILE: .prospector.yml ================================================ ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## pep257: disable: - D213 pylint: run: false frosted: run: false mypy: run: false pyroma: run: false ================================================ FILE: CHANGELOG.md ================================================ See [`releases`](https://github.com/oceanprotocol/ocean.py/releases). ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ > ocean.py: a Python library to privately & securely publish, exchange, and consume data. ⚠ Note: as of early 2025, this codebase is not being maintained. It might work, it might not. If you find a bug, feel free to report it, but do not expect it to be fixed. Ocean.py helps data scientists earn $ from their AI models, track provenance of data & compute, and get more data. (More details [here](https://docs.oceanprotocol.com/data-science).) Ocean.py makes these tasks easy: - **Publish** data services: data feeds, REST APIs, downloadable files or compute-to-data. Create an ERC721 **data NFT** for each service, and ERC20 **datatoken** for access (1.0 datatokens to access). - **Sell** datatokens via for a fixed price. Sell data NFTs. - **Transfer** data NFTs & datatokens to another owner, and **all other ERC721 & ERC20 actions** using [web3.py](https://web3py.readthedocs.io). ocean.py is part of the [Ocean Protocol](https://www.oceanprotocol.com) toolset. This is in beta state. If you run into problems, please open up a [new issue](/issues). ## 🏄 Quickstart Follow these steps in sequence to ramp into Ocean. 1. **[Install Ocean](READMEs/install.md)** 2. **Setup:** - **[Remote](READMEs/setup-remote.md)** (Win, MacOS, Linux) - *or* **[Local](READMEs/setup-local.md)** (Linux only) 3. **[Walk through main flow](READMEs/main-flow.md)**: publish asset, post for free / for sale, dispense it / buy it, and consume it ### Tools - [Define gas strategy](READMEs/gas-strategy-remote.md) - auto-determine gas fee for remote networks - [Search & filter data](READMEs/search-and-filter-assets.md) - find assets by tag - [Custody-light flow](READMEs/custody-light-flow.md) - consume a free & a priced asset without custody ### Use-case flows - [Challenge DF](https://github.com/oceanprotocol/predict-eth) - prize $$ to predict future ETH price - [Data Farming](READMEs/df.md) - curate data assets, earn rewards ### On-chain key-value store via data NFTs - [Sharing public data on-chain](READMEs/key-value-public.md) - e.g. public AI models - [Sharing private data on-chain](READMEs/key-value-private.md) - e.g. private AI models ### More types of data assets Each of the following shows how to publish & consume a particular type of data. - [C2D](READMEs/c2d-flow.md) - tokenize & monetize AI algorithms via Compute-to-Data - [REST API](READMEs/publish-flow-restapi.md) - Example on Binance ETH price feed - [GraphQL](READMEs/publish-flow-graphql.md) - Example on Ocean Data NFTs - [On-chain data](READMEs/publish-flow-onchain.md) - Example on Ocean swap fees - [Adding credentials](READMEs/publish-flow-credentials.md) - Example on publishing an asset with custom credentials ### Learn more - [Understand config parameters](READMEs/parameters.md) - envvars vs files - [Learn about off-chain services](READMEs/services.md) - Ocean Provider for data services, Aquarius metadata store ## 🦑 Development - **[Developers flow](READMEs/developers.md)** - to further develop ocean.py - [Release process](READMEs/release-process.md) - to do a new release of ocean.py ## 🏛 License Copyright ((C)) 2025 Ocean Protocol Foundation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: READMEs/c2d-flow-more-examples.md ================================================ # Compute-to-Data (C2D) Flow - More Examples Please note that these code snippets are not automatically tested, so they might be out of date. Always refer to the main [c2d-flow README](https://github.com/oceanprotocol/ocean.py/blob/v4main/READMEs/c2d-flow.md) for the latest fully tested c2d flow. ## Example 1: Image Processing Run the [c2d-flow README](https://github.com/oceanprotocol/ocean.py/blob/v4main/READMEs/c2d-flow.md) with the following alterations: ### 3. Alice publishes a dataset In Step #3 where Alice publishes a dataset, use [peppers.tiff](https://sipi.usc.edu/database/database.php?volume=misc&image=13#top): ```python # Specify metadata, using the peppers.tiff image DATA_date_created = "2021-12-28T10:55:11Z" DATA_metadata = { "created": DATA_date_created, "updated": DATA_date_created, "description": "peppers image", "name": "peppers", "type": "dataset", "author": "Trent", "license": "CC0: PublicDomain", } # ocean.py offers multiple file types, but a simple url file should be enough for this example from ocean_lib.structures.file_objects import UrlFile DATA_url_file = UrlFile( url="https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/peppers_and_grayscale/peppers.tiff" ) ``` ### 4. Alice publishes an algorithm In step #4 where Alice publishes an algorithm, use a standard grayscale algorithm: ```python # Specify metadata, using the grayscale algorithm ALGO_date_created = "2021-12-28T10:55:11Z" ALGO_metadata = { "created": ALGO_date_created, "updated": ALGO_date_created, "description": "grayscale", "name": "grayscale", "type": "algorithm", "author": "Trent", "license": "CC0: PublicDomain", "algorithm": { "language": "python", "format": "docker-image", "version": "0.1", "container": { "entrypoint": "python $ALGO", "image": "oceanprotocol/algo_dockers", "tag": "python-branin", # This image provides all the dependencies of the grayscale.py algorithm "checksum": "sha256:8221d20c1c16491d7d56b9657ea09082c0ee4a8ab1a6621fa720da58b09580e4", }, } } # ocean.py offers multiple file types, but a simple url file should be enough for this example from ocean_lib.structures.file_objects import UrlFile ALGO_url_file = UrlFile( url="https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/peppers_and_grayscale/grayscale.py" ) ``` ### Display and Save the Result Install dependencies: `pip install Pillow` Display the image: ```python from PIL import Image import io image = Image.open(io.BytesIO(result)) image.show() ``` Save the image: ```python image.save('grayscale.png') ``` ## Example 2: Logistic Regression for Classification Run the [c2d-flow README](https://github.com/oceanprotocol/ocean.py/blob/v4main/READMEs/c2d-flow.md) with the following alterations: ### 3. Alice publishes a dataset In Step #3 where Alice publishes a dataset, use the [Iris Flower Dataset](https://en.wikipedia.org/wiki/Iris_flower_data_set): ```python # Specify metadata, using the iris dataset DATA_date_created = "2019-12-28T10:55:11Z" DATA_metadata = { "created": DATA_date_created, "updated": DATA_date_created, "description": "The Iris flower dataset is a multivariate dataset to train classification algorithms", "name": "Iris Flower Dataset", "type": "dataset", "author": "Ocean Protocol & Raven Protocol", "license": "MIT", } # ocean.py offers multiple file types, but a simple url file should be enough for this example from ocean_lib.structures.file_objects import UrlFile DATA_url_file = UrlFile( url="https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/iris_and_logisitc_regression/dataset_61_iris.csv" ) ``` ### 4. Alice publishes an algorithm In step #4 where Alice publishes an algorithm, use a standard grayscale algorithm: ```python # Specify metadata, using the Logistic Regression algorithm ALGO_date_created = "2020-01-28T10:55:11Z" ALGO_metadata = { "created": ALGO_date_created, "updated": ALGO_date_created, "description": "Logistic Regression", "name": "Logistic Regression", "type": "algorithm", "author": "Ocean Protocol & Raven Protocol", "license": "MIT", "algorithm": { "language": "python", "format": "docker-image", "version": "0.1", "container": { "entrypoint": "python $ALGO", "image": "oceanprotocol/algo_dockers", "tag": "python-panda", # This image provides all the dependencies of the logistic_regression.py algorithm "checksum": "sha256:7fc268f502935d11ff50c54e3776dda76477648d5d83c2e3c4fdab744390ecf2", }, } } # ocean.py offers multiple file types, but a simple url file should be enough for this example from ocean_lib.structures.file_objects import UrlFile ALGO_url_file = UrlFile( url="https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/iris_and_logisitc_regression/logistic_regression.py" ) ``` ### Display the Result Install dependencies: `pip install numpy, matplotlib` Display the result: ```python import numpy as np import matplotlib.pyplot as plt h = 0.02 # step size in the mesh xx, yy = np.meshgrid(np.arange(3.8, 8.4, h), np.arange(1.5, 4.9, h)) plt.figure(1, figsize=(4, 3)) plt.pcolormesh(xx, yy, model, cmap=plt.cm.Paired) plt.xlabel("Sepal length") plt.ylabel("Sepal width") plt.xlim(xx.min(), xx.max()) plt.ylim(yy.min(), yy.max()) plt.show() ``` ================================================ FILE: READMEs/c2d-flow.md ================================================ # Quickstart: Compute-to-Data (C2D) Flow This quickstart describes a C2D flow, using a remote setup on Mumbai testnet. Here are the steps: 1. Setup 2. Alice publishes dataset 3. Alice publishes algorithm 4. Alice allows the algorithm for C2D for that data asset 5. Bob acquires datatokens for data and algorithm 6. Bob starts a compute job using a free C2D environment (no provider fees) 7. Bob monitors logs / algorithm output Let's go through each step. ## 1. Setup ### 1.1 Install Ocean First, ensure that you've [installed Ocean](install.md) ### 1.2 Install matplotlib This example uses C2D to create a regression model. We'll use the library `matplotlib` to visualize it. In the same console: ```console #ensure you're in the Python virtualenv source venv/bin/activate #install matplotlib pip install matplotlib ``` ### 1.3 Setup remotely Follow [setup-remote.md](setup-remote.md). ## 2. Alice publishes dataset In the same python console: ```python # Publish data NFT, datatoken, and asset for dataset based on url # ocean.py offers multiple file object types. A simple url file is enough for here from ocean_lib.structures.file_objects import UrlFile DATA_url_file = UrlFile( url="https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/branin_and_gpr/branin.arff" ) name = "Branin dataset" (DATA_data_nft, DATA_datatoken, DATA_ddo) = ocean.assets.create_url_asset(name, DATA_url_file.url, {"from": alice}, with_compute=True, wait_for_aqua=True) print(f"DATA_data_nft address = '{DATA_data_nft.address}'") print(f"DATA_datatoken address = '{DATA_datatoken.address}'") print(f"DATA_ddo did = '{DATA_ddo.did}'") ``` To customise the privacy and accessibility of your compute service, add the `compute_values` argument to `create_url_asset` to set values according to the [DDO specs](https://docs.oceanprotocol.com/core-concepts/did-ddo). The function assumes the documented defaults. ## 3. Alice publishes an algorithm In the same Python console: ```python # Publish data NFT & datatoken for algorithm ALGO_url = "https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/branin_and_gpr/gpr.py" name = "grp" (ALGO_data_nft, ALGO_datatoken, ALGO_ddo) = ocean.assets.create_algo_asset(name, ALGO_url, {"from": alice}, wait_for_aqua=True) print(f"ALGO_data_nft address = '{ALGO_data_nft.address}'") print(f"ALGO_datatoken address = '{ALGO_datatoken.address}'") print(f"ALGO_ddo did = '{ALGO_ddo.did}'") ``` ## 4. Alice allows the algorithm for C2D for that data asset In the same Python console: ```python compute_service = DATA_ddo.services[1] compute_service.add_publisher_trusted_algorithm(ALGO_ddo) DATA_ddo = ocean.assets.update(DATA_ddo, {"from": alice}) ``` ## 5. Bob acquires datatokens for data and algorithm In the same Python console: ```python # Alice mints DATA datatokens and ALGO datatokens to Bob. # Alternatively, Bob might have bought these in a market. from ocean_lib.ocean.util import to_wei DATA_datatoken.mint(bob, to_wei(5), {"from": alice}) ALGO_datatoken.mint(bob, to_wei(5), {"from": alice}) ``` ## 6. Bob starts a compute job using a free C2D environment Only inputs needed: DATA_did, ALGO_did. Everything else can get computed as needed. For demo purposes, we will use the free C2D environment, which requires no provider fees. In the same Python console: ```python # Convenience variables DATA_did = DATA_ddo.did ALGO_did = ALGO_ddo.did # Operate on updated and indexed assets DATA_ddo = ocean.assets.resolve(DATA_did) ALGO_ddo = ocean.assets.resolve(ALGO_did) compute_service = DATA_ddo.services[1] algo_service = ALGO_ddo.services[0] free_c2d_env = ocean.compute.get_free_c2d_environment(compute_service.service_endpoint, DATA_ddo.chain_id) from datetime import datetime, timedelta, timezone from ocean_lib.models.compute_input import ComputeInput DATA_compute_input = ComputeInput(DATA_ddo, compute_service) ALGO_compute_input = ComputeInput(ALGO_ddo, algo_service) # Pay for dataset and algo for 1 day datasets, algorithm = ocean.assets.pay_for_compute_service( datasets=[DATA_compute_input], algorithm_data=ALGO_compute_input, consume_market_order_fee_address=bob.address, tx_dict={"from": bob}, compute_environment=free_c2d_env["id"], valid_until=int((datetime.now(timezone.utc) + timedelta(days=1)).timestamp()), consumer_address=free_c2d_env["consumerAddress"], ) assert datasets, "pay for dataset unsuccessful" assert algorithm, "pay for algorithm unsuccessful" # Start compute job job_id = ocean.compute.start( consumer_wallet=bob, dataset=datasets[0], compute_environment=free_c2d_env["id"], algorithm=algorithm, ) print(f"Started compute job with id: {job_id}") ``` ## 7. Bob monitors logs / algorithm output In the same Python console, you can check the job status as many times as needed: ```python # Wait until job is done import time from decimal import Decimal succeeded = False for _ in range(0, 200): status = ocean.compute.status(DATA_ddo, compute_service, job_id, bob) if status.get("dateFinished") and Decimal(status["dateFinished"]) > 0: succeeded = True break time.sleep(5) ``` This will output the status of the current job. Here is a list of possible results: [Operator Service Status description](https://github.com/oceanprotocol/operator-service/blob/main/API.md#status-description). Once the returned status dictionary contains the `dateFinished` key, Bob can retrieve the job results using ocean.compute.result or, more specifically, just the output if the job was successful. For the purpose of this tutorial, let's choose the second option. ```python # Retrieve algorithm output and log files output = ocean.compute.compute_job_result_logs( DATA_ddo, compute_service, job_id, bob )[0] import pickle model = pickle.loads(output) # the gaussian model result assert len(model) > 0, "unpickle result unsuccessful" ``` You can use the result however you like. For the purpose of this example, let's plot it. ```python import numpy from matplotlib import pyplot X0_vec = numpy.linspace(-5., 10., 15) X1_vec = numpy.linspace(0., 15., 15) X0, X1 = numpy.meshgrid(X0_vec, X1_vec) b, c, t = 0.12918450914398066, 1.5915494309189535, 0.039788735772973836 u = X1 - b * X0 ** 2 + c * X0 - 6 r = 10. * (1. - t) * numpy.cos(X0) + 10 Z = u ** 2 + r fig, ax = pyplot.subplots(subplot_kw={"projection": "3d"}) ax.scatter(X0, X1, model, c="r", label="model") pyplot.title("Data + model") pyplot.show() # or pyplot.savefig("test.png") to save the plot as a .png file instead ``` You should see something like this: ![test](https://user-images.githubusercontent.com/4101015/134895548-82e8ede8-d0db-433a-b37e-694de390bca3.png) ## Appendix. Tips & tricks This README has a simple ML algorithm. However, Ocean C2D is not limited to usage in ML. The file [c2d-flow-more-examples.md](https://github.com/oceanprotocol/ocean.py/blob/v4main/READMEs/c2d-flow-more-examples.md) has examples from vision and other fields. In the "publish algorithm" step, to replace the sample algorithm with another one: - Use one of the standard [Ocean algo_dockers images](https://github.com/oceanprotocol/algo_dockers) or publish a custom docker image. - Use the image name and tag in the `container` part of the algorithm metadata. - The image must have basic support for installing dependencies. E.g. "pip" for the case of Python. You can use other languages, of course. - More info: https://docs.oceanprotocol.com/tutorials/compute-to-data-algorithms/) The function to `pay_for_compute_service` automates order starting, order reusing and performs all the necessary Provider and on-chain requests. It modifies the contents of the given ComputeInput as follows: - If the dataset/algorithm contains a `transfer_tx_id` property, it will try to reuse that previous transfer id. If provider fees have expired but the order is still valid, then the order is reused on-chain. - If the dataset/algorithm does not contain a `transfer_tx_id` or the order has expired (based on the Provider's response), then one new order will be created. This means you can reuse the same ComputeInput and you don't need to regenerate it everytime it is sent to `pay_for_compute_service`. This step makes sure you are not paying unnecessary or duplicated fees. If you wish to upgrade the compute resources, you can use any (paid) C2D environment. Inspect the results of `ocean.ocean_compute.get_c2d_environments(service.service_endpoint, DATA_ddo.chain_id)` and `ocean.retrieve_provider_fees_for_compute(datasets, algorithm_data, consumer_address, compute_environment, duration)` for a preview of what you will pay. Don't forget to handle any minting, allowance or approvals on the desired token to ensure transactions pass. ================================================ FILE: READMEs/custody-light-flow.md ================================================ # Custody-light flow It allows orgs to buy & consume data without having custody of assets. We assume you've already (a) [installed Ocean](install.md), and (b) done [local setup](setup-local.md) or [remote setup](setup-remote.md). This flow works for either one, without any changes between them (!) This flow is split in two sections: - free steps: publish free asset with a Datatoken 2 (enterprise template), then consume; - priced steps: publish a priced asset attached to a Datatoken 2 (enterprise template), then buy / consume. Let's go! ## Free steps Steps in this flow: 1. Alice publishes a free asset 2. Bob dispenses funds from the asset's pricing schema 3. Bob consumes the asset ### 1. Alice publishes a free asset In the same Python console: ```python #data info name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" #create data asset from ocean_lib.models.dispenser import DispenserArguments from ocean_lib.ocean.util import to_wei (data_nft, datatoken, ddo) = ocean.assets.create_url_asset(name, url, {"from": alice}, dt_template_index=2, pricing_schema_args=DispenserArguments(to_wei(1), to_wei(1))) #print print("Just published a free asset:") print(f" data_nft: symbol={data_nft.symbol}, address={data_nft.address}") print(f" datatoken: symbol={datatoken.symbol}, address={datatoken.address}") print(f" did={ddo.did}") ``` ### 2. Bob dispenses funds from the asset's pricing schema Bob wants to consume Alice's asset. He can dispense 1.0 datatokens to complete his job. Below, we show the possible approach: ```python provider_fees = ocean.retrieve_provider_fees( ddo, ddo.services[0], publisher_wallet=bob ) tx = datatoken.dispense_and_order(provider_fees, {"from": bob}, consumer=bob.address, service_index=0) ``` ### 3. Bob consumes the asset Bob now has the transaction receipt to prove that he dispensed funds! Time to download the dataset and use it. In the same Python console: ```python # Bob downloads the file. If the connection breaks, Bob can try again asset_dir = ocean.assets.download_asset(ddo, bob, './', tx.transactionHash.hex()) import os file_name = os.path.join(asset_dir, "file0") ``` Let's check that the file is downloaded. In a new console: ```console cd my_project/datafile.did:op:0xAf07... ls branin.arff ``` ## Priced steps Steps in this flow: 1. Alice publishes a priced asset 2. Bob buys funds from the asset's pricing schema 3. Bob consumes the asset ### 1. Alice publishes a free asset In the same Python console: ```python # data info name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" # create data asset from ocean_lib.models.fixed_rate_exchange import ExchangeArguments from ocean_lib.ocean.util import to_wei (data_nft, datatoken, ddo) = ocean.assets.create_url_asset(name, url, {"from": alice}, dt_template_index=2, pricing_schema_args=ExchangeArguments( rate=to_wei(3), base_token_addr=ocean.OCEAN_address, dt_decimals=18 ),) #print print("Just published a priced asset:") print(f" data_nft: symbol={data_nft.symbol}, address={data_nft.address}") print(f" datatoken: symbol={datatoken.symbol}, address={datatoken.address}") print(f" did={ddo.did}") ``` ### 2. Bob buys funds from the asset's pricing schema Bob wants to consume Alice's asset. He can buy 1.0 datatokens to complete his job. Below, we show the possible approach: ```python provider_fees = ocean.retrieve_provider_fees( ddo, ddo.services[0], publisher_wallet=bob ) exchange = datatoken.get_exchanges()[0] OCEAN = ocean.OCEAN_token OCEAN.approve( datatoken.address, to_wei(10), {"from": bob}, ) OCEAN.approve( exchange.address, to_wei(10), {"from": bob}, ) tx = datatoken.buy_DT_and_order(provider_fees, exchange, {"from": bob}, consumer=bob.address, service_index=0) ``` ### 3. Bob consumes the asset Bob now has the transaction receipt to prove that he bought funds from the exchange! Time to download the dataset and use it. In the same Python console: ```python # Bob downloads the file. If the connection breaks, Bob can try again asset_dir = ocean.assets.download_asset(ddo, bob, './', tx.transactionHash.hex()) import os file_name = os.path.join(asset_dir, "file0") ``` Let's check that the file is downloaded. In a new console: ```console cd my_project/datafile.did:op:0xAf07... ls branin.arff ``` ================================================ FILE: READMEs/developers.md ================================================ # Developing ocean.py This README is how to further _develop_ ocean.py. (Compare to the quickstarts which show how to _use_ it.) Steps: 1. **Install dependencies** 2. **Run barge services** 3. **Set up contracts** 4. **Test** 5. **Merge** the changes via a PR 6. **Release** ## 1. Install dependencies ### Prerequisites - Linux/MacOS - Docker, [allowing non-root users](https://www.thegeekdiary.com/run-docker-as-a-non-root-user/) - Python 3.8.5+ ### Do Install In a new console that we'll call the _work_ console (as we'll use it later): ```console # Clone the repo and enter into it git clone https://github.com/oceanprotocol/ocean.py cd ocean.py # Install OS dependencies sudo apt-get install -y python3-dev gcc # Initialize virtual environment and activate it. # Make sure your Python version inside the venv is >=3.8. python3 -m venv venv source venv/bin/activate # Install modules in the environment. pip install -r requirements_dev.txt ``` ## 2. Run barge services In a new console: ```console # grab repo git clone https://github.com/oceanprotocol/barge cd barge # clean up old containers (to be sure) docker system prune -a --volumes # for support of type 2 transactions export GANACHE_FORK=london # Run barge: start Ganache, Provider, Aquarius; deploy contracts; update ~/.ocean # The `--with-c2d` option tells barge to include the Compute-to-Data backend ./start_ocean.sh --with-c2d ``` (Or, [run services separately](services.md).) ## 3. Set up contracts In work console: ```console # set private keys of two local (ganache) accounts export TEST_PRIVATE_KEY1=0x8467415bb2ba7c91084d932276214b11a3dd9bdb2930fefa194b666dd8020b99 export TEST_PRIVATE_KEY2=0x1d751ded5a32226054cd2e71261039b65afb9ee1c746d055dd699b1150a5befc # needed to mint fake OCEAN for testing with ganache export FACTORY_DEPLOYER_PRIVATE_KEY=0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58 ``` Some tests run on Mumbai (e.g. test_mumbai.py), which need fake MATIC. So you also need: ```console # set private keys of two remote accounts export REMOTE_TEST_PRIVATE_KEY1= export REMOTE_TEST_PRIVATE_KEY2= ``` These keys aren't public because bots could eat the fake MATIC. You need to generate your own, and fill them with a faucet; see instructions in remote setup README. Or, [access-protected OPF keys](https://github.com/oceanprotocol/private-keys/blob/main/README.md)). ## 4. Test In work console: ```console # run a single test pytest ocean_lib/models/test/test_data_nft_factory.py::test_start_multiple_order # run all tests in a file pytest ocean_lib/models/test/test_data_nft_factory.py # run all regular tests; see details on pytest markers to select specific suites pytest ``` The README tests are special. Here's how to run them: ```console # need to auto-generate READMEs first mkcodes --github --output tests/generated-readmes/test_{name}.{ext} READMEs # then run the tests pytest tests/readmes/test_readmes.py pytest /tests/integration/remote/test_mumbai_readme.py ``` For envvars that aren't set, `pytest` uses values in `pytest.ini`. ## 5. Merge Merge the changes via a pull request (PR) etc. Specifically, [follow this workflow](https://docs.oceanprotocol.com/concepts/contributing/#fix-or-improve-core-software). ## 6. Release Release for pip etc. Specifically, [follow the Release Process instructions](./release-process.md). ## 7. Appendix: More tests ### 7.1 Pre-commit hooks In main console (with venv on): ```console pre-commit install ``` Now, this will auto-apply isort (import sorting), flake8 (linting) and black (automatic code formatting) to commits. Black formatting is the standard and is checked as part of pull requests. ## 8. Appendix: Contributing to docs You are welcome to contribute to ocean.py docs and READMEs. For clean markdowns in the READMEs folder, we use the `remark` tool for automatic markdown formatting. OCEAN has an official repository containing remark settings, so please follow the instructions [here](https://github.com/oceanprotocol/ocean-remark). ================================================ FILE: READMEs/df.md ================================================ # Quickstart: Data Farming Flow This README shows how to do steps in Ocean Data Farming (DF), where you curate data assets to earn rewards. It also helps to democratize "wash consume" until it becomes unprofitable. Here are the steps: 1. Setup, in Ganache 2. Lock OCEAN for veOCEAN 3. Publish dataset & exchange 4. Allocate veOCEAN to dataset 5. Fake-consume data 6. Collect OCEAN rewards 7. Repeat steps 1-6, for Eth mainnet Let's go through each step. ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md). ## 2. Lock OCEAN for veOCEAN First, let's set some key parameters for veOCEAN and DF. On Ganache, you can use these values as-is. But on Eth mainnet, you must choose your own. In the same Python console: ```python # On your asset, your DCV = DT_price * num_consumes # Your asset gets rewards pro-rata for its DCV compared to other assets' DCVs. DT_price = 100.0 # number of OCEAN needed to buy one datatoken num_consumes = 3 # This is how much OCEAN to lock into veOCEAN. It can be small if you're # the only staker on your asset. If others stake on your asset, your # rewards are pro-rate compared to others' stake in your asset. amt_OCEAN_lock = 10.0 ``` Now, let's lock OCEAN for veOCEAN. In the same Python console: ```python # simulate passage of time, until next Thursday, the start of DF(X) web3 = ocean.config_dict["web3_instance"] provider = web3.provider latest_block = web3.eth.get_block("latest") WEEK = 7 * 86400 # seconds in a week t0 = latest_block.timestamp t1 = t0 // WEEK * WEEK + WEEK # this is a Thursday, because Jan 1 1970 was t2 = t1 + WEEK provider.make_request("evm_increaseTime", [(t1 - t0)]) #we're now at the beginning of the week. So, lock veOCEAN = ocean.veOCEAN OCEAN.approve(veOCEAN.address, to_wei(amt_OCEAN_lock), {"from" : alice}) import math web3 = ocean.config_dict["web3_instance"] latest_block = web3.eth.get_block("latest") veOCEAN.withdraw({ "from": alice, "gas": latest_block.gasLimit, "gasPrice": math.ceil(latest_block["baseFeePerGas"] * 1.2), }) # withdraw old tokens first latest_block = web3.eth.get_block("latest") veOCEAN.create_lock( to_wei(amt_OCEAN_lock), t2, { "from": alice, "gas": latest_block.gasLimit, "gasPrice": math.ceil(latest_block["baseFeePerGas"] * 1.2), }) ``` ## 3. Publish Dataset & Exchange In the same Python console: ```python #data info name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" #create data asset (data_NFT, DT, ddo) = ocean.assets.create_url_asset(name, url, {"from": alice}, wait_for_aqua=False) print(f"Just published asset, with data_NFT.address={data_NFT.address}") #create exchange exchange = DT.create_exchange({"from": alice}, to_wei(DT_price), OCEAN.address) #make datatokens available on the exchange DT.mint(alice, to_wei(num_consumes), {"from": alice}) DT.approve(exchange.address, to_wei(num_consumes), {"from": alice}) ``` ## 4. Stake on dataset To stake, you allocate veOCEAN to dataset. In the same Python console: ```python amt_allocate = 100 #total allocation must be <= 10000 (wei) ocean.ve_allocate.setAllocation(amt_allocate, data_NFT.address, web3.eth.chain_id, {"from": alice}) ``` ## 5. Fake-consume data "Wash consuming" is when the publisher fake-consumes data to drive data consume volume (DCV) to get more rewards. Not healthy for the ecosystem long-term. Good news: if consume fee > weekly rewards, then wash consume becomes unprofitable. DF is set up to make this happen by DF29 (if not sooner). [Details](https://twitter.com/trentmc0/status/1587527525529358336). In the meantime, this README helps level the playing field around wash consume. This step shows how to do fake-consume. ```python # Alice buys datatokens from herself OCEAN_pay = DT_price * num_consumes OCEAN_alice = from_wei(OCEAN.balanceOf(alice)) assert OCEAN_alice >= OCEAN_pay, f"Have just {OCEAN_alice} OCEAN" OCEAN.approve(exchange.address, to_wei(OCEAN_alice), {"from": alice}) exchange.buy_DT(to_wei(num_consumes), {"from": alice}) DT_bal = from_wei(DT.balanceOf(alice)) assert DT_bal >= num_consumes, \ f"Have {DT_bal} datatokens, too few for {num_consumes} consumes" # Alice sends datatokens to the service, to get access. This is the "consume". for i in range(num_consumes): print(f"Consume #{i+1}/{num_consumes}...") ocean.assets.pay_for_access_service(ddo, {"from": alice}) #don't need to call e.g. ocean.assets.download_asset() since wash-consuming ``` ## 6. Collect OCEAN rewards In the same Python console: ```python #simulate passage of time, until next Thursday, which is the start of DF(X+1) WEEK = 7 * 86400 # seconds in a week latest_block = web3.eth.get_block("latest") t0 = latest_block.timestamp t1 = t0 // WEEK * WEEK + WEEK t2 = t1 + WEEK provider.make_request("evm_increaseTime", [(t1 - t0)]) #Rewards can be claimed via code or webapp, at your leisure. Let's do it now. OCEAN_before = from_wei(OCEAN.balanceOf(alice)) ocean.ve_fee_distributor.claim({ "from": alice, "gas": latest_block.gasLimit, "gasPrice": math.ceil(latest_block["baseFeePerGas"] * 1.2), }) OCEAN_after = from_wei(OCEAN.balanceOf(alice)) print(f"Just claimed {OCEAN_after - OCEAN_before} OCEAN rewards") ``` ## 7. Repeat steps 1-6, for Eth mainnet First, you'll need to set up remotely. This will be like [setup-remote.md](setup-remote.md), but for Eth mainnet. We leave this as an exercise to the reader:) Happy Data Farming! ## Appendix. At the beginning of this flows, we created an `ocean` object, which is an instance of class [`Ocean`](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/ocean/ocean.py). It provides convenient access to [DF](https://github.com/oceanprotocol/ocean.py/tree/main/ocean_lib/models/df) & [VE](https://github.com/oceanprotocol/ocean.py/tree/main/ocean_lib/models/ve) Python objects that which wrap [DF](https://github.com/oceanprotocol/contracts/tree/main/contracts/df) & [VE](https://github.com/oceanprotocol/contracts/tree/main/contracts/ve) Solidity contracts: - `ocean.ve_ocean` or `ocean.veOCEAN -> VeOcean` - `ocean.df_rewards -> DFRewards` - `ocean.df_strategy_v1 -> DFStrategyV1` - `ocean.smart_wallet_checker -> SmartWalletChecker` - `ocean.ve_allocate -> VeAllocate` - `ocean.ve_delegation -> VeDelegation` - `ocean.ve_fee_distributor -> VeFeeDistributor` - `ocean.ve_fee_estimate(self) -> VeFeeEstimate` ================================================ FILE: READMEs/gas-strategy-remote.md ================================================ # Quickstart: Use Specific Gas Strategy for Remote Networks This quickstart illustrates the definition of gas strategy in ocean-lib stack in order to confirm the transactions on blockchain as soon as possible in case the network is congested. Here are the steps: 1. Setup 2. Define gas strategy 3. Alice publishes the asset using gas strategy Let's go through each step. ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up remotely](setup-remote.md). ## 2. Define gas strategy Fees are defined for `polygon` & `mumbai` networks. ```python from ocean_lib.web3_internal.utils import get_gas_fees priority_fee, max_fee = get_gas_fees() ``` ## 3. Alice publishes the asset using gas strategy The gas strategy can be added to any `tx_dict`, and this is just an example of usage. ```python #data info name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" tx_dict = { "from": alice, "maxPriorityFeePerGas": priority_fee, "maxFeePerGas": max_fee, } #create data asset (data_nft, datatoken, ddo) = ocean.assets.create_url_asset( name, url, tx_dict=tx_dict ) #print print("Just published a data asset:") print(f" data_nft: symbol={data_nft.symbol}, address={data_nft.address}") print(f" datatoken: symbol={datatoken.symbol}, address={datatoken.address}") print(f" did={ddo.did}") ``` ================================================ FILE: READMEs/install.md ================================================ # Install Ocean ## Prerequisites - Linux, MacOS, or Windows - [Docker](https://docs.docker.com/engine/install/), [Docker Compose](https://docs.docker.com/compose/install/), [allowing non-root users](https://www.thegeekdiary.com/run-docker-as-a-non-root-user/) - Python 3.8.5 - Python 3.10.4, Python 3.11 with some manual alterations ## Install ocean.py library (If you have issues, see "Potential issues & workarounds" section below.) In a new console: ```console # Create your working directory mkdir my_project cd my_project # Initialize virtual environment and activate it. Install artifacts. # Make sure your Python version inside the venv is >=3.8. # Anaconda is not fully supported for now, please use venv python3 -m venv venv source venv/bin/activate # Avoid errors for the step that follows pip install wheel # Install Ocean library. pip install ocean-lib ``` ## Potential issues & workarounds Issue: M1 * `coincurve` or `cryptography` - If you have an Apple M1 processor, `coincurve` and `cryptography` installation may fail due missing packages, which come pre-packaged in other operating systems. - Workaround: ensure you have `autoconf`, `automake`, `libtool` and `pkg-config` installed, e.g. using Homebrew or MacPorts. Issue: Could not build wheels for coincurve - Reasons for this happening are usually missing dependencies. - Workaround: - make sure you have the OS-level development libraries for building Python packages: `python3-dev` and `build-essential` (install e.g. using apt-get) - install the OS-level `libsecp256k1-dev` library (e.g. using apt-get) - install pyproject.toml separately, e.g. `pip install pyproject-toml` - if ocean-lib installation still fails, install coincurve separately e.g. `pip install coincurve`, then retry Issue: MacOS "Unsupported Architecture" - If you run MacOS, you may encounter an "Unsupported Architecture" issue. - Workaround: install including ARCHFLAGS: `ARCHFLAGS="-arch x86_64" pip install ocean-lib`. [Details](https://github.com/oceanprotocol/ocean.py/issues/486). Issue: Dependencies and Python 3.11 - ocean.py depends on the `parsimonious` package. In turn, `parsimonious` depends on `getargsspec`, which doesn't support Python 3.11. The workaround: open the package's expressions.py file (e.g. in ./venv/lib/python3.11/site-packages/parsimonious/expressions.py), and change the line `import getfullargspec as getargsspec` instead of the regular import. ## Next step You've now installed Ocean, great! Next step is setup: - [Remote](setup-remote.md) (Win, MacOS, Linux) - *or* [Local](setup-local.md) (Linux only) ================================================ FILE: READMEs/key-value-private.md ================================================ # Quickstart: Private Sharing of On-Chain Data ## Introduction This quickstart describes how to use Ocean data NFTs to publish data on-chain, then privately share it to multiple parties. It can be used for: 1. **Sharing AI models** of small to medium size, to specified parties. 2. **Sharing AI model predictions** to specified parties. 3. **["Soulbound Tokens"](https://papers.ssrn.com/sol3/Delivery.cfm/SSRN_ID4105763_code1186331.pdf?abstractid=4105763&mirid=1)** approach to Web3 identity, where an individual's attributes are fields in one (or more) data NFTs. 4. **Profile NFTs / "Login with Web3"** where a Dapp accesses userdata. In this case, the code would be running in the browser via [pyscript](https://www.pyscript.org); or it would be an equivalent flow using JS not Python. This can be viewed as a special case of (2). To generalize, this flow is appropriate for: - **Small to medium-sized datasets.** For larger datasets, store the data off-chain and share via Ocean datatokens. - **When the data sharer knows (or can compute) the recipient's public key.** When this isn't known - such as for faucets to serve free data to anyone, or for selling priced data to anyone, then use Ocean datatokens. ## Steps The quickstart follows these steps, for an example of privately sharing an AI model. Steps by AI modeler (Alice): 1. Setup 2. Publish data NFT 3. Encrypt & store on-chain AI model 4. Share encryption key via chain Steps by AI model retriever (Bob): 5. Get encryption key via chain 6. Retrieve from chain & decrypt AI model ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). ## 2. Publish data NFT Here, we publish a data NFT like elsewhere. To make it a soulbound token (SBT). we set `transferable=False`. In the Python console: ```python # Publish an NFT token. Note "transferable=False" data_nft = ocean.data_nft_factory.create({"from": alice}, 'NFT1', 'NFT1', transferable=False) ``` ## 3. Encrypt & store on-chain AI model Here, we'll symmetrically encrypt an AI model, then store it as a key-value pair in a data NFT on-chain. In the Python console: ```python # Key-value pair model_label = "my_MLP" model_value = "" # Compute a symmetric key: unique to this (nft, nft field) and (your priv key) # Therefore you can calculate it anytime from ocean_lib.ocean import crypto symkey = crypto.calc_symkey(data_nft.address + model_label + alice._private_key.hex()) # Symmetrically encrypt AI model model_value_symenc = crypto.sym_encrypt(model_value, symkey) # Save model to chain data_nft.set_data(model_label, model_value_symenc, {"from": alice}) ``` ## 4. Share encryption key via chain There are many possible ways for Alice to share the symkey to Bob. Here, Alice shares it securely on a public channel by encrypting the symkey in a way that only Bob can decrypt: - The public channel is on the same data NFT, on-chain - So that only Bob can decrypt: Alice asymetricallys encrypt the symkey with Bob's public key, for Bob to decrypt with his private key. In the Python console: ```python # Get Bob's public key. There are various ways; see appendix. pubkey = crypto.calc_pubkey(bob._private_key.hex()) # Asymmetrically encrypt symkey, using Bob's public key symkey_asymenc = crypto.asym_encrypt(symkey, pubkey) # Save asymetrically-encrypted symkey to chain data_nft.set_data("symkey", symkey_asymenc, {"from": alice}) ``` ## 5. Get encryption key via chain Whereas the first four steps were done by the AI model sharer (Alice), the remaining steps are done by the AI model receiver (Bob). You're now Bob. In the Python console: ```python # Retrieve the asymetrically-encrypted symkey from chain symkey_asymenc2 = data_nft.get_data("symkey") # Asymetrically decrypt symkey, with Bob's private key symkey2 = crypto.asym_decrypt(symkey_asymenc2, bob._private_key.hex()) ``` ## 6. Retrieve from chain & decrypt AI model In the Python console: ```python # Retrieve the symetrically-encrypted model from chain model_value_symenc2 = data_nft.get_data(model_label) # Symetrically-decrypt the model, with the symkey retrieved in step 5 model_value2 = crypto.sym_decrypt(model_value_symenc2, symkey2) print(f"Loaded model {model_label} = {model_value2}") ``` ## Appendix Step 4 gave one way for Alice to get the Dapp's public key; step 5 gave one way for the Dapp to get the encrypted symkey. Here are more options. On computing public keys: - If you have the private_key, you can compute the public_key (used above) - Hardware wallets don't expose private_keys. And, while they do expose a _root_ public_key, you shouldn't publicly share those because it lets anyone see all your wallets - However, you _can_ compute anyone's public_key from any tx. This is a general solution. Conveniently, Etherscan shows it too. Possible ways for Alice to get Dapp's public key: - Alice auto-computes from any of Dapp's previous txs. - Alice retrieves it from a public-ish registry or api, e.g. etherscan - Dapp computes it from private_key or from past tx, then shares. Possible ways for Alice to share an encrypted symkey, or for Dapp to share public_key: - Directly client-side - Client-side: in a browser with Metamask - [example by FELToken](https://betterprogramming.pub/exchanging-encrypted-data-on-blockchain-using-metamask-a2e65a9a896c). This is a good choice because it does no on-chain txs. - Client-side: in a script. Like done in step 4 above for public key - Over a public channel: - Public channel: write a new key-value pair on the same data NFT. Like done in step 5 above for encrypted symkey. This is a good choice because the Dapp can access the info in future sessions without extra work. - Public channel: a new data NFT for each message - Public channel: traditional: http, email, or any messaging medium ================================================ FILE: READMEs/key-value-public.md ================================================ # Quickstart: On-Chain Key-Value Store for Public Sharing Data NFTs can store arbitrary key-value pairs, to be an on-chain key-value store. They can be used for: 1. **Publicly sharing AI models** of small to medium size 2. **Publicly sharing AI model predictions** 3. **Comments & ratings in Dapps** 4. **Digital Attestations**, e.g. for verifiable credentials This flow is appropriate for: - **Public data**. The next README will explore for private sharing. - **Small to medium-sized datasets.** For larger datasets, store the data off-chain and share via Ocean datatokens. Here are the steps: 1. Setup 2. Publish data NFT 3. Add key-value pair to data NFT 4. Retrieve value from data NFT ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). ## 2. Publish data NFT In Python console: ```python data_nft = ocean.data_nft_factory.create({"from": alice}, 'NFT1', 'NFT1') ``` ## 3. Add key-value pair to data NFT ```python # Key-value pair model_key = "my_MLP" model_value = "" # set data_nft.set_data(model_key, model_value, {"from": alice}) ``` ## 4. Retrieve value from data NFT ```python model_value2 = data_nft.get_data(model_key) print(f"Found that {model_key} = {model_value2}") ``` This data is public, so anyone can retrieve it. That's it! Note the simplicity. All data was stored and retrieved from on-chain. We don't need Ocean Provider or Ocean Aquarius for these use cases (though the latter can help for fast querying & retrieval). Under the hood, it uses [ERC725](https://erc725alliance.org/), which augments ERC721 with a well-defined way to set and get key-value pairs. ## 5. Next step This README showed how to share _public_ key-value data. The next README covers private data. [Let's go there!](key-value-private.md). ================================================ FILE: READMEs/main-flow.md ================================================ # Main flow This step is the fun one! In it, you'll publish a data asset, post for free / for sale, dispense it / buy it, and consume it. We assume you've already (a) [installed Ocean](install.md), and (b) done [local setup](setup-local.md) or [remote setup](setup-remote.md). This flow works for either one, without any changes between them (!) Steps in the flow: 1. Alice publishes dataset 2. Bob gets access to the dataset (faucet, priced, etc) 3. Bob consumes the dataset Let's go! ## 1. Alice publishes dataset In the same Python console: ```python #data info name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" #create data asset (data_nft, datatoken, ddo) = ocean.assets.create_url_asset(name, url, {"from": alice}) #print print("Just published asset:") print(f" data_nft: symbol={data_nft.symbol}, address={data_nft.address}") print(f" datatoken: symbol={datatoken.symbol}, address={datatoken.address}") print(f" did={ddo.did}") ``` You've now published an Ocean asset! - `data_nft` is the base (base IP) - `datatoken` for access by others (licensing) - `ddo` holding metadata (For more info, see [Appendix: Publish Details](#appendix-publish-details).) ## 2. Bob gets access to the dataset Bob wants to consume the dataset that Alice just published. The first step is for Bob to get 1.0 datatokens. Below, we show four possible approaches: - A & B are when Alice is in contact with Bob. She can mint directly to him, or mint to herself and transfer to him. - C is when Alice wants to share access for free, to anyone - D is when Alice wants to sell access In the same Python console: ```python from ocean_lib.ocean.util import to_wei #Approach A: Alice mints datatokens to Bob datatoken.mint(bob, to_wei(1), {"from": alice}) #Approach B: Alice mints for herself, and transfers to Bob datatoken.mint(alice, to_wei(1), {"from": alice}) datatoken.transfer(bob, to_wei(1), {"from": alice}) #Approach C: Alice posts for free, via a dispenser / faucet; Bob requests & gets datatoken.create_dispenser({"from": alice}) datatoken.dispense(to_wei(1), {"from": bob}) #Approach D: Alice posts for sale; Bob buys # D.1 Alice creates exchange price = to_wei(100) exchange = datatoken.create_exchange({"from": alice}, price, ocean.OCEAN_address) # D.2 Alice makes 100 datatokens available on the exchange datatoken.mint(alice, to_wei(100), {"from": alice}) datatoken.approve(exchange.address, to_wei(100), {"from": alice}) # D.3 Bob lets exchange pull the OCEAN needed OCEAN_needed = exchange.BT_needed(to_wei(1), consume_market_fee=0) ocean.OCEAN_token.approve(exchange.address, OCEAN_needed, {"from":bob}) # D.4 Bob buys datatoken exchange.buy_DT(to_wei(1), consume_market_fee=0, tx_dict={"from": bob}) ```` (For more info, see [Appendix: Dispenser / Faucet Details](#appendix-faucet-details) and [Exchange Details](#appendix-exchange-details).) ## 3. Bob consumes the dataset Bob now has the datatoken for the dataset! Time to download the dataset and use it. In the same Python console: ```python # Bob sends a datatoken to the service to get access order_tx_id = ocean.assets.pay_for_access_service(ddo, {"from": bob}).hex() # Bob downloads the file. If the connection breaks, Bob can try again asset_dir = ocean.assets.download_asset(ddo, bob, './', order_tx_id) import os file_name = os.path.join(asset_dir, "file0") ``` Let's check that the file is downloaded. In a new console: ```console cd my_project/datafile.did:op:* cat file0 ``` The *beginning* of the file should contain the following contents: ``` % 1. Title: Branin Function % 3. Number of instances: 225 % 6. Number of attributes: 2 @relation branin @attribute 'x0' numeric @attribute 'x1' numeric @attribute 'y' numeric @data -5.0000,0.0000,308.1291 -3.9286,0.0000,206.1783 ... ``` (For more info, see [Appendix: Consume Details](#appendix-consume-details).) ## Next step You've now gone through the main flow for Ocean, congrats! Where you want to go next is up to you! Some possibilities: - Go back to the [top-level README](../README.md) to explore advanced flows like Compute-to-Data, Predict-ETH, and Data Farming - Review this README's appendices, which expand on the steps above with further flexibility.

Appendix: Publish Details

### Reconstructing Data NFT & Datatoken Anytime in the future, you can reconstruct your data NFT as an object in Python, via: ```console from ocean_lib.models.data_nft import DataNFT config = data_nft_address = data_nft = DataNFT(config, data_nft_address) ``` It's similar for Datatokens. In Python: ```console from ocean_lib.models.datatoken_base import DatatokenBase config = datatoken_address = datatoken = DatatokenBase.get_typed(config, datatoken_address) ``` ### Data NFT Interface Data NFTs implement ERC721 functionality, ERC725 which extends it, and data management functionality on top. ERC721: - Basic spec of a non-fungible token (NFT) - Official spec is at [erc721.org](https://erc721.org/) - Solidity interface: [IERC721Template.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/interfaces/IERC721Template.sol) ERC725: - ERC725X is execution, and Y is key-value store - Official spec is at [eips.ethereum.org](https://eips.ethereum.org/EIPS/eip-725) - Solidity interface: [IERC725X.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/interfaces/IERC725X.sol) (execution) and [IERC725Y.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/interfaces/IERC725Y.sol) (key-value store) The `data_nft` is a Python object of class [DataNFT](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/models/data_nft.py). The DataNFT class directly exposes the Solidity ERC721 & ERC725 interfaces. This means your `data_nft` object has a Python method for every Solidity method! Besides that, DataNFT implements other Python methods like `create_datatoken()` to improve developer experience. And, [ocean_assets.OceanAssets](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/ocean/ocean_assets.py) and other higher-level Python classes / methods work with DataNFT. Ocean's architecture allows for >1 implementations of ERC721, each with its own Solidity template and Python class. Here are the templates: - [ERC721Template.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/templates/ERC721Template.sol), exposed as Python class `DataNFT` - (there's just one template so far; we can expect more in the future) ### Datatoken Interface Datatokens implement ERC20 functionality, and data management functionality on top. ERC20: - Basic spec of a fungible token standard - Official spec is at [eips.ethereum.org](https://eips.ethereum.org/EIPS/eip-20) - Solidity interface: [IERC20Template.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/interfaces/IERC20Template.sol) Ocean's architecture allows for >1 implementations of ERC20, each with its own "template". Here are the templates so far (we can expect more over time). Template 1: - Solidity: [ERC20Template.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/templates/ERC20Template.sol) - Python wrapper: [Datatoken1](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/models/datatoken.py). It has a Python method for every Solidity method. - Implements methods like `start_order()`, `create_exchange()`, and `create_dispenser()` to enhance developer experience. Template 2: - Inherits from template 1 in both Solidity and Python. - Solidity: [ERC20TemplateEnterprise.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/templates/ERC20TemplateEnterprise.sol) - Python wrapper: [Datatoken2](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/models/datatoken2.py) - New method: [`buy_DT_and_order()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken2.py#L20). This uses just 1 tx to do both actions at once (versus 2 txs for template 1). - New method: [`dispense_and_order()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken2.py#L70). Similarly, uses just 1 tx. Below you can find an explanatory table describing the template attributes: Template # | Class Label | Allows dispense by default? | Allows non-custody of datatokens? | Combines txs? | Allows non-custody of url? :----: | :----: | :----: | :----: | :----: | :----: 1 | Datatoken1 | Y | N | N | N 2 | Datatoken2 | N | Y | Y | N ### DIDs and DDOs DDOs get returned in `create()` calls. Think of them as metadata, following a well-defined format. Let's get more specific. A [DID](https://w3c-ccg.github.io/did-spec/) is a decentralized identifier. A DID Document (DDO) is a JSON blob that holds information about the DID. Given a DID, a resolver will return the DDO of that DID. An Ocean _asset_ has a DID and DDO, alongside a data NFT and >=0 datatokens. The DDO should include metadata about the asset, and define access in at least one service. Only owners or delegated users can modify the DDO. DDOs follow a schema - a pre-specified structure of possible metadata fields. Ocean Aquarius helps in reading, decrypting, and searching through encrypted DDO data from the chain. [Ocean docs](https://docs.oceanprotocol.com/core-concepts/did-ddo) have further info yet. ### Publish Flexibility Here's an example similar to the `create()` step above, but exposes more fine-grained control. In the same python console: ```python # Specify metadata and services, using the Branin test dataset date_created = "2021-12-28T10:55:11Z" metadata = { "created": date_created, "updated": date_created, "description": "Branin dataset", "name": "Branin dataset", "type": "dataset", "author": "Trent", "license": "CC0: PublicDomain", } # Use "UrlFile" asset type. (There are other options) from ocean_lib.structures.file_objects import UrlFile url_file = UrlFile( url="https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" ) # Publish data asset from ocean_lib.models.datatoken_base import DatatokenArguments _, _, ddo = ocean.assets.create( metadata, {"from": alice}, datatoken_args=[DatatokenArguments(files=[url_file])], ) ``` ### DDO Encryption or Compression The DDO is stored on-chain. It's encrypted and compressed by default. Therefore it supports GDPR "right-to-be-forgotten" compliance rules by default. You can control this during create(): - To disable encryption, use `ocean.assets.create(..., encrypt_flag=False)`. - To disable compression, use `ocean.assets.create(..., compress_flag=False)`. - To disable both, use `ocean.assets.create(..., encrypt_flag=False, compress_flag=False)`. ### Create _just_ a data NFT Calling `create()` like above generates a data NFT, a datatoken for that NFT, and a ddo. This is the most common case. However, sometimes you may want _just_ the data NFT, e.g. if using a data NFT as a simple key-value store. Here's how: ```python data_nft = ocean.data_nft_factory.create({"from": alice}, 'NFT1', 'NFT1') ``` If you call `create()` after this, you can pass in an argument `data_nft_address:string` and it will use that NFT rather than creating a new one. ### Create a datatoken from a data NFT Calling `create()` like above generates a data NFT, a datatoken for that NFT, and a ddo object. However, we may want a second datatoken. Or, we may have started with _just_ the data NFT, and want to add a datatoken to it. Here's how: ```python datatoken = data_nft.create_datatoken({"from": alice}, "Datatoken 1", "DT1") ``` If you call `create()` after this, you can pass in an argument `deployed_datatokens:List[Datatoken1]` and it will use those datatokens during creation.

Appendix: Dispenser / Faucet Details

### Dispenser Interface We access dispenser (faucet) functionality from two complementary places: datatokens, and `Dispenser` object. A given datatoken can create exactly one dispenser for that datatoken. **Interface via datatokens:** - [`datatoken.create_dispenser()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken.py#L337) - implemented in DatatokenBase, inherited by Datatoken2 - [`datatoken.dispense()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken.py#L380) - "" - [`datatoken.dispense_and_order()` - implemented in Datatoken1](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken.py#L439) and [in Datatoken2](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken_enterprise.py#L70). The latter only needs one tx to dispense and order. - [`datatoken.dispenser_status()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken.py#L403) - returns a [`DispenserStatus`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/dispenser.py#L16) object **Interface via [`Dispenser`](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/models/dispenser.py) Python class:** - You can access its instance via `ocean.dispenser`. - `Dispenser` wraps [Dispenser.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/pools/dispenser/Dispenser.sol) Solidity implementation, to expose `Dispenser.sol`'s methods as Python methods. - Note that `Dispenser` is _global_ across all datatokens. Therefore calls to the Solidity contract - or Python calls that pass through - provide the datatoken address as an argument. - Example call: `ocean.dispenser.setAllowedSwapper(datatoken_addr, ZERO_ADDRESS, {"from": alice})` ### Flexibility in Creating a Dispenser `datatoken.create_dispenser()` can take these optional arguments: - `max_tokens` - maximum number of tokens to dispense. The default is a large number. - `max_balance` - maximum balance of requester. The default is a large number. - `with_mint` - allow minting - `allowed_swapper` - swapper user address A call would look like `create_dispenser({"from": alice}, max_tokens=max_tokens, max_balance=max_balance)`. ### Dispenser Status To learn about dispenser status: ```python status = datatoken.dispenser_status() print(f"For datatoken {datatoken.address}:") print(status) ``` It will output something like: ```text For datatoken 0x92cA723B61CbD933390aA58b83e1F00cedf4ebb6: DispenserStatus: active = True owner_address = 0x1234 is_minter = True max_tokens = 1000 (10000000000000000000000 wei) max_balance = 10 (100000000000000000000 wei) balance = 1 allowed_swapper = anyone can request ``` ### Who can request tokens from a faucet Template 1 (`Datatoken1`): - Anyone can call `datatoken.dispense()` to request tokens. Template 2 (`Datatoken2`): - Option A. Anyone can `datatoken.dispense_and_order()` to request tokens, and order. - Option B. Not anyone can call `datatoken.dispense()` by default. To allow anyone, the publisher does: `ocean.dispenser.setAllowedSwapper(datatoken_address, ZERO_ADDRESS, {"from" : publisher_wallet})`, where `ZERO_ADDRESS` is `0x00..00`. Details: `Dispenser.sol` has an attribute `allowed_swapper` to govern who can call `dispense()`. A value of `0x00...0` allows anyone. Template 1 has `ZERO_ADDRESS` as a default value; template 2 does not. However, template 2 allows anyone to call `dispense_and_order()`, independent of the value of `allowed_swapper`.

Appendix: Exchange Details

### Exchange Interface We access exchange functionality from three complementary places: datatokens, [`OneExchange`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L117) object, and (if needed) [`FixedRateExchange`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L106) object. A given datatoken can create one or more `OneExchange` objects. **Interface via datatokens:** - [`datatoken.create_exchange()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken.py#L237) - Returns a `OneExchange` object. - [`datatoken.get_exchanges()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/datatoken.py#L312) - Returns a list of `OneExchange` objects. Once you've got a `OneExchange` object, most interactions are with it. **Interface via [`OneExchange`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L117) Python class:** - [`BT_needed()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L150) - # basetokens (BTs) needed, to buy target # datatokens (DTs) - [`BT_received()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L167) - # BTs you receive, in selling given # DTs - [`buy_DT()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L184) - spend BTs to buy DTs - [`sell_DT()`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L230) - sell DTs, receive back BTs - and more, including get/set rate (price), toggle on/off, get/set fees, update balances While most interactions are with `OneExchange` described above, sometimes we may want to access the `FixedRateExchange` object. **Interface via [`FixedRateExchange`](https://github.com/oceanprotocol/ocean.py/blob/4aa12afd8a933d64bc2ed68d1e5359d0b9ae62f9/ocean_lib/models/fixed_rate_exchange.py#L106) Python class:** - You can access its instance via `ocean.fixed_rate_exchange`. - `FixedRateExchange` wraps [FixedRateExchange.sol](https://github.com/oceanprotocol/contracts/blob/main/contracts/pools/fixedRate/FixedRateExchange.sol) Solidity implementation, to expose `Dispenser.sol`'s methods as Python methods. - Note that `FixedRateExchange` is _global_ across all datatokens. Therefore calls to the Solidity contract - or Python calls that pass through - provide the exchange id address as an argument. It's exchange id, and not datatoken, because there may be >1 exchange for a given datatoken. - Example call: `ocean.fixed_rate_exchange.getRate(exchange_id)` returns rate (price). ### Flexibility in Creating an Exchange When Alice posted the dataset for sale via `create_exchange()`, she used OCEAN. Alternatively, she could have used H2O, the OCEAN-backed stable asset. Or, she could have used USDC, DAI, RAI, WETH, or other, for a slightly higher fee (0.2% vs 0.1%). ### Exchange Status Here's how to see all the exchanges that list the datatoken. In the Python console: ```python exchanges = datatoken.get_exchanges() # list of OneExchange ``` To learn more about the exchange status: ```python print(exchange.details) print(exchange.exchange_fees_info) ``` It will output something like: ```text >>> print(exchange.details) ExchangeDetails: datatoken = 0xdA3cf7aE9b28E1A9B5F295201d9AcbEf14c43019 base_token = 0x24f42342C7C171a66f2B7feB5c712471bED92A97 fixed_rate (price) = 1.0 (1000000000000000000 wei) active = True dt_supply = 99.0 (99000000000000000000 wei) bt_supply = 1.0 (1000000000000000000 wei) dt_balance = 0.0 (0 wei) bt_balance = 1.0 (1000000000000000000 wei) with_mint = False dt_decimals = 18 bt_decimals = 18 owner = 0x02354A1F160A3fd7ac8b02ee91F04104440B28E7 >>> print(exchange.exchange_fees_info) ExchangeFeeInfo: publish_market_fee = 0.0 (0 wei) publish_market_fee_available = 0.0 (0 wei) publish_market_fee_collector = 0x02354A1F160A3fd7ac8b02ee91F04104440B28E7 opc_fee = 0.001 (1000000000000000 wei) ocean_fee_available (to opc) = 0.001 (1000000000000000 wei) ```

Create asset and pricing schema simultaneously

Ocean Assets allows you to bundle several common scenarios as a single transaction, thus lowering gas fees. Any of the `ocean.assets.create__asset()` functions can also take an optional parameter that describes a bundled pricing schema (Dispenser or Fixed Rate Exchange). This can be either a DispenserArguments or an ExchangeArguments object. The parameters for these Arguments classes are identical to those for creating the object itself. E.g. adding `pricing_schema_args=DispenserArguments(to_wei(1), to_wei(1))` to the `create` function is equivalent to performing the creation and creating a dispenser later using `dt.create_dispenser(to_wei(1), to_wei(1))`. Here is an example involving an exchange: ```python from ocean_lib.models.fixed_rate_exchange import ExchangeArguments (data_nft, datatoken, ddo) = ocean.assets.create_url_asset( name, url, {"from": alice}, pricing_schema_args=ExchangeArguments(rate=to_wei(3), base_token_addr=ocean.OCEAN_address, dt_decimals=18) ) assert len(datatoken.get_exchanges()) == 1 ```

Appendix: Consume Details

### Consume General To "consume" an asset typically means placing an "order", where you pass in 1.0 datatokens and get back a url. Then, you typically download the asset from the url. For more information, search for "order" in this README or related code. ### About ARFF format The file is in ARFF format, used by some AI/ML tools. Our example has two input variables (x0, x1) and one output. ```console % 1. Title: Branin Function % 3. Number of instances: 225 % 6. Number of attributes: 2 @relation branin @attribute 'x0' numeric @attribute 'x1' numeric @attribute 'y' numeric @data -5.0000,0.0000,308.1291 -3.9286,0.0000,206.1783 ... ```

Appendix: Ocean Instance

At the beginning of most flows, we create an `ocean` object, which is an instance of class [`Ocean`](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/ocean/ocean.py). It exposes useful information, including the following. Config dict attribute: - `ocean.config_dict` or `ocean.config -> dict` OCEAN token: - `ocean.OCEAN_address -> str` - `ocean.OCEAN_token` or `ocean.OCEAN -> Datatoken` Ocean smart contracts: - `ocean.data_nft_factory -> DataNFTFactoryContract` - `ocean.dispenser -> Dispenser` - faucets for free data - `ocean.fixed_rate_exchange -> FixedRateExchange` - exchanges for priced data Simple getters: - `ocean.get_nft_token(self, token_address: str) -> DataNFT` - `ocean.get_datatoken(self, token_address: str) -> Datatoken` - `ocean.def get_user_orders(self, address: str, datatoken: str)` - (and some others that are more complex) It also provides Python wrappers to veOCEAN and Data Farming contracts. See [df.md](df.md) for details. ================================================ FILE: READMEs/parameters.md ================================================ # On Config Parameters We can set any config parameter using the config dictionary. An `Ocean` instance will hold a `config_dict` that holds various config parameters. These parameters need to get set using the ExampleConfig class. This is set based on what's input to `Ocean` constructor: 1. dict input: `Ocean({'METADATA_CACHE_URI':..})`, in which case you have to build the web3 instance manually 2. use boilerplate from example config, which also sets the web3 instance to be used by each contract ## Example Here is an example for (1): dict input, filled from envvars ```python import os from ocean_lib.ocean.ocean import Ocean from ocean_lib.example_config import get_web3 network_url = "https://your-rpc.com" d = { 'METADATA_CACHE_URI': "https://v4.aquarius.oceanprotocol.com", 'PROVIDER_URL' : "https://v4.provider.goerli.oceanprotocol.com", "web3_instance": get_web3(network_url) } ocean = Ocean(d) ``` ## Further details For the most precise description of config parameter logic, see the [Ocean() constructor implementation](https://github.com/oceanprotocol/ocean.py/blob/main/ocean_lib/ocean/ocean.py). ================================================ FILE: READMEs/predict-eth.md ================================================ This has been moved [here](https://github.com/oceanprotocol/predict-eth). ================================================ FILE: READMEs/profile-nfts-flow.md ================================================ # Quickstart: Profile NFTs This is a flow showing how to do "login with Web3" with the help of Ocean data NFTs. In this flow, a dapp is not only connected to the user's wallet, but it can access profile data that the user has privately shared to it. Interestingly, these NFTs are also essentially [Soulbound Tokens](https://papers.ssrn.com/sol3/Delivery.cfm/SSRN_ID4105763_code1186331.pdf?abstractid=4105763&mirid=1) as well:) Here are the steps: 1. Setup 2. Alice publishes data NFT 3. Alice adds key-value pair to data NFT. 'value' encrypted with a symmetric key 'symkey' 4. Alice gets Dapp's public_key 5. Alice encrypts symkey with Dapp's public key and shares to Dapp 6. Dapp gets & decrypts symkey, then gets & decrypts original 'value' ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). ## 2. Alice publishes data NFT We publish a data NFT like elsewhere, except we set `transferable=False` (and skip print statements). In the Python console: ```python # Publish an NFT token. Note "transferable=False" data_nft = ocean.data_nft_factory.create({"from": alice}, 'NFT1', 'NFT1', transferable=False) ``` ## 3. Alice adds key-value pair to data NFT. 'value' encrypted with a symmetric key 'symkey' ```python # imports from base64 import b64encode from cryptography.fernet import Fernet from eth_account.messages import encode_defunct from hashlib import sha256 from ocean_lib.web3_internal.utils import sign_with_key from web3.main import Web3 # Key-value pair profiledata_name = "fav_color" profiledata_val = "blue" # Prep key for setter. Contract/ERC725 requires keccak256 hash profiledata_name_hash = Web3.keccak(text=profiledata_name) # Choose a symkey where: # - sharing it unlocks only this field: make unique to this data nft & field # - only Alice can compute it: make it a function of her private key # - is hardware wallet friendly: uses Alice's digital signature not private key preimage = data_nft.address + profiledata_name preimage = sha256(preimage.encode('utf-8')).hexdigest() prefix = "\x19Ethereum Signed Message:\n32" msg = Web3.solidity_keccak( ["bytes", "bytes"], [Web3.to_bytes(text=prefix), Web3.to_bytes(text=preimage)] ) signed = sign_with_key(msg, alice._private_key.hex()) symkey = b64encode(str(signed).encode('ascii'))[:43] + b'=' # bytes # Prep value for setter profiledata_val_encr_hex = Fernet(symkey).encrypt(profiledata_val.encode('utf-8')).hex() # set data_nft.setNewData(profiledata_name_hash, profiledata_val_encr_hex, {"from": alice}) ``` ## 4. Alice gets Dapp's public_key There are various ways to compute public_key, and for Alice to get it (see Appendix). Here, the Dapp computes public_key from its private_key, then shares with Alice client-side within the script. ```python from eth_keys import keys from eth_utils import decode_hex dapp_private_key = os.getenv('TEST_PRIVATE_KEY2') dapp_private_key_obj = keys.PrivateKey(decode_hex(dapp_private_key)) dapp_public_key = str(dapp_private_key_obj.public_key) # str dapp_address = dapp_private_key_obj.public_key.to_address() # str ``` ## 5. Alice encrypts symkey with Dapp's public key and shares to Dapp There are various ways for Alice to share the encrypted symkey to the Dapp (see Appendix). Here, Alice writes a new key-value pair on the same data NFT. This approach allows the Dapp to access the info in future sessions without extra work. ```python from ecies import encrypt as asymmetric_encrypt symkey_name = (profiledata_name + ':for:' + dapp_address[:10]) # str symkey_name_hash = Web3.keccak(text=symkey_name) symkey_val_encr = asymmetric_encrypt(dapp_public_key, symkey) # bytes symkey_val_encr_hex = symkey_val_encr.hex() # hex # arg types: key=bytes32, value=bytes, wallet=wallet data_nft.setNewData(symkey_name_hash, symkey_val_encr_hex, {"from": alice}) ``` ## 6. Dapp gets & decrypts symkey, then gets & decrypts original 'value' ```python from ecies import decrypt as asymmetric_decrypt # symkey_name_hash = symkey_val_encr2 = data_nft.getData(symkey_name_hash) symkey2 = asymmetric_decrypt(dapp_private_key, symkey_val_encr2) # profiledata_name_hash = profiledata_val_encr_hex2 = data_nft.getData(profiledata_name_hash) profiledata_val2_bytes = Fernet(symkey).decrypt(profiledata_val_encr_hex2) profiledata_val2 = profiledata_val2_bytes.decode('utf-8') print(f"Dapp found profiledata {profiledata_name} = {profiledata_val2}") ``` ## Appendix Step 4 gave one way for Alice to get the Dapp's public key; step 5 gave one way for the Dapp to get the encrypted symkey. Here are more options. On computing public keys: - If you have the private_key, you can compute the public_key (used above) - Hardware wallets don't expose private_keys. And, while they do expose a _root_ public_key, you shouldn't publicly share those because it lets anyone see all your wallets - However, you _can_ compute anyone's public_key from any tx. This is a general solution. Conveniently, Etherscan shows it too. Possible ways for Alice to get Dapp's public key: - Alice auto-computes from any of Dapp's previous txs. - Alice retrieves it from a public-ish registry or api, e.g. etherscan - Dapp computes it from private_key or from past tx, then shares. Possible ways for Alice to share an encrypted symkey, or for Dapp to share public_key: - Directly client-side - Client-side: in a browser with Metamask - [example by FELToken](https://betterprogramming.pub/exchanging-encrypted-data-on-blockchain-using-metamask-a2e65a9a896c). This is a good choice because it does no on-chain txs. - Client-side: in a script. Like done in step 4 above for public key - Over a public channel: - Public channel: write a new key-value pair on the same data NFT. Like done in step 5 above for encrypted symkey. This is a good choice because the Dapp can access the info in future sessions without extra work. - Public channel: a new data NFT for each message - Public channel: traditional: http, email, or any messaging medium ================================================ FILE: READMEs/publish-flow-credentials.md ================================================ # Quickstart: Publish & Metadata update Flows for credentials This quickstart describes how to use credentials in order to limit access to a dataset. Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). Here are the steps: 1. Publish a dataset that can only be accessed by Alice and Bob. Everyone else will be denied. 2. Update the dataset so only Bob will be denied, everyone else will have access. Let's go through each step. ## 2. Carlos publishes the API asset, allowing only Alice and Bob as consumers ```python url = 'http://www.example.net' name = "Restricted dataset" credentials = { "allow": [{"type": "address", "values": [alice.address, bob.address]}], "deny": [], } #create asset (data_nft, datatoken, ddo) = ocean.assets.create_url_asset(name, url, {"from": carlos}, credentials = credentials) print(f"Just published asset, with did={ddo.did}") ``` That's it! You've created a data asset which is accesible only to Alice and Bob. Consume here is just like in [consume-flow](consume-flow.md). ## 2. Carlos updates the asset, allowing everyone, but denying Bob Using the ddo directly, or later using `ddo=ocean.assets.resolve()` ```python ddo.credentials = { "allow": [], "deny": [{"type": "address", "values": [bob.address]}], } ddo = ocean.assets.update(ddo, {"from": carlos}) ``` That's it! Now everyone can access the dataset, except Bob. Consume here is just like in [consume-flow](consume-flow.md). For more information about credentials, you can refer to [docs](https://docs.oceanprotocol.com/core-concepts/did-ddo#credentials). ================================================ FILE: READMEs/publish-flow-graphql.md ================================================ # Quickstart: Publish & Consume Flow for GraphQL data type This quickstart describes a flow to publish & consume GraphQL-style URIs. In our example, the data asset is a query to find data NFTs via ocean-subgraph. Here are the steps: 1. Setup 2. Publish dataset 3. Consume dataset Let's go through each step. ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). ## 2. Publish dataset In the same Python console: ```python #data info name = "Data NFTs in Ocean" url="https://v4.subgraph.goerli.oceanprotocol.com/subgraphs/name/oceanprotocol/ocean-subgraph" query="""query{ nfts(orderBy: createdTimestamp,orderDirection:desc){ id symbol createdTimestamp } } """ #create asset (data_nft, datatoken, ddo) = ocean.assets.create_graphql_asset(name, url, query, {"from": alice}) print(f"Just published asset, with did={ddo.did}") ``` That's it! You've created a data asset of "GraphqlQuery" asset type. It includes a data NFT, a datatoken for the data NFT, and metadata. ## 3. Consume dataset Consume here is just like in [consume-flow](consume-flow.md). The file downloaded is a .json. From that, use the python `json` library to parse it as desired. ================================================ FILE: READMEs/publish-flow-onchain.md ================================================ # Quickstart: Publish & Consume Flow using onchain data source This quickstart describes a flow to publish & consume onchain data source Here are the steps: 1. Setup 2. Publish dataset 3. Consume dataset Let's go through each step. ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). ## 2. Publish dataset In the same Python console: ```python #data info from ocean_lib.ocean.util import get_address_of_type name = "swapOceanFee function call" contract_address = get_address_of_type(config, "Router") contract_abi = { "inputs": [], "name": "swapOceanFee", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function", } #create asset (data_nft, datatoken, ddo) = ocean.assets.create_onchain_asset(name, contract_address, contract_abi, {"from": alice}) print(f"Just published asset, with did={ddo.did}") ``` That's it! You've created a data asset of "SmartContractCall" asset type. It includes a data NFT, a datatoken for the data NFT, and metadata. ## 3. Consume the dataset (Consume here is just like in [consume-flow](READMEs/consume-flow.md]. The file downloaded is a .json. From that, use the python `json` library to parse it as desired.) ================================================ FILE: READMEs/publish-flow-restapi.md ================================================ # Quickstart: Publish & Consume Flow for REST API-style URIs This quickstart describes a flow to publish Kraken REST API of OCEAN-USD pair price feed, to make it available as free data asset on Ocean, and to consume it. Here are the steps: 1. Setup 2. Alice publishes the API asset 3. Alice creates a faucet for the asset 4. Bob gets a free datatoken, then consumes it Let's go through each step. ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). ## 2. Alice publishes the API asset In the same Python console: ```python # Data info name = "Kraken API OCEAN-USD price feed" pair = 'OCEANUSD' # Choose the trading pair interval = '1440' # Choose the time interval in minutes (1440 for daily) from datetime import datetime, timedelta end_datetime = datetime.now() start_datetime = end_datetime - timedelta(days=7) # The previous week since = int(start_datetime.timestamp() * 1000) # Choose the start time in Unix timestamp url = f'https://api.kraken.com/0/public/OHLC?pair={pair}&interval={interval}&since={since}' #create asset (data_nft, datatoken, ddo) = ocean.assets.create_url_asset(name, url, {"from": alice}) print(f"Just published asset, with did={ddo.did}") ``` ### 3. Alice creates a faucet for the asset In the same Python console: ```python datatoken.create_dispenser({"from": alice}) ``` ### 4. Bob gets a free datatoken, then consumes it Now, you're Bob. First, download the file. In the same Python console: ```python # Set asset did. Practically, you'd get this from Ocean Market. _This_ example uses prior info. ddo_did = ddo.did # Bob gets a free datatoken, sends it to the service, and downloads datatoken.dispense(to_wei(1), {"from": bob}) order_tx_id = ocean.assets.pay_for_access_service(ddo, {"from": bob}).hex() asset_dir = ocean.assets.download_asset(ddo, bob, './', order_tx_id) import os file_name = os.path.join(asset_dir, 'file0') ``` Now, load the file and use its data. The data follows the Kraken docs specs for Data, [here](https://docs.kraken.com/rest/#tag/Market-Data/operation/getOHLCData). In the same Python console: ```python # Load from file into memory with open(file_name, "r") as file: # Data is a string with the result inside. data_str = file.read().rstrip().replace("'", '"') import json data = json.loads(data_str) # Data is a list of lists # -Outer dictionary contains 2 keys, one for errors and one for the result with the pair. # -Inner dictionary have 9 entries each: Kline open time, Open price, High price, Low price, close Price, Vol, .. # Get close price close_prices = [float(data_at_day[4]) for data_at_day in data['result'][pair]] print(f"close prices: {close_prices}") ``` ================================================ FILE: READMEs/release-process.md ================================================ # The ocean.py Release Process ## Step 0: Update documentation - If your changes affect what docs.oceanprotocol.com shows, then make changes in the docs repo https://github.com/oceanprotocol/docs and change ## Step 1: Bump version and push changes - Identify the current version. It's listed at [pypi.org/project/ocean-lib](https://pypi.org/project/ocean-lib/), in this repo in [.bumpversion.cfg](../.bumpversion.cfg), and elsewhere. - Create a new local feature branch, e.g. `git checkout -b feature/bumpversion-to-v1.2.5` - Ensure you're in virtual env: `source venv/bin/activate` - Run `./bumpversion.sh` to bump the project version, as follows: - To bump the major version (v**X**.Y.Z): `./bumpversion.sh major` - To bump the minor version (vX.**Y**.Z): `./bumpversion.sh minor` - To bump the patch version (vX.Y.**Z**): `./bumpversion.sh patch` - (Ocean.py follows [semantic versioning](https://semver.org/).) - Commit the changes to the feature branch. For example: `git commit -m "Bump version v1.2.4 -> v1.2.5"` - Push the feature branch to GitHub. `git push origin feature/bumpversion-to-v1.2.5` ## Step 2: Merge changes to main branch - Make a pull request from the just-pushed branch. - Wait for all the tests to pass! - Merge the pull request into the `main` branch. ## Step 3: Release - To make a GitHub release (which creates a Git tag): - Go to the ocean.py repo's Releases page - Click "Draft a new release". - For tag version, put something like `v1.2.5` - For release title, put the same value (like `v1.2.5`). - For the target, select the `main` branch, or the just-merged commit. - Describe the main changes. (In the future, these will come from the changelog.) - Click "Publish release". ## Step 4: Verify - GitHub Actions will detect the release (a new tag) and run the deployment and publishing to PyPi. - Check PyPI for the new release at ================================================ FILE: READMEs/search-and-filter-assets.md ================================================ # Quickstart: Search Assets Flow This quickstart describes how assets can be found by their `tags` from Aquarius. ## 1. Setup Ensure that you've already (a) [installed Ocean](install.md), and (b) [set up locally](setup-local.md) or [remotely](setup-remote.md). ## 2. Alice publishes datasets Now, you're Alice. ```python #data info url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" # Created a list of tags for the following assets tags = [ ["Branin dataset 1", "test", "ganache", "best asset"], ["Branin dataset 2", "test", "ocean"], ["Branin dataset 3", "AI", "dataset", "testing"], ] # Publish few assets for testing for tag in tags: name = tag[0] tx_dict = {"from": alice} metadata = ocean.assets.__class__.default_metadata(name, tx_dict) metadata.update({"tags": tag[1:]}) (data_NFT, datatoken, ddo) = ocean.assets.create_url_asset(name, url, tx_dict, metadata=metadata) print(f"Just published asset, with did={ddo.did}") ``` ## 3. Alice filters assets by their `tags` Alice can filter the assets by a certain tag and after can retrieve the necessary information afterwards. ```python # Get a list of assets filtered by a given tag. # All assets that contain the specified tag name tag = "test" all_ddos = ocean.assets.search(tag) # Filter just by the `tags` key filtered_ddos = list( filter( lambda a: tag in a.metadata["tags"], list(filter(lambda a: "tags" in a.metadata.keys(), all_ddos)), ) ) # Make sure that the provided tag is valid. assert len(filtered_ddos) > 0, "Assets not found with this tag." # Retrieve the wanted information from assets. for ddo in filtered_ddos: print(f"ddo.did :{ddo.did}") print(f"ddo.metadata :{ddo.metadata}") print(f"ddo.nft :{ddo.nft}") print(f"ddo.datatokens :{ddo.datatokens}") ``` ## Running custom queries You can run any custom ES query using OceanAssets. For example: ```python results = ocean.assets.query( { "query": { "query_string": { "query": "Branin dataset 1", "fields": ["metadata.name"], } } } ) assert results[0].metadata["name"] == "Branin dataset 1" ``` ================================================ FILE: READMEs/services.md ================================================ # About Ocean off-chain services ## Introduction Ocean uses these off-chain services: - [Ocean Provider](https://github.com/oceanprotocol/provider) is for data services. Specifically, it's a REST API serving requests for two types of data services: static urls (for downloading data) and compute services. It's run by the marketplace or the data publisher. - [Ocean Aquarius](https://github.com/oceanprotocol/aquarius) is metadata cache REST API. This helps to aid search in marketplaces. We now describe how to use these, for each of: - Local Services: Default - Local Services: Non-Default - Remote Services: Default - Remote Services: Non-Default ### Local Services: Default When you follow [local setup](READMEs/setup-local.md), you will use Barge. Barge runs its own Ganache, and also its own Provider and Aquarius. You don't need to do more. ### Local Services: Non-Default Instead of pointing to existing services (in Barge), you can run your own. Here's how. Open a new console, and get provider running: ```console docker run oceanprotocol/provider:latest ``` Open another new console, and get aquarius running: ```console docker run oceanprotocol/aquarius:latest ``` Here are the urls for the local services, for use in the config dict. - Provider url: `http://127.0.0.1:8030` - Aquarius url: `http://127.0.0.1:5000` Remember, here's how the config dict is set. ```python from ocean_lib.example_config import get_config_dict config = get_config_dict() # returns a dict # (then, here you can update the config dict as you wish) ocean = Ocean(config) ``` ### Remote Services: Default For convenience, Ocean Protocol Foundation (OPF) runs an instance of Provider, and of Aquarius. [Ocean network docs](https://docs.oceanprotocol.com/core-concepts/networks) gives the urls. When you follow [remote setup](READMEs/setup-remote.md), it will default to use these OPF-run Provider and Aquarius. You don't need to do more. ### Remote Services: Non-Default You can run your own Provider or Aquarius, like shown above. And then point to it from your config dict. You can also point to a Provider or Aquarius instance run by a 3rd party. Simply point to it from your config dict. ================================================ FILE: READMEs/setup-local.md ================================================ # Local Setup Here, we do setup for local testing. We assume you've already [installed Ocean](install.md). ## 1. Download barge and run services Ocean `barge` runs ganache (local blockchain), Provider (data service), and Aquarius (metadata cache). Barge helps you quickly become familiar with Ocean, because the local blockchain has low latency and no transaction fees. Accordingly, many READMEs use it. However, if you plan to only use Ocean with remote services, you don't need barge. Note: if you are running MacOS or Windows, we recommend to go directly to [Remote Setup](setup-remote.md). Why: Barge uses Docker, which behaves badly on MacOS and Windows. We're working to address this [here](https://github.com/oceanprotocol/ocean.py/issues/1313). In a new console: ```console # Grab repo git clone https://github.com/oceanprotocol/barge cd barge # Clean up old containers (to be sure) docker system prune -a --volumes # Run barge: start Ganache, Provider, Aquarius; deploy contracts; update ~/.ocean # for support of type 2 transactions export GANACHE_FORK=london ./start_ocean.sh ``` Now that we have barge running, we can mostly ignore its console while it runs. ## 2. Set envvars From here on, go to a console different than Barge. (E.g. the console where you installed Ocean, or a new one.) First, ensure that you're in the working directory, with venv activated: ```console cd my_project source venv/bin/activate ``` Then, set keys in readmes. As a Linux user, you'll use "`export`". In the same console: ```console # keys for alice and bob in readmes export TEST_PRIVATE_KEY1=0x8467415bb2ba7c91084d932276214b11a3dd9bdb2930fefa194b666dd8020b99 export TEST_PRIVATE_KEY2=0x1d751ded5a32226054cd2e71261039b65afb9ee1c746d055dd699b1150a5befc export TEST_PRIVATE_KEY3=0x732fbb7c355aa8898f4cff92fa7a6a947339eaf026a08a51f171199e35a18ae0 # key for minting fake OCEAN export FACTORY_DEPLOYER_PRIVATE_KEY=0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58 ``` ## 3. Setup in Python In the same console, run Python console: ```console python ``` In the Python console: ```python # Create Ocean instance from ocean_lib.example_config import get_config_dict config = get_config_dict("http://localhost:8545") from ocean_lib.ocean.ocean import Ocean ocean = Ocean(config) # Create OCEAN object. Barge auto-created OCEAN, and ocean instance knows OCEAN = ocean.OCEAN_token # Mint fake OCEAN to Alice & Bob from ocean_lib.ocean.mint_fake_ocean import mint_fake_OCEAN mint_fake_OCEAN(config) # Create Alice's wallet import os from eth_account import Account alice_private_key = os.getenv("TEST_PRIVATE_KEY1") alice = Account.from_key(private_key=alice_private_key) assert ocean.wallet_balance(alice) > 0, "Alice needs ETH" assert OCEAN.balanceOf(alice) > 0, "Alice needs OCEAN" # Create additional wallets. While some flows just use Alice wallet, it's simpler to do all here. bob_private_key = os.getenv('TEST_PRIVATE_KEY2') bob = Account.from_key(private_key=bob_private_key) assert ocean.wallet_balance(bob) > 0, "Bob needs ETH" assert OCEAN.balanceOf(bob) > 0, "Bob needs OCEAN" carlos_private_key = os.getenv('TEST_PRIVATE_KEY3') carlos = Account.from_key(private_key=carlos_private_key) assert ocean.wallet_balance(carlos) > 0, "Carlos needs ETH" assert OCEAN.balanceOf(carlos) > 0, "Carlos needs OCEAN" # Compact wei <> eth conversion from ocean_lib.ocean.util import to_wei, from_wei ``` ## 4. Next step You've now set up everything you need for local testing, congrats! The next step - the fun one - is to walk through the [main flow](main-flow.md). In it, you'll publish a data asset, post for free / for sale, dispense it / buy it, and consume it. Because you've set up for local, you'll be doing all these steps on the local network. ================================================ FILE: READMEs/setup-remote.md ================================================ # Remote Setup Here, we do setup for Mumbai, the testnet for Polygon. It's similar for other remote chains. We assume you've already [installed Ocean](install.md). Here, we will: 1. Configure networks 2. Create two accounts - `REMOTE_TEST_PRIVATE_KEY1` and `2` 3. Get fake MATIC on Mumbai 4. Get fake OCEAN on Mumbai 5. Set envvars 6. Set up Alice and Bob wallets in Python Let's go! ## 1. Configure Networks ### 1.1 Setup network RPC URLs for all desired networks All [Ocean chain deployments](https://docs.oceanprotocol.com/discover/networks) (Eth mainnet, Polygon, etc) are supported. ## 2. Create EVM Accounts (One-Time) An EVM account is singularly defined by its private key. Its address is a function of that key. Let's generate two accounts! In a new or existing console, run Python. ```console python ``` In the Python console: ```python from eth_account.account import Account account1 = Account.create() account2 = Account.create() print(f""" REMOTE_TEST_PRIVATE_KEY1={account1.key.hex()}, ADDRESS1={account1.address} REMOTE_TEST_PRIVATE_KEY2={account2.key.hex()}, ADDRESS2={account2.address} """) ``` Then, hit Ctrl-C to exit the Python console. Now, you have two EVM accounts (address & private key). Save them somewhere safe, like a local file or a password manager. These accounts will work on any EVM-based chain: production chains like Eth mainnet and Polygon, and testnets like Goerli and Mumbai. Here, we'll use them for Mumbai. ## 3. Get (fake) MATIC on Mumbai We need the a network's native token to pay for transactions on the network. [ETH](https://ethereum.org/en/get-eth/) is the native token for Ethereum mainnet; [MATIC](https://polygon.technology/matic-token/) is the native token for Polygon, and [(fake) MATIC](https://faucet.polygon.technology/) is the native token for Mumbai. To get free (fake) MATIC on Mumbai: 1. Go to the faucet https://faucet.polygon.technology/. Ensure you've selected "Mumbai" network and "MATIC" token. 2. Request funds for ADDRESS1 3. Request funds for ADDRESS2 You can confirm receiving funds by going to the following url, and seeing your reported MATIC balance: `https://mumbai.polygonscan.com/address/` ## 4. Get (fake) OCEAN on Mumbai [OCEAN](https://oceanprotocol.com/token) can be used as a data payment token, and locked into veOCEAN for Data Farming / curation. The READMEs show how to use OCEAN in both cases. - OCEAN is an ERC20 token with a finite supply, rooted in Ethereum mainnet at address [`0x967da4048cD07aB37855c090aAF366e4ce1b9F48`](https://etherscan.io/token/0x967da4048cD07aB37855c090aAF366e4ce1b9F48). - OCEAN on other production chains derives from the Ethereum mainnet OCEAN. OCEAN on Polygon (mOCEAN) is at [`0x282d8efce846a88b159800bd4130ad77443fa1a1`](https://polygonscan.com/token/0x282d8efce846a88b159800bd4130ad77443fa1a1). - (Fake) OCEAN is on each testnet. Fake OCEAN on Mumbai is at [`0xd8992Ed72C445c35Cb4A2be468568Ed1079357c8`](https://mumbai.polygonscan.com/token/0xd8992Ed72C445c35Cb4A2be468568Ed1079357c8). To get free (fake) OCEAN on Mumbai: 1. Go to the faucet https://faucet.mumbai.oceanprotocol.com/ 2. Request funds for ADDRESS1 3. Request funds for ADDRESS2 You can confirm receiving funds by going to the following url, and seeing your reported OCEAN balance: `https://mumbai.polygonscan.com/token/0xd8992Ed72C445c35Cb4A2be468568Ed1079357c8?a=` ## 5. Set envvars As usual, Linux/MacOS needs "`export`" and Windows needs "`set`". In the console: #### Linux & MacOS users: ```console # For accounts: set private keys export REMOTE_TEST_PRIVATE_KEY1= export REMOTE_TEST_PRIVATE_KEY2= export MUMBAI_RPC_URL= # exported used for convenience/security, you can also use the direct URL string later ``` #### Windows users: ```console # For accounts: set private keys set REMOTE_TEST_PRIVATE_KEY1= set REMOTE_TEST_PRIVATE_KEY2= set MUMBAI_RPC_URL= # exported used for convenience/security, you can also use the direct URL string later ``` Optionally, chainlist.org has other RPCs for [Mumbai](https://chainlist.org/chain/80001) and [Polygon](https://chainlist.org/chain/137). ## 6. Setup in Python In your working console, run Python: ```console python ``` In the Python console: ```python # Create Ocean instance import os from ocean_lib.example_config import get_config_dict from ocean_lib.ocean.ocean import Ocean config = get_config_dict(os.getenv("MUMBAI_RPC_URL")) # you can also input the string directly ocean = Ocean(config) # Create OCEAN object. ocean_lib knows where OCEAN is on all remote networks OCEAN = ocean.OCEAN_token # Create Alice's wallet from eth_account import Account alice_private_key = os.getenv('REMOTE_TEST_PRIVATE_KEY1') alice = Account.from_key(private_key=alice_private_key) assert ocean.wallet_balance(alice) > 0, "Alice needs MATIC" assert OCEAN.balanceOf(alice) > 0, "Alice needs OCEAN" # Create Bob's wallet. While some flows just use Alice wallet, it's simpler to do all here. bob_private_key = os.getenv('REMOTE_TEST_PRIVATE_KEY2') bob = Account.from_key(private_key=bob_private_key) assert ocean.wallet_balance(bob) > 0, "Bob needs MATIC" assert OCEAN.balanceOf(bob) > 0, "Bob needs OCEAN" # Compact wei <> eth conversion from ocean_lib.ocean.util import to_wei, from_wei ``` If you get a gas-related error like `transaction underpriced`, you'll need to change the `priority_fee` or `max_fee`. ## Next step You've now set up everything you need for testing on a remote chain, congrats! it's similar for any remote chain. The next step is to walk through the [main flow](main-flow.md). In it, you'll publish a data asset, post for free / for sale, dispense it / buy it, and consume it. Because you've set up for remote, you'll be doing all these steps on the remote network. ================================================ FILE: READMEs/using-clef.md ================================================ # Using hardware wallets with ocean.py This README describes how to setup ocean.py with hardware wallets. We assume you've already (a) [installed Ocean](install.md), configured any environment variables necessary and created the Ocean object as described in (b) done [local setup](setup-local.md) or [remote setup](setup-remote.md). These instructions are applicable to both local and remote setup. If you intend to use hardware wallets ONLY, then you can skip the wallet creation parts in the setup instructions. ## 1. Setting up and running Clef ocean.py allows the use of hardware wallets via [Clef](https://geth.ethereum.org/docs/clef/tutorial), an account management tool included within [Geth](https://geth.ethereum.org/) To use a hardware wallet with ocean.py, start by [installing Geth](https://geth.ethereum.org/docs/install-and-build/installing-geth). Once finished, type the following command in a bash console and follow the on-screen prompts to set of Clef: ```console clef init ``` If you need to create a new account, you can use the command `clef newaccount`. For other usefull commands, please consult the [Clef documentation](https://geth.ethereum.org/docs/tools/clef/introduction). Once Clef is configured, run it in a bash console as needed, i.e. ```console # you can use a different chain if needed clef --chainid 8996 ``` You can also customise your run, e.g. `clef --chainid 8996 --advanced`. Keep the clef console open, you will be required to approve transactions and input your password when so requested. ## 2. Connect ocean.py to Clef via Brownie In your Python console where you have setup the Ocean object: ```python from ocean_lib.web3_internal.clef import get_clef_accounts clef_accounts = get_clef_accounts() ``` Approve the connection from the Clef console. This will add your Clef account to the `accounts` array. You can now use the Clef account instead of any wallet argument, e.g. when publishing or consuming DDOs. ```python # pick up the account for convenience clef_account = clef_accounts[index] # make sure account is funded. Let's transfer some ether and OCEAN from alice from ocean_lib.ocean.util import send_ether send_ether(config, alice, clef_account.address, to_wei(4)) OCEAN.transfer(clef_account, to_wei(4), {"from": alice}) # publish and download an asset name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" (data_nft, datatoken, ddo) = ocean.assets.create_url_asset(name, url, {"from": clef_account}) datatoken.mint(clef_account, to_wei(1), {"from": clef_account}) order_tx_id = ocean.assets.pay_for_access_service(ddo, {"from": clef_account}) ocean.assets.download_asset(ddo, clef_account, './', order_tx_id) ``` Please note that you need to consult your clef console periodically to approve transactions and input your password if needed. You can use the ClefAccount object seamlessly, in any transaction, just like regular Accounts. Simply send your transaction with `{"from": clef_account}` where needed. ================================================ FILE: bumpversion.sh ================================================ #!/bin/bash ## ## Copyright 2023 Ocean Protocol Foundation ## SPDX-License-Identifier: Apache-2.0 ## set -x set -e usage(){ echo "Usage: $0 {major|minor|patch} [--tag]" exit 1 } if ! [ -x "$(command -v bumpversion)" ]; then echo 'Error: bumpversion is not installed.' >&2 exit 1 elif ! git diff-index --quiet HEAD -- >/dev/null 2>&1; then echo 'There are local changes in your the git repository. Please commit or stash them before bumping version.' >&2 exit 1 fi if [ "$#" -lt 1 ]; then echo "Illegal number of parameters" usage elif [[ $1 != 'major' && $1 != 'minor' && $1 != 'patch' ]]; then echo 'First argument must be {major|minor|patch}' usage fi if [[ $2 == '--tag' ]]; then if git branch --contains $(git rev-parse --verify HEAD) | grep -E 'main'; then eval "bumpversion --tag --commit $1" else echo "Only main tags can be tagged" exit 1 fi else eval "bumpversion --no-tag $1" fi ================================================ FILE: conftest.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # # This is the global conftest. EVERY SINGLE TEST looks at this # - For tests that use ganache, import conftest_ganache.py. Just don't put it here. # - For tests that use remote networks, do your own thing. Just don't put it here. ================================================ FILE: conftest_ganache.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from typing import Tuple import pytest from ocean_lib.example_config import get_config_dict from ocean_lib.models.data_nft import DataNFT from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.models.datatoken1 import Datatoken1 from ocean_lib.models.factory_router import FactoryRouter from ocean_lib.models.fixed_rate_exchange import FixedRateExchange from ocean_lib.ocean.util import get_address_of_type, send_ether, to_wei from ocean_lib.web3_internal.contract_utils import get_contracts_addresses_all_networks from tests.resources.helper_functions import ( deploy_erc721_erc20, get_another_consumer_wallet, get_consumer_ocean_instance, get_consumer_wallet, get_factory_deployer_wallet, get_file1, get_file2, get_file3, get_ganache_wallet, get_provider_wallet, get_publisher_ocean_instance, get_publisher_wallet, get_wallet, setup_logging, ) _NETWORK = "ganache" setup_logging() @pytest.fixture(autouse=True) def setup_all(request, config, ocean_token): # a test can skip setup_all() via decorator "@pytest.mark.nosetup_all" if "nosetup_all" in request.keywords: return wallet = get_ganache_wallet() if not wallet: return if not get_contracts_addresses_all_networks(config): print("Can not find adddresses.") return balance = config["web3_instance"].eth.get_balance(wallet.address) assert balance >= to_wei(10), "Need more ETH" amt_distribute = to_wei(1000) ocean_token.mint(wallet, to_wei(2000), {"from": wallet}) for w in (get_publisher_wallet(), get_consumer_wallet()): balance = config["web3_instance"].eth.get_balance(w.address) if balance < to_wei(2): send_ether(config, wallet, w.address, to_wei(4)) if ocean_token.balanceOf(w) < to_wei(100): ocean_token.mint(w, amt_distribute, {"from": wallet}) @pytest.fixture def config(): return get_config_dict() @pytest.fixture def publisher_ocean(): return get_publisher_ocean_instance() @pytest.fixture def basic_asset(publisher_ocean, publisher_wallet): name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" (data_nft, datatoken, ddo) = publisher_ocean.assets.create_url_asset( name, url, {"from": publisher_wallet} ) assert ddo.nft["name"] == name assert len(ddo.datatokens) == 1 return (data_nft, datatoken, ddo) @pytest.fixture def consumer_ocean(): return get_consumer_ocean_instance() @pytest.fixture def publisher_wallet(): return get_publisher_wallet() @pytest.fixture def consumer_wallet(): return get_consumer_wallet() @pytest.fixture def another_consumer_wallet(): return get_another_consumer_wallet() @pytest.fixture def factory_deployer_wallet(config): return get_factory_deployer_wallet(config) @pytest.fixture def ocean_address(config) -> str: return get_address_of_type(config, "Ocean") @pytest.fixture def ocean_token(config, ocean_address) -> Datatoken1: return Datatoken1(config, ocean_address) @pytest.fixture def factory_router(config): return FactoryRouter(config, get_address_of_type(config, "Router")) @pytest.fixture def data_nft_factory(config): return DataNFTFactoryContract(config, get_address_of_type(config, "ERC721Factory")) @pytest.fixture def provider_wallet(): return get_provider_wallet() @pytest.fixture def file1(): return get_file1() @pytest.fixture def file2(): return get_file2() @pytest.fixture def file3(): return get_file3() @pytest.fixture def FRE(config) -> FixedRateExchange: return FixedRateExchange(config, get_address_of_type(config, "FixedPrice")) @pytest.fixture def data_nft(config, publisher_wallet) -> DataNFT: return deploy_erc721_erc20(config, publisher_wallet) @pytest.fixture def data_NFT_and_DT(config, publisher_wallet) -> Tuple[DataNFT, Datatoken1]: return deploy_erc721_erc20(config, publisher_wallet, publisher_wallet) @pytest.fixture def DT(data_NFT_and_DT) -> Datatoken1: (_, DT) = data_NFT_and_DT return DT # aliases @pytest.fixture def OCEAN(ocean_token) -> Datatoken1: return ocean_token @pytest.fixture def alice(publisher_wallet): return publisher_wallet @pytest.fixture def bob(consumer_wallet): return consumer_wallet @pytest.fixture def carlos(): return get_wallet(8) @pytest.fixture def dan(): return get_wallet(7) ================================================ FILE: ocean_lib/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Initialises ocean lib package.""" __author__ = """OceanProtocol""" # fmt: off __version__ = '3.1.2' # fmt: on ================================================ FILE: ocean_lib/agreements/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/agreements/consumable.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from enforce_typing import enforce_types class ConsumableCodes: """ Contains constant values for: - OK - ASSET_DISABLED - CONNECTIVITY_FAIL - CREDENTIAL_NOT_IN_ALLOW_LIST - CREDENTIAL_IN_DENY_LIST """ OK = 0 ASSET_DISABLED = 1 CONNECTIVITY_FAIL = 2 CREDENTIAL_NOT_IN_ALLOW_LIST = 3 CREDENTIAL_IN_DENY_LIST = 4 ASSET_UNLISTED = 5 class MalformedCredential(Exception): pass class AssetNotConsumable(Exception): @enforce_types def __init__(self, consumable_code: int) -> None: self.consumable_code = consumable_code ================================================ FILE: ocean_lib/agreements/service_types.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Agreements module.""" class ServiceTypes: """Types of Service allowed in ocean protocol DDO services for V4.""" ASSET_ACCESS = "access" CLOUD_COMPUTE = "compute" AUTHORIZATION = "wss" class ServiceTypesNames: DEFAULT_ACCESS_NAME = "Download service" DEFAULT_COMPUTE_NAME = "Compute service" ================================================ FILE: ocean_lib/aquarius/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Ocean Aquarius module.""" from .aquarius import Aquarius # noqa ================================================ FILE: ocean_lib/aquarius/aquarius.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """ Aquarius module. Help to communicate with the metadata store. """ import json import logging import time from typing import Optional, Tuple, Union from enforce_typing import enforce_types from ocean_lib.assets.ddo import DDO from ocean_lib.http_requests.requests_session import get_requests_session logger = logging.getLogger("aquarius") class Aquarius: """Aquarius wrapper to call different endpoint of aquarius component.""" @enforce_types def __init__(self, aquarius_url: str) -> None: """ This class wraps Aquarius REST API. :param aquarius_url: Url of the aquarius instance. """ assert aquarius_url, f'Invalid url "{aquarius_url}"' # :HACK: if "/api/aquarius/assets" in aquarius_url: aquarius_url = aquarius_url[: aquarius_url.find("/api/aquarius/assets")] self.requests_session = get_requests_session() try: response = self.requests_session.get(f"{aquarius_url}") except Exception: response = None if not response or response.status_code != 200: raise Exception(f"Invalid or unresponsive aquarius url {aquarius_url}") self.base_url = f"{aquarius_url}/api/aquarius/assets" logging.debug(f"Aquarius connected at {aquarius_url}") logging.debug(f"Aquarius API documentation at {aquarius_url}/api/v1/docs") logging.debug(f"Metadata assets (DDOs) at {self.base_url}") @classmethod def get_instance(cls, metadata_cache_uri: str) -> "Aquarius": return cls(metadata_cache_uri) @enforce_types def get_ddo(self, did: str) -> Optional[DDO]: """Retrieve ddo for a given did.""" response = self.requests_session.get(f"{self.base_url}/ddo/{did}") if response.status_code == 200: response_dict = response.json() return DDO.from_dict(response_dict) return None @enforce_types def ddo_exists(self, did: str) -> bool: """Is this DDO in Aqua?""" response = self.requests_session.get(f"{self.base_url}/ddo/{did}").content return f"Asset DID {did} not found in Elasticsearch" not in str(response) @enforce_types def get_ddo_metadata(self, did: str) -> dict: """Returns a given DDO's "metadata" field values""" response = self.requests_session.get(f"{self.base_url}/metadata/{did}") if response.status_code == 200: return response.json() return {} @enforce_types def query_search(self, search_query: dict) -> list: """ Search using a query. Currently implemented is the MongoDB query model to search for documents according to: https://docs.mongodb.com/manual/tutorial/query-documents/ And an Elastic Search driver, which implements a basic parser to convert the query into elastic search format. Example: query_search({"price":[0,10]}) :param search_query: Python dictionary, query following elasticsearch syntax :return: List of DDO """ response = self.requests_session.post( f"{self.base_url}/query", data=json.dumps(search_query), headers={"content-type": "application/json"}, ) if response.status_code == 200: return response.json()["hits"]["hits"] raise ValueError(f"Unable to search for DDO: {response.content}") @enforce_types def validate_ddo(self, ddo: DDO) -> Tuple[bool, Union[list, dict]]: """Does the DDO conform to the Ocean DDO schema? Schema definition: https://docs.oceanprotocol.com/core-concepts/did-ddo """ ddo_dict = ddo.as_dictionary() data = json.dumps(ddo_dict, separators=(",", ":")).encode("utf-8") response = self.requests_session.post( f"{self.base_url.replace('/v1/', '/')}/ddo/validate", data=data, headers={"content-type": "application/octet-stream"}, ) parsed_response = response.json() if parsed_response.get("hash"): return True, parsed_response return False, parsed_response @enforce_types def wait_for_ddo(self, did: str, timeout=60): start = time.time() ddo = None while not ddo: ddo = self.get_ddo(did) if not ddo: time.sleep(0.2) if time.time() - start > timeout: break return ddo @enforce_types def wait_for_ddo_update(self, ddo: DDO, tx: str): start = time.time() ddo2 = None while True: try: ddo2 = self.get_ddo(ddo.did) except ValueError: pass if not ddo2: time.sleep(0.2) elif ddo2.event.get("tx") == tx: logger.debug( f"Transaction matching the given tx id detected in metadata store. ddo2.event = {ddo2.event}" ) break elapsed_time = time.time() - start if elapsed_time > 60: break return ddo2 ================================================ FILE: ocean_lib/aquarius/test/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/aquarius/test/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: ocean_lib/aquarius/test/test_aquarius.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.aquarius.aquarius import Aquarius from ocean_lib.assets.ddo import DDO from ocean_lib.example_config import METADATA_CACHE_URI @pytest.mark.unit def test_init(): """Tests initialisation of Aquarius objects.""" aqua = Aquarius("http://172.15.0.5:5000/api/aquarius/assets") assert aqua.base_url == "http://172.15.0.5:5000/api/aquarius/assets" @pytest.mark.integration def test_aqua_functions_for_single_ddo(publisher_ocean, publisher_wallet, basic_asset): """Tests against single-ddo functions of Aquarius.""" aquarius = publisher_ocean.assets._aquarius _, _, ddo1 = basic_asset metadata1 = ddo1.metadata ddo2 = aquarius.wait_for_ddo(ddo1.did) assert ddo2.metadata == ddo1.metadata ddo3 = publisher_ocean.assets.resolve(ddo1.did) assert ddo3.did == ddo1.did, "Aquarius could not resolve the did." assert ddo3.did == ddo2.did, "Aquarius could not resolve the did." aqua_uri = publisher_ocean.config_dict.get("METADATA_CACHE_URI") ddo4 = Aquarius.get_instance(aqua_uri).get_ddo(ddo2.did) assert isinstance(ddo4, DDO) assert ddo4.did == ddo2.did, "Aquarius could not resolve the did." metadata2 = aquarius.get_ddo_metadata(ddo2.did) assert metadata2 == metadata1 @pytest.mark.unit def test_invalid_search_query(): """Tests query search with an invalid query.""" aquarius = Aquarius.get_instance(METADATA_CACHE_URI) search_query = "not_a_dict" with pytest.raises(TypeError): aquarius.query_search(search_query=search_query) @pytest.mark.unit def test_empty_responses(): aquarius = Aquarius.get_instance(METADATA_CACHE_URI) assert aquarius.get_ddo_metadata("inexistent_ddo") == {} ================================================ FILE: ocean_lib/assets/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/assets/asset_downloader.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import logging import os from typing import Optional, Union from enforce_typing import enforce_types from ocean_lib.agreements.consumable import AssetNotConsumable, ConsumableCodes from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.services.service import Service logger = logging.getLogger(__name__) @enforce_types def download_asset_files( ddo: DDO, service: Service, consumer_wallet, destination: str, order_tx_id: Union[str, bytes], index: Optional[int] = None, userdata: Optional[dict] = None, ) -> str: """Download asset data file or result file from compute job. :param ddo: DDO instance :param service: Sevice instance :param consumer_wallet: Wallet instance of the consumer :param destination: Path, str :param order_tx_id: hex str or hex bytes the transaction hash of the startOrder tx :param index: Index of the document that is going to be downloaded, Optional[int] :param userdata: Dict of additional data from user :return: asset folder path, str """ data_provider = DataServiceProvider if not service.service_endpoint: logger.error( 'Consume asset failed, service definition is missing the "serviceEndpoint".' ) raise AssertionError( 'Consume asset failed, service definition is missing the "serviceEndpoint".' ) if index is not None: assert isinstance(index, int), logger.error("index has to be an integer.") assert index >= 0, logger.error("index has to be 0 or a positive integer.") consumable_result = is_consumable( ddo, service, {"type": "address", "value": consumer_wallet.address}, with_connectivity_check=True, userdata=userdata, ) if consumable_result != ConsumableCodes.OK: raise AssetNotConsumable(consumable_result) service_index_in_asset = ddo.get_index_of_service(service) asset_folder = os.path.join( destination, f"datafile.{ddo.did},{service_index_in_asset}" ) if not os.path.exists(asset_folder): os.makedirs(asset_folder) data_provider.download( did=ddo.did, service=service, tx_id=order_tx_id, consumer_wallet=consumer_wallet, destination_folder=asset_folder, index=index, userdata=userdata, ) return asset_folder @enforce_types def is_consumable( ddo: DDO, service: Service, credential: Optional[dict] = None, with_connectivity_check: bool = True, userdata: Optional[dict] = None, ) -> bool: """Checks whether an asset is consumable and returns a ConsumableCode.""" if ddo.is_disabled: return ConsumableCodes.ASSET_DISABLED if with_connectivity_check and not DataServiceProvider.check_asset_file_info( ddo.did, service.id, service.service_endpoint, userdata=userdata ): return ConsumableCodes.CONNECTIVITY_FAIL # to be parameterized in the future, can implement other credential classes if ddo.requires_address_credential: return ddo.validate_access(credential) return ConsumableCodes.OK ================================================ FILE: ocean_lib/assets/credentials.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from typing import Optional from enforce_typing import enforce_types from ocean_lib.agreements.consumable import ConsumableCodes, MalformedCredential class AddressCredentialMixin: @enforce_types def get_addresses_of_class(self, access_class: str = "allow") -> list: """Get a filtered list of addresses from credentials (use with allow/deny).""" address_entry = self.get_address_entry_of_class(access_class) if not address_entry: return [] if "values" not in address_entry: raise MalformedCredential("No values key in the address credential.") return [addr.lower() for addr in address_entry["values"]] @enforce_types def requires_credential(self) -> bool: """Checks whether the ddo requires an address credential.""" allowed_addresses = self.get_addresses_of_class("allow") denied_addresses = self.get_addresses_of_class("deny") return bool(allowed_addresses or denied_addresses) @enforce_types def validate_access(self, credential: Optional[dict] = None) -> int: """Checks a credential dictionary against the address allow/deny lists.""" address = simplify_credential_to_address(credential) allowed_addresses = self.get_addresses_of_class("allow") denied_addresses = self.get_addresses_of_class("deny") if not address and not self.requires_credential(): return ConsumableCodes.OK if allowed_addresses and address.lower() not in allowed_addresses: return ConsumableCodes.CREDENTIAL_NOT_IN_ALLOW_LIST if not allowed_addresses and address.lower() in denied_addresses: return ConsumableCodes.CREDENTIAL_IN_DENY_LIST return ConsumableCodes.OK @enforce_types def add_address_to_access_class( self, address: str, access_class: str = "allow" ) -> None: """Adds an address to an address list (either allow or deny).""" address = address.lower() if not self.credentials or access_class not in self.credentials: self.credentials[access_class] = [{"type": "address", "values": [address]}] return address_entry = self.get_address_entry_of_class(access_class) if not address_entry: self.credentials[access_class].append( {"type": "address", "values": [address]} ) return lc_addresses = self.get_addresses_of_class(access_class) if address not in lc_addresses: lc_addresses.append(address) address_entry["values"] = lc_addresses @enforce_types def remove_address_from_access_class( self, address: str, access_class: str = "allow" ) -> None: """Removes an address from an address list (either allow or deny)i.""" address = address.lower() if not self.credentials or access_class not in self.credentials: return address_entry = self.get_address_entry_of_class(access_class) if not address_entry: return lc_addresses = self.get_addresses_of_class(access_class) if address not in lc_addresses: return lc_addresses.remove(address) address_entry["values"] = lc_addresses @enforce_types def get_address_entry_of_class(self, access_class: str = "allow") -> Optional[dict]: """Get address credentials entry of the specified access class. access_class = "allow" or "deny".""" entries = self.credentials.get(access_class, []) address_entries = [entry for entry in entries if entry.get("type") == "address"] return address_entries[0] if address_entries else None @enforce_types def simplify_credential_to_address(credential: Optional[dict]) -> Optional[str]: """Extracts address value from credential dictionary.""" if not credential: return None if not credential.get("value"): raise MalformedCredential("Received empty address.") return credential["value"] ================================================ FILE: ocean_lib/assets/ddo.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import copy import logging from typing import Optional from enforce_typing import enforce_types from ocean_lib.assets.credentials import AddressCredentialMixin from ocean_lib.data_provider.fileinfo_provider import FileInfoProvider from ocean_lib.ocean.util import create_checksum from ocean_lib.services.service import Service logger = logging.getLogger("ddo") class DDO(AddressCredentialMixin): """Create, import, export, validate DDO objects.""" @enforce_types def __init__( self, did: Optional[str] = None, context: Optional[list] = None, chain_id: Optional[int] = None, nft_address: Optional[str] = None, metadata: Optional[dict] = None, services: Optional[list] = None, credentials: Optional[dict] = None, nft: Optional[dict] = None, datatokens: Optional[list] = None, event: Optional[dict] = None, stats: Optional[dict] = None, ) -> None: self.did = did self.context = context or ["https://w3id.org/did/v1"] self.chain_id = chain_id self.nft_address = nft_address self.metadata = metadata self.version = "4.1.0" self.services = services or [] self.credentials = credentials or {} self.nft = nft self.datatokens = datatokens self.event = event self.stats = stats @property @enforce_types def requires_address_credential(self) -> bool: """Checks if an address credential is required on this ddo.""" return self.requires_credential() @property @enforce_types def allowed_addresses(self) -> list: """Lists addresses that are explicitly allowed in credentials.""" return self.get_addresses_of_class("allow") @property @enforce_types def denied_addresses(self) -> list: """Lists addresses that are explicitly denied in credentials.""" return self.get_addresses_of_class("deny") @enforce_types def add_address_to_allow_list(self, address: str) -> None: """Adds an address to allowed addresses list.""" self.add_address_to_access_class(address, "allow") @enforce_types def add_address_to_deny_list(self, address: str) -> None: """Adds an address to the denied addresses list.""" self.add_address_to_access_class(address, "deny") @enforce_types def remove_address_from_allow_list(self, address: str) -> None: """Removes address from allow list (if it exists).""" self.remove_address_from_access_class(address, "allow") @enforce_types def remove_address_from_deny_list(self, address: str) -> None: """Removes address from deny list (if it exists).""" self.remove_address_from_access_class(address, "deny") @classmethod @enforce_types def from_dict(cls, dictionary: dict) -> "DDO": """Import a JSON dict into this DDO.""" values = copy.deepcopy(dictionary) services = ( [] if "services" not in values else [Service.from_dict(value) for value in values.pop("services")] ) args = [ values.pop("id", None), values.pop("@context", None), values.pop("chainId", None), values.pop("nftAddress", None), values.pop("metadata", None), services, values.pop("credentials", None), values.pop("nft", None), values.pop("datatokens", None), values.pop("event", None), values.pop("stats", None), ] if args[0] is None: return UnavailableDDO(*args) return cls(*args) @enforce_types def as_dictionary(self) -> dict: """ Return the DDO as a JSON dict. :return: dict """ data = { "@context": self.context, "id": self.did, "version": self.version, "chainId": self.chain_id, } data["nftAddress"] = self.nft_address services = [value.as_dictionary() for value in self.services] args = ["metadata", "credentials", "nft", "datatokens", "event", "stats"] attrs = list( filter( lambda attr: not not attr[1], map(lambda attr: (attr, getattr(self, attr, None)), args), ) ) attrs.append(("services", services)) data.update(attrs) return data @enforce_types def add_service(self, service: Service) -> None: """ Add a service to the list of services on the V4 DDO. :param service: To add service, Service """ service.encrypt_files(self.nft_address, self.chain_id) logger.debug( f"Adding service with service type {service.type} with did {self.did}" ) self.services.append(service) @enforce_types def create_compute_service( self, service_id: str, service_endpoint: str, datatoken_address: str, files, compute_values: Optional[dict] = None, timeout: Optional[int] = 3600, ) -> None: if not compute_values: compute_values = { "allowRawAlgorithm": False, "allowNetworkAccess": True, "publisherTrustedAlgorithms": [], "publisherTrustedAlgorithmPublishers": [], } compute_service = Service( service_id=service_id, service_type="compute", service_endpoint=service_endpoint, datatoken=datatoken_address, files=files, timeout=timeout, compute_values=compute_values, ) self.add_service(compute_service) @enforce_types def get_service_by_id(self, service_id: str) -> Service: """Return Service with the given id. Return None if service with the given id not found.""" return next( (service for service in self.services if service.id == service_id), None ) @enforce_types def get_service_by_index(self, service_index: int) -> Service: """Return Service with the given index. Return None if service with the given index not found.""" return ( self.services[service_index] if service_index < len(self.services) else None ) @enforce_types def get_index_of_service(self, service: Service) -> int: """Return index of the given Service. Return None if service was not found.""" return next( ( index for index, this_service in enumerate(self.services) if this_service.id == service.id ), None, ) @enforce_types def generate_trusted_algorithms(self) -> dict: """Returns a trustedAlgorithm dictionary for service at index 0.""" resp = FileInfoProvider.fileinfo( self.did, self.get_service_by_index(0), with_checksum=True ) files_checksum = [resp_item["checksum"] for resp_item in resp.json()] container = self.metadata["algorithm"]["container"] return { "did": self.did, "filesChecksum": "".join(files_checksum), "containerSectionChecksum": create_checksum( container["entrypoint"] + container["checksum"] ), } @property def is_disabled(self) -> bool: return not self.metadata or (self.nft and self.nft["state"] not in [0, 5]) class UnavailableDDO(DDO): pass ================================================ FILE: ocean_lib/assets/test/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/assets/test/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: ocean_lib/assets/test/test_asset_downloader.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os from unittest.mock import patch import pytest from requests.exceptions import InvalidURL from ocean_lib.agreements.consumable import AssetNotConsumable, ConsumableCodes from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.assets.asset_downloader import download_asset_files, is_consumable from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.models.datatoken_base import TokenFeeInfo from ocean_lib.ocean.util import to_wei from ocean_lib.services.service import Service from tests.resources.ddo_helpers import get_first_service_by_type, get_sample_ddo @pytest.mark.unit def test_is_consumable(): ddo_dict = get_sample_ddo() ddo = DDO.from_dict(ddo_dict) service_dict = ddo_dict["services"][0] service = Service.from_dict(service_dict) with patch( "ocean_lib.assets.test.test_asset_downloader.DataServiceProvider.check_asset_file_info", return_value=False, ): assert ( is_consumable(ddo, service, {}, True) == ConsumableCodes.CONNECTIVITY_FAIL ) with patch( "ocean_lib.assets.test.test_asset_downloader.DataServiceProvider.check_asset_file_info", return_value=True, ): assert ( is_consumable(ddo, service, {"type": "address", "value": "0xdddd"}, True) == ConsumableCodes.CREDENTIAL_NOT_IN_ALLOW_LIST ) @pytest.mark.unit def test_ocean_assets_download_failure(publisher_wallet): """Tests that downloading from an empty service raises an AssertionError.""" ddo_dict = get_sample_ddo() ddo = DDO.from_dict(ddo_dict) access_service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS) access_service.service_endpoint = None ddo.services[0] = access_service with pytest.raises(AssertionError): download_asset_files( ddo, access_service, publisher_wallet, "test_destination", "test_order_tx_id", ) @pytest.mark.unit def test_invalid_provider_uri(publisher_wallet): """Tests with invalid provider URI that raise AssertionError.""" ddo_dict = get_sample_ddo() ddo = DDO.from_dict(ddo_dict) ddo.services[0].service_endpoint = "http://nothing-here.com" with pytest.raises(InvalidURL): download_asset_files( ddo, ddo.services[0], publisher_wallet, "test_destination", "test_order_tx_id", ) @pytest.mark.unit def test_invalid_state(publisher_wallet): """Tests different scenarios that raise AssetNotConsumable.""" ddo_dict = get_sample_ddo() ddo = DDO.from_dict(ddo_dict) ddo.nft["state"] = 1 with pytest.raises(AssetNotConsumable): download_asset_files( ddo, ddo.services[0], publisher_wallet, "test_destination", "test_order_tx_id", ) ddo.metadata = [] with pytest.raises(AssetNotConsumable): download_asset_files( ddo, ddo.services[0], publisher_wallet, "test_destination", "test_order_tx_id", ) @pytest.mark.integration def test_ocean_assets_download_indexes(publisher_wallet): """Tests different values of indexes that raise AssertionError.""" ddo_dict = get_sample_ddo() ddo = DDO.from_dict(ddo_dict) index = range(3) with pytest.raises(TypeError): download_asset_files( ddo, ddo.services[0], publisher_wallet, "test_destination", "test_order_tx_id", index=index, ) index = -1 with pytest.raises(AssertionError): download_asset_files( ddo, ddo.services[0], publisher_wallet, "test_destination", "test_order_tx_id", index=index, ) @pytest.mark.integration def test_ocean_assets_download_destination_file( tmpdir, publisher_ocean, publisher_wallet, basic_asset, ): """Convert tmpdir: py._path.local.LocalPath to str, satisfy enforce-typing.""" data_provider = DataServiceProvider data_nft, datatoken, ddo = basic_asset access_service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS) datatoken.mint( publisher_wallet.address, to_wei(50), {"from": publisher_wallet}, ) initialize_response = data_provider.initialize( did=ddo.did, service=access_service, consumer_address=publisher_wallet.address, ) provider_fees = initialize_response.json()["providerFee"] consume_market_fees = TokenFeeInfo( address=publisher_wallet.address, token=datatoken.address, ) receipt = datatoken.start_order( consumer=publisher_wallet.address, service_index=ddo.get_index_of_service(access_service), provider_fees=provider_fees, consume_market_fees=consume_market_fees, tx_dict={"from": publisher_wallet}, ) orders = publisher_ocean.get_user_orders( publisher_wallet.address, datatoken.address ) assert datatoken.address in [order.address for order in orders] assert receipt.transactionHash.hex() in [ order.transactionHash.hex() for order in orders ] written_path = download_asset_files( ddo, access_service, publisher_wallet, str(tmpdir), receipt.transactionHash.hex(), ) assert os.path.exists(written_path) # index not found, even though tx_id exists with pytest.raises(AssertionError): download_asset_files( ddo, ddo.services[0], publisher_wallet, str(tmpdir), receipt.transactionHash.hex(), index=4, ) ================================================ FILE: ocean_lib/assets/test/test_ddo.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.agreements.consumable import MalformedCredential from ocean_lib.assets.credentials import simplify_credential_to_address from ocean_lib.assets.ddo import DDO from ocean_lib.services.service import Service from tests.resources.ddo_helpers import ( get_key_from_v4_sample_ddo, get_sample_ddo, get_sample_ddo_with_compute_service, ) @pytest.mark.unit def test_ddo_utils(): """Tests the structure of a JSON format of the V4 DDO.""" ddo_dict = get_sample_ddo() assert isinstance(ddo_dict, dict) assert isinstance(ddo_dict, dict) assert ddo_dict["@context"] == ["https://w3id.org/did/v1"] context = ddo_dict["@context"] assert ( ddo_dict["id"] == "did:op:d32696f71f3318c92bcf325e2e51e6e8299c0eb6d362ddcfa77d2a3e0c1237b5" ) did = ddo_dict["id"] assert ddo_dict["version"] == "4.1.0" assert ddo_dict["chainId"] == 8996 chain_id = ddo_dict["chainId"] assert ddo_dict["metadata"] == { "created": "2020-11-15T12:27:48Z", "updated": "2021-05-17T21:58:02Z", "description": "Sample description", "name": "Sample asset", "type": "dataset", "author": "OPF", "license": "https://market.oceanprotocol.com/terms", } metadata = ddo_dict["metadata"] assert isinstance(ddo_dict["services"], list) assert ddo_dict["services"] == [ { "id": "1", "type": "access", "files": "0x0000", "name": "Download service", "description": "Download service", "datatokenAddress": "0x123", "serviceEndpoint": "http://172.15.0.4:8030", "timeout": 0, } ] services = [ Service.from_dict(value) for value in ddo_dict["services"] if isinstance(value, dict) ] assert ddo_dict["credentials"] == { "allow": [{"type": "address", "values": ["0x123", "0x456"]}], "deny": [{"type": "address", "values": ["0x2222", "0x333"]}], } credentials = ddo_dict["credentials"] assert ddo_dict["nft"] == { "address": "0xCc708430E6a174BD4639A979F578A2176A0FA3fA", "name": "Ocean Protocol Asset v4", "symbol": "OCEAN-A-v4", "owner": "0x0000000", "state": 0, "created": "2000-10-31T01:30:00", } nft = ddo_dict["nft"] assert ddo_dict["datatokens"] == [ { "address": "0x000000", "name": "Datatoken 1", "symbol": "DT-1", "serviceId": "1", } ] datatokens = ddo_dict["datatokens"] assert ddo_dict["event"] == { "tx": "0x8d127de58509be5dfac600792ad24cc9164921571d168bff2f123c7f1cb4b11c", "block": 12831214, "from": "0xAcca11dbeD4F863Bb3bC2336D3CE5BAC52aa1f83", "contract": "0x1a4b70d8c9DcA47cD6D0Fb3c52BB8634CA1C0Fdf", "datetime": "2000-10-31T01:30:00", } event = ddo_dict["event"] # Sample ddo assert ddo_dict["stats"] == {"consumes": 4} stats = ddo_dict["stats"] ddo = DDO( did=did, context=context, chain_id=chain_id, metadata=metadata, services=services, credentials=credentials, nft=nft, nft_address="0xCc708430E6a174BD4639A979F578A2176A0FA3fA", datatokens=datatokens, event=event, stats=stats, ) ddo_dict_v2 = ddo.as_dictionary() ddo_v2 = DDO.from_dict(ddo_dict_v2) assert ddo_v2.as_dictionary() == ddo_dict @pytest.mark.unit def test_add_service(): """Tests adding a compute service.""" ddo_dict = get_sample_ddo() ddo = DDO.from_dict(ddo_dict) compute_values = { "namespace": "ocean-compute", "cpus": 2, "gpus": 4, "gpuType": "NVIDIA Tesla V100 GPU", "memory": "128M", "volumeSize": "2G", "allowRawAlgorithm": False, "allowNetworkAccess": True, "publisherTrustedAlgorithmPublishers": ["0x234", "0x235"], "publisherTrustedAlgorithms": [ { "did": "did:op:123", "filesChecksum": "100", "containerSectionChecksum": "200", }, { "did": "did:op:124", "filesChecksum": "110", "containerSectionChecksum": "210", }, ], } ddo.create_compute_service( service_id="2", service_endpoint="http://172.15.0.4:8030", datatoken_address="0x124", files="0x0001", compute_values=compute_values, ) assert len(ddo.as_dictionary()["services"]) > 1 expected_access_service = get_key_from_v4_sample_ddo( key="services", file_name="ddo_v4_with_compute_service.json" )[0] assert ddo.as_dictionary()["services"][0] == expected_access_service expected_compute_service = get_key_from_v4_sample_ddo( key="services", file_name="ddo_v4_with_compute_service.json" )[1] assert ddo.as_dictionary()["services"][1]["id"] == expected_compute_service["id"] assert ( ddo.as_dictionary()["services"][1]["name"] == expected_compute_service["name"] ) assert ( ddo.as_dictionary()["services"][1]["description"] == expected_compute_service["description"] ) assert ( ddo.as_dictionary()["services"][1]["serviceEndpoint"] == expected_compute_service["serviceEndpoint"] ) assert ( ddo.as_dictionary()["services"][1]["datatokenAddress"] == expected_compute_service["datatokenAddress"] ) assert ( ddo.as_dictionary()["services"][1]["files"] == expected_compute_service["files"] ) assert ( ddo.as_dictionary()["services"][1]["timeout"] == expected_compute_service["timeout"] ) assert ( ddo.as_dictionary()["services"][1]["compute"] == expected_compute_service["compute"] ) @pytest.mark.unit def test_get_service_by_id(): """Tests retrieving services from the V4 DDO.""" ddo_dict = get_sample_ddo_with_compute_service() ddo = DDO.from_dict(ddo_dict) expected_access_service = get_key_from_v4_sample_ddo( key="services", file_name="ddo_v4_with_compute_service.json" )[0] assert ddo.get_service_by_id("1").as_dictionary() == expected_access_service expected_compute_service = get_key_from_v4_sample_ddo( key="services", file_name="ddo_v4_with_compute_service.json" )[1] assert ddo.get_service_by_id("2").as_dictionary() == expected_compute_service @pytest.mark.unit def test_credentials(): ddo_dict = get_sample_ddo_with_compute_service() ddo = DDO.from_dict(ddo_dict) assert ddo.requires_address_credential assert ddo.allowed_addresses == ["0x123", "0x456"] assert ddo.denied_addresses == ["0x2222", "0x333"] ddo.add_address_to_allow_list("0xaAA") assert "0xaaa" in ddo.allowed_addresses ddo.remove_address_from_allow_list("0xaAA") assert "0xaaa" not in ddo.allowed_addresses ddo.remove_address_from_allow_list("0xaAA") ddo.add_address_to_deny_list("0xaAA") assert "0xaaa" in ddo.denied_addresses ddo.remove_address_from_deny_list("0xaAA") assert "0xaaa" not in ddo.denied_addresses assert ddo.validate_access({"type": "address", "value": "0x123"}) == 0 # not allowed assert ddo.validate_access({"type": "address", "value": "0x444"}) == 3 ddo_dict = get_sample_ddo_with_compute_service() del ddo_dict["credentials"]["allow"] ddo = DDO.from_dict(ddo_dict) assert ddo.validate_access({"type": "address", "value": "0x444"}) == 0 # specifically denied assert ddo.validate_access({"type": "address", "value": "0x333"}) == 4 ddo_dict = get_sample_ddo_with_compute_service() del ddo_dict["credentials"]["allow"][0]["values"] ddo = DDO.from_dict(ddo_dict) with pytest.raises( MalformedCredential, match="No values key in the address credential" ): ddo.get_addresses_of_class("allow") ddo_dict = get_sample_ddo_with_compute_service() del ddo_dict["credentials"] ddo = DDO.from_dict(ddo_dict) ddo.remove_address_from_allow_list("0xAA") ddo.add_address_to_allow_list("0xAA") ddo_dict = get_sample_ddo_with_compute_service() ddo_dict["credentials"]["allow"] = [] ddo = DDO.from_dict(ddo_dict) ddo.remove_address_from_allow_list("0xAA") ddo.add_address_to_allow_list("0xAA") @pytest.mark.unit def test_credential_simplification(): assert simplify_credential_to_address(None) is None with pytest.raises(MalformedCredential, match="Received empty address."): simplify_credential_to_address({"malformed": "no value"}) assert ( simplify_credential_to_address({"type": "address", "value": "0x11"}) == "0x11" ) @pytest.mark.unit def test_is_disabled(): ddo_dict = get_sample_ddo() for state in range(6): ddo_dict["nft"]["state"] = state ddo = DDO.from_dict(ddo_dict) # adhere to https://docs.oceanprotocol.com/core-concepts/did-ddo#state if state in [0, 5]: assert not ddo.is_disabled else: assert ddo.is_disabled ================================================ FILE: ocean_lib/data_provider/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/data_provider/base.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Provider module.""" import logging import os import re from json import JSONDecodeError from math import ceil from typing import Dict, List, Optional, Tuple, Union from unittest.mock import Mock import requests from enforce_typing import enforce_types from requests.exceptions import InvalidURL from requests.models import PreparedRequest, Response from requests.sessions import Session from ocean_lib.exceptions import DataProviderException from ocean_lib.http_requests.requests_session import get_requests_session from ocean_lib.web3_internal.clef import ClefAccount from ocean_lib.web3_internal.utils import sign_with_clef, sign_with_key logger = logging.getLogger(__name__) class DataServiceProviderBase: """DataServiceProviderBase class.""" _http_client = get_requests_session() provider_info = None @staticmethod @enforce_types def get_http_client() -> Session: """Get the http client.""" return DataServiceProviderBase._http_client @staticmethod @enforce_types def set_http_client(http_client: Session) -> None: """Set the http client to something other than the default `requests`.""" DataServiceProviderBase._http_client = http_client @staticmethod @enforce_types def sign_message(wallet, msg: str, provider_uri: str) -> Tuple[str, str]: method, nonce_endpoint = DataServiceProviderBase.build_endpoint( "nonce", provider_uri ) nonce_response = DataServiceProviderBase._http_method( method, url=nonce_endpoint, params={"userAddress": wallet.address} ) if ( not nonce_response or not hasattr(nonce_response, "status_code") or nonce_response.status_code != 200 or "nonce" not in nonce_response.json() ): current_nonce = 0 else: current_nonce = int(ceil(float(nonce_response.json()["nonce"]))) nonce = current_nonce + 1 print(f"signing message with nonce {nonce}: {msg}, account={wallet.address}") if isinstance(wallet, ClefAccount): return nonce, str(sign_with_clef(f"{msg}{nonce}", wallet)) return nonce, str(sign_with_key(f"{msg}{nonce}", wallet._private_key.hex())) @staticmethod @enforce_types def get_url(config_dict: dict) -> str: """ Return the DataProvider component url. :return: Url, str """ return _remove_slash(config_dict.get("PROVIDER_URL")) @staticmethod @enforce_types def get_service_endpoints(provider_uri: str) -> Dict[str, List[str]]: """ Return the service endpoints from the provider URL. """ provider_info = DataServiceProviderBase._http_method( "get", url=provider_uri ).json() return provider_info["serviceEndpoints"] @staticmethod @enforce_types def get_c2d_environments(provider_uri: str, chain_id: int) -> Optional[str]: """ Return the provider address """ try: method, envs_endpoint = DataServiceProviderBase.build_endpoint( "computeEnvironments", provider_uri, {"chainId": chain_id} ) environments = DataServiceProviderBase._http_method( method, url=envs_endpoint ).json() if str(chain_id) not in environments: logger.warning( "You might be using an older provider. ocean.py can not verify the chain id." ) return environments return environments[str(chain_id)] except (requests.exceptions.RequestException, KeyError): pass return [] @staticmethod @enforce_types def get_provider_address(provider_uri: str, chain_id: int) -> Optional[str]: """ Return the provider address """ try: provider_info = DataServiceProviderBase._http_method( "get", provider_uri ).json() if "providerAddress" in provider_info: logger.warning( "You might be using an older provider. ocean.py can not verify the chain id." ) return provider_info["providerAddress"] return provider_info["providerAddresses"][str(chain_id)] except requests.exceptions.RequestException: pass return None @staticmethod @enforce_types def get_root_uri(service_endpoint: str) -> str: provider_uri = service_endpoint if "/api" in provider_uri: i = provider_uri.find("/api") provider_uri = provider_uri[:i] parts = provider_uri.split("/") if len(parts) < 2: raise InvalidURL(f"InvalidURL {service_endpoint}.") if parts[-2] == "services": provider_uri = "/".join(parts[:-2]) result = _remove_slash(provider_uri) if not result: raise InvalidURL(f"InvalidURL {service_endpoint}.") try: root_result = "/".join(parts[0:3]) response = requests.get(root_result).json() except (requests.exceptions.RequestException, JSONDecodeError): raise InvalidURL(f"InvalidURL {service_endpoint}.") if "providerAddresses" not in response: if "providerAddress" in response: logger.warning( "You might be using an older provider. ocean.py can not verify the chain id." ) else: raise InvalidURL( f"Invalid Provider URL {service_endpoint}, no providerAddresses." ) return result @staticmethod @enforce_types def is_valid_provider(provider_uri: str) -> bool: try: DataServiceProviderBase.get_root_uri(provider_uri) except InvalidURL: return False return True @staticmethod @enforce_types def build_endpoint( service_name: str, provider_uri: str, params: Optional[dict] = None ) -> Tuple[str, str]: provider_uri = DataServiceProviderBase.get_root_uri(provider_uri) service_endpoints = DataServiceProviderBase.get_service_endpoints(provider_uri) method, url = service_endpoints[service_name] url = urljoin(provider_uri, url) if params: req = PreparedRequest() req.prepare_url(url, params) url = req.url return method, url @staticmethod @enforce_types def write_file( response: Response, destination_folder: Union[str, bytes, os.PathLike], index: int, ) -> None: """ Write the response content in a file in the destination folder. :param response: Response :param destination_folder: Destination folder, string :param index: file index :return: None """ if response.status_code != 200: logger.warning(f"consume failed: {response.reason}") return with open(os.path.join(destination_folder, f"file{index}"), "wb") as f: for chunk in response.iter_content(chunk_size=4096): f.write(chunk) logger.info(f"Saved downloaded file in {f.name}") @staticmethod @enforce_types def _validate_content_disposition(header: str) -> bool: pattern = re.compile(r"\\|\.\.|/") return not bool(pattern.findall(header)) @staticmethod @enforce_types def _get_file_name(response: Response) -> Optional[str]: try: if not DataServiceProviderBase._validate_content_disposition( response.headers.get("content-disposition") ): logger.error( "Invalid content disposition format. It was not possible to get the file name." ) return None return re.match( r"attachment;filename=(.+)", response.headers.get("content-disposition"), )[1] except Exception as e: logger.warning(f"It was not possible to get the file name. {e}") return None @staticmethod @enforce_types def _http_method(method: str, *args, **kwargs) -> Optional[Union[Mock, Response]]: try: return getattr(DataServiceProviderBase._http_client, method.lower())( *args, **kwargs ) except Exception: logger.error( f"Error invoking http method {method}: args={str(args)}, kwargs={str(kwargs)}" ) raise @staticmethod @enforce_types def check_response( response, endpoint_name: str, endpoint: str, payload: Union[Dict, bytes], success_codes: Optional[List] = None, exception_type=DataProviderException, ): if not response or not hasattr(response, "status_code"): if isinstance(response, Response) and response.status_code == 400: error = response.json().get( "error", response.json().get("errors", "unknown error") ) raise DataProviderException(f"{endpoint_name} failed: {error}") response_content = getattr(response, "content", "") raise DataProviderException( f"Failed to get a response for request: {endpoint_name}={endpoint}, payload={payload}, response is {response_content}" ) if not success_codes: success_codes = [200] if response.status_code not in success_codes: msg = ( f"request failed at the {endpoint_name}" f"{endpoint}, reason {response.text}, status {response.status_code}" ) logger.error(msg) raise exception_type(msg) return None @enforce_types def urljoin(*args) -> str: trailing_slash = "/" if args[-1].endswith("/") else "" return "/".join(map(lambda x: str(x).strip("/"), args)) + trailing_slash def _remove_slash(path: str) -> str: path = path[:-1] if path.endswith("/") else path path = path[1:] if path.startswith("/") else path return path ================================================ FILE: ocean_lib/data_provider/data_encryptor.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Provider module.""" import json import logging from typing import Union from enforce_typing import enforce_types from requests.models import Response from ocean_lib.data_provider.base import DataServiceProviderBase from ocean_lib.exceptions import OceanEncryptAssetUrlsError logger = logging.getLogger(__name__) class DataEncryptor(DataServiceProviderBase): """DataEncryptor class.""" @staticmethod @enforce_types def encrypt( objects_to_encrypt: Union[list, str, bytes, dict], provider_uri: str, chain_id: int, ) -> Response: if isinstance(objects_to_encrypt, dict): data = json.dumps(objects_to_encrypt, separators=(",", ":")) payload = data.encode("utf-8") elif isinstance(objects_to_encrypt, str): payload = objects_to_encrypt.encode("utf-8") else: payload = objects_to_encrypt method, encrypt_endpoint = DataServiceProviderBase.build_endpoint( "encrypt", provider_uri, {"chainId": chain_id} ) response = DataServiceProviderBase._http_method( method, encrypt_endpoint, data=payload, headers={"Content-type": "application/octet-stream"}, ) DataServiceProviderBase.check_response( response, "encryptEndpoint", encrypt_endpoint, payload, [201], OceanEncryptAssetUrlsError, ) logger.info( f"Asset urls encrypted successfully, encrypted urls str: {response.text}," f" encryptedEndpoint {encrypt_endpoint}" ) return response ================================================ FILE: ocean_lib/data_provider/data_service_provider.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Provider module.""" import json import logging from json import JSONDecodeError from pathlib import Path from typing import Any, Dict, List, Optional, Union from enforce_typing import enforce_types from requests.models import PreparedRequest, Response from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.data_provider.base import DataServiceProviderBase from ocean_lib.data_provider.fileinfo_provider import FileInfoProvider from ocean_lib.http_requests.requests_session import get_requests_session from ocean_lib.models.compute_input import ComputeInput from ocean_lib.structures.algorithm_metadata import AlgorithmMetadata logger = logging.getLogger(__name__) class DataServiceProvider(DataServiceProviderBase): """DataServiceProvider class. The main functions available are: - consume_service - run_compute_service (not implemented yet) """ _http_client = get_requests_session() provider_info = None @staticmethod @enforce_types def initialize( did: str, service: Any, # Can not add Service typing due to enforce_type errors. consumer_address: str, userdata: Optional[Dict] = None, ) -> Response: method, initialize_endpoint = DataServiceProvider.build_endpoint( "initialize", service.service_endpoint ) payload = { "documentId": did, "serviceId": service.id, "consumerAddress": consumer_address, } if userdata is not None: userdata = json.dumps(userdata) payload["userdata"] = userdata response = DataServiceProvider._http_method( method, url=initialize_endpoint, params=payload ) DataServiceProviderBase.check_response( response, "initializeEndpoint", initialize_endpoint, payload ) logger.info( f"Service initialized successfully" f" initializeEndpoint {initialize_endpoint}" ) return response @staticmethod @enforce_types def initialize_compute( datasets: List[Dict[str, Any]], algorithm_data: Dict[str, Any], service_endpoint, consumer_address: str, compute_environment: str, valid_until: int, ) -> Response: """This function initializes compute services. To determine the Provider instance that will be called, we rely on the first dataset. The first dataset is also required to have a compute service. """ ( method, initialize_compute_endpoint, ) = DataServiceProvider.build_endpoint("initializeCompute", service_endpoint) payload = { "datasets": datasets, "algorithm": algorithm_data, "compute": { "env": compute_environment, "validUntil": valid_until, }, "consumerAddress": consumer_address, } response = DataServiceProvider._http_method( method, initialize_compute_endpoint, data=json.dumps(payload), headers={"content-type": "application/json"}, ) DataServiceProviderBase.check_response( response, "initializeComputeEndpoint", initialize_compute_endpoint, payload ) logger.info( f"Service initialized successfully" f" initializeComputeEndpoint {initialize_compute_endpoint}" ) return response @staticmethod @enforce_types def download( did: str, service: Any, # Can not add Service typing due to enforce_type errors. tx_id: Union[str, bytes], consumer_wallet, destination_folder: Union[str, Path], index: Optional[int] = None, userdata: Optional[Dict] = None, ) -> None: service_endpoint = service.service_endpoint fileinfo_response = FileInfoProvider.fileinfo(did, service, userdata=userdata) files = fileinfo_response.json() indexes = range(len(files)) if index is not None: assert isinstance(index, int), logger.error("index has to be an integer.") assert index >= 0, logger.error("index has to be 0 or a positive integer.") assert index < len(files), logger.error( "index can not be bigger than the number of files" ) indexes = [index] method, download_endpoint = DataServiceProvider.build_endpoint( "download", service_endpoint ) payload = { "documentId": did, "serviceId": service.id, "consumerAddress": consumer_wallet.address, "transferTxId": tx_id, } if userdata: userdata = json.dumps(userdata) payload["userdata"] = userdata for i in indexes: payload["fileIndex"] = i payload["nonce"], payload["signature"] = DataServiceProvider.sign_message( consumer_wallet, did, provider_uri=service_endpoint ) response = DataServiceProvider._http_method( method, url=download_endpoint, params=payload, stream=True, timeout=3 ) DataServiceProviderBase.check_response( response, "downloadEndpoint", download_endpoint, payload ) DataServiceProvider.write_file(response, destination_folder, i) logger.info( f"DDO downloaded successfully" f" downloadEndpoint {download_endpoint}" ) @staticmethod # @enforce_types omitted due to subscripted generics error def start_compute_job( dataset_compute_service: Any, # Can not add Service typing due to enforce_type errors. consumer, dataset: ComputeInput, compute_environment: str, algorithm: Optional[ComputeInput] = None, algorithm_meta: Optional[AlgorithmMetadata] = None, algorithm_custom_data: Optional[str] = None, input_datasets: Optional[List[ComputeInput]] = None, ) -> Dict[str, Any]: """ Start a compute job. Either algorithm or algorithm_meta must be defined. :param dataset_compute_service: :param consumer: hex str the ethereum address of the consumer executing the compute job :param dataset: ComputeInput dataset with a compute service :param compute_environment: str compute environment id :param algorithm: ComputeInput algorithm witha download service. :param algorithm_meta: AlgorithmMetadata algorithm metadata :param algorithm_custom_data: dict customizable algo parameters (ie. no of iterations, etc) :param input_datasets: List[ComputeInput] additional input datasets :return job_info dict """ assert ( algorithm or algorithm_meta ), "either an algorithm did or an algorithm meta must be provided." assert ( hasattr(dataset_compute_service, "type") and dataset_compute_service.type == ServiceTypes.CLOUD_COMPUTE ), "invalid compute service" payload = DataServiceProvider._prepare_compute_payload( consumer=consumer, dataset=dataset, compute_environment=compute_environment, dataset_compute_service=dataset_compute_service, algorithm=algorithm, algorithm_meta=algorithm_meta, algorithm_custom_data=algorithm_custom_data, input_datasets=input_datasets, ) logger.info(f"invoke start compute endpoint with this url: {payload}") method, compute_endpoint = DataServiceProvider.build_endpoint( "computeStart", dataset_compute_service.service_endpoint ) response = DataServiceProvider._http_method( method, compute_endpoint, data=json.dumps(payload), headers={"content-type": "application/json"}, ) logger.debug( f"got DataProvider execute response: {response.content} with status-code {response.status_code} " ) DataServiceProviderBase.check_response( response, "computeStartEndpoint", compute_endpoint, payload, [200, 201] ) try: job_info = json.loads(response.content.decode("utf-8")) return job_info[0] if isinstance(job_info, list) else job_info except KeyError as err: logger.error(f"Failed to extract jobId from response: {err}") raise KeyError(f"Failed to extract jobId from response: {err}") except JSONDecodeError as err: logger.error(f"Failed to parse response json: {err}") raise @staticmethod @enforce_types def stop_compute_job( did: str, job_id: str, dataset_compute_service: Any, consumer, # Can not add Service typing due to enforce_type errors. ) -> Dict[str, Any]: """ :param did: hex str the DDO id :param job_id: str id of compute job that was returned from `start_compute_job` :param dataset_compute_service: :param consumer of the consumer's account :return: bool whether the job was stopped successfully """ _, compute_stop_endpoint = DataServiceProvider.build_endpoint( "computeStop", dataset_compute_service.service_endpoint ) return DataServiceProvider._send_compute_request( "put", did, job_id, compute_stop_endpoint, consumer ) @staticmethod @enforce_types def delete_compute_job( did: str, job_id: str, dataset_compute_service: Any, consumer, # Can not add Service typing due to enforce_type errors. ) -> Dict[str, str]: """ :param did: hex str the DDO id :param job_id: str id of compute job that was returned from `start_compute_job` :param dataset_compute_service: :param consumer of the consumer's account :return: bool whether the job was deleted successfully """ method, compute_delete_endpoint = DataServiceProvider.build_endpoint( "computeDelete", dataset_compute_service.service_endpoint ) return DataServiceProvider._send_compute_request( method, did, job_id, compute_delete_endpoint, consumer ) @staticmethod @enforce_types def compute_job_status( did: str, job_id: str, dataset_compute_service: Any, consumer, # Can not add Service typing due to enforce_type errors. ) -> Dict[str, Any]: """ :param did: hex str the DDO id :param job_id: str id of compute job that was returned from `start_compute_job` :param dataset_compute_service: :param consumer of the consumer's account :return: dict of job_id to status info. When job_id is not provided, this will return status for each job_id that exist for the did """ method, compute_status_endpoint = DataServiceProvider.build_endpoint( "computeStatus", dataset_compute_service.service_endpoint ) return DataServiceProvider._send_compute_request( method, did, job_id, compute_status_endpoint, consumer ) @staticmethod @enforce_types def compute_job_result( job_id: str, index: int, dataset_compute_service: Any, consumer ) -> bytes: """ :param job_id: str id of compute job that was returned from `start_compute_job` :param index: int compute result index :param dataset_compute_service: :param consumer of the consumer's account :return: dict of job_id to result urls. """ nonce, signature = DataServiceProvider.sign_message( consumer, f"{consumer.address}{job_id}{str(index)}", provider_uri=dataset_compute_service.service_endpoint, ) req = PreparedRequest() params = { "signature": signature, "nonce": nonce, "jobId": job_id, "index": index, "consumerAddress": consumer.address, } (method, compute_job_result_endpoint) = DataServiceProvider.build_endpoint( "computeResult", dataset_compute_service.service_endpoint ) req.prepare_url(compute_job_result_endpoint, params) compute_job_result_file_url = req.url logger.info( f"invoke the computeResult endpoint with this url: {compute_job_result_file_url}" ) response = DataServiceProvider._http_method(method, compute_job_result_file_url) DataServiceProviderBase.check_response( response, "jobResultEndpoint", compute_job_result_endpoint, params ) return response.content @staticmethod @enforce_types def compute_job_result_logs( ddo: Any, job_id: str, dataset_compute_service: Any, consumer, log_type="output", ) -> List[Dict[str, Any]]: """ :param job_id: str id of compute job that was returned from `start_compute_job` :param dataset_compute_service: :param consumer of the consumer's account :return: dict of job_id to result urls. """ status = DataServiceProvider.compute_job_status( ddo.did, job_id, dataset_compute_service, consumer ) function_result = [] for i in range(len(status["results"])): result = None result_type = status["results"][i]["type"] result = DataServiceProvider.compute_job_result( job_id, i, dataset_compute_service, consumer ) # Extract algorithm output if result_type == log_type: function_result.append(result) return function_result @staticmethod @enforce_types def _send_compute_request( http_method: str, did: str, job_id: str, service_endpoint: str, consumer ) -> Dict[str, Any]: nonce, signature = DataServiceProvider.sign_message( consumer, f"{consumer.address}{job_id}{did}", provider_uri=service_endpoint, ) req = PreparedRequest() payload = { "consumerAddress": consumer.address, "documentId": did, "jobId": job_id, "nonce": nonce, "signature": signature, } req.prepare_url(service_endpoint, payload) logger.info(f"invoke compute endpoint with this url: {req.url}") response = DataServiceProvider._http_method(http_method, req.url) logger.debug( f"got provider execute response: {response.content} with status-code {response.status_code} " ) DataServiceProviderBase.check_response( response, "compute Endpoint", req.url, payload ) resp_content = json.loads(response.content.decode("utf-8")) if isinstance(resp_content, list): return resp_content[0] return resp_content @staticmethod # @enforce_types omitted due to subscripted generics error def _prepare_compute_payload( consumer, dataset: ComputeInput, dataset_compute_service: Any, # Can not add Service typing due to enforce_type errors. compute_environment: str, algorithm: Optional[ComputeInput] = None, algorithm_meta: Optional[AlgorithmMetadata] = None, algorithm_custom_data: Optional[str] = None, input_datasets: Optional[List[ComputeInput]] = None, ) -> Dict[str, Any]: assert ( algorithm or algorithm_meta ), "either an algorithm did or an algorithm meta must be provided." if algorithm_meta: assert isinstance(algorithm_meta, AlgorithmMetadata), ( f"expecting a AlgorithmMetadata type " f"for `algorithm_meta`, got {type(algorithm_meta)}" ) input_datasets = input_datasets if input_datasets else [] _input_datasets = [] for _input in input_datasets: for req_key in ["did", "transfer_tx_id", "service_id"]: assert getattr( _input, req_key ), f"The received dataset does not have a {req_key}." # TODO: is the nonce correct here? # Should it be the one from the compute service or a dataset? nonce, signature = DataServiceProvider.sign_message( consumer, f"{consumer.address}{dataset.did}", provider_uri=dataset_compute_service.service_endpoint, ) payload = { "dataset": { "documentId": dataset.did, "serviceId": dataset.service_id, "transferTxId": dataset.transfer_tx_id, }, "environment": compute_environment, "algorithm": {}, "signature": signature, "nonce": nonce, "consumerAddress": consumer.address, "additionalInputs": _input_datasets or [], } if dataset.userdata: payload["dataset"]["userdata"] = dataset.userdata if algorithm: payload.update( { "algorithm": { "documentId": algorithm.did, "serviceId": algorithm.service_id, "transferTxId": algorithm.transfer_tx_id, } } ) if algorithm.userdata: payload["algorithm"]["userdata"] = algorithm.userdata if algorithm_custom_data: payload["algorithm"]["algocustomdata"] = algorithm_custom_data else: payload["algorithm"] = algorithm_meta.as_dictionary() return payload @staticmethod @enforce_types def check_single_file_info(url_object: dict, provider_uri: str) -> bool: method, endpoint = DataServiceProvider.build_endpoint("fileinfo", provider_uri) response = DataServiceProvider._http_method(method, endpoint, json=url_object) if response.status_code != 200: return False return any([file_info["valid"] for file_info in response.json()]) @staticmethod @enforce_types def check_asset_file_info( did: str, service_id: str, provider_uri: str, userdata: Optional[dict] = None ) -> bool: if not did: return False method, endpoint = DataServiceProvider.build_endpoint("fileinfo", provider_uri) data = {"did": did, "serviceId": service_id} if userdata is not None: data["userdata"] = userdata response = DataServiceProvider._http_method(method, endpoint, json=data) if not response or response.status_code != 200: return False return any([file_info["valid"] for file_info in response.json()]) ================================================ FILE: ocean_lib/data_provider/fileinfo_provider.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Provider module.""" import logging from typing import Any, Optional from enforce_typing import enforce_types from requests.models import Response from ocean_lib.data_provider.base import DataServiceProviderBase from ocean_lib.http_requests.requests_session import get_requests_session logger = logging.getLogger(__name__) class FileInfoProvider(DataServiceProviderBase): """DataServiceProvider class. The main functions available are: - consume_service - run_compute_service (not implemented yet) """ _http_client = get_requests_session() provider_info = None @staticmethod @enforce_types def fileinfo( did: str, service: Any, with_checksum: bool = False, userdata: Optional[dict] = None, ) -> Response: # Can not add Service typing due to enforce_type errors. method, fileinfo_endpoint = DataServiceProviderBase.build_endpoint( "fileinfo", service.service_endpoint ) payload = {"did": did, "serviceId": service.id} if userdata is not None: payload["userdata"] = userdata if with_checksum: payload["checksum"] = 1 response = DataServiceProviderBase._http_method( method, fileinfo_endpoint, json=payload ) DataServiceProviderBase.check_response( response, "fileInfoEndpoint", fileinfo_endpoint, payload ) logger.info( f"Retrieved asset files successfully" f" FileInfoEndpoint {fileinfo_endpoint} from did {did} with service id {service.id}" ) return response ================================================ FILE: ocean_lib/data_provider/test/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/data_provider/test/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: ocean_lib/data_provider/test/test_base.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from requests.models import Response from ocean_lib.data_provider.base import DataServiceProviderBase def test_validate_content_disposition(): header = "./../../my/relative/path" res = DataServiceProviderBase._validate_content_disposition(header) assert not res header = "somehtml.html" res = DataServiceProviderBase._validate_content_disposition(header) assert res def test_get_file_name(caplog): response = Response() response.headers["content-disposition"] = "./../../my/relative/path" file_name = DataServiceProviderBase._get_file_name(response) assert not file_name assert ( "Invalid content disposition format. It was not possible to get the file name." in caplog.text ) response.headers["content-disposition"] = "attachment;filename=somehtml.html" file_name = DataServiceProviderBase._get_file_name(response) assert file_name == "somehtml.html" ================================================ FILE: ocean_lib/data_provider/test/test_data_service_provider.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json from datetime import datetime, timedelta, timezone from unittest.mock import Mock import ecies import pytest from requests.exceptions import InvalidURL from requests.models import Response from web3.main import Web3 from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.base import DataServiceProviderBase, urljoin from ocean_lib.data_provider.data_encryptor import DataEncryptor from ocean_lib.data_provider.data_service_provider import DataServiceProvider as DataSP from ocean_lib.data_provider.fileinfo_provider import FileInfoProvider from ocean_lib.example_config import DEFAULT_PROVIDER_URL from ocean_lib.exceptions import DataProviderException, OceanEncryptAssetUrlsError from ocean_lib.http_requests.requests_session import get_requests_session from ocean_lib.models.compute_input import ComputeInput from ocean_lib.services.service import Service from tests.resources.ddo_helpers import ( get_first_service_by_type, get_registered_asset_with_access_service, ) from tests.resources.mocks.http_client_mock import ( TEST_SERVICE_ENDPOINTS, HttpClientEmptyMock, HttpClientEvilMock, HttpClientNiceMock, ) @pytest.fixture def with_evil_client(): http_client = HttpClientEvilMock() DataSP.set_http_client(http_client) yield DataSP.set_http_client(get_requests_session()) @pytest.fixture def with_nice_client(): http_client = HttpClientNiceMock() DataSP.set_http_client(http_client) yield DataSP.set_http_client(get_requests_session()) @pytest.fixture def with_empty_client(): http_client = HttpClientEmptyMock() DataSP.set_http_client(http_client) yield DataSP.set_http_client(get_requests_session()) def test_set_http_client(with_nice_client): """Tests that a custom http client can be set on the DataServiceProvider.""" assert isinstance(DataSP.get_http_client(), HttpClientNiceMock) @pytest.mark.unit def test_initialize_fails(): """Tests failures of initialize endpoint.""" mock_service = Service( service_id="some_service_id", service_type="some_service_type", service_endpoint="http://mock/", datatoken="some_dt", files="some_files", timeout=0, ) with pytest.raises( InvalidURL, match=f"InvalidURL {mock_service.service_endpoint}." ): DataSP.initialize( "some_did", mock_service, "some_consumer_address", userdata={"test_dict_key": "test_dict_value"}, ) mock_service.service_endpoint = DEFAULT_PROVIDER_URL with pytest.raises( DataProviderException, match="initializeEndpoint failed", ): DataSP.initialize( "some_did", mock_service, "some_consumer_address", userdata={"test_dict_key": "test_dict_value"}, ) @pytest.mark.unit def test_start_compute_job_fails_empty(consumer_wallet): """Tests failures of compute job from endpoint with empty response.""" mock_service = Service( service_id="some_service_id", service_type="compute", service_endpoint="http://mock/", datatoken="some_dt", files="some_files", timeout=0, compute_values=dict(), ) mock_ddo = DDO() with pytest.raises( InvalidURL, match=f"InvalidURL {mock_service.service_endpoint}." ): DataSP.start_compute_job( dataset_compute_service=mock_service, consumer=consumer_wallet, dataset=ComputeInput(mock_ddo, mock_service, "tx_id"), compute_environment="some_env", algorithm=ComputeInput(DDO(), mock_service, "tx_id"), ) mock_service.service_endpoint = DEFAULT_PROVIDER_URL with pytest.raises( DataProviderException, match="The dataset.documentId field is required." ): DataSP.start_compute_job( dataset_compute_service=mock_service, consumer=consumer_wallet, dataset=ComputeInput(DDO(), mock_service, "tx"), compute_environment="some_env", algorithm=ComputeInput(DDO(), mock_service, "tx"), ) @pytest.mark.unit def test_send_compute_request_failure(with_evil_client, provider_wallet): """Tests failure of compute request from endpoint with non-200 response.""" with pytest.raises(Exception): DataSP._send_compute_request( "post", "some_did", "some_job_id", "http://mock/", provider_wallet ) @pytest.mark.unit def test_compute_job_result_fails(provider_wallet): """Tests failure of compute job starting.""" mock_service = Service( service_id="some_service_id", service_type="some_service_type", service_endpoint="http://mock", datatoken="some_dt", files="some_files", timeout=0, compute_values=dict(), ) with pytest.raises( InvalidURL, match=f"InvalidURL {mock_service.service_endpoint}." ): DataSP.compute_job_result("some_job_id", 0, mock_service, provider_wallet) @pytest.mark.unit def test_delete_job_result(provider_wallet): """Tests a failure & a success of compute job deletion.""" mock_service = Service( service_id="some_service_id", service_type="some_service_type", service_endpoint="http://mock/", datatoken="some_dt", files="some_files", timeout=0, compute_values=dict(), ) # Failure of compute job deletion. with pytest.raises( InvalidURL, match=f"InvalidURL {mock_service.service_endpoint}." ): DataSP.delete_compute_job( "some_did", "some_job_id", mock_service, provider_wallet ) # Success of compute job deletion. mock_service.service_endpoint = DEFAULT_PROVIDER_URL DataSP.delete_compute_job("some_did", "some_job_id", mock_service, provider_wallet) @pytest.mark.integration def test_encrypt(provider_wallet, file1, file2): """Tests successful encrypt job.""" key = provider_wallet._private_key.hex() # Encrypt file objects res = {"files": [file1.to_dict(), file2.to_dict()]} result = DataEncryptor.encrypt(res, DEFAULT_PROVIDER_URL, 8996) encrypted_files = result.content.decode("utf-8") assert result.status_code == 201 assert result.headers["Content-type"] == "text/plain" assert encrypted_files.startswith("0x") if isinstance(encrypted_files, str): encrypted_files = Web3.to_bytes(hexstr=encrypted_files) decrypted_document = ecies.decrypt(key, encrypted_files) decrypted_document_string = decrypted_document.decode("utf-8") assert decrypted_document_string == json.dumps(res, separators=(",", ":")) # Encrypt a simple string test_string = "hello_world" encrypt_result = DataEncryptor.encrypt(test_string, DEFAULT_PROVIDER_URL, 8996) encrypted_document = encrypt_result.content.decode("utf-8") assert result.status_code == 201 assert result.headers["Content-type"] == "text/plain" assert result.content.decode("utf-8").startswith("0x") if isinstance(encrypted_document, str): encrypted_document = Web3.to_bytes(hexstr=encrypted_document) decrypted_document = ecies.decrypt(key, encrypted_document) decrypted_document_string = decrypted_document.decode("utf-8") assert decrypted_document_string == test_string @pytest.mark.integration def test_fileinfo_and_initialize(publisher_wallet, publisher_ocean, ocean_address): """Test both fileinfo and initialize to avoid publishing 2 assets.""" _, _, ddo = get_registered_asset_with_access_service( publisher_ocean, publisher_wallet, more_files=True ) access_service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS) fileinfo_result = FileInfoProvider.fileinfo( ddo.did, access_service, with_checksum=True ) assert fileinfo_result.status_code == 200 files_info = fileinfo_result.json() assert len(files_info) == 2 for file_index, file in enumerate(files_info): assert file["index"] == file_index assert file["checksum"] assert file["valid"] is True matches = "text/plain" if file_index == 0 else "text/xml" assert file["contentType"] == matches initialize_result = DataSP.initialize( did=ddo.did, service=access_service, consumer_address=publisher_wallet.address, ) assert initialize_result assert initialize_result.status_code == 200 response_json = initialize_result.json() assert response_json["providerFee"]["providerFeeAmount"] == "0" assert response_json["providerFee"]["providerFeeToken"] == ocean_address @pytest.mark.unit def test_invalid_file_name(): """Tests that no filename is returned if attachment headers are found.""" response = Mock(spec=Response) response.headers = {"no_good": "headers at all"} assert DataSP._get_file_name(response) is None @pytest.mark.integration def test_expose_endpoints(): """Tests that the DataServiceProvider exposes all service endpoints.""" service_endpoints = TEST_SERVICE_ENDPOINTS provider_uri = DEFAULT_PROVIDER_URL valid_endpoints = DataSP.get_service_endpoints(provider_uri) assert len(valid_endpoints) == len(service_endpoints) assert [ valid_endpoints[key] for key in set(service_endpoints) & set(valid_endpoints) ] @pytest.mark.integration def test_c2d_environments(): """Tests that the test ocean-compute env exists on the DataServiceProvider.""" provider_uri = DEFAULT_PROVIDER_URL c2d_envs = DataSP.get_c2d_environments(provider_uri, 8996) c2d_env_ids = [elem["id"] for elem in c2d_envs] assert "ocean-compute" in c2d_env_ids, "ocean-compute env not found." @pytest.mark.integration def test_provider_address(): """Tests that a provider address exists on the DataServiceProvider.""" provider_uri = DEFAULT_PROVIDER_URL provider_address = DataSP.get_provider_address(provider_uri, 8996) assert provider_address, "Failed to get provider address." assert DataSP.get_provider_address("not a url", 8996) is None @pytest.mark.integration def test_get_root_uri(): """Tests extraction of base URLs from various inputs.""" uri = "https://v4.provider.mainnet.oceanprotocol.com" assert DataSP.is_valid_provider(uri) assert DataSP.get_root_uri(uri) == uri assert DataSP.get_root_uri("http://localhost:8030") == "http://localhost:8030" assert ( DataSP.get_root_uri("http://localhost:8030/api/services/") == "http://localhost:8030" ) assert DataSP.get_root_uri("http://localhost:8030/api") == "http://localhost:8030" assert ( DataSP.get_root_uri("http://localhost:8030/services") == "http://localhost:8030/services" ) assert ( DataSP.get_root_uri("http://localhost:8030/services/download") == "http://localhost:8030" ) assert ( DataSP.get_root_uri("http://localhost:8030/api/services") == "http://localhost:8030" ) assert ( DataSP.get_root_uri("http://localhost:8030/api/services/") == "http://localhost:8030" ) assert not DataSP.is_valid_provider("thisIsNotAnURL") with pytest.raises(InvalidURL): DataSP.get_root_uri("thisIsNotAnURL") with pytest.raises(InvalidURL): # URL is of correct format but unreachable DataSP.get_root_uri("http://thisisaurl.but/itshouldnt") with pytest.raises(InvalidURL): # valid URL, but no provider address DataSP.get_root_uri("http://oceanprotocol.com") with pytest.raises(InvalidURL): DataSP.get_root_uri("//") @pytest.mark.integration def test_build_endpoint(): """Tests that service endpoints are correctly built from URL and service name.""" def get_service_endpoints(_provider_uri=None): _endpoints = TEST_SERVICE_ENDPOINTS.copy() _endpoints.update({"newEndpoint": ["GET", "/api/services/newthing"]}) return _endpoints original_func = DataServiceProviderBase.get_service_endpoints DataServiceProviderBase.get_service_endpoints = get_service_endpoints endpoints = get_service_endpoints() uri = "http://localhost:8030" method, endpnt = DataSP.build_endpoint("newEndpoint", provider_uri=uri) assert endpnt == urljoin(uri, endpoints["newEndpoint"][1]) uri = "http://localhost:8030/api/services/newthing" method, endpnt = DataSP.build_endpoint("download", provider_uri=uri) assert method == endpoints["download"][0] assert endpnt == urljoin(DataSP.get_root_uri(uri), endpoints["download"][1]) DataSP.get_service_endpoints = original_func @pytest.mark.integration def test_build_specific_endpoints(): """Tests that a specific list of agreed endpoints is supported on the DataServiceProvider.""" endpoints = TEST_SERVICE_ENDPOINTS def get_service_endpoints(_provider_uri=None): return TEST_SERVICE_ENDPOINTS.copy() original_func = DataSP.get_service_endpoints DataSP.get_service_endpoints = get_service_endpoints provider_uri = DEFAULT_PROVIDER_URL base_uri = DataSP.get_root_uri(DEFAULT_PROVIDER_URL) assert ( DataSP.build_endpoint("encrypt", provider_uri, {"chainId": 8996})[1] == urljoin(base_uri, endpoints["encrypt"][1]) + "?chainId=8996" ) for key in [ "fileinfo", "download", "initialize", "initializeCompute", "computeStatus", "computeStart", "computeStop", "computeDelete", ]: assert DataSP.build_endpoint(key, provider_uri)[1] == urljoin( base_uri, endpoints[key][1] ) DataSP.get_service_endpoints = original_func @pytest.mark.integration def test_check_single_file_info(): assert DataSP.check_single_file_info( {"url": "http://www.google.com", "type": "url"}, provider_uri="http://172.15.0.4:8030", ) assert not DataSP.check_single_file_info( {"url": "http://www.google.com"}, provider_uri="http://172.15.0.4:8030" ) assert not DataSP.check_single_file_info({}, provider_uri="http://172.15.0.4:8030") @pytest.mark.unit def test_encrypt_failure(): """Tests encrypt failures.""" http_client = HttpClientEvilMock() DataEncryptor.set_http_client(http_client) with pytest.raises(OceanEncryptAssetUrlsError): DataEncryptor.encrypt({}, DEFAULT_PROVIDER_URL, 8996) http_client = HttpClientEmptyMock() DataSP.set_http_client(http_client) with pytest.raises(DataProviderException): DataEncryptor.encrypt({}, DEFAULT_PROVIDER_URL, 8996) DataSP.set_http_client(get_requests_session()) @pytest.mark.unit def test_fileinfo_failure(): """Tests successful fileinfo failures.""" service = Mock(spec=Service) service.service_endpoint = "http://172.15.0.4:8030" service.id = "abc" http_client = HttpClientEvilMock() DataSP.set_http_client(http_client) with pytest.raises(DataProviderException): FileInfoProvider.fileinfo("0xabc", service) http_client = HttpClientEmptyMock() DataSP.set_http_client(http_client) with pytest.raises(DataProviderException): FileInfoProvider.fileinfo("0xabc", service) DataSP.set_http_client(get_requests_session()) @pytest.mark.unit def test_initialize_failure(): """Tests initialize failures.""" service = Mock(spec=Service) service.service_endpoint = "http://172.15.0.4:8030" service.id = "abc" http_client = HttpClientEvilMock() DataSP.set_http_client(http_client) with pytest.raises(DataProviderException): DataSP.initialize("0xabc", service, "0x") http_client = HttpClientEmptyMock() DataSP.set_http_client(http_client) with pytest.raises(DataProviderException): DataSP.initialize("0xabc", service, "0x") DataSP.set_http_client(get_requests_session()) @pytest.mark.unit def test_initialize_compute_failure(): """Tests initialize_compute failures.""" service = Mock(spec=Service) service.service_endpoint = "http://172.15.0.4:8030" service.id = "abc" ddo = Mock(spec=DDO) ddo.did = "0x0" compute_input = ComputeInput(ddo, service) http_client = HttpClientEvilMock() DataSP.set_http_client(http_client) valid_until = int((datetime.now(timezone.utc) + timedelta(days=1)).timestamp()) with pytest.raises( DataProviderException, match="request failed at the initializeComputeEndpoint" ): DataSP.initialize_compute( [compute_input.as_dictionary()], compute_input.as_dictionary(), service.service_endpoint, "0x0", "test", valid_until, ) http_client = HttpClientEmptyMock() DataSP.set_http_client(http_client) with pytest.raises(DataProviderException, match="Failed to get a response"): DataSP.initialize_compute( [compute_input.as_dictionary()], compute_input.as_dictionary(), service.service_endpoint, "0x0", "test", valid_until, ) DataSP.set_http_client(get_requests_session()) @pytest.mark.unit def test_job_result_failure(consumer_wallet): """Tests compute job result failures.""" service = Mock(spec=Service) service.service_endpoint = "http://172.15.0.4:8030" service.id = "abc" http_client = HttpClientEvilMock() DataSP.set_http_client(http_client) with pytest.raises(DataProviderException): DataSP.compute_job_result("0xabc", 0, service, consumer_wallet) DataSP.set_http_client(get_requests_session()) @pytest.mark.unit def test_check_asset_failure(): """Tests check_asset_file_info failures.""" assert DataSP.check_asset_file_info("", "", DEFAULT_PROVIDER_URL) is False http_client = HttpClientEvilMock() DataSP.set_http_client(http_client) assert DataSP.check_asset_file_info("test", "", DEFAULT_PROVIDER_URL) is False http_client = HttpClientEmptyMock() DataSP.set_http_client(http_client) assert DataSP.check_asset_file_info("test", "", DEFAULT_PROVIDER_URL) is False DataSP.set_http_client(get_requests_session()) ================================================ FILE: ocean_lib/example_config.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import copy import logging import os from pathlib import Path from typing import Optional import addresses from enforce_typing import enforce_types from web3 import Web3 from web3.exceptions import ExtraDataLengthError from ocean_lib.web3_internal.http_provider import get_web3_connection_provider logging.basicConfig(level=logging.INFO) DEFAULT_METADATA_CACHE_URI = "http://172.15.0.5:5000" METADATA_CACHE_URI = "https://v4.aquarius.oceanprotocol.com" DEFAULT_PROVIDER_URL = "http://172.15.0.4:8030" config_defaults = { "METADATA_CACHE_URI": "http://172.15.0.5:5000", "PROVIDER_URL": "http://172.15.0.4:8030", "DOWNLOADS_PATH": "consume-downloads", } PROVIDER_PER_NETWORK = { 1: "https://v4.provider.mainnet.oceanprotocol.com", 5: "https://v4.provider.goerli.oceanprotocol.com", 10: "https://v4.provider.oceanprotocol.com", 56: "https://v4.provider.bsc.oceanprotocol.com", 137: "https://v4.provider.polygon.oceanprotocol.com", 246: "https://v4.provider.energyweb.oceanprotocol.com", 1285: "https://v4.provider.moonriver.oceanprotocol.com", 1287: "https://v4.provider.moonbase.oceanprotocol.com", 80001: "https://v4.provider.mumbai.oceanprotocol.com", 58008: "https://v4.provider.oceanprotocol.com", 8996: DEFAULT_PROVIDER_URL, } NAME_PER_NETWORK = { 1: "mainnet", 5: "goerli", 10: "optimism", 56: "bsc", 137: "polygon", 246: "energyweb", 1285: "moonriver", 1287: "moonbase", 80001: "mumbai", 58008: "sepolia", 8996: "development", } def get_config_dict(network_url: Optional[str] = None) -> dict: """Return config dict containing default values for a given network. Chain ID is determined by querying the RPC specified by network_url. """ if not network_url: network_url = "http://localhost:8545" config_dict = copy.deepcopy(config_defaults) config_dict["web3_instance"] = get_web3(network_url) config_dict["CHAIN_ID"] = config_dict["web3_instance"].eth.chain_id chain_id = config_dict["CHAIN_ID"] if chain_id not in PROVIDER_PER_NETWORK: raise ValueError("The chain id for the specific RPC could not be fetched!") config_dict["PROVIDER_URL"] = PROVIDER_PER_NETWORK[chain_id] config_dict["NETWORK_NAME"] = NAME_PER_NETWORK[chain_id] if chain_id != 8996: config_dict["METADATA_CACHE_URI"] = METADATA_CACHE_URI if os.getenv("ADDRESS_FILE"): base_file = os.getenv("ADDRESS_FILE") address_file = os.path.expanduser(base_file) elif chain_id == 8996: # this is auto-created when barge is run base_file = "~/.ocean/ocean-contracts/artifacts/address.json" address_file = os.path.expanduser(base_file) else: # `contract_addresses` comes from "ocean-contracts" pypi library, # a JSON blob holding addresses of contract deployments, per network address_file = ( Path(os.path.join(addresses.__file__, "..", "address.json")) .expanduser() .resolve() ) assert os.path.exists(address_file), f"Could not find address_file={address_file}." config_dict["ADDRESS_FILE"] = address_file return config_dict @enforce_types def get_web3(network_url: str) -> Web3: """ Return a web3 instance connected via the given network_url. Adds POA middleware when connecting to the Rinkeby Testnet. A note about using the `rinkeby` testnet: Web3 py has an issue when making some requests to `rinkeby` - the issue is described here: https://github.com/ethereum/web3.py/issues/549 - and the fix is here: https://web3py.readthedocs.io/en/latest/middleware.html#geth-style-proof-of-authority """ provider = get_web3_connection_provider(network_url) web3 = Web3(provider) try: web3.eth.get_block("latest") except ExtraDataLengthError: from web3.middleware import geth_poa_middleware web3.middleware_onion.inject(geth_poa_middleware, layer=0) web3.strict_bytes_type_checking = False return web3 ================================================ FILE: ocean_lib/exceptions.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # class OceanEncryptAssetUrlsError(Exception): """Error invoking the encrypt endpoint.""" class InsufficientBalance(Exception): """The token balance is insufficient.""" class AquariusError(Exception): """Error invoking an Aquarius metadata service endpoint.""" class VerifyTxFailed(Exception): """Transaction verification failed.""" class TransactionFailed(Exception): """Transaction has failed.""" class DataProviderException(Exception): """Exception from Provider endpoints.""" ================================================ FILE: ocean_lib/http_requests/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/http_requests/requests_session.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from enforce_typing import enforce_types from requests.adapters import HTTPAdapter from requests.sessions import Session @enforce_types class TimeoutHTTPAdapter(HTTPAdapter): def __init__(self, *args, **kwargs): self.timeout = 30 if "timeout" in kwargs: self.timeout = kwargs["timeout"] del kwargs["timeout"] super().__init__(*args, **kwargs) def send(self, request, **kwargs): # timeout = kwargs.get("timeout") # if timeout is None: kwargs["timeout"] = self.timeout return super().send(request, **kwargs) def get_requests_session() -> Session: """ Set connection pool maxsize and block value to avoid `connection pool full` warnings. :return: requests session """ session = Session() session.mount( "http://", TimeoutHTTPAdapter( pool_connections=25, pool_maxsize=25, pool_block=True, max_retries=1, timeout=30, ), ) session.mount( "https://", TimeoutHTTPAdapter( pool_connections=25, pool_maxsize=25, pool_block=True, max_retries=1, timeout=30, ), ) return session ================================================ FILE: ocean_lib/models/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/models/compute_input.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from typing import Dict, Optional, Union from enforce_typing import enforce_types from ocean_lib.assets.ddo import DDO from ocean_lib.services.service import Service from ocean_lib.web3_internal.constants import ZERO_ADDRESS class ComputeInput: @enforce_types def __init__( self, ddo: DDO, service: Service, transfer_tx_id: Union[str, bytes] = None, userdata: Optional[Dict] = None, consume_market_order_fee_token: Optional[str] = None, consume_market_order_fee_amount: Optional[int] = None, ) -> None: """Initialise and validate arguments.""" assert ddo and service is not None, "bad argument values." if userdata: assert isinstance(userdata, dict), "Userdata must be a dictionary." self.ddo = ddo self.did = ddo.did self.transfer_tx_id = transfer_tx_id self.service = service self.service_id = service.id self.userdata = userdata self.consume_market_order_fee_token = ( consume_market_order_fee_token if consume_market_order_fee_token else ZERO_ADDRESS ) self.consume_market_order_fee_amount = ( consume_market_order_fee_amount if consume_market_order_fee_amount else 0 ) @enforce_types def as_dictionary(self) -> Dict[str, Union[str, Dict]]: res = { "documentId": self.did, "serviceId": self.service_id, } if self.userdata: res["userdata"] = self.userdata if self.transfer_tx_id: res["transferTxId"] = self.transfer_tx_id return res ================================================ FILE: ocean_lib/models/data_nft.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json from base64 import b64encode from enum import IntEnum, IntFlag from typing import Optional from enforce_typing import enforce_types from web3 import Web3 from web3.logs import DISCARD from ocean_lib.models.datatoken_base import DatatokenArguments, DatatokenBase from ocean_lib.ocean.util import ( create_checksum, get_address_of_type, get_args_object, get_from_address, ) from ocean_lib.web3_internal.constants import ZERO_ADDRESS from ocean_lib.web3_internal.contract_base import ContractBase """ def addManager(address: str) -> None: add a manager role to the address provided as a parameter :param address: address of interest :return: None def addMultipleUsersToRoles(addresses: list, roles: list): add multiple users to multiple roles, mapping each address to the corresponding role in the list :param addresses: list of addresses :param roles: list of roles :return: None def addTo725StoreList(address: str) -> None: add a role for storing datatokens to the address provided as a parameter :param address: address of interest :return: None def addToCreateERC20List(address: str) -> None: add a role for deploying datatokens to the address provided as a parameter :param address: address of interest :return: None def addToMetadataList(address: str) -> None: add a role for updating metadata to the address provided as a parameter :param address: address of interest :return: None def approve(dst: str, tokenId: int) -> None: approve a token for address :param address: destination address :param tokenId: token Id :return: None def balance() -> int: get token balance :return: int def balanceOf(address: str) -> int: get token balance for specific address :param address: address of interest :return: int def baseURI() -> str: get token baseURI :return: str def cleanPermissions() -> None: reset all permissions on token, must include the tx_dict with the publisher as transaction sender :return: None def executeCall(operation: int, dst: str, value: int, data: bytes) -> str: execute call :param operation: int representation of the operation to run :param dst: destination account address :param value: amount in wei :param data: operation data :return: transaction tx_id def getApproved(tokenId: int) -> None: get approved address for a specific token Id :param tokenId: token Id :return: address def getData(key: bytes32) -> bytes: get data assigned on token for specific key :param key: key of interest :return: value def getId() -> int: get token Id :return: id def getMetaData() -> tuple: get medatadata of token :return: tuple of decryptor url, decryptor address, metadata state, hasMetaData) def getPermissions(user: str) -> tuple: get user permissions :param user: account address of interest :return: tuple of boolean values for manager role, deployer role, metadata updater role, store role def getTokensList() -> list: get list of ERC20 tokens deployed on this NFT :return: list of token addresses def hasMetaData() -> bool: :return: True if token has metadata, False otherwise def isApprovedForAll(owner: str, operator: str) -> bool: returns if the operator is allowed to manage all the assets of owner. :param owner: address of owner :param operator: address of operator :return: bool def metaDataDecryptorAddress() -> str: :return: address of metadata decryptor def metaDataDecryptorUrl() -> str: :return: url of metadata decryptor def metaDataState() -> int: :return: metadata state according to convention def name() -> str: :return: name of token def ownerOf(tokenId: int) -> str: get owner for a specific token Id :param tokenId: token Id :return: owner address def removeFrom725StoreList(address: str) -> None: remove role for storing datatokens to the address provided as a parameter :param address: address of interest :return: None def removeFromCreateERC20List(address: str) -> None: remove role for deploying datatokens to the address provided as a parameter :param address: address of interest :return: None def removeFromMetadataList(address: str) -> None: remove role for updating metadata to the address provided as a parameter :param address: address of interest :return: None def removeManager(address: str) -> None: remove manager role for the address provided as a parameter :param address: address of interest :return: None def safeTransferFrom(from: str, to: str, token_id: int) -> TransactionReceipt: transfer ownership from one address to another :param from: address of current owner account :param to: address of destination account :param token_id: token Id :return: TransactionReceipt def setApprovalForAll(operator: str, bool approved) -> None: approve or remove address as an operator for the token :param operator: address of operator :param approved: True for approval, False to revoke :return: None def setBaseURI(base_uri: str) -> None: set token baseURI :param base_uri: :return: None def setDataERC20(key: bytes, value: bytes) -> None: set a key, value pair on the token :param key: :param bytes: :return: None def setMetaData( metaDataState: int, metaDataDecryptorUrl: str, metaDataDecryptorAddress: str, flags: bytes, data: bytes, metaDataHash: bytes, metadataProofs: list ) -> None: set token metadata, must include tx_dict with an authorized metadata updater as the sender :param metaDataState: metadata state as an int according to convention :param metaDataDecryptorUrl: metadata decryptor url :param metaDataDecryptorAddress: metadata decryptor address :param flags: encrypt/compress flags :param data: metadata (encoded as bytes) :param metaDataHash: metadata hash :param metadataProofs: list of tuples of valudator address and v, r, s signature values retrieved from validator :return: None def setMetaDataAndTokenURI( metadataAndTokenURI: tuple, ) -> None: similar to setMetaData, set token metadata and token URI, must include tx_dict with an authorized metadata updater as the sender :param metadataAndTokenURI: tuple of the form (state, decryptor url, decryptor address, flags, data, hash, tokenURI, proofs) :return: None def setMetaDataState(metaDataState: int) -> None: set token metadata state, must include tx_dict with an authorized metadata updater as the sender :param metaDataState: metadata state as an int according to convention :return: None def setNewData(key: bytes, value: bytes) -> None: set a key, value pair on the token :param key: :param bytes: :return: None def setTokenURI(tokenURI: str) -> None: set token URI, must include tx_dict with an authorized metadata updater as the sender :param tokenURI: token URI :return: None def symbol() -> str: :return: symbol of token def tokenByIndex(index: int) -> int: Returns a token ID at a given index of all the tokens stored by the contract :param index: int, index of a token :return: int id of the token def tokenOfOwnerByIndex(owner: str, index: int) -> int: Returns a token ID owned by owner at a given index of all the tokens stored by the contract :param owner: owner address :param index: int, index of a token :return: int id of the token def tokenURI() -> str: :return: tokenURI of token def totalSupply() -> int: :return: total supply of token def transferFrom(from: str, to: str, token_id: int) -> TransactionReceipt: transfer ownership from one address to another :param from: address of current owner account :param to: address of destination account :param token_id: token Id :return: TransactionReceipt def transferable() -> bool: :return: True if token is transferable, False otherwise def withdrawETH() -> None: withdraws all available ETH into the owner account :return: None The following functions are wrapped with ocean.py helpers, but you can use the raw form if needed: createERC20 """ class DataNFTPermissions(IntEnum): MANAGER = 0 DEPLOY_DATATOKEN = 1 UPDATE_METADATA = 2 STORE = 3 class MetadataState(IntEnum): ACTIVE = 0 END_OF_LIFE = 1 DEPRECATED = 2 REVOKED = 3 TEMPORARILY_DISABLED = 4 class Flags(IntFlag): PLAIN = 0 COMPRESSED = 1 ENCRYPTED = 2 def to_byte(self): return self.to_bytes(1, "big") @enforce_types class DataNFT(ContractBase): CONTRACT_NAME = "ERC721Template" def create_datatoken(self, tx_dict, *args, **kwargs) -> DatatokenBase: datatoken_args = get_args_object(args, kwargs, DatatokenArguments) return datatoken_args.create_datatoken(self, tx_dict) def calculate_did(self): chain_id = self.config_dict["CHAIN_ID"] return f"did:op:{create_checksum(self.address + str(chain_id))}" def set_data(self, field_label: str, field_value: str, tx_dict: dict): """Set key/value data via ERC725, with strings for key/value""" field_label_hash = Web3.keccak(text=field_label) # to keccak256 hash field_value_bytes = field_value.encode() # to array of bytes tx = self.setNewData(field_label_hash, field_value_bytes, tx_dict) return tx def get_data(self, field_label: str) -> str: """Get key/value data via ERC725, with strings for key/value""" field_label_hash = Web3.keccak(text=field_label) # to keccak256 hash field_value_hex = self.getData(field_label_hash) field_value = field_value_hex.decode("ascii") return field_value class DataNFTArguments: def __init__( self, name: str, symbol: str, template_index: Optional[int] = 1, additional_datatoken_deployer: Optional[str] = None, additional_metadata_updater: Optional[str] = None, uri: Optional[str] = None, transferable: Optional[bool] = None, owner: Optional[str] = None, ): """ :param name: str name of data NFT if creating a new one :param symbol: str symbol of data NFT if creating a new one :param template_index: int template index of the data NFT, by default is 1. :param additional_datatoken_deployer: str address of an additional ERC20 deployer. :param additional_metadata_updater: str address of an additional metadata updater. :param uri: str URL of the data NFT. """ self.name = name self.symbol = symbol or name self.template_index = template_index self.additional_datatoken_deployer = ( additional_datatoken_deployer or ZERO_ADDRESS ) self.additional_metadata_updater = additional_metadata_updater or ZERO_ADDRESS self.uri = uri or self.get_default_token_uri() self.transferable = transferable or True self.owner = owner def get_default_token_uri(self): data = { "name": self.name, "symbol": self.symbol, "background_color": "141414", "image_data": "data:image/svg+xml,%3Csvg viewBox='0 0 99 99' fill='undefined' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='%23ff409277' d='M0,99L0,29C9,24 19,19 31,19C42,18 55,23 67,25C78,26 88,23 99,21L99,99Z'/%3E%3Cpath fill='%23ff4092bb' d='M0,99L0,43C9,45 18,47 30,48C41,48 54,46 66,45C77,43 88,43 99,43L99,99Z'%3E%3C/path%3E%3Cpath fill='%23ff4092ff' d='M0,99L0,78C10,75 20,72 31,71C41,69 53,69 65,70C76,70 87,72 99,74L99,99Z'%3E%3C/path%3E%3C/svg%3E", } return b"data:application/json;base64," + b64encode( json.dumps(data, separators=(",", ":")).encode("utf-8") ) def deploy_contract(self, config_dict, tx_dict) -> DataNFT: from ocean_lib.models.data_nft_factory import ( # isort:skip DataNFTFactoryContract, ) address = get_address_of_type(config_dict, DataNFTFactoryContract.CONTRACT_NAME) data_nft_factory = DataNFTFactoryContract(config_dict, address) wallet_address = get_from_address(tx_dict) receipt = data_nft_factory.deployERC721Contract( self.name, self.symbol, self.template_index, self.additional_metadata_updater, self.additional_datatoken_deployer, self.uri, self.transferable, self.owner or wallet_address, tx_dict, ) registered_event = ( data_nft_factory.contract.events.NFTCreated().process_receipt( receipt, errors=DISCARD )[0] ) data_nft_address = registered_event.args.newTokenAddress return DataNFT(config_dict, data_nft_address) ================================================ FILE: ocean_lib/models/data_nft_factory.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from typing import List, Optional, Union from enforce_typing import enforce_types from web3.exceptions import BadFunctionCallOutput from web3.logs import DISCARD from ocean_lib.models.data_nft import DataNFT, DataNFTArguments from ocean_lib.models.datatoken_base import DatatokenBase from ocean_lib.models.erc721_token_factory_base import ERC721TokenFactoryBase from ocean_lib.models.fixed_rate_exchange import FixedRateExchange, OneExchange from ocean_lib.ocean.util import get_address_of_type, get_args_object, get_from_address from ocean_lib.structures.abi_tuples import MetadataProof, OrderData, ReuseOrderData from ocean_lib.web3_internal.contract_base import ContractBase """ def balance() -> int: get token balance :return: int def getCurrentNFTCount() -> int: get current NFT count :return: int def getCurrentNFTTemplateCount() -> int: get current NFT template count (should be always 1 in current ocean.py) :return: int def getCurrentTemplateCount() -> int: get current ERC20 template count (should be always 2 in current ocean.py) :return: int def getCurrentTokenCount() -> int: get current ERC20 token count :return: int def getNFTTemplate(index: int) -> tuple: get NFT template details for specific index :param index: index of the NFT template :return: tuple of the form (address, valid), where address is the template address and valid is a boolean value indicating template existence def getTokenTemplate(index: int) -> tuple: get ERC20 template details for specific index :param index: index of the ERC20 template :return: if template exists, tuple of the form (address, True), where address is the template address; otherwise throws an exception def owner() -> str: get owner address of the contract :return: str The following functions are wrapped with ocean.py helpers, but you can use the raw form if needed: createNftWithErc20 createNftWithErc20WithDispenser createNftWithErc20WithFixedRate createNftWithMetaData createToken deployERC721Contract erc20List erc721List reuseMultipleTokenOrder startMultipleTokenOrder """ class DataNFTFactoryContract(ERC721TokenFactoryBase): CONTRACT_NAME = "ERC721Factory" @enforce_types def verify_nft(self, nft_address: str) -> bool: """Checks that a token was registered.""" data_nft_contract = DataNFT(self.config_dict, nft_address) try: data_nft_contract.getId() return True except BadFunctionCallOutput: return False def create(self, tx_dict, *args, **kwargs): data_nft_args = get_args_object(args, kwargs, DataNFTArguments) return data_nft_args.deploy_contract(self.config_dict, tx_dict) @enforce_types def start_multiple_token_order(self, orders: List[OrderData], tx_dict: dict) -> str: """An order contains the following keys: - tokenAddress, str - consumer, str - serviceIndex, int - providerFeeAddress, str - providerFeeToken, str - providerFeeAmount (in Wei), int - providerData, bytes - v, int - r, bytes - s, bytes """ for order in orders: order._replace( token_address=ContractBase.to_checksum_address(order.token_address) ) order._replace(consumer=ContractBase.to_checksum_address(order.consumer)) provider_fees = list(order.provider_fees) provider_fees[0] = ContractBase.to_checksum_address(order.provider_fees[0]) provider_fees[1] = ContractBase.to_checksum_address(order.provider_fees[1]) order._replace(provider_fees=tuple(provider_fees)) consume_fees = list(order.consume_fees) consume_fees[0] = ContractBase.to_checksum_address(order.consume_fees[0]) consume_fees[1] = ContractBase.to_checksum_address(order.consume_fees[1]) order._replace(consume_fees=tuple(consume_fees)) return self.startMultipleTokenOrder(orders, tx_dict) @enforce_types def reuse_multiple_token_order( self, reuse_orders: List[ReuseOrderData], tx_dict: dict ) -> str: for order in reuse_orders: order._replace( token_address=ContractBase.to_checksum_address(order.token_address) ) provider_fees = list(order.provider_fees) provider_fees[0] = ContractBase.to_checksum_address(order.provider_fees[0]) provider_fees[1] = ContractBase.to_checksum_address(order.provider_fees[1]) return self.reuseMultipleTokenOrder(reuse_orders, tx_dict) @enforce_types def create_with_erc20( self, data_nft_args, datatoken_args, tx_dict: dict, ) -> str: wallet_address = get_from_address(tx_dict) receipt = self.createNftWithErc20( ( data_nft_args.name, data_nft_args.symbol, data_nft_args.template_index, data_nft_args.uri, data_nft_args.transferable, ContractBase.to_checksum_address(data_nft_args.owner or wallet_address), ), ( datatoken_args.template_index, [datatoken_args.name, datatoken_args.symbol], [ ContractBase.to_checksum_address( datatoken_args.minter or wallet_address ), ContractBase.to_checksum_address( datatoken_args.fee_manager or wallet_address ), ContractBase.to_checksum_address( datatoken_args.publish_market_order_fees.address ), ContractBase.to_checksum_address( datatoken_args.publish_market_order_fees.token ), ], [datatoken_args.cap, datatoken_args.publish_market_order_fees.amount], datatoken_args.bytess, ), tx_dict, ) registered_nft_event = self.contract.events.NFTCreated().process_receipt( receipt, errors=DISCARD )[0] data_nft_address = registered_nft_event.args.newTokenAddress data_nft_token = DataNFT(self.config_dict, data_nft_address) registered_token_event = self.contract.events.TokenCreated().process_receipt( receipt, errors=DISCARD )[0] datatoken_address = registered_token_event.args.newTokenAddress datatoken = DatatokenBase.get_typed(self.config_dict, datatoken_address) return data_nft_token, datatoken @enforce_types def create_with_erc20_and_fixed_rate( self, data_nft_args, datatoken_args, fixed_price_args, tx_dict: dict, ) -> str: wallet_address = get_from_address(tx_dict) receipt = self.createNftWithErc20WithFixedRate( ( data_nft_args.name, data_nft_args.symbol, data_nft_args.template_index, data_nft_args.uri, data_nft_args.transferable, ContractBase.to_checksum_address(data_nft_args.owner or wallet_address), ), ( datatoken_args.template_index, [datatoken_args.name, datatoken_args.symbol], [ ContractBase.to_checksum_address( datatoken_args.minter or wallet_address ), ContractBase.to_checksum_address( datatoken_args.fee_manager or wallet_address ), ContractBase.to_checksum_address( datatoken_args.publish_market_order_fees.address ), ContractBase.to_checksum_address( datatoken_args.publish_market_order_fees.token ), ], [datatoken_args.cap, datatoken_args.publish_market_order_fees.amount], datatoken_args.bytess, ), fixed_price_args.to_tuple(self.config_dict, tx_dict), tx_dict, ) registered_nft_event = self.contract.events.NFTCreated().process_receipt( receipt, errors=DISCARD )[0] data_nft_address = registered_nft_event.args.newTokenAddress data_nft_token = DataNFT(self.config_dict, data_nft_address) registered_token_event = self.contract.events.TokenCreated().process_receipt( receipt, errors=DISCARD )[0] datatoken_address = registered_token_event.args.newTokenAddress datatoken = DatatokenBase.get_typed(self.config_dict, datatoken_address) registered_fixed_rate_event = ( self.contract.events.NewFixedRate().process_receipt( receipt, errors=DISCARD )[0] ) exchange_id = registered_fixed_rate_event.args.exchangeId fixed_rate_exchange = FixedRateExchange( self.config_dict, get_address_of_type(self.config_dict, "FixedPrice") ) exchange = OneExchange(fixed_rate_exchange, exchange_id) return data_nft_token, datatoken, exchange @enforce_types def create_with_erc20_and_dispenser( self, data_nft_args, datatoken_args, dispenser_args, tx_dict: dict, ) -> str: wallet_address = get_from_address(tx_dict) receipt = self.createNftWithErc20WithDispenser( ( data_nft_args.name, data_nft_args.symbol, data_nft_args.template_index, data_nft_args.uri, data_nft_args.transferable, ContractBase.to_checksum_address(data_nft_args.owner or wallet_address), ), ( datatoken_args.template_index, [datatoken_args.name, datatoken_args.symbol], [ ContractBase.to_checksum_address( datatoken_args.minter or wallet_address ), ContractBase.to_checksum_address( datatoken_args.fee_manager or wallet_address ), ContractBase.to_checksum_address( datatoken_args.publish_market_order_fees.address ), ContractBase.to_checksum_address( datatoken_args.publish_market_order_fees.token ), ], [datatoken_args.cap, datatoken_args.publish_market_order_fees.amount], datatoken_args.bytess, ), dispenser_args.to_tuple(self.config_dict), tx_dict, ) registered_nft_event = self.contract.events.NFTCreated().process_receipt( receipt, errors=DISCARD )[0] data_nft_address = registered_nft_event.args.newTokenAddress data_nft_token = DataNFT(self.config_dict, data_nft_address) registered_token_event = self.contract.events.TokenCreated().process_receipt( receipt, errors=DISCARD )[0] datatoken_address = registered_token_event.args.newTokenAddress datatoken = DatatokenBase.get_typed(self.config_dict, datatoken_address) registered_dispenser_event = ( self.contract.events.DispenserCreated().process_receipt( receipt, errors=DISCARD )[0] ) assert registered_dispenser_event.args.datatokenAddress == datatoken_address return data_nft_token, datatoken @enforce_types def create_with_metadata( self, data_nft_args, metadata_state: int, metadata_decryptor_url: str, metadata_decryptor_address: bytes, metadata_flags: bytes, metadata_data: Union[str, bytes], metadata_data_hash: Union[str, bytes], metadata_proofs: List[MetadataProof], tx_dict: dict, ) -> str: wallet_address = get_from_address(tx_dict) receipt = self.createNftWithMetaData( ( data_nft_args.name, data_nft_args.symbol, data_nft_args.template_index, data_nft_args.uri, data_nft_args.transferable, ContractBase.to_checksum_address(data_nft_args.owner or wallet_address), ), ( metadata_state, metadata_decryptor_url, metadata_decryptor_address, metadata_flags, metadata_data, metadata_data_hash, metadata_proofs, ), tx_dict, ) registered_nft_event = self.contract.events.NFTCreated().process_receipt( receipt, errors=DISCARD )[0] data_nft_address = registered_nft_event.args.newTokenAddress data_nft_token = DataNFT(self.config_dict, data_nft_address) return data_nft_token @enforce_types def search_exchange_by_datatoken( self, fixed_rate_exchange: FixedRateExchange, datatoken: str, exchange_owner: Optional[str] = None, ) -> list: datatoken_contract = DatatokenBase.get_typed(self.config_dict, datatoken) exchange_addresses_and_ids = datatoken_contract.getFixedRates() return ( exchange_addresses_and_ids if exchange_owner is None else [ exchange_address_and_id for exchange_address_and_id in exchange_addresses_and_ids if fixed_rate_exchange.getExchange(exchange_address_and_id[1])[0] == exchange_owner ] ) @enforce_types def get_token_address(self, receipt): event = self.contract.events.NFTCreated().process_receipt( receipt, errors=DISCARD )[0] return event.args.newTokenAddress @enforce_types def check_datatoken(self, datatoken_address: str) -> bool: return self.erc20List(datatoken_address) @enforce_types def check_nft(self, nft_address: str) -> bool: return self.erc721List(nft_address) == nft_address ================================================ FILE: ocean_lib/models/datatoken1.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import logging from typing import Any, Optional from enforce_typing import enforce_types from ocean_lib.models.datatoken_base import DatatokenBase, TokenFeeInfo from ocean_lib.ocean.util import from_wei, get_from_address, to_wei from ocean_lib.web3_internal.constants import ZERO_ADDRESS from ocean_lib.web3_internal.contract_base import ContractBase checksum_addr = ContractBase.to_checksum_address logger = logging.getLogger("ocean") """ def addMinter(address: str) -> None: add a minter for the datatoken :param address: address of interest :return: None def addPaymentManager(address: str) -> None: add a payment manager for the datatoken :param address: address of interest :return: None def allowance(owner_addr: str, spender_addr: str) -> int: get token allowance for spender from owner :param owner_addr: address of owner :param spender_addr: address of owner :return: int allowance def approve(address: str, amount: int) -> None: approve tokens for a specific address in the given amount :param address: address of interest :param amount: amount in int :return: None def authERC20(index: int) -> tuple: get user permissions on ERC20 for specific index :param index: index of interest :return: tuple of tuples of boolean values for minter role, payment manager role def balance() -> int: get token balance :return: int def balanceOf(address: str) -> int: get token balance for specific address :param address: address of interest :return: int def burn(amount: int) -> None: burn a specific amount of tokens :param amount: amount in int :return: None def burnFrom(address: str, amount: int) -> None: burn a specific amount of tokens from an account :param address: address of the burner account :param amount: amount in int :return: None def cap() -> int: get token cap :return: int def cleanPermissions() -> None: reset all permissions on token, must include the tx_dict with the publisher as transaction sender :return: None def decimals() -> int: get token decimals :return: int def decreaseAllowance(address: str, amount: int) -> None: decrease the allowance for an address by a specific amount :param address: address of the account :param amount: amount to subtract in int :return: None def getERC721Address() -> str: get address of ERC721 token :return: str def getId() -> int: get token Id :return: id def getPaymentCollector() -> str: get payment collector address :return: address of payment collector def getPermissions(user: str) -> tuple: get user permissions :param user: account address of interest :return: tuple of boolean values for minter role, payment manager role def increaseAllowance(address: str, amount: int) -> None: increase the allowance for an address by a specific amount :param address: address of the account :param amount: amount to add in int :return: None def isERC20Deployer(address: str) -> bool: returns whether an address has ERC20 Deployer role :param address: address of interest :return: bool def isMinter(address: str) -> bool: returns whether an address has minter role :param address: address of interest :return: bool def mint(address: str, amount: int) -> None: mints am amount of tokens for the given address, requires tx_dict with a datatoken minter as the sender returns whether an address has minter role :param address: address of interest :param amount: amount to mint :return: None def name() -> str: :return: name of token def removeMinter(address: str) -> None: remove minter role for the datatoken :param address: address of interest :return: None def removePaymentManager(address: str) -> None: remove payment manager role for the datatoken :param address: address of interest :return: None def setData(key: bytes, value: bytes) -> None: set a key, value pair on the token :param key: :param bytes: :return: None def setPaymentCollector(address: str) -> None: set payment collector address :param address: address of payment collector :return: None def setPublishingMarketFee(address: str, token: str, amount: int) -> None: set publishing market fee :param address: address of the intended receiver :param token: address of the token to receive fees in :param amount: amount of intended fee :return: None def symbol() -> str: :return: symbol of token def totalSupply() -> int: :return: total supply of token def transfer(to: str, amount: int) -> None: transfer an amount of tokens from transaction sender to address, requires tx_dict with a datatoken minter as the sender :param to: address of destination account :param amount: amount to transfer :return: None def transferFrom(from: str, to: str, amount: int) -> None: transfer an amount of tokens from one address to another, requires tx_dict with a datatoken minter as the sender :param from: address of current owner account :param to: address of destination account :param amount: amount to transfer :return: None def withdrawETH() -> None: withdraws all available ETH into the owner account :return: None The following functions are wrapped with ocean.py helpers, but you can use the raw form if needed: createDispenser createFixedRate getDispensers getFixedRates getPublishingMarketFee reuseOrder startOrder """ class Datatoken1(DatatokenBase): CONTRACT_NAME = "ERC20Template" BASE = 10**18 BASE_COMMUNITY_FEE_PERCENTAGE = BASE / 1000 BASE_MARKET_FEE_PERCENTAGE = BASE / 1000 # =========================================================================== # consume def dispense_and_order( self, provider_fees: dict, tx_dict: dict, consumer: Optional[str] = None, service_index: int = 1, consume_market_fees=None, ) -> str: if not consumer: consumer = get_from_address(tx_dict) if not consume_market_fees: consume_market_fees = TokenFeeInfo() buyer_addr = get_from_address(tx_dict) bal = from_wei(self.balanceOf(buyer_addr)) if bal < 1.0: dispensers = self.get_dispensers() assert dispensers, "there are no dispensers for this datatoken" dispenser = dispensers[0] # catch key failure modes st = dispenser.status(self.address) active, allowedSwapper = st[0], st[6] if not active: raise ValueError("No active dispenser for datatoken") if allowedSwapper not in [ZERO_ADDRESS, buyer_addr]: raise ValueError(f"Not allowed. allowedSwapper={allowedSwapper}") # Try to dispense. If other issues, they'll pop out dispenser.dispense(self.address, to_wei(1), buyer_addr, tx_dict) return self.start_order( consumer=ContractBase.to_checksum_address(consumer), service_index=service_index, provider_fees=provider_fees, consume_market_fees=consume_market_fees, tx_dict=tx_dict, ) @enforce_types def buy_DT_and_order( self, provider_fees: dict, exchange: Any, tx_dict: dict, consumer: Optional[str] = None, service_index: int = 1, consume_market_fees=None, ) -> str: if not consumer: consumer = get_from_address(tx_dict) exchanges = self.get_exchanges() assert exchanges, "there are no fixed rate exchanges for this datatoken" # import now, to avoid circular import from ocean_lib.models.fixed_rate_exchange import OneExchange if not consume_market_fees: consume_market_fees = TokenFeeInfo() if not isinstance(exchange, OneExchange): exchange = exchanges[0] exchange.buy_DT( datatoken_amt=to_wei(1), consume_market_fee_addr=consume_market_fees.address, consume_market_fee=consume_market_fees.amount, tx_dict=tx_dict, ) return self.start_order( consumer=ContractBase.to_checksum_address(consumer), service_index=service_index, provider_fees=provider_fees, consume_market_fees=consume_market_fees, tx_dict=tx_dict, ) ================================================ FILE: ocean_lib/models/datatoken2.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from typing import Any, Optional, Union from enforce_typing import enforce_types from ocean_lib.models.datatoken_base import DatatokenBase, TokenFeeInfo from ocean_lib.ocean.util import get_from_address, to_wei from ocean_lib.web3_internal.constants import ZERO_ADDRESS from ocean_lib.web3_internal.contract_base import ContractBase checksum_addr = ContractBase.to_checksum_address """ Datatoken2 retains all the functions from Datatoken model. The different functions are redundant (wrapped by ocean.py in helpers): buyFromDispenserAndOrder buyFromFreAndOrder """ class Datatoken2(DatatokenBase): CONTRACT_NAME = "ERC20TemplateEnterprise" @enforce_types def buy_DT_and_order( self, provider_fees: dict, exchange: Any, tx_dict: dict, consumer: Optional[str] = None, service_index: int = 1, consume_market_fees=None, max_base_token_amount: Optional[Union[int, str]] = None, consume_market_swap_fee_amount: Optional[Union[int, str]] = 0, consume_market_swap_fee_address: Optional[str] = ZERO_ADDRESS, ) -> str: if not consumer: consumer = get_from_address(tx_dict) exchanges = self.get_exchanges() assert exchanges, "there are no fixed rate exchanges for this datatoken" # import now, to avoid circular import from ocean_lib.models.fixed_rate_exchange import OneExchange if not isinstance(exchange, OneExchange): exchange = exchanges[0] if not consume_market_fees: consume_market_fees = TokenFeeInfo() if not max_base_token_amount: amt_needed = exchange.BT_needed(to_wei(1), consume_market_fees.amount) max_base_token_amount = amt_needed return self.buyFromFreAndOrder( ( ContractBase.to_checksum_address(consumer), service_index, ( checksum_addr(provider_fees["providerFeeAddress"]), checksum_addr(provider_fees["providerFeeToken"]), int(provider_fees["providerFeeAmount"]), provider_fees["v"], provider_fees["r"], provider_fees["s"], provider_fees["validUntil"], provider_fees["providerData"], ), consume_market_fees.to_tuple(), ), ( ContractBase.to_checksum_address(exchange.address), exchange.exchange_id, max_base_token_amount, consume_market_swap_fee_amount, ContractBase.to_checksum_address(consume_market_swap_fee_address), ), tx_dict, ) @enforce_types def dispense_and_order( self, provider_fees: dict, tx_dict: dict, consumer: Optional[str] = None, service_index: int = 1, consume_market_fees=None, ) -> str: if not consume_market_fees: consume_market_fees = TokenFeeInfo() if not consumer: consumer = get_from_address(tx_dict) dispensers = self.get_dispensers() assert dispensers, "there are no dispensers for this datatoken" dispenser = dispensers[0] return self.buyFromDispenserAndOrder( ( ContractBase.to_checksum_address(consumer), service_index, ( checksum_addr(provider_fees["providerFeeAddress"]), checksum_addr(provider_fees["providerFeeToken"]), int(provider_fees["providerFeeAmount"]), provider_fees["v"], provider_fees["r"], provider_fees["s"], provider_fees["validUntil"], provider_fees["providerData"], ), consume_market_fees.to_tuple(), ), ContractBase.to_checksum_address(dispenser.address), tx_dict, ) ================================================ FILE: ocean_lib/models/datatoken_base.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import logging from abc import ABC from enum import IntEnum from typing import Any, Dict, List, Optional, Tuple, Union from enforce_typing import enforce_types from web3.logs import DISCARD from web3.main import Web3 from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.models.fixed_rate_exchange import OneExchange from ocean_lib.ocean.util import ( get_address_of_type, get_args_object, get_from_address, get_ocean_token_address, str_with_wei, ) from ocean_lib.services.service import Service from ocean_lib.structures.file_objects import FilesType from ocean_lib.web3_internal.constants import MAX_UINT256, ZERO_ADDRESS from ocean_lib.web3_internal.contract_base import ContractBase checksum_addr = ContractBase.to_checksum_address logger = logging.getLogger("ocean") class TokenFeeInfo: def __init__( self, address: Optional[str] = None, token: Optional[str] = None, amount: Optional[int] = 0, ): self.address = ( Web3.to_checksum_address(address.lower()) if address else ZERO_ADDRESS ) self.token = Web3.to_checksum_address(token.lower()) if token else ZERO_ADDRESS self.amount = amount def to_tuple(self): return (self.address, self.token, self.amount) @classmethod def from_tuple(cls, tup): address, token, amount = tup return cls(address, token, amount) def __str__(self): s = ( f"TokenFeeInfo: \n" f" address = {self.address}\n" f" token = {self.token}\n" f" amount = {str_with_wei(self.amount)}\n" ) return s class DatatokenArguments: def __init__( self, name: Optional[str] = "Datatoken 1", symbol: Optional[str] = "DT1", template_index: Optional[int] = 1, minter: Optional[str] = None, fee_manager: Optional[str] = None, publish_market_order_fees: Optional = None, bytess: Optional[List[bytes]] = None, services: Optional[list] = None, files: Optional[List[FilesType]] = None, consumer_parameters: Optional[List[Dict[str, Any]]] = None, cap: Optional[int] = None, ): if template_index == 2 and not cap: raise Exception("Cap is needed for Datatoken Template 2 token deployment.") self.cap = cap if template_index == 2 else MAX_UINT256 self.name = name self.symbol = symbol self.template_index = template_index self.minter = minter self.fee_manager = fee_manager self.bytess = bytess or [b""] self.services = services self.files = files self.consumer_parameters = consumer_parameters self.publish_market_order_fees = publish_market_order_fees or TokenFeeInfo() self.set_default_fees_at_deploy = not publish_market_order_fees def create_datatoken(self, data_nft, tx_dict, with_services=False): config_dict = data_nft.config_dict OCEAN_address = get_ocean_token_address(config_dict) initial_list = data_nft.getTokensList() wallet_address = get_from_address(tx_dict) if self.set_default_fees_at_deploy: self.publish_market_order_fees = TokenFeeInfo( address=wallet_address, token=OCEAN_address ) data_nft.createERC20( self.template_index, [self.name, self.symbol], [ ContractBase.to_checksum_address(self.minter or wallet_address), ContractBase.to_checksum_address(self.fee_manager or wallet_address), self.publish_market_order_fees.address, self.publish_market_order_fees.token, ], [self.cap, self.publish_market_order_fees.amount], self.bytess, tx_dict, ) new_elements = [ item for item in data_nft.getTokensList() if item not in initial_list ] assert len(new_elements) == 1, "new datatoken has no address" datatoken = DatatokenBase.get_typed(config_dict, new_elements[0]) logger.info( f"Successfully created datatoken with address " f"{datatoken.address}." ) if with_services: if not self.services: self.services = [ datatoken.build_access_service( service_id="0", service_endpoint=config_dict.get("PROVIDER_URL"), files=self.files, consumer_parameters=self.consumer_parameters, ) ] else: for service in self.services: service.datatoken = datatoken.address return datatoken class DatatokenRoles(IntEnum): MINTER = 0 PAYMENT_MANAGER = 1 class DatatokenBase(ABC, ContractBase): CONTRACT_NAME = "ERC20Template" BASE = 10**18 BASE_COMMUNITY_FEE_PERCENTAGE = BASE / 1000 BASE_MARKET_FEE_PERCENTAGE = BASE / 1000 # =========================================================================== # consume @staticmethod def get_typed(config, address): from ocean_lib.models.datatoken1 import Datatoken1 from ocean_lib.models.datatoken2 import Datatoken2 datatoken = Datatoken1(config, address) try: template_id = datatoken.getId() except Exception: template_id = 1 return datatoken if template_id == 1 else Datatoken2(config, address) @enforce_types def start_order( self, consumer: str, service_index: int, provider_fees: dict, tx_dict: dict, consume_market_fees=None, ) -> str: if not consume_market_fees: consume_market_fees = TokenFeeInfo() return self.startOrder( checksum_addr(consumer), service_index, ( checksum_addr(provider_fees["providerFeeAddress"]), checksum_addr(provider_fees["providerFeeToken"]), int(provider_fees["providerFeeAmount"]), provider_fees["v"], provider_fees["r"], provider_fees["s"], provider_fees["validUntil"], provider_fees["providerData"], ), consume_market_fees.to_tuple(), tx_dict, ) @enforce_types def reuse_order( self, order_tx_id: Union[str, bytes], provider_fees: dict, tx_dict: dict, ) -> str: return self.reuseOrder( order_tx_id, ( checksum_addr(provider_fees["providerFeeAddress"]), checksum_addr(provider_fees["providerFeeToken"]), int(provider_fees["providerFeeAmount"]), provider_fees["v"], provider_fees["r"], provider_fees["s"], provider_fees["validUntil"], provider_fees["providerData"], ), tx_dict, ) @enforce_types def get_start_order_logs( self, consumer_address: Optional[str] = None, from_block: Optional[int] = 0, to_block: Optional[int] = "latest", ) -> Tuple: topic0 = self.get_event_signature("OrderStarted") topics = [topic0] if consumer_address: topic1 = f"0x000000000000000000000000{consumer_address[2:].lower()}" topics = [topic0, topic1] web3 = self.config_dict["web3_instance"] event_filter = web3.eth.filter( { "topics": topics, "toBlock": to_block, "fromBlock": from_block, } ) orders = [] for log in event_filter.get_all_entries(): receipt = web3.eth.wait_for_transaction_receipt(log.transactionHash) processed_events = self.contract.events.OrderStarted().process_receipt( receipt, errors=DISCARD ) for processed_event in processed_events: orders.append(processed_event) return orders # ====================================================================== # Priced data: fixed-rate exchange @enforce_types def create_exchange( self, tx_dict: dict, *args, **kwargs ) -> Union[OneExchange, tuple]: """ For this datatoken, create a single fixed-rate exchange (OneExchange). This wraps the smart contract method Datatoken.createFixedRate() with a simpler interface Main params: - rate - how many base tokens does 1 datatoken cost? In wei or str - base_token_addr - e.g. OCEAN address - tx_dict - e.g. {"from": alice_wallet} Optional params, with good defaults - owner_addr - publish_market_fee_collector - fee going to publish mkt - publish_market_fee - in wei or str, e.g. int(1e15) or "0.001 ether" do they need to by supplied/allowed by participants like base token? - allowed_swapper - if ZERO_ADDRESS, anyone can swap - full_info - return just OneExchange, or (OneExchange, ) Return - exchange - OneExchange - (maybe) tx_receipt """ # import now, to avoid circular import from ocean_lib.models.fixed_rate_exchange import ExchangeArguments, OneExchange exchange_args = get_args_object(args, kwargs, ExchangeArguments) args_tup = exchange_args.to_tuple(self.config_dict, tx_dict, self.decimals()) tx = self.createFixedRate(*(args_tup + (tx_dict,))) event = self.contract.events.NewFixedRate().process_receipt(tx, errors=DISCARD)[ 0 ] exchange_id = event.args.exchangeId FRE = self._FRE() exchange = OneExchange(FRE, exchange_id) return (exchange, tx) if kwargs.get("full_info") else exchange @enforce_types def get_exchanges(self, only_active=True) -> list: """return List[OneExchange] - all the exchanges for this datatoken""" # import now, to avoid circular import from ocean_lib.models.fixed_rate_exchange import FixedRateExchange, OneExchange exchanges = [] addrs_and_exchange_ids = self.getFixedRates() exchanges = [ OneExchange(FixedRateExchange(self.config_dict, address), exchange_id) for address, exchange_id in addrs_and_exchange_ids ] if not only_active: return exchanges return [exchange for exchange in exchanges if exchange.is_active()] @enforce_types def _FRE(self): """Return FixedRateExchange - global across all exchanges""" # import now, to avoid circular import from ocean_lib.models.fixed_rate_exchange import FixedRateExchange FRE_addr = get_address_of_type(self.config_dict, "FixedPrice") return FixedRateExchange(self.config_dict, FRE_addr) # ====================================================================== # Free data: dispenser faucet @enforce_types def create_dispenser(self, tx_dict: dict, *args, **kwargs): """ For this datataken, create a dispenser faucet for free tokens. This wraps the smart contract method Datatoken.createDispenser() with a simpler interface. :param: max_tokens - max # tokens to dispense, in wei :param: max_balance - max balance of requester :tx_dict: e.g. {"from": alice_wallet} :return: tx """ # already created, so nothing to do if self.dispenser_status().active: return from ocean_lib.models.dispenser import DispenserArguments # isort:skip dispenser_args = get_args_object(args, kwargs, DispenserArguments) args_tup = dispenser_args.to_tuple(self.config_dict) # do contract tx tx = self.createDispenser(*(args_tup + (tx_dict,))) return tx @enforce_types def get_dispensers(self, only_active=True) -> list: """return List[Dispenser] - all the dispensers for this datatoken""" # import here to avoid circular import from ocean_lib.models.dispenser import Dispenser dispensers = [] addrs = self.getDispensers() dispensers = [Dispenser(self.config_dict, address) for address in addrs] if not only_active: return dispensers return [disp for disp in dispensers if disp.status(self.address)] @enforce_types def dispense(self, amount: Union[int, str], tx_dict: dict): """ Dispense free tokens via the dispenser faucet. :param: amount - number of tokens to dispense, in wei :tx_dict: e.g. {"from": alice_wallet} :return: tx """ # args for contract tx datatoken_addr = self.address from_addr = get_from_address(tx_dict) # do contract tx tx = self._ocean_dispenser().dispense( datatoken_addr, amount, from_addr, tx_dict ) return tx @enforce_types def dispenser_status(self): """:return: DispenserStatus object""" # import here to avoid circular import from ocean_lib.models.dispenser import DispenserStatus status_tup = self._ocean_dispenser().status(self.address) return DispenserStatus(status_tup) @enforce_types def _ocean_dispenser(self): """:return: Dispenser object""" # import here to avoid circular import from ocean_lib.models.dispenser import Dispenser dispenser_addr = get_address_of_type(self.config_dict, "Dispenser") return Dispenser(self.config_dict, dispenser_addr) @enforce_types def build_access_service( self, service_id: str, service_endpoint: str, files: List[FilesType], timeout: Optional[int] = 3600, consumer_parameters=None, ) -> Service: return Service( service_id=service_id, service_type=ServiceTypes.ASSET_ACCESS, service_endpoint=service_endpoint, datatoken=self.address, files=files, timeout=timeout, consumer_parameters=consumer_parameters, ) def get_publish_market_order_fees(self): return TokenFeeInfo.from_tuple(self.getPublishingMarketFee()) def get_from_pricing_schema_and_order(self, *args, **kwargs): dispensers = self.dispenser_status().active exchanges = self.get_exchanges() if not dispensers and not exchanges: raise ValueError("No pricing schemas found") if dispensers: kwargs.pop("consume_market_swap_fee_amount", None) kwargs.pop("consume_market_swap_fee_address", None) return self.dispense_and_order(*args, **kwargs) exchange = self.get_exchanges()[0] kwargs["exchange"] = exchange consume_market_fees = kwargs.get("consume_market_fees") if not consume_market_fees: consume_market_fees = TokenFeeInfo() wallet = kwargs["tx_dict"]["from"] amt_needed = exchange.BT_needed( Web3.to_wei(1, "ether"), consume_market_fees.amount ) base_token = DatatokenBase.get_typed( exchange._FRE.config_dict, exchange.details.base_token ) base_token_balance = base_token.balanceOf(wallet.address) if base_token_balance < amt_needed: raise ValueError( f"Your token balance {base_token_balance} {base_token.symbol()} is not sufficient " f"to execute the requested service. This service " f"requires {amt_needed} {base_token.symbol()}." ) if self.getId() == 1: approve_address = exchange.address kwargs.pop("consume_market_swap_fee_amount", None) kwargs.pop("consume_market_swap_fee_address", None) else: approve_address = self.address kwargs["max_base_token_amount"] = amt_needed base_token.approve( approve_address, amt_needed, {"from": wallet}, ) return self.buy_DT_and_order(*args, **kwargs) class MockERC20(DatatokenBase): CONTRACT_NAME = "MockERC20" def getId(self): return 1 class MockOcean(DatatokenBase): CONTRACT_NAME = "MockOcean" def getId(self): return 1 ================================================ FILE: ocean_lib/models/df/df_rewards.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class DFRewards(ContractBase): CONTRACT_NAME = "DFRewards" ================================================ FILE: ocean_lib/models/df/df_strategy_v1.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class DFStrategyV1(ContractBase): CONTRACT_NAME = "DFStrategyV1" ================================================ FILE: ocean_lib/models/df/test/conftest.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from conftest_ganache import * @pytest.fixture def ocean(publisher_ocean): return publisher_ocean ================================================ FILE: ocean_lib/models/df/test/test_df_rewards.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest @pytest.mark.unit def test1(ocean): # df-py/util/test has thorough tests, so keep it super-simple here assert ocean.df_rewards.address is not None ================================================ FILE: ocean_lib/models/df/test/test_df_strategy_v1.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest @pytest.mark.unit def test1(ocean): # df-py/util/test has thorough tests, so keep it super-simple here assert ocean.df_rewards.address is not None ================================================ FILE: ocean_lib/models/dispenser.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from typing import Optional, Union from enforce_typing import enforce_types from ocean_lib.ocean.util import from_wei, get_address_of_type from ocean_lib.web3_internal.constants import MAX_UINT256, ZERO_ADDRESS from ocean_lib.web3_internal.contract_base import ContractBase """ def activate(dt_addr: str, max_tokens: int, max_balance: int) -> None: activate dispenser after deactivation :param dt_addr: datatoken address of ERC20 :param max_tokens: maximum amount of tokens :param max_balance: maximum token balance :return: None def balance() -> int: get dispenser balance :return: balance in int def deactivate(dt_addr: str) -> None: deactivate dispenser :param dt_addr: datatoken address of ERC20 :return: None def dispense(dt_addr: str, amount: int, destination: str) -> None: dispense an amount of tokens to a given destination address, requires tx_dict with a sender that can dispense :param dt_addr: address of the ERC20 token :param amount: amount to dispense :param destination: address of the account to receive dispensed tokens :return: None def getId() -> int: get dispenser id :return: dispenser id def ownerWithdraw(dt_addr: str) -> None: withdraw datatokens from dispenser, requires tx_dict with a sender that can dispense :param dt_addr: address of the ERC20 token. If missing, will withdraw all. :return: None def setAllowedSwapper(dt_addr: str, new_swapper_addr: str) -> None: set allowed swapper to a new address :param dt_addr: address of the ERC20 token :param new_swapper_addr: address of the account to be set as swapper :return: None The following functions are wrapped with ocean.py helpers, but you can use the raw form if needed: status -> you can use the datatoken.dispenser_status() function as a better shorthand create -> you can use the datatoken.create_dispenser() function as a better shorthand datatokensList -> a list of datatokens served by this dispenser, but we recommend retrieving each dispenser from its datatoken object """ class Dispenser(ContractBase): CONTRACT_NAME = "Dispenser" class DispenserArguments: def __init__( self, max_tokens: Optional[Union[int, str]] = MAX_UINT256, max_balance: Optional[Union[int, str]] = MAX_UINT256, with_mint: Optional[bool] = True, allowed_swapper: Optional[str] = ZERO_ADDRESS, ): self.max_tokens = max_tokens self.max_balance = max_balance self.with_mint = with_mint self.allowed_swapper = ContractBase.to_checksum_address(allowed_swapper) def to_tuple(self, config_dict): dispenser_address = get_address_of_type(config_dict, "Dispenser") return ( ContractBase.to_checksum_address(dispenser_address), self.max_tokens, self.max_balance, bool(self.with_mint), self.allowed_swapper, ) class DispenserStatus: """Status of dispenser smart contract, for a given datatoken""" def __init__(self, status_tup): """ :param:status_tup -- returned from Dispenser.sol::status(dt_addr) which is (bool active, address owner, bool isMinter, uint256 maxTokens, uint256 maxBalance, uint256 balance, address allowedSwapper) """ t = status_tup self.active: bool = t[0] self.owner_address: str = t[1] self.is_minter: bool = t[2] self.max_tokens: int = t[3] self.max_balance: int = t[4] self.balance: int = t[5] self.allowed_swapper: int = t[6] def __str__(self): s = ( f"DispenserStatus: " f" active = {self.active}\n" f" owner_address = {self.owner_address}\n" f" balance (of tokens) = {_strWithWei(self.balance)}\n" f" is_minter (can mint more tokens?) = {self.is_minter}\n" f" max_tokens (to dispense) = {_strWithWei(self.max_tokens)}\n" f" max_balance (of requester) = {_strWithWei(self.max_balance)}\n" ) if self.allowed_swapper.lower() == ZERO_ADDRESS.lower(): s += " allowed_swapper = anyone can request\n" else: s += f" allowed_swapper = {self.allowed_swapper}\n" return s @enforce_types def _strWithWei(x_wei: int) -> str: return f"{from_wei(x_wei)} ({x_wei} wei)" ================================================ FILE: ocean_lib/models/erc721_token_factory_base.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from abc import ABC from ocean_lib.web3_internal.contract_base import ContractBase class ERC721TokenFactoryBase(ABC, ContractBase): pass ================================================ FILE: ocean_lib/models/factory_router.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class FactoryRouter(ContractBase): CONTRACT_NAME = "FactoryRouter" ================================================ FILE: ocean_lib/models/fixed_rate_exchange.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from typing import Optional, Union from enforce_typing import enforce_types from ocean_lib.models.factory_router import FactoryRouter from ocean_lib.ocean.util import get_address_of_type, get_from_address, str_with_wei from ocean_lib.web3_internal.constants import MAX_UINT256, ZERO_ADDRESS from ocean_lib.web3_internal.contract_base import ContractBase checksum_addr = ContractBase.to_checksum_address @enforce_types class ExchangeArguments: def __init__( self, rate: Union[int, str], base_token_addr: str, owner_addr: Optional[str] = None, publish_market_fee_collector: Optional[str] = None, publish_market_fee: Union[int, str] = 0, allowed_swapper: str = ZERO_ADDRESS, full_info: bool = False, dt_decimals: Optional[int] = None, ): self.rate = rate self.base_token_addr = base_token_addr self.owner_addr = owner_addr self.publish_market_fee_collector = publish_market_fee_collector self.allowed_swapper = checksum_addr(allowed_swapper) self.rate = rate self.publish_market_fee = publish_market_fee self.dt_decimals = dt_decimals def to_tuple(self, config_dict, tx_dict, dt_decimals=None): FRE_addr = get_address_of_type(config_dict, "FixedPrice") if not self.owner_addr: self.owner_addr = get_from_address(tx_dict) if not self.publish_market_fee_collector: self.publish_market_fee_collector = get_from_address(tx_dict) if not self.dt_decimals: if not dt_decimals: raise Exception( "Must configure dt decimals either on arg creation or usage." ) self.dt_decimals = dt_decimals # TODO: move to top now? from ocean_lib.models.datatoken_base import DatatokenBase # isort:skip self.BT = DatatokenBase.get_typed(config_dict, self.base_token_addr) return ( FRE_addr, [ checksum_addr(self.BT.address), checksum_addr(self.owner_addr), self.publish_market_fee_collector, self.allowed_swapper, ], [ self.BT.decimals(), self.dt_decimals, self.rate, self.publish_market_fee, 1, # with mint ], ) """ Attributes: MAX_FEE MIN_FEE MIN_RATE Functions: def balance() -> int: returns FRE balance :return: balance def getExchanges() -> tuple: get list of exchange contracts addresses :return: tuple of contract addresses for each exchange def generateExchangeId(bt_addr: str, dt_addr) -> str: retrieve exchange id based on a pair of basetoken-datatoken address pair :param bt_addr: address of base token :param dt_addr: address of datatoken :return: exchange id def getId() -> int: get exchange id :return: id def getNumberOfExchanges() -> int: get number of exchange contracts :return: number of exchanges The following functions are wrapped with ocean.py helpers, especially in the OneExchange class, but you can use the raw form if needed. buyDT calcBaseInGivenOutDT calcBaseOutGivenInDT collectBT collectDT collectMarketFee collectOceanFee createWithDecimals -> raw creation of the exchange contract, much easier to use from datatoken.create_exchange() getAllowedSwapper getBTSupply getDTSupply getExchange getFeesInfo getMarketFee getOPCFee getRate isActive sellDT setAllowedSwapper setRate toggleExchangeState toggleMintState updateMarketFee updateMarketFeeCollector """ @enforce_types class ExchangeDetails: def __init__(self, details_tup): """ :param:details_tup -- returned from FixedRateExchange.sol::getExchange(exchange_id) which is (exchangeOwner, datatoken, .., withMint) """ t = details_tup self.owner: str = t[0] self.datatoken: str = t[1] self.dt_decimals: int = t[2] self.base_token: str = t[3] self.bt_decimals: int = t[4] self.fixed_rate: int = t[5] self.active: bool = t[6] self.dt_supply: int = t[7] self.bt_supply: int = t[8] self.dt_balance: int = t[9] self.bt_balance: int = t[10] self.with_mint: bool = t[11] def __str__(self): s = ( f"ExchangeDetails: \n" f" datatoken = {self.datatoken}\n" f" base_token = {self.base_token}\n" f" fixed_rate (price) = {str_with_wei(self.fixed_rate)}\n" f" active = {self.active}\n" f" dt_supply = {str_with_wei(self.dt_supply)}\n" f" bt_supply = {str_with_wei(self.bt_supply)}\n" f" dt_balance = {str_with_wei(self.dt_balance)}\n" f" bt_balance = {str_with_wei(self.bt_balance)}\n" f" with_mint = {self.with_mint}\n" f" dt_decimals = {self.dt_decimals}\n" f" bt_decimals = {self.bt_decimals}\n" f" owner = {self.owner}\n" ) return s @enforce_types class ExchangeFeeInfo: def __init__(self, fees_tup): """ :param:details_tup -- returned from FixedRateExchange.sol::getFeesInfo(exchange_id) which is {(publish)market(swap)Fee, ..., oceanFeeAvailable} """ t = fees_tup self.publish_market_fee: int = t[0] self.publish_market_fee_collector: str = t[1] # address self.opc_fee: int = t[2] self.publish_market_fee_available = t[3] # in base tokens self.ocean_fee_available = t[4] # in base tokens. Goes to OPC def __str__(self): s = ( f"ExchangeFeeInfo: \n" f" publish_market_fee = {str_with_wei(self.publish_market_fee)}\n" f" publish_market_fee_available" f" = {str_with_wei(self.publish_market_fee_available)}\n" f" publish_market_fee_collector" f" = {self.publish_market_fee_collector}\n" f" opc_fee" f" = {str_with_wei(self.opc_fee)}\n" f" ocean_fee_available (to opc)" f" = {str_with_wei(self.ocean_fee_available)}\n" ) return s @enforce_types class BtNeeded: def __init__(self, tup): self.base_token_amount = tup[0] self.ocean_fee_amount = tup[1] self.publish_market_fee_amount = tup[2] self.consume_market_fee_amount = tup[3] @enforce_types class BtReceived: def __init__(self, tup): self.base_token_amount = tup[0] self.ocean_fee_amount = tup[1] self.publish_market_fee_amount = tup[2] self.consume_market_fee_amount = tup[3] @enforce_types class FixedRateExchange(ContractBase): CONTRACT_NAME = "FixedRateExchange" def get_opc_collector(self) -> str: """Returns address that collects fees for Ocean Protocol Community""" router_addr = self.router() router = FactoryRouter(self.config_dict, router_addr) return router.getOPCCollector() @enforce_types class OneExchange: """ Clean object-oriented class for a sole exchange, between two tokens. It's a bit like FixedRateExchange, but for just one exchange_id. Therefore its methods don't need the exchange_id argument. While it doesn't have a corresponding smart contract. It can be viewed as a slice of the FixedRateExchange contract. """ @enforce_types def __init__(self, FRE: FixedRateExchange, exchange_id): self._FRE = FRE self._id = exchange_id @property def FRE(self): return self._FRE @property def exchange_id(self): return self._id @property def address(self): return self._FRE.address # From here on, the methods have a 1:1 mapping to FixedRateExchange.sol. # In some cases, there's a rename for better clarity # It's easy to tell the original method name: see what this class calls. @enforce_types def BT_needed( self, DT_amt: Union[int, str], consume_market_fee: Union[int, str], full_info: bool = False, ) -> Union[int, BtNeeded]: """ Returns an int - how many BTs you need, to buy target amt of DTs. Or, for an object with all details, set full_info=True. """ tup = self._FRE.calcBaseInGivenOutDT(self._id, DT_amt, consume_market_fee) bt_needed_obj = BtNeeded(tup) if full_info: return bt_needed_obj return bt_needed_obj.base_token_amount @enforce_types def BT_received( self, DT_amt: Union[int, str], consume_market_fee: Union[int, str], full_info: bool = False, ) -> Union[int, BtReceived]: """ Returns an int - how many BTs you receive, in selling given amt of DTs. Or, for an object with all details, set full_info=True. """ tup = self._FRE.calcBaseOutGivenInDT(self._id, DT_amt, consume_market_fee) bt_recd_obj = BtReceived(tup) if full_info: return bt_recd_obj return bt_recd_obj.base_token_amount @enforce_types def buy_DT( self, datatoken_amt: Union[int, str], tx_dict: dict, max_basetoken_amt=MAX_UINT256, consume_market_fee_addr: Optional[str] = ZERO_ADDRESS, consume_market_fee: Optional[Union[int, str]] = 0, ): """ Buy datatokens via fixed-rate exchange. This wraps the smart contract method FixedRateExchange.buyDT() with a simpler interface. Params: - datatoken_amt - how many DT to buy? In wei, or str - max_basetoken_amt - maximum to spend - consume_market_fee_addr - market facilitating this swap - consume_market_fee - fee charged by market that's facilitating - tx_dict - e.g. {"from": alice_wallet} """ # import now, to avoid circular import # TODO: maybe we can move it now? from ocean_lib.models.datatoken_base import DatatokenBase details = self.details BT = DatatokenBase.get_typed(self._FRE.config_dict, details.base_token) buyer_addr = get_from_address(tx_dict) BT_needed = self.BT_needed(datatoken_amt, consume_market_fee) assert BT.balanceOf(buyer_addr) >= BT_needed, "not enough funds" tx = self._FRE.buyDT( self._id, datatoken_amt, max_basetoken_amt, consume_market_fee_addr, consume_market_fee, tx_dict, ) return tx @enforce_types def sell_DT( self, datatoken_amt: Union[int, str], tx_dict: dict, min_basetoken_amt: Union[int, str] = 0, consume_market_fee_addr: str = ZERO_ADDRESS, consume_market_fee: Union[int, str] = 0, ): """ Sell datatokens to the exchange, in return for e.g. OCEAN This wraps the smart contract method FixedRateExchange.sellDT() with a simpler interface. Main params: - datatoken_amt - how many DT to sell? In wei, or str - min_basetoken_amt - min basetoken to get back - consume_market_fee_addr - market facilitating this swap - consume_market_fee - fee charged by market that's facilitating - tx_dict - e.g. {"from": alice_wallet} """ tx = self._FRE.sellDT( self._id, datatoken_amt, min_basetoken_amt, consume_market_fee_addr, consume_market_fee, tx_dict, ) return tx @enforce_types def collect_BT(self, amount: Union[int, str], tx_dict: dict): """ This exchange collects fees denominated in base tokens, and records updates into its `bt_balance`. *This method* triggers the exchange to send `amount` fees to the datatoken's payment collector (ERC20.getPaymentCollector) 'amount' must be <= this exchange's bt_balance, of course. Anyone can call this method, since the receiver is constant. """ return self._FRE.collectBT(self._id, amount, tx_dict) @enforce_types def collect_DT(self, amount: Union[int, str], tx_dict: dict): """ This exchange collects fees denominated in datatokens, and records updates into its `dt_balance`. *This method* triggers the exchange to send `amount` fees to the datatoken's payment collector (ERC20.getPaymentCollector) 'amount' must be <= this exchange's dt_balance, of course. Anyone can call this method, since the receiver is constant. """ return self._FRE.collectDT(self._id, amount, tx_dict) @enforce_types def collect_publish_market_fee(self, tx_dict: dict): """ This exchange collects fees for the publishing market, and records updates into its `market_fee_available`. *This method* triggers the exchange to send all available market fees to this exchange's market fee collector (`market_fee_collector`). Anyone can call this method, since the receiver is constant. """ return self._FRE.collectMarketFee(self._id, tx_dict) @enforce_types def collect_opc_fee(self, tx_dict: dict): """ This exchange collects fees for the Ocean Protocol Community (OPC), and records updates into its `ocean_fee_available`. *This method* triggers the exchange to send all available OPC fees to the OPC Collector (router.getOPCCollector). Anyone can call this method, since the receiver is constant. """ return self._FRE.collectOceanFee(self._id, tx_dict) @enforce_types def update_publish_market_fee_collector(self, new_addr: str, tx_dict): """Update which address collects the publish market swap fees""" return self._FRE.updateMarketFeeCollector(self._id, new_addr, tx_dict) @enforce_types def update_publish_market_fee(self, new_amt: Union[str, int], tx_dict): """Update the value of the publish market swap fee""" return self._FRE.updateMarketFee(self._id, new_amt, tx_dict) @enforce_types def get_publish_market_fee(self) -> int: """Return the value of the publish market swap fee""" return self._FRE.getMarketFee(self._id) @enforce_types def set_rate(self, new_rate: Union[int, str], tx_dict: dict): """Set price = # base tokens needed to buy 1 datatoken""" return self._FRE.setRate(self._id, new_rate, tx_dict) @enforce_types def toggle_active(self, tx_dict: dict): """Switch whether 'active' from True <> False""" return self._FRE.toggleExchangeState(self._id, tx_dict) @enforce_types def set_allowed_swapper(self, new_addr: str, tx_dict: dict): """Set allowed swapper. ZERO_ADDRESS means anyone can swap.""" return self._FRE.setAllowedSwapper(self._id, new_addr, tx_dict) @enforce_types def get_rate(self) -> int: """Get price = # base tokens needed to buy 1 datatoken""" return self._FRE.getRate(self._id) @enforce_types def get_dt_supply(self) -> int: """Return the current supply of datatokens in this exchange""" return self._FRE.getDTSupply(self._id) @enforce_types def get_bt_supply(self) -> int: """Return the current supply of base tokens in this exchange""" return self._FRE.getBTSupply(self._id) @property def details(self) -> ExchangeDetails: """Get all the exchange's details, as an object""" tup = self._FRE.getExchange(self._id) return ExchangeDetails(tup) @enforce_types def get_allowed_swapper(self) -> str: """Get allowed swapper. ZERO_ADDRESS means anyone can swap.""" return self._FRE.getAllowedSwapper(self._id) @property def exchange_fees_info(self) -> ExchangeFeeInfo: """Get fee information for this exchange, as an object""" tup = self._FRE.getFeesInfo(self._id) return ExchangeFeeInfo(tup) @enforce_types def is_active(self) -> bool: """Get whether exchange is 'active'""" return self._FRE.isActive(self._id) ================================================ FILE: ocean_lib/models/test/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/models/test/conftest.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from conftest_ganache import * ================================================ FILE: ocean_lib/models/test/test_data_nft.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json from base64 import b64decode import pytest from web3 import Web3 from web3.logs import DISCARD from ocean_lib.models.data_nft import DataNFTPermissions from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.models.datatoken_base import ( DatatokenArguments, DatatokenBase, TokenFeeInfo, ) from ocean_lib.ocean.util import get_address_of_type, to_wei BLOB = "f8929916089218bdb4aa78c3ecd16633afd44b8aef89299160" @pytest.mark.unit def test_permissions( publisher_wallet, consumer_wallet, another_consumer_wallet, config, data_nft, ): """Tests permissions' functions.""" assert data_nft.name() == "NFT" assert data_nft.symbol() == "NFTSYMBOL" assert data_nft.balanceOf(publisher_wallet.address) == 1 # Tests if the NFT was initialized assert data_nft.isInitialized() # Tests adding a manager successfully data_nft.addManager(consumer_wallet.address, {"from": publisher_wallet}) assert data_nft.getPermissions(consumer_wallet.address)[DataNFTPermissions.MANAGER] token_uri = data_nft.tokenURI(1).replace("data:application/json;base64,", "") decoded_token_uri = json.loads(b64decode(token_uri)) assert decoded_token_uri["name"] == "NFT" assert decoded_token_uri["symbol"] == "NFTSYMBOL" assert decoded_token_uri["background_color"] == "141414" assert decoded_token_uri["image_data"].startswith("data:image/svg+xm") # Tests failing clearing permissions with pytest.raises(Exception, match="not NFTOwner"): data_nft.cleanPermissions({"from": another_consumer_wallet}) # Tests clearing permissions data_nft.addToCreateERC20List(publisher_wallet.address, {"from": publisher_wallet}) data_nft.addToCreateERC20List( another_consumer_wallet.address, {"from": publisher_wallet} ) assert data_nft.getPermissions(publisher_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] assert data_nft.getPermissions(another_consumer_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] # Still is not the NFT owner, cannot clear permissions then with pytest.raises(Exception, match="not NFTOwner"): data_nft.cleanPermissions({"from": another_consumer_wallet}) data_nft.cleanPermissions({"from": publisher_wallet}) assert not ( data_nft.getPermissions(publisher_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] ) assert not ( data_nft.getPermissions(consumer_wallet.address)[DataNFTPermissions.MANAGER] ) assert not ( data_nft.getPermissions(another_consumer_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] ) # Tests failing adding a new manager by another user different from the NFT owner data_nft.addManager(publisher_wallet.address, {"from": publisher_wallet}) assert data_nft.getPermissions(publisher_wallet.address)[DataNFTPermissions.MANAGER] assert not ( data_nft.getPermissions(consumer_wallet.address)[DataNFTPermissions.MANAGER] ) with pytest.raises(Exception, match="not NFTOwner"): data_nft.addManager(another_consumer_wallet.address, {"from": consumer_wallet}) assert not ( data_nft.getPermissions(another_consumer_wallet.address)[ DataNFTPermissions.MANAGER ] ) # Tests removing manager data_nft.addManager(consumer_wallet.address, {"from": publisher_wallet}) assert data_nft.getPermissions(consumer_wallet.address)[DataNFTPermissions.MANAGER] data_nft.removeManager(consumer_wallet.address, {"from": publisher_wallet}) assert not ( data_nft.getPermissions(consumer_wallet.address)[DataNFTPermissions.MANAGER] ) # Tests failing removing a manager if it has not the NFT owner role data_nft.addManager(consumer_wallet.address, {"from": publisher_wallet}) assert data_nft.getPermissions(consumer_wallet.address)[DataNFTPermissions.MANAGER] with pytest.raises(Exception, match="not NFTOwner"): data_nft.removeManager(publisher_wallet.address, {"from": consumer_wallet}) assert data_nft.getPermissions(publisher_wallet.address)[DataNFTPermissions.MANAGER] # Tests removing the NFT owner from the manager role data_nft.removeManager(publisher_wallet.address, {"from": publisher_wallet}) assert not ( data_nft.getPermissions(publisher_wallet.address)[DataNFTPermissions.MANAGER] ) data_nft.addManager(publisher_wallet.address, {"from": publisher_wallet}) assert data_nft.getPermissions(publisher_wallet.address)[DataNFTPermissions.MANAGER] # Tests failing calling execute_call function if the user is not manager assert not ( data_nft.getPermissions(another_consumer_wallet.address)[ DataNFTPermissions.MANAGER ] ) with pytest.raises(Exception, match="NOT MANAGER"): data_nft.executeCall( 0, consumer_wallet.address, 10, Web3.to_hex(text="SomeData"), {"from": another_consumer_wallet}, ) # Tests calling execute_call with a manager role assert data_nft.getPermissions(publisher_wallet.address)[DataNFTPermissions.MANAGER] tx = data_nft.executeCall( 0, consumer_wallet.address, 10, Web3.to_hex(text="SomeData"), {"from": consumer_wallet}, ) assert tx, "Could not execute call to consumer." # Tests setting new data data_nft.addTo725StoreList(consumer_wallet.address, {"from": publisher_wallet}) assert data_nft.getPermissions(consumer_wallet.address)[DataNFTPermissions.STORE] data_nft.setNewData( b"ARBITRARY_KEY", b"SomeData", {"from": consumer_wallet}, ) assert data_nft.getData(b"ARBITRARY_KEY").hex() == b"SomeData".hex() # Tests failing setting new data if user has not STORE UPDATER role. assert not ( data_nft.getPermissions(another_consumer_wallet.address)[ DataNFTPermissions.STORE ] ) with pytest.raises(Exception, match="NOT STORE UPDATER"): data_nft.setNewData( b"ARBITRARY_KEY", b"SomeData", {"from": another_consumer_wallet}, ) # Tests failing setting ERC20 data with pytest.raises(Exception, match="NOT ERC20 Contract"): data_nft.setDataERC20( b"FOO_KEY", b"SomeData", {"from": consumer_wallet}, ) assert data_nft.getData(b"FOO_KEY").hex() == b"".hex() def test_add_and_remove_permissions( publisher_wallet, consumer_wallet, config, data_nft ): # Assert consumer has no permissions permissions = data_nft.getPermissions(consumer_wallet.address) assert not permissions[DataNFTPermissions.MANAGER] assert not permissions[DataNFTPermissions.DEPLOY_DATATOKEN] assert not permissions[DataNFTPermissions.UPDATE_METADATA] assert not permissions[DataNFTPermissions.STORE] # Grant consumer all permissions, one by one data_nft.addManager(consumer_wallet.address, {"from": publisher_wallet}) data_nft.addToCreateERC20List(consumer_wallet.address, {"from": publisher_wallet}) data_nft.addToMetadataList(consumer_wallet.address, {"from": publisher_wallet}) data_nft.addTo725StoreList(consumer_wallet.address, {"from": publisher_wallet}) # Assert consumer has all permissions permissions = data_nft.getPermissions(consumer_wallet.address) assert permissions[DataNFTPermissions.MANAGER] assert permissions[DataNFTPermissions.DEPLOY_DATATOKEN] assert permissions[DataNFTPermissions.UPDATE_METADATA] assert permissions[DataNFTPermissions.STORE] # Revoke all consumer permissions, one by one data_nft.removeManager(consumer_wallet.address, {"from": publisher_wallet}) data_nft.removeFromCreateERC20List( consumer_wallet.address, {"from": publisher_wallet} ) data_nft.removeFromMetadataList(consumer_wallet.address, {"from": publisher_wallet}) data_nft.removeFrom725StoreList(consumer_wallet.address, {"from": publisher_wallet}) # Assert consumer has no permissions permissions = data_nft.getPermissions(consumer_wallet.address) assert not permissions[DataNFTPermissions.MANAGER] assert not permissions[DataNFTPermissions.DEPLOY_DATATOKEN] assert not permissions[DataNFTPermissions.UPDATE_METADATA] assert not permissions[DataNFTPermissions.STORE] @pytest.mark.unit def test_success_update_metadata(publisher_wallet, consumer_wallet, config, data_nft): """Tests updating the metadata flow.""" assert not ( data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.UPDATE_METADATA ] ) data_nft.addToMetadataList(consumer_wallet.address, {"from": publisher_wallet}) metadata_info = data_nft.getMetaData() assert not metadata_info[3] receipt = data_nft.setMetaData( 1, "http://myprovider:8030", b"0x123", Web3.to_bytes(hexstr=BLOB), Web3.to_bytes(hexstr=BLOB), Web3.to_bytes(hexstr=BLOB), [], {"from": consumer_wallet}, ) event = data_nft.contract.events.MetadataCreated().process_receipt( receipt, errors=DISCARD )[0] assert event.args.decryptorUrl == "http://myprovider:8030" metadata_info = data_nft.getMetaData() assert metadata_info[3] assert metadata_info[0] == "http://myprovider:8030" receipt = data_nft.setMetaData( 1, "http://foourl", b"0x123", Web3.to_bytes(hexstr=BLOB), Web3.to_bytes(hexstr=BLOB), Web3.to_bytes(hexstr=BLOB), [], {"from": consumer_wallet}, ) event = data_nft.contract.events.MetadataUpdated().process_receipt( receipt, errors=DISCARD )[0] assert event.args.decryptorUrl == "http://foourl" metadata_info = data_nft.getMetaData() assert metadata_info[3] assert metadata_info[0] == "http://foourl" # Update tokenURI and set metadata in one call receipt = data_nft.setMetaDataAndTokenURI( ( 1, "http://foourl", b"0x123", Web3.to_bytes(hexstr=BLOB), Web3.to_bytes(hexstr=BLOB), Web3.to_bytes(hexstr=BLOB), 1, "https://anothernewurl.com/nft/", [], ), {"from": publisher_wallet}, ) event = data_nft.contract.events.TokenURIUpdate().process_receipt( receipt, errors=DISCARD )[0] assert event.args.tokenURI == "https://anothernewurl.com/nft/" assert event.args.updatedBy == publisher_wallet.address event = data_nft.contract.events.MetadataUpdated().process_receipt( receipt, errors=DISCARD )[0] assert event.args.decryptorUrl == "http://foourl" metadata_info = data_nft.getMetaData() assert metadata_info[3] assert metadata_info[0] == "http://foourl" # Consumer self-revokes permission to update metadata data_nft.removeFromMetadataList(consumer_wallet.address, {"from": consumer_wallet}) assert not data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.UPDATE_METADATA ] def test_fails_update_metadata(consumer_wallet, publisher_wallet, config, data_nft): """Tests failure of calling update metadata function when the role of the user is not METADATA UPDATER.""" assert not ( data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.UPDATE_METADATA ] ) with pytest.raises(Exception, match="NOT METADATA_ROLE"): data_nft.setMetaData( 1, "http://myprovider:8030", b"0x123", BLOB.encode("utf-8"), BLOB, BLOB, [], {"from": consumer_wallet}, ) @pytest.mark.unit def test_create_datatoken( publisher_wallet, consumer_wallet, config, data_nft_factory: DataNFTFactoryContract, data_nft, ): """Tests calling create an ERC20 by the owner.""" assert data_nft.getPermissions(publisher_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] datatoken = data_nft.create_datatoken( {"from": publisher_wallet}, "DT1", "DT1Symbol", fee_manager=consumer_wallet.address, ) assert datatoken, "Could not create ERC20." dt_ent = data_nft.create_datatoken( {"from": publisher_wallet}, datatoken_args=DatatokenArguments( template_index=2, name="Datatoken2DT1", symbol="Datatoken2DT1Symbol", minter=publisher_wallet.address, fee_manager=consumer_wallet.address, bytess=[b""], cap=to_wei(0.1), ), ) assert dt_ent, "Could not create datatoken template 2 with explicit parameters" dt_ent = data_nft.create_datatoken( {"from": publisher_wallet}, name="Datatoken2DT1", symbol="Datatoken2DT1Symbol", cap=to_wei(0.1), ) assert dt_ent, "Could not create datatoken template 2 with implicit parameters." def test_create_datatoken_with_usdc_order_fee( config: dict, publisher_wallet, data_nft_factory: DataNFTFactoryContract, data_nft ): """Create an ERC20 with order fees ( 5 USDC, going to publishMarketAddress)""" usdc = DatatokenBase.get_typed(config, get_address_of_type(config, "MockUSDC")) publish_market_order_fee_amount_in_wei = to_wei(5) dt = data_nft.create_datatoken( {"from": publisher_wallet}, DatatokenArguments( name="DT1", symbol="DT1Symbol", publish_market_order_fees=TokenFeeInfo( address=publisher_wallet.address, token=usdc.address, amount=publish_market_order_fee_amount_in_wei, ), ), ) # Check publish fee info publish_market_fees = dt.get_publish_market_order_fees() assert publish_market_fees.address == publisher_wallet.address assert publish_market_fees.token == usdc.address assert publish_market_fees.amount == publish_market_order_fee_amount_in_wei @pytest.mark.unit def test_create_datatoken_with_non_owner( publisher_wallet, consumer_wallet, data_nft_factory: DataNFTFactoryContract, config, data_nft, ): """Tests creating an ERC20 token by wallet other than nft owner""" # Assert consumer cannot create ERC20 assert not data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] # Grant consumer permission to create ERC20 data_nft.addToCreateERC20List(consumer_wallet.address, {"from": publisher_wallet}) assert data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] # Consumer creates ERC20 dt = data_nft.create_datatoken( {"from": consumer_wallet}, DatatokenArguments( name="DT1", symbol="DT1Symbol", minter=publisher_wallet.address, fee_manager=publisher_wallet.address, ), ) assert dt, "Failed to create ERC20 token." # Consumer self-revokes permission to create ERC20 data_nft.removeFromCreateERC20List( consumer_wallet.address, {"from": consumer_wallet} ) assert not data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] @pytest.mark.unit def test_fail_creating_erc20( consumer_wallet, publisher_wallet, config, data_nft, ): """Tests failure for creating ERC20 token.""" assert not ( data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] ) with pytest.raises(Exception, match="NOT ERC20DEPLOYER_ROLE"): data_nft.create_datatoken( {"from": consumer_wallet}, name="DT1", symbol="DT1Symbol", minter=publisher_wallet.address, ) @pytest.mark.unit def test_erc721_datatoken_functions( publisher_wallet, consumer_wallet, config, data_NFT_and_DT, ): """Tests ERC721 Template functions for ERC20 tokens.""" data_nft, datatoken = data_NFT_and_DT assert len(data_nft.getTokensList()) == 1 assert data_nft.isDeployed(datatoken.address) assert not data_nft.isDeployed(consumer_wallet.address) receipt = data_nft.setTokenURI( 1, "https://newurl.com/nft/", {"from": publisher_wallet}, ) registered_event = data_nft.contract.events.TokenURIUpdate().process_receipt( receipt, errors=DISCARD )[0] assert registered_event, "Cannot find TokenURIUpdate event." assert registered_event.args.updatedBy == publisher_wallet.address assert registered_event.args.tokenID == 1 assert registered_event.args.blockNumber == receipt.blockNumber assert data_nft.tokenURI(1) == "https://newurl.com/nft/" assert data_nft.tokenURI(1) == registered_event.args.tokenURI # Tests failing setting token URI by another user with pytest.raises(Exception, match="not NFTOwner"): data_nft.setTokenURI( 1, "https://foourl.com/nft/", {"from": consumer_wallet}, ) # Tests transfer functions datatoken.mint( consumer_wallet.address, to_wei(0.2), {"from": publisher_wallet}, ) assert datatoken.balanceOf(consumer_wallet.address) == to_wei(0.2) assert data_nft.ownerOf(1) == publisher_wallet.address data_nft.transferFrom( publisher_wallet.address, consumer_wallet.address, 1, {"from": publisher_wallet}, ) assert data_nft.balanceOf(publisher_wallet.address) == 0 assert data_nft.ownerOf(1) == consumer_wallet.address assert data_nft.getPermissions(consumer_wallet.address)[ DataNFTPermissions.DEPLOY_DATATOKEN ] data_nft.create_datatoken( {"from": consumer_wallet}, name="DT1", symbol="DT1Symbol", minter=publisher_wallet.address, ) with pytest.raises(Exception, match="NOT MINTER"): datatoken.mint( consumer_wallet.address, to_wei(1), {"from": consumer_wallet}, ) datatoken.addMinter(consumer_wallet.address, {"from": consumer_wallet}) datatoken.mint( consumer_wallet.address, to_wei(0.2), {"from": consumer_wallet}, ) assert datatoken.balanceOf(consumer_wallet.address) == to_wei(0.4) @pytest.mark.unit def test_fail_transfer_function(consumer_wallet, publisher_wallet, config, data_nft): """Tests failure of using the transfer functions.""" with pytest.raises( Exception, match="transfer caller is not owner nor approved", ): data_nft.transferFrom( publisher_wallet.address, consumer_wallet.address, 1, {"from": consumer_wallet}, ) # Tests for safe transfer as well with pytest.raises( Exception, match="transfer caller is not owner nor approved", ): data_nft.safeTransferFrom( publisher_wallet.address, consumer_wallet.address, 1, {"from": consumer_wallet}, ) def test_transfer_nft( config, publisher_wallet, consumer_wallet, factory_router, data_nft_factory, publisher_ocean, ): """Tests transferring the NFT before deploying an ERC20, a pool, a FRE.""" data_nft = data_nft_factory.create( {"from": publisher_wallet}, "NFT to TRANSFER", "NFTtT", additional_datatoken_deployer=consumer_wallet.address, ) assert data_nft.name() == "NFT to TRANSFER" assert data_nft.symbol() == "NFTtT" receipt = data_nft.safeTransferFrom( publisher_wallet.address, consumer_wallet.address, 1, {"from": publisher_wallet}, ) transfer_event = data_nft.contract.events.Transfer().process_receipt( receipt, errors=DISCARD )[0] assert getattr(transfer_event.args, "from") == publisher_wallet.address assert transfer_event.args.to == consumer_wallet.address assert data_nft.balanceOf(consumer_wallet.address) == 1 assert data_nft.balanceOf(publisher_wallet.address) == 0 assert data_nft.isERC20Deployer(consumer_wallet.address) assert data_nft.ownerOf(1) == consumer_wallet.address # Consumer is not the additional ERC20 deployer, but will be after the NFT transfer data_nft = data_nft_factory.create({"from": publisher_wallet}, "NFT1", "NFT") receipt = data_nft.safeTransferFrom( publisher_wallet.address, consumer_wallet.address, 1, {"from": publisher_wallet}, ) transfer_event = data_nft.contract.events.Transfer().process_receipt( receipt, errors=DISCARD )[0] assert getattr(transfer_event.args, "from") == publisher_wallet.address assert transfer_event.args.to == consumer_wallet.address assert data_nft.isERC20Deployer(consumer_wallet.address) # Creates an ERC20 datatoken = data_nft.create_datatoken( {"from": consumer_wallet}, "DT1", "DT1Symbol", publish_market_order_fees=TokenFeeInfo( address=publisher_wallet.address, ), ) assert datatoken, "Failed to create ERC20 token." assert not datatoken.isMinter(publisher_wallet.address) assert datatoken.isMinter(consumer_wallet.address) datatoken.addMinter(publisher_wallet.address, {"from": consumer_wallet}) assert datatoken.getPermissions(publisher_wallet.address)[ 0 ] # publisher is minter now OCEAN = publisher_ocean.OCEAN_token OCEAN.approve(factory_router.address, to_wei(10000), {"from": consumer_wallet}) # Make consumer the publish market order fee address instead of publisher receipt = datatoken.setPublishingMarketFee( consumer_wallet.address, OCEAN.address, to_wei(1), {"from": publisher_wallet}, ) set_publishing_fee_event = ( datatoken.contract.events.PublishMarketFeeChanged().process_receipt( receipt, errors=DISCARD )[0] ) assert set_publishing_fee_event, "Cannot find PublishMarketFeeChanged event." publish_fees = datatoken.get_publish_market_order_fees() assert publish_fees.address == consumer_wallet.address assert publish_fees.token == OCEAN.address assert publish_fees.amount == to_wei(1) def test_nft_transfer_with_fre( config, OCEAN, publisher_wallet, consumer_wallet, data_NFT_and_DT, ): """Tests transferring the NFT before deploying an ERC20, a FRE.""" data_nft, datatoken = data_NFT_and_DT assert datatoken.isMinter(publisher_wallet.address) # The NFT owner (publisher) has ERC20 deployer role & can deploy an exchange exchange = datatoken.create_exchange( rate=to_wei(1), base_token_addr=OCEAN.address, publish_market_fee=to_wei(0.01), tx_dict={"from": publisher_wallet}, ) # Exchange should have supply and fees setup # (Don't test thoroughly here, since exchange has its own unit tests) details = exchange.details assert details.owner == publisher_wallet.address assert details.datatoken == datatoken.address assert details.fixed_rate == to_wei(1) # Now do a transfer receipt = data_nft.safeTransferFrom( publisher_wallet.address, consumer_wallet.address, 1, {"from": publisher_wallet}, ) transfer_event = data_nft.contract.events.Transfer().process_receipt( receipt, errors=DISCARD )[0] assert getattr(transfer_event.args, "from") == publisher_wallet.address assert transfer_event.args.to == consumer_wallet.address assert data_nft.balanceOf(consumer_wallet) == 1 assert data_nft.balanceOf(publisher_wallet) == 0 assert data_nft.isERC20Deployer(consumer_wallet) assert data_nft.ownerOf(1) == consumer_wallet.address permissions = datatoken.getPermissions(consumer_wallet) assert not permissions[0] # the newest owner is not the minter datatoken.addMinter(consumer_wallet, {"from": consumer_wallet}) assert datatoken.permissions(consumer_wallet)[0] # Consumer wallet has not become the owner of the publisher's exchange details = exchange.details assert details.owner == publisher_wallet.address assert details.active @pytest.mark.unit def test_fail_create_datatoken( config, publisher_wallet, consumer_wallet, another_consumer_wallet, data_nft_factory ): """Tests multiple failures for creating ERC20 token.""" data_nft = data_nft_factory.create({"from": publisher_wallet}, "DT1", "DTSYMBOL") data_nft.addToCreateERC20List(consumer_wallet.address, {"from": publisher_wallet}) # Should fail to create a specific ERC20 Template if the index is ZERO with pytest.raises(Exception, match="Template index doesnt exist"): data_nft.create_datatoken( {"from": consumer_wallet}, template_index=0, name="DT1", symbol="DT1Symbol", ) # Should fail to create a specific ERC20 Template if the index doesn't exist with pytest.raises(Exception, match="Template index doesnt exist"): data_nft.create_datatoken( {"from": consumer_wallet}, template_index=4, name="DT1", symbol="DT1Symbol", ) # Should fail to create a specific ERC20 Template if the user is not added on the ERC20 deployers list assert data_nft.getPermissions(another_consumer_wallet.address)[1] is False with pytest.raises(Exception, match="NOT ERC20DEPLOYER_ROLE"): data_nft.create_datatoken( {"from": another_consumer_wallet}, template_index=1, name="DT1", symbol="DT1Symbol", ) @pytest.mark.unit def test_datatoken_cap(publisher_wallet, consumer_wallet, data_nft_factory): # create NFT with ERC20 with pytest.raises(Exception, match="Cap is needed for Datatoken Template 2"): DatatokenArguments(template_index=2, name="DTB1", symbol="EntDT1Symbol") @pytest.mark.unit def test_nft_owner_transfer(config, publisher_wallet, consumer_wallet, data_NFT_and_DT): """Test erc721 ownership transfer on token transfer""" data_nft, datatoken = data_NFT_and_DT assert data_nft.ownerOf(1) == publisher_wallet.address with pytest.raises(Exception, match="transfer of token that is not own"): data_nft.transferFrom( consumer_wallet.address, publisher_wallet.address, 1, {"from": publisher_wallet}, ) data_nft.transferFrom( publisher_wallet.address, consumer_wallet.address, 1, {"from": publisher_wallet} ) assert data_nft.balanceOf(publisher_wallet.address) == 0 assert data_nft.ownerOf(1) == consumer_wallet.address # Owner is not NFT owner anymore, nor has any other role, neither older users with pytest.raises(Exception, match="NOT ERC20DEPLOYER_ROLE"): data_nft.create_datatoken( {"from": publisher_wallet}, name="DT1", symbol="DT1Symbol", ) with pytest.raises(Exception, match="NOT MINTER"): datatoken.mint(publisher_wallet.address, 10, {"from": publisher_wallet}) # NewOwner now owns the NFT, is already Manager by default and has all roles data_nft.create_datatoken( {"from": consumer_wallet}, name="DT1", symbol="DT1Symbol", ) datatoken.addMinter(consumer_wallet.address, {"from": consumer_wallet}) datatoken.mint(consumer_wallet.address, 20, {"from": consumer_wallet}) assert datatoken.balanceOf(consumer_wallet.address) == 20 def test_set_get_data(data_nft, alice): # Key-value pair key = "fav_color" value = "blue" # set data data_nft.set_data(key, value, {"from": alice}) # retrieve data value2 = data_nft.get_data(key) # test assert value2 == value ================================================ FILE: ocean_lib/models/test/test_data_nft_factory.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from web3.logs import DISCARD from web3.main import Web3 from ocean_lib.models.data_nft import DataNFT, DataNFTArguments from ocean_lib.models.datatoken1 import Datatoken1 from ocean_lib.models.datatoken_base import ( DatatokenArguments, DatatokenBase, TokenFeeInfo, ) from ocean_lib.models.dispenser import Dispenser, DispenserArguments from ocean_lib.models.fixed_rate_exchange import ExchangeArguments from ocean_lib.ocean.util import create_checksum, get_address_of_type, to_wei from ocean_lib.structures.abi_tuples import OrderData, ReuseOrderData from ocean_lib.web3_internal.constants import ZERO_ADDRESS from ocean_lib.web3_internal.utils import split_signature from tests.resources.helper_functions import get_non_existent_nft_template @pytest.mark.unit def test_nft_creation( config, publisher_wallet, consumer_wallet, data_nft_factory, ): """Tests the utils functions.""" data_nft = data_nft_factory.create({"from": publisher_wallet}, "DT1", "DTSYMBOL") assert data_nft.name() == "DT1" assert data_nft.symbol() == "DTSYMBOL" # Tests current NFT count current_nft_count = data_nft_factory.getCurrentNFTCount() data_nft = data_nft_factory.create({"from": publisher_wallet}, "DT2", "DTSYMBOL1") assert data_nft_factory.getCurrentNFTCount() == current_nft_count + 1 # Tests get NFT template nft_template_address = get_address_of_type(config, DataNFT.CONTRACT_NAME, "1") nft_template = data_nft_factory.getNFTTemplate(1) assert nft_template[0] == nft_template_address assert nft_template[1] is True # Tests creating successfully an ERC20 token data_nft.addToCreateERC20List(consumer_wallet.address, {"from": publisher_wallet}) datatoken = data_nft.create_datatoken( {"from": publisher_wallet}, DatatokenArguments( "DT1P", "DT1Symbol", fee_manager=consumer_wallet.address, ), ) assert datatoken, "Failed to create ERC20 token." # Tests templateCount function (one of them should be the Enterprise template) assert data_nft_factory.templateCount() == 3 # Tests datatoken template list datatoken_template_address = get_address_of_type( config, Datatoken1.CONTRACT_NAME, "1" ) template = data_nft_factory.getTokenTemplate(1) assert template[0] == datatoken_template_address assert template[1] is True # Tests current token template (one of them should be the Enterprise template) assert data_nft_factory.getCurrentTemplateCount() == 3 @pytest.mark.unit def test_combo_functions( config, publisher_wallet, consumer_wallet, data_nft_factory, ): """Tests the utils functions.""" # Tests creating NFT with ERC20 successfully data_nft_token2, datatoken2 = data_nft_factory.create_with_erc20( DataNFTArguments("72120Bundle", "72Bundle"), DatatokenArguments( "DTB1", "DT1Symbol", fee_manager=consumer_wallet.address, ), {"from": publisher_wallet}, ) assert data_nft_token2.name() == "72120Bundle" assert data_nft_token2.symbol() == "72Bundle" assert datatoken2.name() == "DTB1" assert datatoken2.symbol() == "DT1Symbol" # Tests creating NFT with ERC20 and with Fixed Rate Exchange successfully. fixed_rate_address = get_address_of_type(config, "FixedPrice") # Create ERC20 data token for fees. datatoken = data_nft_token2.create_datatoken( {"from": publisher_wallet}, DatatokenArguments( "DT1P", "DT1SymbolP", fee_manager=consumer_wallet.address, publish_market_order_fees=TokenFeeInfo( address=publisher_wallet.address, token=ZERO_ADDRESS, amount=to_wei(0.0005), ), ), ) assert datatoken, "Failed to create ERC20 token." fee_datatoken_address = datatoken.address ( data_nft_token4, datatoken4, one_fixed_rate, ) = data_nft_factory.create_with_erc20_and_fixed_rate( DataNFTArguments("72120Bundle", "72Bundle"), DatatokenArguments( "DTWithPool", "DTP", fee_manager=consumer_wallet.address, ), ExchangeArguments( base_token_addr=fee_datatoken_address, publish_market_fee_collector=consumer_wallet.address, rate=to_wei(1), publish_market_fee=to_wei(0.001), dt_decimals=18, ), tx_dict={"from": publisher_wallet}, ) assert data_nft_token4.name() == "72120Bundle" assert data_nft_token4.symbol() == "72Bundle" assert datatoken4.name() == "DTWithPool" assert datatoken4.symbol() == "DTP" assert one_fixed_rate.address == fixed_rate_address # Tests creating NFT with ERC20 and with Dispenser successfully. dispenser_address = get_address_of_type(config, Dispenser.CONTRACT_NAME) data_nft_token5, datatoken5 = data_nft_factory.create_with_erc20_and_dispenser( DataNFTArguments("72120Bundle", "72Bundle"), DatatokenArguments( "DTWithPool", "DTP", fee_manager=consumer_wallet.address, ), DispenserArguments(to_wei(1), to_wei(1)), tx_dict={"from": publisher_wallet}, ) assert data_nft_token5.name() == "72120Bundle" assert data_nft_token5.symbol() == "72Bundle" assert datatoken5.name() == "DTWithPool" assert datatoken5.symbol() == "DTP" _ = Dispenser(config, dispenser_address) # Create a new erc721 with metadata in one single call and get address data_nft = data_nft_factory.create_with_metadata( DataNFTArguments("72120Bundle", "72Bundle"), metadata_state=1, metadata_decryptor_url="http://myprovider:8030", metadata_decryptor_address=b"0x123", metadata_flags=bytes(0), metadata_data=Web3.to_hex(text="my cool metadata."), metadata_data_hash=create_checksum("my cool metadata."), metadata_proofs=[], tx_dict={"from": publisher_wallet}, ) assert ( data_nft.name() == "72120Bundle" ), "NFT name doesn't match with the expected one." metadata_info = data_nft.getMetaData() assert metadata_info[3] is True assert metadata_info[0] == "http://myprovider:8030" @pytest.mark.unit def test_start_multiple_order( config, publisher_wallet, consumer_wallet, another_consumer_wallet, data_nft_factory ): """Tests the utils functions.""" data_nft = data_nft_factory.create({"from": publisher_wallet}, "DT1", "DTSYMBOL") assert data_nft.name() == "DT1" assert data_nft.symbol() == "DTSYMBOL" assert data_nft_factory.check_nft(data_nft.address) # Tests current NFT count current_nft_count = data_nft_factory.getCurrentNFTCount() data_nft = data_nft_factory.create( {"from": publisher_wallet}, DataNFTArguments("DT2", "DTSYMBOL1") ) assert data_nft.name() == "DT2" assert data_nft_factory.getCurrentNFTCount() == current_nft_count + 1 # Tests get NFT template nft_template_address = get_address_of_type(config, DataNFT.CONTRACT_NAME, "1") nft_template = data_nft_factory.getNFTTemplate(1) assert nft_template[0] == nft_template_address assert nft_template[1] is True # Tests creating successfully an ERC20 token data_nft.addToCreateERC20List(consumer_wallet.address, {"from": publisher_wallet}) datatoken = data_nft.create_datatoken( {"from": consumer_wallet}, DatatokenArguments( name="DT1", symbol="DT1Symbol", minter=publisher_wallet.address, ), ) assert datatoken, "Failed to create ERC20 token." # Tests templateCount function (one of them should be the Enterprise template) assert data_nft_factory.templateCount() == 3 # Tests datatoken template list datatoken_template_address = get_address_of_type( config, Datatoken1.CONTRACT_NAME, "1" ) template = data_nft_factory.getTokenTemplate(1) assert template[0] == datatoken_template_address assert template[1] is True # Tests current token template (one of them should be the Enterprise template) assert data_nft_factory.getCurrentTemplateCount() == 3 # Tests datatoken can be checked as deployed by the factory assert data_nft_factory.check_datatoken(datatoken.address) # Tests starting multiple token orders successfully datatoken = DatatokenBase.get_typed(config, datatoken.address) dt_amount = to_wei(2) mock_dai_contract_address = get_address_of_type(config, "MockDAI") assert datatoken.balanceOf(consumer_wallet.address) == 0 datatoken.addMinter(consumer_wallet.address, {"from": publisher_wallet}) datatoken.mint(consumer_wallet.address, dt_amount, {"from": consumer_wallet}) assert datatoken.balanceOf(consumer_wallet.address) == dt_amount datatoken.approve(data_nft_factory.address, dt_amount, {"from": consumer_wallet}) datatoken.setPaymentCollector( another_consumer_wallet.address, {"from": publisher_wallet} ) provider_fee_token = mock_dai_contract_address provider_fee_amount = 0 provider_fee_address = publisher_wallet.address # provider_data = json.dumps({"timeout": 0}, separators=(",", ":")) provider_data = b"\x00" message = Web3.solidity_keccak( ["bytes", "address", "address", "uint256", "uint256"], [ provider_data, provider_fee_address, provider_fee_token, provider_fee_amount, 0, ], ) signed = config["web3_instance"].eth.sign(provider_fee_address, data=message) signature = split_signature(signed) order_data = OrderData( datatoken.address, consumer_wallet.address, 1, ( provider_fee_address, provider_fee_token, provider_fee_amount, signature.v, signature.r, signature.s, 0, provider_data, ), ( consumer_wallet.address, mock_dai_contract_address, 0, ), ) orders = [order_data, order_data] receipt = data_nft_factory.start_multiple_token_order( orders, {"from": consumer_wallet} ) registered_erc20_start_order_event = ( datatoken.contract.events.OrderStarted().process_receipt( receipt, errors=DISCARD )[0] ) assert receipt, "Failed starting multiple token orders." assert registered_erc20_start_order_event.args.consumer == consumer_wallet.address assert datatoken.balanceOf(consumer_wallet.address) == 0 assert datatoken.balanceOf(datatoken.getPaymentCollector()) == (dt_amount * 0.97) reuse_order = ReuseOrderData( order_data.token_address, receipt.transactionHash, ( provider_fee_address, provider_fee_token, provider_fee_amount, signature.v, signature.r, signature.s, 0, provider_data, ), ) receipt = data_nft_factory.reuse_multiple_token_order( [reuse_order], {"from": consumer_wallet} ) reused_event = datatoken.contract.events.OrderReused().process_receipt( receipt, errors=DISCARD )[0] assert reused_event @pytest.mark.unit def test_fail_get_templates(data_nft_factory): """Tests multiple failures for getting tokens' templates.""" # Should fail to get the Datatoken template if index = 0 with pytest.raises(Exception, match="Template index doesnt exist"): data_nft_factory.getTokenTemplate(0) # Should fail to get the Datatoken template if index > templateCount with pytest.raises(Exception, match="Template index doesnt exist"): data_nft_factory.getTokenTemplate(4) @pytest.mark.unit def test_nonexistent_template_index(data_nft_factory, publisher_wallet): """Test erc721 non existent template creation fail""" non_existent_nft_template = get_non_existent_nft_template( data_nft_factory, check_first=10 ) assert non_existent_nft_template >= 0, "Non existent NFT template not found." with pytest.raises(Exception, match="Template index doesnt exist"): data_nft_factory.create( {"from": publisher_wallet}, DataNFTArguments( "DT1", "DTSYMBOL", template_index=non_existent_nft_template ), ) ================================================ FILE: ocean_lib/models/test/test_datatoken.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from web3.logs import DISCARD from web3.main import Web3 from ocean_lib.models.datatoken_base import DatatokenRoles, TokenFeeInfo from ocean_lib.ocean.util import get_address_of_type, to_wei from ocean_lib.web3_internal.constants import MAX_UINT256 from tests.resources.helper_functions import get_mock_provider_fees @pytest.mark.unit def test_main( publisher_wallet, consumer_wallet, another_consumer_wallet, data_NFT_and_DT ): """Tests successful function calls""" data_nft, datatoken = data_NFT_and_DT # Check datatoken params assert datatoken.getId() == 1 assert datatoken.name() == "DT1" assert datatoken.symbol() == "DT1Symbol" assert datatoken.decimals() == 18 assert datatoken.cap() == MAX_UINT256 # Check data NFT address assert datatoken.getERC721Address() == data_nft.address # Check that the Datatoken contract is initialized assert datatoken.isInitialized() # Check publish market payment collector assert datatoken.getPaymentCollector() == publisher_wallet.address # Set payment collector to consumer datatoken.setPaymentCollector( consumer_wallet.address, {"from": publisher_wallet}, ) assert datatoken.getPaymentCollector() == consumer_wallet.address # Check minter permissions assert datatoken.getPermissions(publisher_wallet.address)[DatatokenRoles.MINTER] assert datatoken.isMinter(publisher_wallet.address) # Mint Datatoken to user2 from publisher datatoken.mint(consumer_wallet.address, 1, {"from": publisher_wallet}) assert datatoken.balanceOf(consumer_wallet.address) == 1 # Add minter assert not datatoken.getPermissions(consumer_wallet.address)[DatatokenRoles.MINTER] datatoken.addMinter(consumer_wallet.address, {"from": publisher_wallet}) assert datatoken.getPermissions(consumer_wallet.address)[DatatokenRoles.MINTER] # Mint Datatoken to user2 from consumer datatoken.mint(consumer_wallet.address, 1, {"from": consumer_wallet}) assert datatoken.balanceOf(consumer_wallet.address) == 2 # Should succeed to removeMinter if erc20Deployer datatoken.removeMinter(consumer_wallet.address, {"from": publisher_wallet}) assert not datatoken.getPermissions(consumer_wallet.address)[DatatokenRoles.MINTER] # Should succeed to addFeeManager if erc20Deployer (permission to deploy the erc20Contract at 721 level) assert not datatoken.getPermissions(consumer_wallet.address)[ DatatokenRoles.PAYMENT_MANAGER ] datatoken.addPaymentManager(consumer_wallet.address, {"from": publisher_wallet}) assert datatoken.getPermissions(consumer_wallet.address)[ DatatokenRoles.PAYMENT_MANAGER ] # Should succeed to removeFeeManager if erc20Deployer datatoken.removePaymentManager(consumer_wallet.address, {"from": publisher_wallet}) assert not datatoken.getPermissions(consumer_wallet.address)[ DatatokenRoles.PAYMENT_MANAGER ] # Should succeed to setData if erc20Deployer value = Web3.to_hex(text="SomeData") key = Web3.keccak(hexstr=datatoken.address) datatoken.setData(value, {"from": publisher_wallet}) assert Web3.to_hex(data_nft.getData(key)) == value # Should succeed to call cleanPermissions if NFTOwner datatoken.cleanPermissions({"from": publisher_wallet}) permissions = datatoken.getPermissions(publisher_wallet.address) assert not permissions[DatatokenRoles.MINTER] assert not permissions[DatatokenRoles.PAYMENT_MANAGER] with pytest.raises(Exception, match="NOT ERC20DEPLOYER_ROLE"): data_nft.create_datatoken( {"from": another_consumer_wallet}, name="DT1", symbol="DT1Symbol", ) def test_start_order(config, publisher_wallet, consumer_wallet, data_NFT_and_DT): """Tests startOrder functionality without publish fees, consume fees.""" data_nft, datatoken = data_NFT_and_DT # Mint datatokens to use datatoken.mint(consumer_wallet.address, to_wei(10), {"from": publisher_wallet}) datatoken.mint(publisher_wallet.address, to_wei(10), {"from": publisher_wallet}) # Set the fee collector address datatoken.setPaymentCollector( get_address_of_type(config, "OPFCommunityFeeCollector"), {"from": publisher_wallet}, ) provider_fees = get_mock_provider_fees("MockUSDC", publisher_wallet) receipt = datatoken.start_order( consumer=consumer_wallet.address, service_index=1, provider_fees=provider_fees, consume_market_fees=TokenFeeInfo( address=publisher_wallet.address, token=datatoken.address, ), tx_dict={"from": publisher_wallet}, ) # Check erc20 balances assert datatoken.balanceOf(publisher_wallet.address) == to_wei(9) assert datatoken.balanceOf( get_address_of_type(config, "OPFCommunityFeeCollector") ) == to_wei(1) provider_fee_address = publisher_wallet.address provider_data = provider_fees["providerData"] provider_message = Web3.solidity_keccak( ["bytes32", "bytes"], [receipt.transactionHash, provider_data], ) provider_signed = config["web3_instance"].eth.sign( provider_fee_address, data=provider_message ) message = Web3.solidity_keccak( ["bytes"], [Web3.to_hex(Web3.to_bytes(text="12345"))], ) consumer_signed = config["web3_instance"].eth.sign( consumer_wallet.address, data=message ) receipt_interm = datatoken.orderExecuted( receipt.transactionHash, provider_data, provider_signed, Web3.to_hex(Web3.to_bytes(text="12345")), consumer_signed, consumer_wallet.address, {"from": publisher_wallet}, ) executed_event = datatoken.contract.events.OrderExecuted().process_receipt( receipt_interm, errors=DISCARD )[0] assert executed_event.args.orderTxId == receipt.transactionHash assert executed_event.args.providerAddress == provider_fee_address # Tests exceptions for order_executed consumer_signed = config["web3_instance"].eth.sign( provider_fee_address, data=message ) with pytest.raises(Exception, match="Consumer signature check failed"): datatoken.orderExecuted( receipt.transactionHash, provider_data, provider_signed, Web3.to_hex(Web3.to_bytes(text="12345")), consumer_signed, consumer_wallet.address, {"from": publisher_wallet}, ) message = Web3.solidity_keccak( ["bytes"], [Web3.to_hex(Web3.to_bytes(text="12345"))], ) consumer_signed = config["web3_instance"].eth.sign( consumer_wallet.address, data=message ) with pytest.raises(Exception, match="Provider signature check failed"): datatoken.orderExecuted( receipt.transactionHash, provider_data, consumer_signed, Web3.to_hex(Web3.to_bytes(text="12345")), consumer_signed, consumer_wallet.address, {"from": publisher_wallet}, ) # Tests reuses order receipt_interm = datatoken.reuse_order( receipt.transactionHash, provider_fees=provider_fees, tx_dict={"from": publisher_wallet}, ) reused_event = datatoken.contract.events.OrderReused().process_receipt( receipt_interm, errors=DISCARD )[0] assert reused_event, "Cannot find OrderReused event" assert reused_event.args.orderTxId == receipt.transactionHash assert reused_event.args.caller == publisher_wallet.address provider_fee_event = datatoken.contract.events.ProviderFee().process_receipt( receipt_interm, errors=DISCARD )[0] assert provider_fee_event, "Cannot find ProviderFee event" # Set and get publishing market fee params datatoken.setPublishingMarketFee( publisher_wallet.address, get_address_of_type(config, "MockUSDC"), to_wei(1.2), {"from": publisher_wallet}, ) publish_fees = datatoken.get_publish_market_order_fees() # PublishMarketFeeAddress set previously assert publish_fees.address == publisher_wallet.address # PublishMarketFeeToken set previously assert publish_fees.token == get_address_of_type(config, "MockUSDC") # PublishMarketFeeAmount set previously assert publish_fees.amount == to_wei(1.2) # Fee collector assert datatoken.getPaymentCollector() == get_address_of_type( config, "OPFCommunityFeeCollector" ) # Publisher should succeed to burn some consumer's tokens using burnFrom initial_total_supply = datatoken.totalSupply() initial_consumer_balance = datatoken.balanceOf(consumer_wallet.address) # Approve publisher to burn datatoken.approve(publisher_wallet.address, to_wei(10), {"from": consumer_wallet}) allowance = datatoken.allowance(consumer_wallet.address, publisher_wallet.address) assert allowance == to_wei(10) datatoken.burnFrom(consumer_wallet.address, to_wei(2), {"from": publisher_wallet}) assert datatoken.totalSupply() == initial_total_supply - to_wei(2) assert datatoken.balanceOf( consumer_wallet.address ) == initial_consumer_balance - to_wei(2) # Test transterFrom too initial_consumer_balance = datatoken.balanceOf(consumer_wallet.address) datatoken.transferFrom( consumer_wallet.address, publisher_wallet.address, to_wei(1), {"from": publisher_wallet}, ) assert datatoken.balanceOf( consumer_wallet.address ) == initial_consumer_balance - to_wei(1) # Consumer should be able to burn his tokens too initial_consumer_balance = datatoken.balanceOf(consumer_wallet.address) datatoken.burn(to_wei(1), {"from": consumer_wallet}) assert datatoken.balanceOf( consumer_wallet.address ) == initial_consumer_balance - to_wei(1) # Consumer should be able to transfer too initial_consumer_balance = datatoken.balanceOf(consumer_wallet.address) datatoken.transfer(publisher_wallet.address, to_wei(1), {"from": consumer_wallet}) assert datatoken.balanceOf( consumer_wallet.address ) == initial_consumer_balance - to_wei(1) @pytest.mark.unit def test_exceptions(consumer_wallet, config, publisher_wallet, DT): """Tests revert statements in contracts functions""" datatoken = DT # Should fail to mint if wallet is not a minter with pytest.raises(Exception, match="NOT MINTER"): datatoken.mint( consumer_wallet.address, to_wei(1), {"from": consumer_wallet}, ) # Should fail to set new FeeCollector if not NFTOwner with pytest.raises(Exception, match="NOT PAYMENT MANAGER or OWNER"): datatoken.setPaymentCollector( consumer_wallet.address, {"from": consumer_wallet}, ) # Should fail to addMinter if not erc20Deployer (permission to deploy the erc20Contract at 721 level) with pytest.raises(Exception, match="NOT DEPLOYER ROLE"): datatoken.addMinter(consumer_wallet.address, {"from": consumer_wallet}) # Should fail to removeMinter even if it's minter with pytest.raises(Exception, match="NOT DEPLOYER ROLE"): datatoken.removeMinter(consumer_wallet.address, {"from": consumer_wallet}) # Should fail to addFeeManager if not erc20Deployer (permission to deploy the erc20Contract at 721 level) with pytest.raises(Exception, match="NOT DEPLOYER ROLE"): datatoken.addPaymentManager(consumer_wallet.address, {"from": consumer_wallet}) # Should fail to removeFeeManager if NOT erc20Deployer with pytest.raises(Exception, match="NOT DEPLOYER ROLE"): datatoken.removePaymentManager( consumer_wallet.address, {"from": consumer_wallet} ) # Should fail to setData if NOT erc20Deployer with pytest.raises(Exception, match="NOT DEPLOYER ROLE"): datatoken.setData(Web3.to_hex(text="SomeData"), {"from": consumer_wallet}) # Should fail to call cleanPermissions if NOT NFTOwner with pytest.raises(Exception, match="not NFTOwner"): datatoken.cleanPermissions({"from": consumer_wallet}) # Clean from nft should work shouldn't be callable by publisher or consumer, only by erc721 contract with pytest.raises(Exception, match="NOT 721 Contract"): datatoken.cleanFrom721({"from": consumer_wallet}) ================================================ FILE: ocean_lib/models/test/test_datatoken_order_both_templates.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from datetime import datetime import pytest from ocean_lib.models.datatoken_base import DatatokenBase, TokenFeeInfo from ocean_lib.ocean.util import from_wei, get_address_of_type, to_wei from ocean_lib.web3_internal.constants import MAX_UINT256 from tests.resources.helper_functions import deploy_erc721_erc20, get_mock_provider_fees valid_until = int(datetime(2032, 12, 31).timestamp()) @pytest.mark.unit def test_dispense_and_order_with_non_defaults( config, publisher_wallet, consumer_wallet, factory_deployer_wallet, ): """Tests dispense_and_order function of the Datatoken Template 2""" _, DT = deploy_erc721_erc20(config, publisher_wallet, publisher_wallet, 2) USDC = DatatokenBase.get_typed(config, get_address_of_type(config, "MockUSDC")) DAI = DatatokenBase.get_typed(config, get_address_of_type(config, "MockDAI")) _ = DT.create_dispenser( max_tokens=to_wei(1), max_balance=to_wei(1), tx_dict={"from": publisher_wallet} ) status = DT.dispenser_status() assert status.active assert status.owner_address == publisher_wallet.address assert status.is_minter # ALLOWED_SWAPPER == ZERO means anyone should be able to request dispense # However, ERC20TemplateEnterprise.sol has a quirk where this isn't allowed # Below, we test the quirk. match_s = "This address is not allowed to request DT" with pytest.raises(Exception, match=match_s): DT.dispense(to_wei(1), {"from": consumer_wallet}) consume_fee_amount = to_wei(2) consume_fee_address = consumer_wallet.address DT.setPublishingMarketFee( consume_fee_address, USDC.address, consume_fee_amount, {"from": publisher_wallet}, ) publish_market_fees = DT.get_publish_market_order_fees() USDC.transfer( publisher_wallet.address, publish_market_fees.amount, {"from": factory_deployer_wallet}, ) USDC.approve( DT.address, consume_fee_amount, {"from": publisher_wallet}, ) DAI.transfer( publisher_wallet.address, consume_fee_amount, {"from": factory_deployer_wallet}, ) DAI.approve( DT.address, consume_fee_amount, {"from": publisher_wallet}, ) provider_fees = get_mock_provider_fees( "MockDAI", publisher_wallet, valid_until=valid_until ) opf_collector_address = get_address_of_type(config, "OPFCommunityFeeCollector") balance_opf_consume_before = DAI.balanceOf(opf_collector_address) publish_bal_before = USDC.balanceOf(consumer_wallet.address) tx = DT.dispense_and_order( consumer=consume_fee_address, provider_fees=provider_fees, consume_market_fees=TokenFeeInfo( address=consume_fee_address, token=DAI.address, ), tx_dict={"from": publisher_wallet}, ) assert tx assert DT.totalSupply() == to_wei(0) balance_opf_consume = DAI.balanceOf(opf_collector_address) balance_publish = USDC.balanceOf(publish_market_fees.address) assert balance_opf_consume - balance_opf_consume_before == 0 assert balance_publish - publish_bal_before == to_wei(2) assert DT.balanceOf(DT.getPaymentCollector()) == 0 @pytest.mark.unit @pytest.mark.parametrize("template_index", [1, 2]) def test_dispense_and_order_with_defaults( config, publisher_wallet, consumer_wallet, factory_deployer_wallet, template_index ): """Tests dispense_and_order function of the Datatoken1 and Datatoken2""" _, DT = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet, template_index ) _ = DT.create_dispenser( max_tokens=to_wei(1), max_balance=to_wei(1), tx_dict={"from": publisher_wallet}, ) provider_fees = get_mock_provider_fees( "MockDAI", publisher_wallet, valid_until=valid_until ) tx = DT.dispense_and_order( consumer=consumer_wallet.address, provider_fees=provider_fees, tx_dict={"from": publisher_wallet}, ) assert tx assert DT.totalSupply() == (to_wei(0) if template_index == 2 else to_wei(1)) @pytest.mark.unit @pytest.mark.parametrize("template_index", [1, 2]) def test_buy_DT_and_order( config, publisher_wallet, consumer_wallet, factory_deployer_wallet, another_consumer_wallet, template_index, ): """Tests buy_DT_and_order function of the Datatoken1 and Datatoken2""" _, DT = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet, template_index ) USDC = DatatokenBase.get_typed(config, get_address_of_type(config, "MockUSDC")) DAI = DatatokenBase.get_typed(config, get_address_of_type(config, "MockDAI")) exchange = DT.create_exchange( rate=to_wei(1), base_token_addr=USDC.address, tx_dict={"from": publisher_wallet}, publish_market_fee=to_wei(0.1), ) assert exchange.details.active assert exchange.details.with_mint if template_index == 2: with pytest.raises(Exception, match="This address is not allowed to swap"): exchange.buy_DT( datatoken_amt=to_wei(1), max_basetoken_amt=to_wei(1), tx_dict={"from": consumer_wallet}, ) consume_fee_amount = to_wei(2) consume_fee_address = consumer_wallet.address DT.setPublishingMarketFee( consume_fee_address, USDC.address, consume_fee_amount, {"from": publisher_wallet}, ) publish_market_fees = DT.get_publish_market_order_fees() USDC.transfer( publisher_wallet.address, publish_market_fees.amount + to_wei(3), {"from": factory_deployer_wallet}, ) USDC.approve( DT.address, MAX_UINT256, {"from": publisher_wallet}, ) USDC.approve( exchange.address, MAX_UINT256, {"from": publisher_wallet}, ) DAI.transfer( publisher_wallet.address, consume_fee_amount, {"from": factory_deployer_wallet}, ) DAI.approve(DT.address, consume_fee_amount, {"from": publisher_wallet}) provider_fees = get_mock_provider_fees( "MockDAI", publisher_wallet, valid_until=valid_until ) consume_bal1 = DAI.balanceOf(consume_fee_address) publish_bal1 = USDC.balanceOf(consumer_wallet.address) args = { "consumer": another_consumer_wallet.address, "provider_fees": provider_fees, "consume_market_fees": TokenFeeInfo( address=consume_fee_address, token=DAI.address, ), "exchange": exchange, "tx_dict": {"from": publisher_wallet}, } if template_index == 2: args["max_base_token_amount"] = to_wei(2.5) args["consume_market_swap_fee_amount"] = to_wei(0.001) # 1e15 => 0.1% args["consume_market_swap_fee_address"] = another_consumer_wallet.address tx = DT.buy_DT_and_order(**args) assert tx if template_index == 2: assert DT.totalSupply() == to_wei(0) consume_bal2 = DAI.balanceOf(consume_fee_address) publish_bal2 = USDC.balanceOf(publish_market_fees.address) assert from_wei(consume_bal2) == from_wei(consume_bal1) assert from_wei(publish_bal2) == from_wei(publish_bal1) + 2.0 if template_index == 2: assert from_wei(DT.balanceOf(DT.getPaymentCollector())) == 0 ================================================ FILE: ocean_lib/models/test/test_dispenser.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.models.dispenser import Dispenser, DispenserStatus from ocean_lib.ocean.util import from_wei, get_address_of_type, to_wei from ocean_lib.web3_internal.constants import MAX_UINT256, ZERO_ADDRESS from tests.resources.helper_functions import deploy_erc721_erc20 @pytest.mark.unit def test_DispenserStatus(): """Test DispenserStatus object""" # set status_tup active = True owner = "0x1234" is_minter = True max_tokens = 456 max_bal = 789 bal = 3 swpr = ZERO_ADDRESS # allowed swapper. ZERO_ADDRESS = anyone can request # create the object status_tup = (active, owner, is_minter, max_tokens, max_bal, bal, swpr) status = DispenserStatus(status_tup) # verify status assert isinstance(status, DispenserStatus) assert status.active assert status.owner_address == "0x1234" assert status.is_minter is True assert status.max_tokens == 456 assert status.max_balance == 789 assert status.balance == 3 assert status.allowed_swapper == ZERO_ADDRESS # verify __str__ s = str(status) assert "DispenserStatus" in s assert "active = True" in s assert "owner_address = 0x1234" in s assert "is_minter" in s assert "max_tokens" in s assert "max_balance" in s assert "balance" in s assert "allowed_swapper = anyone" in s @pytest.mark.unit def test_main_flow_via_simple_ux_and_good_defaults( config, publisher_wallet, consumer_wallet, ): """ Tests main flow, via simple interface Datatoken.create_dispenser(). Focus on the basic steps. Use good defaults for max_tokens, max_balance, more. """ _, datatoken = deploy_erc721_erc20(config, publisher_wallet, publisher_wallet) # basic steps datatoken.create_dispenser({"from": publisher_wallet}) datatoken.dispense(to_wei(3), {"from": consumer_wallet}) # check balance bal = datatoken.balanceOf(consumer_wallet.address) assert from_wei(bal) == 3 # check status status = datatoken.dispenser_status() assert isinstance(status, DispenserStatus) assert status.active assert status.owner_address == publisher_wallet.address assert status.is_minter is True assert status.max_tokens == MAX_UINT256 assert status.max_balance == MAX_UINT256 assert status.balance == 0 # 0, not 3, because it mints on the fly assert status.allowed_swapper == ZERO_ADDRESS # anyone can request @pytest.mark.unit def test_main_flow_via_simple_ux_and_setting_token_counts( config, publisher_wallet, consumer_wallet, ): """ Tests main flow, via simple interface Datatoken.create_dispenser(). Focus on the basic steps. Set values for max_tokens, max_balance. """ _, datatoken = deploy_erc721_erc20(config, publisher_wallet, publisher_wallet) # set params max_tokens = to_wei(456) # max # tokens to dispense max_balance = to_wei(789) # max balance of requester # basic steps datatoken.create_dispenser({"from": publisher_wallet}, max_tokens, max_balance) datatoken.dispense(to_wei(3), {"from": consumer_wallet}) # check status status = datatoken.dispenser_status() assert from_wei(status.max_tokens) == 456 assert from_wei(status.max_balance) == 789 assert status.balance == 0 @pytest.mark.unit def test_main_flow_via_contract_directly( config, publisher_wallet, consumer_wallet, factory_deployer_wallet, ): """ Tests main flow, via direct calls to smart contracts (more args). Has not just basic steps, but also advanced actions. """ _, datatoken = deploy_erc721_erc20(config, publisher_wallet, publisher_wallet) # get the dispenser dispenser = Dispenser(config, get_address_of_type(config, "Dispenser")) # Tests publisher creates a dispenser with minter role _ = datatoken.create_dispenser({"from": publisher_wallet}, to_wei(1), to_wei(1)) # Tests publisher gets the dispenser status dispenser_status = dispenser.status(datatoken.address) assert dispenser_status[0] is True assert dispenser_status[1] == publisher_wallet.address assert dispenser_status[2] is True # Tests consumer requests more datatokens then allowed transaction reverts with pytest.raises(Exception, match="Amount too high"): dispenser.dispense( datatoken.address, to_wei(20), consumer_wallet.address, {"from": consumer_wallet}, ) # Tests consumer requests data tokens _ = dispenser.dispense( datatoken.address, to_wei(1), consumer_wallet.address, {"from": consumer_wallet}, ) # Tests consumer requests more datatokens then exceeds maxBalance with pytest.raises(Exception, match="Caller balance too high"): dispenser.dispense( datatoken.address, to_wei(1), consumer_wallet.address, {"from": consumer_wallet}, ) # Tests publisher deactivates the dispenser dispenser.deactivate(datatoken.address, {"from": publisher_wallet}) status = dispenser.status(datatoken.address) assert status[0] is False # Tests factory deployer should fail to get data tokens with pytest.raises(Exception, match="Dispenser not active"): dispenser.dispense( datatoken.address, to_wei(0.00001), factory_deployer_wallet.address, {"from": factory_deployer_wallet}, ) # Tests consumer should fail to activate a dispenser for a token for he is not a minter with pytest.raises(Exception, match="Invalid owner"): dispenser.activate( datatoken.address, to_wei(1), to_wei(1), {"from": consumer_wallet}, ) def test_dispenser_creation_without_minter(config, publisher_wallet, consumer_wallet): """Tests dispenser creation without a minter role.""" _, datatoken = deploy_erc721_erc20(config, publisher_wallet, publisher_wallet) # get the dispenser dispenser = Dispenser(config, get_address_of_type(config, "Dispenser")) datatoken.create_dispenser( {"from": publisher_wallet}, to_wei(1), to_wei(1), with_mint=False ) # Tests consumer requests data tokens but they are not minted with pytest.raises(Exception, match="Not enough reserves"): dispenser.dispense( datatoken.address, to_wei(1), consumer_wallet.address, {"from": consumer_wallet}, ) # Tests publisher mints tokens and transfer them to the dispenser. datatoken.mint( dispenser.address, to_wei(1), {"from": publisher_wallet}, ) # Tests consumer requests data tokens dispenser.dispense( datatoken.address, to_wei(1), consumer_wallet.address, {"from": consumer_wallet}, ) # Tests publisher withdraws all datatokens dispenser.ownerWithdraw(datatoken.address, {"from": publisher_wallet}) status = dispenser.status(datatoken.address) assert status[5] == 0 ================================================ FILE: ocean_lib/models/test/test_exchange_fees.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from web3.logs import DISCARD from ocean_lib.models.datatoken1 import Datatoken1 from ocean_lib.models.factory_router import FactoryRouter from ocean_lib.models.fixed_rate_exchange import FixedRateExchange, OneExchange from ocean_lib.models.test.test_factory_router import ( OPC_SWAP_FEE_APPROVED, OPC_SWAP_FEE_NOT_APPROVED, ) from ocean_lib.ocean.util import from_wei, get_address_of_type, to_wei from ocean_lib.web3_internal.constants import MAX_UINT256, ZERO_ADDRESS from tests.resources.helper_functions import ( convert_bt_amt_to_dt, get_wallet, int_units, transfer_bt_if_balance_lte, ) @pytest.mark.unit @pytest.mark.parametrize( "bt_name, publish_market_swap_fee, consume_market_swap_fee, bt_per_dt", [ # Min fees ("Ocean", 0, 0, 1), ("MockUSDC", 0, 0, 1), # Happy path ("Ocean", 0.003, 0.005, 1), ("MockDAI", 0.003, 0.005, 1), ("MockUSDC", 0.003, 0.005, 1), # Max fees ("Ocean", 0.1, 0.1, 1), ("MockUSDC", 0.1, 0.1, 1), # Min rate. Rate must be => 1e10 wei ("Ocean", 0.003, 0.005, 0.000000010000000000), ("MockUSDC", 0.003, 0.005, 0.000000010000000000), # High rate. There is no maximum ("Ocean", 0.003, 0.005, 1000), ("MockUSDC", 0.003, 0.005, 1000), ], ) def test_exchange_swap_fees( config: dict, factory_deployer_wallet, bob, alice, DT, bt_name: str, publish_market_swap_fee: str, consume_market_swap_fee: str, bt_per_dt: str, ): """ Tests fixed rate exchange swap fees with OCEAN, DAI, and USDC as base token OCEAN is an approved base token with 18 decimals (OPC Fee = 0.1%) DAI is a non-approved base token with 18 decimals (OPC Fee = 0.2%) USDC is a non-approved base token with 6 decimals (OPC Fee = 0.2%) """ bt_deployer_wallet = factory_deployer_wallet consume_market_swap_fee_collector = get_wallet(7) router = FactoryRouter(config, get_address_of_type(config, "Router")) FRE = FixedRateExchange(config, get_address_of_type(config, "FixedPrice")) bt = Datatoken1(config, get_address_of_type(config, bt_name)) dt = DT transfer_bt_if_balance_lte( config=config, bt_address=bt.address, from_wallet=bt_deployer_wallet, recipient=alice.address, min_balance=int_units("1500", bt.decimals()), amount_to_transfer=int_units("1500", bt.decimals()), ) transfer_bt_if_balance_lte( config=config, bt_address=bt.address, from_wallet=bt_deployer_wallet, recipient=bob.address, min_balance=int_units("1500", bt.decimals()), amount_to_transfer=int_units("1500", bt.decimals()), ) publish_market_swap_fee = to_wei(publish_market_swap_fee) consume_market_swap_fee = to_wei(consume_market_swap_fee) bt_per_dt_in_wei = to_wei(bt_per_dt) exchange = dt.create_exchange( rate=bt_per_dt_in_wei, base_token_addr=bt.address, tx_dict={"from": alice}, owner_addr=alice.address, publish_market_fee_collector=alice.address, publish_market_fee=publish_market_swap_fee, allowed_swapper=ZERO_ADDRESS, ) fees = exchange.exchange_fees_info # Verify fee collectors are configured correctly assert fees.publish_market_fee_collector == alice.address # Verify fees are configured correctly if router.isApprovedToken(bt.address): assert fees.opc_fee == OPC_SWAP_FEE_APPROVED else: assert fees.opc_fee == OPC_SWAP_FEE_NOT_APPROVED assert FRE.getOPCFee(bt.address) == fees.opc_fee == router.getOPCFee(bt.address) assert ( exchange.get_publish_market_fee() == publish_market_swap_fee == fees.publish_market_fee ) # Verify 0 fees have been collected so far assert fees.publish_market_fee_available == 0 assert fees.ocean_fee_available == 0 # Verify that rate is configured correctly assert exchange.get_rate() == bt_per_dt_in_wei # Verify exchange starting balance and supply details = exchange.details assert details.bt_balance == 0 assert details.dt_balance == 0 assert details.dt_supply == dt.cap() # Grant infinite approvals for exchange to spend bob's BT and DT dt.approve(exchange.address, MAX_UINT256, {"from": bob}) bt.approve(exchange.address, MAX_UINT256, {"from": bob}) one_base_token = int_units("1", bt.decimals()) dt_per_bt_in_wei = to_wei(1.0 / float(bt_per_dt)) buy_or_sell_dt_and_verify_balances_swap_fees( "buy", convert_bt_amt_to_dt(one_base_token, bt.decimals(), dt_per_bt_in_wei), config, exchange, consume_market_swap_fee_collector.address, consume_market_swap_fee, bob, ) buy_or_sell_dt_and_verify_balances_swap_fees( "sell", convert_bt_amt_to_dt(one_base_token, bt.decimals(), dt_per_bt_in_wei), config, exchange, consume_market_swap_fee_collector.address, consume_market_swap_fee, bob, ) # Collect BT collect_bt_or_dt_and_verify_balances( bt.address, config, exchange, bob, ) # Collect DT collect_bt_or_dt_and_verify_balances( dt.address, config, exchange, bob, ) # Update publish market swap fee new_publish_market_swap_fee = to_wei(0.09) exchange.update_publish_market_fee(new_publish_market_swap_fee, {"from": alice}) assert exchange.exchange_fees_info.publish_market_fee == new_publish_market_swap_fee # Increase rate (base tokens per datatoken) by 1 new_bt_per_dt_in_wei = bt_per_dt_in_wei + to_wei(1) exchange.set_rate(new_bt_per_dt_in_wei, {"from": alice}) assert exchange.get_rate() == new_bt_per_dt_in_wei new_dt_per_bt_in_wei = to_wei(1.0 / from_wei(new_bt_per_dt_in_wei)) buy_or_sell_dt_and_verify_balances_swap_fees( "buy", convert_bt_amt_to_dt(one_base_token, bt.decimals(), new_dt_per_bt_in_wei), config, exchange, consume_market_swap_fee_collector.address, consume_market_swap_fee, bob, ) # Make Bob the new market fee collector exchange.update_publish_market_fee_collector(bob.address, {"from": alice}) # Collect publish market fee collect_fee_and_verify_balances( "publish_market_fee", config, exchange, bob, ) # Collect ocean fee collect_fee_and_verify_balances( "ocean_fee", config, exchange, bob, ) def buy_or_sell_dt_and_verify_balances_swap_fees( action: str, # "buy" or "sell" dt_amount: int, config: dict, exchange: OneExchange, consume_market_swap_fee_address: str, consume_market_swap_fee: int, bob, ): details = exchange.details bt = Datatoken1(config, details.base_token) dt = Datatoken1(config, details.datatoken) # Get balances before swap BT_bob1 = bt.balanceOf(bob) DT_bob1 = dt.balanceOf(bob) BT_exchange1 = details.bt_balance DT_exchange1 = details.dt_balance BT_publish_market_fee_avail1 = ( exchange.exchange_fees_info.publish_market_fee_available ) BT_opc_fee_avail1 = exchange.exchange_fees_info.ocean_fee_available BT_consume_market_fee_avail1 = bt.balanceOf(consume_market_swap_fee_address) if action == "buy": method = exchange.buy_DT min_or_max_bt = MAX_UINT256 elif action == "sell": method = exchange.sell_DT min_or_max_bt = 0 else: raise ValueError(action) # buy_DT() or sell_DT() tx = method( dt_amount, {"from": bob}, min_or_max_bt, consume_market_swap_fee_address, consume_market_swap_fee, ) # Get balances after swap details = exchange.details BT_bob2 = bt.balanceOf(bob) DT_bob2 = dt.balanceOf(bob) BT_exchange2 = details.bt_balance DT_exchange2 = details.dt_balance # Get Swapped event swapped_event = exchange._FRE.contract.events.Swapped().process_receipt( tx, errors=DISCARD )[0] BT_publish_market_fee_amt = swapped_event.args.marketFeeAmount BT_consume_market_fee_amt = swapped_event.args.consumeMarketFeeAmount BT_opc_fee_amt = swapped_event.args.oceanFeeAmount if swapped_event.args.tokenOutAddress == dt.address: BT_amt_swapped = swapped_event.args.baseTokenSwappedAmount DT_amt_swapped = swapped_event.args.datatokenSwappedAmount assert (BT_bob1 - BT_amt_swapped) == BT_bob2 assert (DT_bob1 + DT_amt_swapped) == DT_bob2 assert ( BT_exchange1 + BT_amt_swapped - BT_publish_market_fee_amt - BT_opc_fee_amt - BT_consume_market_fee_amt == BT_exchange2 ) # When buying DT, exchange DT bal doesn't change bc exchange *mints* DT assert DT_exchange1 == DT_exchange2 elif swapped_event.args.tokenOutAddress == bt.address: DT_amt_swapped = swapped_event.args.datatokenSwappedAmount BT_amt_swapped = swapped_event.args.baseTokenSwappedAmount assert (DT_bob1 - DT_amt_swapped) == DT_bob2 assert (BT_bob1 + BT_amt_swapped) == BT_bob2 assert ( BT_exchange1 - BT_amt_swapped - BT_publish_market_fee_amt - BT_opc_fee_amt - BT_consume_market_fee_amt == BT_exchange2 ) assert DT_exchange1 + DT_amt_swapped == DT_exchange2 else: raise ValueError(swapped_event["tokenOutAddress"]) # Get current fee balances # Exchange fees are always base tokens BT_publish_market_fee_avail2 = ( exchange.exchange_fees_info.publish_market_fee_available ) BT_opc_fee_avail2 = exchange.exchange_fees_info.ocean_fee_available BT_consume_market_fee_avail2 = bt.balanceOf(consume_market_swap_fee_address) # Check fees assert ( BT_publish_market_fee_avail1 + BT_publish_market_fee_amt == BT_publish_market_fee_avail2 ) assert ( BT_consume_market_fee_avail1 + BT_consume_market_fee_amt == BT_consume_market_fee_avail2 ) assert BT_opc_fee_avail1 + BT_opc_fee_amt == BT_opc_fee_avail2 def collect_bt_or_dt_and_verify_balances( token_address: str, config: dict, exchange: OneExchange, from_wallet, ): """Collet BT or Collect DT and verify balances""" details = exchange.details dt = Datatoken1(config, details.datatoken) bt = Datatoken1(config, details.base_token) publish_market = dt.getPaymentCollector() if token_address == dt.address: exchange_token_bal1 = details.dt_balance token = dt balance_index = "dt_balance" method = exchange.collect_DT else: exchange_token_bal1 = details.bt_balance token = bt balance_index = "bt_balance" method = exchange.collect_BT publish_market_token_bal1 = token.balanceOf(publish_market) method(exchange_token_bal1, {"from": from_wallet}) details = exchange.details if balance_index == "dt_balance": exchange_token_bal2 = details.dt_balance else: exchange_token_bal2 = details.bt_balance publish_market_token_bal2 = token.balanceOf(publish_market) assert exchange_token_bal2 == 0 assert publish_market_token_bal1 + exchange_token_bal1 == publish_market_token_bal2 def collect_fee_and_verify_balances( fee_type: str, config: dict, exchange: OneExchange, from_wallet, ): """Collect publish_market or opc fees, and verify balances""" FRE = FixedRateExchange(config, get_address_of_type(config, "FixedPrice")) bt = Datatoken1(config, exchange.details.base_token) if fee_type == "publish_market_fee": BT_exchange_fee_avail1 = ( exchange.exchange_fees_info.publish_market_fee_available ) method = exchange.collect_publish_market_fee fee_collector = exchange.exchange_fees_info.publish_market_fee_collector elif fee_type == "ocean_fee": BT_exchange_fee_avail1 = exchange.exchange_fees_info.ocean_fee_available method = exchange.collect_opc_fee fee_collector = FRE.get_opc_collector() else: raise ValueError(fee_type) BT_fee_collector1 = bt.balanceOf(fee_collector) method({"from": from_wallet}) if fee_type == "publish_market_fee": BT_exchange_fee_avail2 = ( exchange.exchange_fees_info.publish_market_fee_available ) else: BT_exchange_fee_avail2 = exchange.exchange_fees_info.ocean_fee_available BT_fee_collector2 = bt.balanceOf(fee_collector) assert BT_exchange_fee_avail2 == 0 assert (BT_fee_collector1 + BT_exchange_fee_avail1) == BT_fee_collector2 ================================================ FILE: ocean_lib/models/test/test_exchange_main.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import time import pytest from web3.logs import DISCARD from ocean_lib.models.fixed_rate_exchange import ( BtNeeded, BtReceived, ExchangeDetails, ExchangeFeeInfo, ) from ocean_lib.models.test.test_factory_router import OPC_SWAP_FEE_APPROVED from ocean_lib.ocean.util import from_wei, to_wei from ocean_lib.web3_internal.constants import MAX_UINT256, ZERO_ADDRESS @pytest.mark.unit def test_with_defaults(OCEAN, DT, alice, bob): # ========================================================================= # Create exchange exchange = DT.create_exchange( rate=to_wei(3), base_token_addr=OCEAN.address, tx_dict={"from": alice}, ) # Alice makes 100 datatokens available on the exchange DT.mint(alice.address, to_wei(100), {"from": alice}) DT.approve(exchange.address, to_wei(100), {"from": alice}) # Bob lets exchange pull the OCEAN needed OCEAN.approve(exchange.address, MAX_UINT256, {"from": bob}) details = exchange.details initial_dt_supply = from_wei(details.dt_supply) # Bob buys 2 datatokens DT_bob1 = DT.balanceOf(bob) _ = exchange.buy_DT(datatoken_amt=to_wei(2), tx_dict={"from": bob}) assert from_wei(DT.balanceOf(bob)) == from_wei(DT_bob1) + 2 # all exchanges for this DT exchanges = DT.get_exchanges() assert len(exchanges) == 1 assert exchanges[0].exchange_id == exchange.exchange_id # Test details details = exchange.details assert details.owner == alice.address assert details.datatoken == DT.address assert details.dt_decimals == DT.decimals() assert from_wei(details.fixed_rate) == 3 assert details.active assert from_wei(details.dt_supply) == initial_dt_supply - 2 assert from_wei(details.bt_supply) == 2 * 3 assert from_wei(details.dt_balance) == 0 assert from_wei(details.bt_balance) == 2 * 3 # Fees tests fees = exchange.exchange_fees_info assert from_wei(fees.publish_market_fee) == 0 # publish mkt swap fee assert fees.publish_market_fee_collector == alice.address # for publish mkt swaps assert ( from_wei(fees.opc_fee) == 0.001 == from_wei(OPC_SWAP_FEE_APPROVED) ) # 0.1% *if* BT approved assert from_wei(fees.publish_market_fee_available) == 0 # for publish mkt swaps assert from_wei(fees.ocean_fee_available) == 2 * 3 * 0.001 FRE = exchange.FRE assert FRE.get_opc_collector()[:2] == "0x", FRE.get_opc_collector() assert from_wei(FRE.getOPCFee(ZERO_ADDRESS)) == 0.002 # 0.2% bc BT not approved # Test other attributes assert exchange.BT_needed(to_wei(1.0), 0) >= to_wei(3) assert exchange.BT_received(to_wei(1.0), 0) >= to_wei(2) assert from_wei(exchange.get_rate()) == 3 assert exchange.get_allowed_swapper() == ZERO_ADDRESS assert exchange.is_active() # ========================================================================== # Bob sells DT to the exchange DT_sell = to_wei(1.5) DT_bob1 = DT.balanceOf(bob) OCEAN_bob1 = OCEAN.balanceOf(bob) DT.approve(exchange.address, DT_sell, {"from": bob}) exchange.sell_DT(DT_sell, tx_dict={"from": bob}) assert DT.balanceOf(bob) < DT_bob1 assert OCEAN.balanceOf(bob) > OCEAN_bob1 # ========================================================================== # Change other stuff # Alice changes market fee collector exchange.update_publish_market_fee_collector(bob.address, {"from": alice}) # Test deactivating exchange assert exchange.details.owner == alice.address assert exchange.is_active() exchange.toggle_active({"from": alice}) assert not exchange.is_active() exchange.toggle_active({"from": alice}) assert exchange.is_active() # Test setting rate exchange.set_rate(to_wei(1.1), {"from": alice}) assert from_wei(exchange.get_rate()) == 1.1 @pytest.mark.unit def test_with_nondefaults(OCEAN, DT, alice, bob, carlos, dan, FRE): # ================================================================= # Alice creates exchange. Bob's the owner, and carlos gets fees! rate = to_wei(1) publish_market_fee = to_wei(0.09) publish_market_fee_collector = alice.address consume_market_fee = to_wei(0.02) consume_market_fee_addr = dan.address n_exchanges1 = FRE.getNumberOfExchanges() exchange, tx = DT.create_exchange( rate=rate, base_token_addr=OCEAN.address, owner_addr=bob.address, publish_market_fee_collector=publish_market_fee_collector, publish_market_fee=publish_market_fee, allowed_swapper=carlos.address, full_info=True, tx_dict={"from": alice}, ) assert tx is not None assert FRE.getNumberOfExchanges() == (n_exchanges1 + 1) assert len(FRE.getExchanges()) == (n_exchanges1 + 1) # Test, focusing on difference from default assert exchange.details.owner == bob.address assert exchange.details.with_mint assert exchange.exchange_fees_info.publish_market_fee == publish_market_fee assert exchange.exchange_fees_info.publish_market_fee_collector == alice.address assert exchange.get_allowed_swapper() == carlos.address # ================================================================= # Alice makes 100 datatokens available on the exchange DT.mint(alice.address, to_wei(100), {"from": alice}) DT.approve(exchange.address, to_wei(100), {"from": alice}) # ================================================================== # Carlos buys DT. (Carlos spends OCEAN, Bob spends DT) DT_buy = to_wei(11) OCEAN_needed = exchange.BT_needed(DT_buy, consume_market_fee) OCEAN.transfer(carlos.address, OCEAN_needed, {"from": bob}) # give carlos OCN DT_carlos1 = DT.balanceOf(carlos) OCEAN_carlos1 = OCEAN.balanceOf(carlos) OCEAN.approve(exchange.address, OCEAN_needed, {"from": carlos}) tx = exchange.buy_DT( datatoken_amt=DT_buy, max_basetoken_amt=MAX_UINT256, consume_market_fee_addr=consume_market_fee_addr, consume_market_fee=consume_market_fee, tx_dict={"from": carlos}, ) assert DT.balanceOf(carlos) == (DT_carlos1 + DT_buy) assert OCEAN.balanceOf(carlos) == (OCEAN_carlos1 - OCEAN_needed) # ========================================================================== # Carlos sells DT to the exchange DT_sell = to_wei(10) DT_exchange1 = exchange.details.dt_supply OCEAN_exchange1 = exchange.details.bt_supply DT_carlos1 = DT.balanceOf(carlos) OCEAN_carlos1 = OCEAN.balanceOf(carlos) DT.approve(exchange.address, DT_sell, {"from": carlos}) exchange.sell_DT( DT_sell, min_basetoken_amt=0, consume_market_fee_addr=consume_market_fee_addr, consume_market_fee=consume_market_fee, tx_dict={"from": carlos}, ) # Carlos should now have more OCEAN, and fewer DT OCEAN_received = exchange.BT_received(DT_sell, consume_market_fee) OCEAN_carlos2 = OCEAN.balanceOf(carlos) DT_carlos2 = DT.balanceOf(carlos) assert pytest.approx(from_wei(OCEAN_carlos2), 0.01) == ( from_wei(OCEAN_carlos1) + from_wei(OCEAN_received) ) assert from_wei(DT_carlos2) == (from_wei(DT_carlos1) - from_wei(DT_sell)) # Test exchange's DT & OCEAN supply details = exchange.details OCEAN_to_exchange = to_wei(from_wei(details.fixed_rate) * from_wei(DT_sell)) assert from_wei(details.dt_supply) == from_wei(DT_exchange1) - from_wei(DT_sell) assert from_wei(details.bt_supply) == from_wei(OCEAN_exchange1) - from_wei( OCEAN_to_exchange ) # ========================================================================== # As payment collector, Alice collects DT payments & BT (OCEAN) payments assert DT.getPaymentCollector() == alice.address DT_alice1 = DT.balanceOf(alice) receipt = exchange.collect_DT(details.dt_balance, {"from": alice}) event = exchange._FRE.contract.events.TokenCollected().process_receipt( receipt, errors=DISCARD )[0] DT_received = event.args.amount assert event.args.to == alice.address DT_expected = DT_alice1 + DT_received OCEAN_alice1 = OCEAN.balanceOf(alice) receipt = exchange.collect_BT(details.bt_balance, {"from": alice}) event = exchange._FRE.contract.events.TokenCollected().process_receipt( receipt, errors=DISCARD )[0] OCEAN_received = event.args.amount assert event.args.to == alice.address OCEAN_expected = OCEAN_alice1 + OCEAN_received st_time = time.time() # loop to ensure chain's updated (shouldn't need!!) while (time.time() - st_time) < 5 and DT.balanceOf(alice) == DT_alice1: time.sleep(0.2) assert from_wei(DT.balanceOf(alice.address)) == from_wei(DT_expected) assert from_wei(OCEAN.balanceOf(alice.address)) == from_wei(OCEAN_expected) # ========================================================================== # As publish market fee collector, Alice collects fees fees = exchange.exchange_fees_info assert fees.publish_market_fee > 0 assert fees.publish_market_fee_available > 0 OCEAN_alice1 = OCEAN.balanceOf(alice.address) exchange.collect_publish_market_fee({"from": alice}) OCEAN_expected = OCEAN_alice1 + fees.publish_market_fee_available st_time = time.time() # loop to ensure chain's updated (shouldn't need!!) while (time.time() - st_time) < 5 and OCEAN.balanceOf(alice) == OCEAN_alice1: time.sleep(0.2) assert from_wei(OCEAN.balanceOf(alice)) == from_wei(OCEAN_expected) @pytest.mark.unit def test_ExchangeDetails(): owner = "0xabc" datatoken = "0xdef" dt_decimals = 18 base_token = "0x123" bt_decimals = 10 fixed_rate = to_wei(0.01) active = False dt_supply = to_wei(100) bt_supply = to_wei(101) dt_balance = to_wei(10) bt_balance = to_wei(11) with_mint = True tup = [ owner, datatoken, dt_decimals, base_token, bt_decimals, fixed_rate, active, dt_supply, bt_supply, dt_balance, bt_balance, with_mint, ] details = ExchangeDetails(tup) assert details.owner == owner assert details.datatoken == datatoken assert details.dt_decimals == dt_decimals assert details.base_token == base_token assert details.bt_decimals == bt_decimals assert details.fixed_rate == fixed_rate assert details.active == active assert details.dt_supply == dt_supply assert details.bt_supply == bt_supply assert details.dt_balance == dt_balance assert details.bt_balance == bt_balance assert details.with_mint == with_mint # Test str. Don't need to be thorough s = str(details) assert "ExchangeDetails" in s assert f"datatoken = {datatoken}" in s assert "rate " in s @pytest.mark.unit def test_ExchangeFeeInfo(): mkt_fee = to_wei(0.03) mkt_fee_coll = "0xabc" opc_fee = to_wei(0.04) mkt_avail = to_wei(0.5) opc_avail = to_wei(0.6) tup = [mkt_fee, mkt_fee_coll, opc_fee, mkt_avail, opc_avail] fees = ExchangeFeeInfo(tup) assert fees.publish_market_fee == mkt_fee assert fees.publish_market_fee_collector == mkt_fee_coll assert fees.opc_fee == opc_fee assert fees.publish_market_fee_available == mkt_avail assert fees.ocean_fee_available == opc_avail # Test str. Don't need to be thorough s = str(fees) assert "ExchangeFeeInfo" in s assert f"publish_market_fee_collector = {mkt_fee_coll}" in s @pytest.mark.unit def test_BtNeeded(): a, b, c, d = 1, 2, 3, 4 # not realistic values, fyi bt_needed = BtNeeded([a, b, c, d]) assert bt_needed.base_token_amount == a assert bt_needed.ocean_fee_amount == b assert bt_needed.publish_market_fee_amount == c assert bt_needed.consume_market_fee_amount == d @pytest.mark.unit def test_BtReceived(): a, b, c, d = 1, 2, 3, 4 # not realistic values, fyi bt_recd = BtReceived([a, b, c, d]) assert bt_recd.base_token_amount == a assert bt_recd.ocean_fee_amount == b assert bt_recd.publish_market_fee_amount == c assert bt_recd.consume_market_fee_amount == d ================================================ FILE: ocean_lib/models/test/test_factory_router.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from web3 import Web3 from ocean_lib.models.factory_router import FactoryRouter from ocean_lib.ocean.util import get_address_of_type, to_wei from ocean_lib.web3_internal.constants import ZERO_ADDRESS # Constants copied from FactoryRouter.sol, used for testing purposes OPC_SWAP_FEE_APPROVED = to_wei(0.001) # 0.1% OPC_SWAP_FEE_NOT_APPROVED = to_wei(0.002) # 0.2% OPC_CONSUME_FEE = to_wei(0.03) # 0.03 DT OPC_PROVIDER_FEE = to_wei(0) # 0% # FactoryRouter methods @pytest.mark.unit def test_router_owner(factory_router: FactoryRouter): assert Web3.is_checksum_address(factory_router.routerOwner()) @pytest.mark.unit def test_swap_ocean_fee(factory_router: FactoryRouter): assert factory_router.swapOceanFee() == OPC_SWAP_FEE_APPROVED @pytest.mark.unit def test_swap_non_ocean_fee(factory_router: FactoryRouter): assert factory_router.swapNonOceanFee() == OPC_SWAP_FEE_NOT_APPROVED @pytest.mark.unit def test_is_approved_token( config: dict, factory_router: FactoryRouter, ocean_address: str ): """Tests that Ocean token has been added to the mapping""" assert factory_router.isApprovedToken(ocean_address) assert not (factory_router.isApprovedToken(ZERO_ADDRESS)) @pytest.mark.unit def test_is_fixed_rate_contract(config: dict, factory_router: FactoryRouter): """Tests that fixedRateExchange address is added to the mapping""" assert factory_router.isFixedRateContract(get_address_of_type(config, "FixedPrice")) @pytest.mark.unit def test_is_dispenser_contract(config: dict, factory_router: FactoryRouter): assert factory_router.isDispenserContract(get_address_of_type(config, "Dispenser")) @pytest.mark.unit def test_get_opc_fee(config: dict, factory_router: FactoryRouter, ocean_address: str): assert factory_router.getOPCFee(ocean_address) == OPC_SWAP_FEE_APPROVED assert factory_router.getOPCFee(ZERO_ADDRESS) == OPC_SWAP_FEE_NOT_APPROVED @pytest.mark.unit def test_get_opc_fees(factory_router: FactoryRouter): assert factory_router.getOPCFees() == [ OPC_SWAP_FEE_APPROVED, OPC_SWAP_FEE_NOT_APPROVED, ] @pytest.mark.unit def test_get_opc_consume_fee(factory_router: FactoryRouter): assert factory_router.getOPCConsumeFee() == OPC_CONSUME_FEE @pytest.mark.unit def test_get_opc_provider_fee(factory_router: FactoryRouter): assert factory_router.getOPCProviderFee() == OPC_PROVIDER_FEE @pytest.mark.unit def test_opc_collector(config: dict, factory_router: FactoryRouter): assert factory_router.getOPCCollector() == get_address_of_type( config, "OPFCommunityFeeCollector" ) ================================================ FILE: ocean_lib/models/test/test_fake_ocean.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os import pytest from eth_account import Account from ocean_lib.ocean.mint_fake_ocean import mint_fake_OCEAN from ocean_lib.ocean.util import to_wei @pytest.mark.unit def test_direct_call(config, consumer_wallet, factory_deployer_wallet, ocean_token): bal_before = ocean_token.balanceOf(consumer_wallet.address) amt_distribute = to_wei(1000) ocean_token.mint( consumer_wallet.address, amt_distribute, {"from": factory_deployer_wallet} ) bal_after = ocean_token.balanceOf(consumer_wallet.address) assert bal_after == (bal_before + amt_distribute) @pytest.mark.unit def test_use_mint_fake_ocean(config, factory_deployer_wallet, ocean_token): expected_amt_distribute = to_wei(2000) mint_fake_OCEAN(config) for key_label in ["TEST_PRIVATE_KEY1", "TEST_PRIVATE_KEY2", "TEST_PRIVATE_KEY3"]: key = os.environ.get(key_label) if not key: continue w = Account.from_key(private_key=key) assert ocean_token.balanceOf(w.address) >= expected_amt_distribute ================================================ FILE: ocean_lib/models/ve/smart_wallet_checker.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class SmartWalletChecker(ContractBase): CONTRACT_NAME = "SmartWalletChecker" ================================================ FILE: ocean_lib/models/ve/test/conftest.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from conftest_ganache import * @pytest.fixture def ocean(publisher_ocean): return publisher_ocean @pytest.fixture def ve_allocate(publisher_ocean): return publisher_ocean.ve_allocate ================================================ FILE: ocean_lib/models/ve/test/test_smart_wallet_checker.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest @pytest.mark.unit def test1(ocean): # df-py/util/test has thorough tests, so keep it super-simple here assert ocean.smart_wallet_checker.address is not None ================================================ FILE: ocean_lib/models/ve/test/test_ve_allocate.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from web3.logs import DISCARD from tests.resources.helper_functions import get_wallet @pytest.mark.unit def test_single_allocation(ve_allocate): """getveAllocation should return the correct allocation.""" accounts = [ get_wallet(1), get_wallet(2), get_wallet(3), ] nftaddr1 = accounts[0].address nftaddr2 = accounts[1].address nftaddr3 = accounts[2].address ve_allocate.setAllocation(100, nftaddr1, 1, {"from": accounts[0]}) assert ve_allocate.getveAllocation(accounts[0], nftaddr1, 1) == 100 ve_allocate.setAllocation(25, nftaddr2, 1, {"from": accounts[0]}) assert ve_allocate.getveAllocation(accounts[0], nftaddr2, 1) == 25 ve_allocate.setAllocation(50, nftaddr3, 1, {"from": accounts[0]}) assert ve_allocate.getveAllocation(accounts[0], nftaddr3, 1) == 50 ve_allocate.setAllocation(0, nftaddr2, 1, {"from": accounts[0]}) assert ve_allocate.getveAllocation(accounts[0], nftaddr2, 1) == 0 @pytest.mark.unit def test_single_events(ve_allocate): """Test emitted events.""" accounts = [ get_wallet(1), get_wallet(2), ] nftaddr1 = accounts[1].address tx = ve_allocate.setAllocation(100, nftaddr1, 1, {"from": accounts[0]}) event = ve_allocate.contract.events.AllocationSet().process_receipt( tx, errors=DISCARD )[0] assert event.args.sender == accounts[0].address assert event.args.nft == accounts[1].address assert event.args.chainId == 1 assert event.args.amount == 100 @pytest.mark.unit def test_batch_allocation(ve_allocate): """getveAllocation should return the correct allocation.""" accounts = [ get_wallet(1), get_wallet(2), ] nftaddr1 = accounts[0].address nftaddr2 = accounts[1].address ve_allocate.setBatchAllocation( [50, 50], [nftaddr1, nftaddr2], [1, 1], {"from": accounts[0]} ) assert ve_allocate.getveAllocation(accounts[0], nftaddr1, 1) == 50 @pytest.mark.unit def test_batch_events(ve_allocate): """Test emitted events.""" accounts = [ get_wallet(1), get_wallet(2), ] nftaddr1 = accounts[1].address nftaddr2 = accounts[1].address tx = ve_allocate.setBatchAllocation( [25, 75], [nftaddr1, nftaddr2], [1, 1], {"from": accounts[0]} ) event = ve_allocate.contract.events.AllocationSetMultiple().process_receipt( tx, errors=DISCARD )[0] assert event.args.sender == accounts[0].address assert event.args.nft == [nftaddr1, nftaddr2] assert event.args.chainId == [1, 1] assert event.args.amount == [25, 75] ================================================ FILE: ocean_lib/models/ve/test/test_ve_delegation.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest @pytest.mark.unit def test1(ocean): # df-py/util/test has thorough tests, so keep it super-simple here assert ocean.ve_delegation.address is not None ================================================ FILE: ocean_lib/models/ve/test/test_ve_fee_distributor.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest @pytest.mark.unit def test1(ocean, consumer_wallet): # df-py/util/test/veOcean/test_estimateClaim.py has thorough tests # therefore, we keep it super-simple here assert ocean.ve_fee_distributor.address is not None ocean.ve_fee_distributor.user_epoch_of(consumer_wallet) ocean.ve_fee_distributor.time_cursor_of(consumer_wallet) # this is the most important call from a user standpoint. $$ :) ocean.ve_fee_distributor.claim({"from": consumer_wallet}) ================================================ FILE: ocean_lib/models/ve/test/test_ve_fee_estimate.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest @pytest.mark.unit def test1(ocean): # df-py/util/test has thorough tests, so keep it super-simple here assert ocean.ve_fee_estimate.address is not None ================================================ FILE: ocean_lib/models/ve/test/test_ve_ocean.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import math import pytest from ocean_lib.ocean.util import from_wei, send_ether, to_wei WEEK = 7 * 86400 MAXTIME = 4 * 365 * 86400 # 4 years @pytest.mark.unit def test_ve_ocean1(ocean, factory_deployer_wallet, ocean_token): OCEAN = ocean.OCEAN_token veOCEAN = ocean.veOCEAN # inspiration from df-py/util/test/veOcean/test_lock.py assert veOCEAN.symbol() == "veOCEAN" OCEAN = ocean_token web3 = ocean.config_dict["web3_instance"] alice_wallet = ( web3.eth.account.create() ) # new account avoids "withdraw old tokens first" send_ether( ocean.config_dict, factory_deployer_wallet, alice_wallet.address, to_wei(1) ) TA = to_wei(0.0001) OCEAN.mint(alice_wallet.address, TA, {"from": factory_deployer_wallet}) latest_block = web3.eth.get_block("latest") veOCEAN.checkpoint({"from": factory_deployer_wallet, "gas": latest_block.gasLimit}) OCEAN.approve(veOCEAN.address, TA, {"from": alice_wallet}) latest_block = ocean.config_dict["web3_instance"].eth.get_block("latest") t0 = latest_block.timestamp # ve funcs use block.timestamp, not chain.time() t1 = t0 // WEEK * WEEK + WEEK # this is a Thursday, because Jan 1 1970 was t2 = t1 + WEEK provider = web3.provider provider.make_request("evm_increaseTime", [(t1 - t0)]) assert OCEAN.balanceOf(alice_wallet.address) != 0 latest_block = web3.eth.get_block("latest") veOCEAN.create_lock( TA, t2, { "from": alice_wallet, "gas": latest_block.gasLimit, "gasPrice": math.ceil(latest_block["baseFeePerGas"] * 1.2), }, ) assert OCEAN.balanceOf(alice_wallet.address) == 0 epoch = veOCEAN.user_point_epoch(alice_wallet) assert epoch != 0 assert veOCEAN.get_last_user_slope(alice_wallet) != 0 latest_block = web3.eth.get_block("latest") alice_vote_power = float( from_wei(veOCEAN.balanceOf(alice_wallet, latest_block.timestamp)) ) expected_vote_power = float(from_wei(TA)) * WEEK / MAXTIME assert alice_vote_power == pytest.approx(expected_vote_power, TA / 20.0) provider.make_request("evm_increaseTime", [t2]) provider.make_request("evm_mine", []) latest_block = web3.eth.get_block("latest") veOCEAN.withdraw( { "from": alice_wallet, "gas": latest_block.gasLimit, "gasPrice": math.ceil(latest_block["baseFeePerGas"] * 1.2), } ) assert OCEAN.balanceOf(alice_wallet.address) == TA latest_block = web3.eth.get_block("latest") assert veOCEAN.get_last_user_slope(alice_wallet) == 0 assert veOCEAN.balanceOf(alice_wallet, latest_block.timestamp) == 0 ================================================ FILE: ocean_lib/models/ve/ve_allocate.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class VeAllocate(ContractBase): CONTRACT_NAME = "veAllocate" ================================================ FILE: ocean_lib/models/ve/ve_delegation.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class VeDelegation(ContractBase): CONTRACT_NAME = "veDelegation" ================================================ FILE: ocean_lib/models/ve/ve_fee_distributor.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class VeFeeDistributor(ContractBase): CONTRACT_NAME = "veFeeDistributor" ================================================ FILE: ocean_lib/models/ve/ve_fee_estimate.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class VeFeeEstimate(ContractBase): CONTRACT_NAME = "veFeeEstimate" ================================================ FILE: ocean_lib/models/ve/ve_ocean.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from ocean_lib.web3_internal.contract_base import ContractBase class VeOcean(ContractBase): CONTRACT_NAME = "veOCEAN" ================================================ FILE: ocean_lib/ocean/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/ocean/crypto.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from base64 import b64encode from hashlib import sha256 from cryptography.fernet import Fernet from ecies import decrypt as asymmetric_decrypt from ecies import encrypt as asymmetric_encrypt from enforce_typing import enforce_types from eth_keys import keys from eth_utils import decode_hex @enforce_types def calc_symkey(base_str: str) -> str: """Compute a symmetric private key that's a function of the base_str""" base_b = base_str.encode("utf-8") # bytes hash_b = sha256(base_b).hexdigest() symkey_b = b64encode(hash_b.encode("ascii"))[:43] + b"=" # bytes symkey = symkey_b.decode("ascii") return symkey @enforce_types def sym_encrypt(value: str, symkey: str) -> str: """Symmetrically encrypt a value, e.g. ready to store in set_data()""" value_b = value.encode("utf-8") # bytes symkey_b = symkey.encode("utf-8") # bytes value_enc_b = Fernet(symkey_b).encrypt(value_b) # main work. bytes value_enc = value_enc_b.decode("ascii") # ascii str return value_enc @enforce_types def sym_decrypt(value_enc: str, symkey: str) -> str: """Symmetrically decrypt a value, e.g. retrieved from get_data()""" value_enc_b = value_enc.encode("utf-8") symkey_b = symkey.encode("utf-8") value_b = Fernet(symkey_b).decrypt(value_enc_b) # main work value = value_b.decode("ascii") return value @enforce_types def calc_pubkey(privkey: str) -> str: privkey_obj = keys.PrivateKey(decode_hex(privkey)) pubkey = str(privkey_obj.public_key) # str return pubkey @enforce_types def asym_encrypt(value: str, pubkey: str) -> str: """Asymmetrically encrypt a value, e.g. ready to store in set_data()""" value_b = value.encode("utf-8") # binary value_enc_b = asymmetric_encrypt(pubkey, value_b) # main work. binary value_enc_h = value_enc_b.hex() # hex str return value_enc_h @enforce_types def asym_decrypt(value_enc_h: str, privkey: str) -> str: """Asymmetrically decrypt a value, e.g. retrieved from get_data()""" value_enc_b = decode_hex(value_enc_h) # bytes value_b = asymmetric_decrypt(privkey, value_enc_b) # main work. bytes value = value_b.decode("ascii") return value ================================================ FILE: ocean_lib/ocean/mint_fake_ocean.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os from enforce_typing import enforce_types from eth_account import Account from ocean_lib.models.datatoken_base import DatatokenBase from ocean_lib.ocean.util import get_ocean_token_address, send_ether, to_wei @enforce_types def mint_fake_OCEAN(config: dict) -> None: """ Does the following: 1. Mints tokens 2. Distributes tokens to TEST_PRIVATE_KEY1 and TEST_PRIVATE_KEY2 """ deployer_wallet = Account.from_key( private_key=os.getenv("FACTORY_DEPLOYER_PRIVATE_KEY") ) OCEAN_token = DatatokenBase.get_typed( config, address=get_ocean_token_address(config) ) amt_distribute = to_wei(2000) OCEAN_token.mint(deployer_wallet.address, to_wei(20000), {"from": deployer_wallet}) for key_label in ["TEST_PRIVATE_KEY1", "TEST_PRIVATE_KEY2", "TEST_PRIVATE_KEY3"]: key = os.environ.get(key_label) if not key: continue w = Account.from_key(private_key=key) if OCEAN_token.balanceOf(w.address) < amt_distribute: OCEAN_token.mint(w.address, amt_distribute, {"from": deployer_wallet}) if config["web3_instance"].eth.get_balance(w.address) < to_wei(2): send_ether(config, deployer_wallet, w.address, to_wei(4)) ================================================ FILE: ocean_lib/ocean/ocean.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Ocean module.""" import json import logging from typing import Dict, List, Optional, Type, Union from enforce_typing import enforce_types from web3.datastructures import AttributeDict from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.example_config import config_defaults from ocean_lib.models.compute_input import ComputeInput from ocean_lib.models.data_nft import DataNFT from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.models.datatoken_base import DatatokenBase from ocean_lib.models.df.df_rewards import DFRewards from ocean_lib.models.df.df_strategy_v1 import DFStrategyV1 from ocean_lib.models.dispenser import Dispenser from ocean_lib.models.factory_router import FactoryRouter from ocean_lib.models.fixed_rate_exchange import FixedRateExchange from ocean_lib.models.ve.smart_wallet_checker import SmartWalletChecker from ocean_lib.models.ve.ve_allocate import VeAllocate from ocean_lib.models.ve.ve_delegation import VeDelegation from ocean_lib.models.ve.ve_fee_distributor import VeFeeDistributor from ocean_lib.models.ve.ve_fee_estimate import VeFeeEstimate from ocean_lib.models.ve.ve_ocean import VeOcean from ocean_lib.ocean.ocean_assets import OceanAssets from ocean_lib.ocean.ocean_compute import OceanCompute from ocean_lib.ocean.util import get_address_of_type, get_ocean_token_address from ocean_lib.services.service import Service from ocean_lib.structures.algorithm_metadata import AlgorithmMetadata logger = logging.getLogger("ocean") class Ocean: """The Ocean class is the entry point into Ocean Protocol.""" @enforce_types def __init__(self, config_dict: Dict, data_provider: Optional[Type] = None) -> None: """Initialize Ocean class. Usage: Make a new Ocean instance `ocean = Ocean({...})` This class provides the main top-level functions in ocean protocol: 1. Publish assets metadata and associated services - Each asset is assigned a unique DID and a DID Document (DDO) - The DDO contains the asset's services including the metadata - The DID is registered on-chain with a URL of the metadata store to retrieve the DDO from `ddo = ocean.assets.create(metadata, publisher_wallet)` 2. Discover/Search ddos via the current configured metadata store (Aquarius) - Usage: `ddos_list = ocean.assets.search('search text')` An instance of Ocean is parameterized by a `Config` instance. :param config_dict: variable definitions :param data_provider: `DataServiceProvider` instance """ config_errors = {} for key, value in config_defaults.items(): if key not in config_dict: config_errors[key] = "required" continue if not isinstance(config_dict[key], type(value)): config_errors[key] = f"must be {type(value).__name__}" if "web3_instance" not in config_dict: config_errors["web3_instance"] = "required" if "NETWORK_NAME" not in config_dict: config_errors["NETWORK_NAME"] = "required" if config_errors: raise Exception(json.dumps(config_errors)) self.config_dict = config_dict if not data_provider: data_provider = DataServiceProvider self.assets = OceanAssets(self.config_dict, data_provider) self.compute = OceanCompute(self.config_dict, data_provider) logger.debug("Ocean instance initialized: ") # ====================================================================== # OCEAN @property @enforce_types def OCEAN_address(self) -> str: return get_ocean_token_address(self.config) @property @enforce_types def OCEAN_token(self) -> DatatokenBase: return DatatokenBase.get_typed(self.config, self.OCEAN_address) @property @enforce_types def OCEAN(self): # alias for OCEAN_token return self.OCEAN_token # ====================================================================== # objects for singleton smart contracts @property @enforce_types def data_nft_factory(self) -> DataNFTFactoryContract: return DataNFTFactoryContract(self.config, self._addr("ERC721Factory")) @property @enforce_types def dispenser(self) -> Dispenser: return Dispenser(self.config, self._addr("Dispenser")) @property @enforce_types def fixed_rate_exchange(self) -> FixedRateExchange: return FixedRateExchange(self.config, self._addr("FixedPrice")) @property @enforce_types def factory_router(self) -> FactoryRouter: return FactoryRouter(self.config, self._addr("Router")) # ====================================================================== # token getters @enforce_types def get_nft_token(self, token_address: str) -> DataNFT: """ :param token_address: Token contract address, str :return: `DataNFT` instance """ return DataNFT(self.config, token_address) @enforce_types def get_datatoken(self, token_address: str) -> DatatokenBase: """ :param token_address: Token contract address, str :return: `Datatoken1` or `Datatoken2` instance """ return DatatokenBase.get_typed(self.config, token_address) # ====================================================================== # orders @enforce_types def get_user_orders(self, address: str, datatoken: str) -> List[AttributeDict]: """ :return: List of orders `[Order]` """ dt = DatatokenBase.get_typed(self.config_dict, datatoken) _orders = [] for log in dt.get_start_order_logs(address): a = dict(log.args.items()) a["amount"] = int(log.args.amount) a["address"] = log.address a["transactionHash"] = log.transactionHash a = AttributeDict(a.items()) _orders.append(a) return _orders # ====================================================================== # provider fees @enforce_types def retrieve_provider_fees( self, ddo: DDO, access_service: Service, publisher_wallet ) -> dict: initialize_response = DataServiceProvider.initialize( ddo.did, access_service, consumer_address=publisher_wallet.address ) initialize_data = initialize_response.json() provider_fees = initialize_data["providerFee"] return provider_fees @enforce_types def retrieve_provider_fees_for_compute( self, datasets: List[ComputeInput], algorithm_data: Union[ComputeInput, AlgorithmMetadata], consumer_address: str, compute_environment: str, valid_until: int, ) -> dict: initialize_compute_response = DataServiceProvider.initialize_compute( [x.as_dictionary() for x in datasets], algorithm_data.as_dictionary(), datasets[0].service.service_endpoint, consumer_address, compute_environment, valid_until, ) return initialize_compute_response.json() # ====================================================================== # DF/VE properties (alphabetical) @property @enforce_types def df_rewards(self) -> DFRewards: return DFRewards(self.config, self._addr("DFRewards")) @property @enforce_types def df_strategy_v1(self) -> DFStrategyV1: return DFStrategyV1(self.config, self._addr("DFStrategyV1")) @property @enforce_types def smart_wallet_checker(self) -> SmartWalletChecker: return SmartWalletChecker(self.config, self._addr("SmartWalletChecker")) @property @enforce_types def ve_allocate(self) -> VeAllocate: return VeAllocate(self.config, self._addr("veAllocate")) @property @enforce_types def ve_delegation(self) -> VeDelegation: return VeDelegation(self.config, self._addr("veDelegation")) @property @enforce_types def ve_fee_distributor(self) -> VeFeeDistributor: return VeFeeDistributor(self.config, self._addr("veFeeDistributor")) @property @enforce_types def ve_fee_estimate(self) -> VeFeeEstimate: return VeFeeEstimate(self.config, self._addr("veFeeEstimate")) @property @enforce_types def ve_ocean(self) -> VeOcean: return VeOcean(self.config, self._addr("veOCEAN")) @property @enforce_types def veOCEAN(self) -> VeOcean: # alias for ve_ocean return self.ve_ocean # ====================================================================== # helpers @property @enforce_types def config(self) -> dict: # alias for config_dict return self.config_dict @enforce_types def _addr(self, type_str: str) -> str: return get_address_of_type(self.config, type_str) def wallet_balance(self, w): return self.config["web3_instance"].eth.get_balance(w.address) ================================================ FILE: ocean_lib/ocean/ocean_assets.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Ocean module.""" import json import logging import lzma import os from datetime import datetime from typing import List, Optional, Tuple, Type, Union from enforce_typing import enforce_types from ocean_lib.agreements.consumable import AssetNotConsumable, ConsumableCodes from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.aquarius import Aquarius from ocean_lib.assets.asset_downloader import download_asset_files, is_consumable from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_encryptor import DataEncryptor from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.exceptions import AquariusError, InsufficientBalance from ocean_lib.models.compute_input import ComputeInput from ocean_lib.models.data_nft import DataNFT, DataNFTArguments from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.models.datatoken_base import ( DatatokenArguments, DatatokenBase, TokenFeeInfo, ) from ocean_lib.models.dispenser import DispenserArguments from ocean_lib.models.fixed_rate_exchange import ExchangeArguments from ocean_lib.ocean.util import ( create_checksum, get_address_of_type, get_args_object, get_from_address, to_wei, ) from ocean_lib.services.service import Service from ocean_lib.structures.algorithm_metadata import AlgorithmMetadata from ocean_lib.structures.file_objects import ( ArweaveFile, FilesType, GraphqlQuery, SmartContractCall, UrlFile, ) from ocean_lib.web3_internal.constants import ZERO_ADDRESS logger = logging.getLogger("ocean") class AssetArguments: def __init__( self, wait_for_aqua: bool = True, dt_template_index: Optional[int] = 1, pricing_schema_args: Optional[ Union[DispenserArguments, ExchangeArguments] ] = None, metadata: Optional[dict] = None, with_compute: Optional[bool] = False, compute_values: Optional[dict] = None, credentials: Optional[dict] = None, ): self.wait_for_aqua = wait_for_aqua self.dt_template_index = dt_template_index self.pricing_schema_args = pricing_schema_args self.metadata = metadata self.with_compute = with_compute self.compute_values = compute_values self.credentials = credentials if credentials else {"allow": [], "deny": []} class OceanAssets: """Ocean asset class for V4.""" @enforce_types def __init__(self, config_dict, data_provider: Type[DataServiceProvider]) -> None: """Initialises OceanAssets object.""" self._config_dict = config_dict self._chain_id = config_dict["CHAIN_ID"] self._metadata_cache_uri = config_dict.get("METADATA_CACHE_URI") self._data_provider = data_provider downloads_path = os.path.join(os.getcwd(), "downloads") self._downloads_path = config_dict.get("DOWNLOADS_PATH", downloads_path) self._aquarius = Aquarius.get_instance(self._metadata_cache_uri) self.data_nft_factory = DataNFTFactoryContract( self._config_dict, get_address_of_type(config_dict, "ERC721Factory") ) @enforce_types def validate(self, ddo: DDO) -> Tuple[bool, list]: """ Validate that the ddo is ok to be stored in aquarius. :param ddo: DDO. :return: (bool, list) list of errors, empty if valid """ # Validation by Aquarius validation_result, validation_errors = self._aquarius.validate_ddo(ddo) if not validation_result: msg = f"DDO has validation errors: {validation_errors}" logger.error(msg) raise ValueError(msg) return validation_result, validation_errors @staticmethod @enforce_types def _encrypt_ddo( ddo: DDO, provider_uri: str, encrypt_flag: Optional[bool] = True, compress_flag: Optional[bool] = True, ): # Process the DDO ddo_dict = ddo.as_dictionary() ddo_string = json.dumps(ddo_dict, separators=(",", ":")) ddo_bytes = ddo_string.encode("utf-8") ddo_hash = create_checksum(ddo_string) # Plain DDO if not encrypt_flag and not compress_flag: flags = bytes([0]) document = ddo_bytes return document, flags, ddo_hash # Only compression, not encrypted if compress_flag and not encrypt_flag: flags = bytes([1]) # Compress DDO document = lzma.compress(ddo_bytes) return document, flags, ddo_hash # Only encryption, not compressed if encrypt_flag and not compress_flag: flags = bytes([2]) # Encrypt DDO encrypt_response = DataEncryptor.encrypt( objects_to_encrypt=ddo_string, provider_uri=provider_uri, chain_id=ddo.chain_id, ) document = encrypt_response.text return document, flags, ddo_hash # Encrypted & compressed flags = bytes([3]) # Compress DDO compressed_document = lzma.compress(ddo_bytes) # Encrypt DDO encrypt_response = DataEncryptor.encrypt( objects_to_encrypt=compressed_document, provider_uri=provider_uri, chain_id=ddo.chain_id, ) document = encrypt_response.text return document, flags, ddo_hash @staticmethod @enforce_types def _assert_ddo_metadata(metadata: dict): assert isinstance( metadata, dict ), f"Expected metadata of type dict, got {type(metadata)}" asset_type = metadata.get("type") assert asset_type in ( "dataset", "algorithm", ), f"Invalid/unsupported asset type {asset_type}" assert "name" in metadata, "Must have name in metadata." @enforce_types def create_algo_asset( self, name: str, url: str, tx_dict: dict, image: str = "oceanprotocol/algo_dockers", tag: str = "python-branin", checksum: str = "sha256:8221d20c1c16491d7d56b9657ea09082c0ee4a8ab1a6621fa720da58b09580e4", *args, **kwargs, ) -> tuple: """Create asset of type "algorithm", having UrlFiles, with good defaults""" asset_args = get_args_object(args, kwargs, AssetArguments) if not asset_args.metadata: metadata = OceanAssets.default_metadata(name, tx_dict, "algorithm") metadata["algorithm"] = { "language": "python", "format": "docker-image", "version": "0.1", "container": { "entrypoint": "python $ALGO", "image": image, "tag": tag, "checksum": checksum, }, } asset_args.metadata = metadata files = [UrlFile(url)] return self.create_bundled(files, tx_dict, asset_args) @enforce_types def create_url_asset( self, name: str, url: str, tx_dict: dict, *args, **kwargs, ) -> tuple: """Create asset of type "data", having UrlFiles, with good defaults""" asset_args = get_args_object(args, kwargs, AssetArguments) if not asset_args.metadata: asset_args.metadata = OceanAssets.default_metadata(name, tx_dict) files = [UrlFile(url)] return self.create_bundled(files, tx_dict, asset_args) @enforce_types def create_arweave_asset( self, name: str, transaction_id: str, tx_dict: dict, *args, **kwargs ) -> tuple: """Create asset of type "data", having UrlFiles, with good defaults""" asset_args = get_args_object(args, kwargs, AssetArguments) if not asset_args.metadata: asset_args.metadata = OceanAssets.default_metadata(name, tx_dict) files = [ArweaveFile(transaction_id)] return self.create_bundled(files, tx_dict, asset_args) @enforce_types def create_graphql_asset( self, name: str, url: str, query: str, tx_dict: dict, *args, **kwargs ) -> tuple: """Create asset of type "data", having GraphqlQuery files, w good defaults""" asset_args = get_args_object(args, kwargs, AssetArguments) if not asset_args.metadata: asset_args.metadata = OceanAssets.default_metadata(name, tx_dict) files = [GraphqlQuery(url, query)] return self.create_bundled(files, tx_dict, asset_args) @enforce_types def create_onchain_asset( self, name: str, contract_address: str, contract_abi: dict, tx_dict: dict, wait_for_aqua: bool = True, dt_template_index: Optional[int] = 1, pricing_schema_args: Optional[ Union[DispenserArguments, ExchangeArguments] ] = None, *args, **kwargs, ) -> tuple: """Create asset of type "data", having SmartContractCall files, w defaults""" chain_id = self._chain_id onchain_data = SmartContractCall(contract_address, chain_id, contract_abi) files = [onchain_data] asset_args = get_args_object(args, kwargs, AssetArguments) if not asset_args.metadata: asset_args.metadata = OceanAssets.default_metadata(name, tx_dict) return self.create_bundled(files, tx_dict, asset_args) @classmethod @enforce_types def default_metadata(cls, name: str, tx_dict: dict, type="dataset") -> dict: address = get_from_address(tx_dict) date_created = datetime.now().isoformat() metadata = { "created": date_created, "updated": date_created, "description": name, "name": name, "type": type, "author": address[:7], "license": "CC0: PublicDomain", } return metadata @enforce_types def create_bundled( self, files: List[FilesType], tx_dict: dict, asset_args: AssetArguments ): provider_uri = DataServiceProvider.get_url(self._config_dict) self._assert_ddo_metadata(asset_args.metadata) name = asset_args.metadata["name"] data_nft_args = DataNFTArguments(name, name) if asset_args.dt_template_index == 2: datatoken_args = DatatokenArguments( f"{name}: DT1", files=files, template_index=2, cap=to_wei(100) ) else: datatoken_args = DatatokenArguments(f"{name}: DT1", files=files) if not asset_args.pricing_schema_args: data_nft, datatoken = self.data_nft_factory.create_with_erc20( data_nft_args, datatoken_args, tx_dict ) if isinstance(asset_args.pricing_schema_args, DispenserArguments): data_nft, datatoken = self.data_nft_factory.create_with_erc20_and_dispenser( data_nft_args, datatoken_args, asset_args.pricing_schema_args, tx_dict ) if isinstance(asset_args.pricing_schema_args, ExchangeArguments): ( data_nft, datatoken, _, ) = self.data_nft_factory.create_with_erc20_and_fixed_rate( data_nft_args, datatoken_args, asset_args.pricing_schema_args, tx_dict ) ddo = DDO() # Generate the did, add it to the ddo. ddo.did = data_nft.calculate_did() # Check if it's already registered first! if self._aquarius.ddo_exists(ddo.did): raise AquariusError( f"Asset id {ddo.did} is already registered to another asset." ) ddo.chain_id = self._chain_id ddo.metadata = asset_args.metadata ddo.credentials = asset_args.credentials ddo.nft_address = data_nft.address access_service = datatoken.build_access_service( service_id="0", service_endpoint=provider_uri, files=files, ) ddo.add_service(access_service) if asset_args.with_compute or asset_args.compute_values: ddo.create_compute_service( "1", provider_uri, datatoken.address, files, asset_args.compute_values, ) # Validation by Aquarius _, proof = self.validate(ddo) proof = ( proof["publicKey"], proof["v"], proof["r"][0], proof["s"][0], ) document, flags, ddo_hash = self._encrypt_ddo(ddo, provider_uri, True, True) wallet_address = get_from_address(tx_dict) data_nft.setMetaData( 0, provider_uri, wallet_address.encode("utf-8"), flags, document, ddo_hash, [proof], tx_dict, ) # Fetch the ddo on chain if asset_args.wait_for_aqua: ddo = self._aquarius.wait_for_ddo(ddo.did) return (data_nft, datatoken, ddo) # Don't enforce types due to error: # TypeError: Subscripted generics cannot be used with class and instance checks def create( self, metadata: dict, tx_dict: dict, credentials: Optional[dict] = None, data_nft_address: Optional[str] = None, data_nft_args: Optional[DataNFTArguments] = None, deployed_datatokens: Optional[List[DatatokenBase]] = None, services: Optional[list] = None, datatoken_args: Optional[List["DatatokenArguments"]] = None, encrypt_flag: Optional[bool] = True, compress_flag: Optional[bool] = True, wait_for_aqua: bool = True, ) -> Optional[DDO]: """Register an asset on-chain. Asset = {data_NFT, >=0 datatokens, DDO} Creating/deploying a DataNFT contract and in the Metadata store (Aquarius). :param metadata: dict conforming to the Metadata accepted by Ocean Protocol. :param publisher_wallet: account of the publisher registering this asset. :param credentials: credentials dict necessary for the asset. construct the serviceEndpoint for the `access` (download) service :param data_nft_address: hex str the address of the data NFT. The new asset will be associated with this data NFT address. :param data_nft_args: object of DataNFTArguments type if creating a new one :param deployed_datatokens: list of datatokens which are already deployed. :param encrypt_flag: bool for encryption of the DDO. :param compress_flag: bool for compression of the DDO. :param wait_for_aqua: wait to ensure ddo's updated in aquarius? :return: tuple of (data_nft, datatokens, ddo) """ self._assert_ddo_metadata(metadata) provider_uri = DataServiceProvider.get_url(self._config_dict) if not data_nft_address: data_nft_args = data_nft_args or DataNFTArguments( metadata["name"], metadata["name"] ) data_nft = data_nft_args.deploy_contract(self._config_dict, tx_dict) # register on-chain if not data_nft: logger.warning("Creating new NFT failed.") return None, None, None logger.info(f"Successfully created NFT with address {data_nft.address}.") else: data_nft = DataNFT(self._config_dict, data_nft_address) # Create DDO object ddo = DDO() # Generate the did, add it to the ddo. ddo.did = data_nft.calculate_did() # Check if it's already registered first! if self._aquarius.ddo_exists(ddo.did): raise AquariusError( f"Asset id {ddo.did} is already registered to another asset." ) ddo.chain_id = self._chain_id ddo.metadata = metadata ddo.credentials = credentials if credentials else {"allow": [], "deny": []} ddo.nft_address = data_nft.address datatokens = [] if not deployed_datatokens: services = [] for datatoken_arg in datatoken_args: new_dt = datatoken_arg.create_datatoken( data_nft, tx_dict, with_services=True ) datatokens.append(new_dt) services.extend(datatoken_arg.services) for service in services: ddo.add_service(service) else: if not services: logger.warning("services required with deployed_datatokens.") return None, None, None datatokens = deployed_datatokens dt_addresses = [] for datatoken in datatokens: if deployed_datatokens[0].address not in data_nft.getTokensList(): logger.warning( "some deployed_datatokens don't belong to the given data nft." ) return None, None, None dt_addresses.append(datatoken.address) for service in services: if service.datatoken not in dt_addresses: logger.warning("Datatoken services mismatch.") return None, None, None ddo.add_service(service) # Validation by Aquarius _, proof = self.validate(ddo) proof = ( proof["publicKey"], proof["v"], proof["r"][0], proof["s"][0], ) document, flags, ddo_hash = self._encrypt_ddo( ddo, provider_uri, encrypt_flag, compress_flag ) wallet_address = get_from_address(tx_dict) data_nft.setMetaData( 0, provider_uri, wallet_address.encode("utf-8"), flags, document, ddo_hash, [proof], tx_dict, ) # Fetch the ddo on chain if wait_for_aqua: ddo = self._aquarius.wait_for_ddo(ddo.did) return (data_nft, datatokens, ddo) @enforce_types def update( self, ddo: DDO, tx_dict: dict, provider_uri: Optional[str] = None, encrypt_flag: Optional[bool] = True, compress_flag: Optional[bool] = True, ) -> Optional[DDO]: """Update a ddo on-chain. :param ddo - DDO to update :param publisher_wallet - who published this ddo :param provider_uri - URL of service provider. This will be used as base to construct the serviceEndpoint for the `access` (download) service :param encrypt_flag - encrypt this DDO? :param compress_flag - compress this DDO? :return - the updated DDO, or None if updated ddo not found in aquarius """ self._assert_ddo_metadata(ddo.metadata) if not provider_uri: provider_uri = DataServiceProvider.get_url(self._config_dict) assert ddo.nft_address, "need nft address to update a ddo" data_nft = DataNFT(self._config_dict, ddo.nft_address) assert ddo.chain_id == self._chain_id for service in ddo.services: service.encrypt_files(ddo.nft_address, ddo.chain_id) # Validation by Aquarius validation_result, errors_or_proof = self.validate(ddo) if not validation_result: msg = f"DDO has validation errors: {errors_or_proof}" logger.error(msg) raise ValueError(msg) document, flags, ddo_hash = self._encrypt_ddo( ddo, provider_uri, encrypt_flag, compress_flag ) proof = ( errors_or_proof["publicKey"], errors_or_proof["v"], errors_or_proof["r"][0], errors_or_proof["s"][0], ) wallet_address = get_from_address(tx_dict) tx_result = data_nft.setMetaData( 0, provider_uri, wallet_address.encode("utf-8"), flags, document, ddo_hash, [proof], tx_dict, ) ddo = self._aquarius.wait_for_ddo_update(ddo, tx_result.transactionHash.hex()) return ddo @enforce_types def resolve(self, did: str) -> "DDO": return self._aquarius.get_ddo(did) @enforce_types def search(self, text: str) -> list: """ Search for DDOs in aquarius that contain the target text string :param text - target string :return - List of DDOs that match with the query """ logger.info(f"Search for DDOs containing text: {text}") text = text.replace(":", "\\:").replace("\\\\:", "\\:") return [ DDO.from_dict(ddo_dict["_source"]) for ddo_dict in self._aquarius.query_search( {"query": {"query_string": {"query": text}}} ) if "_source" in ddo_dict ] @enforce_types def query(self, query: dict) -> list: """ Search for DDOs in aquarius with a search query dict :param query - dict with query parameters More info at: https://docs.oceanprotocol.com/api-references/aquarius-rest-api :return - List of DDOs that match the query. """ logger.info(f"Search for DDOs matching query: {query}") return [ DDO.from_dict(ddo_dict["_source"]) for ddo_dict in self._aquarius.query_search(query) if "_source" in ddo_dict ] @enforce_types def download_asset( self, ddo: DDO, consumer_wallet, destination: str, order_tx_id: Union[str, bytes], service: Optional[Service] = None, index: Optional[int] = None, userdata: Optional[dict] = None, ) -> str: service = service or ddo.services[0] # fill in good default if index is not None: assert isinstance(index, int), logger.error("index has to be an integer.") assert index >= 0, logger.error("index has to be 0 or a positive integer.") assert ( service and service.type == ServiceTypes.ASSET_ACCESS ), f"Service with type {ServiceTypes.ASSET_ACCESS} is not found." path: str = download_asset_files( ddo, service, consumer_wallet, destination, order_tx_id, index, userdata ) return path @enforce_types def pay_for_access_service( self, ddo: DDO, tx_dict: dict, service: Optional[Service] = None, consume_market_fees: Optional[TokenFeeInfo] = None, consumer_address: Optional[str] = None, userdata: Optional[dict] = None, consume_market_swap_fee_amount: Optional[int] = 0, consume_market_swap_fee_address: Optional[str] = ZERO_ADDRESS, ): # fill in good defaults as needed service = service or ddo.services[0] wallet_address = get_from_address(tx_dict) consumer_address = consumer_address or wallet_address consumable_result = is_consumable( ddo, service, {"type": "address", "value": wallet_address}, userdata=userdata, ) if consumable_result != ConsumableCodes.OK: raise AssetNotConsumable(consumable_result) data_provider = DataServiceProvider initialize_args = { "did": ddo.did, "service": service, "consumer_address": consumer_address, } initialize_response = data_provider.initialize(**initialize_args) provider_fees = initialize_response.json()["providerFee"] params = { "consumer": consumer_address, "service_index": ddo.get_index_of_service(service), "provider_fees": provider_fees, "consume_market_fees": consume_market_fees, "tx_dict": tx_dict, } # main work... dt = DatatokenBase.get_typed(self._config_dict, service.datatoken) balance = dt.balanceOf(wallet_address) if balance < to_wei(1): try: params[ "consume_market_swap_fee_amount" ] = consume_market_swap_fee_amount params[ "consume_market_swap_fee_address" ] = consume_market_swap_fee_address receipt = dt.get_from_pricing_schema_and_order(**params) except Exception: receipt = None if receipt: return receipt raise InsufficientBalance( f"Your token balance {balance} {dt.symbol()} is not sufficient " f"to execute the requested service. This service " f"requires 1 wei." ) receipt = dt.start_order(**params) return receipt.transactionHash @enforce_types def pay_for_compute_service( self, datasets: List[ComputeInput], algorithm_data: Union[ComputeInput, AlgorithmMetadata], compute_environment: str, valid_until: int, consume_market_order_fee_address: str, tx_dict: dict, consumer_address: Optional[str] = None, ): data_provider = DataServiceProvider wallet_address = get_from_address(tx_dict) if not consumer_address: consumer_address = wallet_address initialize_response = data_provider.initialize_compute( [x.as_dictionary() for x in datasets], algorithm_data.as_dictionary(), datasets[0].service.service_endpoint, consumer_address, compute_environment, valid_until, ) result = initialize_response.json() for i, item in enumerate(result["datasets"]): self._start_or_reuse_order_based_on_initialize_response( datasets[i], item, TokenFeeInfo( consume_market_order_fee_address, datasets[i].consume_market_order_fee_token, datasets[i].consume_market_order_fee_amount, ), tx_dict, consumer_address, ) if "algorithm" in result: self._start_or_reuse_order_based_on_initialize_response( algorithm_data, result["algorithm"], TokenFeeInfo( address=consume_market_order_fee_address, token=algorithm_data.consume_market_order_fee_token, amount=algorithm_data.consume_market_order_fee_amount, ), tx_dict, consumer_address, ) return datasets, algorithm_data return datasets, None @enforce_types def _start_or_reuse_order_based_on_initialize_response( self, asset_compute_input: ComputeInput, item: dict, consume_market_fees: TokenFeeInfo, tx_dict: dict, consumer_address: Optional[str] = None, ): provider_fees = item.get("providerFee") valid_order = item.get("validOrder") if valid_order and not provider_fees: asset_compute_input.transfer_tx_id = valid_order return service = asset_compute_input.service dt = DatatokenBase.get_typed(self._config_dict, service.datatoken) if valid_order and provider_fees: asset_compute_input.transfer_tx_id = dt.reuse_order( valid_order, provider_fees=provider_fees, tx_dict=tx_dict ).transactionHash.hex() return asset_compute_input.transfer_tx_id = dt.start_order( consumer=consumer_address, service_index=asset_compute_input.ddo.get_index_of_service(service), provider_fees=provider_fees, consume_market_fees=consume_market_fees, tx_dict=tx_dict, ).transactionHash.hex() ================================================ FILE: ocean_lib/ocean/ocean_compute.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import logging from typing import Any, Dict, List, Optional, Type from enforce_typing import enforce_types from ocean_lib.agreements.consumable import AssetNotConsumable, ConsumableCodes from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.aquarius import Aquarius from ocean_lib.assets.asset_downloader import is_consumable from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.models.compute_input import ComputeInput from ocean_lib.services.service import Service from ocean_lib.structures.algorithm_metadata import AlgorithmMetadata logger = logging.getLogger("ocean") class OceanCompute: @enforce_types def __init__( self, config_dict: dict, data_provider: Type[DataServiceProvider] ) -> None: """Initialises OceanCompute class.""" self._config_dict = config_dict self._data_provider = data_provider @enforce_types def start( self, consumer_wallet, dataset: ComputeInput, compute_environment: str, algorithm: Optional[ComputeInput] = None, algorithm_meta: Optional[AlgorithmMetadata] = None, algorithm_algocustomdata: Optional[dict] = None, additional_datasets: List[ComputeInput] = [], ) -> str: metadata_cache_uri = self._config_dict.get("METADATA_CACHE_URI") ddo = Aquarius.get_instance(metadata_cache_uri).get_ddo(dataset.did) service = ddo.get_service_by_id(dataset.service_id) assert ( ServiceTypes.CLOUD_COMPUTE == service.type ), "service at serviceId is not of type compute service." consumable_result = is_consumable( ddo, service, {"type": "address", "value": consumer_wallet.address}, with_connectivity_check=True, ) if consumable_result != ConsumableCodes.OK: raise AssetNotConsumable(consumable_result) # Start compute job job_info = self._data_provider.start_compute_job( dataset_compute_service=service, consumer=consumer_wallet, dataset=dataset, compute_environment=compute_environment, algorithm=algorithm, algorithm_meta=algorithm_meta, algorithm_custom_data=algorithm_algocustomdata, input_datasets=additional_datasets, ) return job_info["jobId"] @enforce_types def status(self, ddo: DDO, service: Service, job_id: str, wallet) -> Dict[str, Any]: """ Gets job status. :param ddo: DDO offering the compute service of this job :param service: compute service of this job :param job_id: str id of the compute job :param wallet: Wallet instance :return: dict the status for an existing compute job, keys are (ok, status, statusText) """ job_info = self._data_provider.compute_job_status( ddo.did, job_id, service, wallet ) job_info.update({"ok": job_info.get("status") not in (31, 32, None)}) return job_info @enforce_types def result( self, ddo: DDO, service: Service, job_id: str, index: int, wallet ) -> Dict[str, Any]: """ Gets job result. :param ddo: DDO offering the compute service of this job :param service: compute service of this job :param job_id: str id of the compute job :param index: compute result index :param wallet: Wallet instance :return: dict the results/logs urls for an existing compute job, keys are (did, urls, logs) """ result = self._data_provider.compute_job_result(job_id, index, service, wallet) return result @enforce_types def compute_job_result_logs( self, ddo: DDO, service: Service, job_id: str, wallet, log_type="output", ) -> Dict[str, Any]: """ Gets job output if exists. :param ddo: DDO offering the compute service of this job :param service: compute service of this job :param job_id: str id of the compute job :param wallet: Wallet instance :return: dict the results/logs urls for an existing compute job, keys are (did, urls, logs) """ result = self._data_provider.compute_job_result_logs( ddo, job_id, service, wallet, log_type ) return result @enforce_types def stop(self, ddo: DDO, service: Service, job_id: str, wallet) -> Dict[str, Any]: """ Attempt to stop the running compute job. :param ddo: DDO offering the compute service of this job :param job_id: str id of the compute job :param wallet: Wallet instance :return: dict the status for the stopped compute job, keys are (ok, status, statusText) """ job_info = self._data_provider.stop_compute_job( ddo.did, job_id, service, wallet ) job_info.update({"ok": job_info.get("status") not in (31, 32, None)}) return job_info @enforce_types def get_c2d_environments(self, service_endpoint: str, chain_id: int) -> str: return DataServiceProvider.get_c2d_environments(service_endpoint, chain_id) @enforce_types def get_free_c2d_environment(self, service_endpoint: str, chain_id) -> str: environments = self.get_c2d_environments(service_endpoint, chain_id) return next(env for env in environments if float(env["priceMin"]) == float(0)) ================================================ FILE: ocean_lib/ocean/test/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: ocean_lib/ocean/test/test_crypto.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from enforce_typing import enforce_types from ocean_lib.ocean import crypto @enforce_types def test_symkey(): base_str = "foo" symkey = crypto.calc_symkey(base_str) assert isinstance(symkey, str) wrong_symkey = crypto.calc_symkey("testwrong") assert wrong_symkey != symkey, "NOK : wrong_sym_key is the same as sym_key" @enforce_types def test_sym_encrypt_decrypt(): symkey = crypto.calc_symkey("1234") value = "hello there" value_enc = crypto.sym_encrypt(value, symkey) assert value_enc != value value2 = crypto.sym_decrypt(value_enc, symkey) assert value2 == value @enforce_types def test_asym_encrypt_decrypt(alice): privkey = alice._private_key.hex() # str pubkey = crypto.calc_pubkey(privkey) # str value = "hello there" value_enc = crypto.asym_encrypt(value, pubkey) assert value_enc != value value2 = crypto.asym_decrypt(value_enc, privkey) assert value2 == value ================================================ FILE: ocean_lib/ocean/test/test_ocean.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.models.datatoken1 import Datatoken1 from ocean_lib.models.df.df_rewards import DFRewards from ocean_lib.models.df.df_strategy_v1 import DFStrategyV1 from ocean_lib.models.dispenser import Dispenser from ocean_lib.models.factory_router import FactoryRouter from ocean_lib.models.fixed_rate_exchange import FixedRateExchange from ocean_lib.models.ve.smart_wallet_checker import SmartWalletChecker from ocean_lib.models.ve.ve_allocate import VeAllocate from ocean_lib.models.ve.ve_delegation import VeDelegation from ocean_lib.models.ve.ve_fee_distributor import VeFeeDistributor from ocean_lib.models.ve.ve_fee_estimate import VeFeeEstimate from ocean_lib.models.ve.ve_ocean import VeOcean from tests.resources.helper_functions import deploy_erc721_erc20 @pytest.mark.unit def test_nft_factory(config, publisher_ocean, publisher_wallet): data_nft, datatoken = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet ) ocean = publisher_ocean assert ocean.data_nft_factory assert ocean.get_nft_token(data_nft.address).address == data_nft.address assert ocean.get_datatoken(datatoken.address).address == datatoken.address @pytest.mark.unit def test_contract_objects(publisher_ocean): ocean = publisher_ocean assert ocean.OCEAN_address[:2] == "0x" assert isinstance(ocean.OCEAN_token, Datatoken1) assert isinstance(ocean.OCEAN, Datatoken1) assert ocean.OCEAN_address == ocean.OCEAN_token.address assert ocean.OCEAN_address == ocean.OCEAN.address assert isinstance(ocean.data_nft_factory, DataNFTFactoryContract) assert isinstance(ocean.dispenser, Dispenser) assert isinstance(ocean.fixed_rate_exchange, FixedRateExchange) assert isinstance(ocean.factory_router, FactoryRouter) assert isinstance(ocean.df_rewards, DFRewards) assert isinstance(ocean.df_strategy_v1, DFStrategyV1) assert isinstance(ocean.smart_wallet_checker, SmartWalletChecker) assert isinstance(ocean.ve_allocate, VeAllocate) assert isinstance(ocean.ve_delegation, VeDelegation) assert isinstance(ocean.ve_fee_distributor, VeFeeDistributor) assert isinstance(ocean.ve_fee_estimate, VeFeeEstimate) assert isinstance(ocean.ve_ocean, VeOcean) assert isinstance(ocean.veOCEAN, VeOcean) assert ocean.config == ocean.config_dict # test alias ================================================ FILE: ocean_lib/ocean/test/test_ocean_assets.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import copy from datetime import datetime, timezone from unittest.mock import patch import pytest from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.example_config import DEFAULT_PROVIDER_URL from ocean_lib.exceptions import AquariusError, InsufficientBalance from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.models.datatoken_base import DatatokenArguments, TokenFeeInfo from ocean_lib.models.dispenser import DispenserArguments from ocean_lib.models.fixed_rate_exchange import ExchangeArguments from ocean_lib.ocean.ocean_assets import OceanAssets from ocean_lib.ocean.util import get_address_of_type, to_wei from ocean_lib.services.service import Service from ocean_lib.web3_internal.utils import get_gas_fees from tests.resources.ddo_helpers import ( build_credentials_dict, build_default_services, get_default_files, get_default_metadata, get_first_service_by_type, get_registered_asset_with_access_service, get_sample_ddo, ) from tests.resources.helper_functions import deploy_erc721_erc20 @pytest.mark.integration def test_register_asset(publisher_ocean): invalid_did = "did:op:0123456789" assert publisher_ocean.assets.resolve(invalid_did) is None @pytest.mark.integration def test_update(publisher_ocean, publisher_wallet, config): data_nft, _, ddo = get_registered_asset_with_access_service( publisher_ocean, publisher_wallet ) new_metadata = copy.deepcopy(ddo.metadata) # Update metadata _description = "Updated description" new_metadata["description"] = _description new_metadata["updated"] = datetime.now(timezone.utc).isoformat() ddo.metadata = new_metadata # Update credentials _new_credentials = { "allow": [{"type": "address", "values": ["0x123", "0x456"]}], "deny": [{"type": "address", "values": ["0x2222", "0x333"]}], } ddo.credentials = _new_credentials ddo2 = publisher_ocean.assets.update(ddo, {"from": publisher_wallet}) # Check metadata update assert ddo2.datatokens == ddo.datatokens assert len(ddo2.services) == len(ddo.services) assert ddo2.services[0].as_dictionary() == ddo.services[0].as_dictionary() assert ddo2.credentials == ddo.credentials assert ddo2.metadata["description"] == _description assert ddo2.metadata["updated"] == new_metadata["updated"] # Check credentials update assert ddo2.credentials == _new_credentials, "Credentials were not updated." # Check flags update registered_token_event = data_nft.get_logs( "MetadataUpdated", ddo2.event.get("block"), config["web3_instance"].eth.get_block("latest").number, ) assert registered_token_event[0].args.get("flags") == bytes([3]) @pytest.mark.integration def test_update_datatokens(publisher_ocean, publisher_wallet, config, file2): _, datatoken = deploy_erc721_erc20(config, publisher_wallet, publisher_wallet) _, _, ddo = get_registered_asset_with_access_service( publisher_ocean, publisher_wallet ) files = [file2] # Add new existing datatoken with service ddo_orig = copy.deepcopy(ddo) access_service = Service( service_id="3", service_type=ServiceTypes.ASSET_ACCESS, service_endpoint=DEFAULT_PROVIDER_URL, datatoken=datatoken.address, files=files, timeout=0, ) ddo.datatokens.append( { "address": datatoken.address, "name": datatoken.name(), "symbol": datatoken.symbol(), "serviceId": access_service.id, } ) ddo.services.append(access_service) ddo2 = publisher_ocean.assets.update(ddo, {"from": publisher_wallet}) assert len(ddo2.datatokens) == len(ddo_orig.datatokens) + 1 assert len(ddo2.services) == len(ddo_orig.services) + 1 assert ddo2.datatokens[1].get("address") == datatoken.address assert ddo2.datatokens[0].get("address") == ddo_orig.datatokens[0].get("address") assert ddo2.services[0].datatoken == ddo_orig.datatokens[0].get("address") assert ddo2.services[1].datatoken == datatoken.address # Delete datatoken ddo3 = copy.deepcopy(ddo2) metadata3 = copy.deepcopy(ddo2.metadata) _description = "Test delete datatoken" metadata3["description"] = _description metadata3["updated"] = datetime.now(timezone.utc).isoformat() removed_dt = ddo3.datatokens.pop() ddo3.services = [ service for service in ddo3.services if service.datatoken != removed_dt.get("address") ] ddo2_prev_datatokens = ddo2.datatokens ddo4 = publisher_ocean.assets.update(ddo3, {"from": publisher_wallet}) assert ddo4, "Can't read ddo after update." assert len(ddo4.datatokens) == 1 assert ddo4.datatokens[0].get("address") == ddo2_prev_datatokens[0].get("address") assert ddo4.services[0].datatoken == ddo2_prev_datatokens[0].get("address") nft_token = publisher_ocean.get_nft_token(ddo4.nft["address"]) bn = config["web3_instance"].eth.get_block("latest").number updated_event = nft_token.get_logs("MetadataUpdated", bn, bn)[0] assert updated_event.args.updatedBy == publisher_wallet.address validation_event = nft_token.get_logs("MetadataValidated", bn, bn)[0] assert validation_event.args.validator.startswith("0x") assert updated_event.transactionHash == validation_event.transactionHash @pytest.mark.integration def test_ocean_assets_search(): # skipping as tested by the search-and-filter readme assert True @pytest.mark.integration def test_ocean_assets_validate(publisher_ocean): ddo_dict = get_sample_ddo() ddo = DDO.from_dict(ddo_dict) assert publisher_ocean.assets.validate( ddo ), "ddo should be valid, unless the schema changed" ddo_dict = get_sample_ddo() ddo_dict["id"] = "something not conformant" ddo = DDO.from_dict(ddo_dict) with pytest.raises(ValueError): publisher_ocean.assets.validate(ddo) @pytest.mark.integration def test_ocean_assets_algorithm(): # skipped because it is covered by c2d tests assert True @pytest.mark.unit def test_download_fails(publisher_ocean, publisher_wallet): with patch("ocean_lib.ocean.ocean_assets.OceanAssets.resolve") as mock: ddo = DDO.from_dict(get_sample_ddo()) mock.return_value = ddo with pytest.raises(AssertionError): publisher_ocean.assets.download_asset( ddo, publisher_wallet, destination="", order_tx_id="", service=ddo.services[0], index=-4, ) with pytest.raises(TypeError): publisher_ocean.assets.download_asset( ddo, publisher_wallet, destination="", order_tx_id="", service=ddo.services[0], index="string_index", ) @pytest.mark.integration def test_create_bad_metadata(publisher_ocean, publisher_wallet): metadata = { "created": "2020-11-15T12:27:48Z", "updated": "2021-05-17T21:58:02Z", "description": "Sample description", # name missing intentionally "type": "dataset", "author": "OPF", "license": "https://market.oceanprotocol.com/terms", } with pytest.raises(AssertionError): get_registered_asset_with_access_service( publisher_ocean, publisher_wallet, metadata ) metadata["name"] = "Sample asset" metadata.pop("type") with pytest.raises(AssertionError): get_registered_asset_with_access_service( publisher_ocean, publisher_wallet, metadata ) @pytest.mark.integration def test_create_url_asset(): # skipped because this functionality is intrinsic to the basic_asset fixture assert True @pytest.mark.integration def test_plain_asset_with_one_datatoken(publisher_ocean, publisher_wallet, config): data_nft_factory = DataNFTFactoryContract( config, get_address_of_type(config, "ERC721Factory") ) metadata = get_default_metadata() files = get_default_files() # Publisher deploys NFT contract data_nft = data_nft_factory.create({"from": publisher_wallet}, "NFT1", "NFTSYMBOL") _, _, ddo = publisher_ocean.assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, data_nft_address=data_nft.address, datatoken_args=[DatatokenArguments(files=files)], ) assert ddo, "The ddo is not created." assert ddo.nft["name"] == "NFT1" assert ddo.nft["symbol"] == "NFTSYMBOL" assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "Datatoken 1" assert ddo.datatokens[0]["symbol"] == "DT1" assert ddo.credentials == build_credentials_dict() @pytest.mark.integration def test_plain_asset_multiple_datatokens(publisher_ocean, publisher_wallet, config): data_nft_factory = DataNFTFactoryContract( config, get_address_of_type(config, "ERC721Factory") ) metadata = get_default_metadata() files = get_default_files() data_nft = data_nft_factory.create({"from": publisher_wallet}, "NFT2", "NFT2SYMBOL") _, _, ddo = publisher_ocean.assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, data_nft_address=data_nft.address, datatoken_args=[ DatatokenArguments("Datatoken 2", "DT2", files=files), DatatokenArguments("Datatoken 3", "DT3", files=files), ], ) assert ddo, "The ddo is not created." assert ddo.nft["name"] == "NFT2" assert ddo.nft["symbol"] == "NFT2SYMBOL" assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "Datatoken 2" assert ddo.datatokens[0]["symbol"] == "DT2" assert ddo.datatokens[1]["name"] == "Datatoken 3" assert ddo.datatokens[1]["symbol"] == "DT3" assert len(ddo.services) == 2 assert len(ddo.datatokens) == 2 assert ddo.credentials == build_credentials_dict() datatoken_names = [] for datatoken in ddo.datatokens: datatoken_names.append(datatoken["name"]) assert datatoken_names[0] == "Datatoken 2" assert datatoken_names[1] == "Datatoken 3" @pytest.mark.integration def test_plain_asset_multiple_services(publisher_ocean, publisher_wallet, config): data_nft, datatoken = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet ) metadata = get_default_metadata() files = get_default_files() access_service = Service( service_id="0", service_type=ServiceTypes.ASSET_ACCESS, service_endpoint=DEFAULT_PROVIDER_URL, datatoken=datatoken.address, files=files, timeout=0, ) # Set the compute values for compute service compute_values = { "namespace": "ocean-compute", "cpus": 2, "gpus": 4, "gpuType": "NVIDIA Tesla V100 GPU", "memory": "128M", "volumeSize": "2G", "allowRawAlgorithm": False, "allowNetworkAccess": True, } compute_service = Service( service_id="1", service_type=ServiceTypes.CLOUD_COMPUTE, service_endpoint=DEFAULT_PROVIDER_URL, datatoken=datatoken.address, files=files, timeout=3600, compute_values=compute_values, ) _, _, ddo = publisher_ocean.assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, services=[access_service, compute_service], data_nft_address=data_nft.address, deployed_datatokens=[datatoken], ) assert ddo, "The ddo is not created." assert ddo.nft["name"] == "NFT" assert ddo.nft["symbol"] == "NFTSYMBOL" assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "DT1" assert ddo.datatokens[0]["symbol"] == "DT1Symbol" assert ddo.datatokens[0]["address"] == datatoken.address assert ddo.credentials == build_credentials_dict() assert ddo.services[1].compute_values == compute_values @pytest.mark.integration def test_encrypted_asset(publisher_ocean, publisher_wallet, config): data_nft, datatoken = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet ) metadata = get_default_metadata() services = build_default_services(config, datatoken) _, _, ddo = publisher_ocean.assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, data_nft_address=data_nft.address, deployed_datatokens=[datatoken], services=services, encrypt_flag=True, ) assert ddo, "The ddo is not created." assert ddo.nft["name"] == "NFT" assert ddo.nft["symbol"] == "NFTSYMBOL" assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "DT1" assert ddo.datatokens[0]["symbol"] == "DT1Symbol" assert ddo.datatokens[0]["address"] == datatoken.address @pytest.mark.integration def test_compressed_asset(publisher_ocean, publisher_wallet, config): data_nft, datatoken = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet ) metadata = get_default_metadata() services = build_default_services(config, datatoken) _, _, ddo = publisher_ocean.assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, services=services, data_nft_address=data_nft.address, deployed_datatokens=[datatoken], compress_flag=True, ) assert ddo, "The ddo is not created." assert ddo.nft["name"] == "NFT" assert ddo.nft["symbol"] == "NFTSYMBOL" assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "DT1" assert ddo.datatokens[0]["symbol"] == "DT1Symbol" assert ddo.datatokens[0]["address"] == datatoken.address @pytest.mark.integration def test_compressed_and_encrypted_asset(publisher_ocean, publisher_wallet, config): data_nft, datatoken = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet ) metadata = get_default_metadata() services = build_default_services(config, datatoken) _, _, ddo = publisher_ocean.assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, services=services, data_nft_address=data_nft.address, deployed_datatokens=[datatoken], encrypt_flag=True, compress_flag=True, ) assert ddo, "The ddo is not created." assert ddo.nft["name"] == "NFT" assert ddo.nft["symbol"] == "NFTSYMBOL" assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "DT1" assert ddo.datatokens[0]["symbol"] == "DT1Symbol" assert ddo.datatokens[0]["address"] == datatoken.address @pytest.mark.unit def test_asset_creation_errors(publisher_ocean, publisher_wallet, config): data_nft, datatoken = deploy_erc721_erc20( config, publisher_wallet, publisher_wallet ) metadata = get_default_metadata() with patch("ocean_lib.aquarius.aquarius.Aquarius.ddo_exists") as mock: mock.return_value = True with pytest.raises(AquariusError): publisher_ocean.assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, services=[], data_nft_address=data_nft.address, deployed_datatokens=[datatoken], encrypt_flag=True, ) @pytest.mark.integration def test_create_algo_asset(publisher_ocean, publisher_wallet): ocean = publisher_ocean name = "Branin dataset" url = "https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/branin_and_gpr/gpr.py" (data_nft, datatoken, ddo) = ocean.assets.create_algo_asset( name, url, {"from": publisher_wallet} ) assert ddo.nft["name"] == name # thorough testing is below, on create() directly assert len(ddo.datatokens) == 1 @pytest.mark.integration def test_create_url_asset_with_gas_strategy( config, publisher_wallet, consumer_wallet, consumer_ocean, OCEAN ): """Directly models READMEs/gas-strategy-remote.md behavior""" data_provider = DataServiceProvider ocean_assets = OceanAssets(config, data_provider) priority_fee, max_fee = get_gas_fees() name = "Branin dataset" url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" tx_dict = { "from": publisher_wallet, "maxPriorityFeePerGas": priority_fee, "maxFeePerGas": max_fee, } ddo = ocean_assets.create_url_asset( name, url, tx_dict, dt_template_index=1, wait_for_aqua=False, ) assert ddo @pytest.mark.integration def test_create_pricing_schemas( config, publisher_wallet, consumer_wallet, consumer_ocean, OCEAN ): data_provider = DataServiceProvider ocean_assets = OceanAssets(config, data_provider) url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" ddo_set = {} for dt_template_index in [2, 1]: ddo_set[dt_template_index] = {} # No pricing schema ddo_set[dt_template_index]["np"] = ocean_assets.create_url_asset( "Data NFTs in Ocean", url, {"from": publisher_wallet}, dt_template_index=dt_template_index, wait_for_aqua=False, ) ddo_set[dt_template_index]["disp"] = ocean_assets.create_url_asset( "Data NFTs in Ocean", url, {"from": publisher_wallet}, dt_template_index=dt_template_index, pricing_schema_args=DispenserArguments(to_wei(1), to_wei(1)), wait_for_aqua=False, ) ddo_set[dt_template_index]["ex"] = ocean_assets.create_url_asset( "Data NFTs in Ocean", url, {"from": publisher_wallet}, dt_template_index=dt_template_index, pricing_schema_args=ExchangeArguments( rate=to_wei(3), base_token_addr=OCEAN.address, dt_decimals=18 ), ) for dt_template_index in [2, 1]: data_nft_np, dt_np, ddo_np = ddo_set[dt_template_index]["np"] data_nft_disp, dt_disp, ddo_disp = ddo_set[dt_template_index]["disp"] data_nft_ex, dt_ex, ddo_ex = ddo_set[dt_template_index]["ex"] ddo_np = ocean_assets._aquarius.wait_for_ddo(ddo_np.did) ddo_disp = ocean_assets._aquarius.wait_for_ddo(ddo_disp.did) ddo_ex = ocean_assets._aquarius.wait_for_ddo(ddo_ex.did) assert not dt_np.dispenser_status().active assert dt_np.get_exchanges() == [] # pay_for_access service has insufficient balance and can't buy or dispense empty_wallet = config["web3_instance"].eth.account.create() with pytest.raises(InsufficientBalance): ocean_assets.pay_for_access_service( ddo_np, {"from": empty_wallet}, get_first_service_by_type(ddo_np, "access"), TokenFeeInfo(address=empty_wallet.address, token=dt_np.address), ) assert dt_disp.dispenser_status().active assert dt_disp.get_exchanges() == [] # pay_for_access service has insufficient balance but dispenses automatically _ = consumer_ocean.assets.pay_for_access_service( ddo_disp, {"from": consumer_wallet} ) assert not dt_ex.dispenser_status().active assert len(dt_ex.get_exchanges()) == 1 assert dt_ex.get_exchanges()[0].details.base_token == OCEAN.address # pay_for_access service has insufficient balance but buys 1 datatoken automatically from the exchange _ = consumer_ocean.assets.pay_for_access_service( ddo_ex, {"from": consumer_wallet} ) ================================================ FILE: ocean_lib/ocean/test/test_util.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from web3 import Web3 from ocean_lib.ocean import util from ocean_lib.ocean.util import ( from_wei, get_address_of_type, get_ocean_token_address, str_with_wei, to_wei, ) @pytest.mark.unit def test_get_ocean_token_address(config): addresses = util.get_contracts_addresses(config) assert addresses assert isinstance(addresses, dict) assert "Ocean" in addresses address = get_ocean_token_address(config) assert Web3.is_checksum_address(address), "It is not a checksum token address." assert address == Web3.to_checksum_address(addresses["Ocean"]) @pytest.mark.unit def test_get_address_by_type(config): addresses = util.get_contracts_addresses(config) address = get_address_of_type(config, "Ocean") assert Web3.is_checksum_address(address), "It is not a checksum token address." assert address == Web3.to_checksum_address(addresses["Ocean"]) @pytest.mark.unit def test_get_address_of_type_failure(config): with pytest.raises(KeyError): get_address_of_type(config, "", "non-existent-key") @pytest.mark.unit def test_wei(): assert from_wei(int(1234 * 1e18)) == 1234 assert from_wei(int(12.34 * 1e18)) == 12.34 assert from_wei(int(0.1234 * 1e18)) == 0.1234 assert to_wei(1234) == 1234 * 1e18 and type(to_wei(1234)) == int assert to_wei(12.34) == 12.34 * 1e18 and type(to_wei(12.34)) == int assert to_wei(0.1234) == 0.1234 * 1e18 and type(to_wei(0.1234)) == int assert str_with_wei(int(12.34 * 1e18)) == "12.34 (12340000000000000000 wei)" ================================================ FILE: ocean_lib/ocean/util.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import hashlib from typing import Optional, Union from enforce_typing import enforce_types from web3.main import Web3 from ocean_lib.web3_internal.contract_utils import get_contracts_addresses GANACHE_URL = "http://127.0.0.1:8545" @enforce_types def get_address_of_type( config_dict: dict, address_type: str, key: Optional[str] = None ) -> str: addresses = get_contracts_addresses(config_dict) if address_type not in addresses.keys(): raise KeyError(f"{address_type} address is not set in the config file") address = ( addresses[address_type] if not isinstance(addresses[address_type], dict) else addresses[address_type].get(key, addresses[address_type]["1"]) ) return Web3.to_checksum_address(address.lower()) @enforce_types def get_ocean_token_address(config_dict: dict) -> str: """Returns the Ocean token address for given network or web3 instance Requires either network name or web3 instance. """ addresses = get_contracts_addresses(config_dict) return ( Web3.to_checksum_address(addresses.get("Ocean").lower()) if addresses else None ) @enforce_types def create_checksum(text: str) -> str: """ :return: str """ return hashlib.sha256(text.encode("utf-8")).hexdigest() @enforce_types def from_wei(amt_wei: int): return float(amt_wei / 1e18) @enforce_types def to_wei(amt_eth) -> int: return int(amt_eth * 1e18) @enforce_types def str_with_wei(amt_wei: int) -> str: return f"{from_wei(amt_wei)} ({amt_wei} wei)" @enforce_types def get_from_address(tx_dict: dict) -> str: address = tx_dict["from"].address return Web3.to_checksum_address(address.lower()) @enforce_types def get_args_object(args, kwargs, args_class): args_to_use = None if args and isinstance(args[0], args_class): args_to_use = args[0] elif kwargs: for key, value in kwargs.items(): if isinstance(value, args_class): args_to_use = value break if not args_to_use: args_to_use = args_class(*args, **kwargs) return args_to_use @enforce_types def send_ether( config, from_wallet, to_address: str, amount: Union[int, float], priority_fee=None ): if not Web3.is_checksum_address(to_address): to_address = Web3.to_checksum_address(to_address) web3 = config["web3_instance"] chain_id = web3.eth.chain_id tx = { "from": from_wallet.address, "to": to_address, "value": amount, "chainId": chain_id, "nonce": web3.eth.get_transaction_count(from_wallet.address), "type": 2, } tx["gas"] = web3.eth.estimate_gas(tx) if not priority_fee: priority_fee = web3.eth.max_priority_fee base_fee = web3.eth.get_block("latest")["baseFeePerGas"] tx["maxPriorityFeePerGas"] = priority_fee tx["maxFeePerGas"] = base_fee * 2 + priority_fee signed_tx = web3.eth.account.sign_transaction(tx, from_wallet._private_key) tx_hash = web3.eth.send_raw_transaction(signed_tx.rawTransaction) return web3.eth.wait_for_transaction_receipt(tx_hash) ================================================ FILE: ocean_lib/services/consumer_parameters.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """ Consumer Parameters Class for V4 To handle the consumerParameters key of a service in a DDO record """ import copy import logging from distutils.util import strtobool from typing import Any, Dict, List, Optional from enforce_typing import enforce_types logger = logging.getLogger(__name__) class ConsumerParameters: def __init__( self, name: str, type: str, label: str, required: bool, default: str, description: str, options: Optional[List[str]] = None, ) -> None: fn_args = locals().copy() for attr_name in ConsumerParameters.required_attrs(): setattr(self, attr_name, fn_args[attr_name]) if options is not None and not isinstance(options, list): raise TypeError("Options should be a list") self.options = options @classmethod def from_dict( cls, consumer_parameters_dict: Dict[str, Any] ) -> "ConsumerParameters": """Create a ConsumerParameters object from a JSON string.""" cpd = copy.deepcopy(consumer_parameters_dict) missing_attributes = [ x for x in ConsumerParameters.required_attrs() if x not in cpd.keys() ] if missing_attributes: raise TypeError( "ConsumerParameters is missing the keys " + ", ".join(missing_attributes) ) required = cpd["required"] if "required" in cpd else None return cls( cpd["name"], cpd["type"], cpd["label"], bool(strtobool(required)) if isinstance(required, str) else required, cpd["default"], cpd["description"], cpd.pop("options", None), ) @enforce_types def as_dictionary(self) -> Dict[str, Any]: """Return the consume parameters object as a python dictionary.""" result = { attr_name: getattr(self, attr_name) for attr_name in ConsumerParameters.required_attrs() } if self.options is not None: result["options"] = self.options return result @staticmethod @enforce_types def required_attrs(): return [ "name", "type", "label", "required", "default", "description", ] ================================================ FILE: ocean_lib/services/service.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """ Service Class for V4 To handle service items in a DDO record """ import copy import logging import re from typing import Any, Dict, List, Optional, Union from enforce_typing import enforce_types from requests.models import Response from web3.main import Web3 from ocean_lib.agreements.service_types import ServiceTypes, ServiceTypesNames from ocean_lib.data_provider.data_encryptor import DataEncryptor from ocean_lib.services.consumer_parameters import ConsumerParameters from ocean_lib.structures.file_objects import FilesType logger = logging.getLogger(__name__) class Service: """Service class to create validate service in a V4 DDO.""" def __init__( self, service_id: str, service_type: str, service_endpoint: Optional[str], datatoken: Optional[str], files: Optional[Union[List[FilesType], str]], timeout: Optional[int], compute_values: Optional[Dict[str, Any]] = None, name: Optional[str] = None, description: Optional[str] = None, additional_information: Optional[Dict[str, Any]] = None, consumer_parameters=None, ) -> None: """Initialize NFT Service instance.""" self.id = service_id self.type = service_type self.service_endpoint = service_endpoint self.datatoken = datatoken self.files = files self.timeout = timeout self.compute_values = compute_values self.name = name self.description = description self.additional_information = None self.consumer_parameters = consumer_parameters if consumer_parameters: try: self.consumer_parameters = [ ConsumerParameters.from_dict(cp_dict) for cp_dict in consumer_parameters ] except AttributeError: raise TypeError("ConsumerParameters should be a list of dictionaries.") if additional_information: self.additional_information = additional_information if not name or not description: service_to_default_name = { ServiceTypes.ASSET_ACCESS: ServiceTypesNames.DEFAULT_ACCESS_NAME, ServiceTypes.CLOUD_COMPUTE: ServiceTypesNames.DEFAULT_COMPUTE_NAME, } if service_type in service_to_default_name: self.name = service_to_default_name[service_type] self.description = service_to_default_name[service_type] @classmethod def from_dict(cls, service_dict: Dict[str, Any]) -> "Service": """Create a service object from a JSON string.""" sd = copy.deepcopy(service_dict) service_type = sd.pop("type", None) if not service_type: logger.error( 'Service definition in DDO document is missing the "type" key/value.' ) raise IndexError return cls( sd.pop("id", None), service_type, sd.pop("serviceEndpoint", None), sd.pop("datatokenAddress", None), sd.pop("files", None), sd.pop("timeout", None), sd.pop("compute", None), sd.pop("name", None), sd.pop("description", None), sd.pop("additionalInformation", None), sd.pop("consumerParameters", None), ) def get_trusted_algorithms(self) -> list: return self.compute_values.get("publisherTrustedAlgorithms", []) def get_trusted_algorithm_publishers(self) -> list: return self.compute_values.get("publisherTrustedAlgorithmPublishers", []) # Not type provided due to circular imports def add_publisher_trusted_algorithm(self, algo_ddo) -> list: """ :return: List of trusted algos """ if self.type != ServiceTypes.CLOUD_COMPUTE: raise AssertionError("Service is not compute type") initial_trusted_algos_v4 = self.get_trusted_algorithms() # remove algo_did if already in the list trusted_algos = [ ta for ta in initial_trusted_algos_v4 if ta["did"] != algo_ddo.did ] initial_count = len(trusted_algos) generated_trusted_algo_dict = algo_ddo.generate_trusted_algorithms() trusted_algos.append(generated_trusted_algo_dict) # update with the new list self.compute_values["publisherTrustedAlgorithms"] = trusted_algos assert ( len(self.compute_values["publisherTrustedAlgorithms"]) > initial_count ), "New trusted algorithm was not added. Failed when updating the privacy key. " return trusted_algos def add_publisher_trusted_algorithm_publisher(self, publisher_address: str) -> list: trusted_algo_publishers = [ Web3.to_checksum_address(tp) for tp in self.get_trusted_algorithm_publishers() ] publisher_address = Web3.to_checksum_address(publisher_address) if publisher_address in trusted_algo_publishers: return trusted_algo_publishers initial_len = len(trusted_algo_publishers) trusted_algo_publishers.append(publisher_address) # update with the new list self.compute_values[ "publisherTrustedAlgorithmPublishers" ] = trusted_algo_publishers assert ( len(self.compute_values["publisherTrustedAlgorithmPublishers"]) > initial_len ), "New trusted algorithm was not added. Failed when updating the privacy key. " return trusted_algo_publishers def as_dictionary(self) -> Dict[str, Any]: """Return the service as a python dictionary.""" # camelCase to snake case dict, matching the dict value to the attribute name key_names = { x: re.sub("([A-Z]+)", r"_\1", x).lower() for x in [ "name", "description", "id", "type", "files", "datatokenAddress", "serviceEndpoint", "timeout", "additionalInformation", "consumerParameters", ] } key_names["datatokenAddress"] = "datatoken" optional_keys = [ "name", "description", "additionalInformation", "consumerParameters", ] values = {} if self.type == "compute": if "compute" in self.compute_values: values.update(self.compute_values) else: values["compute"] = self.compute_values for key, attr_name in key_names.items(): value = getattr(self, attr_name) if isinstance(value, object) and hasattr(value, "as_dictionary"): value = value.as_dictionary() elif isinstance(value, list): value = [ v.as_dictionary() if hasattr(v, "as_dictionary") else v for v in value ] if key in optional_keys and value is None: continue values[key] = value return values def remove_publisher_trusted_algorithm(self, algo_did: str) -> list: """Returns a trusted algorithms list after removal.""" trusted_algorithms = self.get_trusted_algorithms() if not trusted_algorithms: raise ValueError( f"Algorithm {algo_did} is not in trusted algorithms of this asset." ) trusted_algorithms = [ta for ta in trusted_algorithms if ta["did"] != algo_did] trusted_algo_publishers = self.get_trusted_algorithm_publishers() self.update_compute_values( trusted_algorithms, trusted_algo_publishers, allow_network_access=True, allow_raw_algorithm=False, ) assert ( self.compute_values["publisherTrustedAlgorithms"] == trusted_algorithms ), "New trusted algorithm was not removed. Failed when updating the list of trusted algorithms. " return trusted_algorithms def remove_publisher_trusted_algorithm_publisher( self, publisher_address: str ) -> list: """ :return: List of trusted algo publishers not containing `publisher_address`. """ trusted_algorithm_publishers = [ tp.lower() for tp in self.get_trusted_algorithm_publishers() ] publisher_address = publisher_address.lower() if not trusted_algorithm_publishers: raise ValueError( f"Publisher {publisher_address} is not in trusted algorithm publishers of this asset." ) trusted_algorithm_publishers = [ tp for tp in trusted_algorithm_publishers if tp != publisher_address ] trusted_algorithms = self.get_trusted_algorithms() self.update_compute_values( trusted_algorithms, trusted_algorithm_publishers, True, False ) assert ( self.compute_values["publisherTrustedAlgorithmPublishers"] == trusted_algorithm_publishers ), "New trusted algorithm publisher was not removed. Failed when updating the list of trusted algo publishers. " return trusted_algorithm_publishers def update_compute_values( self, trusted_algorithms: List, trusted_algo_publishers: Optional[List], allow_network_access: bool, allow_raw_algorithm: bool, ) -> None: """Set the `trusted_algorithms` on the compute service. - An assertion is raised if this asset has no compute service - Updates the compute service in place - Adds the trusted algorithms under privacy.publisherTrustedAlgorithms :param trusted_algorithms: list of dicts, each dict contain the keys ("containerSectionChecksum", "filesChecksum", "did") :param trusted_algo_publishers: list of strings, addresses of trusted publishers :param allow_network_access: bool -- set to True to allow network access to all the algorithms that belong to this dataset :param allow_raw_algorithm: bool -- determine whether raw algorithms (i.e. unpublished) can be run on this dataset :return: None :raises AssertionError if this asset has no `ServiceTypes.CLOUD_COMPUTE` service """ assert not trusted_algorithms or isinstance(trusted_algorithms, list) assert ( self.type == ServiceTypes.CLOUD_COMPUTE is not None ), "this asset does not have a compute service." for ta in trusted_algorithms: assert isinstance( ta, dict ), f"item in list of trusted_algorithms must be a dict, got {ta}" assert ( "did" in ta ), f"dict in list of trusted_algorithms is expected to have a `did` key, got {ta.keys()}." if not self.compute_values: self.compute_values = {} self.compute_values["publisherTrustedAlgorithms"] = trusted_algorithms self.compute_values[ "publisherTrustedAlgorithmPublishers" ] = trusted_algo_publishers self.compute_values["allowNetworkAccess"] = allow_network_access self.compute_values["allowRawAlgorithm"] = allow_raw_algorithm @enforce_types def encrypt_files(self, nft_address: str, chain_id: int) -> Response: if self.files and isinstance(self.files, str): return files = list(map(lambda file: file.to_dict(), self.files)) encrypt_response = DataEncryptor.encrypt( { "datatokenAddress": self.datatoken, "nftAddress": nft_address, "files": files, }, self.service_endpoint, chain_id, ) self.files = encrypt_response.content.decode("utf-8") ================================================ FILE: ocean_lib/services/test/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: ocean_lib/services/test/test_consumer_parameters.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.services.consumer_parameters import ConsumerParameters @pytest.mark.unit def test_consumer_parameters(): """Tests the Consumer Parameters key/object.""" cp_dict = { "name": "test_key", "type": "string", "label": "test_key_label", "required": True, "default": "value", "description": "this is a test key", "options": ["a", "b"], } cp_object = ConsumerParameters.from_dict(cp_dict) assert cp_object.as_dictionary() == cp_dict cp_dict.pop("options") cp_object = ConsumerParameters.from_dict(cp_dict) assert cp_object.as_dictionary() == cp_dict cp_dict["required"] = "false" # explicitly false, not missing cp_object = ConsumerParameters.from_dict(cp_dict) assert cp_object.as_dictionary()["required"] is False cp_dict["options"] = "not an array" with pytest.raises(TypeError): cp_object = ConsumerParameters.from_dict(cp_dict) cp_dict.pop("options") cp_dict.pop("type") cp_dict.pop("label") with pytest.raises(TypeError, match="is missing the keys type, label"): cp_object = ConsumerParameters.from_dict(cp_dict) ================================================ FILE: ocean_lib/services/test/test_service.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from unittest.mock import Mock, patch import pytest from requests.models import Response from ocean_lib.assets.ddo import DDO from ocean_lib.services.service import Service from tests.resources.ddo_helpers import ( get_sample_algorithm_ddo, get_sample_ddo, get_sample_ddo_with_compute_service, ) @pytest.mark.unit def test_service(): """Tests that the get_cost function for ServiceAgreement returns the correct value.""" ddo_dict = get_sample_ddo() service_dict = ddo_dict["services"][0] service_dict["additionalInformation"] = {"message": "Sample DDO"} cp_dict = { "name": "some_key", "type": "string", "label": "test_key_label", "required": True, "default": "value", "description": "this is a test key", } service_dict["consumerParameters"] = [cp_dict] sa = Service.from_dict(service_dict) assert sa.id == "1" assert sa.name == "Download service" assert sa.type == "access" assert sa.service_endpoint == "http://172.15.0.4:8030" assert sa.datatoken == "0x123" assert sa.additional_information == {"message": "Sample DDO"} assert sa.as_dictionary() == { "id": "1", "type": "access", "serviceEndpoint": "http://172.15.0.4:8030", "datatokenAddress": "0x123", "files": "0x0000", "timeout": 0, "name": "Download service", "description": "Download service", "additionalInformation": {"message": "Sample DDO"}, "consumerParameters": [cp_dict], } ddo_dict = get_sample_ddo() service_dict = ddo_dict["services"][0] del service_dict["type"] with pytest.raises(IndexError): Service.from_dict(service_dict) ddo_dict = get_sample_ddo() service_dict = ddo_dict["services"][0] service_dict["consumerParameters"] = "not a list" with pytest.raises(TypeError): sa = Service.from_dict(service_dict) service_dict["consumerParameters"] = ["not a dict"] with pytest.raises(TypeError): sa = Service.from_dict(service_dict) service_dict["consumerParameters"] = True with pytest.raises(TypeError): sa = Service.from_dict(service_dict) @pytest.mark.unit def test_additional_information(): """Tests a complex structure of additional information key.""" ddo_dict = get_sample_ddo() service_dict = ddo_dict["services"][0] service_dict["additionalInformation"] = { "message": "Sample DDO", "some_list": ["a", "b", "c"], "nested_dict": {"some_key": "value"}, } sa = Service.from_dict(service_dict) assert sa.additional_information == { "message": "Sample DDO", "some_list": ["a", "b", "c"], "nested_dict": {"some_key": "value"}, } assert sa.additional_information["message"] == "Sample DDO" assert sa.additional_information["some_list"][0] == "a" assert sa.additional_information["some_list"][1] == "b" assert sa.additional_information["some_list"][2] == "c" assert sa.additional_information["nested_dict"]["some_key"] == "value" assert sa.as_dictionary() == { "id": "1", "type": "access", "serviceEndpoint": "http://172.15.0.4:8030", "datatokenAddress": "0x123", "files": "0x0000", "timeout": 0, "name": "Download service", "description": "Download service", "additionalInformation": { "message": "Sample DDO", "some_list": ["a", "b", "c"], "nested_dict": {"some_key": "value"}, }, } @pytest.mark.unit def test_trusted_algo_functions(publisher_ocean): algorithm_ddo = get_sample_algorithm_ddo(filename="ddo_algorithm2.json") algorithm_ddo.did = "did:op:123" algorithm_ddo_v2 = get_sample_algorithm_ddo(filename="ddo_algorithm2.json") algorithm_ddo_v2.did = "did:op:1234" algorithm_ddo_v3 = get_sample_algorithm_ddo(filename="ddo_algorithm2.json") algorithm_ddo_v3.did = "did:op:3333" ddo_dict = get_sample_ddo_with_compute_service() service_dict = ddo_dict["services"][1] compute_service = Service.from_dict(service_dict) assert compute_service.type == "compute" # remove an existing algorithm to publisher_trusted_algorithms list new_publisher_trusted_algorithms = ( compute_service.remove_publisher_trusted_algorithm(algorithm_ddo.did) ) assert ( new_publisher_trusted_algorithms is not None ), "Remove process of a trusted algorithm failed." assert len(new_publisher_trusted_algorithms) == 1 # remove a trusted algorithm that does not belong to publisher_trusted_algorithms list new_publisher_trusted_algorithms = ( compute_service.remove_publisher_trusted_algorithm(algorithm_ddo_v3.did) ) assert len(new_publisher_trusted_algorithms) == 1 @pytest.mark.unit def test_utilitary_functions_for_trusted_algorithm_publishers(publisher_ocean): """Tests adding/removing trusted algorithms in the DDO metadata.""" ddo = DDO.from_dict(get_sample_ddo_with_compute_service()) compute_service = ddo.services[1] assert compute_service.type == "compute" web3 = publisher_ocean.config_dict["web3_instance"] addr1 = web3.eth.account.create().address compute_service.compute_values["publisherTrustedAlgorithmPublishers"] = [addr1] addr2 = web3.eth.account.create().address # add a new trusted algorithm to the publisher_trusted_algorithms list new_publisher_trusted_algo_publishers = ( compute_service.add_publisher_trusted_algorithm_publisher(addr2) ) assert ( new_publisher_trusted_algo_publishers is not None ), "Added a new trusted algorithm failed. The list is empty." assert len(new_publisher_trusted_algo_publishers) == 2 # add an existing algorithm to publisher_trusted_algorithms list new_publisher_trusted_algo_publishers = ( compute_service.add_publisher_trusted_algorithm_publisher(addr2.upper()) ) assert len(new_publisher_trusted_algo_publishers) == 2 # remove an existing algorithm to publisher_trusted_algorithms list new_publisher_trusted_algo_publishers = ( compute_service.remove_publisher_trusted_algorithm_publisher(addr2.upper()) ) assert len(new_publisher_trusted_algo_publishers) == 1 addr3 = web3.eth.account.create().address # remove a trusted algorithm that does not belong to publisher_trusted_algorithms list new_publisher_trusted_algo_publishers = ( compute_service.remove_publisher_trusted_algorithm_publisher(addr3) ) assert len(new_publisher_trusted_algo_publishers) == 1 @pytest.mark.unit def test_inexistent_removals(): ddo_dict = get_sample_ddo_with_compute_service() del ddo_dict["services"][1]["compute"]["publisherTrustedAlgorithms"] ddo = DDO.from_dict(ddo_dict) compute_service = ddo.services[1] with pytest.raises( Exception, match="Algorithm notadid is not in trusted algorithms" ): compute_service.remove_publisher_trusted_algorithm("notadid") ddo_dict = get_sample_ddo_with_compute_service() del ddo_dict["services"][1]["compute"]["publisherTrustedAlgorithmPublishers"] ddo = DDO.from_dict(ddo_dict) compute_service = ddo.services[1] with pytest.raises( Exception, match="Publisher notadid is not in trusted algorithm publishers" ): compute_service.remove_publisher_trusted_algorithm_publisher("notadid") @pytest.mark.unit def test_utilitary_functions_for_trusted_algorithms(publisher_ocean): """Tests adding/removing trusted algorithms in the DDO metadata.""" algorithm_ddo = get_sample_algorithm_ddo(filename="ddo_algorithm2.json") algorithm_ddo.did = "did:op:123" algorithm_ddo_v2 = get_sample_algorithm_ddo(filename="ddo_algorithm2.json") algorithm_ddo_v2.did = "did:op:1234" algorithm_ddo_v3 = get_sample_algorithm_ddo(filename="ddo_algorithm2.json") algorithm_ddo_v3.did = "did:op:3333" ddo = DDO.from_dict( get_sample_ddo_with_compute_service("ddo_v4_with_compute_service2.json") ) with patch("ocean_lib.assets.ddo.FileInfoProvider.fileinfo") as mock: the_response = Mock(spec=Response) the_response.json.return_value = [ { "checksum": "5ce12db0cc7f13f963b1af3b5df7cab4fd3ffae16c8af7e6e416570d197dcc61" } ] mock.return_value = the_response publisher_trusted_algorithms = [algorithm_ddo.generate_trusted_algorithms()] assert len(publisher_trusted_algorithms) == 1 compute_service = ddo.services[1] assert compute_service.type == "compute" assert ( compute_service.compute_values["publisherTrustedAlgorithms"] == publisher_trusted_algorithms ) with patch("ocean_lib.assets.ddo.FileInfoProvider.fileinfo") as mock: the_response = Mock(spec=Response) the_response.json.return_value = [ { "checksum": "5ce12db0cc7f13f963b1af3b5df7cab4fd3ffae16c8af7e6e416570d197dcc61" } ] mock.return_value = the_response new_publisher_trusted_algorithms = ( compute_service.add_publisher_trusted_algorithm( algorithm_ddo_v2, ) ) assert ( new_publisher_trusted_algorithms is not None ), "Added a new trusted algorithm failed. The list is empty." assert len(new_publisher_trusted_algorithms) == 2 with patch("ocean_lib.assets.ddo.FileInfoProvider.fileinfo") as mock: the_response = Mock(spec=Response) the_response.json.return_value = [ { "checksum": "5ce12db0cc7f13f963b1af3b5df7cab4fd3ffae16c8af7e6e416570d197dcc61" } ] mock.return_value = the_response new_publisher_trusted_algorithms = ( compute_service.add_publisher_trusted_algorithm(algorithm_ddo) ) assert new_publisher_trusted_algorithms is not None for trusted_algorithm in publisher_trusted_algorithms: assert ( trusted_algorithm["did"] == algorithm_ddo.did ), "Added a different algorithm besides the existing ones." assert len(new_publisher_trusted_algorithms) == 2 @pytest.mark.unit def test_add_trusted_algorithm_no_compute_service(publisher_ocean): """Tests if the DDO has or not a compute service.""" algorithm_ddo = get_sample_algorithm_ddo("ddo_algorithm2.json") algorithm_ddo.did = "did:op:0x666" ddo = DDO.from_dict(get_sample_ddo()) access_service = ddo.services[0] assert access_service.type == "access" with pytest.raises(AssertionError, match="Service is not compute type"): access_service.add_publisher_trusted_algorithm(algorithm_ddo) ================================================ FILE: ocean_lib/structures/abi_tuples.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """Defines NamedTuples `Stakes`, `OrderData`, `Operations`""" from enum import Enum from typing import NamedTuple class OperationType(Enum): SwapExactIn = 0 SwapExactOut = 1 FixedRate = 2 Dispenser = 3 Operations = NamedTuple( "Operations", [ ("exchange_id", bytes), ("source", str), ("operation", OperationType), ("token_in", str), ("amounts_in", int), ("token_out", str), ("amounts_out", int), ("max_price", int), ("swap_market_fee", int), ("market_fee_address", int), ], ) Stakes = NamedTuple( "Stakes", [ ("pool_address", str), ("token_amount_in", int), ("min_pool_amount_out", int), ], ) OrderData = NamedTuple( "OrderData", [ ("token_address", str), ("consumer", str), ("service_index", int), ("provider_fees", tuple), ("consume_fees", tuple), ], ) ReuseOrderData = NamedTuple( "ReuseOrderData", [ ("token_address", str), ("order_tx_id", str), ("provider_fees", tuple), ], ) MetadataProof = NamedTuple( "MetadataProof", [("validator_address", str), ("v", int), ("r", bytes), ("s", bytes)], ) ================================================ FILE: ocean_lib/structures/algorithm_metadata.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json from typing import Any, Dict from enforce_typing import enforce_types from ocean_lib.services.consumer_parameters import ConsumerParameters class AlgorithmMetadata: @enforce_types def __init__(self, metadata_dict: Dict[str, Any]) -> None: """Initialises AlgorithmMetadata object.""" self.url = metadata_dict.get("url", "") self.rawcode = metadata_dict.get("rawcode", "") self.language = metadata_dict.get("language", "") self.format = metadata_dict.get("format", "") self.version = metadata_dict.get("version", "") container = metadata_dict.get("container", dict()) self.container_entry_point = container.get("entrypoint", "") self.container_image = container.get("image", "") self.container_tag = container.get("tag", "") self.container_checksum = container.get("checksum", "") consumer_parameters = metadata_dict.get("consumerParameters", []) try: self.consumer_parameters = [ ConsumerParameters.from_dict(cp_dict) for cp_dict in consumer_parameters ] except AttributeError: raise TypeError("ConsumerParameters should be a list of dictionaries.") @enforce_types def is_valid(self) -> bool: return bool( self.container_image and self.container_tag and self.container_entry_point ) @enforce_types def as_json_str(self) -> str: return json.dumps(self.as_dictionary()) @enforce_types def as_dictionary(self) -> Dict[str, Any]: result = { "meta": { "url": self.url, "rawcode": self.rawcode, "language": self.language, "version": self.version, "container": { "entrypoint": self.container_entry_point, "image": self.container_image, "tag": self.container_tag, "checksum": self.container_checksum, }, } } if self.consumer_parameters: consumer_parameters = [x.as_dictionary() for x in self.consumer_parameters] result["meta"]["consumerParameters"] = consumer_parameters return result ================================================ FILE: ocean_lib/structures/file_objects.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from abc import abstractmethod from typing import Optional, Protocol from enforce_typing import enforce_types class FilesType(Protocol): @enforce_types @abstractmethod def to_dict(self) -> dict: raise NotImplementedError class UrlFile(FilesType): @enforce_types def __init__( self, url: str, method: Optional[str] = None, headers: Optional[dict] = None ) -> None: self.url = url self.method = method self.headers = headers self.type = "url" @enforce_types def to_dict(self) -> dict: result = {"type": self.type, "url": self.url} if self.method: result["method"] = self.method if self.headers: result["headers"] = self.headers return result class IpfsFile(FilesType): @enforce_types def __init__(self, hash: str) -> None: self.hash = hash self.type = "ipfs" @enforce_types def to_dict(self) -> dict: return {"type": self.type, "hash": self.hash} class GraphqlQuery(FilesType): @enforce_types def __init__(self, url: str, query: str, headers: Optional[dict] = None) -> None: self.url = url self.query = query self.headers = headers self.type = "graphql" @enforce_types def to_dict(self) -> dict: result = {"type": self.type, "url": self.url, "query": self.query} if self.headers: result["headers"] = self.headers return result class SmartContractCall(FilesType): @enforce_types def __init__(self, address: str, chainId: int, abi: dict) -> None: self.address = address self.abi = abi self.chainId = chainId self.type = "smartcontract" @enforce_types def to_dict(self) -> dict: return { "type": self.type, "address": self.address, "abi": self.abi, "chainId": self.chainId, } class ArweaveFile(FilesType): @enforce_types def __init__(self, transaction_id: str) -> None: self.transaction_id = transaction_id self.type = "arweave" @enforce_types def to_dict(self) -> dict: return {"type": self.type, "transactionId": self.transaction_id} @enforce_types def FilesTypeFactory(file_obj: dict) -> FilesType: """Factory Method""" if file_obj["type"] == "url": return UrlFile( file_obj["url"], method=file_obj.get("method", "GET"), headers=file_obj.get("headers"), ) elif file_obj["type"] == "arweave": return ArweaveFile(file_obj["transactionId"]) elif file_obj["type"] == "ipfs": return IpfsFile(file_obj["hash"]) elif file_obj["type"] == "graphql": return GraphqlQuery(file_obj["url"], query=file_obj.get("query")) elif file_obj["type"] == "smartcontract": return SmartContractCall( address=file_obj.get("address"), chainId=file_obj.get("chainId"), abi=file_obj.get("abi"), ) else: raise Exception("Unrecognized file type") ================================================ FILE: ocean_lib/structures/test/test_algorithm_metadata.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json import pytest from ocean_lib.structures.algorithm_metadata import AlgorithmMetadata @pytest.mark.unit def test_algorithm_metadata(): algo_metadata = AlgorithmMetadata( { "rawcode": "", "language": "Node.js", "format": "docker-image", "version": "0.1", "container": { "entrypoint": "node $ALGO", "image": "ubuntu", "tag": "latest", "checksum": "44e10daa6637893f4276bb8d7301eb35306ece50f61ca34dcab550", }, "consumerParameters": [ { "name": "some_key", "type": "string", "label": "test_key_label", "required": True, "default": "value", "description": "this is a test key", } ], } ) assert algo_metadata.is_valid() assert "rawcode" in json.loads(algo_metadata.as_json_str())["meta"] assert ( algo_metadata.as_dictionary()["meta"]["consumerParameters"][0]["name"] == "some_key" ) ================================================ FILE: ocean_lib/structures/test/test_file_objects.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.structures.file_objects import FilesTypeFactory, IpfsFile, UrlFile @pytest.mark.unit def test_url_file(): url_file = UrlFile(url="https://url.com") assert url_file.to_dict() == {"type": "url", "url": "https://url.com"} url_file = UrlFile(url="https://url.com", method="POST") assert url_file.to_dict() == { "type": "url", "url": "https://url.com", "method": "POST", } url_file = UrlFile(url="https://url.com", headers={"test": "test_header"}) assert url_file.to_dict() == { "type": "url", "url": "https://url.com", "headers": {"test": "test_header"}, } @pytest.mark.unit def test_ipfs_file(): ipfs_file = IpfsFile(hash="abc") assert ipfs_file.to_dict() == {"type": "ipfs", "hash": "abc"} @pytest.mark.unit def test_filetype_factory(): factory_file = FilesTypeFactory( { "type": "url", "url": "https://url.com", "method": "GET", } ) assert factory_file.url == "https://url.com" factory_file = FilesTypeFactory( { "type": "ipfs", "hash": "abc", } ) assert factory_file.hash == "abc" with pytest.raises(Exception): factory_file = FilesTypeFactory({"type": "somethingelse"}) ================================================ FILE: ocean_lib/test/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/test/test_config.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json import pytest from web3 import Web3 from ocean_lib.ocean.ocean import Ocean @pytest.mark.unit def test_metadataCacheUri_config_key(): """Tests that the metadata_cache_uri config property can be set using the `metadataCacheUri` config dict key when created via the Ocean __init__""" config_dict = { "NETWORK_NAME": "development", "METADATA_CACHE_URI": "http://v4.aquarius.oceanprotocol.com", "PROVIDER_URL": "http://172.15.0.4:8030", "DOWNLOADS_PATH": "consume-downloads", "ADDRESS_FILE": "~/.ocean/ocean-contracts/artifacts/address.json", "CHAIN_ID": 8996, "web3_instance": Web3(), } ocean_instance = Ocean(config_dict=config_dict) assert ( "http://v4.aquarius.oceanprotocol.com" == ocean_instance.config_dict["METADATA_CACHE_URI"] ) @pytest.mark.unit def test_incomplete(): """Tests that the metadata_cache_uri config property can be set using the `metadataCacheUri` config dict key when created via the Ocean __init__""" config_dict = { "METADATA_CACHE_URI": "http://ItWorked.com", } with pytest.raises(Exception) as exception_info: Ocean(config_dict=config_dict) exception_response = json.loads(exception_info.value.args[0]) assert exception_response["NETWORK_NAME"] == "required" ================================================ FILE: ocean_lib/test/test_example_config.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.example_config import ( DEFAULT_METADATA_CACHE_URI, DEFAULT_PROVIDER_URL, METADATA_CACHE_URI, get_config_dict, ) from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.ocean.util import get_address_of_type from ocean_lib.web3_internal.contract_utils import get_contracts_addresses @pytest.mark.unit def test_ganache_example_config(): """Tests the config structure of ganache network.""" config = get_config_dict() assert config["METADATA_CACHE_URI"] == DEFAULT_METADATA_CACHE_URI assert config["PROVIDER_URL"] == DEFAULT_PROVIDER_URL @pytest.mark.unit def test_polygon_example_config(): """Tests the config structure of Polygon network.""" config = get_config_dict("https://polygon-rpc.com") assert config["METADATA_CACHE_URI"] == METADATA_CACHE_URI assert config["PROVIDER_URL"] == "https://v4.provider.polygon.oceanprotocol.com" @pytest.mark.unit def test_get_address_of_type(): config = get_config_dict("https://polygon-rpc.com") data_nft_factory = get_address_of_type(config, DataNFTFactoryContract.CONTRACT_NAME) addresses = get_contracts_addresses(config) assert addresses[DataNFTFactoryContract.CONTRACT_NAME] == data_nft_factory ================================================ FILE: ocean_lib/web3_internal/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/web3_internal/clef.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import sys from pathlib import Path from web3 import HTTPProvider, IPCProvider from web3.main import Web3 def get_clef_accounts(uri: str = None, timeout: int = 120) -> None: provider = None if uri is None: if sys.platform == "win32": uri = "http://localhost:8550/" else: uri = Path.home().joinpath(".clef/clef.ipc").as_posix() try: if Path(uri).exists(): provider = IPCProvider(uri, timeout=timeout) except OSError: if uri is not None and uri.startswith("http"): provider = HTTPProvider(uri, {"timeout": timeout}) if provider is None: raise ValueError( "Unknown URI, must be IPC socket path or URL starting with 'http'" ) response = provider.make_request("account_list", []) if "error" in response: raise ValueError(response["error"]["message"]) clef_accounts = [ClefAccount(address, provider) for address in response["result"]] return clef_accounts class ClefAccount: def __init__(self, address: str, provider: [HTTPProvider, IPCProvider]) -> None: self.address = Web3.to_checksum_address(address) self.provider = provider ================================================ FILE: ocean_lib/web3_internal/constants.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """ This module holds following default values and constants. """ ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" MAX_UINT256 = 2**256 - 1 MAX_INT256 = 2**255 - 1 MIN_INT256 = 2**255 * -1 ================================================ FILE: ocean_lib/web3_internal/contract_base.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """All contracts inherit from `ContractBase` class.""" import logging from typing import List, Optional from enforce_typing import enforce_types from eth_typing import ChecksumAddress from web3._utils.abi import abi_to_signature from web3.exceptions import MismatchedABI from web3.logs import DISCARD from web3.main import Web3 from ocean_lib.web3_internal.clef import ClefAccount from ocean_lib.web3_internal.contract_utils import load_contract logger = logging.getLogger(__name__) def function_wrapper(contract, web3, contract_functions, func_name): # direct function calls if hasattr(contract, func_name): return getattr(contract, func_name) # contract functions def wrap(*args, **kwargs): args2 = list(args) tx_dict = None # retrieve tx dict from either args or kwargs if args and isinstance(args[-1], dict): tx_dict = args[-1] if args[-1].get("from") else None args2 = list(args[:-1]) if "tx_dict" in kwargs: tx_dict = kwargs["tx_dict"] if kwargs["tx_dict"].get("from") else None del kwargs["tx_dict"] # use addresses instead of wallets when doing the call for arg in args2: if hasattr(arg, "address"): args2 = list(args2) args2[args2.index(arg)] = arg.address func = getattr(contract_functions, func_name) result = func(*args2, **kwargs) func_signature = abi_to_signature(result.abi) # view/pure functions don't need "from" key in tx_dict if not tx_dict and result.abi["stateMutability"] not in ["view", "pure"]: raise Exception("Needs tx_dict with 'from' key.") # if it's a view/pure function, just call it if result.abi["stateMutability"] in ["view", "pure"]: return result.call() else: # if it's a transaction, build and send it wallet = tx_dict["from"] tx_dict2 = tx_dict.copy() tx_dict2["nonce"] = web3.eth.get_transaction_count(wallet.address) tx_dict2["from"] = tx_dict["from"].address result = result.build_transaction(tx_dict2) # sign with wallet private key and send transaction if isinstance(wallet, ClefAccount): for k, v in result.items(): result[k] = Web3.to_hex(v) if not isinstance(v, str) else v raw_signed_tx = wallet.provider.make_request( "account_signTransaction", [result, func_signature] ) raw_signed_tx = raw_signed_tx["result"]["raw"] else: signed_tx = web3.eth.account.sign_transaction( result, wallet._private_key ) raw_signed_tx = signed_tx.rawTransaction receipt = web3.eth.send_raw_transaction(raw_signed_tx) return web3.eth.wait_for_transaction_receipt(receipt) return wrap class ContractBase(object): """Base class for all contract objects.""" CONTRACT_NAME = None @enforce_types def __init__(self, config_dict: dict, address: Optional[str]) -> None: """Initialises Contract Base object.""" assert ( self.contract_name ), "contract_name property needs to be implemented in subclasses." self.config_dict = config_dict self.contract = load_contract( config_dict["web3_instance"], self.contract_name, address ) assert not address or (self.contract.address.lower() == address.lower()) transferable = [ x for x in dir(self.contract.functions) if not x.startswith("_") ] # transfer contract functions to ContractBase object for function in transferable: setattr( self, function, function_wrapper( self.contract, config_dict["web3_instance"], self.contract.functions, function, ), ) @enforce_types def __str__(self) -> str: """Returns contract `name @ address.`""" return f"{self.contract_name} @ {self.address}" @property @enforce_types def contract_name(self) -> str: """Returns the contract name""" return self.CONTRACT_NAME @staticmethod @enforce_types def to_checksum_address(address: str) -> ChecksumAddress: """ Validate the address provided. :param address: Address, hex str :return: address, hex str """ return Web3.to_checksum_address(address.lower()) @enforce_types def get_event_signature(self, event_name: str) -> str: try: e = getattr(self.contract.events, event_name) except MismatchedABI: raise ValueError( f"Event {event_name} not found in {self.CONTRACT_NAME} contract." ) abi = e().abi types = [param["type"] for param in abi["inputs"]] sig_str = f'{event_name}({",".join(types)})' return Web3.keccak(text=sig_str).hex() @enforce_types def get_logs( self, event_name: str, from_block: Optional[int] = 0, to_block: Optional[int] = "latest", ) -> List: topic = self.get_event_signature(event_name) web3 = self.config_dict["web3_instance"] event_filter = web3.eth.filter( { "topics": [topic], "toBlock": to_block, "fromBlock": from_block, } ) events = [] for log in event_filter.get_all_entries(): receipt = web3.eth.wait_for_transaction_receipt(log.transactionHash) fn = getattr(self.contract.events, event_name) processed_events = fn().process_receipt(receipt, errors=DISCARD) for processed_event in processed_events: events.append(processed_event) return events ================================================ FILE: ocean_lib/web3_internal/contract_utils.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json import logging import os from pathlib import Path from typing import Any, Dict, Optional from enforce_typing import enforce_types from web3.contract import Contract from web3.main import Web3 import artifacts # noqa logger = logging.getLogger(__name__) GANACHE_URL = "http://127.0.0.1:8545" @enforce_types def get_contract_definition(contract_name: str) -> Dict[str, Any]: """Returns the abi JSON for a contract name.""" path = os.path.join(artifacts.__file__, "..", f"{contract_name}.json") path = Path(path).expanduser().resolve() if not path.exists(): raise TypeError("Contract name does not exist in artifacts.") with open(path) as f: return json.load(f) @enforce_types def load_contract(web3: Web3, contract_name: str, address: Optional[str]) -> Contract: """Loads a contract using its name and address.""" contract_definition = get_contract_definition(contract_name) abi = contract_definition["abi"] bytecode = contract_definition["bytecode"] return web3.eth.contract(address=address, abi=abi, bytecode=bytecode) @enforce_types def get_contracts_addresses_all_networks(config: dict): """Get addresses, across *all* networks, from info in ADDRESS_FILE""" address_file = config.get("ADDRESS_FILE") address_file = os.path.expanduser(address_file) if address_file else None if not address_file or not os.path.exists(address_file): raise Exception(f"Could not find address_file={address_file}.") with open(address_file) as f: addresses = json.load(f) return addresses @enforce_types def get_contracts_addresses(config: dict) -> Optional[Dict[str, str]]: """Get addresses for given NETWORK_NAME, from info in ADDRESS_FILE""" network_name = config["NETWORK_NAME"] addresses = get_contracts_addresses_all_networks(config) network_addresses = [val for key, val in addresses.items() if key == network_name] if not network_addresses: address_file = config.get("ADDRESS_FILE") raise Exception( f"Address not found for network_name={network_name}." f" Please check your address_file={address_file}." ) return _checksum_contract_addresses(network_addresses=network_addresses[0]) @enforce_types # Check singnet/snet-cli#142 (comment). You need to provide a lowercase address then call web3.to_checksum_address() # for software safety. def _checksum_contract_addresses( network_addresses: Dict[str, Any] ) -> Optional[Dict[str, Any]]: for key, value in network_addresses.items(): if key == "chainId": continue if isinstance(value, int): continue if isinstance(value, dict): for k, v in value.items(): value.update({k: Web3.to_checksum_address(v.lower())}) else: network_addresses.update({key: Web3.to_checksum_address(value.lower())}) return network_addresses ================================================ FILE: ocean_lib/web3_internal/http_provider.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from web3 import HTTPProvider, WebsocketProvider from ocean_lib.web3_internal.request import make_post_request GANACHE_URL = "http://127.0.0.1:8545" POLYGON_URL = "https://rpc.polygon.oceanprotocol.com" SUPPORTED_NETWORK_NAMES = { "rinkeby", "kovan", "ganache", "mainnet", "ropsten", "polygon", } class CustomHTTPProvider(HTTPProvider): """ Override requests to control the connection pool to make it blocking. """ def make_request(self, method, params): self.logger.debug( "Making request HTTP. URI: %s, Method: %s", self.endpoint_uri, method ) request_data = self.encode_rpc_request(method, params) raw_response = make_post_request( self.endpoint_uri, request_data, **self.get_request_kwargs() ) response = self.decode_rpc_response(raw_response) self.logger.debug( "Getting response HTTP. URI: %s, " "Method: %s, Response: %s", self.endpoint_uri, method, response, ) return response def get_web3_connection_provider(network_url): if network_url.startswith("http"): return CustomHTTPProvider(network_url) elif network_url.startswith("ws"): return WebsocketProvider(network_url) raise Exception(f"Unsupported network url: {network_url}") ================================================ FILE: ocean_lib/web3_internal/request.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """ This is copied from Web3 python library to control the `requests` session parameters. """ import lru import requests from requests.adapters import HTTPAdapter from web3._utils.caching import generate_cache_key def _remove_session(key, session): session.close() _session_cache = lru.LRU(8, callback=_remove_session) def _get_session(*args, **kwargs): cache_key = generate_cache_key((args, kwargs)) if cache_key not in _session_cache: # This is the main change from original Web3 `_get_session` session = requests.sessions.Session() session.mount( "http://", HTTPAdapter(pool_connections=25, pool_maxsize=25, pool_block=True), ) session.mount( "https://", HTTPAdapter(pool_connections=25, pool_maxsize=25, pool_block=True), ) _session_cache[cache_key] = session return _session_cache[cache_key] def make_post_request(endpoint_uri, data, *args, **kwargs): kwargs.setdefault("timeout", 10) session = _get_session(endpoint_uri) version = "TODO" # TODO version_header = {"User-Agent": f"OceanAquarius/{version}"} if "headers" in kwargs: kwargs["headers"].update(version_header) else: kwargs["headers"] = version_header response = session.post(endpoint_uri, data=data, *args, **kwargs) response.raise_for_status() return response.content ================================================ FILE: ocean_lib/web3_internal/test/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: ocean_lib/web3_internal/test/conftest.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # # Directory ../ocean_lib/models/test holds test_btoken.py and more. # Those tests grab ../ocean_lib/models/test/conftest.py, which # sets up convenient-to-use wallets/accounts for Alice & Bob, datatokens, more. # *This* directory wants similar items. To avoid code repetition, # here we simply import that conftest's contents. from conftest_ganache import * ================================================ FILE: ocean_lib/web3_internal/test/test_contract_base.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pytest from ocean_lib.ocean.util import get_address_of_type from ocean_lib.web3_internal.constants import ZERO_ADDRESS from ocean_lib.web3_internal.contract_base import ContractBase from tests.resources.helper_functions import get_wallet class MyFactory(ContractBase): CONTRACT_NAME = "ERC721Factory" @pytest.mark.unit def test_name_is_None(config): with pytest.raises(Exception): # self.name will become None, triggering the error ContractBase(config, None) @pytest.mark.unit def test_main(config): alice_wallet = get_wallet(1) # test super-simple functionality of child nft_factory_address = get_address_of_type(config, "ERC721Factory") factory = MyFactory(config, nft_factory_address) factory.deployERC721Contract( "NFT", "NFTS", 1, ZERO_ADDRESS, ZERO_ADDRESS, "http://someurl", True, alice_wallet.address, {"from": alice_wallet}, ) # test attributes assert factory.contract_name == "ERC721Factory" assert factory.contract is not None assert factory.contract.address == nft_factory_address assert ContractBase.to_checksum_address(nft_factory_address) == nft_factory_address # test methods assert factory.contract_name == "ERC721Factory" assert factory.address == nft_factory_address assert str(factory) == f"{factory.contract_name} @ {factory.address}" assert factory.createToken assert factory.getCurrentTokenCount assert factory.getTokenTemplate ================================================ FILE: ocean_lib/web3_internal/test/test_contract_utils.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from web3.main import Web3 from ocean_lib.web3_internal.contract_utils import _checksum_contract_addresses def test_checksum_contract_addresses(monkeypatch): addresses = { "chainId": 7, "test": "0x20802d1a9581b94e51db358c09e0818d6bd071b4", "dict": {"address": "0xe2dd09d719da89e5a3d0f2549c7e24566e947260"}, } assert Web3.is_checksum_address(addresses["test"]) is False assert Web3.is_checksum_address(addresses["dict"]["address"]) is False checksum_addresses = _checksum_contract_addresses(addresses) assert Web3.is_checksum_address(checksum_addresses["test"]) is True assert Web3.is_checksum_address(addresses["dict"]["address"]) is True ================================================ FILE: ocean_lib/web3_internal/test/test_wallet.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os import pytest from ocean_lib.ocean.util import to_wei from tests.resources.helper_functions import generate_wallet @pytest.mark.unit def test_generating_wallets(ocean_token, config): generated_wallet = generate_wallet() assert generated_wallet.address, "Wallet has not an address." assert config["web3_instance"].eth.get_balance(generated_wallet.address) == to_wei( 3 ) assert ocean_token.balanceOf(generated_wallet.address) == to_wei(50) env_key_labels = [ "TEST_PRIVATE_KEY1", "TEST_PRIVATE_KEY2", "TEST_PRIVATE_KEY3", "TEST_PRIVATE_KEY4", "TEST_PRIVATE_KEY5", "TEST_PRIVATE_KEY6", "TEST_PRIVATE_KEY7", "TEST_PRIVATE_KEY8", "FACTORY_DEPLOYER_PRIVATE_KEY", "PROVIDER_PRIVATE_KEY", ] env_private_keys = [] for key_label in env_key_labels: key = os.environ.get(key_label) env_private_keys.append(key) assert generated_wallet._private_key.hex() not in env_private_keys ================================================ FILE: ocean_lib/web3_internal/utils.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import logging from collections import namedtuple from typing import Any, Union import requests from enforce_typing import enforce_types from eth_keys import KeyAPI from eth_keys.backends import NativeECCBackend from hexbytes.main import HexBytes from web3.main import Web3 from ocean_lib.web3_internal.clef import ClefAccount Signature = namedtuple("Signature", ("v", "r", "s")) logger = logging.getLogger(__name__) keys = KeyAPI(NativeECCBackend) @enforce_types def to_32byte_hex(val: int) -> str: """ :param val: :return: """ return Web3.to_hex(Web3.to_bytes(val).rjust(32, b"\0")) @enforce_types def sign_with_clef(message_hash: str, wallet: ClefAccount) -> str: message_hash = Web3.solidity_keccak( ["bytes"], [Web3.to_bytes(text=message_hash)], ) orig_sig = wallet.provider.make_request( "account_signData", ["data/plain", wallet.address, message_hash.hex()] )["result"] return orig_sig @enforce_types def sign_with_key(message_hash: Union[HexBytes, str], key: str) -> str: if isinstance(message_hash, str): message_hash = Web3.solidity_keccak( ["bytes"], [Web3.to_bytes(text=message_hash)], ) pk = keys.PrivateKey(Web3.to_bytes(hexstr=key)) prefix = "\x19Ethereum Signed Message:\n32" signable_hash = Web3.solidity_keccak( ["bytes", "bytes"], [Web3.to_bytes(text=prefix), Web3.to_bytes(message_hash)] ) return keys.ecdsa_sign(message_hash=signable_hash, private_key=pk) @enforce_types def split_signature(signature: Any) -> Signature: """ :param web3: :param signature: signed message hash, hex str :return: """ assert len(signature) == 65, ( f"invalid signature, " f"expecting bytes of length 65, got {len(signature)}" ) v = Web3.to_int(signature[-1]) r = to_32byte_hex(int.from_bytes(signature[:32], "big")) s = to_32byte_hex(int.from_bytes(signature[32:64], "big")) if v != 27 and v != 28: v = 27 + v % 2 return Signature(v, r, s) @enforce_types def get_gas_fees() -> tuple: # Polygon & Mumbai uses EIP-1559. So, dynamically determine priority fee gas_resp = requests.get("https://gasstation.polygon.technology/v2") return ( Web3.to_wei(gas_resp.json()["fast"]["maxPriorityFee"], "gwei"), Web3.to_wei(gas_resp.json()["fast"]["maxFee"], "gwei"), ) ================================================ FILE: pyproject.toml ================================================ [tool.black] exclude = ''' ( setup.py ) ''' ================================================ FILE: pytest.ini ================================================ # pytest.ini [pytest] filterwarnings = ignore::DeprecationWarning ignore:.*The event signature did not match the provided ABI*:UserWarning ignore:.*Event log does not contain enough topics for the given ABI.*:UserWarning markers = nosetup_all: do not call setup_all unit integration # generated readmes run through test_readmes.py, separately and programatically addopts = --ignore=tests/generated-readmes env = D:TEST_PRIVATE_KEY1=0x8467415bb2ba7c91084d932276214b11a3dd9bdb2930fefa194b666dd8020b99 D:TEST_PRIVATE_KEY2=0x1d751ded5a32226054cd2e71261039b65afb9ee1c746d055dd699b1150a5befc D:TEST_PRIVATE_KEY3=0x732fbb7c355aa8898f4cff92fa7a6a947339eaf026a08a51f171199e35a18ae0 D:TEST_PRIVATE_KEY4=0xef4b441145c1d0f3b4bc6d61d29f5c6e502359481152f869247c7a4244d45209 D:TEST_PRIVATE_KEY5=0x5d75837394b078ce97bc289fa8d75e21000573520bfa7784a9d28ccaae602bf8 D:TEST_PRIVATE_KEY6=0x1f990f8b013fc5c7955e0f8746f11ded231721b9cf3f99ff06cdc03492b28090 D:TEST_PRIVATE_KEY7=0x8683d6511213ac949e093ca8e9179514d4c56ce5ea9b83068f723593f913b1ab D:TEST_PRIVATE_KEY8=0x1263dc73bef43a9da06149c7e598f52025bf4027f1d6c13896b71e81bb9233fb D:FACTORY_DEPLOYER_PRIVATE_KEY=0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58 D:PROVIDER_PRIVATE_KEY=0xfd5c1ccea015b6d663618850824154a3b3fb2882c46cefb05b9a93fea8c3d215 D:ADDRESS_FILE=~/.ocean/ocean-contracts/artifacts/address.json ================================================ FILE: requirements_dev.txt ================================================ # pip install ocean-lib # is for end users. That will install the packages # listed in the install_requires list of setup.py. # pip install -r requirements_dev.txt # is for the developers of ocean-lib, so doing that should # install all the Python packages listed # in the install_requires list of setup.py # and also the 'dev' list in the extras_require dict -e .[dev] ================================================ FILE: setup.cfg ================================================ # Here is how we view setup.cfg vs setup.py: https://stackoverflow.com/a/46090408/14214722 [bdist_wheel] universal = 1 [aliases] # Define setup.py command aliases here test = pytest [tool:pytest] collect_ignore = ['setup.py'] [isort] multi_line_output = 3 include_trailing_comma = True force_grid_wrap = 0 use_parentheses = True ensure_newline_before_comments = True line_length = 88 [flake8] max-line-length=8888888888 ignore=E203,W503 ================================================ FILE: setup.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # """The setup script.""" # Copyright 2018 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 from setuptools import find_namespace_packages, setup with open("README.md", encoding="utf8") as readme_file: readme = readme_file.read() # Installed by pip install ocean-lib # or pip install -e . install_requirements = [ "ocean-contracts==2.0.3", "coloredlogs==15.0.1", "requests>=2.21.0", "pytz", # used minimally and unlikely to change, common dependency "enforce-typing==1.0.0.post1", "eciespy==0.4.1", "cryptography==41.0.7", "web3==6.14.0", # web3.py requires eth-abi, requests, and more, # so those will be installed too. # See https://github.com/ethereum/web3.py/blob/master/setup.py ] # Required to run setup.py: setup_requirements = ["pytest-runner"] test_requirements = [ "codacy-coverage==1.3.11", "coverage==7.4.1", "mccabe==0.7.0", "pytest==8.0.0", "pytest-watch==4.2.0", "pytest-env", # common dependency "matplotlib", # just used in a readme test and unlikely to change, common dependency "mkcodes==0.1.1", "pytest-sugar==0.9.7", ] # Possibly required by developers of ocean-lib: dev_requirements = [ "bumpversion==0.6.0", "pkginfo==1.9.6", "twine==4.0.2", "watchdog==3.0.0", "isort", "flake8", "black", "pre-commit", "licenseheaders", ] packages = find_namespace_packages(include=["ocean_lib*"], exclude=["*test*"]) setup( author="ocean-core-team", author_email="devops@oceanprotocol.com", classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Natural Language :: English", "Programming Language :: Python :: 3.8", ], description="🐳 Ocean protocol library.", extras_require={ "test": test_requirements, "dev": dev_requirements + test_requirements, }, install_requires=install_requirements, license="Apache Software License 2.0", long_description=readme, long_description_content_type="text/markdown", include_package_data=True, keywords="ocean-lib", name="ocean-lib", packages=packages, setup_requires=setup_requirements, test_suite="tests", tests_require=test_requirements, url="https://github.com/oceanprotocol/ocean.py", # fmt: off # bumpversion.sh needs single-quotes version='3.1.2', # fmt: on zip_safe=False, ) ================================================ FILE: tests/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: tests/flows/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: tests/flows/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: tests/flows/test_start_reuse_order_fees.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # from datetime import datetime, timedelta, timezone from time import sleep from typing import Tuple import pytest from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.example_config import get_config_dict from ocean_lib.models.data_nft import DataNFT from ocean_lib.models.datatoken_base import DatatokenBase, TokenFeeInfo from ocean_lib.models.factory_router import FactoryRouter from ocean_lib.ocean.ocean_assets import OceanAssets from ocean_lib.ocean.util import get_address_of_type, to_wei from ocean_lib.services.service import Service from ocean_lib.structures.file_objects import FilesType from ocean_lib.web3_internal.constants import MAX_UINT256 from tests.resources.ddo_helpers import get_default_metadata, get_first_service_by_type from tests.resources.helper_functions import ( deploy_erc721_erc20, get_file1, get_provider_fees, get_publisher_wallet, get_wallet, int_units, transfer_bt_if_balance_lte, ) fee_parametrisation = [ # Small fees (0, "Ocean", "5", "6", "7"), (1, "MockDAI", "5", "6", "7"), (2, "MockUSDC", "5", "6", "7"), # Zero fees (3, "Ocean", "0", "0", "0"), (4, "MockUSDC", "0", "0", "0"), # Min fees ( 5, "Ocean", "0.000000000000000001", # 1 wei "0.000000000000000001", # 1 wei "0.000000000000000001", # 1 wei ), (6, "MockUSDC", "0.000001", "0.000001", "0.000001"), # Smallest USDC amounts # Large fees (7, "Ocean", "500", "600", "700"), (8, "MockUSDC", "500", "600", "700"), ] class TestStartReuseOrderFees(object): @classmethod def setup_class(self): self.ddos = [] self.dts = [] self.services = [] self.start_order_receipts = [] config = get_config_dict() publisher_wallet = get_publisher_wallet() file1 = get_file1() for index in range(9): data_nft = deploy_erc721_erc20(config, publisher_wallet) publish_market_wallet = get_wallet(4) bt = DatatokenBase.get_typed( config, get_address_of_type(config, fee_parametrisation[index][1]) ) publish_market_order_fee = int_units( fee_parametrisation[index][2], bt.decimals() ) ddo, service, dt = create_asset_with_order_fee_and_timeout( config=config, file=file1, data_nft=data_nft, publisher_wallet=publisher_wallet, publish_market_order_fees=TokenFeeInfo( address=publish_market_wallet.address, token=bt.address, amount=publish_market_order_fee, ), timeout=3600, wait_for_aqua=False, ) self.ddos.append(ddo) self.dts.append(dt) self.services.append(service) @pytest.mark.unit @pytest.mark.parametrize( "param_index, base_token_name, publish_market_order_fee_in_unit, consume_market_order_fee_in_unit, provider_fee_in_unit", fee_parametrisation, ) def test_start_order_fees( self, config: dict, publisher_wallet, consumer_wallet, provider_wallet, factory_deployer_wallet, factory_router: FactoryRouter, param_index: int, base_token_name: str, publish_market_order_fee_in_unit: str, consume_market_order_fee_in_unit: str, provider_fee_in_unit: str, ): bt = DatatokenBase.get_typed( config, get_address_of_type(config, base_token_name) ) dt = self.dts[param_index] publish_market_wallet = get_wallet(4) consume_market_wallet = get_wallet(5) data_provider = DataServiceProvider ocean_assets = OceanAssets(config, data_provider) ddo = ocean_assets._aquarius.wait_for_ddo(self.ddos[param_index].did) publish_market_order_fee = int_units( publish_market_order_fee_in_unit, bt.decimals() ) # Send base tokens to the consumer so they can pay for fees transfer_bt_if_balance_lte( config=config, bt_address=bt.address, from_wallet=factory_deployer_wallet, recipient=consumer_wallet.address, min_balance=int_units("2000", bt.decimals()), amount_to_transfer=int_units("2000", bt.decimals()), ) # Mint 50 datatokens in consumer wallet from publisher. dt.mint( consumer_wallet.address, to_wei(50), {"from": publisher_wallet}, ) opc_collector_address = factory_router.getOPCCollector() if base_token_name == "Ocean" and publish_market_order_fee_in_unit == "500": bt.mint( consumer_wallet.address, int_units("2000", bt.decimals()), {"from": factory_deployer_wallet}, ) # Get balances publisher_bt_balance_before = bt.balanceOf(publisher_wallet.address) publisher_dt_balance_before = dt.balanceOf(publisher_wallet.address) publish_market_bt_balance_before = bt.balanceOf(publish_market_wallet.address) publish_market_dt_balance_before = dt.balanceOf(publish_market_wallet.address) consume_market_bt_balance_before = bt.balanceOf(consume_market_wallet.address) consume_market_dt_balance_before = dt.balanceOf(consume_market_wallet.address) consumer_bt_balance_before = bt.balanceOf(consumer_wallet.address) consumer_dt_balance_before = dt.balanceOf(consumer_wallet.address) provider_bt_balance_before = bt.balanceOf(provider_wallet.address) provider_dt_balance_before = dt.balanceOf(provider_wallet.address) opc_bt_balance_before = bt.balanceOf(opc_collector_address) opc_dt_balance_before = dt.balanceOf(opc_collector_address) # Get provider fees provider_fee = int_units(provider_fee_in_unit, bt.decimals()) valid_for_two_hours = int( (datetime.now(timezone.utc) + timedelta(hours=2)).timestamp() ) provider_fees = get_provider_fees( provider_wallet, bt.address, provider_fee, valid_for_two_hours, ) # Grant datatoken infinite approval to spend consumer's base tokens bt.approve(dt.address, MAX_UINT256, {"from": consumer_wallet}) # Start order for consumer consume_market_order_fee = int_units( consume_market_order_fee_in_unit, bt.decimals() ) start_order_receipt = dt.start_order( consumer=consumer_wallet.address, service_index=ddo.get_index_of_service(self.services[param_index]), provider_fees=provider_fees, consume_market_fees=TokenFeeInfo( address=consume_market_wallet.address, token=bt.address, amount=consume_market_order_fee, ), tx_dict={"from": consumer_wallet}, ) self.start_order_receipts.append(start_order_receipt) # Get balances publisher_bt_balance_after = bt.balanceOf(publisher_wallet.address) publisher_dt_balance_after = dt.balanceOf(publisher_wallet.address) publish_market_bt_balance_after = bt.balanceOf(publish_market_wallet.address) publish_market_dt_balance_after = dt.balanceOf(publish_market_wallet.address) consume_market_bt_balance_after = bt.balanceOf(consume_market_wallet.address) consume_market_dt_balance_after = dt.balanceOf(consume_market_wallet.address) consumer_bt_balance_after = bt.balanceOf(consumer_wallet.address) consumer_dt_balance_after = dt.balanceOf(consumer_wallet.address) provider_bt_balance_after = bt.balanceOf(provider_wallet.address) provider_dt_balance_after = dt.balanceOf(provider_wallet.address) opc_bt_balance_after = bt.balanceOf(opc_collector_address) opc_dt_balance_after = dt.balanceOf(opc_collector_address) # Get order fee amount assert dt.get_publish_market_order_fees().amount == publish_market_order_fee # Get Ocean community fee amount opc_order_fee = factory_router.getOPCConsumeFee() assert opc_order_fee == to_wei(0.03) one_datatoken = to_wei(1) # Check balances assert publisher_bt_balance_before == publisher_bt_balance_after assert ( publisher_dt_balance_before + one_datatoken - opc_order_fee == publisher_dt_balance_after ) assert ( publish_market_bt_balance_before + publish_market_order_fee == publish_market_bt_balance_after ) assert publish_market_dt_balance_before == publish_market_dt_balance_after assert ( consume_market_bt_balance_before + consume_market_order_fee == consume_market_bt_balance_after ) assert consume_market_dt_balance_before == consume_market_dt_balance_after assert ( consumer_bt_balance_before - publish_market_order_fee - consume_market_order_fee - provider_fee == consumer_bt_balance_after ) assert consumer_dt_balance_before - one_datatoken == consumer_dt_balance_after assert provider_bt_balance_before + provider_fee == provider_bt_balance_after assert provider_dt_balance_before == provider_dt_balance_after assert opc_bt_balance_before == opc_bt_balance_after assert opc_dt_balance_before + opc_order_fee == opc_dt_balance_after @pytest.mark.unit @pytest.mark.parametrize( "param_index", range(9), ) def test_reuse_order_fees( self, config: dict, publisher_wallet, consumer_wallet, provider_wallet, param_index, ): publish_market_wallet = get_wallet(4) consume_market_wallet = get_wallet(5) base_token_name = fee_parametrisation[param_index][1] provider_fee_in_unit = fee_parametrisation[param_index][4] bt = DatatokenBase.get_typed( config, get_address_of_type(config, base_token_name) ) dt = self.dts[param_index] start_order_receipt = self.start_order_receipts[param_index] # Reuse order where: # Order: valid # Provider fees: valid # Simulate valid provider fees by setting them to 0 reuse_order_with_mock_provider_fees( provider_fee_in_unit="0", start_order_tx_id=start_order_receipt.transactionHash, bt=bt, dt=dt, publisher_wallet=publisher_wallet, publish_market_wallet=publish_market_wallet, consume_market_wallet=consume_market_wallet, consumer_wallet=consumer_wallet, provider_wallet=provider_wallet, ) # Reuse order where: # Order: valid # Provider fees: expired # Simulate expired provider fees by setting them to non-zero reuse_order_with_mock_provider_fees( provider_fee_in_unit=provider_fee_in_unit, start_order_tx_id=start_order_receipt.transactionHash, bt=bt, dt=dt, publisher_wallet=publisher_wallet, publish_market_wallet=publish_market_wallet, consume_market_wallet=consume_market_wallet, consumer_wallet=consumer_wallet, provider_wallet=provider_wallet, ) # Sleep for 6 seconds, long enough for order to expire sleep(6) # Reuse order where: # Order: expired # Provider fees: valid # Simulate valid provider fees by setting them to 0 reuse_order_with_mock_provider_fees( provider_fee_in_unit="0", start_order_tx_id=start_order_receipt.transactionHash, bt=bt, dt=dt, publisher_wallet=publisher_wallet, publish_market_wallet=publish_market_wallet, consume_market_wallet=consume_market_wallet, consumer_wallet=consumer_wallet, provider_wallet=provider_wallet, ) # Reuse order where: # Order: expired # Provider fees: expired # Simulate expired provider fees by setting them to non-zero reuse_order_with_mock_provider_fees( provider_fee_in_unit=provider_fee_in_unit, start_order_tx_id=start_order_receipt.transactionHash, bt=bt, dt=dt, publisher_wallet=publisher_wallet, publish_market_wallet=publish_market_wallet, consume_market_wallet=consume_market_wallet, consumer_wallet=consumer_wallet, provider_wallet=provider_wallet, ) def create_asset_with_order_fee_and_timeout( config: dict, file: FilesType, data_nft: DataNFT, publisher_wallet, publish_market_order_fees, timeout: int, wait_for_aqua: bool = True, ) -> Tuple[DDO, Service, DatatokenBase]: # Create datatoken with order fee datatoken = data_nft.create_datatoken( {"from": publisher_wallet}, name="Datatoken 1", symbol="DT1", publish_market_order_fees=publish_market_order_fees, ) data_provider = DataServiceProvider ocean_assets = OceanAssets(config, data_provider) metadata = get_default_metadata() files = [file] # Create service with timeout service = Service( service_id="5", service_type=ServiceTypes.ASSET_ACCESS, service_endpoint=data_provider.get_url(config), datatoken=datatoken.address, files=files, timeout=timeout, ) # Publish asset data_nft, datatokens, ddo = ocean_assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, services=[service], data_nft_address=data_nft.address, deployed_datatokens=[datatoken], wait_for_aqua=wait_for_aqua, ) service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS) return ddo, service, datatokens[0] def reuse_order_with_mock_provider_fees( provider_fee_in_unit: str, start_order_tx_id: str, bt: DatatokenBase, dt: DatatokenBase, publisher_wallet, publish_market_wallet, consume_market_wallet, consumer_wallet, provider_wallet, ): """Call reuse_order, and verify the balances/fees are correct""" router = FactoryRouter(bt.config_dict, dt.router()) opc_collector_address = router.getOPCCollector() # Get balances before reuse_order publisher_bt_balance_before = bt.balanceOf(publisher_wallet.address) publisher_dt_balance_before = dt.balanceOf(publisher_wallet.address) publish_market_bt_balance_before = bt.balanceOf(publish_market_wallet.address) publish_market_dt_balance_before = dt.balanceOf(publish_market_wallet.address) consume_market_bt_balance_before = bt.balanceOf(consume_market_wallet.address) consume_market_dt_balance_before = dt.balanceOf(consume_market_wallet.address) consumer_bt_balance_before = bt.balanceOf(consumer_wallet.address) consumer_dt_balance_before = dt.balanceOf(consumer_wallet.address) provider_bt_balance_before = bt.balanceOf(provider_wallet.address) provider_dt_balance_before = dt.balanceOf(provider_wallet.address) opc_bt_balance_before = bt.balanceOf(opc_collector_address) opc_dt_balance_before = dt.balanceOf(opc_collector_address) # Mock provider fees provider_fee = int_units(provider_fee_in_unit, bt.decimals()) valid_until = int((datetime.now(timezone.utc) + timedelta(seconds=10)).timestamp()) provider_fees = get_provider_fees( provider_wallet, bt.address, provider_fee, valid_until, ) # Reuse order dt.reuse_order( order_tx_id=start_order_tx_id, provider_fees=provider_fees, tx_dict={"from": consumer_wallet}, ) # Get balances after reuse_order publisher_bt_balance_after = bt.balanceOf(publisher_wallet.address) publisher_dt_balance_after = dt.balanceOf(publisher_wallet.address) publish_market_bt_balance_after = bt.balanceOf(publish_market_wallet.address) publish_market_dt_balance_after = dt.balanceOf(publish_market_wallet.address) consume_market_bt_balance_after = bt.balanceOf(consume_market_wallet.address) consume_market_dt_balance_after = dt.balanceOf(consume_market_wallet.address) consumer_bt_balance_after = bt.balanceOf(consumer_wallet.address) consumer_dt_balance_after = dt.balanceOf(consumer_wallet.address) provider_bt_balance_after = bt.balanceOf(provider_wallet.address) provider_dt_balance_after = dt.balanceOf(provider_wallet.address) opc_bt_balance_after = bt.balanceOf(opc_collector_address) opc_dt_balance_after = dt.balanceOf(opc_collector_address) # Check balances assert publisher_bt_balance_before == publisher_bt_balance_after assert publisher_dt_balance_before == publisher_dt_balance_after assert publish_market_bt_balance_before == publish_market_bt_balance_after assert publish_market_dt_balance_before == publish_market_dt_balance_after assert consume_market_bt_balance_before == consume_market_bt_balance_after assert consume_market_dt_balance_before == consume_market_dt_balance_after assert consumer_bt_balance_before - provider_fee == consumer_bt_balance_after assert consumer_dt_balance_before == consumer_dt_balance_after assert provider_bt_balance_before + provider_fee == provider_bt_balance_after assert provider_dt_balance_before == provider_dt_balance_after assert opc_bt_balance_before == opc_bt_balance_after assert opc_dt_balance_before == opc_dt_balance_after ================================================ FILE: tests/generated-readmes/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: tests/integration/ganache/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: tests/integration/ganache/test_compute_flow.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import time from datetime import datetime, timedelta, timezone from typing import List, Optional import pytest import requests from attr import dataclass from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.example_config import get_config_dict from ocean_lib.exceptions import DataProviderException from ocean_lib.models.compute_input import ComputeInput from ocean_lib.models.datatoken_base import DatatokenBase from ocean_lib.ocean.ocean import Ocean from ocean_lib.ocean.ocean_assets import OceanAssets from ocean_lib.ocean.util import to_wei from ocean_lib.structures.algorithm_metadata import AlgorithmMetadata from tests.resources.ddo_helpers import ( get_first_service_by_type, get_registered_asset_with_compute_service, ) from tests.resources.helper_functions import ( get_consumer_wallet, get_publisher_ocean_instance, get_publisher_wallet, ) class TestComputeFlow(object): @classmethod def setup_class(self): publisher_wallet = get_publisher_wallet() consumer_wallet = get_consumer_wallet() publisher_ocean = get_publisher_ocean_instance() _, _, ddo = get_registered_asset_with_compute_service( publisher_ocean, publisher_wallet ) self.dataset_with_compute_service = ddo _, _, ddo = get_registered_asset_with_compute_service( publisher_ocean, publisher_wallet, allow_raw_algorithms=True ) self.dataset_with_compute_service_allow_raw_algo = ddo _, _, ddo = get_registered_asset_with_compute_service( publisher_ocean, publisher_wallet ) self.dataset_for_update = ddo _, _, ddo = get_registered_asset_with_compute_service( publisher_ocean, publisher_wallet, trusted_algorithm_publishers=[publisher_wallet.address], ) self.dataset_with_compute_service_and_trusted_publisher = ddo self.algorithm = get_algorithm(publisher_wallet, publisher_ocean) self.algorithm_with_different_publisher = get_algorithm( consumer_wallet, publisher_ocean ) self.published_ddos = {} def wait_for_ddo(self, ddo): if ddo.did not in self.published_ddos: config = get_config_dict() data_provider = DataServiceProvider ocean_assets = OceanAssets(config, data_provider) ddo = ocean_assets._aquarius.wait_for_ddo(ddo.did) self.published_ddos[ddo.did] = ddo return self.published_ddos[ddo.did] @pytest.mark.integration def test_compute_raw_algo( self, publisher_wallet, publisher_ocean, consumer_wallet, raw_algorithm, ): """Tests that a compute job with a raw algorithm starts properly.""" self.wait_for_ddo(self.dataset_with_compute_service_allow_raw_algo) run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service_allow_raw_algo, None ), algorithm_meta=raw_algorithm, ) self.wait_for_ddo(self.dataset_with_compute_service) with pytest.raises(DataProviderException, match="no_raw_algo_allowed"): run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service, None ), algorithm_meta=raw_algorithm, ) @pytest.mark.integration def test_compute_registered_algo( self, publisher_wallet, publisher_ocean, consumer_wallet, ): """Tests that a compute job with a registered algorithm starts properly.""" self.wait_for_ddo(self.dataset_with_compute_service) self.wait_for_ddo(self.algorithm) run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service, None ), algorithm_and_userdata=AssetAndUserdata(self.algorithm, None), ) @pytest.mark.integration def test_compute_reuse_order( self, publisher_wallet, publisher_ocean, consumer_wallet, ): """Tests that a compute job with a registered algorithm starts properly.""" self.wait_for_ddo(self.dataset_with_compute_service) self.wait_for_ddo(self.algorithm) run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service, None ), algorithm_and_userdata=AssetAndUserdata(self.algorithm, None), scenarios=["reuse_order"], ) @pytest.mark.integration def test_compute_multi_inputs( self, publisher_wallet, publisher_ocean, consumer_wallet, basic_asset, ): """Tests that a compute job with additional Inputs (multiple assets) starts properly.""" _, _, dataset_with_access_service = basic_asset self.wait_for_ddo(self.dataset_with_compute_service) self.wait_for_ddo(self.algorithm) run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service, None ), algorithm_and_userdata=AssetAndUserdata(self.algorithm, None), additional_datasets_and_userdata=[ AssetAndUserdata( dataset_with_access_service, {"test_key": "test_value"} ) ], ) @pytest.mark.integration def test_compute_trusted_algorithm(self): # functionality covered in test_compute_update_trusted_algorithm assert True @pytest.mark.integration def test_compute_update_trusted_algorithm( self, publisher_wallet, publisher_ocean, consumer_wallet, ): self.wait_for_ddo(self.dataset_for_update) self.wait_for_ddo(self.algorithm) trusted_algo_list = [self.algorithm.generate_trusted_algorithms()] compute_service = get_first_service_by_type(self.dataset_for_update, "compute") compute_service.update_compute_values( trusted_algorithms=trusted_algo_list, trusted_algo_publishers=[], allow_network_access=True, allow_raw_algorithm=False, ) updated_dataset = publisher_ocean.assets.update( self.dataset_for_update, {"from": publisher_wallet} ) # Expect to pass when trusted algorithm is used run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata(updated_dataset, None), algorithm_and_userdata=AssetAndUserdata(self.algorithm, None), scenarios=["with_result"], ) self.wait_for_ddo(self.algorithm_with_different_publisher) # Expect to fail when non-trusted algorithm is used with pytest.raises( DataProviderException, match="not_trusted_algo", ): run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata(updated_dataset, None), algorithm_and_userdata=AssetAndUserdata( self.algorithm_with_different_publisher, None ), scenarios=["with_result"], ) @pytest.mark.integration def test_compute_trusted_publisher( self, publisher_wallet, publisher_ocean, consumer_wallet, ): self.wait_for_ddo(self.dataset_with_compute_service_and_trusted_publisher) self.wait_for_ddo(self.algorithm) # Expect to pass when algorithm with trusted publisher is used run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service_and_trusted_publisher, None ), algorithm_and_userdata=AssetAndUserdata(self.algorithm, None), ) self.wait_for_ddo(self.algorithm_with_different_publisher) # Expect to fail when algorithm with non-trusted publisher is used with pytest.raises(DataProviderException, match="not_trusted_algo_publisher"): run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service_and_trusted_publisher, None ), algorithm_and_userdata=AssetAndUserdata( self.algorithm_with_different_publisher, None ), ) @pytest.mark.integration def test_compute_just_provider_fees( self, publisher_wallet, publisher_ocean, consumer_wallet, ): self.wait_for_ddo(self.dataset_with_compute_service) self.wait_for_ddo(self.algorithm) """Tests that the correct compute provider fees are calculated.""" run_compute_test( ocean_instance=publisher_ocean, publisher_wallet=publisher_wallet, consumer_wallet=consumer_wallet, dataset_and_userdata=AssetAndUserdata( self.dataset_with_compute_service, None ), algorithm_and_userdata=AssetAndUserdata(self.algorithm, None), scenarios=["just_fees"], ) def get_algorithm(wallet, ocean_instance): # Setup algorithm meta to run raw algorithm _, _, ddo = ocean_instance.assets.create_algo_asset( name="Sample asset", url="https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/branin_and_gpr/gpr.py", tx_dict={"from": wallet}, image="oceanprotocol/algo_dockers", tag="python-branin", checksum="sha256:8221d20c1c16491d7d56b9657ea09082c0ee4a8ab1a6621fa720da58b09580e4", wait_for_aqua=False, ) # verify the asset is available in Aquarius ocean_instance.assets.resolve(ddo.did) return ddo @pytest.fixture def raw_algorithm(): req = requests.get( "https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js" ) return AlgorithmMetadata( { "rawcode": req.text, "language": "Node.js", "format": "docker-image", "version": "0.1", "container": { "entrypoint": "python $ALGO", "image": "oceanprotocol/algo_dockers", "tag": "python-branin", "checksum": "sha256:8221d20c1c16491d7d56b9657ea09082c0ee4a8ab1a6621fa720da58b09580e4", }, } ) @dataclass class AssetAndUserdata: ddo: DDO userdata: Optional[dict] def _mint_and_build_compute_input( dataset_and_userdata: AssetAndUserdata, service_type: str, publisher_wallet, consumer_wallet, ocean_instance: Ocean, ) -> ComputeInput: service = get_first_service_by_type(dataset_and_userdata.ddo, service_type) datatoken = DatatokenBase.get_typed(ocean_instance.config_dict, service.datatoken) minter = ( consumer_wallet if datatoken.isMinter(consumer_wallet.address) else publisher_wallet ) datatoken.mint(consumer_wallet.address, to_wei(10), {"from": minter}) return ComputeInput( dataset_and_userdata.ddo, service, userdata=dataset_and_userdata.userdata, consume_market_order_fee_token=datatoken.address, consume_market_order_fee_amount=0, ) def run_compute_test( ocean_instance: Ocean, publisher_wallet, consumer_wallet, dataset_and_userdata: AssetAndUserdata, algorithm_and_userdata: Optional[AssetAndUserdata] = None, algorithm_meta: Optional[AlgorithmMetadata] = None, algorithm_algocustomdata: Optional[dict] = None, additional_datasets_and_userdata: List[AssetAndUserdata] = [], scenarios: Optional[List[str]] = None, ): """Helper function to bootstrap compute job creation and status checking.""" assert ( algorithm_and_userdata or algorithm_meta ), "either algorithm_and_userdata or algorithm_meta must be provided." if not scenarios: scenarios = [] datasets = [ _mint_and_build_compute_input( dataset_and_userdata, ServiceTypes.CLOUD_COMPUTE, publisher_wallet, consumer_wallet, ocean_instance, ) ] # build additional datasets for asset_and_userdata in additional_datasets_and_userdata: service_type = ServiceTypes.ASSET_ACCESS if not get_first_service_by_type(asset_and_userdata.ddo, service_type): service_type = ServiceTypes.CLOUD_COMPUTE datasets.append( _mint_and_build_compute_input( asset_and_userdata, service_type, publisher_wallet, consumer_wallet, ocean_instance, ) ) # Order algo download service (aka. access service) algorithm = None if algorithm_and_userdata: algorithm = _mint_and_build_compute_input( algorithm_and_userdata, ServiceTypes.ASSET_ACCESS, publisher_wallet, consumer_wallet, ocean_instance, ) service = get_first_service_by_type( dataset_and_userdata.ddo, ServiceTypes.CLOUD_COMPUTE ) try: free_c2d_env = ocean_instance.compute.get_free_c2d_environment( service.service_endpoint, 8996 ) except StopIteration: assert False, "No free c2d environment found." time_difference = ( timedelta(hours=1) if "reuse_order" not in scenarios else timedelta(seconds=30) ) valid_until = int((datetime.now(timezone.utc) + time_difference).timestamp()) if "just_fees" in scenarios: fees_response = ocean_instance.retrieve_provider_fees_for_compute( datasets, algorithm if algorithm else algorithm_meta, consumer_address=free_c2d_env["consumerAddress"], compute_environment=free_c2d_env["id"], valid_until=valid_until, ) assert "algorithm" in fees_response assert len(fees_response["datasets"]) == 1 return datasets, algorithm = ocean_instance.assets.pay_for_compute_service( datasets, algorithm if algorithm else algorithm_meta, consumer_address=free_c2d_env["consumerAddress"], compute_environment=free_c2d_env["id"], valid_until=valid_until, consume_market_order_fee_address=consumer_wallet.address, tx_dict={"from": consumer_wallet}, ) # Start compute job job_id = ocean_instance.compute.start( consumer_wallet, datasets[0], free_c2d_env["id"], algorithm, algorithm_meta, algorithm_algocustomdata, datasets[1:], ) status = ocean_instance.compute.status( dataset_and_userdata.ddo, service, job_id, consumer_wallet ) print(f"got job status: {status}") assert ( status and status["ok"] ), f"something not right about the compute job, got status: {status}" status = ocean_instance.compute.stop( dataset_and_userdata.ddo, service, job_id, consumer_wallet ) print(f"got job status after requesting stop: {status}") assert status, f"something not right about the compute job, got status: {status}" if "with_result" in scenarios: succeeded = False for _ in range(0, 300): status = ocean_instance.compute.status( dataset_and_userdata.ddo, service, job_id, consumer_wallet ) # wait until job is done, see: # https://github.com/oceanprotocol/operator-service/blob/main/API.md#status-description if status["status"] > 60: succeeded = True break time.sleep(5) print(f"got status: {status}") assert succeeded, "compute job unsuccessful" log_file = ocean_instance.compute.compute_job_result_logs( dataset_and_userdata.ddo, service, job_id, consumer_wallet, "algorithmLog" ) print(f"got algo log file: {str(log_file)}") _ = ocean_instance.compute.result( dataset_and_userdata.ddo, service, job_id, 0, consumer_wallet ) prev_dt_tx_id = datasets[0].transfer_tx_id prev_algo_tx_id = algorithm.transfer_tx_id # retry initialize but all orders are already valid datasets, algorithm = ocean_instance.assets.pay_for_compute_service( datasets, algorithm if algorithm else algorithm_meta, consumer_address=free_c2d_env["consumerAddress"], compute_environment=free_c2d_env["id"], valid_until=valid_until, consume_market_order_fee_address=consumer_wallet.address, tx_dict={"from": consumer_wallet}, ) # transferTxId was not updated assert datasets[0].transfer_tx_id == prev_dt_tx_id assert algorithm.transfer_tx_id == prev_algo_tx_id if "reuse_order" in scenarios: prev_dt_tx_id = datasets[0].transfer_tx_id prev_algo_tx_id = algorithm.transfer_tx_id # ensure order expires time.sleep(time_difference.seconds + 1) valid_until = int((datetime.now(timezone.utc) + time_difference).timestamp()) datasets, algorithm = ocean_instance.assets.pay_for_compute_service( datasets, algorithm if algorithm else algorithm_meta, consumer_address=free_c2d_env["consumerAddress"], compute_environment=free_c2d_env["id"], valid_until=valid_until, consume_market_order_fee_address=consumer_wallet.address, tx_dict={"from": consumer_wallet}, ) assert datasets[0].transfer_tx_id != prev_dt_tx_id assert algorithm.transfer_tx_id != prev_algo_tx_id job_id = ocean_instance.compute.start( consumer_wallet, datasets[0], free_c2d_env["id"], algorithm, algorithm_meta, algorithm_algocustomdata, datasets[1:], ) assert job_id, "can not reuse order" ================================================ FILE: tests/integration/ganache/test_consume_flow.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os import shutil import pytest from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.ocean.ocean_assets import OceanAssets from ocean_lib.ocean.util import get_address_of_type, to_wei from tests.resources.ddo_helpers import get_first_service_by_type ARWEAVE_TRANSACTION_ID = "a4qJoQZa1poIv5guEzkfgZYSAD0uYm7Vw4zm_tCswVQ" @pytest.mark.integration @pytest.mark.parametrize("asset_type", ["simple", "graphql", "onchain", "arweave"]) def test_consume_asset( config: dict, publisher_wallet, consumer_wallet, basic_asset, asset_type ): data_provider = DataServiceProvider ocean_assets = OceanAssets(config, data_provider) if asset_type == "simple": data_nft, dt, ddo = basic_asset elif asset_type == "arweave": data_nft, dt, ddo = ocean_assets.create_arweave_asset( "Data NFTs in Ocean", ARWEAVE_TRANSACTION_ID, {"from": publisher_wallet} ) elif asset_type == "onchain": abi = { "inputs": [], "name": "swapOceanFee", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function", } router_address = get_address_of_type(config, "Router") data_nft, dt, ddo = ocean_assets.create_onchain_asset( "Data NFTs in Ocean", router_address, abi, {"from": publisher_wallet} ) else: url = "http://172.15.0.15:8030/graphql" query = """ query{ indexingStatuses{ subgraph chains node } } """ data_nft, dt, ddo = ocean_assets.create_graphql_asset( "Data NFTs in Ocean", url, query, {"from": publisher_wallet} ) assert ddo, "The ddo is not created." assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address if asset_type != "simple": assert ddo.datatokens[0]["name"] == "Data NFTs in Ocean: DT1" service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS) # Mint 50 datatokens in consumer wallet from publisher. Max cap = 100 dt.mint( consumer_wallet.address, to_wei(50), {"from": publisher_wallet}, ) # Initialize service response = data_provider.initialize( did=ddo.did, service=service, consumer_address=consumer_wallet.address ) assert response assert response.status_code == 200 assert response.json()["providerFee"] provider_fees = response.json()["providerFee"] # Start order for consumer receipt = dt.start_order( consumer=consumer_wallet.address, service_index=ddo.get_index_of_service(service), provider_fees=provider_fees, tx_dict={"from": consumer_wallet}, ) # Download file destination = config["DOWNLOADS_PATH"] if not os.path.isabs(destination): destination = os.path.abspath(destination) if os.path.exists(destination) and len(os.listdir(destination)) > 0: list( map( lambda d: shutil.rmtree(os.path.join(destination, d)), os.listdir(destination), ) ) if not os.path.exists(destination): os.makedirs(destination) assert len(os.listdir(destination)) == 0 ocean_assets.download_asset( ddo, consumer_wallet, destination, receipt.transactionHash.hex(), service, ) assert ( len(os.listdir(os.path.join(destination, os.listdir(destination)[0]))) == 1 ), "The asset folder is empty." ================================================ FILE: tests/integration/ganache/test_disconnecting_components.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import threading import time import pytest import requests from ocean_lib.assets.ddo import DDO from ocean_lib.data_provider.data_encryptor import DataEncryptor from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.example_config import DEFAULT_PROVIDER_URL from ocean_lib.ocean.ocean import Ocean exception_flag = 0 def test_with_wrong_provider(config, caplog): """Tests encrypt with a good provider URL and then switch to a bad one.""" config["PROVIDER_URL"] = DEFAULT_PROVIDER_URL updating_thread = threading.Thread( target=_update_with_wrong_component, args=(config,), ) updating_thread.start() _iterative_encrypt(config) updating_thread.join() assert "Asset urls encrypted successfully" in caplog.text assert exception_flag == 1 def test_with_wrong_aquarius(publisher_wallet, caplog, monkeypatch, config): """Tests DDO creation with a good config.ini and then switch to a bad one.""" config["METADATA_CACHE_URI"] = "http:://not-valid-aqua.com" with pytest.raises(Exception, match="Invalid or unresponsive aquarius url"): ocean = Ocean(config, DataServiceProvider) config["METADATA_CACHE_URI"] = "http://172.15.0.5:5000" ocean = Ocean(config, DataServiceProvider) # force a bad URL, assuming initial Ocean and Aquarius objects were created successfully ocean.assets._aquarius.base_url = "http://not-valid-aqua.com" with pytest.raises(Exception): ocean.assets._aquarius.validate_ddo(DDO()) def _create_ddo(ocean, publisher): global exception_flag time.sleep(5) try: ocean.assets.create_url_asset( "Sample asset", "https://foo.txt", {"from": publisher} ) except requests.exceptions.InvalidURL as err: exception_flag = 1 assert err.args[0] == "InvalidURL http://foourl.com." except requests.exceptions.ConnectionError as e: exception_flag = 2 assert ( e.args[0] .args[0] .startswith("HTTPConnectionPool(host='fooaqua.com', port=80)") ) def _iterative_create_ddo(mock_ocean, publisher): time.sleep(10) _create_ddo(mock_ocean.return_value, publisher) def _iterative_encrypt(mock): global exception_flag for _ in range(5): try: DataEncryptor.encrypt({}, mock["PROVIDER_URL"], 8996) except requests.exceptions.InvalidURL as err: exception_flag = 1 assert err.args[0] == "InvalidURL http://foourl.com." time.sleep(1) def _update_with_wrong_component(mock): time.sleep(2) mock["PROVIDER_URL"] = "http://foourl.com" ================================================ FILE: tests/integration/ganache/test_graphql.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json import os import shutil import pytest from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.ocean.ocean import Ocean from ocean_lib.ocean.util import to_wei from tests.resources.ddo_helpers import get_first_service_by_type @pytest.mark.integration def test_consume_simple_graphql_query( config: dict, publisher_wallet, consumer_wallet, ): data_provider = DataServiceProvider ocean = Ocean(config) url = "http://172.15.0.15:8030/graphql" query = """ query{ indexingStatuses{ subgraph chains node } } """ data_nft, dt, ddo = ocean.assets.create_graphql_asset( "Data NFTs in Ocean", url, query, {"from": publisher_wallet} ) assert ddo, "The ddo is not created." assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "Data NFTs in Ocean: DT1" service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS) # Mint 50 datatokens in consumer wallet from publisher. Max cap = 100 dt.mint( consumer_wallet.address, to_wei(50), {"from": publisher_wallet}, ) # Initialize service response = data_provider.initialize( did=ddo.did, service=service, consumer_address=consumer_wallet.address ) assert response assert response.status_code == 200 assert response.json()["providerFee"] provider_fees = response.json()["providerFee"] # Start order for consumer receipt = dt.start_order( consumer=consumer_wallet.address, service_index=ddo.get_index_of_service(service), provider_fees=provider_fees, tx_dict={"from": consumer_wallet}, ) # Download file destination = config["DOWNLOADS_PATH"] if not os.path.isabs(destination): destination = os.path.abspath(destination) if os.path.exists(destination) and len(os.listdir(destination)) > 0: list( map( lambda d: shutil.rmtree(os.path.join(destination, d)), os.listdir(destination), ) ) if not os.path.exists(destination): os.makedirs(destination) assert len(os.listdir(destination)) == 0 ocean.assets.download_asset( ddo, consumer_wallet, destination, receipt.transactionHash.hex(), service, ) file_path = os.path.join(destination, os.listdir(destination)[0]) assert len(os.listdir(file_path)) == 1, "The asset folder is empty." with open(os.path.join(file_path, os.listdir(file_path)[0])) as f: contents = f.readlines() content = json.loads(contents[0]) assert "data" in content.keys() assert "indexingStatuses" in content["data"].keys() ================================================ FILE: tests/integration/ganache/test_market_flow.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os import pytest from web3.main import Web3 from ocean_lib.models.datatoken_base import TokenFeeInfo from ocean_lib.ocean.util import to_wei from tests.resources.helper_functions import get_another_consumer_ocean_instance @pytest.mark.integration @pytest.mark.parametrize("consumer_type", ["publisher", "another_user"]) def test_market_flow( publisher_wallet, consumer_wallet, basic_asset, consumer_ocean, another_consumer_wallet, consumer_type, ): """Tests that an order is correctly placed on the market. The parameter implicit_none sends the payload with an empty key as the delegated consumer. The parameter explicit_none sends None as the delegated consumer, explicitly.""" consumer_ocean = consumer_ocean another_consumer_ocean = get_another_consumer_ocean_instance(use_provider_mock=True) data_nft, datatoken, ddo = basic_asset service = ddo.services[0] # Mint data tokens and assign to publisher datatoken.mint( publisher_wallet.address, to_wei(50), {"from": publisher_wallet}, ) # Give the consumer some datatokens so they can order the service datatoken.transfer(consumer_wallet.address, to_wei(10), {"from": publisher_wallet}) # Place order for the download service if consumer_type == "publisher": order_tx_id = consumer_ocean.assets.pay_for_access_service( ddo, {"from": consumer_wallet}, service=service, consume_market_fees=TokenFeeInfo(token=datatoken.address), ).hex() asset_folder = consumer_ocean.assets.download_asset( ddo, consumer_wallet, consumer_ocean.config_dict["DOWNLOADS_PATH"], order_tx_id, service, ) else: order_tx_id = consumer_ocean.assets.pay_for_access_service( ddo, {"from": consumer_wallet}, service=service, consume_market_fees=TokenFeeInfo( address=another_consumer_wallet.address, token=datatoken.address, ), consumer_address=another_consumer_wallet.address, ).hex() asset_folder = consumer_ocean.assets.download_asset( ddo, another_consumer_wallet, another_consumer_ocean.config_dict["DOWNLOADS_PATH"], order_tx_id, service, ) assert len(os.listdir(asset_folder)) >= 1, "The asset folder is empty." orders = consumer_ocean.get_user_orders(consumer_wallet.address, datatoken.address) assert ( orders ), f"no orders found using the order history: datatoken {datatoken.address}, consumer {consumer_wallet.address}" orders = consumer_ocean.get_user_orders( consumer_wallet.address, Web3.to_checksum_address(datatoken.address), ) assert ( orders ), f"no orders found using the order history: datatoken {datatoken.address}, consumer {consumer_wallet.address}" @pytest.mark.integration def test_pay_for_access_service_good_default( basic_asset, publisher_wallet, consumer_wallet, consumer_ocean, ): data_nft, datatoken, ddo = basic_asset service = ddo.services[0] # Mint datatokens to consumer datatoken.mint(consumer_wallet.address, to_wei(50), {"from": publisher_wallet}) # Place order for the download service # - Here, use good defaults for service, and fee-related args order_tx_id = consumer_ocean.assets.pay_for_access_service( ddo, {"from": consumer_wallet} ).hex() asset_folder = consumer_ocean.assets.download_asset( ddo, consumer_wallet, consumer_ocean.config_dict["DOWNLOADS_PATH"], order_tx_id, service, ) # basic check. Leave thorough checks to other tests here assert len(os.listdir(asset_folder)) >= 1, "The asset folder is empty." ================================================ FILE: tests/integration/ganache/test_onchain.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os import shutil import pytest from ocean_lib.agreements.service_types import ServiceTypes from ocean_lib.data_provider.data_service_provider import DataServiceProvider from ocean_lib.models.datatoken_base import DatatokenArguments, DatatokenBase from ocean_lib.ocean.ocean_assets import OceanAssets from ocean_lib.ocean.util import get_address_of_type, to_wei from ocean_lib.structures.file_objects import SmartContractCall from tests.resources.ddo_helpers import get_first_service_by_type @pytest.mark.integration def test_consume_parametrized_onchain_data( config: dict, publisher_wallet, consumer_wallet, ): data_provider = DataServiceProvider ocean_assets = OceanAssets(config, data_provider) metadata = { "created": "2020-11-15T12:27:48Z", "updated": "2021-05-17T21:58:02Z", "description": "Sample description", "name": "Sample asset", "type": "dataset", "author": "OPF", "license": "https://market.oceanprotocol.com/terms", } abi = { "inputs": [{"internalType": "address", "name": "baseToken", "type": "address"}], "name": "getOPCFee", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function", } router_address = get_address_of_type(config, "Router") onchain_data = SmartContractCall( address=router_address, chainId=config["web3_instance"].eth.chain_id, abi=abi, ) files = [onchain_data] # to consume dataset, consumer needs to send a value for nftAddress consumer_parameters = [ { "name": "baseToken", "type": "text", "label": "baseToken", "required": True, "description": "baseToken to check for fee", "default": "0x0000000000000000000000000000000000000000", } ] # Publish a plain asset with one data token on chain dt_arg = DatatokenArguments(files=files, consumer_parameters=consumer_parameters) data_nft, _, ddo = ocean_assets.create( metadata=metadata, tx_dict={"from": publisher_wallet}, datatoken_args=[dt_arg], ) assert ddo, "The ddo is not created." assert ddo.nft["name"] == "Sample asset" assert ddo.nft["symbol"] == "Sample asset" assert ddo.nft["address"] == data_nft.address assert ddo.nft["owner"] == publisher_wallet.address assert ddo.datatokens[0]["name"] == "Datatoken 1" assert ddo.datatokens[0]["symbol"] == "DT1" service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS) dt = DatatokenBase.get_typed(config, ddo.datatokens[0]["address"]) # Mint 50 datatokens in consumer wallet from publisher. Max cap = 100 dt.mint( consumer_wallet.address, to_wei(50), {"from": publisher_wallet}, ) # Initialize service response = data_provider.initialize( did=ddo.did, service=service, consumer_address=consumer_wallet.address ) assert response assert response.status_code == 200 assert response.json()["providerFee"] provider_fees = response.json()["providerFee"] # Start order for consumer receipt = dt.start_order( consumer=consumer_wallet.address, service_index=ddo.get_index_of_service(service), provider_fees=provider_fees, tx_dict={"from": consumer_wallet}, ) # Download file destination = config["DOWNLOADS_PATH"] if not os.path.isabs(destination): destination = os.path.abspath(destination) if os.path.exists(destination) and len(os.listdir(destination)) > 0: list( map( lambda d: shutil.rmtree(os.path.join(destination, d)), os.listdir(destination), ) ) if not os.path.exists(destination): os.makedirs(destination) assert len(os.listdir(destination)) == 0 # this is where user is sending the required consumer_parameters userdata = {"baseToken": ddo.nft_address.lower()} ocean_assets.download_asset( ddo, consumer_wallet, destination, receipt.transactionHash.hex(), service, userdata=userdata, ) dir_files = os.listdir(os.path.join(destination, os.listdir(destination)[0])) assert len(dir_files) == len(files), "The asset folder is empty." ================================================ FILE: tests/integration/remote/__init__.py ================================================ ================================================ FILE: tests/integration/remote/test_mumbai_main.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os from ocean_lib.example_config import get_config_dict from ocean_lib.ocean.ocean import Ocean from . import util def _get_mumbai_rpc(): infura_id = os.getenv("WEB3_INFURA_PROJECT_ID") if not infura_id: return "https://rpc-mumbai.maticvigil.com" return f"https://polygon-mumbai.infura.io/v3/{infura_id}" def test_nonocean_tx(tmp_path, monkeypatch): """Do a simple non-Ocean tx on Mumbai. Only use Ocean config""" monkeypatch.delenv("ADDRESS_FILE") # setup config = get_config_dict(_get_mumbai_rpc()) ocean = Ocean(config) (alice_wallet, bob_wallet) = util.get_wallets() # Do a simple-as-possible test that uses ocean stack, while accounting for gotchas util.do_nonocean_tx_and_handle_gotchas(ocean, alice_wallet, bob_wallet) def test_ocean_tx__create(tmp_path, monkeypatch): """On Mumbai, do a simple Ocean tx: create""" monkeypatch.delenv("ADDRESS_FILE") # setup config = get_config_dict(_get_mumbai_rpc()) ocean = Ocean(config) (alice_wallet, _) = util.get_wallets() # Do a simple-as-possible test that uses ocean stack, while accounting for gotchas util.do_ocean_tx_and_handle_gotchas(ocean, alice_wallet) ================================================ FILE: tests/integration/remote/test_mumbai_readme.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pathlib import runpy from . import util def test_simple_remote_readme(monkeypatch): monkeypatch.delenv("ADDRESS_FILE") (ref_alice_wallet, _) = util.get_wallets() # README generation command: # mkcodes --github --output tests/generated-readmes/test_{name}.{ext} READMEs script = pathlib.Path( __file__, "..", "..", "..", "generated-readmes", "test_setup-remote.py" ) try: result = runpy.run_path(str(script), run_name="__main__") except AssertionError as e: # skip if zero funds in account if "Alice needs MATIC" in str(e) or "Bob needs MATIC" in str(e): return raise (e) ocean = result["ocean"] alice = result["alice"] # at this point, this script should have set up ocean and the wallets # make sure that the script used REMOTE_TEST_PRIVATE_KEY1 wallet, like reference wallet assert alice.address == ref_alice_wallet.address # besides what the readme script does, is it actually able to do more? util.do_ocean_tx_and_handle_gotchas(ocean, alice) ================================================ FILE: tests/integration/remote/test_polygon.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os import pytest from ocean_lib.example_config import get_config_dict from ocean_lib.ocean.ocean import Ocean from . import util def _get_polygon_rpc(): infura_id = os.getenv("WEB3_INFURA_PROJECT_ID") if not infura_id: return "https://polygon-rpc.com" return f"https://polygon-mainnet.infura.io/v3/{infura_id}" @pytest.mark.integration def test_ocean_tx__create(tmp_path, monkeypatch): """On Polygon, do a simple Ocean tx: create""" monkeypatch.delenv("ADDRESS_FILE") # setup config = get_config_dict(_get_polygon_rpc()) ocean = Ocean(config) (alice_wallet, _) = util.get_wallets() # Do a simple-as-possible test that uses ocean stack, while accounting for gotchas util.do_ocean_tx_and_handle_gotchas(ocean, alice_wallet) ================================================ FILE: tests/integration/remote/util.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os import random import string import time import warnings from enforce_typing import enforce_types from eth_account import Account from ocean_lib.ocean.util import send_ether, to_wei from ocean_lib.web3_internal.utils import get_gas_fees @enforce_types def remote_config_mumbai(tmp_path): config = { "NETWORK_NAME": "mumbai", "METADATA_CACHE_URI": "https://v4.aquarius.oceanprotocol.com", "PROVIDER_URL": "https://v4.provider.mumbai.oceanprotocol.com", "DOWNLOADS_PATH": "consume-downloads", } return config @enforce_types def remote_config_polygon(tmp_path): config = { "NETWORK_NAME": "polygon", "METADATA_CACHE_URI": "https://v4.aquarius.oceanprotocol.com", "PROVIDER_URL": "https://v4.provider.polygon.oceanprotocol.com", "DOWNLOADS_PATH": "consume-downloads", } return config @enforce_types def get_wallets(): alice_private_key = os.getenv("REMOTE_TEST_PRIVATE_KEY1") bob_private_key = os.getenv("REMOTE_TEST_PRIVATE_KEY2") instrs = "You must set it. It must hold Mumbai MATIC." assert alice_private_key, f"Need envvar REMOTE_TEST_PRIVATE_KEY1. {instrs}" assert bob_private_key, f"Need envvar REMOTE_TEST_PRIVATE_KEY2. {instrs}" # wallets alice_wallet = Account.from_key(private_key=alice_private_key) bob_wallet = Account.from_key(private_key=bob_private_key) print(f"alice_wallet.address = '{alice_wallet.address}'") print(f"bob_wallet.address = '{bob_wallet.address}'") return (alice_wallet, bob_wallet) @enforce_types def do_nonocean_tx_and_handle_gotchas(ocean, alice_wallet, bob_wallet): """Call wallet.transfer(), but handle several gotchas for this test use case: - if the test has to repeat, there are nonce errors. Avoid via unique - if there are insufficient funds, since they're hard to replace automatically in remote testnets, then just skip """ # Simplest possible tx: Alice send Bob some fake MATIC web3 = ocean.config_dict["web3_instance"] bob_eth_before = web3.eth.get_balance(bob_wallet.address) normalized_unixtime = time.time() / 1e9 amt_send = 1e-8 * (random.random() + normalized_unixtime) print("Do a send-Ether tx...") try: priority_fee, _ = get_gas_fees() send_ether( ocean.config_dict, alice_wallet, bob_wallet.address, to_wei(amt_send), priority_fee=priority_fee, ) bob_eth_after = web3.eth.get_balance(bob_wallet.address) except Exception as e: if error_is_skippable(str(e)): warnings.warn(UserWarning(f"Warning: EVM reported error: {e}")) return raise (e) assert bob_eth_after > bob_eth_before print("Success") @enforce_types def do_ocean_tx_and_handle_gotchas(ocean, alice_wallet): """Call create() from data NFT, but handle several gotchas for this test use case: - if the test has to repeat, there are nonce errors. Avoid via unique - if there are insufficient funds, since they're hard to replace automatically in remote testnets, then just skip """ # Alice publish data NFT # avoid "replacement transaction underpriced" error: make each tx diff't symbol = random_chars() print("Call create() from data NFT, and wait for it to complete...") num_retries = 2 while num_retries != 0: try: priority_fee, max_fee = get_gas_fees() data_nft = ocean.data_nft_factory.create( { "from": alice_wallet, "maxPriorityFeePerGas": priority_fee, "maxFeePerGas": max_fee, "gas": ocean.config_dict["web3_instance"] .eth.get_block("latest") .gasLimit, }, symbol, symbol, ) data_nft_symbol = data_nft.symbol() break except Exception as e: if error_is_skippable(str(e)): warnings.warn(UserWarning(f"Warning: EVM reported error: {e}")) return if "Tx dropped" in str(e): num_retries -= 1 warnings.warn( UserWarning(f"Warning: EVM reported error: {e}\n Retrying...") ) continue raise (e) assert data_nft_symbol == symbol print("Success") @enforce_types def error_is_skippable(error_s: str) -> bool: return ( "insufficient funds" in error_s or "underpriced" in error_s or "exceeds block gas limit" in error_s or "exceeds the configured cap" in error_s or "No contract deployed at" in error_s or "nonce too low" in error_s or "Internal error" in error_s or "execution reverted" in error_s or "No data was returned - the call likely reverted" in error_s or "The field extraData is 97 bytes, but should be 32." in error_s ) @enforce_types def random_chars() -> str: cand_chars = string.ascii_uppercase + string.digits s = "".join(random.choices(cand_chars, k=8)) + str(time.time()) return s ================================================ FILE: tests/readmes/conftest.py ================================================ from conftest_ganache import * ================================================ FILE: tests/readmes/test_readmes.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import pathlib import runpy import pytest # This file tests READMEs on local chain (ganache). # For tests of READMEs on remote chains, see tests/integration/remote/ scripts = pathlib.Path(__file__, "..", "..", "generated-readmes").resolve().glob("*.py") script_names = [script.name for script in scripts if script.name != "__init__.py"] class TestReadmes(object): @classmethod def setup_class(self): globs = {} prerequisite = pathlib.Path( __file__, "..", "..", "generated-readmes/test_setup-local.py", ) result = runpy.run_path(str(prerequisite), run_name="__main__") for key in ["os", "config", "ocean", "alice", "bob", "carlos"]: globs[key] = result[key] self.globs = globs @pytest.mark.parametrize("script_name", script_names) def test_script_execution(self, script_name): # README generation command: # mkcodes --github --output tests/generated-readmes/test_{name}.{ext} READMEs skippable = [ "c2d-flow-more-examples", "developers", "df", "install", "parameters", "predict-eth", "services", "setup-local", "setup-remote", "publish-flow-credentials", "publish-flow-restapi", # TODO: fix and restore "gas-strategy-remote", "using-clef", # no way to approve transactions through automatic readme ] if script_name.replace("test_", "").replace(".py", "") in skippable: return script = pathlib.Path(__file__, "..", "..", "generated-readmes", script_name) runpy.run_path(str(script), run_name="__main__", init_globals=self.globs) ================================================ FILE: tests/resources/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: tests/resources/ddo/ddo_algorithm.json ================================================ { "@context": "https://w3id.org/did/v1", "created": "2019-02-08T08:13:49Z", "id": "did:op:8d1b4d73e7af4634958f071ab8dfe7ab0df14019", "proof": { "created": "2019-02-08T08:13:41Z", "creator": "0x37BB53e3d293494DE59fBe1FF78500423dcFd43B", "signatureValue": "did:op:0bc278fee025464f8012b811d1bce8e22094d0984e4e49139df5d5ff7a028bdf", "type": "DDOIntegritySignature", "checksum": { "0": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "1": "0x999999952b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3" } }, "verifiableCredential": [ { "@context": [ "https://www.w3.org/2018/credentials/v1", "https://www.w3.org/2018/credentials/examples/v1" ], "id": "1872", "type": [ "read", "update", "deactivate" ], "issuer": "0x610D9314EDF2ced7681BA1633C33fdb8cF365a12", "issuanceDate": "2019-01-01T19:73:24Z", "credentialSubject": { "id": "0x89328493849328493284932" }, "proof": { "type": "RsaSignature2018", "created": "2019-01-01T19:73:24Z", "proofPurpose": "assertionMethod", "signatureValue": "ABCJSDAO23...1tzjn4w==" } } ], "service": [ { "index": 0, "serviceEndpoint": "http://localhost:5000/api/v1/aquarius/assets/ddo/{did}", "type": "metadata", "attributes": { "main": { "author": "John Doe", "dateCreated": "2019-02-08T08:13:49Z", "license": "CC-BY", "name": "My super algorithm", "type": "algorithm", "algorithm": { "language": "scala", "format": "docker-image", "version": "0.1", "container": { "entrypoint": "node $ALGO", "image": "node", "tag": "10" } }, "files": [ { "name": "build_model", "url": "https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js", "index": 0, "checksum": "efb2c764274b745f5fc37f97c6b0e761", "contentLength": "4535431", "contentType": "text/plain", "encoding": "UTF-8", "compression": "zip" } ] }, "additionalInformation": { "description": "Workflow to aggregate weather information", "tags": [ "weather", "uk", "2011", "workflow", "aggregation" ], "copyrightHolder": "John Doe" } } } ] } ================================================ FILE: tests/resources/ddo/ddo_algorithm2.json ================================================ { "@context": ["https://w3id.org/did/v1"], "id": "did:op:ACce67694eD2848dd683c651Dab7Af823b7dd123", "version": "4.1.0", "chainId": 8996, "nftAddress": "0xabc", "metadata": { "created": "2020-11-15T12:27:48Z", "updated": "2021-05-17T21:58:02Z", "description": "Sample description", "name": "Sample asset", "type": "algorithm", "author": "OPF", "license": "https://market.oceanprotocol.com/terms", "algorithm": { "container": { "entrypoint": "python $ALGO", "image": "oceanprotocol/algo_dockers", "tag": "python-branin", "checksum": "sha256:8221d20c1c16491d7d56b9657ea09082c0ee4a8ab1a6621fa720da58b09580e4" } } }, "services": [ { "id": "1", "type": "compute", "files": "0x0000", "name": "Download service", "description": "Download service", "datatokenAddress": "0x123", "serviceEndpoint": "http://172.15.0.4:8030", "timeout": 0 } ], "credentials": { "allow": [ ], "deny": [ ] }, "nft": { "address": "0x000000", "name": "Ocean Protocol Asset v4", "symbol": "OCEAN-A-v4", "owner": "0x0000000", "state": 0, "created": "2000-10-31T01:30:00" }, "datatokens": [ { "address": "0x000000", "name": "Datatoken 1", "symbol": "DT-1", "serviceId": "1" } ], "event": { "tx": "0x8d127de58509be5dfac600792ad24cc9164921571d168bff2f123c7f1cb4b11c", "block": 12831214, "from": "0xAcca11dbeD4F863Bb3bC2336D3CE5BAC52aa1f83", "contract": "0x1a4b70d8c9DcA47cD6D0Fb3c52BB8634CA1C0Fdf", "datetime": "2000-10-31T01:30:00" }, "stats": { "consumes": 4 } } ================================================ FILE: tests/resources/ddo/ddo_sa_sample.json ================================================ { "@context": "https://w3id.org/did/v1", "created": "2019-02-08T08:13:49Z", "updated": "2019-03-08T08:13:49Z", "id": "did:op:8d1b4d73e7af4634958f071ab8dfe7ab0df14019", "proof": { "created": "2019-02-08T08:13:41Z", "creator": "0x37BB53e3d293494DE59fBe1FF78500423dcFd43B", "signatureValue": "did:op:0bc278fee025464f8012b811d1bce8e22094d0984e4e49139df5d5ff7a028bdf", "type": "DDOIntegritySignature", "checksum": { "0": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "1": "0x999999952b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3" } }, "verifiableCredential": [ { "@context": [ "https://www.w3.org/2018/credentials/v1", "https://www.w3.org/2018/credentials/examples/v1" ], "id": "1872", "type": [ "read", "update", "deactivate" ], "issuer": "0x610D9314EDF2ced7681BA1633C33fdb8cF365a12", "issuanceDate": "2019-01-01T19:73:24Z", "credentialSubject": { "id": "0x89328493849328493284932" }, "proof": { "type": "RsaSignature2018", "created": "2019-01-01T19:73:24Z", "proofPurpose": "assertionMethod", "signatureValue": "ABCJSDAO23...1tzjn4w==" } } ], "service": [ { "index": 0, "serviceEndpoint": "http://localhost:5000/api/v1/aquarius/assets/ddo/{did}", "type": "metadata", "attributes": { "encryptedFiles": "0x2e48ceefcca7abb024f90c87c676fce8f7913f889605a349c08c0c4a822c69ad651e122cc81db4fbb52938ac627786491514f37a2ebfd04fd98ec726f1d9061ed52f13fde132222af34d9af8ec358429cf45fc669f81a607185cb9a8150df3cbb2b4e3e382fb16429be228ddd920f061b78dd54701025fac8aab976239fb31a5b60a57393e96a338324c5ac8a5600a1247339c4835533cecdb5b53caf6b6f9d6478b579b7426f650a4154a20d18a9d49f509770af62647a57fc174741b47af3c8beeaaa76bee276cce8fba1f3fec0e1c", "main": { "author": "Met Office", "dateCreated": "2019-02-08T08:13:49Z", "files": [ { "url": "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt", "index": 0, "checksum": "efb2c764274b745f5fc37f97c6b0e761", "contentLength": "4535431", "contentType": "text/csv", "encoding": "UTF-8", "compression": "zip" } ], "license": "CC-BY", "name": "UK Weather information 2011", "type": "dataset" }, "additionalInformation": { "description": "Weather information of UK including temperature and humidity", "tags": [ "weather", "uk", "2011", "temperature", "humidity" ], "workExample": "423432fsd,51.509865,-0.118092,2011-01-01T10:55:11+00:00,7.2,68", "copyrightHolder": "Met Office", "links": [ { "name": "Sample of Asset Data", "type": "sample", "url": "https://foo.com/sample.csv" }, { "name": "Data Format Definition", "type": "format", "url": "https://foo.com/sampl2.csv" } ], "inLanguage": "en", "updateFrecuency": "yearly", "structuredMarkup": [ { "uri": "http://skos.um.es/unescothes/C01194/jsonld", "mediaType": "application/ld+json" }, { "uri": "http://skos.um.es/unescothes/C01194/turtle", "mediaType": "text/turtle" } ] }, "curation": { "numVotes": 123, "rating": 0.0, "schema": "Binary Votting", "isListed": true } } }, { "type": "access", "index": 1, "serviceEndpoint": "http://localhost:8030", "attributes": { "main": { "name": "dataAssetAccessServiceAgreement", "creator": "", "datePublished": "2019-02-08T08:13:49Z", "cost": "1.0", "timeout": 36000 }, "additionalInformation": { "description": "" } } } ] } ================================================ FILE: tests/resources/ddo/ddo_sa_sample_disabled.json ================================================ { "@context": "https://w3id.org/did/v1", "created": "2019-02-08T08:13:49Z", "updated": "2019-03-08T08:13:49Z", "id": "did:op:8d1b4d73e7af4634958f071ab8dfe7ab0df14019", "proof": { "created": "2019-02-08T08:13:41Z", "creator": "0x37BB53e3d293494DE59fBe1FF78500423dcFd43B", "signatureValue": "did:op:0bc278fee025464f8012b811d1bce8e22094d0984e4e49139df5d5ff7a028bdf", "type": "DDOIntegritySignature", "checksum": { "0": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "1": "0x999999952b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3" } }, "verifiableCredential": [ { "@context": [ "https://www.w3.org/2018/credentials/v1", "https://www.w3.org/2018/credentials/examples/v1" ], "id": "1872", "type": [ "read", "update", "deactivate" ], "issuer": "0x610D9314EDF2ced7681BA1633C33fdb8cF365a12", "issuanceDate": "2019-01-01T19:73:24Z", "credentialSubject": { "id": "0x89328493849328493284932" }, "proof": { "type": "RsaSignature2018", "created": "2019-01-01T19:73:24Z", "proofPurpose": "assertionMethod", "signatureValue": "ABCJSDAO23...1tzjn4w==" } } ], "service": [ { "index": 0, "serviceEndpoint": "http://localhost:5000/api/v1/aquarius/assets/ddo/{did}", "type": "metadata", "attributes": { "encryptedFiles": "0x2e48ceefcca7abb024f90c87c676fce8f7913f889605a349c08c0c4a822c69ad651e122cc81db4fbb52938ac627786491514f37a2ebfd04fd98ec726f1d9061ed52f13fde132222af34d9af8ec358429cf45fc669f81a607185cb9a8150df3cbb2b4e3e382fb16429be228ddd920f061b78dd54701025fac8aab976239fb31a5b60a57393e96a338324c5ac8a5600a1247339c4835533cecdb5b53caf6b6f9d6478b579b7426f650a4154a20d18a9d49f509770af62647a57fc174741b47af3c8beeaaa76bee276cce8fba1f3fec0e1c", "status": { "isOrderDisabled": true }, "main": { "author": "Met Office", "dateCreated": "2019-02-08T08:13:49Z", "files": [ { "url": "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt", "index": 0, "checksum": "efb2c764274b745f5fc37f97c6b0e761", "contentLength": "4535431", "contentType": "text/csv", "encoding": "UTF-8", "compression": "zip" } ], "license": "CC-BY", "name": "UK Weather information 2011", "type": "dataset" }, "additionalInformation": { "description": "Weather information of UK including temperature and humidity", "tags": [ "weather", "uk", "2011", "temperature", "humidity" ], "workExample": "423432fsd,51.509865,-0.118092,2011-01-01T10:55:11+00:00,7.2,68", "copyrightHolder": "Met Office", "links": [ { "name": "Sample of Asset Data", "type": "sample", "url": "https://foo.com/sample.csv" }, { "name": "Data Format Definition", "type": "format", "url": "https://foo.com/sampl2.csv" } ], "inLanguage": "en", "updateFrecuency": "yearly", "structuredMarkup": [ { "uri": "http://skos.um.es/unescothes/C01194/jsonld", "mediaType": "application/ld+json" }, { "uri": "http://skos.um.es/unescothes/C01194/turtle", "mediaType": "text/turtle" } ] }, "curation": { "numVotes": 123, "rating": 0.0, "schema": "Binary Votting", "isListed": true } } }, { "type": "access", "index": 1, "serviceEndpoint": "http://localhost:8030", "attributes": { "main": { "name": "dataAssetAccessServiceAgreement", "creator": "", "datePublished": "2019-02-08T08:13:49Z", "cost": "1.0", "timeout": 36000 }, "additionalInformation": { "description": "" } } } ] } ================================================ FILE: tests/resources/ddo/ddo_sa_sample_with_credentials.json ================================================ { "@context": "https://w3id.org/did/v1", "created": "2019-02-08T08:13:49Z", "updated": "2019-03-08T08:13:49Z", "id": "did:op:8d1b4d73e7af4634958f071ab8dfe7ab0df14019", "proof": { "created": "2019-02-08T08:13:41Z", "creator": "0x37BB53e3d293494DE59fBe1FF78500423dcFd43B", "signatureValue": "did:op:0bc278fee025464f8012b811d1bce8e22094d0984e4e49139df5d5ff7a028bdf", "type": "DDOIntegritySignature", "checksum": { "0": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "1": "0x999999952b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3" } }, "verifiableCredential": [ { "@context": [ "https://www.w3.org/2018/credentials/v1", "https://www.w3.org/2018/credentials/examples/v1" ], "id": "1872", "type": [ "read", "update", "deactivate" ], "issuer": "0x610D9314EDF2ced7681BA1633C33fdb8cF365a12", "issuanceDate": "2019-01-01T19:73:24Z", "credentialSubject": { "id": "0x89328493849328493284932" }, "proof": { "type": "RsaSignature2018", "created": "2019-01-01T19:73:24Z", "proofPurpose": "assertionMethod", "signatureValue": "ABCJSDAO23...1tzjn4w==" } } ], "service": [ { "index": 0, "serviceEndpoint": "http://localhost:5000/api/v1/aquarius/assets/ddo/{did}", "type": "metadata", "attributes": { "encryptedFiles": "0x2e48ceefcca7abb024f90c87c676fce8f7913f889605a349c08c0c4a822c69ad651e122cc81db4fbb52938ac627786491514f37a2ebfd04fd98ec726f1d9061ed52f13fde132222af34d9af8ec358429cf45fc669f81a607185cb9a8150df3cbb2b4e3e382fb16429be228ddd920f061b78dd54701025fac8aab976239fb31a5b60a57393e96a338324c5ac8a5600a1247339c4835533cecdb5b53caf6b6f9d6478b579b7426f650a4154a20d18a9d49f509770af62647a57fc174741b47af3c8beeaaa76bee276cce8fba1f3fec0e1c", "main": { "author": "Met Office", "dateCreated": "2019-02-08T08:13:49Z", "files": [ { "url": "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt", "index": 0, "checksum": "efb2c764274b745f5fc37f97c6b0e761", "contentLength": "4535431", "contentType": "text/csv", "encoding": "UTF-8", "compression": "zip" } ], "license": "CC-BY", "name": "UK Weather information 2011", "type": "dataset" }, "additionalInformation": { "description": "Weather information of UK including temperature and humidity", "tags": [ "weather", "uk", "2011", "temperature", "humidity" ], "workExample": "423432fsd,51.509865,-0.118092,2011-01-01T10:55:11+00:00,7.2,68", "copyrightHolder": "Met Office", "links": [ { "name": "Sample of Asset Data", "type": "sample", "url": "https://foo.com/sample.csv" }, { "name": "Data Format Definition", "type": "format", "url": "https://foo.com/sampl2.csv" } ], "inLanguage": "en", "updateFrecuency": "yearly", "structuredMarkup": [ { "uri": "http://skos.um.es/unescothes/C01194/jsonld", "mediaType": "application/ld+json" }, { "uri": "http://skos.um.es/unescothes/C01194/turtle", "mediaType": "text/turtle" } ] }, "curation": { "numVotes": 123, "rating": 0.0, "schema": "Binary Votting", "isListed": true } } }, { "type": "access", "index": 1, "serviceEndpoint": "http://localhost:8030", "attributes": { "main": { "name": "dataAssetAccessServiceAgreement", "creator": "", "datePublished": "2019-02-08T08:13:49Z", "cost": "1.0", "timeout": 36000 }, "additionalInformation": { "description": "" } } } ], "credentials": { "allow": [ {"type": "address", "values": ["0x123", "0x456A"]} ], "deny": [ {"type": "address", "values": ["0x2222", "0x333"]} ] } } ================================================ FILE: tests/resources/ddo/ddo_sample_algorithm.json ================================================ { "@context": "https://w3id.org/did/v1", "created": "2019-02-08T08:13:49Z", "id": "did:op:8d1b4d73e7af4634958f071ab8dfe7ab0df14019", "proof": { "created": "2019-02-08T08:13:41Z", "creator": "0x37BB53e3d293494DE59fBe1FF78500423dcFd43B", "signatureValue": "did:op:0bc278fee025464f8012b811d1bce8e22094d0984e4e49139df5d5ff7a028bdf", "type": "DDOIntegritySignature", "checksum": { "0": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "1": "0x999999952b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3" } }, "verifiableCredential": [ { "@context": [ "https://www.w3.org/2018/credentials/v1", "https://www.w3.org/2018/credentials/examples/v1" ], "id": "1872", "type": [ "read", "update", "deactivate" ], "issuer": "0x610D9314EDF2ced7681BA1633C33fdb8cF365a12", "issuanceDate": "2019-01-01T19:73:24Z", "credentialSubject": { "id": "0x89328493849328493284932" }, "proof": { "type": "RsaSignature2018", "created": "2019-01-01T19:73:24Z", "proofPurpose": "assertionMethod", "signatureValue": "ABCJSDAO23...1tzjn4w==" } } ], "service": [ { "index": 0, "serviceEndpoint": "http://localhost:5000/api/v1/aquarius/assets/ddo/{did}", "type": "metadata", "attributes": { "main": { "author": "John Doe", "dateCreated": "2019-02-08T08:13:49Z", "license": "CC-BY", "name": "My super algorithm", "type": "algorithm", "algorithm": { "language": "scala", "format": "docker-image", "version": "0.1", "container": { "entrypoint": "node $ALGO", "image": "node", "tag": "10" } }, "files": [ { "name": "build_model", "url": "https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js", "index": 0, "checksum": "efb2c764274b745f5fc37f97c6b0e761", "contentLength": "4535431", "contentType": "text/plain", "encoding": "UTF-8", "compression": "zip" } ] }, "additionalInformation": { "description": "Workflow to aggregate weather information", "tags": [ "weather", "uk", "2011", "workflow", "aggregation" ], "copyrightHolder": "John Doe" } } } ] } ================================================ FILE: tests/resources/ddo/ddo_v4_sample.json ================================================ { "@context": ["https://w3id.org/did/v1"], "id": "did:op:d32696f71f3318c92bcf325e2e51e6e8299c0eb6d362ddcfa77d2a3e0c1237b5", "version": "4.1.0", "chainId": 8996, "nftAddress": "0xCc708430E6a174BD4639A979F578A2176A0FA3fA", "metadata": { "created": "2020-11-15T12:27:48Z", "updated": "2021-05-17T21:58:02Z", "description": "Sample description", "name": "Sample asset", "type": "dataset", "author": "OPF", "license": "https://market.oceanprotocol.com/terms" }, "services": [ { "id": "1", "type": "access", "files": "0x0000", "name": "Download service", "description": "Download service", "datatokenAddress": "0x123", "serviceEndpoint": "http://172.15.0.4:8030", "timeout": 0 } ], "credentials": { "allow": [ { "type": "address", "values": ["0x123", "0x456"] } ], "deny": [ { "type": "address", "values": ["0x2222", "0x333"] } ] }, "nft": { "address": "0xCc708430E6a174BD4639A979F578A2176A0FA3fA", "name": "Ocean Protocol Asset v4", "symbol": "OCEAN-A-v4", "owner": "0x0000000", "state": 0, "created": "2000-10-31T01:30:00" }, "datatokens": [ { "address": "0x000000", "name": "Datatoken 1", "symbol": "DT-1", "serviceId": "1" } ], "event": { "tx": "0x8d127de58509be5dfac600792ad24cc9164921571d168bff2f123c7f1cb4b11c", "block": 12831214, "from": "0xAcca11dbeD4F863Bb3bC2336D3CE5BAC52aa1f83", "contract": "0x1a4b70d8c9DcA47cD6D0Fb3c52BB8634CA1C0Fdf", "datetime": "2000-10-31T01:30:00" }, "stats": { "consumes": 4 } } ================================================ FILE: tests/resources/ddo/ddo_v4_with_compute_service.json ================================================ { "@context": ["https://w3id.org/did/v1"], "id": "did:op:d32696f71f3318c92bcf325e2e51e6e8299c0eb6d362ddcfa77d2a3e0c1237b5", "version": "4.1.0", "chainId": 8996, "nftAddress": "0xCc708430E6a174BD4639A979F578A2176A0FA3fA", "metadata": { "created": "2020-11-15T12:27:48Z", "updated": "2021-05-17T21:58:02Z", "description": "Sample description", "name": "Sample asset", "type": "dataset", "author": "OPF", "license": "https://market.oceanprotocol.com/terms" }, "services": [ { "id": "1", "type": "access", "files": "0x0000", "name": "Download service", "description": "Download service", "datatokenAddress": "0x123", "serviceEndpoint": "http://172.15.0.4:8030", "timeout": 0 }, { "id": "2", "type": "compute", "files": "0x0001", "name": "Compute service", "description": "Compute service", "datatokenAddress": "0x124", "serviceEndpoint": "http://172.15.0.4:8030", "timeout": 3600, "compute": { "namespace": "ocean-compute", "cpus": 2, "gpus": 4, "gpuType": "NVIDIA Tesla V100 GPU", "memory": "128M", "volumeSize": "2G", "allowRawAlgorithm": false, "allowNetworkAccess": true, "publisherTrustedAlgorithmPublishers": ["0x234", "0x235"], "publisherTrustedAlgorithms": [ { "did": "did:op:123", "filesChecksum": "100", "containerSectionChecksum": "200" }, { "did": "did:op:124", "filesChecksum": "110", "containerSectionChecksum": "210" } ] } } ], "credentials": { "allow": [ { "type": "address", "values": ["0x123", "0x456"] } ], "deny": [ { "type": "address", "values": ["0x2222", "0x333"] } ] }, "nft": { "address": "0xCc708430E6a174BD4639A979F578A2176A0FA3fA", "name": "Ocean Protocol Asset v4", "symbol": "OCEAN-A-v4", "owner": "0x0000000", "state": 0, "created": "2000-10-31T01:30:00" }, "datatokens": [ { "address": "0x000000", "name": "Datatoken 1", "symbol": "DT-1", "serviceId": "1" }, { "address": "0x000001", "name": "Datatoken 2", "symbol": "DT-2", "serviceId": "2" } ], "event": { "tx": "0x8d127de58509be5dfac600792ad24cc9164921571d168bff2f123c7f1cb4b11c", "block": 12831214, "from": "0xAcca11dbeD4F863Bb3bC2336D3CE5BAC52aa1f83", "contract": "0x1a4b70d8c9DcA47cD6D0Fb3c52BB8634CA1C0Fdf", "datetime": "2000-10-31T01:30:00" }, "stats": { "consumes": 4 } } ================================================ FILE: tests/resources/ddo/ddo_v4_with_compute_service2.json ================================================ { "@context": ["https://w3id.org/did/v1"], "id": "did:op:d32696f71f3318c92bcf325e2e51e6e8299c0eb6d362ddcfa77d2a3e0c1237b5", "version": "4.1.0", "chainId": 8996, "nftAddress": "0xCc708430E6a174BD4639A979F578A2176A0FA3fA", "metadata": { "created": "2020-11-15T12:27:48Z", "updated": "2021-05-17T21:58:02Z", "description": "Sample description", "name": "Sample asset", "type": "dataset", "author": "OPF", "license": "https://market.oceanprotocol.com/terms" }, "services": [ { "id": "1", "type": "access", "files": "0x0000", "name": "Download service", "description": "Download service", "datatokenAddress": "0x123", "serviceEndpoint": "http://172.15.0.4:8030", "timeout": 0 }, { "id": "2", "type": "compute", "files": "0x0001", "name": "Compute service", "description": "Compute service", "datatokenAddress": "0x124", "serviceEndpoint": "http://172.15.0.4:8030", "timeout": 3600, "compute": { "namespace": "ocean-compute", "cpus": 2, "gpus": 4, "gpuType": "NVIDIA Tesla V100 GPU", "memory": "128M", "volumeSize": "2G", "allowRawAlgorithm": false, "allowNetworkAccess": true, "publisherTrustedAlgorithmPublishers": ["0xabc"], "publisherTrustedAlgorithms": [ { "did": "did:op:123", "filesChecksum":"5ce12db0cc7f13f963b1af3b5df7cab4fd3ffae16c8af7e6e416570d197dcc61", "containerSectionChecksum": "77ac49b8c4ad209694f7143cec62d67ee4c6d2d5edf1f3f4e09ca4173b22610f" } ] } } ], "credentials": { "allow": [ { "type": "address", "values": ["0x123", "0x456"] } ], "deny": [ { "type": "address", "values": ["0x2222", "0x333"] } ] }, "nft": { "address": "0xCc708430E6a174BD4639A979F578A2176A0FA3fA", "name": "Ocean Protocol Asset v4", "symbol": "OCEAN-A-v4", "owner": "0x0000000", "state": 0, "created": "2000-10-31T01:30:00" }, "datatokens": [ { "address": "0x000000", "name": "Datatoken 1", "symbol": "DT-1", "serviceId": "1" }, { "address": "0x000001", "name": "Datatoken 2", "symbol": "DT-2", "serviceId": "2" } ], "event": { "tx": "0x8d127de58509be5dfac600792ad24cc9164921571d168bff2f123c7f1cb4b11c", "block": 12831214, "from": "0xAcca11dbeD4F863Bb3bC2336D3CE5BAC52aa1f83", "contract": "0x1a4b70d8c9DcA47cD6D0Fb3c52BB8634CA1C0Fdf", "datetime": "2000-10-31T01:30:00" }, "stats": { "consumes": 4 } } ================================================ FILE: tests/resources/ddo/ddo_with_compute_service.json ================================================ { "@context": "https://w3id.org/future-method/v1", "created": "2019-04-09T19:02:11Z", "id": "did:op:8d1b4d73e7af4634958f071ab8dfe7ab0df14019", "proof": { "created": "2019-04-09T19:02:11Z", "creator": "0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e", "signatureValue": "1cd57300733bcbcda0beb59b3e076de6419c0d7674e7befb77820b53c79e3aa8f1776effc64cf088bad8cb694cc4d71ebd74a13b2f75893df5a53f3f318f6cf828", "type": "DDOIntegritySignature" }, "service": [ { "type": "metadata", "index": 0, "serviceEndpoint": "http://myaquarius.org/api/v1/provider/assets/metadata/{did}", "attributes": { "main": { "author": "Met Office", "dateCreated": "2019-02-08T08:13:49Z", "files": [ { "url": "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt", "index": 0, "checksum": "efb2c764274b745f5fc37f97c6b0e764", "contentLength": "4535431", "contentType": "text/csv", "encoding": "UTF-8", "compression": "zip" } ], "license": "CC-BY", "name": "UK Weather information 2011", "type": "dataset" }, "additionalInformation": {} } }, { "type": "compute", "index": 2, "serviceEndpoint": "http://myprovider.org", "templateId": "", "attributes": { "main": { "name": "dataAssetComputingServiceAgreement", "creator": "0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e", "datePublished": "2019-04-09T19:02:11Z", "cost": "1.0", "timeout": 86400, "privacy": {}, "provider": { "type": "Azure", "description": "", "environment": { "cluster": { "type": "Kubernetes", "url": "http://10.0.0.17/xxx" }, "supportedContainers": [ { "image": "tensorflow/tensorflow", "tag": "latest", "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" }, { "image": "tensorflow/tensorflow", "tag": "latest", "checksum": "sha256:cb57ecfa6ebbefd8ffc7f75c0f00e57a7fa739578a429b6f72a0df19315deadc" } ], "supportedServers": [ { "serverId": "1", "serverType": "xlsize", "price": "50", "cpu": "16", "gpu": "0", "memory": "128gb", "disk": "160gb", "maxExecutionTime": 86400 }, { "serverId": "2", "serverType": "medium", "price": "10", "cpu": "2", "gpu": "0", "memory": "8gb", "disk": "80gb", "maxExecutionTime": 86400 } ] } } }, "additionalInformation": {} } } ] } ================================================ FILE: tests/resources/ddo/valid_metadata.json ================================================ { "main": { "name": "10 Monkey Species Small", "dateCreated": "2012-02-01T10:55:11Z", "author": "Mario", "license": "CC0: Public Domain", "files": [ { "index": 0, "contentType": "application/zip", "checksum": "2bf9d229d110d1976cdf85e9f3256c7f", "checksumType": "MD5", "contentLength": "12057507", "url": "https://s3.amazonaws.com/datacommons-seeding-us-east/10_Monkey_Species_Small/assets/training.zip" }, { "index": 1, "contentType": "text/text", "checksum": "354d19c0733c47ef3a6cce5b633116b0", "checksumType": "MD5", "contentLength": "928", "url": "https://s3.amazonaws.com/datacommons-seeding-us-east/10_Monkey_Species_Small/assets/monkey_labels.txt" }, { "index": 2, "contentType": "application/zip", "url": "https://s3.amazonaws.com/datacommons-seeding-us-east/10_Monkey_Species_Small/assets/validation.zip" } ], "type": "dataset" } } ================================================ FILE: tests/resources/ddo_helpers.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json import os import pathlib from typing import List from ocean_lib.assets.ddo import DDO from ocean_lib.models.datatoken_base import DatatokenArguments from ocean_lib.ocean.ocean import Ocean from ocean_lib.services.service import Service from ocean_lib.structures.file_objects import UrlFile from tests.resources.helper_functions import get_file1, get_file2 def get_resource_path(dir_name, file_name): base = os.path.realpath(__file__).split(os.path.sep)[1:-1] if dir_name: return pathlib.Path(os.path.join(os.path.sep, *base, dir_name, file_name)) else: return pathlib.Path(os.path.join(os.path.sep, *base, file_name)) def get_key_from_v4_sample_ddo(key, file_name="ddo_v4_sample.json"): path = get_resource_path("ddo", file_name) with open(path, "r") as file_handle: ddo = file_handle.read() ddo_dict = json.loads(ddo) return ddo_dict.pop(key, None) def get_sample_ddo(file_name="ddo_v4_sample.json") -> dict: path = get_resource_path("ddo", file_name) with open(path, "r") as file_handle: ddo = file_handle.read() return json.loads(ddo) def get_sample_ddo_with_compute_service( filename="ddo_v4_with_compute_service.json", ) -> dict: path = get_resource_path("ddo", filename) with open(path, "r") as file_handle: ddo = file_handle.read() return json.loads(ddo) def get_sample_algorithm_ddo(filename="ddo_algorithm.json") -> DDO: path = get_resource_path("ddo", filename) assert path.exists(), f"{path} does not exist!" with open(path, "r") as file_handle: metadata = file_handle.read() alg_dict = json.loads(metadata) return DDO.from_dict(alg_dict) def get_default_metadata(): return get_key_from_v4_sample_ddo("metadata") def get_default_files(): return [get_file1(), get_file2()] def build_default_services(config, datatoken): files = get_default_files() services = [ datatoken.build_access_service( service_id="0", service_endpoint=config.get("PROVIDER_URL"), files=files, ) ] return services def build_credentials_dict() -> dict: """Build a credentials dict, used for testing.""" return {"allow": [], "deny": []} def get_registered_asset_with_access_service( ocean_instance, publisher_wallet, metadata=None, more_files=False ): url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff" files = [UrlFile(url)] if not more_files else [UrlFile(url), get_file2()] if not metadata: metadata = get_default_metadata() data_nft, dts, ddo = ocean_instance.assets.create( metadata, {"from": publisher_wallet}, datatoken_args=[DatatokenArguments("Branin: DT1", "DT1", files=files)], ) return data_nft, dts[0], ddo def get_registered_asset_with_compute_service( ocean_instance: Ocean, publisher_wallet, allow_raw_algorithms: bool = False, trusted_algorithms: List[DDO] = [], trusted_algorithm_publishers: List[str] = [], ): # Set the compute values for compute service compute_values = { "allowRawAlgorithm": allow_raw_algorithms, "allowNetworkAccess": True, "publisherTrustedAlgorithms": trusted_algorithms, "publisherTrustedAlgorithmPublishers": trusted_algorithm_publishers, } return ocean_instance.assets.create_url_asset( "Branin", "https://raw.githubusercontent.com/oceanprotocol/c2d-examples/main/branin_and_gpr/branin.arff", tx_dict={"from": publisher_wallet}, compute_values=compute_values, wait_for_aqua=False, ) def get_first_service_by_type(ddo, service_type: str) -> Service: """Return the first Service with the given service type.""" return next((service for service in ddo.services if service.type == service_type)) ================================================ FILE: tests/resources/helper_functions.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import json import logging import logging.config import os import secrets from datetime import datetime, timezone from decimal import Decimal from typing import Any, Dict, Optional, Tuple, Union import coloredlogs from enforce_typing import enforce_types from eth_account import Account from web3 import Web3 from ocean_lib.example_config import get_config_dict from ocean_lib.models.data_nft import DataNFT from ocean_lib.models.data_nft_factory import DataNFTFactoryContract from ocean_lib.models.datatoken_base import DatatokenBase from ocean_lib.ocean.ocean import Ocean from ocean_lib.ocean.util import get_address_of_type, send_ether, to_wei from ocean_lib.structures.file_objects import FilesTypeFactory from ocean_lib.web3_internal.constants import ZERO_ADDRESS from ocean_lib.web3_internal.utils import sign_with_key, split_signature from tests.resources.mocks.data_provider_mock import DataProviderMock _NETWORK = "ganache" @enforce_types def get_wallet(index: int): return Account.from_key(private_key=os.getenv(f"TEST_PRIVATE_KEY{index}")) @enforce_types def get_publisher_wallet(): return get_wallet(1) @enforce_types def get_consumer_wallet(): return get_wallet(2) @enforce_types def get_another_consumer_wallet(): return get_wallet(3) @enforce_types def get_provider_wallet(): return Account.from_key(private_key=os.getenv("PROVIDER_PRIVATE_KEY")) def get_factory_deployer_wallet(config): if config["NETWORK_NAME"] == "development": return get_ganache_wallet() private_key = os.environ.get("FACTORY_DEPLOYER_PRIVATE_KEY") if not private_key: return None config = get_config_dict() return Account.from_key(private_key=private_key) def get_ganache_wallet(): return Account.from_key( private_key="0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58" ) @enforce_types def generate_wallet(): """Generates wallets on the fly with funds.""" config = get_config_dict() secret = secrets.token_hex(32) private_key = "0x" + secret new_wallet = Account.from_key(private_key=private_key) deployer_wallet = get_factory_deployer_wallet(config) send_ether( config, deployer_wallet, new_wallet.address, to_wei(3), ) ocean = Ocean(config) OCEAN = ocean.OCEAN_token OCEAN.transfer(new_wallet, to_wei(50), {"from": deployer_wallet}) return new_wallet def get_ocean_instance_prerequisites(use_provider_mock=False) -> Ocean: config_dict = get_config_dict() data_provider = DataProviderMock if use_provider_mock else None return Ocean(config_dict, data_provider=data_provider) @enforce_types def get_publisher_ocean_instance(use_provider_mock=False) -> Ocean: ocn = get_ocean_instance_prerequisites(use_provider_mock) ocn.main_account = get_publisher_wallet() return ocn @enforce_types def get_consumer_ocean_instance(use_provider_mock: bool = False) -> Ocean: ocn = get_ocean_instance_prerequisites(use_provider_mock) ocn.main_account = get_consumer_wallet() return ocn @enforce_types def get_another_consumer_ocean_instance(use_provider_mock: bool = False) -> Ocean: ocn = get_ocean_instance_prerequisites(use_provider_mock) ocn.main_account = get_another_consumer_wallet() return ocn @enforce_types def setup_logging( default_level=logging.INFO, ): """Logging setup.""" logging.basicConfig(level=default_level) coloredlogs.install(level=default_level) @enforce_types def deploy_erc721_erc20( config_dict: dict, data_nft_publisher, datatoken_minter: Optional = None, template_index: Optional[int] = 1, ) -> Union[DataNFT, Tuple[DataNFT, DatatokenBase]]: """Helper function to deploy an DataNFT using data_nft_publisher Wallet and an Datatoken data token with the newly DataNFT using datatoken_minter Wallet if the wallet is provided. :rtype: Union[DataNFT, Tuple[DataNFT, DatatokenBase]] """ data_nft_factory = DataNFTFactoryContract( config_dict, get_address_of_type(config_dict, "ERC721Factory") ) data_nft = data_nft_factory.create({"from": data_nft_publisher}, "NFT", "NFTSYMBOL") if not datatoken_minter: return data_nft datatoken_cap = to_wei(100) if template_index == 2 else None datatoken = data_nft.create_datatoken( {"from": data_nft_publisher}, template_index=template_index, cap=datatoken_cap, name="DT1", symbol="DT1Symbol", minter=datatoken_minter.address, ) return data_nft, datatoken @enforce_types def get_non_existent_nft_template( data_nft_factory: DataNFTFactoryContract, check_first=20 ) -> int: """Helper function to find a non existent ERC721 template among the first *check_first* templates of an Data NFT Factory contract. Returns -1 if template was found. """ for template_nbr in range(check_first): [address, _] = data_nft_factory.getNFTTemplate(template_nbr) if address == ZERO_ADDRESS: return template_nbr return -1 @enforce_types def send_mock_usdc_to_address(config: dict, recipient: str, amount: int) -> int: """Helper function to send mock usdc to an arbitrary recipient address if factory_deployer has enough balance to send. Returns the transferred balance. """ factory_deployer = get_factory_deployer_wallet(config) mock_usdc = DatatokenBase.get_typed(config, get_address_of_type(config, "MockUSDC")) initial_recipient_balance = mock_usdc.balanceOf(recipient) if mock_usdc.balanceOf(factory_deployer) >= amount: mock_usdc.transfer(recipient, amount, factory_deployer) return mock_usdc.balanceOf(recipient) - initial_recipient_balance @enforce_types def transfer_bt_if_balance_lte( config: dict, bt_address: str, from_wallet, recipient: str, min_balance: int, amount_to_transfer: int, ) -> int: """Helper function to send an arbitrary amount of ocean to recipient address if recipient's ocean balance is less or equal to min_balance and from_wallet has enough ocean balance to send. Returns the transferred ocean amount. """ base_token = DatatokenBase.get_typed(config, bt_address) initial_recipient_balance = base_token.balanceOf(recipient) if ( initial_recipient_balance <= min_balance and base_token.balanceOf(from_wallet) >= amount_to_transfer ): base_token.transfer(recipient, amount_to_transfer, {"from": from_wallet}) return base_token.balanceOf(recipient) - initial_recipient_balance @enforce_types def get_provider_fees( provider_wallet, provider_fee_token: str, provider_fee_amount: int, valid_until: int, compute_env: str = None, timestamp: int = None, ) -> Dict[str, Any]: """Copied and adapted from https://github.com/oceanprotocol/provider/blob/b9eb303c3470817d11b3bba01a49f220953ed963/ocean_provider/utils/provider_fees.py#L22-L74 Keep this in sync with the corresponding provider fee logic when it changes! """ provider_fee_address = provider_wallet.address provider_data = json.dumps( { "environment": compute_env, "timestamp": datetime.now(timezone.utc).timestamp(), }, separators=(",", ":"), ) message_hash = Web3.solidity_keccak( ["bytes", "address", "address", "uint256", "uint256"], [ Web3.to_hex(Web3.to_bytes(text=provider_data)), provider_fee_address, provider_fee_token, provider_fee_amount, valid_until, ], ) signed = sign_with_key(message_hash, os.getenv("PROVIDER_PRIVATE_KEY")) provider_fee = { "providerFeeAddress": provider_fee_address, "providerFeeToken": provider_fee_token, "providerFeeAmount": str(provider_fee_amount), "providerData": Web3.to_hex(Web3.to_bytes(text=provider_data)), # make it compatible with last openzepellin https://github.com/OpenZeppelin/openzeppelin-contracts/pull/1622 "v": (signed.v + 27) if signed.v <= 1 else signed.v, "r": Web3.to_hex(Web3.to_bytes(signed.r).rjust(32, b"\0")), "s": Web3.to_hex(Web3.to_bytes(signed.s).rjust(32, b"\0")), "validUntil": valid_until, } return provider_fee def convert_bt_amt_to_dt( bt_amount: int, bt_decimals: int, dt_per_bt_in_wei: int, ) -> int: """Convert base tokens to equivalent datatokens, accounting for differences in decimals and exchange rate. dt_per_bt_in_wei = 1 / bt_per_dt = 1 / price Datatokens always have 18 decimals, even if base tokens don't. """ bt_amount_wei = bt_amount bt_amount_float = float(bt_amount_wei) / 10**bt_decimals dt_per_bt_float = float(dt_per_bt_in_wei) / 10**18 # price always has 18 dec dt_amount_float = bt_amount_float * dt_per_bt_float dt_amount_wei = int(dt_amount_float * 10**18) return dt_amount_wei def get_file1(): file1_dict = { "type": "url", "url": "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt", "method": "GET", } return FilesTypeFactory(file1_dict) def get_file2(): file2_dict = { "type": "url", "url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-abstract10.xml.gz-rss.xml", "method": "GET", } return FilesTypeFactory(file2_dict) def get_file3(): file3_dict = { "type": "url", "url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-abstract10.xml.gz", "method": "GET", } return FilesTypeFactory(file3_dict) def int_units(amount, num_decimals): decimal_amount = Decimal(amount) unit_value = Decimal(10) ** num_decimals return int(decimal_amount * unit_value) @enforce_types def get_mock_provider_fees(mock_type, wallet, valid_until=0): config = get_config_dict() provider_fee_address = wallet.address provider_fee_token = get_address_of_type(config, mock_type) provider_fee_amount = 0 provider_data = json.dumps({"timeout": 0}, separators=(",", ":")) message = Web3.solidity_keccak( ["bytes", "address", "address", "uint256", "uint256"], [ Web3.to_hex(Web3.to_bytes(text=provider_data)), wallet.address, provider_fee_token, provider_fee_amount, valid_until, ], ) signed = config["web3_instance"].eth.sign(wallet.address, data=message) signature = split_signature(signed) return { "providerFeeAddress": provider_fee_address, "providerFeeToken": provider_fee_token, "providerFeeAmount": provider_fee_amount, "v": signature.v, "r": signature.r, "s": signature.s, "validUntil": valid_until, "providerData": Web3.to_hex(Web3.to_bytes(text=provider_data)), } ================================================ FILE: tests/resources/keys/key_file_1.json ================================================ { "id": "50aa801a-8d66-1402-1fa4-d8987868c2ce", "version": 3, "crypto": { "cipher": "aes-128-ctr", "cipherparams": { "iv": "a874e6fe50a5bb088826c45560dc1b7e" }, "ciphertext": "2383c6aa50c744b6558e77b5dcec6137f647c81f10f71f22a87321fd1306056c", "kdf": "pbkdf2", "kdfparams": { "c": 10240, "dklen": 32, "prf": "hmac-sha256", "salt": "eca6ccc9fbb0bdc3a516c7576808ba5031669e6878f3bb95624ddb46449e119c" }, "mac": "14e9a33a45ae32f88a0bd5aac14521c1fcf14f56fd55c1a1c080b2f81ddb8d44" }, "address": "068ed00cf0441e4829d9784fcbe7b9e26d4bd8d0", "name": "", "meta": "{}" } ================================================ FILE: tests/resources/keys/key_file_2.json ================================================ { "id": "0902d04b-f26e-5c1f-e3ae-78d2c1cb16e7", "version": 3, "crypto": { "cipher": "aes-128-ctr", "cipherparams": { "iv": "6a829fe7bc656d85f6c2e9fd21784952" }, "ciphertext": "1bfec0b054a648af8fdd0e85662206c65a4af0ed15fede4ad41ca9ab7b504ce2", "kdf": "pbkdf2", "kdfparams": { "c": 10240, "dklen": 32, "prf": "hmac-sha256", "salt": "95f96b5ee22dd537e06076eb8d7078eb7275d29af935782fe476696b11be50e5" }, "mac": "4af2215c3cd9447a5b0512d7d1c3ea5a4435981e1c8f48bf71d7a49c0e5b4986" }, "address": "00bd138abd70e2f00903268f3db08f2d25677c9e", "name": "Validator0", "meta": "{}" } ================================================ FILE: tests/resources/mocks/__init__.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # ================================================ FILE: tests/resources/mocks/data_provider_mock.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import os from ocean_lib.data_provider.data_service_provider import DataServiceProvider class DataProviderMock(DataServiceProvider): def __init__(self, ocean_instance=None, wallet=None): """Initialises DataProviderMock object.""" if not ocean_instance: from tests.resources.helper_functions import get_publisher_ocean_instance ocean_instance = get_publisher_ocean_instance(use_provider_mock=True) self.ocean_instance = ocean_instance self.wallet = wallet if not wallet: from tests.resources.helper_functions import get_publisher_wallet self.wallet = get_publisher_wallet() @staticmethod def consume_service( did, service_endpoint, wallet_address, files, destination_folder, *_, **__ ): for f in files: with open( os.path.join(destination_folder, os.path.basename(f["url"])), "w" ) as of: of.write(f"mock data {did}.{service_endpoint}.{wallet_address}") @staticmethod def start_compute_job(*args, **kwargs): return True @staticmethod def stop_compute_job(*args, **kwargs): return True @staticmethod def delete_compute_job(*args, **kwargs): return True @staticmethod def compute_job_status(*args, **kwargs): return True @staticmethod def compute_job_result(*args, **kwargs): return True @staticmethod def compute_job_result_file(*args, **kwargs): return True @staticmethod def get_url(config): return DataServiceProvider.get_url(config) ================================================ FILE: tests/resources/mocks/http_client_mock.py ================================================ # # Copyright 2023 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # import inspect import json from unittest.mock import Mock from requests.models import Response from requests.sessions import Session TEST_SERVICE_ENDPOINTS = { "computeDelete": ["DELETE", "/api/services/compute"], "computeStart": ["POST", "/api/services/compute"], "computeStatus": ["GET", "/api/services/compute"], "computeStop": ["PUT", "/api/services/compute"], "computeResult": ["GET", "/api/services/computeResult"], "download": ["GET", "/api/services/download"], "encrypt": ["POST", "/api/services/encrypt"], "decrypt": ["POST", "/api/services/decrypt"], "fileinfo": ["POST", "/api/services/fileinfo"], "initialize": ["GET", "/api/services/initialize"], "initializeCompute": ["POST", "/api/services/initializeCompute"], "nonce": ["GET", "/api/services/nonce"], "computeEnvironments": ["GET", "/api/services/computeEnvironments"], "create_auth_token": ["GET", "/api/services/createAuthToken"], "delete_auth_token": ["DELETE", "/api/services/deleteAuthToken"], "validateContainer": ["POST", "/api/services/validateContainer"], } class HttpClientMockBase(Session): """Parent class for all HTTPClient mocks.""" @classmethod def get(cls, *args, **kwargs): """Handles the base case of service endpoints.""" is_get_endpoints_request = False for _, _, _, fn, _, _ in inspect.getouterframes(inspect.currentframe()): if fn == "get_service_endpoints": is_get_endpoints_request = True if is_get_endpoints_request: the_response = Mock(spec=Response) the_response.status_code = 200 the_response.json.return_value = { "serviceEndpoints": TEST_SERVICE_ENDPOINTS } return the_response return cls.specific_get(*args, **kwargs) class HttpClientEvilMock(HttpClientMockBase): """Mock that generally returns 400 results and errors.""" @staticmethod def post(*args, **kwargs): the_response = Mock(spec=Response) the_response.status_code = 400 the_response.text = "Bad request (mocked)." return the_response @staticmethod def specific_get(*args, **kwargs): the_response = Mock(spec=Response) the_response.status_code = 400 the_response.text = "Bad request (mocked)." return the_response class HttpClientEmptyMock(HttpClientMockBase): """Mock unresponsiveness.""" @staticmethod def post(*args, **kwargs): return None @staticmethod def specific_get(*args, **kwargs): return None class HttpClientNiceMock(HttpClientMockBase): """Mock that returns 200 results and successful responses.""" @staticmethod def specific_get(*args, **kwargs): the_response = Mock(spec=Response) the_response.status_code = 200 the_response.content = '{"good_job": "with_mock"}'.encode("utf-8") return the_response @staticmethod def return_nice_response(indication, *args, **kwargs): the_response = Mock(spec=Response) the_response.status_code = 200 json_result = {"good_job": ("with_mock_" + indication)} the_response.content = json.dumps(json_result).encode("utf-8") return the_response @staticmethod def delete(*args, **kwargs): return HttpClientNiceMock.return_nice_response("delete", *args, **kwargs) @staticmethod def put(*args, **kwargs): return HttpClientNiceMock.return_nice_response("put", *args, **kwargs) @staticmethod def post(*args, **kwargs): return HttpClientNiceMock.return_nice_response("post", *args, **kwargs) ================================================ FILE: tests/resources/test/test_helper_functions.py ================================================ from tests.resources.helper_functions import convert_bt_amt_to_dt def test_convert_bt_amt_to_dt(): bt_decimals = 16 bt_amt_float = 10.0 price_float = 2.0 # price expected_dt_amt_float = bt_amt_float / price_float bt_amt_wei = int(bt_amt_float * 10**bt_decimals) bt_per_dt_float = 1.0 / price_float dt_per_bt_wei = int(bt_per_dt_float * 10**18) dt_amt_wei = convert_bt_amt_to_dt(bt_amt_wei, bt_decimals, dt_per_bt_wei) dt_amt_float = float(dt_amt_wei / 10**18) assert dt_amt_float == expected_dt_amt_float