Repository: lbryio/lbry-sdk Branch: master Commit: e7666f489418 Files: 318 Total size: 3.2 MB Directory structure: gitextract_scmdz25q/ ├── .github/ │ └── workflows/ │ ├── main.yml │ └── release.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── INSTALL.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── SECURITY.md ├── docker/ │ ├── Dockerfile.dht_node │ ├── Dockerfile.wallet_server │ ├── Dockerfile.web │ ├── README.md │ ├── docker-compose-wallet-server.yml │ ├── docker-compose.yml │ ├── hooks/ │ │ └── build │ ├── install_choco.ps1 │ ├── set_build.py │ ├── wallet_server_entrypoint.sh │ └── webconf.yaml ├── docs/ │ └── api.json ├── example_daemon_settings.yml ├── lbry/ │ ├── .dockerignore │ ├── __init__.py │ ├── blob/ │ │ ├── __init__.py │ │ ├── blob_file.py │ │ ├── blob_info.py │ │ ├── blob_manager.py │ │ ├── disk_space_manager.py │ │ └── writer.py │ ├── blob_exchange/ │ │ ├── __init__.py │ │ ├── client.py │ │ ├── downloader.py │ │ ├── serialization.py │ │ └── server.py │ ├── build_info.py │ ├── conf.py │ ├── connection_manager.py │ ├── constants.py │ ├── crypto/ │ │ ├── __init__.py │ │ ├── base58.py │ │ ├── crypt.py │ │ ├── hash.py │ │ └── util.py │ ├── dht/ │ │ ├── __init__.py │ │ ├── blob_announcer.py │ │ ├── constants.py │ │ ├── error.py │ │ ├── node.py │ │ ├── peer.py │ │ ├── protocol/ │ │ │ ├── __init__.py │ │ │ ├── data_store.py │ │ │ ├── distance.py │ │ │ ├── iterative_find.py │ │ │ ├── protocol.py │ │ │ └── routing_table.py │ │ └── serialization/ │ │ ├── __init__.py │ │ ├── bencoding.py │ │ └── datagram.py │ ├── error/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── __init__.py │ │ ├── base.py │ │ └── generate.py │ ├── extras/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── daemon/ │ │ │ ├── __init__.py │ │ │ ├── analytics.py │ │ │ ├── client.py │ │ │ ├── component.py │ │ │ ├── componentmanager.py │ │ │ ├── components.py │ │ │ ├── daemon.py │ │ │ ├── exchange_rate_manager.py │ │ │ ├── json_response_encoder.py │ │ │ ├── migrator/ │ │ │ │ ├── __init__.py │ │ │ │ ├── dbmigrator.py │ │ │ │ ├── migrate10to11.py │ │ │ │ ├── migrate11to12.py │ │ │ │ ├── migrate12to13.py │ │ │ │ ├── migrate13to14.py │ │ │ │ ├── migrate14to15.py │ │ │ │ ├── migrate15to16.py │ │ │ │ ├── migrate1to2.py │ │ │ │ ├── migrate2to3.py │ │ │ │ ├── migrate3to4.py │ │ │ │ ├── migrate4to5.py │ │ │ │ ├── migrate5to6.py │ │ │ │ ├── migrate6to7.py │ │ │ │ ├── migrate7to8.py │ │ │ │ ├── migrate8to9.py │ │ │ │ └── migrate9to10.py │ │ │ ├── security.py │ │ │ ├── storage.py │ │ │ └── undecorated.py │ │ └── system_info.py │ ├── file/ │ │ ├── __init__.py │ │ ├── file_manager.py │ │ ├── source.py │ │ └── source_manager.py │ ├── file_analysis.py │ ├── prometheus.py │ ├── schema/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── __init__.py │ │ ├── attrs.py │ │ ├── base.py │ │ ├── claim.py │ │ ├── compat.py │ │ ├── mime_types.py │ │ ├── purchase.py │ │ ├── result.py │ │ ├── support.py │ │ ├── tags.py │ │ ├── types/ │ │ │ ├── __init__.py │ │ │ ├── v1/ │ │ │ │ ├── __init__.py │ │ │ │ ├── certificate_pb2.py │ │ │ │ ├── fee_pb2.py │ │ │ │ ├── legacy_claim_pb2.py │ │ │ │ ├── metadata_pb2.py │ │ │ │ ├── signature_pb2.py │ │ │ │ ├── source_pb2.py │ │ │ │ └── stream_pb2.py │ │ │ └── v2/ │ │ │ ├── __init__.py │ │ │ ├── claim_pb2.py │ │ │ ├── purchase_pb2.py │ │ │ ├── result_pb2.py │ │ │ ├── support_pb2.py │ │ │ └── wallet.json │ │ └── url.py │ ├── stream/ │ │ ├── __init__.py │ │ ├── background_downloader.py │ │ ├── descriptor.py │ │ ├── downloader.py │ │ ├── managed_stream.py │ │ ├── reflector/ │ │ │ ├── __init__.py │ │ │ ├── client.py │ │ │ └── server.py │ │ └── stream_manager.py │ ├── testcase.py │ ├── torrent/ │ │ ├── __init__.py │ │ ├── session.py │ │ ├── torrent.py │ │ ├── torrent_manager.py │ │ └── tracker.py │ ├── utils.py │ ├── wallet/ │ │ ├── __init__.py │ │ ├── account.py │ │ ├── bcd_data_stream.py │ │ ├── bip32.py │ │ ├── checkpoints.py │ │ ├── claim_proofs.py │ │ ├── coinselection.py │ │ ├── constants.py │ │ ├── database.py │ │ ├── dewies.py │ │ ├── hash.py │ │ ├── header.py │ │ ├── ledger.py │ │ ├── manager.py │ │ ├── mnemonic.py │ │ ├── network.py │ │ ├── orchstr8/ │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ ├── node.py │ │ │ └── service.py │ │ ├── rpc/ │ │ │ ├── __init__.py │ │ │ ├── framing.py │ │ │ ├── jsonrpc.py │ │ │ ├── session.py │ │ │ ├── socks.py │ │ │ └── util.py │ │ ├── script.py │ │ ├── stream.py │ │ ├── tasks.py │ │ ├── transaction.py │ │ ├── udp.py │ │ ├── usage_payment.py │ │ ├── util.py │ │ ├── wallet.py │ │ └── words/ │ │ ├── __init__.py │ │ ├── chinese_simplified.py │ │ ├── english.py │ │ ├── japanese.py │ │ ├── portuguese.py │ │ └── spanish.py │ └── winpaths.py ├── scripts/ │ ├── Dockerfile.lbry_orchstr8 │ ├── check_signature.py │ ├── check_video.py │ ├── checkpoints.py │ ├── checktrie.py │ ├── deploy_dev_wallet_server.sh │ ├── dht_crawler.py │ ├── dht_monitor.py │ ├── dht_node.py │ ├── download_blob_from_peer.py │ ├── find_max_server_load.py │ ├── generate_json_api.py │ ├── hook-coincurve.py │ ├── hook-libtorrent.py │ ├── idea/ │ │ ├── lbry-sdk.iml │ │ ├── modules.xml │ │ └── vcs.xml │ ├── initialize_hub_from_snapshot.sh │ ├── monitor_slow_queries.py │ ├── publish_performance.py │ ├── release.py │ ├── repair_0_31_1_db.py │ ├── sd_hash_sampler.py │ ├── standalone_blob_server.py │ ├── test_claim_search.py │ ├── time_to_first_byte.py │ ├── troubleshoot_p2p_and_dht_webservice.py │ └── wallet_server_monitor.py ├── setup.cfg ├── setup.py ├── tests/ │ ├── __init__.py │ ├── dht_mocks.py │ ├── integration/ │ │ ├── __init__.py │ │ ├── blockchain/ │ │ │ ├── __init__.py │ │ │ ├── test_account_commands.py │ │ │ ├── test_blockchain_reorganization.py │ │ │ ├── test_network.py │ │ │ ├── test_purchase_command.py │ │ │ ├── test_sync.py │ │ │ ├── test_wallet_commands.py │ │ │ └── test_wallet_server_sessions.py │ │ ├── claims/ │ │ │ ├── __init__.py │ │ │ └── test_claim_commands.py │ │ ├── datanetwork/ │ │ │ ├── __init__.py │ │ │ ├── test_dht.py │ │ │ ├── test_file_commands.py │ │ │ └── test_streaming.py │ │ ├── other/ │ │ │ ├── __init__.py │ │ │ ├── test_chris45.py │ │ │ ├── test_cli.py │ │ │ ├── test_exchange_rate_manager.py │ │ │ ├── test_other_commands.py │ │ │ └── test_transcoding.py │ │ ├── takeovers/ │ │ │ ├── __init__.py │ │ │ └── test_resolve_command.py │ │ └── transactions/ │ │ ├── __init__.py │ │ ├── test_internal_transaction_api.py │ │ ├── test_transaction_commands.py │ │ └── test_transactions.py │ ├── test_utils.py │ └── unit/ │ ├── __init__.py │ ├── analytics/ │ │ ├── __init__.py │ │ └── test_track.py │ ├── blob/ │ │ ├── __init__.py │ │ ├── test_blob_file.py │ │ └── test_blob_manager.py │ ├── blob_exchange/ │ │ ├── __init__.py │ │ └── test_transfer_blob.py │ ├── components/ │ │ ├── __init__.py │ │ └── test_component_manager.py │ ├── core/ │ │ ├── __init__.py │ │ └── test_utils.py │ ├── database/ │ │ ├── __init__.py │ │ └── test_SQLiteStorage.py │ ├── dht/ │ │ ├── __init__.py │ │ ├── protocol/ │ │ │ ├── __init__.py │ │ │ ├── test_data_store.py │ │ │ ├── test_distance.py │ │ │ ├── test_kbucket.py │ │ │ ├── test_protocol.py │ │ │ └── test_routing_table.py │ │ ├── serialization/ │ │ │ ├── __init__.py │ │ │ ├── test_bencoding.py │ │ │ └── test_datagram.py │ │ ├── test_blob_announcer.py │ │ ├── test_node.py │ │ └── test_peer.py │ ├── lbrynet_daemon/ │ │ ├── __init__.py │ │ ├── test_Daemon.py │ │ ├── test_allowed_origin.py │ │ ├── test_exchange_rate_manager.py │ │ └── test_mime_types.py │ ├── schema/ │ │ ├── __init__.py │ │ ├── test_claim_from_bytes.py │ │ ├── test_mime_types.py │ │ ├── test_models.py │ │ ├── test_tags.py │ │ └── test_url.py │ ├── stream/ │ │ ├── __init__.py │ │ ├── test_managed_stream.py │ │ ├── test_reflector.py │ │ ├── test_stream_descriptor.py │ │ └── test_stream_manager.py │ ├── test_cli.py │ ├── test_conf.py │ ├── test_utils.py │ ├── torrent/ │ │ ├── __init__.py │ │ └── test_tracker.py │ └── wallet/ │ ├── __init__.py │ ├── key_fixtures.py │ ├── server/ │ │ ├── __init__.py │ │ └── test_migration.py │ ├── test_account.py │ ├── test_bcd_data_stream.py │ ├── test_bip32.py │ ├── test_claim_proofs.py │ ├── test_coinselection.py │ ├── test_database.py │ ├── test_dewies.py │ ├── test_hash.py │ ├── test_headers.py │ ├── test_ledger.py │ ├── test_mnemonic.py │ ├── test_schema_signing.py │ ├── test_script.py │ ├── test_stream_controller.py │ ├── test_transaction.py │ ├── test_utils.py │ └── test_wallet.py └── tox.ini ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/main.yml ================================================ name: ci on: ["push", "pull_request", "workflow_dispatch"] jobs: lint: name: lint runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.9' - name: extract pip cache uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} restore-keys: ${{ runner.os }}-pip- - run: pip install --user --upgrade pip wheel - run: pip install -e .[lint] - run: make lint tests-unit: name: "tests / unit" strategy: matrix: os: - ubuntu-20.04 - macos-13 - windows-2022 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.9' - name: set pip cache dir shell: bash run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV - name: extract pip cache uses: actions/cache@v4 with: path: ${{ env.PIP_CACHE_DIR }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} restore-keys: ${{ runner.os }}-pip- - id: os-name uses: ASzc/change-string-case-action@v6 with: string: ${{ runner.os }} - run: python -m pip install --user --upgrade pip wheel - if: startsWith(runner.os, 'linux') run: pip install -e .[test] - if: startsWith(runner.os, 'linux') env: HOME: /tmp run: make test-unit-coverage - if: startsWith(runner.os, 'linux') != true run: pip install -e .[test] - if: startsWith(runner.os, 'linux') != true env: HOME: /tmp run: coverage run --source=lbry -m unittest tests/unit/test_conf.py - name: submit coverage report env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: tests-unit-${{ steps.os-name.outputs.lowercase }} COVERALLS_PARALLEL: true run: | pip install coveralls coveralls --service=github tests-integration: name: "tests / integration" runs-on: ubuntu-20.04 strategy: matrix: test: - datanetwork - blockchain - claims - takeovers - transactions - other steps: - name: Configure sysctl limits run: | sudo swapoff -a sudo sysctl -w vm.swappiness=1 sudo sysctl -w fs.file-max=262144 sudo sysctl -w vm.max_map_count=262144 - name: Runs Elasticsearch uses: elastic/elastic-github-actions/elasticsearch@master with: stack-version: 7.12.1 - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.9' - if: matrix.test == 'other' run: | sudo apt-get update sudo apt-get install -y --no-install-recommends ffmpeg - name: extract pip cache uses: actions/cache@v4 with: path: ./.tox key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }} restore-keys: txo-integration-${{ matrix.test }}- - run: pip install tox coverage coveralls - if: matrix.test == 'claims' run: rm -rf .tox - run: tox -e ${{ matrix.test }} - name: submit coverage report env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: tests-integration-${{ matrix.test }} COVERALLS_PARALLEL: true run: | coverage combine tests coveralls --service=github coverage: needs: ["tests-unit", "tests-integration"] runs-on: ubuntu-20.04 steps: - name: finalize coverage report submission env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | pip install coveralls coveralls --service=github --finish build: needs: ["lint", "tests-unit", "tests-integration"] name: "build / binary" strategy: matrix: os: - ubuntu-20.04 - macos-13 - windows-2022 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.9' - id: os-name uses: ASzc/change-string-case-action@v6 with: string: ${{ runner.os }} - name: set pip cache dir shell: bash run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV - name: extract pip cache uses: actions/cache@v4 with: path: ${{ env.PIP_CACHE_DIR }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} restore-keys: ${{ runner.os }}-pip- - run: pip install pyinstaller==6.0 - run: pip install -e . - if: startsWith(github.ref, 'refs/tags/v') run: python docker/set_build.py - if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac') name: Build & Run (Unix) run: | pyinstaller --onefile --name lbrynet lbry/extras/cli.py dist/lbrynet --version - if: startsWith(runner.os, 'windows') name: Build & Run (Windows) run: | pip install pywin32==301 pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py dist/lbrynet.exe --version - uses: actions/upload-artifact@v4 with: name: lbrynet-${{ steps.os-name.outputs.lowercase }} path: dist/ release: name: "release" if: startsWith(github.ref, 'refs/tags/v') needs: ["build"] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 - name: upload binaries env: GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }} run: | pip install githubrelease chmod +x lbrynet-macos/lbrynet chmod +x lbrynet-linux/lbrynet zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe ls -lh githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/} githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \ lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/} ================================================ FILE: .github/workflows/release.yml ================================================ name: slack on: release: types: [published] jobs: release: name: "slack notification" runs-on: ubuntu-20.04 steps: - uses: LoveToKnow/slackify-markdown-action@v1.0.0 id: markdown with: text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}" - uses: slackapi/slack-github-action@v1.14.0 env: CHANGELOG: ' ${{ steps.markdown.outputs.text }}' SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }} with: payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }} }' ================================================ FILE: CHANGELOG.md ================================================ # Change Log This changelog is no longer up to date. For release notes, see https://github.com/lbryio/lbry-sdk/releases. ================================================ FILE: CONTRIBUTING.md ================================================ ## Contributing to LBRY https://lbry.tech/contribute ================================================ FILE: INSTALL.md ================================================ # Installing LBRY If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md)! These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users). Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch): [![Setup for development](https://spee.ch/2018-10-04-17-13-54-017046806.png)](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4) ## Prerequisites Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/). After installing Python 3.7, you'll need to install some additional libraries depending on your operating system. Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769) at the moment the `lbrynet` daemon will only work correctly with Python 3.7. If Python 3.8+ is used, the daemon will start but the RPC server may not accept messages, returning the following: ``` Could not connect to daemon. Are you sure it's running? ``` ### macOS macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/). These environment variables also need to be set: ``` PYTHONUNBUFFERED=1 EVENT_NOKQUEUE=1 ``` Remaining dependencies can then be installed by running: ``` brew install python protobuf ``` Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/. ### Linux On Ubuntu (we recommend 18.04 or 20.04), install the following: ``` sudo add-apt-repository ppa:deadsnakes/ppa sudo apt-get update sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf ``` The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7 for those Ubuntu distributions that no longer have it in their official repositories. On Raspbian, you will also need to install `python-pyparsing`. If you're running another Linux distro, install the equivalent of the above packages for your system. ## Installation ### Linux/Mac Clone the repository: ```bash git clone https://github.com/lbryio/lbry-sdk.git cd lbry-sdk ``` Create a Python virtual environment for lbry-sdk: ```bash python3.7 -m venv lbry-venv ``` Activate virtual environment: ```bash source lbry-venv/bin/activate ``` Make sure you're on Python 3.7+ as default in the virtual environment: ```bash python --version ``` Install packages: ```bash make install ``` If you are on Linux and using PyCharm, generates initial configs: ```bash make idea ``` To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder. ```bash (lbry-venv) $ which lbrynet /opt/lbry-sdk/lbry-venv/bin/lbrynet ``` To exit the virtual environment simply use the command `deactivate`. ### Windows Clone the repository: ```bash git clone https://github.com/lbryio/lbry-sdk.git cd lbry-sdk ``` Create a Python virtual environment for lbry-sdk: ```bash python -m venv lbry-venv ``` Activate virtual environment: ```bash lbry-venv\Scripts\activate ``` Install packages: ```bash pip install -e . ``` ## Run the tests ### Elasticsearch For running integration tests, Elasticsearch is required to be available at localhost:9200/ The easiest way to start it is using docker with: ```bash make elastic-docker ``` Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html). To run the unit and integration tests from the repo directory: ``` python -m unittest discover tests.unit python -m unittest discover tests.integration ``` ## Usage To start the API server: ``` lbrynet start ``` Whenever the code inside [lbry-sdk/lbry](./lbry) is modified we should run `make install` to recompile the `lbrynet` executable with the newest code. ## Development When developing, remember to enter the environment, and if you wish start the server interactively. ```bash $ source lbry-venv/bin/activate (lbry-venv) $ python lbry/extras/cli.py start ``` Parameters can be passed in the same way. ```bash (lbry-venv) $ python lbry/extras/cli.py wallet balance ``` If a Python debugger (`pdb` or `ipdb`) is installed we can also start it in this way, set up break points, and step through the code. ```bash (lbry-venv) $ pip install ipdb (lbry-venv) $ ipdb lbry/extras/cli.py ``` Happy hacking! ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2015-2022 LBRY Inc Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ include README.md include CHANGELOG.md include LICENSE recursive-include lbry *.txt *.py ================================================ FILE: Makefile ================================================ .PHONY: install tools lint test test-unit test-unit-coverage test-integration idea install: pip install -e . lint: pylint --rcfile=setup.cfg lbry #mypy --ignore-missing-imports lbry test: test-unit test-integration test-unit: python -m unittest discover tests.unit test-unit-coverage: coverage run --source=lbry -m unittest discover -vv tests.unit test-integration: tox idea: mkdir -p .idea cp -r scripts/idea/* .idea elastic-docker: docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1 ================================================ FILE: README.md ================================================ # LBRY LBRY SDK [![build](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml/badge.svg)](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml) [![coverage](https://coveralls.io/repos/github/lbryio/lbry-sdk/badge.svg)](https://coveralls.io/github/lbryio/lbry-sdk) LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers. LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include: * Built on Python 3.7 and `asyncio`. * Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)). * Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)). * Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)). * Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/wallet)). * Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/extras/daemon)). ## Installation Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing. ## Usage Run `lbrynet start` to launch the API server. By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages. Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning. With the daemon running, `lbrynet commands` will show you a list of commands. The full API is documented [here](https://lbry.tech/api/sdk). ## Running from source Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md) ## Contributing Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link. ## License This project is MIT licensed. For the full license, see [LICENSE](LICENSE). ## Security We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it. ## Contact The primary contact for this project is [@eukreign](mailto:lex@lbry.com). ## Additional information and links The documentation for the API can be found [here](https://lbry.tech/api/sdk). Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings). Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/example_daemon_settings.yml). ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions While we are not at v1.0 yet, only the latest release will be supported. ## Reporting a Vulnerability See https://lbry.com/faq/security ================================================ FILE: docker/Dockerfile.dht_node ================================================ FROM debian:10-slim ARG user=lbry ARG projects_dir=/home/$user ARG db_dir=/database ARG DOCKER_TAG ARG DOCKER_COMMIT=docker ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget \ automake libtool \ tar unzip \ build-essential \ pkg-config \ libleveldb-dev \ python3.7 \ python3-dev \ python3-pip \ python3-wheel \ python3-setuptools && \ update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \ rm -rf /var/lib/apt/lists/* RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user COPY . $projects_dir RUN chown -R $user:$user $projects_dir RUN mkdir -p $db_dir RUN chown -R $user:$user $db_dir USER $user WORKDIR $projects_dir RUN python3 -m pip install -U setuptools pip RUN make install RUN python3 docker/set_build.py RUN rm ~/.cache -rf VOLUME $db_dir ENTRYPOINT ["python3", "scripts/dht_node.py"] ================================================ FILE: docker/Dockerfile.wallet_server ================================================ FROM debian:10-slim ARG user=lbry ARG db_dir=/database ARG projects_dir=/home/$user ARG DOCKER_TAG ARG DOCKER_COMMIT=docker ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget \ tar unzip \ build-essential \ automake libtool \ pkg-config \ libleveldb-dev \ python3.7 \ python3-dev \ python3-pip \ python3-wheel \ python3-cffi \ python3-setuptools && \ update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \ rm -rf /var/lib/apt/lists/* RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user RUN mkdir -p $db_dir RUN chown -R $user:$user $db_dir COPY . $projects_dir RUN chown -R $user:$user $projects_dir USER $user WORKDIR $projects_dir RUN pip install uvloop RUN make install RUN python3 docker/set_build.py RUN rm ~/.cache -rf # entry point ARG host=0.0.0.0 ARG tcp_port=50001 ARG daemon_url=http://lbry:lbry@localhost:9245/ VOLUME $db_dir ENV TCP_PORT=$tcp_port ENV HOST=$host ENV DAEMON_URL=$daemon_url ENV DB_DIRECTORY=$db_dir ENV MAX_SESSIONS=1000000000 ENV MAX_SEND=1000000000000000000 ENV EVENT_LOOP_POLICY=uvloop COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ================================================ FILE: docker/Dockerfile.web ================================================ FROM debian:10-slim ARG user=lbry ARG downloads_dir=/database ARG projects_dir=/home/$user ARG DOCKER_TAG ARG DOCKER_COMMIT=docker ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget \ automake libtool \ tar unzip \ build-essential \ pkg-config \ libleveldb-dev \ python3.7 \ python3-dev \ python3-pip \ python3-wheel \ python3-setuptools && \ update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \ rm -rf /var/lib/apt/lists/* RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user RUN mkdir -p $downloads_dir RUN chown -R $user:$user $downloads_dir COPY . $projects_dir RUN chown -R $user:$user $projects_dir USER $user WORKDIR $projects_dir RUN pip install uvloop RUN make install RUN python3 docker/set_build.py RUN rm ~/.cache -rf # entry point VOLUME $downloads_dir COPY ./docker/webconf.yaml /webconf.yaml ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"] ================================================ FILE: docker/README.md ================================================ ### How to run with docker-compose 1. Edit config file and after that fix permissions with ``` sudo chown -R 999:999 webconf.yaml ``` 2. Start SDK with ``` docker-compose up -d ``` ================================================ FILE: docker/docker-compose-wallet-server.yml ================================================ version: "3" volumes: wallet_server: es01: services: wallet_server: depends_on: - es01 image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release} restart: always network_mode: host ports: - "50001:50001" # rpc port - "2112:2112" # uncomment to enable prometheus volumes: - "wallet_server:/database" environment: - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245 - MAX_QUERY_WORKERS=4 - CACHE_MB=1024 - CACHE_ALL_TX_HASHES= - CACHE_ALL_CLAIM_TXOS= - MAX_SEND=1000000000000000000 - MAX_RECEIVE=1000000000000000000 - MAX_SESSIONS=100000 - HOST=0.0.0.0 - TCP_PORT=50001 - PROMETHEUS_PORT=2112 - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 es01: image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 container_name: es01 environment: - node.name=es01 - discovery.type=single-node - indices.query.bool.max_clause_count=8192 - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms4g -Xmx4g" # no more than 32, remember to disable swap ulimits: memlock: soft: -1 hard: -1 volumes: - es01:/usr/share/elasticsearch/data ports: - 127.0.0.1:9200:9200 ================================================ FILE: docker/docker-compose.yml ================================================ version: '3' services: websdk: image: vshyba/websdk ports: - '5279:5279' - '5280:5280' volumes: - ./webconf.yaml:/webconf.yaml ================================================ FILE: docker/hooks/build ================================================ #!/bin/bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../.." ## make sure we're in the right place. Docker Hub screws this up sometimes echo "docker build dir: $(pwd)" docker build --build-arg DOCKER_TAG=$DOCKER_TAG --build-arg DOCKER_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME . ================================================ FILE: docker/install_choco.ps1 ================================================ # requires powershell and .NET 4+. see https://chocolatey.org/install for more info. $chocoVersion = powershell choco -v if(-not($chocoVersion)){ Write-Output "Chocolatey is not installed, installing now" Write-Output "IF YOU KEEP GETTING THIS MESSAGE ON EVERY BUILD, TRY RESTARTING THE GITLAB RUNNER SO IT GETS CHOCO INTO IT'S ENV" Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) } else{ Write-Output "Chocolatey version $chocoVersion is already installed" } ================================================ FILE: docker/set_build.py ================================================ import sys import os import re import logging import lbry.build_info as build_info_mod log = logging.getLogger() log.addHandler(logging.StreamHandler()) log.setLevel(logging.DEBUG) def _check_and_set(d: dict, key: str, value: str): try: d[key] except KeyError: raise Exception(f"{key} var does not exist in {build_info_mod.__file__}") d[key] = value def main(): build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")} commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA')) if commit_hash is None: raise ValueError("Commit hash not found in env vars") _check_and_set(build_info, "COMMIT_HASH", commit_hash[:6]) docker_tag = os.getenv('DOCKER_TAG') if docker_tag: _check_and_set(build_info, "DOCKER_TAG", docker_tag) _check_and_set(build_info, "BUILD", "docker") else: if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))): _check_and_set(build_info, "BUILD", "release") else: _check_and_set(build_info, "BUILD", "qa") log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()])) with open(build_info_mod.__file__, 'w') as f: f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n") if __name__ == '__main__': sys.exit(main()) ================================================ FILE: docker/wallet_server_entrypoint.sh ================================================ #!/bin/bash # entrypoint for wallet server Docker image set -euo pipefail SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/lbry-leveldb ]]; then files="$(ls)" echo "Downloading wallet snapshot from $SNAPSHOT_URL" wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL" echo "Extracting snapshot..." filename="$(grep -vf <(echo "$files") <(ls))" # finds the file that was not there before case "$filename" in *.tgz|*.tar.gz|*.tar.bz2 ) tar xvf "$filename" --directory /database ;; *.zip ) unzip "$filename" -d /database ;; * ) echo "Don't know how to extract ${filename}. SNAPSHOT COULD NOT BE LOADED" && exit 1 ;; esac rm "$filename" fi /home/lbry/.local/bin/lbry-hub-elastic-sync echo 'starting server' /home/lbry/.local/bin/lbry-hub "$@" ================================================ FILE: docker/webconf.yaml ================================================ allowed_origin: "*" max_key_fee: "0.0 USD" save_files: false save_blobs: false streaming_server: "0.0.0.0:5280" api: "0.0.0.0:5279" data_dir: /tmp download_dir: /tmp wallet_dir: /tmp ================================================ FILE: docs/api.json ================================================ { "main": { "doc": "Ungrouped commands.", "commands": [ { "name": "ffmpeg_find", "description": "Get ffmpeg installation information", "arguments": [], "returns": "(dict) Dictionary of ffmpeg information\n {\n 'available': (bool) found ffmpeg,\n 'which': (str) path to ffmpeg,\n 'analyze_audio_volume': (bool) should ffmpeg analyze audio\n }", "examples": [] }, { "name": "get", "description": "Download stream from a LBRY name.", "arguments": [ { "name": "uri", "type": "str", "description": "uri of the content to download", "is_required": false }, { "name": "file_name", "type": "str", "description": "specified name for the downloaded file, overrides the stream file name", "is_required": false }, { "name": "download_directory", "type": "str", "description": "full path to the directory to download into", "is_required": false }, { "name": "timeout", "type": "int", "description": "download timeout in number of seconds", "is_required": false }, { "name": "save_file", "type": "bool", "description": "save the file to the downloads directory", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "wallet to check for claim purchase receipts", "is_required": false } ], "returns": " {\n \"streaming_url\": \"(str) url to stream the file using range requests\",\n \"completed\": \"(bool) true if download is completed\",\n \"file_name\": \"(str) name of file\",\n \"download_directory\": \"(str) download directory\",\n \"points_paid\": \"(float) credit paid to download file\",\n \"stopped\": \"(bool) true if download is stopped\",\n \"stream_hash\": \"(str) stream hash of file\",\n \"stream_name\": \"(str) stream name\",\n \"suggested_file_name\": \"(str) suggested file name\",\n \"sd_hash\": \"(str) sd hash of file\",\n \"download_path\": \"(str) download path of file\",\n \"mime_type\": \"(str) mime type of file\",\n \"key\": \"(str) key attached to file\",\n \"total_bytes_lower_bound\": \"(int) lower bound file size in bytes\",\n \"total_bytes\": \"(int) file upper bound size in bytes\",\n \"written_bytes\": \"(int) written size in bytes\",\n \"blobs_completed\": \"(int) number of fully downloaded blobs\",\n \"blobs_in_stream\": \"(int) total blobs on stream\",\n \"blobs_remaining\": \"(int) total blobs remaining to download\",\n \"status\": \"(str) downloader status\",\n \"claim_id\": \"(str) None if claim is not found else the claim id\",\n \"txid\": \"(str) None if claim is not found else the transaction id\",\n \"nout\": \"(int) None if claim is not found else the transaction output index\",\n \"outpoint\": \"(str) None if claim is not found else the tx and output\",\n \"metadata\": \"(dict) None if claim is not found else the claim metadata\",\n \"channel_claim_id\": \"(str) None if claim is not found or not signed\",\n \"channel_name\": \"(str) None if claim is not found or not signed\",\n \"claim_name\": \"(str) None if claim is not found else the claim name\",\n \"reflector_progress\": \"(int) reflector upload progress, 0 to 100\",\n \"uploading_to_reflector\": \"(bool) set to True when currently uploading to reflector\"\n }", "examples": [ { "title": "Get a file", "curl": "curl -d'{\"method\": \"get\", \"params\": {\"uri\": \"astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\"}}' http://localhost:5279/", "lbrynet": "lbrynet get astream#ad25e05aa7dc5e9994869040c6103f9a8728db46", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"get\", \"params\": {\"uri\": \"astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"added_on\": 1655141677,\n \"blobs_completed\": 1,\n \"blobs_in_stream\": 1,\n \"blobs_remaining\": 0,\n \"channel_claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"channel_name\": \"@channel\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_name\": \"astream\",\n \"completed\": true,\n \"confirmations\": 4,\n \"content_fee\": null,\n \"download_directory\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpfx0nk2jd\",\n \"download_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpfx0nk2jd/tmpr832hp1x\",\n \"file_name\": \"tmpr832hp1x\",\n \"height\": 214,\n \"is_fully_reflected\": false,\n \"key\": \"66f888fe00cf558494c2fcbd5903d00d\",\n \"metadata\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"mime_type\": \"application/octet-stream\",\n \"nout\": 0,\n \"outpoint\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234:0\",\n \"points_paid\": 0.0,\n \"protobuf\": \"01a2795714b0923650b7fa288618591f0c2f2e5c5961baf110460a527e38b68f0653c1c79f2cae1f57ade55c009fb7578175c9d93d7edb0546d1378e1d33d99df8df0d5cfe9e7a5b63053a4e7ce003f95f5890b27f0a90010a8d010a30fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd120b746d707238333268703178180b22186170706c69636174696f6e2f6f637465742d73747265616d32309ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"purchase_receipt\": null,\n \"reflector_progress\": 0,\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"status\": \"finished\",\n \"stopped\": true,\n \"stream_hash\": \"c48ff9950efbcb78b20d311467b1b0e321069ef5ece96898eb08a27a62d1ef2594cd24f1e8d6993f00c48fd6a4221490\",\n \"stream_name\": \"tmpr832hp1x\",\n \"streaming_url\": \"http://localhost:5280/stream/9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"suggested_file_name\": \"tmpr832hp1x\",\n \"timestamp\": 1655141671,\n \"total_bytes\": 16,\n \"total_bytes_lower_bound\": 0,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"uploading_to_reflector\": false,\n \"written_bytes\": 11\n }\n}" } ] }, { "name": "publish", "description": "Create or replace a stream claim at a given name (use 'stream create/update' for more control).", "arguments": [ { "name": "name", "type": "str", "description": "name of the content (can only consist of a-z A-Z 0-9 and -(dash))", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": false }, { "name": "file_path", "type": "str", "description": "path to file to be associated with name.", "is_required": false }, { "name": "file_name", "type": "str", "description": "name of file to be associated with stream.", "is_required": false }, { "name": "file_hash", "type": "str", "description": "hash of file to be associated with stream.", "is_required": false }, { "name": "validate_file", "type": "bool", "description": "validate that the video container and encodings match common web browser support or that optimization succeeds if specified. FFmpeg is required", "is_required": false }, { "name": "optimize_file", "type": "bool", "description": "transcode the video & audio if necessary to ensure common web browser support. FFmpeg is required", "is_required": false }, { "name": "fee_currency", "type": "string", "description": "specify fee currency", "is_required": false }, { "name": "fee_amount", "type": "decimal", "description": "content download fee", "is_required": false }, { "name": "fee_address", "type": "str", "description": "address where to send fee payments, will use value from --claim_address if not provided", "is_required": false }, { "name": "title", "type": "str", "description": "title of the publication", "is_required": false }, { "name": "description", "type": "str", "description": "description of the publication", "is_required": false }, { "name": "author", "type": "str", "description": "author of the publication. The usage for this field is not the same as for channels. The author field is used to credit an author who is not the publisher and is not represented by the channel. For example, a pdf file of 'The Odyssey' has an author of 'Homer' but may by published to a channel such as '@classics', or to no channel at all", "is_required": false }, { "name": "tags", "type": "list", "description": "add content tags", "is_required": false }, { "name": "languages", "type": "list", "description": "languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant`", "is_required": false }, { "name": "locations", "type": "list", "description": "locations relevant to the stream, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... \"locations\": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: \"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE\" making sure to include colon for blank values, for example to provide only the city: ... --locations=\"::Manchester\" with all values set: ... --locations=\"US:NH:Manchester:03101:42.990605:-71.460989\" optionally, you can just pass the \"LATITUDE:LONGITUDE\": ... --locations=\"42.990605:-71.460989\" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations=\"{'country': 'US', 'state': 'NH'}\"", "is_required": false }, { "name": "license", "type": "str", "description": "publication license", "is_required": false }, { "name": "license_url", "type": "str", "description": "publication license url", "is_required": false }, { "name": "thumbnail_url", "type": "str", "description": "thumbnail url", "is_required": false }, { "name": "release_time", "type": "int", "description": "original public release of content, seconds since UNIX epoch", "is_required": false }, { "name": "width", "type": "int", "description": "image/video width, automatically calculated from media file", "is_required": false }, { "name": "height", "type": "int", "description": "image/video height, automatically calculated from media file", "is_required": false }, { "name": "duration", "type": "int", "description": "audio/video duration in seconds, automatically calculated", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "sd_hash of stream", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of the publisher channel", "is_required": false }, { "name": "channel_name", "type": "str", "description": "name of publisher channel", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "account_id", "type": "str", "description": "account to use for holding the transaction", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the claim is sent to, if not specified it will be determined automatically from the account", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Publish a file", "curl": "curl -d'{\"method\": \"publish\", \"params\": {\"name\": \"a-new-stream\", \"bid\": \"1.0\", \"file_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmp1wt4ndjd\", \"validate_file\": false, \"optimize_file\": false, \"tags\": [], \"languages\": [], \"locations\": [], \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet publish a-new-stream --bid=1.0 --file_path=/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmp1wt4ndjd", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"publish\", \"params\": {\"name\": \"a-new-stream\", \"bid\": \"1.0\", \"file_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmp1wt4ndjd\", \"validate_file\": false, \"optimize_file\": false, \"tags\": [], \"languages\": [], \"locations\": [], \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"0100000001c701b5f207fe0b1abd990f0899a8c8bc4bf6c8ce07d0eafc8a59e44bce4f95a9010000006b483045022100fb66c628da4fb63d7f1ae98d3c2bd15b5e4ae4aa7863891b943989f92c1aed0902202362799eb6f3d1cd66bd230c6710fafa248ad48073a7070e1ba51d24b7c7f453012102f272fd093af2228160e96af70d809c329d9c547375b6d838498880f7c3051290ffffffff0200e1f50500000000bfb50c612d6e65772d73747265616d4c94000a90010a8d010a30fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd120b746d70317774346e646a64180b22186170706c69636174696f6e2f6f637465742d73747265616d32304e61ee6b6a7810750982faa6c805323e33b5d21d2798d4add70c7588640ed8c5f70c822382be5fe87fd5ae356e26896e6d7576a9147ec6c9961e0d398d337699d566702f0ffe1ea07888ac54826311000000001976a914bf0468b0a4fd815b9a4bef7d7c3cd59a67e35a9b88ac00000000\",\n \"inputs\": [\n {\n \"address\": \"n2qN5bEHEQ1keWyR6GbNd15dbHE6z1FoZb\",\n \"amount\": \"3.941448\",\n \"confirmations\": 3,\n \"height\": 217,\n \"nout\": 1,\n \"timestamp\": 1655141675,\n \"txid\": \"a9954fce4be4598afcead007cec8f64bbcc8a899080f99bd1a0bfe07f2b501c7\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"ms5HaDddaUyAAMczcdCbYuTFkuBdAgQxeY\",\n \"amount\": \"1.0\",\n \"claim_id\": \"006db49526758cf95dc2d9cd2354114e67441baa\",\n \"claim_op\": \"create\",\n \"confirmations\": -2,\n \"height\": -2,\n \"meta\": {},\n \"name\": \"a-new-stream\",\n \"normalized_name\": \"a-new-stream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://a-new-stream#006db49526758cf95dc2d9cd2354114e67441baa\",\n \"timestamp\": null,\n \"txid\": \"0c646996c6c30899690e4536037c21e1f2190b89be07af626b3b61f17c4fc431\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmp1wt4ndjd\",\n \"sd_hash\": \"4e61ee6b6a7810750982faa6c805323e33b5d21d2798d4add70c7588640ed8c5f70c822382be5fe87fd5ae356e26896e\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"mxvxe2qg1xUdSoMWFch86uZKS5X3t2Sj3t\",\n \"amount\": \"2.917341\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"0c646996c6c30899690e4536037c21e1f2190b89be07af626b3b61f17c4fc431\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.024107\",\n \"total_input\": \"3.941448\",\n \"total_output\": \"3.917341\",\n \"txid\": \"0c646996c6c30899690e4536037c21e1f2190b89be07af626b3b61f17c4fc431\"\n }\n}" } ] }, { "name": "resolve", "description": "Get the claim that a URL refers to.", "arguments": [ { "name": "urls", "type": "str, list", "description": "one or more urls to resolve", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "wallet to check for claim purchase receipts", "is_required": false }, { "name": "new_sdk_server", "type": "str", "description": "URL of the new SDK server (EXPERIMENTAL)", "is_required": false }, { "name": "include_purchase_receipt", "type": "bool", "description": "lookup and include a receipt if this wallet has purchased the claim being resolved", "is_required": false }, { "name": "include_is_my_output", "type": "bool", "description": "lookup and include a boolean indicating if claim being resolved is yours", "is_required": false }, { "name": "include_sent_supports", "type": "bool", "description": "lookup and sum the total amount of supports you've made to this claim", "is_required": false }, { "name": "include_sent_tips", "type": "bool", "description": "lookup and sum the total amount of tips you've made to this claim (only makes sense when claim is not yours)", "is_required": false }, { "name": "include_received_tips", "type": "bool", "description": "lookup and sum the total amount of tips you've received to this claim (only makes sense when claim is yours)", "is_required": false } ], "returns": "Dictionary of results, keyed by url\n '': {\n If a resolution error occurs:\n 'error': Error message\n\n If the url resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number (or -1 if unknown),\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'confirmations': (int) claim depth,\n 'timestamp': (int) timestamp of the block that included this claim tx,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the url resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the url resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number (or -1 if unknown),\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }", "examples": [ { "title": "Resolve a claim", "curl": "curl -d'{\"method\": \"resolve\", \"params\": {\"urls\": [\"astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\"], \"include_purchase_receipt\": false, \"include_is_my_output\": false, \"include_sent_supports\": false, \"include_sent_tips\": false, \"include_received_tips\": false}}' http://localhost:5279/", "lbrynet": "lbrynet resolve astream#ad25e05aa7dc5e9994869040c6103f9a8728db46", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"resolve\", \"params\": {\"urls\": [\"astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\"], \"include_purchase_receipt\": false, \"include_is_my_output\": false, \"include_sent_supports\": false, \"include_sent_tips\": false, \"include_received_tips\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\": {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5/astream#a\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 4,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"meta\": {\n \"activation_height\": 214,\n \"creation_height\": 213,\n \"creation_timestamp\": 1655141671,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 714,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 213\n },\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"short_url\": \"lbry://astream#a\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 8,\n \"has_signing_key\": false,\n \"height\": 210,\n \"meta\": {\n \"activation_height\": 210,\n \"claims_in_channel\": 2,\n \"creation_height\": 209,\n \"creation_timestamp\": 1655141670,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 710,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 209\n },\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"short_url\": \"lbry://@channel#5\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n }\n }\n}" } ] }, { "name": "routing_table_get", "description": "Get DHT routing information", "arguments": [], "returns": "(dict) dictionary containing routing and peer information\n {\n \"buckets\": {\n : [\n {\n \"address\": (str) peer address,\n \"udp_port\": (int) peer udp port,\n \"tcp_port\": (int) peer tcp port,\n \"node_id\": (str) peer node id,\n }\n ]\n },\n \"node_id\": (str) the local dht node id\n \"prefix_neighbors_count\": (int) the amount of peers sharing the same byte prefix of the local node id\n }", "examples": [] }, { "name": "status", "description": "Get daemon status", "arguments": [], "returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'database': (bool),\n 'dht': (bool),\n 'exchange_rate_manager': (bool),\n 'hash_announcer': (bool),\n 'peer_protocol_server': (bool),\n 'file_manager': (bool),\n 'libtorrent_component': (bool),\n 'upnp': (bool),\n 'wallet': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'connected': (str) host and port of the connected spv server,\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n 'connected_servers': (list) [\n {\n 'host': (str) server hostname,\n 'port': (int) server port,\n 'latency': (int) milliseconds\n }\n ],\n },\n 'libtorrent_component': {\n 'running': (bool) libtorrent was detected and started successfully,\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n 'connections': {\n 'incoming_bps': {\n : (int) bytes per second received,\n },\n 'outgoing_bps': {\n : (int) bytes per second sent,\n },\n 'total_outgoing_mps': (float) megabytes per second sent,\n 'total_incoming_mps': (float) megabytes per second received,\n 'max_outgoing_mbs': (float) maximum bandwidth (megabytes per second) sent, since the\n daemon was started\n 'max_incoming_mbs': (float) maximum bandwidth (megabytes per second) received, since the\n daemon was started\n 'total_sent' : (int) total number of bytes sent since the daemon was started\n 'total_received' : (int) total number of bytes received since the daemon was started\n }\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the stream manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n : (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }", "examples": [ { "title": "Get status", "curl": "curl -d'{\"method\": \"status\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet status", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"status\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"background_downloader\": {\n \"available_free_space_mb\": null,\n \"ongoing_download\": false,\n \"running\": false\n },\n \"blob_manager\": {\n \"connections\": {\n \"incoming_bps\": {},\n \"max_incoming_mbs\": 0.0,\n \"max_outgoing_mbs\": 0.0,\n \"outgoing_bps\": {},\n \"total_incoming_mbs\": 0.0,\n \"total_outgoing_mbs\": 0.0,\n \"total_received\": 0,\n \"total_sent\": 0\n },\n \"finished_blobs\": 0\n },\n \"disk_space\": {\n \"content_blobs_storage_used_mb\": 0,\n \"published_blobs_storage_used_mb\": 0,\n \"running\": true,\n \"seed_blobs_storage_used_mb\": 0,\n \"total_used_mb\": 0\n },\n \"ffmpeg_status\": {\n \"analyze_audio_volume\": true,\n \"available\": true,\n \"which\": \"/opt/homebrew/bin/ffmpeg\"\n },\n \"file_manager\": {\n \"managed_files\": 0\n },\n \"installation_id\": \"3a1UGocRVHs9fAYWhNTRGoq2mpi97NqGEbkofcVkZvJei5xqimckTxfDRdjuKt4Hr6\",\n \"is_running\": true,\n \"skipped_components\": [\n \"dht\",\n \"upnp\",\n \"hash_announcer\",\n \"peer_protocol_server\",\n \"libtorrent_component\"\n ],\n \"startup_status\": {\n \"background_downloader\": true,\n \"blob_manager\": true,\n \"database\": true,\n \"disk_space\": true,\n \"exchange_rate_manager\": true,\n \"file_manager\": true,\n \"tracker_announcer_component\": true,\n \"wallet\": true,\n \"wallet_server_payments\": true\n },\n \"wallet\": {\n \"available_servers\": 1,\n \"best_blockhash\": \"47eb373669435f62580b2855670788eaf07e62551c2d38361e24700f130cab36\",\n \"blocks\": 206,\n \"blocks_behind\": 0,\n \"connected\": \"localhost:50002\",\n \"connected_features\": {\n \"daily_fee\": \"0\",\n \"description\": \"\",\n \"donation_address\": \"\",\n \"genesis_hash\": \"6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556\",\n \"hash_function\": \"sha256\",\n \"hosts\": {},\n \"payment_address\": \"\",\n \"protocol_max\": \"0.199.0\",\n \"protocol_min\": \"0.54.0\",\n \"pruning\": null,\n \"server_version\": \"0.107.0\",\n \"trending_algorithm\": \"fast_ar\"\n },\n \"headers_synchronization_progress\": 100,\n \"known_servers\": 0,\n \"servers\": [\n {\n \"availability\": true,\n \"host\": \"localhost\",\n \"latency\": 0.00792012499999828,\n \"port\": 50002\n }\n ]\n },\n \"wallet_server_payments\": {\n \"max_fee\": \"0.0\",\n \"running\": false\n }\n }\n}" } ] }, { "name": "stop", "description": "Stop lbrynet API server.", "arguments": [], "returns": "(string) Shutdown message", "examples": [] }, { "name": "version", "description": "Get lbrynet API server version information", "arguments": [], "returns": "(dict) Dictionary of lbry version information\n {\n 'processor': (str) processor type,\n 'python_version': (str) python version,\n 'platform': (str) platform string,\n 'os_release': (str) os release string,\n 'os_system': (str) os name,\n 'version': (str) lbrynet version,\n 'build': (str) \"dev\" | \"qa\" | \"rc\" | \"release\",\n }", "examples": [ { "title": "Get version", "curl": "curl -d'{\"method\": \"version\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet version", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"version\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"build\": \"dev\",\n \"lbrynet_version\": \"0.109.0\",\n \"os_release\": \"21.5.0\",\n \"os_system\": \"Darwin\",\n \"platform\": \"Darwin-21.5.0-x86_64-i386-64bit\",\n \"processor\": \"i386\",\n \"python_version\": \"3.7.13\",\n \"version\": \"0.109.0\"\n }\n}" } ] } ] }, "account": { "doc": "Create, modify and inspect wallet accounts.", "commands": [ { "name": "account_add", "description": "Add a previously created account from a seed, private key or public key (read-only).\nSpecify --single_key for single address or vanity address accounts.", "arguments": [ { "name": "account_name", "type": "str", "description": "name of the account to add", "is_required": true }, { "name": "seed", "type": "str", "description": "seed to generate new account from", "is_required": false }, { "name": "private_key", "type": "str", "description": "private key for new account", "is_required": false }, { "name": "public_key", "type": "str", "description": "public key for new account", "is_required": false }, { "name": "single_key", "type": "bool", "description": "create single key account, default is multi-key", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": " {\n \"id\": \"account_id\",\n \"is_default\": \"this account is used by default\",\n \"ledger\": \"name of crypto currency and network\",\n \"name\": \"optional account name\",\n \"seed\": \"human friendly words from which account can be recreated\",\n \"encrypted\": \"if account is encrypted\",\n \"private_key\": \"extended private key\",\n \"public_key\": \"extended public key\",\n \"address_generator\": \"settings for generating addresses\",\n \"modified_on\": \"date of last modification to account settings\"\n }", "examples": [ { "title": "Add an account from seed", "curl": "curl -d'{\"method\": \"account_add\", \"params\": {\"account_name\": \"new account\", \"seed\": \"miss ready crop oval canyon such sing powder figure math noodle style\", \"single_key\": false}}' http://localhost:5279/", "lbrynet": "lbrynet account add \"new account\" --seed=\"miss ready crop oval canyon such sing powder figure math noodle style\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_add\", \"params\": {\"account_name\": \"new account\", \"seed\": \"miss ready crop oval canyon such sing powder figure math noodle style\", \"single_key\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"address_generator\": {\n \"change\": {\n \"gap\": 6,\n \"maximum_uses_per_address\": 1\n },\n \"name\": \"deterministic-chain\",\n \"receiving\": {\n \"gap\": 20,\n \"maximum_uses_per_address\": 1\n }\n },\n \"encrypted\": false,\n \"id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"is_default\": false,\n \"ledger\": \"lbc_regtest\",\n \"modified_on\": 1655141657,\n \"name\": \"new account\",\n \"private_key\": \"tprv8ZgxMBicQKsPdmLUTJtRsfwAeL527nfpJE1GhFFLzKtbuuzwYorQyq8guwgzFCQkqs8EuR2MGLkbh3xuEb1pFuAjx1Web8Vk3RXrrLVKwRU\",\n \"public_key\": \"tpubD6NzVbkrYhZ4XENGLxZ2H5bHDMaxH7risXc3ymHeQbgzkQFiBCg1AKkZ66UqGohv1bXg5579Z3r4omYeqMEd1Khhyuf4dssfmBzfiEce23o\",\n \"seed\": \"miss ready crop oval canyon such sing powder figure math noodle style\"\n }\n}" } ] }, { "name": "account_balance", "description": "Return the balance of an account", "arguments": [ { "name": "account_id", "type": "str", "description": "If provided only the balance for this account will be given. Otherwise default account.", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "balance for specific wallet", "is_required": false }, { "name": "confirmations", "type": "int", "description": "Only include transactions with this many confirmed blocks.", "is_required": false } ], "returns": "(decimal) amount of lbry credits in wallet", "examples": [ { "title": "Get default account balance", "curl": "curl -d'{\"method\": \"account_balance\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet account balance", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_balance\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"available\": \"7.999876\",\n \"reserved\": \"0.0\",\n \"reserved_subtotals\": {\n \"claims\": \"0.0\",\n \"supports\": \"0.0\",\n \"tips\": \"0.0\"\n },\n \"total\": \"7.999876\"\n }\n}" }, { "title": "Get balance for specific account by id", "curl": "curl -d'{\"method\": \"account_balance\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"}}' http://localhost:5279/", "lbrynet": "lbrynet account balance \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_balance\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"available\": \"2.0\",\n \"reserved\": \"0.0\",\n \"reserved_subtotals\": {\n \"claims\": \"0.0\",\n \"supports\": \"0.0\",\n \"tips\": \"0.0\"\n },\n \"total\": \"2.0\"\n }\n}" } ] }, { "name": "account_create", "description": "Create a new account. Specify --single_key if you want to use\nthe same address for all transactions (not recommended).", "arguments": [ { "name": "account_name", "type": "str", "description": "name of the account to create", "is_required": true }, { "name": "single_key", "type": "bool", "description": "create single key account, default is multi-key", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": " {\n \"id\": \"account_id\",\n \"is_default\": \"this account is used by default\",\n \"ledger\": \"name of crypto currency and network\",\n \"name\": \"optional account name\",\n \"seed\": \"human friendly words from which account can be recreated\",\n \"encrypted\": \"if account is encrypted\",\n \"private_key\": \"extended private key\",\n \"public_key\": \"extended public key\",\n \"address_generator\": \"settings for generating addresses\",\n \"modified_on\": \"date of last modification to account settings\"\n }", "examples": [ { "title": "Create an account", "curl": "curl -d'{\"method\": \"account_create\", \"params\": {\"account_name\": \"generated account\", \"single_key\": false}}' http://localhost:5279/", "lbrynet": "lbrynet account create \"generated account\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_create\", \"params\": {\"account_name\": \"generated account\", \"single_key\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"address_generator\": {\n \"change\": {\n \"gap\": 6,\n \"maximum_uses_per_address\": 1\n },\n \"name\": \"deterministic-chain\",\n \"receiving\": {\n \"gap\": 20,\n \"maximum_uses_per_address\": 1\n }\n },\n \"encrypted\": false,\n \"id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"is_default\": false,\n \"ledger\": \"lbc_regtest\",\n \"modified_on\": 1655141657,\n \"name\": \"generated account\",\n \"private_key\": \"tprv8ZgxMBicQKsPdmLUTJtRsfwAeL527nfpJE1GhFFLzKtbuuzwYorQyq8guwgzFCQkqs8EuR2MGLkbh3xuEb1pFuAjx1Web8Vk3RXrrLVKwRU\",\n \"public_key\": \"tpubD6NzVbkrYhZ4XENGLxZ2H5bHDMaxH7risXc3ymHeQbgzkQFiBCg1AKkZ66UqGohv1bXg5579Z3r4omYeqMEd1Khhyuf4dssfmBzfiEce23o\",\n \"seed\": \"miss ready crop oval canyon such sing powder figure math noodle style\"\n }\n}" } ] }, { "name": "account_deposit", "description": "Spend a time locked transaction into your account.", "arguments": [ { "name": "txid", "type": "str", "description": "id of the transaction", "is_required": false }, { "name": "nout", "type": "int", "description": "output number in the transaction", "is_required": false }, { "name": "redeem_script", "type": "str", "description": "redeem script for output", "is_required": false }, { "name": "private_key", "type": "str", "description": "private key to sign transaction", "is_required": false }, { "name": "to_account", "type": "str", "description": "deposit to this account", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "limit operation to specific wallet.", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until tx has synced", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "account_fund", "description": "Transfer some amount (or --everything) to an account from another\naccount (can be the same account). Amounts are interpreted as LBC.\nYou can also spread the transfer across a number of --outputs (cannot\nbe used together with --everything).", "arguments": [ { "name": "to_account", "type": "str", "description": "send to this account", "is_required": false }, { "name": "from_account", "type": "str", "description": "spend from this account", "is_required": false }, { "name": "amount", "type": "decimal", "description": "the amount to transfer lbc", "is_required": true }, { "name": "everything", "type": "bool", "description": "transfer everything (excluding claims), default: false.", "is_required": false }, { "name": "outputs", "type": "int", "description": "split payment across many outputs, default: 1.", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "limit operation to specific wallet.", "is_required": false }, { "name": "broadcast", "type": "bool", "description": "actually broadcast the transaction, default: false.", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Transfer 2 LBC from default account to specific account", "curl": "curl -d'{\"method\": \"account_fund\", \"params\": {\"to_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"amount\": \"2.0\", \"everything\": false, \"broadcast\": true}}' http://localhost:5279/", "lbrynet": "lbrynet account fund --to_account=\"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\" --amount=2.0 --broadcast", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_fund\", \"params\": {\"to_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"amount\": \"2.0\", \"everything\": false, \"broadcast\": true}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"01000000012a489c130efda054091c1e91a258ad57772619a9b42ab5e416768e32334e581c000000006a47304402207fd2ffd9fe8363604a428161ace5ad1bb3bab156e9746dcf4d9b784e90abd8c6022024766cb49008eeac5b963a97de0d63611e8a5d7299776eab95598514b18ad1da0121028a467e50cb5555cc3d4ce66944ffbf5566cfe7b8a553e3bf40d4d214d4cb5233ffffffff0200c2eb0b000000001976a9140e92a8bca34238d407745d94fc5b21f0e39f518888ac90d7ae2f000000001976a914169f2f9502bfb8ee9c1d97c315ef13672721638b88ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mqwYxGxBhfFodPpkCDGXK3eiSHpfRPi6SD\",\n \"amount\": \"10.0\",\n \"confirmations\": 6,\n \"height\": 201,\n \"nout\": 0,\n \"timestamp\": 1655141669,\n \"txid\": \"1c584e33328e7616e4b52ab4a919267757ad58a2911e1c0954a0fd0e139c482a\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mgr1So3484shNr8kNMsavaohFwwRm9AnFU\",\n \"amount\": \"2.0\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 0,\n \"timestamp\": null,\n \"txid\": \"f03159770e23cab4da7779dcf1a6809f6f518c61ab540d6a452d0bc5509874b8\",\n \"type\": \"payment\"\n },\n {\n \"address\": \"mhaZrhgfvGv49E47oDupq8wBppSNfhMFam\",\n \"amount\": \"7.999876\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"f03159770e23cab4da7779dcf1a6809f6f518c61ab540d6a452d0bc5509874b8\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.000124\",\n \"total_input\": \"10.0\",\n \"total_output\": \"9.999876\",\n \"txid\": \"f03159770e23cab4da7779dcf1a6809f6f518c61ab540d6a452d0bc5509874b8\"\n }\n}" }, { "title": "Spread LBC between multiple addresses", "curl": "curl -d'{\"method\": \"account_fund\", \"params\": {\"to_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"from_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"amount\": \"1.5\", \"everything\": false, \"outputs\": 2, \"broadcast\": true}}' http://localhost:5279/", "lbrynet": "lbrynet account fund --to_account=\"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\" --from_account=\"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\" --amount=1.5 --outputs=2 --broadcast", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_fund\", \"params\": {\"to_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"from_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"amount\": \"1.5\", \"everything\": false, \"outputs\": 2, \"broadcast\": true}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"0100000001b8749850c50b2d456a0d54ab618c516f9f80a6f1dc7977dab4ca230e775931f0000000006b48304502210087b512d6b2772190e417f1b4252c0905d09c98eab3a229e3a3f2ab3bda11f2c7022030d2be08ee86c4b1e3899cebea7e1a5b68c74022502e57cc7b141c083fac45b401210232b7e745cbafb7b3506b71afe38ba9a1c049aec1426c3e5442b246604a14c5f6ffffffff03c0687804000000001976a9144f694178c5f66475002876866115a8a74177fcba88acc0687804000000001976a9144f694178c5f66475002876866115a8a74177fcba88ac6cb9fa02000000001976a9140e92a8bca34238d407745d94fc5b21f0e39f518888ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mgr1So3484shNr8kNMsavaohFwwRm9AnFU\",\n \"amount\": \"2.0\",\n \"confirmations\": 1,\n \"height\": 207,\n \"nout\": 0,\n \"timestamp\": 1655141670,\n \"txid\": \"f03159770e23cab4da7779dcf1a6809f6f518c61ab540d6a452d0bc5509874b8\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mnkqmWKPUwkeDuq7pndpfrzXFQDmH9QTBo\",\n \"amount\": \"0.75\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 0,\n \"timestamp\": null,\n \"txid\": \"b1ae061be71d91de2b6e5f6c707547ad109dabb510b7dcddf20b96acb8175561\",\n \"type\": \"payment\"\n },\n {\n \"address\": \"mnkqmWKPUwkeDuq7pndpfrzXFQDmH9QTBo\",\n \"amount\": \"0.75\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"b1ae061be71d91de2b6e5f6c707547ad109dabb510b7dcddf20b96acb8175561\",\n \"type\": \"payment\"\n },\n {\n \"address\": \"mgr1So3484shNr8kNMsavaohFwwRm9AnFU\",\n \"amount\": \"0.499859\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 2,\n \"timestamp\": null,\n \"txid\": \"b1ae061be71d91de2b6e5f6c707547ad109dabb510b7dcddf20b96acb8175561\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.000141\",\n \"total_input\": \"2.0\",\n \"total_output\": \"1.999859\",\n \"txid\": \"b1ae061be71d91de2b6e5f6c707547ad109dabb510b7dcddf20b96acb8175561\"\n }\n}" }, { "title": "Transfer all LBC to a specified account", "curl": "curl -d'{\"method\": \"account_fund\", \"params\": {\"from_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"everything\": true, \"broadcast\": true}}' http://localhost:5279/", "lbrynet": "lbrynet account fund --from_account=\"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\" --everything --broadcast", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_fund\", \"params\": {\"from_account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"everything\": true, \"broadcast\": true}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"0100000003615517b8ac960bf2dddcb710b5ab9d10ad4775706c5f6e2bde911de71b06aeb1000000006a473044022070ccee08cbef735f798b9b29f5fc0f3e6541c4c32178b8ef5d8f45c3789e29ff02203df22817e856760c6453e7c869832fa385ab5284e8f8bd97815ded9f2b2a14bb012103087f88b56b3724117f595eabb5ac5282b9e73842f11c47ad1d20de6d856e72b5ffffffff615517b8ac960bf2dddcb710b5ab9d10ad4775706c5f6e2bde911de71b06aeb1010000006b483045022100d183c50ecdff7371e9a411e0252bab59d3b55984ddfc0b8adaa32087b67114a8022020003d8932c41cd71006e00eb346fa624d64b634c45ff12b8c1b3a4fd8b38a58012103087f88b56b3724117f595eabb5ac5282b9e73842f11c47ad1d20de6d856e72b5ffffffff615517b8ac960bf2dddcb710b5ab9d10ad4775706c5f6e2bde911de71b06aeb1020000006a473044022072d277ab8cdbcc3bbe4ffdcf6e34c55ab0de334f5ffbb3417690ac6e6cfea20d02206302c2026251b84b420427baaf91edcd93ab26264aff676816bbc80b70e05c3301210232b7e745cbafb7b3506b71afe38ba9a1c049aec1426c3e5442b246604a14c5f6ffffffff015027eb0b000000001976a914d90aac550e652c485f8c75585fa9bdf7b56c0f0c88ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mnkqmWKPUwkeDuq7pndpfrzXFQDmH9QTBo\",\n \"amount\": \"0.75\",\n \"confirmations\": 1,\n \"height\": 208,\n \"nout\": 0,\n \"timestamp\": 1655141670,\n \"txid\": \"b1ae061be71d91de2b6e5f6c707547ad109dabb510b7dcddf20b96acb8175561\",\n \"type\": \"payment\"\n },\n {\n \"address\": \"mnkqmWKPUwkeDuq7pndpfrzXFQDmH9QTBo\",\n \"amount\": \"0.75\",\n \"confirmations\": 1,\n \"height\": 208,\n \"nout\": 1,\n \"timestamp\": 1655141670,\n \"txid\": \"b1ae061be71d91de2b6e5f6c707547ad109dabb510b7dcddf20b96acb8175561\",\n \"type\": \"payment\"\n },\n {\n \"address\": \"mgr1So3484shNr8kNMsavaohFwwRm9AnFU\",\n \"amount\": \"0.499859\",\n \"confirmations\": 1,\n \"height\": 208,\n \"nout\": 2,\n \"timestamp\": 1655141670,\n \"txid\": \"b1ae061be71d91de2b6e5f6c707547ad109dabb510b7dcddf20b96acb8175561\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"n1JZiGPzhiFUPTysS5PABHjdiRqfWNMSaX\",\n \"amount\": \"1.999604\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 0,\n \"timestamp\": null,\n \"txid\": \"40747da878033aa34fa25310b12e9625d820cfb9be384254863c59ed6bc3c0e8\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.000255\",\n \"total_input\": \"1.999859\",\n \"total_output\": \"1.999604\",\n \"txid\": \"40747da878033aa34fa25310b12e9625d820cfb9be384254863c59ed6bc3c0e8\"\n }\n}" } ] }, { "name": "account_list", "description": "List details of all of the accounts or a specific account.", "arguments": [ { "name": "account_id", "type": "str", "description": "If provided only the balance for this account will be given", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "accounts in specific wallet", "is_required": false }, { "name": "confirmations", "type": "int", "description": "required confirmations (default: 0)", "is_required": false }, { "name": "include_claims", "type": "bool", "description": "include claims, requires than a LBC account is specified (default: false)", "is_required": false }, { "name": "show_seed", "type": "bool", "description": "show the seed for the account", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"id\": \"account_id\",\n \"is_default\": \"this account is used by default\",\n \"ledger\": \"name of crypto currency and network\",\n \"name\": \"optional account name\",\n \"seed\": \"human friendly words from which account can be recreated\",\n \"encrypted\": \"if account is encrypted\",\n \"private_key\": \"extended private key\",\n \"public_key\": \"extended public key\",\n \"address_generator\": \"settings for generating addresses\",\n \"modified_on\": \"date of last modification to account settings\"\n }\n ]\n }", "examples": [ { "title": "List your accounts", "curl": "curl -d'{\"method\": \"account_list\", \"params\": {\"include_claims\": false, \"show_seed\": false}}' http://localhost:5279/", "lbrynet": "lbrynet account list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_list\", \"params\": {\"include_claims\": false, \"show_seed\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address_generator\": {\n \"change\": {\n \"gap\": 6,\n \"maximum_uses_per_address\": 1\n },\n \"name\": \"deterministic-chain\",\n \"receiving\": {\n \"gap\": 20,\n \"maximum_uses_per_address\": 1\n }\n },\n \"certificates\": 0,\n \"coins\": 10.0,\n \"encrypted\": false,\n \"id\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"is_default\": true,\n \"ledger\": \"lbc_regtest\",\n \"name\": \"Account #mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"public_key\": \"tpubD6NzVbkrYhZ4Yj1pAFUmbQ8bvpo4d5aQyWUzVPpagEiAddqLjxzdvfzpxEinbB2JRXCLkHJ8kSSz4scPnw3rnQeYLmXEsr9cXkQLV4ZjwUh\",\n \"satoshis\": 1000000000\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "account_max_address_gap", "description": "Finds ranges of consecutive addresses that are unused and returns the length\nof the longest such range: for change and receiving address chains. This is\nuseful to figure out ideal values to set for 'receiving_gap' and 'change_gap'\naccount settings.", "arguments": [ { "name": "account_id", "type": "str", "description": "account for which to get max gaps", "is_required": true }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(map) maximum gap for change and receiving addresses", "examples": [] }, { "name": "account_remove", "description": "Remove an existing account.", "arguments": [ { "name": "account_id", "type": "str", "description": "id of the account to remove", "is_required": true }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": " {\n \"id\": \"account_id\",\n \"is_default\": \"this account is used by default\",\n \"ledger\": \"name of crypto currency and network\",\n \"name\": \"optional account name\",\n \"seed\": \"human friendly words from which account can be recreated\",\n \"encrypted\": \"if account is encrypted\",\n \"private_key\": \"extended private key\",\n \"public_key\": \"extended public key\",\n \"address_generator\": \"settings for generating addresses\",\n \"modified_on\": \"date of last modification to account settings\"\n }", "examples": [ { "title": "Remove an account", "curl": "curl -d'{\"method\": \"account_remove\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"}}' http://localhost:5279/", "lbrynet": "lbrynet account remove mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_remove\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"address_generator\": {\n \"change\": {\n \"gap\": 6,\n \"maximum_uses_per_address\": 1\n },\n \"name\": \"deterministic-chain\",\n \"receiving\": {\n \"gap\": 20,\n \"maximum_uses_per_address\": 1\n }\n },\n \"encrypted\": false,\n \"id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"is_default\": false,\n \"ledger\": \"lbc_regtest\",\n \"modified_on\": 1655141657,\n \"name\": \"generated account\",\n \"private_key\": \"tprv8ZgxMBicQKsPdmLUTJtRsfwAeL527nfpJE1GhFFLzKtbuuzwYorQyq8guwgzFCQkqs8EuR2MGLkbh3xuEb1pFuAjx1Web8Vk3RXrrLVKwRU\",\n \"public_key\": \"tpubD6NzVbkrYhZ4XENGLxZ2H5bHDMaxH7risXc3ymHeQbgzkQFiBCg1AKkZ66UqGohv1bXg5579Z3r4omYeqMEd1Khhyuf4dssfmBzfiEce23o\",\n \"seed\": \"miss ready crop oval canyon such sing powder figure math noodle style\"\n }\n}" } ] }, { "name": "account_send", "description": "Send the same number of credits to multiple addresses from a specific account (or default account).", "arguments": [ { "name": "account_id", "type": "str", "description": "account to fund the transaction", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until tx has synced", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "account_set", "description": "Change various settings on an account.", "arguments": [ { "name": "account_id", "type": "str", "description": "id of the account to change", "is_required": true }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "default", "type": "bool", "description": "make this account the default", "is_required": false }, { "name": "new_name", "type": "str", "description": "new name for the account", "is_required": false }, { "name": "receiving_gap", "type": "int", "description": "set the gap for receiving addresses", "is_required": false }, { "name": "receiving_max_uses", "type": "int", "description": "set the maximum number of times to use a receiving address", "is_required": false }, { "name": "change_gap", "type": "int", "description": "set the gap for change addresses", "is_required": false }, { "name": "change_max_uses", "type": "int", "description": "set the maximum number of times to use a change address", "is_required": false } ], "returns": " {\n \"id\": \"account_id\",\n \"is_default\": \"this account is used by default\",\n \"ledger\": \"name of crypto currency and network\",\n \"name\": \"optional account name\",\n \"seed\": \"human friendly words from which account can be recreated\",\n \"encrypted\": \"if account is encrypted\",\n \"private_key\": \"extended private key\",\n \"public_key\": \"extended public key\",\n \"address_generator\": \"settings for generating addresses\",\n \"modified_on\": \"date of last modification to account settings\"\n }", "examples": [ { "title": "Modify maximum number of times a change address can be reused", "curl": "curl -d'{\"method\": \"account_set\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"default\": false, \"change_max_uses\": 10}}' http://localhost:5279/", "lbrynet": "lbrynet account set mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn --change_max_uses=10", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"account_set\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\", \"default\": false, \"change_max_uses\": 10}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"address_generator\": {\n \"change\": {\n \"gap\": 6,\n \"maximum_uses_per_address\": 10\n },\n \"name\": \"deterministic-chain\",\n \"receiving\": {\n \"gap\": 20,\n \"maximum_uses_per_address\": 1\n }\n },\n \"encrypted\": false,\n \"id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"is_default\": false,\n \"ledger\": \"lbc_regtest\",\n \"modified_on\": 1655141657,\n \"name\": \"new account\",\n \"private_key\": \"tprv8ZgxMBicQKsPdmLUTJtRsfwAeL527nfpJE1GhFFLzKtbuuzwYorQyq8guwgzFCQkqs8EuR2MGLkbh3xuEb1pFuAjx1Web8Vk3RXrrLVKwRU\",\n \"public_key\": \"tpubD6NzVbkrYhZ4XENGLxZ2H5bHDMaxH7risXc3ymHeQbgzkQFiBCg1AKkZ66UqGohv1bXg5579Z3r4omYeqMEd1Khhyuf4dssfmBzfiEce23o\",\n \"seed\": \"miss ready crop oval canyon such sing powder figure math noodle style\"\n }\n}" } ] } ] }, "address": { "doc": "List, generate and verify addresses.", "commands": [ { "name": "address_is_mine", "description": "Checks if an address is associated with the current wallet.", "arguments": [ { "name": "address", "type": "str", "description": "address to check", "is_required": true }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(bool) true, if address is associated with current wallet", "examples": [ { "title": "Check if address is mine", "curl": "curl -d'{\"method\": \"address_is_mine\", \"params\": {\"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\"}}' http://localhost:5279/", "lbrynet": "lbrynet address is_mine muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"address_is_mine\", \"params\": {\"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": true\n}" } ] }, { "name": "address_list", "description": "List account addresses or details of single address.", "arguments": [ { "name": "address", "type": "str", "description": "just show details for single address", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n \"an address in base58\"\n ]\n }", "examples": [ { "title": "List addresses in default account", "curl": "curl -d'{\"method\": \"address_list\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet address list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"address_list\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mhSwvz7Qfuh343S8WrXpPoPcaxbtw7QM5W\",\n \"pubkey\": \"tpubDA9GDAntyJu4NnUS2eNJcNYev2VpQi2g9SJZHw9jwfJLVkCvDqtTNJCxS2c25i4BbBZwGZWFBHgAbrQ77qzN2vAiqxwj2w7rTEsxFHv1oKV\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mhaZrhgfvGv49E47oDupq8wBppSNfhMFam\",\n \"pubkey\": \"tpubDA9GDAntyJu4H7sF54p2y2XUQKZecnakxWYJpSzRAnKgMxNip8xJVz7piyaZUwivQvz4RxbGpyn8NQPHqJmHsGDfkwXeHFsgJAHWP6Sqpxq\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"miK8WKVbTb25PHkuzGtgjMkUrmrWZe4d7U\",\n \"pubkey\": \"tpubDA9GDAntyJu4Vt4X61z35Cr2HxGTr4PDUpmX5abtbGQ2Zs5EJJ3kAwwhdyCC1EGx4NqiErEXzictM4SxnfQng7jEhiKofkQHkgysqQb6VyQ\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"miihJtUK4WKc6u3gxirkNmKTbPBvFUxt81\",\n \"pubkey\": \"tpubDA9GDAntyJu4JbNXGCuV83JHncjaq7VvaxuGBNNEuMRdk8aXKLQ7xNPncmrtzS5kjKqWQhW79YCve3EfUAvE6dsVn9DQXdWjq9KZQcn1cZs\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"pubkey\": \"tpubDA9GDAntyJu4K82qyamAJ31vsRiLmGAGQAvvjMWbyHf5yJFDQt8sVpGCftFYMmhME5bRYSXzogRbJkthKhHLsgxJURfNofnNiJbNKDiNQLk\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mnHsWiDjPYw27jPzMfqUFM9scdsgHQ6NJg\",\n \"pubkey\": \"tpubDA9GDAntyJu4mwA4W8hg3Wsf5PH3YWgbcoB4dr3Cf77KELoQeD8WfN4avxprcKKd2E4ivZGTJ45ntCimLTrQibBcanGxudpvqCiDfvCocM1\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mp2RdZPTyBs9WrtPodcrSf9rTBSRKCTBqR\",\n \"pubkey\": \"tpubDA9GDAntyJu4ftNy5YcBuVWmiCNsjsEtZHDi45Q3TKDqgaRsQTMCq1STgCV3KhcM3VWANdk2gPaSmF4Acf39WGgcwRjsbfYgCPUDGfkn5cQ\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mpKoa7UDF5mD7bupzZpdnyXxo13vPHAPAj\",\n \"pubkey\": \"tpubDA9GDAntyJu4ygvkf9qDcakZPBDGQzqZ3qkQLWT7odrTx3sEKwMU8S5YRs1siZVLGEGBkya6oMPKB4DDJ8idy78mMujvCJ9xg8MbRDn2zFk\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mqaQqa8sxKjoPtSiacuKZX61v2TASZiK7e\",\n \"pubkey\": \"tpubDA9GDAntyJu56YFCUSaYZTbzDQmLUtpKMhxLFP3BgnRa7QUEQnbZVEYUBSQ8yiMbUhA83Pt72b18NnyWE7EvFP7Nf2gY1ZDq9xTBM9cRcar\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mqdwaA91Fe888P29tm8XqDbutdq4jfmKzV\",\n \"pubkey\": \"tpubDA9GDAntyJu4tKNwyo8vLQ2KTvSTteofqjM4YDEgUxUd1n4o5iBme1QcYF799H7Svi1MiUkQsgsehfEoELY9zY5me6JuALY7fcDTdpL4Kks\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mqwYxGxBhfFodPpkCDGXK3eiSHpfRPi6SD\",\n \"pubkey\": \"tpubDA9GDAntyJu4EExRfNAiaX8BF4aMZodvmbW6fyohsh2KrJtEpsA3RYGpHadnFW2XNQ3AM8rMnrF8XzDett9gYAqcwgX5NfB7aGwZUzFnY7G\",\n \"used_times\": 1\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"ms5HaDddaUyAAMczcdCbYuTFkuBdAgQxeY\",\n \"pubkey\": \"tpubDA9GDAntyJu4djXrpu7QzDQyq3L9vYpePKx7oxhBnhb9iogFm5yhr35v6hY6F2nbTC1cNtNPMj9mdCGAzgRAPcp314RMpdu1vdZ8HbKGzsg\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mtq45ABFdcEBnuG8VjRfyhRW35rzWZa7U9\",\n \"pubkey\": \"tpubDA9GDAntyJu4cfSnoTMmBNQG7PS9nYkWih6c9DdVQ4BsdcYh77xvCqRnYfxUSzEepHt3CbRVQjknCpX3e7di9UdvZB2eUHDrp33EeudNqGW\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mtvRgoqoAZdrqrsTdGYmqKzByKuerqgupK\",\n \"pubkey\": \"tpubDA9GDAntyJu52jLmREuYqCNJmX1uzsAxWsYME8PoUK4wG9KrdDrNRdGwmSwB7XZewgBJ9wu31TeL1cWMwegUc5QzF6xtGcPhPXsuSBtu1DE\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mtwm7SVcxtxXHgkxwLfrPodkuo14wKgNrz\",\n \"pubkey\": \"tpubDA9GDAntyJu4YkkdNxGKiUABfQ1phUrbuQaVS7u7bE7xjKEMMfJ9ga1pb3kijvY6v7TuV9nTDCkng1XJcgBda3btMCh3Fwt9ngGwHRDXm2Z\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"pubkey\": \"tpubDA9GDAntyJu4MyJWHNPKXbWDmkmg1VVMSxumiRmBHQfQKqfr7BGiHRrP4pHXexmorYAXjHkzB4naVB8qp6Cr5JZBuBoD9QGWYmAM9NcQZqP\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mur916Vn2bk7ASmHqtVs294wgU6KeHVxmq\",\n \"pubkey\": \"tpubDA9GDAntyJu4Eixz66KLYzFsprgwbQ1CsTEZjy48cRRHt4Md4mfgJ8ps8h8sp8dg35rD3GD5WuUes9GB5ceVKJehHsZUCtcUxDbExrozfN1\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mv4KFMYgTdH3c2etDRvyybzKAxfCk3rHi5\",\n \"pubkey\": \"tpubDA9GDAntyJu4UuzqvQhgnXkPz2wJNL7zUWZLE7HYP7woQ62wuJnBNt6QLGw2Gc1WRJZ3sa558CsedMJEaFuMa6zUaVmoQysF6WwwT7yuhaF\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mvJRJRn2ZvdLwLsKuAyenCpBkY9QxbegGF\",\n \"pubkey\": \"tpubDA9GDAntyJu4qkyiyr8CRDo12dohq2BcBCY8M7qRELp6xXJncbhjdx4eMZNhSfHd5SWa2qYm4Aupot866XqZ9G2UznaN5PtWF1faNKqBxT4\",\n \"used_times\": 0\n },\n {\n \"account\": \"mhjFSotvK8b2eJHZw2Tgf8czVLDsG2tLyu\",\n \"address\": \"mwSmsEyeJE7CxzLvwoGqoHLWmd4r7nQeeJ\",\n \"pubkey\": \"tpubDA9GDAntyJu4S7YgeBYbzaaCUPadcuhYmgR2Ugrj7yuvkfpGt1qRYkA9uQE8ekfkN2cSU7ywTqRSXmYVm4DBxQM5PbfLE1FGh7vrfyTDZDv\",\n \"used_times\": 0\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 53,\n \"total_pages\": 3\n }\n}" }, { "title": "List addresses in specified account", "curl": "curl -d'{\"method\": \"address_list\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"}}' http://localhost:5279/", "lbrynet": "lbrynet address list --account_id=\"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"address_list\", \"params\": {\"account_id\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mgPgMEPSvErRFvVihFmUrSD6W84u4TA7Hv\",\n \"pubkey\": \"tpubDA9GDAntyJu4VrQXCBni5u3bPJu4qHp6SPKn4eanRKvXfRgQSXHPTT5fW74hcEG68sg2eUPpA1HZeqEQaHJgNZFBuDi6pGenPASE9rwpPkt\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mgr1So3484shNr8kNMsavaohFwwRm9AnFU\",\n \"pubkey\": \"tpubDA9GDAntyJu4GoY24GN59DWm4A3uux8Gv71kD1fw85Nt7o4ZtdACNjM5ZE2gSN1GYrGYXoz8mzXoQRbPUQs4JTqARTSt5Faco4HM41rJTgf\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mguUfyvvbYuEWiXxKWCWp8dsvphC374QKC\",\n \"pubkey\": \"tpubDA9GDAntyJu4MQvij6HQg6Wc2Q2uWj7CAVoMnqsueh118giBDp2TYv1fXdgV2tPPvmXtwDqf99RNrcSn5YXXcELhd9msNEPw9hLPucG6CpE\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mhBjCUYvnJw7pLYyfyG3xzDohfJmwMjntM\",\n \"pubkey\": \"tpubDA9GDAntyJu4Jgaa9TzJ652wFyuLwBgMgaDKNFbMGhu7oNTpkuuB36ALmtRvtFCkozkSrZogj4dcuW246R9m3Nsm2d69oacBzbeiuyPCtQt\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mhfBgZWfPDF7wD6h97admwDYdaNs1SXafX\",\n \"pubkey\": \"tpubDA9GDAntyJu4zZ1UjFJcgC2zx8okqHedbh1scNYtEThVR53SJKr32oGq2LjUoz35Q7gBqQRcv67JiLVjtes5jpuQbjiaEio9iUmHDyQQCfK\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mi2aTSX87vtAg1QJoW6sP61cByjWgmv9sf\",\n \"pubkey\": \"tpubDA9GDAntyJu4MqkRoL5tDyCk12Mah17UjSYoev1pGqe9M5ZsbdmNDpDCpadXX3sgSKfarF7EDoitqnr5ixCvr6ReAF6wdaKMKJnsrVdeNL4\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mjPQuMg9VLMfJBRMTLkK5tc1F4v83EZuSG\",\n \"pubkey\": \"tpubDA9GDAntyJu4REycnhURTm9svTtCshEKdSKfA2hdmDWaM4Nh13NbppVuQ37G5rPQ26B5rfEw8fMQk3hPGnZ62bCB2Qq7b7wXLnnGqqFTNcT\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mjwhcGupeMHK5vPM3SftKjyPycFbFLYw9E\",\n \"pubkey\": \"tpubDA9GDAntyJu4oKFXbsX3iGqRfBiVZqQKinVB28dzjTYJWwP4bEURX6ogBUhvXwUqTDXX84v1Z6gBeoKQxTs3XKt55fk72NHf1VyRYmWco44\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mm29WyoMPQPJq7QVJGjtbxTvmPW1jL4KbF\",\n \"pubkey\": \"tpubDA9GDAntyJu4etnuFGKHEnL5VnoeExkX3o1KxcFyQQGYgRRSmKSYGGQcZjwH4GSpjnLSVhguac3aBRQYqB1C3c2GcD63EeWo4JtMZhExRpe\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mm6ju92EAUoX3Kx8yKMRDvV1Yw8fnMCbwJ\",\n \"pubkey\": \"tpubDA9GDAntyJu4NNzPx3B2mGLM3eqA8ocs77UZeA15ny7XLE3gCJoYCFpF8i6UU5UQc2qDZVwnhmwyUpriaaTCTNWBqZVpKbaCEjKFfvaJuog\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mnBXVNxS4Xs8xg3AzrQbfcjtBSmivXoh4n\",\n \"pubkey\": \"tpubDA9GDAntyJu4TV1sWBpVyD9DWnsH4m6vLKsv7ZurM71veSumgBymKN7BGT7DqCd2udCpDEaeRLX2A9XfD8h84yaJTpCvBiQs9gnQDXKNgi1\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mpPkZYKWvde2ycPeBwCPW4n1ZC4Z9dti8f\",\n \"pubkey\": \"tpubDA9GDAntyJu4F6uszUizCBJBZLn78LwFFXQkcJ6PpqBouLU2iFKVwZ4GPY6r4QvYjrvJ8UPJ8hyemLcxecveUmFeEJmH27ZBC3xkabgfaPn\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mpURY7Y9WPTf4yStztmtdiqnJEip3rzKdz\",\n \"pubkey\": \"tpubDA9GDAntyJu522u6TeHwazDETcpokjNqmmoJkdVDzqrYp6HGDo5RQAWpgBFAM18D9dfrqdoT3SXYShDneC8RLqZySidcuFfh85LaAtTXHaq\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mpaJaBdSVWmhvAx5M1DaHt5zZTbXhFiiLE\",\n \"pubkey\": \"tpubDA9GDAntyJu4sqMt3KuJpS1jL77fsZq2on7goPwczD4kTRCZHaMSFdggg7NsrrJgRUnyPq2t4WmW9EEbf9j5hTbW8ZVQK5hvkxxnWkQW9zY\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mszRRxz2VQZKsAKthY9heSB5xwdakZ6gkw\",\n \"pubkey\": \"tpubDA9GDAntyJu4xMmBekaRLedHLNdx6ksxZajRv7imPf5XFpciQKXDRPCv5rJ9oUDWX2oGp8JccjG4p3TY66wVhxXd77sDD3qDGjnMWJ9ZyFs\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mu6r6EMyp6nnwmydoFDaokhsaKndxxW4HY\",\n \"pubkey\": \"tpubDA9GDAntyJu4LXiLsMzUnVbqkRZv2aVv83v3PrhKatPww5wMr1H6t2bGSNBxTaTbr2Bhf1Ppm8AiE6jnQemf4rrYv5Fz6XujqBQADAm1k74\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mykwrnckz6sZ2KEe5Wpagt7SjPJdXF6m2V\",\n \"pubkey\": \"tpubDA9GDAntyJu4FiP8EBnyFjKgM7BzYai8Q5HZSr2Z4SRy3j2Zs47ivz1KvNZKDqF33Ljs3mrBUFgKycnqKHNY3AKe3WKZj8kyEAuBZNuPtvH\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"myqynCQQm83z7bjz1rJpFzv8aZxheNoosq\",\n \"pubkey\": \"tpubDA9GDAntyJu4YzqpSUC4UfUEEgWMowZmbpJRhRFS6WyLnN1hADYUty5dWxnUg6NccpNmQFL9kBWnoBiLEMoJuSdjbNZwr3tA8LZgh5byreV\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mzSzaNd9c3QoGiXESZcxv9vmwP2V7ujZJg\",\n \"pubkey\": \"tpubDA9GDAntyJu4iUzB1QVZ9ckBNHsM1WTHubjkDHpvCmeJYXvMdw6WABx9uJWQ8nFpEz8gd6sWDxSkkh9wbTTx4QH1XaNN4KSxtqxB3BQcM9V\",\n \"used_times\": 0\n },\n {\n \"account\": \"mqVK8Df2cBxZgUnxGXXd8JhTm1LHvPofdn\",\n \"address\": \"mzjHGbQvE5orzqRjmJxsbKN7jedNGmv4QD\",\n \"pubkey\": \"tpubDA9GDAntyJu4SzcUV91pjutvrUw1m5gRyzDC5BZbhgzMCZ5tCcMPuH9Ej3BoHv4RdVB1U4LbahzGTUdVZFSHqPng2eg9PLVbde4w8J17nGW\",\n \"used_times\": 0\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 26,\n \"total_pages\": 2\n }\n}" } ] }, { "name": "address_unused", "description": "Return an address containing no balance, will create\na new address if there is none.", "arguments": [ { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": " \"an address in base58\"", "examples": [ { "title": "Get an unused address", "curl": "curl -d'{\"method\": \"address_unused\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet address unused", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"address_unused\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\"\n}" } ] } ] }, "blob": { "doc": "Blob management.", "commands": [ { "name": "blob_announce", "description": "Announce blobs to the DHT", "arguments": [ { "name": "blob_hash", "type": "str", "description": "announce a blob, specified by blob_hash", "is_required": false }, { "name": "stream_hash", "type": "str", "description": "announce all blobs associated with stream_hash", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "announce all blobs associated with sd_hash and the sd_hash itself", "is_required": false } ], "returns": "(bool) true if successful", "examples": [] }, { "name": "blob_clean", "description": "Deletes blobs to cleanup disk space", "arguments": [], "returns": "(bool) true if successful", "examples": [] }, { "name": "blob_delete", "description": "Delete a blob", "arguments": [ { "name": "blob_hash", "type": "str", "description": "blob hash of the blob to delete", "is_required": true } ], "returns": "(str) Success/fail message", "examples": [ { "title": "Delete a blob", "curl": "curl -d'{\"method\": \"blob_delete\", \"params\": {\"blob_hash\": \"d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\"}}' http://localhost:5279/", "lbrynet": "lbrynet blob delete d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"blob_delete\", \"params\": {\"blob_hash\": \"d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": \"Deleted d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\"\n}" } ] }, { "name": "blob_get", "description": "Download and return a blob", "arguments": [ { "name": "blob_hash", "type": "str", "description": "blob hash of the blob to get", "is_required": true }, { "name": "timeout", "type": "int", "description": "timeout in number of seconds", "is_required": false } ], "returns": "(str) Success/Fail message or (dict) decoded data", "examples": [] }, { "name": "blob_list", "description": "Returns blob hashes. If not given filters, returns all blobs known by the blob manager", "arguments": [ { "name": "needed", "type": "bool", "description": "only return needed blobs", "is_required": false }, { "name": "finished", "type": "bool", "description": "only return finished blobs", "is_required": false }, { "name": "uri", "type": "str", "description": "filter blobs by stream in a uri", "is_required": false }, { "name": "stream_hash", "type": "str", "description": "filter blobs by stream hash", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "filter blobs in a stream by sd hash, ie the hash of the stream descriptor blob for a stream that has been downloaded", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": "(list) List of blob hashes", "examples": [ { "title": "List your local blobs", "curl": "curl -d'{\"method\": \"blob_list\", \"params\": {\"needed\": false, \"finished\": false}}' http://localhost:5279/", "lbrynet": "lbrynet blob list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"blob_list\", \"params\": {\"needed\": false, \"finished\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n \"d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\",\n \"3413d65026fcabc798f477a457e205e96b3ac921176df62f79a9d90b9c5bed665bb3ad849c8b472c27d13cfbc6440e9f\",\n \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"536763bbb86446ef57a8a45493b12503ea4c74d865692343ff60c3b40431b5a8624341ae1e48aaf601057071cb25b7c3\"\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 4,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "blob_reflect", "description": "Reflects specified blobs", "arguments": [ { "name": "reflector_server", "type": "str", "description": "reflector address", "is_required": false } ], "returns": "(list) reflected blob hashes", "examples": [] }, { "name": "blob_reflect_all", "description": "Reflects all saved blobs", "arguments": [], "returns": "(bool) true if successful", "examples": [] } ] }, "channel": { "doc": "Create, update, abandon and list your channel claims.", "commands": [ { "name": "channel_abandon", "description": "Abandon one of my channel claims.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim_id of the claim to abandon", "is_required": false }, { "name": "txid", "type": "str", "description": "txid of the claim to abandon", "is_required": false }, { "name": "nout", "type": "int", "description": "nout of the claim to abandon", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until abandon is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Abandon a channel claim", "curl": "curl -d'{\"method\": \"channel_abandon\", \"params\": {\"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet channel abandon 595c2e2f0c1f59188628fab7503692b0145779a2", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"channel_abandon\", \"params\": {\"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"010000000164ffcff7cce034b5b519f047c9aa5c4cf0dc52d6e5804d74174140a5c52182ab000000006a4730440220554a49624aa9ba48ff6f6ea0e747e3f0fce88e9fbdaef0ec8336e8367dc8552802203988c9dc46b066b699fe7f93bd2fcad05cd86b309846f65b85c3b32ba98667c6012103687cff1a4970d2e992ba0035de7251dd822cb495c17ec0c5c4543a514227cd65ffffffff0134b7f505000000001976a91406c0b5a2fea95e096d4406846d1eb01a1993d5a388ac00000000\",\n \"inputs\": [\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 9,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mg8fCtz1uf7aZe8UVgzMcBPw1V3yEu4Qry\",\n \"amount\": \"0.999893\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 0,\n \"timestamp\": null,\n \"txid\": \"3691918e1a838c00b759609bfc89effdfab93860a9bc68d06edcc6ee0bb566d5\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.000107\",\n \"total_input\": \"1.0\",\n \"total_output\": \"0.999893\",\n \"txid\": \"3691918e1a838c00b759609bfc89effdfab93860a9bc68d06edcc6ee0bb566d5\"\n }\n}" } ] }, { "name": "channel_create", "description": "Create a new channel by generating a channel private key and establishing an '@' prefixed claim.", "arguments": [ { "name": "name", "type": "str", "description": "name of the channel prefixed with '@'", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": true }, { "name": "allow_duplicate_name", "type": "bool", "description": "create new channel even if one already exists with given name. default: false.", "is_required": false }, { "name": "title", "type": "str", "description": "title of the publication", "is_required": false }, { "name": "description", "type": "str", "description": "description of the publication", "is_required": false }, { "name": "email", "type": "str", "description": "email of channel owner", "is_required": false }, { "name": "website_url", "type": "str", "description": "website url", "is_required": false }, { "name": "featured", "type": "list", "description": "claim_ids of featured content in channel", "is_required": false }, { "name": "tags", "type": "list", "description": "content tags", "is_required": false }, { "name": "languages", "type": "list", "description": "languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant`", "is_required": false }, { "name": "locations", "type": "list", "description": "locations of the channel, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... \"locations\": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: \"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE\" making sure to include colon for blank values, for example to provide only the city: ... --locations=\"::Manchester\" with all values set: ... --locations=\"US:NH:Manchester:03101:42.990605:-71.460989\" optionally, you can just pass the \"LATITUDE:LONGITUDE\": ... --locations=\"42.990605:-71.460989\" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations=\"{'country': 'US', 'state': 'NH'}\"", "is_required": false }, { "name": "thumbnail_url", "type": "str", "description": "thumbnail url", "is_required": false }, { "name": "cover_url", "type": "str", "description": "url of cover image", "is_required": false }, { "name": "account_id", "type": "str", "description": "account to use for holding the transaction", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the channel is sent to, if not specified it will be determined automatically from the account", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Create a channel claim without metadata", "curl": "curl -d'{\"method\": \"channel_create\", \"params\": {\"name\": \"@channel\", \"bid\": \"1.0\", \"featured\": [], \"tags\": [], \"languages\": [], \"locations\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet channel create @channel 1.0", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"channel_create\", \"params\": {\"name\": \"@channel\", \"bid\": \"1.0\", \"featured\": [], \"tags\": [], \"languages\": [], \"locations\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"0100000001b8749850c50b2d456a0d54ab618c516f9f80a6f1dc7977dab4ca230e775931f0010000006a473044022041b5434aab5d73b41aad02687a657c9b99e1ef83fa8eabb08407359cbe6731e30220293f3f8ac7bf6ade85f4a181920b20d6d55b04acbce1edffc9ef7d1f94a88fdc012102ebd9926866ffd2ea504ee0ec7affe0f85f76e8a3c55149680ff74a63bd655123ffffffff0200e1f505000000004cb508406368616e6e656c260012230a2103bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e56d7576a914988d8d19fed515ed8e9d60a1c31c8aed750e441d88acc462a029000000001976a914b3aa073545c0d148feed1b89f24a3c7302c2f53188ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mhaZrhgfvGv49E47oDupq8wBppSNfhMFam\",\n \"amount\": \"7.999876\",\n \"confirmations\": 2,\n \"height\": 207,\n \"nout\": 1,\n \"timestamp\": 1655141670,\n \"txid\": \"f03159770e23cab4da7779dcf1a6809f6f518c61ab540d6a452d0bc5509874b8\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"create\",\n \"confirmations\": -2,\n \"has_signing_key\": true,\n \"height\": -2,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": null,\n \"txid\": \"de80e8db728f73b499eb384e3bd5688d66d522a3f516de4bf60e9fda2ba91964\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\"\n },\n \"value_type\": \"channel\"\n },\n {\n \"address\": \"mwtvw9x13nsVo6xkkrt3RFkk5h6TAjgPdK\",\n \"amount\": \"6.983769\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"de80e8db728f73b499eb384e3bd5688d66d522a3f516de4bf60e9fda2ba91964\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.016107\",\n \"total_input\": \"7.999876\",\n \"total_output\": \"7.983769\",\n \"txid\": \"de80e8db728f73b499eb384e3bd5688d66d522a3f516de4bf60e9fda2ba91964\"\n }\n}" }, { "title": "Create a channel claim with all metadata", "curl": "curl -d'{\"method\": \"channel_create\", \"params\": {\"name\": \"@bigchannel\", \"bid\": \"1.0\", \"title\": \"Big Channel\", \"description\": \"A channel with lots of videos.\", \"email\": \"creator@smallmedia.com\", \"website_url\": \"http://smallmedia.com\", \"featured\": [], \"tags\": [\"music\", \"art\"], \"languages\": [\"pt-BR\", \"uk\"], \"locations\": [\"BR\", \"UA::Kiyv\"], \"thumbnail_url\": \"http://smallmedia.com/logo.jpg\", \"cover_url\": \"http://smallmedia.com/logo.jpg\", \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet channel create @bigchannel 1.0 --title=\"Big Channel\" --description=\"A channel with lots of videos.\" --email=\"creator@smallmedia.com\" --tags=music --tags=art --languages=pt-BR --languages=uk --locations=BR --locations=UA::Kiyv --website_url=\"http://smallmedia.com\" --thumbnail_url=\"http://smallmedia.com/logo.jpg\" --cover_url=\"http://smallmedia.com/logo.jpg\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"channel_create\", \"params\": {\"name\": \"@bigchannel\", \"bid\": \"1.0\", \"title\": \"Big Channel\", \"description\": \"A channel with lots of videos.\", \"email\": \"creator@smallmedia.com\", \"website_url\": \"http://smallmedia.com\", \"featured\": [], \"tags\": [\"music\", \"art\"], \"languages\": [\"pt-BR\", \"uk\"], \"locations\": [\"BR\", \"UA::Kiyv\"], \"thumbnail_url\": \"http://smallmedia.com/logo.jpg\", \"cover_url\": \"http://smallmedia.com/logo.jpg\", \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"010000000164ffcff7cce034b5b519f047c9aa5c4cf0dc52d6e5804d74174140a5c52182ab010000006b483045022100d26fb9b49ef1a48d33c8639425c1bf216127a45b416ec6a51cb4e81bc8d3a4f002205b38ec4db4ac5e80801203b10b4d31e25775f259163b3f79e78da68eb5ba2b180121036a4e85fba940e22e77dae155444e4f11fa6b9436456a668f00cee5560e48eb36ffffffff0200e1f50500000000fd1701b50b406269676368616e6e656c4ced0012740a2102354550c6c7c2b7c777410102a90b1c3ffa1073517c1f60aa43306490a64a1089121663726561746f7240736d616c6c6d656469612e636f6d1a15687474703a2f2f736d616c6c6d656469612e636f6d22202a1e687474703a2f2f736d616c6c6d656469612e636f6d2f6c6f676f2e6a7067420b426967204368616e6e656c4a1e41206368616e6e656c2077697468206c6f7473206f6620766964656f732e52202a1e687474703a2f2f736d616c6c6d656469612e636f6d2f6c6f676f2e6a70675a056d757369635a0361727462050883011820620308ab016a0208206a0908e9011a044b6979766d7576a9149205f209a3c3a7d6025acbbded70700b686f948488acd22cd305000000001976a91499bb3c59d341b872794cea2524fd2884f83c655e88ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mnTTF4Sw8SGHi58DRCRSpYXJosZ61nPijQ\",\n \"amount\": \"1.9993635\",\n \"confirmations\": 1,\n \"height\": 210,\n \"nout\": 1,\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mtq45ABFdcEBnuG8VjRfyhRW35rzWZa7U9\",\n \"amount\": \"1.0\",\n \"claim_id\": \"6e536ae75030d1293b52d056a9bac51b3b02ed2a\",\n \"claim_op\": \"create\",\n \"confirmations\": -2,\n \"has_signing_key\": true,\n \"height\": -2,\n \"meta\": {},\n \"name\": \"@bigchannel\",\n \"normalized_name\": \"@bigchannel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@bigchannel#6e536ae75030d1293b52d056a9bac51b3b02ed2a\",\n \"timestamp\": null,\n \"txid\": \"acba29e905f1a37d12f8371a77e1b6f92c1a45c7ed396b1eceea97c154e9d2e3\",\n \"type\": \"claim\",\n \"value\": {\n \"cover\": {\n \"url\": \"http://smallmedia.com/logo.jpg\"\n },\n \"description\": \"A channel with lots of videos.\",\n \"email\": \"creator@smallmedia.com\",\n \"languages\": [\n \"pt-BR\",\n \"uk\"\n ],\n \"locations\": [\n {\n \"country\": \"BR\"\n },\n {\n \"city\": \"Kiyv\",\n \"country\": \"UA\"\n }\n ],\n \"public_key\": \"02354550c6c7c2b7c777410102a90b1c3ffa1073517c1f60aa43306490a64a1089\",\n \"public_key_id\": \"mpJbNo6rzLmfViJkVay6kxdqCrAfB1BDbe\",\n \"tags\": [\n \"music\",\n \"art\"\n ],\n \"thumbnail\": {\n \"url\": \"http://smallmedia.com/logo.jpg\"\n },\n \"title\": \"Big Channel\",\n \"website_url\": \"http://smallmedia.com\"\n },\n \"value_type\": \"channel\"\n },\n {\n \"address\": \"muXoymiRVLaS1FXJSgPjieSedrWPgUgEkK\",\n \"amount\": \"0.9772565\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"acba29e905f1a37d12f8371a77e1b6f92c1a45c7ed396b1eceea97c154e9d2e3\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.022107\",\n \"total_input\": \"1.9993635\",\n \"total_output\": \"1.9772565\",\n \"txid\": \"acba29e905f1a37d12f8371a77e1b6f92c1a45c7ed396b1eceea97c154e9d2e3\"\n }\n}" } ] }, { "name": "channel_export", "description": "Export channel private key.", "arguments": [ { "name": "channel_id", "type": "str", "description": "claim id of channel to export", "is_required": true }, { "name": "channel_name", "type": "str", "description": "name of channel to export", "is_required": false }, { "name": "account_id", "type": "str", "description": "one or more account ids for accounts to look in for channels, defaults to all accounts.", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(str) serialized channel private key", "examples": [] }, { "name": "channel_import", "description": "Import serialized channel private key (to allow signing new streams to the channel)", "arguments": [ { "name": "channel_data", "type": "str", "description": "serialized channel, as exported by channel export", "is_required": true }, { "name": "wallet_id", "type": "str", "description": "import into specific wallet", "is_required": false } ], "returns": "(dict) Result dictionary", "examples": [] }, { "name": "channel_list", "description": "List my channel claims.", "arguments": [ { "name": "name", "type": "str or list", "description": "channel name", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "channel id", "is_required": false }, { "name": "is_spent", "type": "bool", "description": "shows previous channel updates and abandons", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false }, { "name": "resolve", "type": "bool", "description": "resolves each channel to provide additional metadata", "is_required": false }, { "name": "no_totals", "type": "bool", "description": "do not calculate the total number of pages and items in result set (significant performance boost)", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [ { "title": "List your channel claims", "curl": "curl -d'{\"method\": \"channel_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"resolve\": false, \"no_totals\": false}}' http://localhost:5279/", "lbrynet": "lbrynet channel list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"channel_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"resolve\": false, \"no_totals\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"create\",\n \"confirmations\": 1,\n \"has_signing_key\": true,\n \"height\": 209,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"de80e8db728f73b499eb384e3bd5688d66d522a3f516de4bf60e9fda2ba91964\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\"\n },\n \"value_type\": \"channel\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" }, { "title": "Paginate your channel claims", "curl": "curl -d'{\"method\": \"channel_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"page\": 1, \"page_size\": 20, \"resolve\": false, \"no_totals\": false}}' http://localhost:5279/", "lbrynet": "lbrynet channel list --page=1 --page_size=20", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"channel_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"page\": 1, \"page_size\": 20, \"resolve\": false, \"no_totals\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"create\",\n \"confirmations\": 1,\n \"has_signing_key\": true,\n \"height\": 209,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"de80e8db728f73b499eb384e3bd5688d66d522a3f516de4bf60e9fda2ba91964\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\"\n },\n \"value_type\": \"channel\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "channel_sign", "description": "Signs data using the specified channel signing key.", "arguments": [ { "name": "channel_name", "type": "str", "description": "name of channel used to sign (or use channel id)", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of channel used to sign (or use channel name)", "is_required": false }, { "name": "hexdata", "type": "str", "description": "data to sign, encoded as hexadecimal", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(dict) Signature if successfully made, (None) or an error otherwise\n {\n \"signature\": (str) The signature of the comment,\n \"signing_ts\": (str) The timestamp used to sign the comment,\n }", "examples": [] }, { "name": "channel_update", "description": "Update an existing channel claim.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim_id of the channel to update", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": false }, { "name": "title", "type": "str", "description": "title of the publication", "is_required": false }, { "name": "description", "type": "str", "description": "description of the publication", "is_required": false }, { "name": "email", "type": "str", "description": "email of channel owner", "is_required": false }, { "name": "website_url", "type": "str", "description": "website url", "is_required": false }, { "name": "featured", "type": "list", "description": "claim_ids of featured content in channel", "is_required": false }, { "name": "clear_featured", "type": "bool", "description": "clear existing featured content (prior to adding new ones)", "is_required": false }, { "name": "tags", "type": "list", "description": "add content tags", "is_required": false }, { "name": "clear_tags", "type": "bool", "description": "clear existing tags (prior to adding new ones)", "is_required": false }, { "name": "languages", "type": "list", "description": "languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant`", "is_required": false }, { "name": "clear_languages", "type": "bool", "description": "clear existing languages (prior to adding new ones)", "is_required": false }, { "name": "locations", "type": "list", "description": "locations of the channel, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... \"locations\": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: \"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE\" making sure to include colon for blank values, for example to provide only the city: ... --locations=\"::Manchester\" with all values set: ... --locations=\"US:NH:Manchester:03101:42.990605:-71.460989\" optionally, you can just pass the \"LATITUDE:LONGITUDE\": ... --locations=\"42.990605:-71.460989\" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations=\"{'country': 'US', 'state': 'NH'}\"", "is_required": false }, { "name": "clear_locations", "type": "bool", "description": "clear existing locations (prior to adding new ones)", "is_required": false }, { "name": "thumbnail_url", "type": "str", "description": "thumbnail url", "is_required": false }, { "name": "cover_url", "type": "str", "description": "url of cover image", "is_required": false }, { "name": "account_id", "type": "str", "description": "account in which to look for channel (default: all)", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the channel is sent", "is_required": false }, { "name": "new_signing_key", "type": "bool", "description": "generate a new signing key, will invalidate all previous publishes", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false }, { "name": "replace", "type": "bool", "description": "instead of modifying specific values on the channel, this will clear all existing values and only save passed in values, useful for form submissions where all values are always set", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Update a channel claim", "curl": "curl -d'{\"method\": \"channel_update\", \"params\": {\"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"title\": \"New Channel\", \"featured\": [], \"clear_featured\": false, \"tags\": [], \"clear_tags\": false, \"languages\": [], \"clear_languages\": false, \"locations\": [], \"clear_locations\": false, \"new_signing_key\": false, \"funding_account_ids\": [], \"preview\": false, \"blocking\": false, \"replace\": false}}' http://localhost:5279/", "lbrynet": "lbrynet channel update 595c2e2f0c1f59188628fab7503692b0145779a2 --title=\"New Channel\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"channel_update\", \"params\": {\"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"title\": \"New Channel\", \"featured\": [], \"clear_featured\": false, \"tags\": [], \"clear_tags\": false, \"languages\": [], \"clear_languages\": false, \"locations\": [], \"clear_locations\": false, \"new_signing_key\": false, \"funding_account_ids\": [], \"preview\": false, \"blocking\": false, \"replace\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"01000000026419a92bda9f0ef64bde16f5a322d5668d68d53b4e38eb99b4738f72dbe880de000000006a47304402206e41c2509f2ff2099feb34189e34b30d1e5cb011a119c7d3a50ec48d8d49b987022035cb2af940fb1e1db687fc29e7575c22c3f1f799edd82d656b85a26fb653c78d012103687cff1a4970d2e992ba0035de7251dd822cb495c17ec0c5c4543a514227cd65ffffffffe8c0c36bed593c86544238beb9cf20d825962eb11053a24fa33a0378a87d7440000000006a47304402202b6601a98c683dfc99c2c1997156eb7bf6a45cc627d953b1b7246db450229c6e02201b74c5e5d27c7f3cf2770c7e308826d331dd75c2275e31f6828e42369fd2466b0121026dec9b64cb86838f381490252b2cfa34512e9673b65b6016734e92c1968762ecffffffff0200e1f505000000006eb708406368616e6e656c14a2795714b0923650b7fa288618591f0c2f2e5c59330012230a2103bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5420b4e6577204368616e6e656c6d6d76a914988d8d19fed515ed8e9d60a1c31c8aed750e441d88ac5ec9ea0b000000001976a9144c1f61ac2064778c7f08eda66437d2677befd3c088ac00000000\",\n \"inputs\": [\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"create\",\n \"confirmations\": 1,\n \"has_signing_key\": true,\n \"height\": 209,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"de80e8db728f73b499eb384e3bd5688d66d522a3f516de4bf60e9fda2ba91964\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\"\n },\n \"value_type\": \"channel\"\n },\n {\n \"address\": \"n1JZiGPzhiFUPTysS5PABHjdiRqfWNMSaX\",\n \"amount\": \"1.999604\",\n \"confirmations\": 1,\n \"height\": 209,\n \"nout\": 0,\n \"timestamp\": 1655141670,\n \"txid\": \"40747da878033aa34fa25310b12e9625d820cfb9be384254863c59ed6bc3c0e8\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": -2,\n \"has_signing_key\": true,\n \"height\": -2,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": null,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n {\n \"address\": \"mnTTF4Sw8SGHi58DRCRSpYXJosZ61nPijQ\",\n \"amount\": \"1.9993635\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.0002405\",\n \"total_input\": \"2.999604\",\n \"total_output\": \"2.9993635\",\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\"\n }\n}" } ] } ] }, "claim": { "doc": "List and search all types of claims.", "commands": [ { "name": "claim_list", "description": "List my stream and channel claims.", "arguments": [ { "name": "claim_type", "type": "str or list", "description": "claim type: channel, stream, repost, collection", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "claim id", "is_required": false }, { "name": "channel_id", "type": "str or list", "description": "streams in this channel", "is_required": false }, { "name": "name", "type": "str or list", "description": "claim name", "is_required": false }, { "name": "is_spent", "type": "bool", "description": "shows previous claim updates and abandons", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "has_source", "type": "bool", "description": "list claims containing a source field", "is_required": false }, { "name": "has_no_source", "type": "bool", "description": "list claims not containing a source field", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false }, { "name": "resolve", "type": "bool", "description": "resolves each claim to provide additional metadata", "is_required": false }, { "name": "order_by", "type": "str", "description": "field to order by: 'name', 'height', 'amount'", "is_required": false }, { "name": "no_totals", "type": "bool", "description": "do not calculate the total number of pages and items in result set (significant performance boost)", "is_required": false }, { "name": "include_received_tips", "type": "bool", "description": "calculate the amount of tips received for claim outputs", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [ { "title": "List all your claims", "curl": "curl -d'{\"method\": \"claim_list\", \"params\": {\"claim_type\": [], \"claim_id\": [], \"name\": [], \"is_spent\": false, \"channel_id\": [], \"has_source\": false, \"has_no_source\": false, \"resolve\": false, \"no_totals\": false, \"include_received_tips\": false}}' http://localhost:5279/", "lbrynet": "lbrynet claim list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"claim_list\", \"params\": {\"claim_type\": [], \"claim_id\": [], \"name\": [], \"is_spent\": false, \"channel_id\": [], \"has_source\": false, \"has_no_source\": false, \"resolve\": false, \"no_totals\": false, \"include_received_tips\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 1,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": true,\n \"height\": 210,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 2,\n \"total_pages\": 1\n }\n}" }, { "title": "Paginate your claims", "curl": "curl -d'{\"method\": \"claim_list\", \"params\": {\"claim_type\": [], \"claim_id\": [], \"name\": [], \"is_spent\": false, \"channel_id\": [], \"has_source\": false, \"has_no_source\": false, \"page\": 1, \"page_size\": 20, \"resolve\": false, \"no_totals\": false, \"include_received_tips\": false}}' http://localhost:5279/", "lbrynet": "lbrynet claim list --page=1 --page_size=20", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"claim_list\", \"params\": {\"claim_type\": [], \"claim_id\": [], \"name\": [], \"is_spent\": false, \"channel_id\": [], \"has_source\": false, \"has_no_source\": false, \"page\": 1, \"page_size\": 20, \"resolve\": false, \"no_totals\": false, \"include_received_tips\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 1,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": true,\n \"height\": 210,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 2,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "claim_search", "description": "Search for stream and channel claims on the blockchain.\n\nArguments marked with \"supports equality constraints\" allow prepending the\nvalue with an equality constraint such as '>', '>=', '<' and '<='\neg. --height=\">400000\" would limit results to only claims above 400k block height.\n\nThey also support multiple constraints passed as a list of the args described above.\neg. --release_time=[\">1000000\", \"<2000000\"]", "arguments": [ { "name": "name", "type": "str", "description": "claim name (normalized)", "is_required": false }, { "name": "text", "type": "str", "description": "full text search", "is_required": false }, { "name": "claim_id", "type": "str", "description": "full or partial claim id", "is_required": false }, { "name": "claim_ids", "type": "list", "description": "list of full claim ids", "is_required": false }, { "name": "txid", "type": "str", "description": "transaction id", "is_required": false }, { "name": "nout", "type": "str", "description": "position in the transaction", "is_required": false }, { "name": "channel", "type": "str", "description": "claims signed by this channel (argument is a URL which automatically gets resolved), see --channel_ids if you need to filter by multiple channels at the same time, includes claims with invalid signatures, use in conjunction with --valid_channel_signature", "is_required": false }, { "name": "channel_ids", "type": "list", "description": "claims signed by any of these channels (arguments must be claim ids of the channels), includes claims with invalid signatures, implies --has_channel_signature, use in conjunction with --valid_channel_signature", "is_required": false }, { "name": "not_channel_ids", "type": "list", "description": "exclude claims signed by any of these channels (arguments must be claim ids of the channels)", "is_required": false }, { "name": "has_channel_signature", "type": "bool", "description": "claims with a channel signature (valid or invalid)", "is_required": false }, { "name": "valid_channel_signature", "type": "bool", "description": "claims with a valid channel signature or no signature, use in conjunction with --has_channel_signature to only get claims with valid signatures", "is_required": false }, { "name": "invalid_channel_signature", "type": "bool", "description": "claims with invalid channel signature or no signature, use in conjunction with --has_channel_signature to only get claims with invalid signatures", "is_required": false }, { "name": "limit_claims_per_channel", "type": "int", "description": "only return up to the specified number of claims per channel", "is_required": false }, { "name": "is_controlling", "type": "bool", "description": "winning claims of their respective name", "is_required": false }, { "name": "public_key_id", "type": "str", "description": "only return channels having this public key id, this is the same key as used in the wallet file to map channel certificate private keys: {'public_key_id': 'private key'}", "is_required": false }, { "name": "height", "type": "int", "description": "last updated block height (supports equality constraints)", "is_required": false }, { "name": "timestamp", "type": "int", "description": "last updated timestamp (supports equality constraints)", "is_required": false }, { "name": "creation_height", "type": "int", "description": "created at block height (supports equality constraints)", "is_required": false }, { "name": "creation_timestamp", "type": "int", "description": "created at timestamp (supports equality constraints)", "is_required": false }, { "name": "activation_height", "type": "int", "description": "height at which claim starts competing for name (supports equality constraints)", "is_required": false }, { "name": "expiration_height", "type": "int", "description": "height at which claim will expire (supports equality constraints)", "is_required": false }, { "name": "release_time", "type": "int", "description": "limit to claims self-described as having been released to the public on or after this UTC timestamp, when claim does not provide a release time the publish time is used instead (supports equality constraints)", "is_required": false }, { "name": "amount", "type": "int", "description": "limit by claim value (supports equality constraints)", "is_required": false }, { "name": "support_amount", "type": "int", "description": "limit by supports and tips received (supports equality constraints)", "is_required": false }, { "name": "effective_amount", "type": "int", "description": "limit by total value (initial claim value plus all tips and supports received), this amount is blank until claim has reached activation height (supports equality constraints)", "is_required": false }, { "name": "trending_score", "type": "int", "description": "limit by trending score (supports equality constraints)", "is_required": false }, { "name": "trending_group", "type": "int", "description": "DEPRECATED - instead please use trending_score", "is_required": false }, { "name": "trending_mixed", "type": "int", "description": "DEPRECATED - instead please use trending_score", "is_required": false }, { "name": "trending_local", "type": "int", "description": "DEPRECATED - instead please use trending_score", "is_required": false }, { "name": "trending_global", "type": "int", "description": "DEPRECATED - instead please use trending_score", "is_required": false }, { "name": "reposted_claim_id", "type": "str", "description": "all reposts of the specified original claim id", "is_required": false }, { "name": "reposted", "type": "int", "description": "claims reposted this many times (supports equality constraints)", "is_required": false }, { "name": "claim_type", "type": "str", "description": "filter by 'channel', 'stream', 'repost' or 'collection'", "is_required": false }, { "name": "stream_types", "type": "list", "description": "filter by 'video', 'image', 'document', etc", "is_required": false }, { "name": "media_types", "type": "list", "description": "filter by 'video/mp4', 'image/png', etc", "is_required": false }, { "name": "fee_currency", "type": "string", "description": "specify fee currency: LBC, BTC, USD", "is_required": false }, { "name": "fee_amount", "type": "decimal", "description": "content download fee (supports equality constraints)", "is_required": false }, { "name": "duration", "type": "int", "description": "duration of video or audio in seconds (supports equality constraints)", "is_required": false }, { "name": "any_tags", "type": "list", "description": "find claims containing any of the tags", "is_required": false }, { "name": "all_tags", "type": "list", "description": "find claims containing every tag", "is_required": false }, { "name": "not_tags", "type": "list", "description": "find claims not containing any of these tags", "is_required": false }, { "name": "any_languages", "type": "list", "description": "find claims containing any of the languages", "is_required": false }, { "name": "all_languages", "type": "list", "description": "find claims containing every language", "is_required": false }, { "name": "not_languages", "type": "list", "description": "find claims not containing any of these languages", "is_required": false }, { "name": "any_locations", "type": "list", "description": "find claims containing any of the locations", "is_required": false }, { "name": "all_locations", "type": "list", "description": "find claims containing every location", "is_required": false }, { "name": "not_locations", "type": "list", "description": "find claims not containing any of these locations", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false }, { "name": "order_by", "type": "list", "description": "field to order by, default is descending order, to do an ascending order prepend ^ to the field name, eg. '^amount' available fields: 'name', 'height', 'release_time', 'publish_time', 'amount', 'effective_amount', 'support_amount', 'trending_group', 'trending_mixed', 'trending_local', 'trending_global', 'activation_height'", "is_required": false }, { "name": "no_totals", "type": "bool", "description": "do not calculate the total number of pages and items in result set (significant performance boost)", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "wallet to check for claim purchase receipts", "is_required": false }, { "name": "include_purchase_receipt", "type": "bool", "description": "lookup and include a receipt if this wallet has purchased the claim", "is_required": false }, { "name": "include_is_my_output", "type": "bool", "description": "lookup and include a boolean indicating if claim being resolved is yours", "is_required": false }, { "name": "remove_duplicates", "type": "bool", "description": "removes duplicated content from search by picking either the original claim or the oldest matching repost", "is_required": false }, { "name": "has_source", "type": "bool", "description": "find claims containing a source field", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "find claims where the source stream descriptor hash matches (partially or completely) the given hexadecimal string", "is_required": false }, { "name": "has_no_source", "type": "bool", "description": "find claims not containing a source field", "is_required": false }, { "name": "new_sdk_server", "type": "str", "description": "URL of the new SDK server (EXPERIMENTAL)", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [ { "title": "Search for all claims in channel", "curl": "curl -d'{\"method\": \"claim_search\", \"params\": {\"claim_ids\": [], \"channel\": \"@channel\", \"channel_ids\": [], \"not_channel_ids\": [], \"has_channel_signature\": false, \"valid_channel_signature\": false, \"invalid_channel_signature\": false, \"is_controlling\": false, \"stream_types\": [], \"media_types\": [], \"any_tags\": [], \"all_tags\": [], \"not_tags\": [], \"any_languages\": [], \"all_languages\": [], \"not_languages\": [], \"any_locations\": [], \"all_locations\": [], \"not_locations\": [], \"order_by\": [], \"no_totals\": false, \"include_purchase_receipt\": false, \"include_is_my_output\": false, \"remove_duplicates\": false, \"has_source\": false, \"has_no_source\": false}}' http://localhost:5279/", "lbrynet": "lbrynet claim search --channel=@channel", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"claim_search\", \"params\": {\"claim_ids\": [], \"channel\": \"@channel\", \"channel_ids\": [], \"not_channel_ids\": [], \"has_channel_signature\": false, \"valid_channel_signature\": false, \"invalid_channel_signature\": false, \"is_controlling\": false, \"stream_types\": [], \"media_types\": [], \"any_tags\": [], \"all_tags\": [], \"not_tags\": [], \"any_languages\": [], \"all_languages\": [], \"not_languages\": [], \"any_locations\": [], \"all_locations\": [], \"not_locations\": [], \"order_by\": [], \"no_totals\": false, \"include_purchase_receipt\": false, \"include_is_my_output\": false, \"remove_duplicates\": false, \"has_source\": false, \"has_no_source\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"blocked\": {\n \"channels\": [],\n \"total\": 0\n },\n \"items\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5/astream#a\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 1,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"meta\": {\n \"activation_height\": 214,\n \"creation_height\": 213,\n \"creation_timestamp\": 1655141671,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 714,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 213\n },\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"short_url\": \"lbry://astream#a\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": false,\n \"height\": 210,\n \"meta\": {\n \"activation_height\": 210,\n \"claims_in_channel\": 1,\n \"creation_height\": 209,\n \"creation_timestamp\": 1655141670,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 710,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 209\n },\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"short_url\": \"lbry://@channel#5\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" }, { "title": "Search for claims matching a name", "curl": "curl -d'{\"method\": \"claim_search\", \"params\": {\"name\": \"astream\", \"claim_ids\": [], \"channel_ids\": [], \"not_channel_ids\": [], \"has_channel_signature\": false, \"valid_channel_signature\": false, \"invalid_channel_signature\": false, \"is_controlling\": false, \"stream_types\": [], \"media_types\": [], \"any_tags\": [], \"all_tags\": [], \"not_tags\": [], \"any_languages\": [], \"all_languages\": [], \"not_languages\": [], \"any_locations\": [], \"all_locations\": [], \"not_locations\": [], \"order_by\": [], \"no_totals\": false, \"include_purchase_receipt\": false, \"include_is_my_output\": false, \"remove_duplicates\": false, \"has_source\": false, \"has_no_source\": false}}' http://localhost:5279/", "lbrynet": "lbrynet claim search --name=\"astream\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"claim_search\", \"params\": {\"name\": \"astream\", \"claim_ids\": [], \"channel_ids\": [], \"not_channel_ids\": [], \"has_channel_signature\": false, \"valid_channel_signature\": false, \"invalid_channel_signature\": false, \"is_controlling\": false, \"stream_types\": [], \"media_types\": [], \"any_tags\": [], \"all_tags\": [], \"not_tags\": [], \"any_languages\": [], \"all_languages\": [], \"not_languages\": [], \"any_locations\": [], \"all_locations\": [], \"not_locations\": [], \"order_by\": [], \"no_totals\": false, \"include_purchase_receipt\": false, \"include_is_my_output\": false, \"remove_duplicates\": false, \"has_source\": false, \"has_no_source\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"blocked\": {\n \"channels\": [],\n \"total\": 0\n },\n \"items\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5/astream#a\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 1,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"meta\": {\n \"activation_height\": 214,\n \"creation_height\": 213,\n \"creation_timestamp\": 1655141671,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 714,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 213\n },\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"short_url\": \"lbry://astream#a\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": false,\n \"height\": 210,\n \"meta\": {\n \"activation_height\": 210,\n \"claims_in_channel\": 1,\n \"creation_height\": 209,\n \"creation_timestamp\": 1655141670,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 710,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 209\n },\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"short_url\": \"lbry://@channel#5\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" } ] } ] }, "collection": { "doc": "Create, update, list, resolve, and abandon collections.", "commands": [ { "name": "collection_abandon", "description": "Abandon one of my collection claims.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim_id of the claim to abandon", "is_required": false }, { "name": "txid", "type": "str", "description": "txid of the claim to abandon", "is_required": false }, { "name": "nout", "type": "int", "description": "nout of the claim to abandon", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until abandon is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "collection_create", "description": "Create a new collection.", "arguments": [ { "name": "name", "type": "str", "description": "name of the collection", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": true }, { "name": "claims", "type": "list", "description": "claim ids to be included in the collection", "is_required": false }, { "name": "allow_duplicate_name", "type": "bool", "description": "create new collection even if one already exists with given name. default: false.", "is_required": false }, { "name": "title", "type": "str", "description": "title of the collection", "is_required": false }, { "name": "description", "type": "str", "description": "description of the collection", "is_required": false }, { "name": "tags", "type": "list", "description": "content tags", "is_required": false }, { "name": "clear_languages", "type": "bool", "description": "clear existing languages (prior to adding new ones)", "is_required": false }, { "name": "languages", "type": "list", "description": "languages used by the collection, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant`", "is_required": false }, { "name": "locations", "type": "list", "description": "locations of the collection, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... \"locations\": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: \"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE\" making sure to include colon for blank values, for example to provide only the city: ... --locations=\"::Manchester\" with all values set: ... --locations=\"US:NH:Manchester:03101:42.990605:-71.460989\" optionally, you can just pass the \"LATITUDE:LONGITUDE\": ... --locations=\"42.990605:-71.460989\" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations=\"{'country': 'US', 'state': 'NH'}\"", "is_required": false }, { "name": "thumbnail_url", "type": "str", "description": "thumbnail url", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of the publisher channel", "is_required": false }, { "name": "channel_name", "type": "str", "description": "name of the publisher channel", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "account_id", "type": "str", "description": "account to use for holding the transaction", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the collection is sent to, if not specified it will be determined automatically from the account", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Create a collection of one stream", "curl": "curl -d'{\"method\": \"collection_create\", \"params\": {\"name\": \"tom\", \"bid\": \"1.0\", \"claims\": [\"ad25e05aa7dc5e9994869040c6103f9a8728db46\"], \"allow_duplicate_name\": false, \"tags\": [], \"languages\": [], \"locations\": [], \"channel_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet collection create --name=tom --bid=1.0 --channel_id=595c2e2f0c1f59188628fab7503692b0145779a2 --claims=ad25e05aa7dc5e9994869040c6103f9a8728db46", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"collection_create\", \"params\": {\"name\": \"tom\", \"bid\": \"1.0\", \"claims\": [\"ad25e05aa7dc5e9994869040c6103f9a8728db46\"], \"allow_duplicate_name\": false, \"tags\": [], \"languages\": [], \"locations\": [], \"channel_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"01000000012c1bc49ac50486afeda93e5b8bff99abc845c446215a05d57c24dc3d522dd304010000006b483045022100ced18ea9fe454cf44d94bb9a5d7e6915030fb6c8f79c0df2f68cbbdd30f5913702204c6d7aded8529aef37f9e800c7e4159c02ed6cf1308c4bac32726b76902733fb0121023bf6d860bb1c3b7c71d9d9c985a56bdba1c6f754fa351805f02d89a87ff0222cffffffff0200e1f5050000000091b503746f6d4c6f01a2795714b0923650b7fa288618591f0c2f2e5c591bb9bb615fa343bdb5e2194aa8b18986d5f001413697412ec4b5f1a529c971e90a63a27ebf78a5e512d5f1b87dde99e6e40aa4e9b710240536ec62c3f1b56df51a1812160a1446db28879a3f10c640908694995edca75ae025ad6d7576a914934ab2faddc49cbc3cad5a805f3e79c5c502d14888ac202c7e17000000001976a914e9d5fbcc88ada7ba2a27cdf59e70c57b339cbf9f88ac00000000\",\n \"inputs\": [\n {\n \"address\": \"n44m2TuZKypifAkpPQqEN4VtRkCxgbgkA3\",\n \"amount\": \"4.947555\",\n \"confirmations\": 2,\n \"height\": 215,\n \"nout\": 1,\n \"timestamp\": 1655141672,\n \"txid\": \"04d32d523ddc247cd5055a2146c445c8ab99ff8b5b3ea9edaf8604c59ac41b2c\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mtwm7SVcxtxXHgkxwLfrPodkuo14wKgNrz\",\n \"amount\": \"1.0\",\n \"claim_id\": \"bd463bad61b8e420258f96e817b9540fa134ed23\",\n \"claim_op\": \"create\",\n \"confirmations\": -2,\n \"height\": -2,\n \"is_channel_signature_valid\": true,\n \"meta\": {},\n \"name\": \"tom\",\n \"normalized_name\": \"tom\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://tom#bd463bad61b8e420258f96e817b9540fa134ed23\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 7,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": null,\n \"txid\": \"a9954fce4be4598afcead007cec8f64bbcc8a899080f99bd1a0bfe07f2b501c7\",\n \"type\": \"claim\",\n \"value\": {\n \"claims\": [\n \"ad25e05aa7dc5e9994869040c6103f9a8728db46\"\n ]\n },\n \"value_type\": \"collection\"\n },\n {\n \"address\": \"n2qN5bEHEQ1keWyR6GbNd15dbHE6z1FoZb\",\n \"amount\": \"3.941448\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"a9954fce4be4598afcead007cec8f64bbcc8a899080f99bd1a0bfe07f2b501c7\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.006107\",\n \"total_input\": \"4.947555\",\n \"total_output\": \"4.941448\",\n \"txid\": \"a9954fce4be4598afcead007cec8f64bbcc8a899080f99bd1a0bfe07f2b501c7\"\n }\n}" } ] }, { "name": "collection_list", "description": "List my collection claims.", "arguments": [ { "name": "resolve", "type": "bool", "description": "resolve collection claim", "is_required": false }, { "name": "resolve_claims", "type": "int", "description": "resolve every claim", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [ { "title": "List collections", "curl": "curl -d'{\"method\": \"collection_list\", \"params\": {\"resolve_claims\": 1, \"resolve\": true}}' http://localhost:5279/", "lbrynet": "lbrynet collection list --resolve --resolve_claims=1", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"collection_list\", \"params\": {\"resolve_claims\": 1, \"resolve\": true}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address\": \"mtwm7SVcxtxXHgkxwLfrPodkuo14wKgNrz\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5/tom#b\",\n \"claim_id\": \"bd463bad61b8e420258f96e817b9540fa134ed23\",\n \"claim_op\": \"create\",\n \"claims\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5/astream#a\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 4,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"meta\": {\n \"activation_height\": 214,\n \"creation_height\": 213,\n \"creation_timestamp\": 1655141671,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 714,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 213\n },\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"short_url\": \"lbry://astream#a\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"canonical_url\": \"lbry://@channel#5\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 8,\n \"has_signing_key\": false,\n \"height\": 210,\n \"meta\": {\n \"activation_height\": 210,\n \"claims_in_channel\": 2,\n \"creation_height\": 209,\n \"creation_timestamp\": 1655141670,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 710,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 209\n },\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"short_url\": \"lbry://@channel#5\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n }\n ],\n \"confirmations\": 1,\n \"height\": 217,\n \"is_channel_signature_valid\": true,\n \"meta\": {\n \"activation_height\": 217,\n \"creation_height\": 217,\n \"creation_timestamp\": 1655141675,\n \"effective_amount\": \"1.0\",\n \"expiration_height\": 717,\n \"is_controlling\": true,\n \"reposted\": 0,\n \"support_amount\": \"0.0\",\n \"take_over_height\": 217\n },\n \"name\": \"tom\",\n \"normalized_name\": \"tom\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://tom#bd463bad61b8e420258f96e817b9540fa134ed23\",\n \"short_url\": \"lbry://tom#b\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 8,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141675,\n \"txid\": \"a9954fce4be4598afcead007cec8f64bbcc8a899080f99bd1a0bfe07f2b501c7\",\n \"type\": \"claim\",\n \"value\": {\n \"claims\": [\n \"ad25e05aa7dc5e9994869040c6103f9a8728db46\"\n ]\n },\n \"value_type\": \"collection\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "collection_resolve", "description": "Resolve claims in the collection.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim id of the collection", "is_required": false }, { "name": "url", "type": "str", "description": "url of the collection", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [] }, { "name": "collection_update", "description": "Update an existing collection claim.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim_id of the collection to update", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": false }, { "name": "claims", "type": "list", "description": "claim ids", "is_required": false }, { "name": "clear_claims", "type": "bool", "description": "clear existing claim references (prior to adding new ones)", "is_required": false }, { "name": "title", "type": "str", "description": "title of the collection", "is_required": false }, { "name": "description", "type": "str", "description": "description of the collection", "is_required": false }, { "name": "tags", "type": "list", "description": "add content tags", "is_required": false }, { "name": "clear_tags", "type": "bool", "description": "clear existing tags (prior to adding new ones)", "is_required": false }, { "name": "languages", "type": "list", "description": "languages used by the collection, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant`", "is_required": false }, { "name": "clear_languages", "type": "bool", "description": "clear existing languages (prior to adding new ones)", "is_required": false }, { "name": "locations", "type": "list", "description": "locations of the collection, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... \"locations\": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: \"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE\" making sure to include colon for blank values, for example to provide only the city: ... --locations=\"::Manchester\" with all values set: ... --locations=\"US:NH:Manchester:03101:42.990605:-71.460989\" optionally, you can just pass the \"LATITUDE:LONGITUDE\": ... --locations=\"42.990605:-71.460989\" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations=\"{'country': 'US', 'state': 'NH'}\"", "is_required": false }, { "name": "clear_locations", "type": "bool", "description": "clear existing locations (prior to adding new ones)", "is_required": false }, { "name": "thumbnail_url", "type": "str", "description": "thumbnail url", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of the publisher channel", "is_required": false }, { "name": "channel_name", "type": "str", "description": "name of the publisher channel", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "account_id", "type": "str", "description": "account in which to look for collection (default: all)", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the collection is sent", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false }, { "name": "replace", "type": "bool", "description": "instead of modifying specific values on the collection, this will clear all existing values and only save passed in values, useful for form submissions where all values are always set", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] } ] }, "file": { "doc": "File management.", "commands": [ { "name": "file_delete", "description": "Delete a LBRY file", "arguments": [ { "name": "delete_from_download_dir", "type": "bool", "description": "delete file from download directory, instead of just deleting blobs", "is_required": false }, { "name": "delete_all", "type": "bool", "description": "if there are multiple matching files, allow the deletion of multiple files. Otherwise do not delete anything.", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "delete by file sd hash", "is_required": false }, { "name": "file_name", "type": "str", "description": "delete by file name in downloads folder", "is_required": false }, { "name": "stream_hash", "type": "str", "description": "delete by file stream hash", "is_required": false }, { "name": "rowid", "type": "int", "description": "delete by file row id", "is_required": false }, { "name": "claim_id", "type": "str", "description": "delete by file claim id", "is_required": false }, { "name": "txid", "type": "str", "description": "delete by file claim txid", "is_required": false }, { "name": "nout", "type": "int", "description": "delete by file claim nout", "is_required": false }, { "name": "claim_name", "type": "str", "description": "delete by file claim name", "is_required": false }, { "name": "channel_claim_id", "type": "str", "description": "delete by file channel claim id", "is_required": false }, { "name": "channel_name", "type": "str", "description": "delete by file channel claim name", "is_required": false } ], "returns": "(bool) true if deletion was successful", "examples": [ { "title": "Delete a file", "curl": "curl -d'{\"method\": \"file_delete\", \"params\": {\"delete_from_download_dir\": false, \"delete_all\": false, \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\"}}' http://localhost:5279/", "lbrynet": "lbrynet file delete --claim_id=\"ad25e05aa7dc5e9994869040c6103f9a8728db46\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"file_delete\", \"params\": {\"delete_from_download_dir\": false, \"delete_all\": false, \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": true\n}" } ] }, { "name": "file_list", "description": "List files limited by optional filters", "arguments": [ { "name": "sd_hash", "type": "str", "description": "get file with matching sd hash", "is_required": false }, { "name": "file_name", "type": "str", "description": "get file with matching file name in the downloads folder", "is_required": false }, { "name": "stream_hash", "type": "str", "description": "get file with matching stream hash", "is_required": false }, { "name": "rowid", "type": "int", "description": "get file with matching row id", "is_required": false }, { "name": "added_on", "type": "int", "description": "get file with matching time of insertion", "is_required": false }, { "name": "claim_id", "type": "str", "description": "get file with matching claim id(s)", "is_required": false }, { "name": "outpoint", "type": "str", "description": "get file with matching claim outpoint(s)", "is_required": false }, { "name": "txid", "type": "str", "description": "get file with matching claim txid", "is_required": false }, { "name": "nout", "type": "int", "description": "get file with matching claim nout", "is_required": false }, { "name": "channel_claim_id", "type": "str", "description": "get file with matching channel claim id(s)", "is_required": false }, { "name": "channel_name", "type": "str", "description": "get file with matching channel name", "is_required": false }, { "name": "claim_name", "type": "str", "description": "get file with matching claim name", "is_required": false }, { "name": "blobs_in_stream", "type": "int", "description": "get file with matching blobs in stream", "is_required": false }, { "name": "download_path", "type": "str", "description": "get file with matching download path", "is_required": false }, { "name": "uploading_to_reflector", "type": "bool", "description": "get files currently uploading to reflector", "is_required": false }, { "name": "is_fully_reflected", "type": "bool", "description": "get files that have been uploaded to reflector", "is_required": false }, { "name": "status", "type": "str", "description": "match by status, ( running | finished | stopped )", "is_required": false }, { "name": "completed", "type": "bool", "description": "match only completed", "is_required": false }, { "name": "blobs_remaining", "type": "int", "description": "amount of remaining blobs to download", "is_required": false }, { "name": "sort", "type": "str", "description": "field to sort by (one of the above filter fields)", "is_required": false }, { "name": "comparison", "type": "str", "description": "logical comparison, (eq | ne | g | ge | l | le | in)", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "add purchase receipts from this wallet", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"streaming_url\": \"(str) url to stream the file using range requests\",\n \"completed\": \"(bool) true if download is completed\",\n \"file_name\": \"(str) name of file\",\n \"download_directory\": \"(str) download directory\",\n \"points_paid\": \"(float) credit paid to download file\",\n \"stopped\": \"(bool) true if download is stopped\",\n \"stream_hash\": \"(str) stream hash of file\",\n \"stream_name\": \"(str) stream name\",\n \"suggested_file_name\": \"(str) suggested file name\",\n \"sd_hash\": \"(str) sd hash of file\",\n \"download_path\": \"(str) download path of file\",\n \"mime_type\": \"(str) mime type of file\",\n \"key\": \"(str) key attached to file\",\n \"total_bytes_lower_bound\": \"(int) lower bound file size in bytes\",\n \"total_bytes\": \"(int) file upper bound size in bytes\",\n \"written_bytes\": \"(int) written size in bytes\",\n \"blobs_completed\": \"(int) number of fully downloaded blobs\",\n \"blobs_in_stream\": \"(int) total blobs on stream\",\n \"blobs_remaining\": \"(int) total blobs remaining to download\",\n \"status\": \"(str) downloader status\",\n \"claim_id\": \"(str) None if claim is not found else the claim id\",\n \"txid\": \"(str) None if claim is not found else the transaction id\",\n \"nout\": \"(int) None if claim is not found else the transaction output index\",\n \"outpoint\": \"(str) None if claim is not found else the tx and output\",\n \"metadata\": \"(dict) None if claim is not found else the claim metadata\",\n \"channel_claim_id\": \"(str) None if claim is not found or not signed\",\n \"channel_name\": \"(str) None if claim is not found or not signed\",\n \"claim_name\": \"(str) None if claim is not found else the claim name\",\n \"reflector_progress\": \"(int) reflector upload progress, 0 to 100\",\n \"uploading_to_reflector\": \"(bool) set to True when currently uploading to reflector\"\n }\n ]\n }", "examples": [ { "title": "List local files", "curl": "curl -d'{\"method\": \"file_list\", \"params\": {\"reverse\": false}}' http://localhost:5279/", "lbrynet": "lbrynet file list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"file_list\", \"params\": {\"reverse\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"added_on\": null,\n \"blobs_completed\": 1,\n \"blobs_in_stream\": 1,\n \"blobs_remaining\": 0,\n \"channel_claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"channel_name\": \"@channel\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_name\": \"astream\",\n \"completed\": true,\n \"confirmations\": -1,\n \"content_fee\": null,\n \"download_directory\": null,\n \"download_path\": null,\n \"file_name\": null,\n \"height\": -1,\n \"is_fully_reflected\": true,\n \"key\": \"66f888fe00cf558494c2fcbd5903d00d\",\n \"metadata\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"mime_type\": \"application/octet-stream\",\n \"nout\": 0,\n \"outpoint\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234:0\",\n \"points_paid\": 0.0,\n \"protobuf\": \"01a2795714b0923650b7fa288618591f0c2f2e5c5961baf110460a527e38b68f0653c1c79f2cae1f57ade55c009fb7578175c9d93d7edb0546d1378e1d33d99df8df0d5cfe9e7a5b63053a4e7ce003f95f5890b27f0a90010a8d010a30fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd120b746d707238333268703178180b22186170706c69636174696f6e2f6f637465742d73747265616d32309ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"purchase_receipt\": null,\n \"reflector_progress\": 0,\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"status\": \"finished\",\n \"stopped\": true,\n \"stream_hash\": \"c48ff9950efbcb78b20d311467b1b0e321069ef5ece96898eb08a27a62d1ef2594cd24f1e8d6993f00c48fd6a4221490\",\n \"stream_name\": \"tmpr832hp1x\",\n \"streaming_url\": \"http://localhost:5280/stream/9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"suggested_file_name\": \"tmpr832hp1x\",\n \"timestamp\": null,\n \"total_bytes\": 16,\n \"total_bytes_lower_bound\": 0,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"uploading_to_reflector\": false,\n \"written_bytes\": 0\n },\n {\n \"added_on\": null,\n \"blobs_completed\": 1,\n \"blobs_in_stream\": 1,\n \"blobs_remaining\": 0,\n \"channel_claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"channel_name\": \"@channel\",\n \"claim_id\": \"46bff43fbdffabf12efc867c2b4c7865315b0b76\",\n \"claim_name\": \"blank-image\",\n \"completed\": false,\n \"confirmations\": -1,\n \"content_fee\": null,\n \"download_directory\": null,\n \"download_path\": null,\n \"file_name\": null,\n \"height\": -1,\n \"is_fully_reflected\": true,\n \"key\": \"b2c6c26d1094b988fbf3313086f2a256\",\n \"metadata\": {\n \"author\": \"Picaso\",\n \"description\": \"A blank PNG that is 5x7.\",\n \"fee\": {\n \"address\": \"mnHsWiDjPYw27jPzMfqUFM9scdsgHQ6NJg\",\n \"amount\": \"0.3\",\n \"currency\": \"LBC\"\n },\n \"image\": {\n \"height\": 7,\n \"width\": 5\n },\n \"languages\": [\n \"en\"\n ],\n \"license\": \"Public Domain\",\n \"license_url\": \"http://public-domain.org\",\n \"locations\": [\n {\n \"city\": \"Manchester\",\n \"country\": \"US\",\n \"state\": \"NH\"\n }\n ],\n \"release_time\": \"1655141671\",\n \"source\": {\n \"hash\": \"6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545\",\n \"media_type\": \"image/png\",\n \"name\": \"tmps0do5cfj.png\",\n \"sd_hash\": \"d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\",\n \"size\": \"99\"\n },\n \"stream_type\": \"image\",\n \"tags\": [\n \"blank\",\n \"art\"\n ],\n \"thumbnail\": {\n \"url\": \"http://smallmedia.com/thumbnail.jpg\"\n },\n \"title\": \"Blank Image\"\n },\n \"mime_type\": \"image/png\",\n \"nout\": 0,\n \"outpoint\": \"04d32d523ddc247cd5055a2146c445c8ab99ff8b5b3ea9edaf8604c59ac41b2c:0\",\n \"points_paid\": 0.0,\n \"protobuf\": \"01a2795714b0923650b7fa288618591f0c2f2e5c5946580bd06bec18e960285651ca9130c75fcf4f5b3cd71fa5d97fe7013de1bf7627b08de186dc0f5c4ff33a6e7173b24f7da6a91080ffb840f6f6fbf4897851790ae6010a82010a306c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545120f746d707330646f3563666a2e706e6718632209696d6167652f706e673230d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996120650696361736f1a0d5075626c696320446f6d61696e2218687474703a2f2f7075626c69632d646f6d61696e2e6f726728a7ea9d95063222080112196f4a4f7b3ab55c52fe12f9d8b9b255fb2625522e4b00415bed188087a70e520408051007420b426c616e6b20496d6167654a184120626c616e6b20504e472074686174206973203578372e52252a23687474703a2f2f736d616c6c6d656469612e636f6d2f7468756d626e61696c2e6a70675a05626c616e6b5a03617274620208016a1308ec0112024e481a0a4d616e63686573746572\",\n \"purchase_receipt\": null,\n \"reflector_progress\": 0,\n \"sd_hash\": \"d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\",\n \"status\": \"finished\",\n \"stopped\": true,\n \"stream_hash\": \"eec0d9d4bf368e6e8abe11a0ceaf6f27606f11c1868d415e1470baa316400d6517f6092974ec5c58018b30b3e8e5b884\",\n \"stream_name\": \"tmps0do5cfj.png\",\n \"streaming_url\": \"http://localhost:5280/stream/d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\",\n \"suggested_file_name\": \"tmps0do5cfj.png\",\n \"timestamp\": null,\n \"total_bytes\": 112,\n \"total_bytes_lower_bound\": 96,\n \"txid\": \"04d32d523ddc247cd5055a2146c445c8ab99ff8b5b3ea9edaf8604c59ac41b2c\",\n \"uploading_to_reflector\": false,\n \"written_bytes\": 0\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 2,\n \"total_pages\": 1\n }\n}" }, { "title": "List files matching a parameter", "curl": "curl -d'{\"method\": \"file_list\", \"params\": {\"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\", \"reverse\": false}}' http://localhost:5279/", "lbrynet": "lbrynet file list --claim_id=\"ad25e05aa7dc5e9994869040c6103f9a8728db46\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"file_list\", \"params\": {\"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\", \"reverse\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"added_on\": null,\n \"blobs_completed\": 1,\n \"blobs_in_stream\": 1,\n \"blobs_remaining\": 0,\n \"channel_claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"channel_name\": \"@channel\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_name\": \"astream\",\n \"completed\": true,\n \"confirmations\": 4,\n \"content_fee\": null,\n \"download_directory\": null,\n \"download_path\": null,\n \"file_name\": null,\n \"height\": 214,\n \"is_fully_reflected\": true,\n \"key\": \"66f888fe00cf558494c2fcbd5903d00d\",\n \"metadata\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"mime_type\": \"application/octet-stream\",\n \"nout\": 0,\n \"outpoint\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234:0\",\n \"points_paid\": 0.0,\n \"protobuf\": \"01a2795714b0923650b7fa288618591f0c2f2e5c5961baf110460a527e38b68f0653c1c79f2cae1f57ade55c009fb7578175c9d93d7edb0546d1378e1d33d99df8df0d5cfe9e7a5b63053a4e7ce003f95f5890b27f0a90010a8d010a30fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd120b746d707238333268703178180b22186170706c69636174696f6e2f6f637465742d73747265616d32309ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"purchase_receipt\": null,\n \"reflector_progress\": 0,\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"status\": \"finished\",\n \"stopped\": true,\n \"stream_hash\": \"c48ff9950efbcb78b20d311467b1b0e321069ef5ece96898eb08a27a62d1ef2594cd24f1e8d6993f00c48fd6a4221490\",\n \"stream_name\": \"tmpr832hp1x\",\n \"streaming_url\": \"http://localhost:5280/stream/9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"suggested_file_name\": \"tmpr832hp1x\",\n \"timestamp\": 1655141671,\n \"total_bytes\": 16,\n \"total_bytes_lower_bound\": 0,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"uploading_to_reflector\": false,\n \"written_bytes\": 0\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "file_reflect", "description": "Reflect all the blobs in a file matching the filter criteria", "arguments": [ { "name": "sd_hash", "type": "str", "description": "get file with matching sd hash", "is_required": false }, { "name": "file_name", "type": "str", "description": "get file with matching file name in the downloads folder", "is_required": false }, { "name": "stream_hash", "type": "str", "description": "get file with matching stream hash", "is_required": false }, { "name": "rowid", "type": "int", "description": "get file with matching row id", "is_required": false }, { "name": "reflector", "type": "str", "description": "reflector server, ip address or url by default choose a server from the config", "is_required": false } ], "returns": "(list) list of blobs reflected", "examples": [] }, { "name": "file_save", "description": "Start saving a file to disk.", "arguments": [ { "name": "file_name", "type": "str", "description": "file name to save to", "is_required": false }, { "name": "download_directory", "type": "str", "description": "directory to save into", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "save file with matching sd hash", "is_required": false }, { "name": "stream_hash", "type": "str", "description": "save file with matching stream hash", "is_required": false }, { "name": "rowid", "type": "int", "description": "save file with matching row id", "is_required": false }, { "name": "claim_id", "type": "str", "description": "save file with matching claim id", "is_required": false }, { "name": "txid", "type": "str", "description": "save file with matching claim txid", "is_required": false }, { "name": "nout", "type": "int", "description": "save file with matching claim nout", "is_required": false }, { "name": "claim_name", "type": "str", "description": "save file with matching claim name", "is_required": false }, { "name": "channel_claim_id", "type": "str", "description": "save file with matching channel claim id", "is_required": false }, { "name": "channel_name", "type": "str", "description": "save file with matching channel claim name", "is_required": false } ], "returns": " {\n \"streaming_url\": \"(str) url to stream the file using range requests\",\n \"completed\": \"(bool) true if download is completed\",\n \"file_name\": \"(str) name of file\",\n \"download_directory\": \"(str) download directory\",\n \"points_paid\": \"(float) credit paid to download file\",\n \"stopped\": \"(bool) true if download is stopped\",\n \"stream_hash\": \"(str) stream hash of file\",\n \"stream_name\": \"(str) stream name\",\n \"suggested_file_name\": \"(str) suggested file name\",\n \"sd_hash\": \"(str) sd hash of file\",\n \"download_path\": \"(str) download path of file\",\n \"mime_type\": \"(str) mime type of file\",\n \"key\": \"(str) key attached to file\",\n \"total_bytes_lower_bound\": \"(int) lower bound file size in bytes\",\n \"total_bytes\": \"(int) file upper bound size in bytes\",\n \"written_bytes\": \"(int) written size in bytes\",\n \"blobs_completed\": \"(int) number of fully downloaded blobs\",\n \"blobs_in_stream\": \"(int) total blobs on stream\",\n \"blobs_remaining\": \"(int) total blobs remaining to download\",\n \"status\": \"(str) downloader status\",\n \"claim_id\": \"(str) None if claim is not found else the claim id\",\n \"txid\": \"(str) None if claim is not found else the transaction id\",\n \"nout\": \"(int) None if claim is not found else the transaction output index\",\n \"outpoint\": \"(str) None if claim is not found else the tx and output\",\n \"metadata\": \"(dict) None if claim is not found else the claim metadata\",\n \"channel_claim_id\": \"(str) None if claim is not found or not signed\",\n \"channel_name\": \"(str) None if claim is not found or not signed\",\n \"claim_name\": \"(str) None if claim is not found else the claim name\",\n \"reflector_progress\": \"(int) reflector upload progress, 0 to 100\",\n \"uploading_to_reflector\": \"(bool) set to True when currently uploading to reflector\"\n }", "examples": [ { "title": "Save a file to the downloads directory", "curl": "curl -d'{\"method\": \"file_save\", \"params\": {\"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\"}}' http://localhost:5279/", "lbrynet": "lbrynet file save --sd_hash=\"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"file_save\", \"params\": {\"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"added_on\": 1655141677,\n \"blobs_completed\": 1,\n \"blobs_in_stream\": 1,\n \"blobs_remaining\": 0,\n \"channel_claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"channel_name\": \"@channel\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_name\": \"astream\",\n \"completed\": true,\n \"confirmations\": 4,\n \"content_fee\": null,\n \"download_directory\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpfx0nk2jd\",\n \"download_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpfx0nk2jd/tmpr832hp1x_1\",\n \"file_name\": \"tmpr832hp1x_1\",\n \"height\": 214,\n \"is_fully_reflected\": false,\n \"key\": \"66f888fe00cf558494c2fcbd5903d00d\",\n \"metadata\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"mime_type\": \"application/octet-stream\",\n \"nout\": 0,\n \"outpoint\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234:0\",\n \"points_paid\": 0.0,\n \"protobuf\": \"01a2795714b0923650b7fa288618591f0c2f2e5c5961baf110460a527e38b68f0653c1c79f2cae1f57ade55c009fb7578175c9d93d7edb0546d1378e1d33d99df8df0d5cfe9e7a5b63053a4e7ce003f95f5890b27f0a90010a8d010a30fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd120b746d707238333268703178180b22186170706c69636174696f6e2f6f637465742d73747265616d32309ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"purchase_receipt\": null,\n \"reflector_progress\": 0,\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"status\": \"finished\",\n \"stopped\": true,\n \"stream_hash\": \"c48ff9950efbcb78b20d311467b1b0e321069ef5ece96898eb08a27a62d1ef2594cd24f1e8d6993f00c48fd6a4221490\",\n \"stream_name\": \"tmpr832hp1x\",\n \"streaming_url\": \"http://localhost:5280/stream/9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"suggested_file_name\": \"tmpr832hp1x\",\n \"timestamp\": 1655141671,\n \"total_bytes\": 16,\n \"total_bytes_lower_bound\": 0,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"uploading_to_reflector\": false,\n \"written_bytes\": 11\n }\n}" } ] }, { "name": "file_set_status", "description": "Start or stop downloading a file", "arguments": [ { "name": "status", "type": "str", "description": "one of \"start\" or \"stop\"", "is_required": true }, { "name": "sd_hash", "type": "str", "description": "set status of file with matching sd hash", "is_required": false }, { "name": "file_name", "type": "str", "description": "set status of file with matching file name in the downloads folder", "is_required": false }, { "name": "stream_hash", "type": "str", "description": "set status of file with matching stream hash", "is_required": false }, { "name": "rowid", "type": "int", "description": "set status of file with matching row id", "is_required": false } ], "returns": "(str) Confirmation message", "examples": [] } ] }, "peer": { "doc": "DHT / Blob Exchange peer commands.", "commands": [ { "name": "peer_list", "description": "Get peers for blob hash", "arguments": [ { "name": "blob_hash", "type": "str", "description": "find available peers for this blob hash", "is_required": true }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": "(list) List of contact dictionaries {'address': , 'udp_port': , 'tcp_port': ,\n 'node_id': }", "examples": [] }, { "name": "peer_ping", "description": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.", "arguments": [], "returns": "(str) pong, or {'error': } if an error is encountered", "examples": [] } ] }, "preference": { "doc": "Preferences management.", "commands": [ { "name": "preference_get", "description": "Get preference value for key or all values if not key is passed in.", "arguments": [ { "name": "key", "type": "str", "description": "key associated with value", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(dict) Dictionary of preference(s)", "examples": [ { "title": "Get preferences", "curl": "curl -d'{\"method\": \"preference_get\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet preference get", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"preference_get\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"theme\": \"dark\"\n }\n}" } ] }, { "name": "preference_set", "description": "Set preferences", "arguments": [ { "name": "key", "type": "str", "description": "key associated with value", "is_required": true }, { "name": "value", "type": "str", "description": "key associated with value", "is_required": true }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(dict) Dictionary with key/value of new preference", "examples": [ { "title": "Set preference", "curl": "curl -d'{\"method\": \"preference_set\", \"params\": {\"key\": \"theme\", \"value\": \"dark\"}}' http://localhost:5279/", "lbrynet": "lbrynet preference set \"theme\" \"dark\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"preference_set\", \"params\": {\"key\": \"theme\", \"value\": \"dark\"}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"theme\": \"dark\"\n }\n}" } ] } ] }, "purchase": { "doc": "List and make purchases of claims.", "commands": [ { "name": "purchase_create", "description": "Purchase a claim.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim id of claim to purchase", "is_required": false }, { "name": "url", "type": "str", "description": "lookup claim to purchase by url", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "allow_duplicate_purchase", "type": "bool", "description": "allow purchasing claim_id you already own", "is_required": false }, { "name": "override_max_key_fee", "type": "bool", "description": "ignore max key fee for this purchase", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "purchase_list", "description": "List my claim purchases.", "arguments": [ { "name": "claim_id", "type": "str", "description": "purchases for specific claim", "is_required": false }, { "name": "resolve", "type": "str", "description": "include resolved claim information", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [] } ] }, "settings": { "doc": "Settings management.", "commands": [ { "name": "settings_clear", "description": "Clear daemon settings", "arguments": [], "returns": "(dict) Updated dictionary of daemon settings", "examples": [] }, { "name": "settings_get", "description": "Get daemon settings", "arguments": [], "returns": "(dict) Dictionary of daemon settings\n See ADJUSTABLE_SETTINGS in lbry/conf.py for full list of settings", "examples": [ { "title": "Get settings", "curl": "curl -d'{\"method\": \"settings_get\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet settings get", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"settings_get\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"allowed_origin\": \"\",\n \"announce_head_and_sd_only\": true,\n \"api\": \"localhost:5279\",\n \"audio_encoder\": \"aac -b:a 160k\",\n \"blob_download_timeout\": 30.0,\n \"blob_lru_cache_size\": 0,\n \"blob_storage_limit\": 0,\n \"blockchain_name\": \"lbrycrd_regtest\",\n \"coin_selection_strategy\": \"prefer_confirmed\",\n \"components_to_skip\": [\n \"dht\",\n \"upnp\",\n \"hash_announcer\",\n \"peer_protocol_server\",\n \"libtorrent_component\"\n ],\n \"concurrent_blob_announcers\": 10,\n \"concurrent_hub_requests\": 32,\n \"concurrent_reflector_uploads\": 10,\n \"config\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmp_dufqex7/daemon_settings.yml\",\n \"data_dir\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpfx0nk2jd\",\n \"download_dir\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpfx0nk2jd\",\n \"download_timeout\": 30.0,\n \"ffmpeg_path\": \"\",\n \"fixed_peer_delay\": 2.0,\n \"fixed_peers\": [\n [\n \"127.0.0.1\",\n 5567\n ]\n ],\n \"hub_timeout\": 30.0,\n \"jurisdiction\": null,\n \"known_dht_nodes\": [],\n \"lbryum_servers\": [\n [\n \"localhost\",\n 50002\n ]\n ],\n \"max_connections_per_download\": 4,\n \"max_key_fee\": {\n \"amount\": 50.0,\n \"currency\": \"USD\"\n },\n \"max_wallet_server_fee\": \"0.0\",\n \"network_interface\": \"0.0.0.0\",\n \"network_storage_limit\": 0,\n \"node_rpc_timeout\": 5.0,\n \"peer_connect_timeout\": 3.0,\n \"prometheus_port\": 0,\n \"reflect_streams\": true,\n \"reflector_servers\": [\n [\n \"127.0.0.1\",\n 5566\n ]\n ],\n \"save_blobs\": true,\n \"save_files\": true,\n \"save_resolved_claims\": true,\n \"share_usage_data\": false,\n \"split_buckets_under_index\": 2,\n \"streaming_get\": true,\n \"streaming_server\": \"localhost:5280\",\n \"tcp_port\": 4444,\n \"track_bandwidth\": true,\n \"tracker_servers\": [\n [\n \"tracker.lbry.com\",\n 9252\n ],\n [\n \"tracker.lbry.grin.io\",\n 9252\n ]\n ],\n \"transaction_cache_size\": 10000,\n \"udp_port\": 4444,\n \"use_upnp\": false,\n \"video_bitrate_maximum\": 5000000,\n \"video_encoder\": \"libx264 -crf 24 -preset faster -pix_fmt yuv420p\",\n \"video_scaler\": \"-vf \\\"scale=if(gte(iw\\\\,ih)\\\\,min(1920\\\\,iw)\\\\,-2):if(lt(iw\\\\,ih)\\\\,min(1920\\\\,ih)\\\\,-2)\\\" -maxrate 5500K -bufsize 5000K\",\n \"volume_analysis_time\": 240,\n \"volume_filter\": \"\",\n \"wallet_dir\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpfx0nk2jd\",\n \"wallets\": [\n \"default_wallet\"\n ]\n }\n}" } ] }, { "name": "settings_set", "description": "Set daemon settings", "arguments": [], "returns": "(dict) Updated dictionary of daemon settings", "examples": [ { "title": "Set settings", "curl": "curl -d'{\"method\": \"settings_set\", \"params\": {\"key\": \"tcp_port\", \"value\": 99}}' http://localhost:5279/", "lbrynet": "lbrynet settings set \"tcp_port\" 99", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"settings_set\", \"params\": {\"key\": \"tcp_port\", \"value\": 99}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"tcp_port\": 99\n }\n}" } ] } ] }, "stream": { "doc": "Create, update, abandon, list and inspect your stream claims.", "commands": [ { "name": "stream_abandon", "description": "Abandon one of my stream claims.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim_id of the claim to abandon", "is_required": false }, { "name": "txid", "type": "str", "description": "txid of the claim to abandon", "is_required": false }, { "name": "nout", "type": "int", "description": "nout of the claim to abandon", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until abandon is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Abandon a stream claim", "curl": "curl -d'{\"method\": \"stream_abandon\", \"params\": {\"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\", \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet stream abandon ad25e05aa7dc5e9994869040c6103f9a8728db46", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"stream_abandon\", \"params\": {\"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\", \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"010000000134627f9ba4ba51befc3e15e8f60f7151e171618922ef727e1ee5effa87c47075000000006a47304402206abd1bb758c3bac2e23a22e93bf2166d29a27215eb297fd063f823cd3c68898b0220301c9f4dd874a90a877ab135153ba7bb1d9d93de8ed1a5db6d9653ece6c8e46b01210200ae7b8f7a6220cb0f3699aeb1b2f1ce7825554f50b6cf542e63f3018e9852b6ffffffff0134b7f505000000001976a9143bd49c5746cb0cbba81dc36420123d208f30600488ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 4,\n \"height\": 214,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mkyJrpgUxpSyVdb6bA36F2cZQfZpWa7Pci\",\n \"amount\": \"0.999893\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 0,\n \"timestamp\": null,\n \"txid\": \"3ef9d8954a84d44a1b3ddf95114e6b77bcbd2cfe5af0d509b1b8a3d9aada3182\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.000107\",\n \"total_input\": \"1.0\",\n \"total_output\": \"0.999893\",\n \"txid\": \"3ef9d8954a84d44a1b3ddf95114e6b77bcbd2cfe5af0d509b1b8a3d9aada3182\"\n }\n}" } ] }, { "name": "stream_cost_estimate", "description": "Get estimated cost for a lbry stream", "arguments": [ { "name": "uri", "type": "str", "description": "uri to use", "is_required": true } ], "returns": "(float) Estimated cost in lbry credits, returns None if uri is not\n resolvable", "examples": [] }, { "name": "stream_create", "description": "Make a new stream claim and announce the associated file to lbrynet.", "arguments": [ { "name": "name", "type": "str", "description": "name of the content (can only consist of a-z A-Z 0-9 and -(dash))", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": true }, { "name": "file_path", "type": "str", "description": "path to file to be associated with name.", "is_required": false }, { "name": "file_name", "type": "str", "description": "name of file to be associated with stream.", "is_required": false }, { "name": "file_hash", "type": "str", "description": "hash of file to be associated with stream.", "is_required": false }, { "name": "validate_file", "type": "bool", "description": "validate that the video container and encodings match common web browser support or that optimization succeeds if specified. FFmpeg is required", "is_required": false }, { "name": "optimize_file", "type": "bool", "description": "transcode the video & audio if necessary to ensure common web browser support. FFmpeg is required", "is_required": false }, { "name": "allow_duplicate_name", "type": "bool", "description": "create new claim even if one already exists with given name. default: false.", "is_required": false }, { "name": "fee_currency", "type": "string", "description": "specify fee currency", "is_required": false }, { "name": "fee_amount", "type": "decimal", "description": "content download fee", "is_required": false }, { "name": "fee_address", "type": "str", "description": "address where to send fee payments, will use value from --claim_address if not provided", "is_required": false }, { "name": "title", "type": "str", "description": "title of the publication", "is_required": false }, { "name": "description", "type": "str", "description": "description of the publication", "is_required": false }, { "name": "author", "type": "str", "description": "author of the publication. The usage for this field is not the same as for channels. The author field is used to credit an author who is not the publisher and is not represented by the channel. For example, a pdf file of 'The Odyssey' has an author of 'Homer' but may by published to a channel such as '@classics', or to no channel at all", "is_required": false }, { "name": "tags", "type": "list", "description": "add content tags", "is_required": false }, { "name": "languages", "type": "list", "description": "languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant`", "is_required": false }, { "name": "locations", "type": "list", "description": "locations relevant to the stream, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... \"locations\": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: \"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE\" making sure to include colon for blank values, for example to provide only the city: ... --locations=\"::Manchester\" with all values set: ... --locations=\"US:NH:Manchester:03101:42.990605:-71.460989\" optionally, you can just pass the \"LATITUDE:LONGITUDE\": ... --locations=\"42.990605:-71.460989\" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations=\"{'country': 'US', 'state': 'NH'}\"", "is_required": false }, { "name": "license", "type": "str", "description": "publication license", "is_required": false }, { "name": "license_url", "type": "str", "description": "publication license url", "is_required": false }, { "name": "thumbnail_url", "type": "str", "description": "thumbnail url", "is_required": false }, { "name": "release_time", "type": "int", "description": "original public release of content, seconds since UNIX epoch", "is_required": false }, { "name": "width", "type": "int", "description": "image/video width, automatically calculated from media file", "is_required": false }, { "name": "height", "type": "int", "description": "image/video height, automatically calculated from media file", "is_required": false }, { "name": "duration", "type": "int", "description": "audio/video duration in seconds, automatically calculated", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "sd_hash of stream", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of the publisher channel", "is_required": false }, { "name": "channel_name", "type": "str", "description": "name of the publisher channel", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "account_id", "type": "str", "description": "account to use for holding the transaction", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the claim is sent to, if not specified it will be determined automatically from the account", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Create a stream claim without metadata", "curl": "curl -d'{\"method\": \"stream_create\", \"params\": {\"name\": \"astream\", \"bid\": \"1.0\", \"file_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpr832hp1x\", \"validate_file\": false, \"optimize_file\": false, \"tags\": [], \"languages\": [], \"locations\": [], \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet stream create astream 1.0 /var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpr832hp1x", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"stream_create\", \"params\": {\"name\": \"astream\", \"bid\": \"1.0\", \"file_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmpr832hp1x\", \"validate_file\": false, \"optimize_file\": false, \"tags\": [], \"languages\": [], \"locations\": [], \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"01000000016419a92bda9f0ef64bde16f5a322d5668d68d53b4e38eb99b4738f72dbe880de010000006b483045022100d6963914b5e07f2c89f15f743c924919f51bbccbf1382395e54ba31cf67c9d5f02205cad973318f58d2841bda1f4c27be312881d2c32974b25d612106fa99cf53828012103c1b9bc894048c2d8d7091ed0d92d1b9468343eba19470bba436c0c4ee2cccb96ffffffff0200e1f50500000000bab5076173747265616d4c94000a90010a8d010a30fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd120b746d707238333268703178180b22186170706c69636174696f6e2f6f637465742d73747265616d32309ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c836d7576a9143d39ffed222af9cdea57d6d388ebba0aeda3039d88ac38fb9423000000001976a914152e48d498f73f383b15e73d6504ba35e6f4639e88ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mwtvw9x13nsVo6xkkrt3RFkk5h6TAjgPdK\",\n \"amount\": \"6.983769\",\n \"confirmations\": 4,\n \"height\": 209,\n \"nout\": 1,\n \"timestamp\": 1655141670,\n \"txid\": \"de80e8db728f73b499eb384e3bd5688d66d522a3f516de4bf60e9fda2ba91964\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"create\",\n \"confirmations\": -2,\n \"height\": -2,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"timestamp\": null,\n \"txid\": \"f9e7ec8e2836aca6145f71ac7dba4eb02e333adfda82221e7af8a6720ebda344\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"mhSwvz7Qfuh343S8WrXpPoPcaxbtw7QM5W\",\n \"amount\": \"5.969662\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"f9e7ec8e2836aca6145f71ac7dba4eb02e333adfda82221e7af8a6720ebda344\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.014107\",\n \"total_input\": \"6.983769\",\n \"total_output\": \"6.969662\",\n \"txid\": \"f9e7ec8e2836aca6145f71ac7dba4eb02e333adfda82221e7af8a6720ebda344\"\n }\n}" }, { "title": "Create an image stream claim with all metadata and fee", "curl": "curl -d'{\"method\": \"stream_create\", \"params\": {\"name\": \"blank-image\", \"bid\": \"1.0\", \"file_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmps0do5cfj.png\", \"validate_file\": false, \"optimize_file\": false, \"fee_currency\": \"LBC\", \"fee_amount\": \"0.3\", \"title\": \"Blank Image\", \"description\": \"A blank PNG that is 5x7.\", \"author\": \"Picaso\", \"tags\": [\"blank\", \"art\"], \"languages\": [\"en\"], \"locations\": [\"US:NH:Manchester\"], \"license\": \"Public Domain\", \"license_url\": \"http://public-domain.org\", \"thumbnail_url\": \"http://smallmedia.com/thumbnail.jpg\", \"release_time\": 1655141671, \"channel_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}' http://localhost:5279/", "lbrynet": "lbrynet stream create blank-image 1.0 /var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmps0do5cfj.png --tags=blank --tags=art --languages=en --locations=US:NH:Manchester --fee_currency=LBC --fee_amount=0.3 --title=\"Blank Image\" --description=\"A blank PNG that is 5x7.\" --author=Picaso --license=\"Public Domain\" --license_url=http://public-domain.org --thumbnail_url=\"http://smallmedia.com/thumbnail.jpg\" --release_time=1655141671 --channel_id=\"595c2e2f0c1f59188628fab7503692b0145779a2\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"stream_create\", \"params\": {\"name\": \"blank-image\", \"bid\": \"1.0\", \"file_path\": \"/var/folders/46/44w2zhrx16b8gsvff9dxtr640000gq/T/tmps0do5cfj.png\", \"validate_file\": false, \"optimize_file\": false, \"fee_currency\": \"LBC\", \"fee_amount\": \"0.3\", \"title\": \"Blank Image\", \"description\": \"A blank PNG that is 5x7.\", \"author\": \"Picaso\", \"tags\": [\"blank\", \"art\"], \"languages\": [\"en\"], \"locations\": [\"US:NH:Manchester\"], \"license\": \"Public Domain\", \"license_url\": \"http://public-domain.org\", \"thumbnail_url\": \"http://smallmedia.com/thumbnail.jpg\", \"release_time\": 1655141671, \"channel_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"010000000144a3bd0e72a6f87a1e2282dadf3a332eb04eba7dac715f14a6ac36288eece7f9010000006b483045022100d1327306e6b07c668e21b915e0a9bbe7e08036ffc6a197f469ff458b5108be6e02207c4160f2c7e0344a2dc30beb0697bee536118df5e6cb978de072bb745b2783d60121026a72cb1ef362624b1483427abf7cb2731c1dd582bad08b4521a1e92967cbda32ffffffff0200e1f50500000000fddc01b50b626c616e6b2d696d6167654db10101a2795714b0923650b7fa288618591f0c2f2e5c5946580bd06bec18e960285651ca9130c75fcf4f5b3cd71fa5d97fe7013de1bf7627b08de186dc0f5c4ff33a6e7173b24f7da6a91080ffb840f6f6fbf4897851790ae6010a82010a306c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545120f746d707330646f3563666a2e706e6718632209696d6167652f706e673230d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996120650696361736f1a0d5075626c696320446f6d61696e2218687474703a2f2f7075626c69632d646f6d61696e2e6f726728a7ea9d95063222080112196f4a4f7b3ab55c52fe12f9d8b9b255fb2625522e4b00415bed188087a70e520408051007420b426c616e6b20496d6167654a184120626c616e6b20504e472074686174206973203578372e52252a23687474703a2f2f736d616c6c6d656469612e636f6d2f7468756d626e61696c2e6a70675a05626c616e6b5a03617274620208016a1308ec0112024e481a0a4d616e636865737465726d7576a9144a4f7b3ab55c52fe12f9d8b9b255fb2625522e4b88acac5e7d1d000000001976a914f756abdb56de34ac287ee4b85ca911e2f83894b788ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mhSwvz7Qfuh343S8WrXpPoPcaxbtw7QM5W\",\n \"amount\": \"5.969662\",\n \"confirmations\": 2,\n \"height\": 213,\n \"nout\": 1,\n \"timestamp\": 1655141671,\n \"txid\": \"f9e7ec8e2836aca6145f71ac7dba4eb02e333adfda82221e7af8a6720ebda344\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mnHsWiDjPYw27jPzMfqUFM9scdsgHQ6NJg\",\n \"amount\": \"1.0\",\n \"claim_id\": \"46bff43fbdffabf12efc867c2b4c7865315b0b76\",\n \"claim_op\": \"create\",\n \"confirmations\": -2,\n \"height\": -2,\n \"is_channel_signature_valid\": true,\n \"meta\": {},\n \"name\": \"blank-image\",\n \"normalized_name\": \"blank-image\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://blank-image#46bff43fbdffabf12efc867c2b4c7865315b0b76\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": null,\n \"txid\": \"04d32d523ddc247cd5055a2146c445c8ab99ff8b5b3ea9edaf8604c59ac41b2c\",\n \"type\": \"claim\",\n \"value\": {\n \"author\": \"Picaso\",\n \"description\": \"A blank PNG that is 5x7.\",\n \"fee\": {\n \"address\": \"mnHsWiDjPYw27jPzMfqUFM9scdsgHQ6NJg\",\n \"amount\": \"0.3\",\n \"currency\": \"LBC\"\n },\n \"image\": {\n \"height\": 7,\n \"width\": 5\n },\n \"languages\": [\n \"en\"\n ],\n \"license\": \"Public Domain\",\n \"license_url\": \"http://public-domain.org\",\n \"locations\": [\n {\n \"city\": \"Manchester\",\n \"country\": \"US\",\n \"state\": \"NH\"\n }\n ],\n \"release_time\": \"1655141671\",\n \"source\": {\n \"hash\": \"6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545\",\n \"media_type\": \"image/png\",\n \"name\": \"tmps0do5cfj.png\",\n \"sd_hash\": \"d221fe243afed69b84e6cbc32448260a187449c5123bfa89d33af05af2c8f195a7bbac2bfe02d4a2ba472af217af3996\",\n \"size\": \"99\"\n },\n \"stream_type\": \"image\",\n \"tags\": [\n \"blank\",\n \"art\"\n ],\n \"thumbnail\": {\n \"url\": \"http://smallmedia.com/thumbnail.jpg\"\n },\n \"title\": \"Blank Image\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"n44m2TuZKypifAkpPQqEN4VtRkCxgbgkA3\",\n \"amount\": \"4.947555\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"04d32d523ddc247cd5055a2146c445c8ab99ff8b5b3ea9edaf8604c59ac41b2c\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.022107\",\n \"total_input\": \"5.969662\",\n \"total_output\": \"5.947555\",\n \"txid\": \"04d32d523ddc247cd5055a2146c445c8ab99ff8b5b3ea9edaf8604c59ac41b2c\"\n }\n}" } ] }, { "name": "stream_list", "description": "List my stream claims.", "arguments": [ { "name": "name", "type": "str or list", "description": "stream name", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "stream id", "is_required": false }, { "name": "is_spent", "type": "bool", "description": "shows previous stream updates and abandons", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false }, { "name": "resolve", "type": "bool", "description": "resolves each stream to provide additional metadata", "is_required": false }, { "name": "no_totals", "type": "bool", "description": "do not calculate the total number of pages and items in result set (significant performance boost)", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [ { "title": "List all your stream claims", "curl": "curl -d'{\"method\": \"stream_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"resolve\": false, \"no_totals\": false}}' http://localhost:5279/", "lbrynet": "lbrynet stream list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"stream_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"resolve\": false, \"no_totals\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 1,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"create\",\n \"confirmations\": 2,\n \"height\": 213,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": true,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"timestamp\": 1655141671,\n \"txid\": \"f9e7ec8e2836aca6145f71ac7dba4eb02e333adfda82221e7af8a6720ebda344\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 2,\n \"total_pages\": 1\n }\n}" }, { "title": "Paginate your stream claims", "curl": "curl -d'{\"method\": \"stream_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"page\": 1, \"page_size\": 20, \"resolve\": false, \"no_totals\": false}}' http://localhost:5279/", "lbrynet": "lbrynet stream list --page=1 --page_size=20", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"stream_list\", \"params\": {\"name\": [], \"claim_id\": [], \"is_spent\": false, \"page\": 1, \"page_size\": 20, \"resolve\": false, \"no_totals\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": 1,\n \"height\": 214,\n \"is_channel_signature_valid\": true,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": false,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 5,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": 1655141671,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"create\",\n \"confirmations\": 2,\n \"height\": 213,\n \"is_internal_transfer\": false,\n \"is_my_input\": true,\n \"is_my_output\": true,\n \"is_spent\": true,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"timestamp\": 1655141671,\n \"txid\": \"f9e7ec8e2836aca6145f71ac7dba4eb02e333adfda82221e7af8a6720ebda344\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 2,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "stream_repost", "description": "Creates a claim that references an existing stream by its claim id.", "arguments": [ { "name": "name", "type": "str", "description": "name of the content (can only consist of a-z A-Z 0-9 and -(dash))", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": true }, { "name": "claim_id", "type": "str", "description": "id of the claim being reposted", "is_required": true }, { "name": "allow_duplicate_name", "type": "bool", "description": "create new claim even if one already exists with given name. default: false.", "is_required": false }, { "name": "title", "type": "str", "description": "title of the repost", "is_required": false }, { "name": "description", "type": "str", "description": "description of the repost", "is_required": false }, { "name": "tags", "type": "list", "description": "add repost tags", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of the publisher channel", "is_required": false }, { "name": "channel_name", "type": "str", "description": "name of the publisher channel", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "account_id", "type": "str", "description": "account to use for holding the transaction", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the claim is sent to, if not specified it will be determined automatically from the account", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "stream_update", "description": "Update an existing stream claim and if a new file is provided announce it to lbrynet.", "arguments": [ { "name": "claim_id", "type": "str", "description": "id of the stream claim to update", "is_required": true }, { "name": "bid", "type": "decimal", "description": "amount to back the claim", "is_required": false }, { "name": "file_path", "type": "str", "description": "path to file to be associated with name.", "is_required": false }, { "name": "validate_file", "type": "bool", "description": "validate that the video container and encodings match common web browser support or that optimization succeeds if specified. FFmpeg is required and file_path must be specified.", "is_required": false }, { "name": "optimize_file", "type": "bool", "description": "transcode the video & audio if necessary to ensure common web browser support. FFmpeg is required and file_path must be specified.", "is_required": false }, { "name": "file_name", "type": "str", "description": "override file name, defaults to name from file_path.", "is_required": false }, { "name": "file_size", "type": "str", "description": "override file size, otherwise automatically computed.", "is_required": false }, { "name": "file_hash", "type": "str", "description": "override file hash, otherwise automatically computed.", "is_required": false }, { "name": "fee_currency", "type": "string", "description": "specify fee currency", "is_required": false }, { "name": "fee_amount", "type": "decimal", "description": "content download fee", "is_required": false }, { "name": "fee_address", "type": "str", "description": "address where to send fee payments, will use value from --claim_address if not provided", "is_required": false }, { "name": "clear_fee", "type": "bool", "description": "clear previously set fee", "is_required": false }, { "name": "title", "type": "str", "description": "title of the publication", "is_required": false }, { "name": "description", "type": "str", "description": "description of the publication", "is_required": false }, { "name": "author", "type": "str", "description": "author of the publication. The usage for this field is not the same as for channels. The author field is used to credit an author who is not the publisher and is not represented by the channel. For example, a pdf file of 'The Odyssey' has an author of 'Homer' but may by published to a channel such as '@classics', or to no channel at all", "is_required": false }, { "name": "tags", "type": "list", "description": "add content tags", "is_required": false }, { "name": "clear_tags", "type": "bool", "description": "clear existing tags (prior to adding new ones)", "is_required": false }, { "name": "languages", "type": "list", "description": "languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant`", "is_required": false }, { "name": "clear_languages", "type": "bool", "description": "clear existing languages (prior to adding new ones)", "is_required": false }, { "name": "locations", "type": "list", "description": "locations relevant to the stream, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... \"locations\": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: \"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE\" making sure to include colon for blank values, for example to provide only the city: ... --locations=\"::Manchester\" with all values set: ... --locations=\"US:NH:Manchester:03101:42.990605:-71.460989\" optionally, you can just pass the \"LATITUDE:LONGITUDE\": ... --locations=\"42.990605:-71.460989\" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations=\"{'country': 'US', 'state': 'NH'}\"", "is_required": false }, { "name": "clear_locations", "type": "bool", "description": "clear existing locations (prior to adding new ones)", "is_required": false }, { "name": "license", "type": "str", "description": "publication license", "is_required": false }, { "name": "license_url", "type": "str", "description": "publication license url", "is_required": false }, { "name": "thumbnail_url", "type": "str", "description": "thumbnail url", "is_required": false }, { "name": "release_time", "type": "int", "description": "original public release of content, seconds since UNIX epoch", "is_required": false }, { "name": "width", "type": "int", "description": "image/video width, automatically calculated from media file", "is_required": false }, { "name": "height", "type": "int", "description": "image/video height, automatically calculated from media file", "is_required": false }, { "name": "duration", "type": "int", "description": "audio/video duration in seconds, automatically calculated", "is_required": false }, { "name": "sd_hash", "type": "str", "description": "sd_hash of stream", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of the publisher channel", "is_required": false }, { "name": "channel_name", "type": "str", "description": "name of the publisher channel", "is_required": false }, { "name": "clear_channel", "type": "bool", "description": "remove channel signature", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "account_id", "type": "str", "description": "account in which to look for stream (default: all)", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "claim_address", "type": "str", "description": "address where the claim is sent to, if not specified it will be determined automatically from the account", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false }, { "name": "replace", "type": "bool", "description": "instead of modifying specific values on the stream, this will clear all existing values and only save passed in values, useful for form submissions where all values are always set", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [ { "title": "Update a stream claim to add channel", "curl": "curl -d'{\"method\": \"stream_update\", \"params\": {\"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\", \"validate_file\": false, \"optimize_file\": false, \"clear_fee\": false, \"tags\": [], \"clear_tags\": false, \"languages\": [], \"clear_languages\": false, \"locations\": [], \"clear_locations\": false, \"channel_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"clear_channel\": false, \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false, \"replace\": false}}' http://localhost:5279/", "lbrynet": "lbrynet stream update ad25e05aa7dc5e9994869040c6103f9a8728db46 --channel_id=\"595c2e2f0c1f59188628fab7503692b0145779a2\"", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"stream_update\", \"params\": {\"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\", \"validate_file\": false, \"optimize_file\": false, \"clear_fee\": false, \"tags\": [], \"clear_tags\": false, \"languages\": [], \"clear_languages\": false, \"locations\": [], \"clear_locations\": false, \"channel_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\", \"clear_channel\": false, \"channel_account_id\": [], \"funding_account_ids\": [], \"preview\": false, \"blocking\": false, \"replace\": false}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"height\": -2,\n \"hex\": \"010000000244a3bd0e72a6f87a1e2282dadf3a332eb04eba7dac715f14a6ac36288eece7f9000000006a4730440220117bdbc0e547eda333409733aa38ccb4841a7873db6c4a971fcf8f87a7c1553e022007bf22f8573e95012d4918fea61ba8b04a7a10af93aea95fbbffe187ca17621101210200ae7b8f7a6220cb0f3699aeb1b2f1ce7825554f50b6cf542e63f3018e9852b6ffffffffe3d2e954c197eace1e6b39edc7451a2cf9b6e1771a37f8127da3f105e929baac010000006a473044022100e29082b0d9e24dc0a9622856ab499725155e379a323575be3ccda6472970e330021f652654fcbad99cbadbf22cf8f876a597f672ba86d4b5e53e400af23914ff200121038af0e4f3c8d08223532f5ade7d5da0203b676b3445a8791c955f8d0f868b2c18ffffffff0200e1f50500000000fd2301b7076173747265616d1446db28879a3f10c640908694995edca75ae025ad4ce801a2795714b0923650b7fa288618591f0c2f2e5c5961baf110460a527e38b68f0653c1c79f2cae1f57ade55c009fb7578175c9d93d7edb0546d1378e1d33d99df8df0d5cfe9e7a5b63053a4e7ce003f95f5890b27f0a90010a8d010a30fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd120b746d707238333268703178180b22186170706c69636174696f6e2f6f637465742d73747265616d32309ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c836d6d76a9143d39ffed222af9cdea57d6d388ebba0aeda3039d88ac22abd205000000001976a91434ab169b2f32ccd430bc73f2c2116f48b907267d88ac00000000\",\n \"inputs\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"create\",\n \"confirmations\": 1,\n \"height\": 213,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"timestamp\": 1655141671,\n \"txid\": \"f9e7ec8e2836aca6145f71ac7dba4eb02e333adfda82221e7af8a6720ebda344\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"muXoymiRVLaS1FXJSgPjieSedrWPgUgEkK\",\n \"amount\": \"0.9772565\",\n \"confirmations\": 3,\n \"height\": 211,\n \"nout\": 1,\n \"timestamp\": 1655141670,\n \"txid\": \"acba29e905f1a37d12f8371a77e1b6f92c1a45c7ed396b1eceea97c154e9d2e3\",\n \"type\": \"payment\"\n }\n ],\n \"outputs\": [\n {\n \"address\": \"mm6gzeSV7hiGxtAv3rQjo3sRYtCDGD4t2M\",\n \"amount\": \"1.0\",\n \"claim_id\": \"ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"claim_op\": \"update\",\n \"confirmations\": -2,\n \"height\": -2,\n \"is_channel_signature_valid\": true,\n \"meta\": {},\n \"name\": \"astream\",\n \"normalized_name\": \"astream\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://astream#ad25e05aa7dc5e9994869040c6103f9a8728db46\",\n \"signing_channel\": {\n \"address\": \"muRaaMs12imkZJQofvBuLbAFYAs6TiQPAQ\",\n \"amount\": \"1.0\",\n \"claim_id\": \"595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"claim_op\": \"update\",\n \"confirmations\": 4,\n \"has_signing_key\": true,\n \"height\": 210,\n \"meta\": {},\n \"name\": \"@channel\",\n \"normalized_name\": \"@channel\",\n \"nout\": 0,\n \"permanent_url\": \"lbry://@channel#595c2e2f0c1f59188628fab7503692b0145779a2\",\n \"timestamp\": 1655141670,\n \"txid\": \"ab8221c5a5404117744d80e5d652dcf04c5caac947f019b5b534e0ccf7cfff64\",\n \"type\": \"claim\",\n \"value\": {\n \"public_key\": \"03bbf11fc85401781301f36897b5b01b0aef440db5d6fb0747900b8c69e40e55e5\",\n \"public_key_id\": \"moF9EgZauGirwqpwZ7NgVsCAxddcPABTfm\",\n \"title\": \"New Channel\"\n },\n \"value_type\": \"channel\"\n },\n \"timestamp\": null,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"claim\",\n \"value\": {\n \"source\": {\n \"hash\": \"fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd\",\n \"media_type\": \"application/octet-stream\",\n \"name\": \"tmpr832hp1x\",\n \"sd_hash\": \"9ddea316b511d9f720b1f67f5958cac381fdac5d1d3beabc754b9a1220a891024e983241e2fc7549f0f89ea0636c6c83\",\n \"size\": \"11\"\n },\n \"stream_type\": \"binary\"\n },\n \"value_type\": \"stream\"\n },\n {\n \"address\": \"mkKSPTEMxFHfbYE8dYMmczn3BCJRg4LoHQ\",\n \"amount\": \"0.9769245\",\n \"confirmations\": -2,\n \"height\": -2,\n \"nout\": 1,\n \"timestamp\": null,\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\",\n \"type\": \"payment\"\n }\n ],\n \"total_fee\": \"0.000332\",\n \"total_input\": \"1.9772565\",\n \"total_output\": \"1.9769245\",\n \"txid\": \"7570c487faefe51e7e72ef22896171e151710ff6e8153efcbe51baa49b7f6234\"\n }\n}" } ] } ] }, "support": { "doc": "Create, list and abandon all types of supports.", "commands": [ { "name": "support_abandon", "description": "Abandon supports, including tips, of a specific claim, optionally\nkeeping some amount as supports.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim_id of the support to abandon", "is_required": false }, { "name": "txid", "type": "str", "description": "txid of the claim to abandon", "is_required": false }, { "name": "nout", "type": "int", "description": "nout of the claim to abandon", "is_required": false }, { "name": "keep", "type": "decimal", "description": "amount of lbc to keep as support", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to use", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until abandon is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "support_create", "description": "Create a support or a tip for name claim.", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim_id of the claim to support", "is_required": true }, { "name": "amount", "type": "decimal", "description": "amount of support", "is_required": true }, { "name": "tip", "type": "bool", "description": "send support to claim owner, default: false.", "is_required": false }, { "name": "channel_id", "type": "str", "description": "claim id of the supporters identity channel", "is_required": false }, { "name": "channel_name", "type": "str", "description": "name of the supporters identity channel", "is_required": false }, { "name": "channel_account_id", "type": "str", "description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.", "is_required": false }, { "name": "account_id", "type": "str", "description": "account to use for holding the transaction", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "funding_account_ids", "type": "list", "description": "ids of accounts to fund this transaction", "is_required": false }, { "name": "comment", "type": "str", "description": "add a comment to the support", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until transaction is in mempool", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "support_list", "description": "List staked supports and sent/received tips.", "arguments": [ { "name": "name", "type": "str or list", "description": "claim name", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "claim id", "is_required": false }, { "name": "received", "type": "bool", "description": "only show received (tips)", "is_required": false }, { "name": "sent", "type": "bool", "description": "only show sent (tips)", "is_required": false }, { "name": "staked", "type": "bool", "description": "only show my staked supports", "is_required": false }, { "name": "is_spent", "type": "bool", "description": "show abandoned supports", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false }, { "name": "no_totals", "type": "bool", "description": "do not calculate the total number of pages and items in result set (significant performance boost)", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [] }, { "name": "support_sum", "description": "List total staked supports for a claim, grouped by the channel that signed the support.\n\nIf claim_id is a channel claim, you can use --include_channel_content to also include supports for\ncontent claims in the channel.\n\n!!!! NOTE: PAGINATION DOES NOT DO ANYTHING AT THE MOMENT !!!!!", "arguments": [ { "name": "claim_id", "type": "str", "description": "claim id", "is_required": false }, { "name": "new_sdk_server", "type": "str", "description": "URL of the new SDK server (EXPERIMENTAL)", "is_required": false }, { "name": "include_channel_content", "type": "bool", "description": "if claim_id is for a channel, include supports for claims in that channel", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n \"glorious data in dictionary\"\n ]\n }", "examples": [] } ] }, "sync": { "doc": "Wallet synchronization.", "commands": [ { "name": "sync_apply", "description": "Apply incoming synchronization data, if provided, and return a sync hash and update wallet data.\n\nWallet must be unlocked to perform this operation.\n\nIf \"encrypt-on-disk\" preference is True and supplied password is different from local password,\nor there is no local password (because local wallet was not encrypted), then the supplied password\nwill be used for local encryption (overwriting previous local encryption password).", "arguments": [ { "name": "password", "type": "str", "description": "password to decrypt incoming and encrypt outgoing data", "is_required": false }, { "name": "data", "type": "str", "description": "incoming sync data, if any", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "wallet being sync'ed", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until any new accounts have sync'ed", "is_required": false } ], "returns": "(map) sync hash and data", "examples": [] }, { "name": "sync_hash", "description": "Deterministic hash of the wallet.", "arguments": [ { "name": "wallet_id", "type": "str", "description": "wallet for which to generate hash", "is_required": false } ], "returns": "(str) sha256 hash of wallet", "examples": [] } ] }, "tracemalloc": { "doc": "Controls and queries tracemalloc memory tracing tools for troubleshooting.", "commands": [ { "name": "tracemalloc_disable", "description": "Disable tracemalloc memory tracing", "arguments": [], "returns": "(bool) is it tracing?", "examples": [] }, { "name": "tracemalloc_enable", "description": "Enable tracemalloc memory tracing", "arguments": [], "returns": "(bool) is it tracing?", "examples": [] }, { "name": "tracemalloc_top", "description": "Show most common objects, the place that created them and their size.", "arguments": [ { "name": "items", "type": "int", "description": "maximum items to return, from the most common", "is_required": true } ], "returns": "(dict) dictionary containing most common objects in memory\n {\n \"line\": (str) filename and line number where it was created,\n \"code\": (str) code that created it,\n \"size\": (int) size in bytes, for each \"memory block\",\n \"count\" (int) number of memory blocks\n }", "examples": [] } ] }, "transaction": { "doc": "Transaction management.", "commands": [ { "name": "transaction_list", "description": "List transactions belonging to wallet", "arguments": [ { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": "(list) List of transactions\n\n {\n \"claim_info\": (list) claim info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) bid amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"abandon_info\": (list) abandon info if in txn [{\n \"address\": (str) address of abandoned claim,\n \"balance_delta\": (float) returned amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"confirmations\": (int) number of confirmations for the txn,\n \"date\": (str) date and time of txn,\n \"fee\": (float) txn fee,\n \"support_info\": (list) support info if in txn [{\n \"address\": (str) address of support,\n \"balance_delta\": (float) support amount,\n \"amount\": (float) support amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"is_tip\": (bool),\n \"nout\": (int) nout\n }],\n \"timestamp\": (int) timestamp,\n \"txid\": (str) txn id,\n \"update_info\": (list) update info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) credited/debited\n \"amount\": (float) absolute amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"value\": (float) value of txn\n }", "examples": [ { "title": "List your transactions", "curl": "curl -d'{\"method\": \"transaction_list\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet transaction list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"transaction_list\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"abandon_info\": [],\n \"claim_info\": [],\n \"confirmations\": 1,\n \"date\": \"2022-06-13 13:34\",\n \"fee\": \"-0.000124\",\n \"purchase_info\": [],\n \"support_info\": [],\n \"timestamp\": 1655141670,\n \"txid\": \"f03159770e23cab4da7779dcf1a6809f6f518c61ab540d6a452d0bc5509874b8\",\n \"update_info\": [],\n \"value\": \"0.0\"\n },\n {\n \"abandon_info\": [],\n \"claim_info\": [],\n \"confirmations\": 7,\n \"date\": \"2022-06-13 13:34\",\n \"fee\": \"0.0\",\n \"purchase_info\": [],\n \"support_info\": [],\n \"timestamp\": 1655141669,\n \"txid\": \"1c584e33328e7616e4b52ab4a919267757ad58a2911e1c0954a0fd0e139c482a\",\n \"update_info\": [],\n \"value\": \"10.0\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 2,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "transaction_show", "description": "Get a decoded transaction from a txid", "arguments": [ { "name": "txid", "type": "str", "description": "txid of the transaction", "is_required": true } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] } ] }, "txo": { "doc": "List and sum transaction outputs.", "commands": [ { "name": "txo_list", "description": "List my transaction outputs.", "arguments": [ { "name": "type", "type": "str or list", "description": "claim type: stream, channel, support, purchase, collection, repost, other", "is_required": false }, { "name": "txid", "type": "str or list", "description": "transaction id of outputs", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "claim id", "is_required": false }, { "name": "channel_id", "type": "str or list", "description": "claims in this channel", "is_required": false }, { "name": "not_channel_id", "type": "str or list", "description": "claims not in this channel", "is_required": false }, { "name": "name", "type": "str or list", "description": "claim name", "is_required": false }, { "name": "is_spent", "type": "bool", "description": "only show spent txos", "is_required": false }, { "name": "is_not_spent", "type": "bool", "description": "only show not spent txos", "is_required": false }, { "name": "is_my_input_or_output", "type": "bool", "description": "txos which have your inputs or your outputs, if using this flag the other related flags are ignored (--is_my_output, --is_my_input, etc)", "is_required": false }, { "name": "is_my_output", "type": "bool", "description": "show outputs controlled by you", "is_required": false }, { "name": "is_not_my_output", "type": "bool", "description": "show outputs not controlled by you", "is_required": false }, { "name": "is_my_input", "type": "bool", "description": "show outputs created by you", "is_required": false }, { "name": "is_not_my_input", "type": "bool", "description": "show outputs not created by you", "is_required": false }, { "name": "exclude_internal_transfers", "type": "bool", "description": "excludes any outputs that are exactly this combination: \"--is_my_input --is_my_output --type=other\" this allows to exclude \"change\" payments, this flag can be used in combination with any of the other flags", "is_required": false }, { "name": "include_received_tips", "type": "bool", "description": "calculate the amount of tips received for claim outputs", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false }, { "name": "resolve", "type": "bool", "description": "resolves each claim to provide additional metadata", "is_required": false }, { "name": "order_by", "type": "str", "description": "field to order by: 'name', 'height', 'amount' and 'none'", "is_required": false }, { "name": "no_totals", "type": "bool", "description": "do not calculate the total number of pages and items in result set (significant performance boost)", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [] }, { "name": "txo_plot", "description": "Plot transaction output sum over days.", "arguments": [ { "name": "type", "type": "str or list", "description": "claim type: stream, channel, support, purchase, collection, repost, other", "is_required": false }, { "name": "txid", "type": "str or list", "description": "transaction id of outputs", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "claim id", "is_required": false }, { "name": "name", "type": "str or list", "description": "claim name", "is_required": false }, { "name": "channel_id", "type": "str or list", "description": "claims in this channel", "is_required": false }, { "name": "not_channel_id", "type": "str or list", "description": "claims not in this channel", "is_required": false }, { "name": "is_spent", "type": "bool", "description": "only show spent txos", "is_required": false }, { "name": "is_not_spent", "type": "bool", "description": "only show not spent txos", "is_required": false }, { "name": "is_my_input_or_output", "type": "bool", "description": "txos which have your inputs or your outputs, if using this flag the other related flags are ignored (--is_my_output, --is_my_input, etc)", "is_required": false }, { "name": "is_my_output", "type": "bool", "description": "show outputs controlled by you", "is_required": false }, { "name": "is_not_my_output", "type": "bool", "description": "show outputs not controlled by you", "is_required": false }, { "name": "is_my_input", "type": "bool", "description": "show outputs created by you", "is_required": false }, { "name": "is_not_my_input", "type": "bool", "description": "show outputs not created by you", "is_required": false }, { "name": "exclude_internal_transfers", "type": "bool", "description": "excludes any outputs that are exactly this combination: \"--is_my_input --is_my_output --type=other\" this allows to exclude \"change\" payments, this flag can be used in combination with any of the other flags", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "days_back", "type": "int", "description": "number of days back from today (not compatible with --start_day, --days_after, --end_day)", "is_required": false }, { "name": "start_day", "type": "date", "description": "start on specific date (YYYY-MM-DD) (instead of --days_back)", "is_required": false }, { "name": "days_after", "type": "int", "description": "end number of days after --start_day (instead of --end_day)", "is_required": false }, { "name": "end_day", "type": "date", "description": "end on specific date (YYYY-MM-DD) (instead of --days_after)", "is_required": false } ], "returns": "List[Dict]", "examples": [] }, { "name": "txo_spend", "description": "Spend transaction outputs, batching into multiple transactions as necessary.", "arguments": [ { "name": "type", "type": "str or list", "description": "claim type: stream, channel, support, purchase, collection, repost, other", "is_required": false }, { "name": "txid", "type": "str or list", "description": "transaction id of outputs", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "claim id", "is_required": false }, { "name": "channel_id", "type": "str or list", "description": "claims in this channel", "is_required": false }, { "name": "not_channel_id", "type": "str or list", "description": "claims not in this channel", "is_required": false }, { "name": "name", "type": "str or list", "description": "claim name", "is_required": false }, { "name": "is_my_input", "type": "bool", "description": "show outputs created by you", "is_required": false }, { "name": "is_not_my_input", "type": "bool", "description": "show outputs not created by you", "is_required": false }, { "name": "exclude_internal_transfers", "type": "bool", "description": "excludes any outputs that are exactly this combination: \"--is_my_input --is_my_output --type=other\" this allows to exclude \"change\" payments, this flag can be used in combination with any of the other flags", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until abandon is in mempool", "is_required": false }, { "name": "batch_size", "type": "int", "description": "number of txos to spend per transactions", "is_required": false }, { "name": "include_full_tx", "type": "bool", "description": "include entire tx in output and not just the txid", "is_required": false } ], "returns": " [\n {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }\n ]", "examples": [] }, { "name": "txo_sum", "description": "Sum of transaction outputs.", "arguments": [ { "name": "type", "type": "str or list", "description": "claim type: stream, channel, support, purchase, collection, repost, other", "is_required": false }, { "name": "txid", "type": "str or list", "description": "transaction id of outputs", "is_required": false }, { "name": "claim_id", "type": "str or list", "description": "claim id", "is_required": false }, { "name": "name", "type": "str or list", "description": "claim name", "is_required": false }, { "name": "channel_id", "type": "str or list", "description": "claims in this channel", "is_required": false }, { "name": "not_channel_id", "type": "str or list", "description": "claims not in this channel", "is_required": false }, { "name": "is_spent", "type": "bool", "description": "only show spent txos", "is_required": false }, { "name": "is_not_spent", "type": "bool", "description": "only show not spent txos", "is_required": false }, { "name": "is_my_input_or_output", "type": "bool", "description": "txos which have your inputs or your outputs, if using this flag the other related flags are ignored (--is_my_output, --is_my_input, etc)", "is_required": false }, { "name": "is_my_output", "type": "bool", "description": "show outputs controlled by you", "is_required": false }, { "name": "is_not_my_output", "type": "bool", "description": "show outputs not controlled by you", "is_required": false }, { "name": "is_my_input", "type": "bool", "description": "show outputs created by you", "is_required": false }, { "name": "is_not_my_input", "type": "bool", "description": "show outputs not created by you", "is_required": false }, { "name": "exclude_internal_transfers", "type": "bool", "description": "excludes any outputs that are exactly this combination: \"--is_my_input --is_my_output --type=other\" this allows to exclude \"change\" payments, this flag can be used in combination with any of the other flags", "is_required": false }, { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false } ], "returns": "int", "examples": [] } ] }, "utxo": { "doc": "Unspent transaction management.", "commands": [ { "name": "utxo_list", "description": "List unspent transaction outputs", "arguments": [ { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict results to specific wallet", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ]\n }", "examples": [] }, { "name": "utxo_release", "description": "When spending a UTXO it is locally locked to prevent double spends;\noccasionally this can result in a UTXO being locked which ultimately\ndid not get spent (failed to broadcast, spend transaction was not\naccepted by blockchain node, etc). This command releases the lock\non all UTXOs in your account.", "arguments": [ { "name": "account_id", "type": "str", "description": "id of the account to query", "is_required": false }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "None", "examples": [] } ] }, "wallet": { "doc": "Create, modify and inspect wallets.", "commands": [ { "name": "wallet_add", "description": "Add existing wallet.", "arguments": [ { "name": "wallet_id", "type": "str", "description": "wallet file name", "is_required": true } ], "returns": " {\n \"id\": \"wallet_id\",\n \"name\": \"optional wallet name\"\n }", "examples": [] }, { "name": "wallet_balance", "description": "Return the balance of a wallet", "arguments": [ { "name": "wallet_id", "type": "str", "description": "balance for specific wallet", "is_required": false }, { "name": "confirmations", "type": "int", "description": "Only include transactions with this many confirmed blocks.", "is_required": false } ], "returns": "(decimal) amount of lbry credits in wallet", "examples": [] }, { "name": "wallet_create", "description": "Create a new wallet.", "arguments": [ { "name": "wallet_id", "type": "str", "description": "wallet file name", "is_required": true }, { "name": "skip_on_startup", "type": "bool", "description": "don't add wallet to daemon_settings.yml", "is_required": false }, { "name": "create_account", "type": "bool", "description": "generates the default account", "is_required": false }, { "name": "single_key", "type": "bool", "description": "used with --create_account, creates single-key account", "is_required": false } ], "returns": " {\n \"id\": \"wallet_id\",\n \"name\": \"optional wallet name\"\n }", "examples": [] }, { "name": "wallet_decrypt", "description": "Decrypt an encrypted wallet, this will remove the wallet password. The wallet must be unlocked to decrypt it", "arguments": [ { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(bool) true if wallet is decrypted, otherwise false", "examples": [] }, { "name": "wallet_encrypt", "description": "Encrypt an unencrypted wallet with a password", "arguments": [ { "name": "new_password", "type": "str", "description": "password to encrypt account", "is_required": true }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(bool) true if wallet is decrypted, otherwise false", "examples": [] }, { "name": "wallet_list", "description": "List wallets.", "arguments": [ { "name": "wallet_id", "type": "str", "description": "show specific wallet only", "is_required": false }, { "name": "page", "type": "int", "description": "page to return during paginating", "is_required": false }, { "name": "page_size", "type": "int", "description": "number of items on page during pagination", "is_required": false } ], "returns": " {\n \"page\": \"Page number of the current items.\",\n \"page_size\": \"Number of items to show on a page.\",\n \"total_pages\": \"Total number of pages.\",\n \"total_items\": \"Total number of items.\",\n \"items\": [\n {\n \"id\": \"wallet_id\",\n \"name\": \"optional wallet name\"\n }\n ]\n }", "examples": [ { "title": "List your wallets", "curl": "curl -d'{\"method\": \"wallet_list\", \"params\": {}}' http://localhost:5279/", "lbrynet": "lbrynet wallet list", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"wallet_list\", \"params\": {}}).json()", "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"items\": [\n {\n \"id\": \"my_wallet.json\",\n \"name\": \"Wallet\"\n }\n ],\n \"page\": 1,\n \"page_size\": 20,\n \"total_items\": 1,\n \"total_pages\": 1\n }\n}" } ] }, { "name": "wallet_lock", "description": "Lock an unlocked wallet", "arguments": [ { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(bool) true if wallet is locked, otherwise false", "examples": [] }, { "name": "wallet_reconnect", "description": "Reconnects ledger network client, applying new configurations.", "arguments": [], "returns": "None", "examples": [] }, { "name": "wallet_remove", "description": "Remove an existing wallet.", "arguments": [ { "name": "wallet_id", "type": "str", "description": "name of wallet to remove", "is_required": true } ], "returns": " {\n \"id\": \"wallet_id\",\n \"name\": \"optional wallet name\"\n }", "examples": [] }, { "name": "wallet_send", "description": "Send the same number of credits to multiple addresses using all accounts in wallet to\nfund the transaction and the default account to receive any change.", "arguments": [ { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false }, { "name": "change_account_id", "type": "str", "description": "account where change will go", "is_required": false }, { "name": "funding_account_ids", "type": "str", "description": "accounts to fund the transaction", "is_required": false }, { "name": "preview", "type": "bool", "description": "do not broadcast the transaction", "is_required": false }, { "name": "blocking", "type": "bool", "description": "wait until tx has synced", "is_required": false } ], "returns": " {\n \"txid\": \"hash of transaction in hex\",\n \"height\": \"block where transaction was recorded\",\n \"inputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"outputs\": [\n {\n \"txid\": \"hash of transaction in hex\",\n \"nout\": \"position in the transaction\",\n \"height\": \"block where transaction was recorded\",\n \"amount\": \"value of the txo as a decimal\",\n \"address\": \"address of who can spend the txo\",\n \"confirmations\": \"number of confirmed blocks\",\n \"is_change\": \"payment to change address, only available when it can be determined\",\n \"is_received\": \"true if txo was sent from external account to this account\",\n \"is_spent\": \"true if txo is spent\",\n \"is_mine\": \"payment to one of your accounts, only available when it can be determined\",\n \"type\": \"one of 'claim', 'support' or 'purchase'\",\n \"name\": \"when type is 'claim' or 'support', this is the claim name\",\n \"claim_id\": \"when type is 'claim', 'support' or 'purchase', this is the claim id\",\n \"claim_op\": \"when type is 'claim', this determines if it is 'create' or 'update'\",\n \"value\": \"when type is 'claim' or 'support' with payload, this is the decoded protobuf payload\",\n \"value_type\": \"determines the type of the 'value' field: 'channel', 'stream', etc\",\n \"protobuf\": \"hex encoded raw protobuf version of 'value' field\",\n \"permanent_url\": \"when type is 'claim' or 'support', this is the long permanent claim URL\",\n \"claim\": \"for purchase outputs only, metadata of purchased claim\",\n \"reposted_claim\": \"for repost claims only, metadata of claim being reposted\",\n \"signing_channel\": \"for signed claims only, metadata of signing channel\",\n \"is_channel_signature_valid\": \"for signed claims only, whether signature is valid\",\n \"purchase_receipt\": \"metadata for the purchase transaction associated with this claim\"\n }\n ],\n \"total_input\": \"sum of inputs as a decimal\",\n \"total_output\": \"sum of outputs, sans fee, as a decimal\",\n \"total_fee\": \"fee amount\",\n \"hex\": \"entire transaction encoded in hex\"\n }", "examples": [] }, { "name": "wallet_status", "description": "Status of wallet including encryption/lock state.", "arguments": [ { "name": "wallet_id", "type": "str", "description": "status of specific wallet", "is_required": false } ], "returns": "Dictionary of wallet status information.", "examples": [] }, { "name": "wallet_unlock", "description": "Unlock an encrypted wallet", "arguments": [ { "name": "password", "type": "str", "description": "password to use for unlocking", "is_required": true }, { "name": "wallet_id", "type": "str", "description": "restrict operation to specific wallet", "is_required": false } ], "returns": "(bool) true if wallet is unlocked, otherwise false", "examples": [] } ] } } ================================================ FILE: example_daemon_settings.yml ================================================ # This is an example daemon_settings.yml file. # See https://lbry.tech/resources/daemon-settings for all configuration keys and values share_usage_data: True lbryum_servers: - lbryumx1.lbry.com:50001 - lbryumx2.lbry.com:50001 - lbryumx4.lbry.com:50001 blockchain_name: lbrycrd_main data_dir: /home/lbry/.lbrynet download_directory: /home/lbry/downloads save_blobs: true save_files: false dht_node_port: 4444 peer_port: 3333 use_upnp: true #components_to_skip: # - hash_announcer # - blob_server # - dht ================================================ FILE: lbry/.dockerignore ================================================ .git .tox __pycache__ dist lbry.egg-info docs tests ================================================ FILE: lbry/__init__.py ================================================ __version__ = "0.113.0" version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name ================================================ FILE: lbry/blob/__init__.py ================================================ from lbry.utils import get_lbry_hash_obj MAX_BLOB_SIZE = 2 * 2 ** 20 # digest_size is in bytes, and blob hashes are hex encoded BLOBHASH_LENGTH = get_lbry_hash_obj().digest_size * 2 ================================================ FILE: lbry/blob/blob_file.py ================================================ import os import re import time import asyncio import binascii import logging import typing import contextlib from io import BytesIO from cryptography.hazmat.primitives.ciphers import Cipher, modes from cryptography.hazmat.primitives.ciphers.algorithms import AES from cryptography.hazmat.primitives.padding import PKCS7 from cryptography.hazmat.backends import default_backend from lbry.utils import get_lbry_hash_obj from lbry.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError from lbry.blob import MAX_BLOB_SIZE, BLOBHASH_LENGTH from lbry.blob.blob_info import BlobInfo from lbry.blob.writer import HashBlobWriter log = logging.getLogger(__name__) HEXMATCH = re.compile("^[a-f,0-9]+$") BACKEND = default_backend() def is_valid_blobhash(blobhash: str) -> bool: """Checks whether the blobhash is the correct length and contains only valid characters (0-9, a-f) @param blobhash: string, the blobhash to check @return: True/False """ return len(blobhash) == BLOBHASH_LENGTH and HEXMATCH.match(blobhash) def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]: cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND) padder = PKCS7(AES.block_size).padder() encryptor = cipher.encryptor() encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize() digest = get_lbry_hash_obj() digest.update(encrypted) return encrypted, digest.hexdigest() def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes: if len(data) != length: raise ValueError("unexpected length") cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND) unpadder = PKCS7(AES.block_size).unpadder() decryptor = cipher.decryptor() return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize() class AbstractBlob: """ A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash This class is non-io specific """ __slots__ = [ 'loop', 'blob_hash', 'length', 'blob_completed_callback', 'blob_directory', 'writers', 'verified', 'writing', 'readers', 'added_on', 'is_mine', ] def __init__( self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None, blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False, ): self.loop = loop self.blob_hash = blob_hash self.length = length self.blob_completed_callback = blob_completed_callback self.blob_directory = blob_directory self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {} self.verified: asyncio.Event = asyncio.Event() self.writing: asyncio.Event = asyncio.Event() self.readers: typing.List[typing.BinaryIO] = [] self.added_on = added_on or time.time() self.is_mine = is_mine if not is_valid_blobhash(blob_hash): raise InvalidBlobHashError(blob_hash) def __del__(self): if self.writers or self.readers: log.warning("%s not closed before being garbage collected", self.blob_hash) self.close() @contextlib.contextmanager def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]: raise NotImplementedError() @contextlib.contextmanager def reader_context(self) -> typing.ContextManager[typing.BinaryIO]: if not self.is_readable(): raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers") with self._reader_context() as reader: try: self.readers.append(reader) yield reader finally: if reader in self.readers: self.readers.remove(reader) def _write_blob(self, blob_bytes: bytes) -> asyncio.Task: raise NotImplementedError() def set_length(self, length) -> None: if self.length is not None and length == self.length: return if self.length is None and 0 <= length <= MAX_BLOB_SIZE: self.length = length return log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length) def get_length(self) -> typing.Optional[int]: return self.length def get_is_verified(self) -> bool: return self.verified.is_set() def is_readable(self) -> bool: return self.verified.is_set() def is_writeable(self) -> bool: return not self.writing.is_set() def write_blob(self, blob_bytes: bytes): if not self.is_writeable(): raise OSError("cannot open blob for writing") try: self.writing.set() self._write_blob(blob_bytes) finally: self.writing.clear() def close(self): while self.writers: _, writer = self.writers.popitem() if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed(): writer.finished.cancel() while self.readers: reader = self.readers.pop() if reader: reader.close() def delete(self): self.close() self.verified.clear() self.length = None async def sendfile(self, writer: asyncio.StreamWriter) -> int: """ Read and send the file to the writer and return the number of bytes sent """ if not self.is_readable(): raise OSError('blob files cannot be read') with self.reader_context() as handle: try: return await self.loop.sendfile(writer.transport, handle, count=self.get_length()) except (ConnectionError, BrokenPipeError, RuntimeError, OSError, AttributeError): return -1 def decrypt(self, key: bytes, iv: bytes) -> bytes: """ Decrypt a BlobFile to plaintext bytes """ with self.reader_context() as reader: return decrypt_blob_bytes(reader.read(), self.length, key, iv) @classmethod async def create_from_unencrypted( cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes, unencrypted: bytes, blob_num: int, added_on: int, is_mine: bool, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None, ) -> BlobInfo: """ Create an encrypted BlobFile from plaintext bytes """ blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted) length = len(blob_bytes) blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir, added_on, is_mine) writer = blob.get_blob_writer() writer.write(blob_bytes) await blob.verified.wait() return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), added_on, blob_hash, is_mine) def save_verified_blob(self, verified_bytes: bytes): if self.verified.is_set(): return def update_events(_): self.verified.set() self.writing.clear() if self.is_writeable(): self.writing.set() task = self._write_blob(verified_bytes) task.add_done_callback(update_events) if self.blob_completed_callback: task.add_done_callback(lambda _: self.blob_completed_callback(self)) def get_blob_writer(self, peer_address: typing.Optional[str] = None, peer_port: typing.Optional[int] = None) -> HashBlobWriter: if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed(): raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}") fut = asyncio.Future() writer = HashBlobWriter(self.blob_hash, self.get_length, fut) self.writers[(peer_address, peer_port)] = writer def remove_writer(_): if (peer_address, peer_port) in self.writers: del self.writers[(peer_address, peer_port)] fut.add_done_callback(remove_writer) def writer_finished_callback(finished: asyncio.Future): try: err = finished.exception() if err: raise err verified_bytes = finished.result() while self.writers: _, other = self.writers.popitem() if other is not writer: other.close_handle() self.save_verified_blob(verified_bytes) except (InvalidBlobHashError, InvalidDataError) as error: log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error)) except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError): pass fut.add_done_callback(writer_finished_callback) return writer class BlobBuffer(AbstractBlob): """ An in-memory only blob """ def __init__( self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None, blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False ): self._verified_bytes: typing.Optional[BytesIO] = None super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine) @contextlib.contextmanager def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]: if not self.is_readable(): raise OSError("cannot open blob for reading") try: yield self._verified_bytes finally: if self._verified_bytes: self._verified_bytes.close() self._verified_bytes = None self.verified.clear() def _write_blob(self, blob_bytes: bytes): async def write(): if self._verified_bytes: raise OSError("already have bytes for blob") self._verified_bytes = BytesIO(blob_bytes) return self.loop.create_task(write()) def delete(self): if self._verified_bytes: self._verified_bytes.close() self._verified_bytes = None return super().delete() def __del__(self): super().__del__() if self._verified_bytes: self.delete() class BlobFile(AbstractBlob): """ A blob existing on the local file system """ def __init__( self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None, blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False ): super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine) if not blob_directory or not os.path.isdir(blob_directory): raise OSError(f"invalid blob directory '{blob_directory}'") self.file_path = os.path.join(self.blob_directory, self.blob_hash) if self.file_exists: file_size = int(os.stat(self.file_path).st_size) if length and length != file_size: log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size) self.delete() else: self.length = file_size self.verified.set() @property def file_exists(self): return os.path.isfile(self.file_path) def is_writeable(self) -> bool: return super().is_writeable() and not os.path.isfile(self.file_path) def get_blob_writer(self, peer_address: typing.Optional[str] = None, peer_port: typing.Optional[str] = None) -> HashBlobWriter: if self.file_exists: raise OSError(f"File already exists '{self.file_path}'") return super().get_blob_writer(peer_address, peer_port) @contextlib.contextmanager def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]: handle = open(self.file_path, 'rb') try: yield handle finally: handle.close() def _write_blob(self, blob_bytes: bytes): def _write_blob(): with open(self.file_path, 'wb') as f: f.write(blob_bytes) async def write_blob(): await self.loop.run_in_executor(None, _write_blob) return self.loop.create_task(write_blob()) def delete(self): super().delete() if os.path.isfile(self.file_path): os.remove(self.file_path) @classmethod async def create_from_unencrypted( cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes, unencrypted: bytes, blob_num: int, added_on: float, is_mine: bool, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None ) -> BlobInfo: if not blob_dir or not os.path.isdir(blob_dir): raise OSError(f"cannot create blob in directory: '{blob_dir}'") return await super().create_from_unencrypted( loop, blob_dir, key, iv, unencrypted, blob_num, added_on, is_mine, blob_completed_callback ) ================================================ FILE: lbry/blob/blob_info.py ================================================ import typing class BlobInfo: __slots__ = [ 'blob_hash', 'blob_num', 'length', 'iv', 'added_on', 'is_mine' ] def __init__( self, blob_num: int, length: int, iv: str, added_on, blob_hash: typing.Optional[str] = None, is_mine=False): self.blob_hash = blob_hash self.blob_num = blob_num self.length = length self.iv = iv self.added_on = added_on self.is_mine = is_mine def as_dict(self) -> typing.Dict: d = { 'length': self.length, 'blob_num': self.blob_num, 'iv': self.iv, } if self.blob_hash: # non-terminator blobs have a blob hash d['blob_hash'] = self.blob_hash return d ================================================ FILE: lbry/blob/blob_manager.py ================================================ import os import typing import asyncio import logging from lbry.utils import LRUCacheWithMetrics from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob from lbry.stream.descriptor import StreamDescriptor from lbry.connection_manager import ConnectionManager if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.dht.protocol.data_store import DictDataStore from lbry.extras.daemon.storage import SQLiteStorage log = logging.getLogger(__name__) class BlobManager: def __init__(self, loop: asyncio.AbstractEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config', node_data_store: typing.Optional['DictDataStore'] = None): """ This class stores blobs on the hard disk blob_dir - directory where blobs are stored storage - SQLiteStorage object """ self.loop = loop self.blob_dir = blob_dir self.storage = storage self._node_data_store = node_data_store self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\ else self._node_data_store.completed_blobs self.blobs: typing.Dict[str, AbstractBlob] = {} self.config = config self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics( self.config.blob_lru_cache_size) self.connection_manager = ConnectionManager(loop) def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None, is_mine: bool = False): if self.config.save_blobs or ( is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))): return BlobFile( self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine ) return BlobBuffer( self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine ) def get_blob(self, blob_hash, length: typing.Optional[int] = None, is_mine: bool = False): if blob_hash in self.blobs: if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer): buffer = self.blobs.pop(blob_hash) if blob_hash in self.completed_blob_hashes: self.completed_blob_hashes.remove(blob_hash) self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine) if buffer.is_readable(): with buffer.reader_context() as reader: self.blobs[blob_hash].write_blob(reader.read()) if length and self.blobs[blob_hash].length is None: self.blobs[blob_hash].set_length(length) else: self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine) return self.blobs[blob_hash] def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool: if not is_valid_blobhash(blob_hash): raise ValueError(blob_hash) if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)): return False if blob_hash in self.blobs: return self.blobs[blob_hash].get_is_verified() return self._get_blob(blob_hash, length).get_is_verified() async def setup(self) -> bool: def get_files_in_blob_dir() -> typing.Set[str]: if not self.blob_dir: return set() return { item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name) } in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir) to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir) if to_add: self.completed_blob_hashes.update(to_add) # check blobs that aren't set as finished but were seen on disk await self.ensure_completed_blobs_status(in_blobfiles_dir - to_add) if self.config.track_bandwidth: self.connection_manager.start() return True def stop(self): self.connection_manager.stop() while self.blobs: _, blob = self.blobs.popitem() blob.close() self.completed_blob_hashes.clear() def get_stream_descriptor(self, sd_hash): return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash)) def blob_completed(self, blob: AbstractBlob) -> asyncio.Task: if blob.blob_hash is None: raise Exception("Blob hash is None") if not blob.length: raise Exception("Blob has a length of 0") if isinstance(blob, BlobFile): if blob.blob_hash not in self.completed_blob_hashes: self.completed_blob_hashes.add(blob.blob_hash) return self.loop.create_task(self.storage.add_blobs( (blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=True) ) else: return self.loop.create_task(self.storage.add_blobs( (blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False) ) async def ensure_completed_blobs_status(self, blob_hashes: typing.Iterable[str]): """Ensures that completed blobs from a given list of blob hashes are set as 'finished' in the database.""" to_add = [] for blob_hash in blob_hashes: if not self.is_blob_verified(blob_hash): continue blob = self.get_blob(blob_hash) to_add.append((blob.blob_hash, blob.length, blob.added_on, blob.is_mine)) if len(to_add) > 500: await self.storage.add_blobs(*to_add, finished=True) to_add.clear() return await self.storage.add_blobs(*to_add, finished=True) def delete_blob(self, blob_hash: str): if not is_valid_blobhash(blob_hash): raise Exception("invalid blob hash to delete") if blob_hash not in self.blobs: if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)): os.remove(os.path.join(self.blob_dir, blob_hash)) else: self.blobs.pop(blob_hash).delete() if blob_hash in self.completed_blob_hashes: self.completed_blob_hashes.remove(blob_hash) async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True): for blob_hash in blob_hashes: self.delete_blob(blob_hash) if delete_from_db: await self.storage.delete_blobs_from_db(blob_hashes) ================================================ FILE: lbry/blob/disk_space_manager.py ================================================ import asyncio import logging log = logging.getLogger(__name__) class DiskSpaceManager: def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60, analytics=None): self.config = config self.db = db self.blob_manager = blob_manager self.cleaning_interval = cleaning_interval self.running = False self.task = None self.analytics = analytics self._used_space_bytes = None async def get_free_space_mb(self, is_network_blob=False): limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit space_used_mb = await self.get_space_used_mb() space_used_mb = space_used_mb['network_storage'] if is_network_blob else space_used_mb['content_storage'] return max(0, limit_mb - space_used_mb) async def get_space_used_bytes(self): self._used_space_bytes = await self.db.get_stored_blob_disk_usage() return self._used_space_bytes async def get_space_used_mb(self, cached=True): cached = cached and self._used_space_bytes is not None space_used_bytes = self._used_space_bytes if cached else await self.get_space_used_bytes() return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()} async def clean(self): await self._clean(False) await self._clean(True) async def _clean(self, is_network_blob=False): space_used_mb = await self.get_space_used_mb(cached=False) if is_network_blob: space_used_mb = space_used_mb['network_storage'] else: space_used_mb = space_used_mb['content_storage'] + space_used_mb['private_storage'] storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit if self.analytics: asyncio.create_task( self.analytics.send_disk_space_used(space_used_mb, storage_limit_mb, is_network_blob) ) delete = [] available = storage_limit_mb - space_used_mb if storage_limit_mb == 0 if not is_network_blob else available >= 0: return 0 for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False, is_network_blob=is_network_blob): delete.append(blob_hash) available += int(file_size/1024.0/1024.0) if available >= 0: break if delete: await self.db.stop_all_files() await self.blob_manager.delete_blobs(delete, delete_from_db=True) self._used_space_bytes = None return len(delete) async def cleaning_loop(self): while self.running: await asyncio.sleep(self.cleaning_interval) await self.clean() async def start(self): self.running = True self.task = asyncio.create_task(self.cleaning_loop()) self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service.")) async def stop(self): if self.running: self.running = False self.task.cancel() ================================================ FILE: lbry/blob/writer.py ================================================ import typing import logging import asyncio from io import BytesIO from lbry.error import InvalidBlobHashError, InvalidDataError from lbry.utils import get_lbry_hash_obj log = logging.getLogger(__name__) class HashBlobWriter: def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int], finished: asyncio.Future): self.expected_blob_hash = expected_blob_hash self.get_length = get_length self.buffer = BytesIO() self.finished = finished self.finished.add_done_callback(lambda *_: self.close_handle()) self._hashsum = get_lbry_hash_obj() self.len_so_far = 0 def __del__(self): if self.buffer is not None: log.warning("Garbage collection was called, but writer was not closed yet") self.close_handle() def calculate_blob_hash(self) -> str: return self._hashsum.hexdigest() def closed(self): return self.buffer is None or self.buffer.closed def write(self, data: bytes): expected_length = self.get_length() if not expected_length: raise OSError("unknown blob length") if self.buffer is None: log.warning("writer has already been closed") if not self.finished.done(): self.finished.cancel() return raise OSError('I/O operation on closed file') self._hashsum.update(data) self.len_so_far += len(data) if self.len_so_far > expected_length: self.finished.set_exception(InvalidDataError( f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}' )) self.close_handle() return self.buffer.write(data) if self.len_so_far == expected_length: blob_hash = self.calculate_blob_hash() if blob_hash != self.expected_blob_hash: self.finished.set_exception(InvalidBlobHashError( f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}" )) elif self.finished and not (self.finished.done() or self.finished.cancelled()): self.finished.set_result(self.buffer.getvalue()) self.close_handle() def close_handle(self): if not self.finished.done(): self.finished.cancel() if self.buffer is not None: self.buffer.close() self.buffer = None ================================================ FILE: lbry/blob_exchange/__init__.py ================================================ ================================================ FILE: lbry/blob_exchange/client.py ================================================ import asyncio import time import logging import typing import binascii from typing import Optional from lbry.error import InvalidBlobHashError, InvalidDataError from lbry.blob_exchange.serialization import BlobResponse, BlobRequest from lbry.utils import cache_concurrent if typing.TYPE_CHECKING: from lbry.blob.blob_file import AbstractBlob from lbry.blob.writer import HashBlobWriter from lbry.connection_manager import ConnectionManager log = logging.getLogger(__name__) class BlobExchangeClientProtocol(asyncio.Protocol): def __init__(self, loop: asyncio.AbstractEventLoop, peer_timeout: typing.Optional[float] = 10, connection_manager: typing.Optional['ConnectionManager'] = None): self.loop = loop self.peer_port: typing.Optional[int] = None self.peer_address: typing.Optional[str] = None self.transport: typing.Optional[asyncio.Transport] = None self.peer_timeout = peer_timeout self.connection_manager = connection_manager self.writer: typing.Optional['HashBlobWriter'] = None self.blob: typing.Optional['AbstractBlob'] = None self._blob_bytes_received = 0 self._response_fut: typing.Optional[asyncio.Future] = None self.buf = b'' # this is here to handle the race when the downloader is closed right as response_fut gets a result self.closed = asyncio.Event() def data_received(self, data: bytes): if self.connection_manager: if not self.peer_address: addr_info = self.transport.get_extra_info('peername') self.peer_address, self.peer_port = addr_info # assert self.peer_address is not None self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data)) if not self.transport or self.transport.is_closing(): log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port, binascii.hexlify(data)) if self._response_fut and not self._response_fut.done(): self._response_fut.cancel() return if not self._response_fut: log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.") return self.close() if self._blob_bytes_received and not self.writer.closed(): return self._write(data) response = BlobResponse.deserialize(self.buf + data) if not response.responses and not self._response_fut.done(): self.buf += data return else: self.buf = b'' if response.responses and self.blob: blob_response = response.get_blob_response() if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash: # set the expected length for the incoming blob if we didn't know it self.blob.set_length(blob_response.length) elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash: # the server started sending a blob we didn't request log.warning("%s started sending blob we didn't request %s instead of %s", self.peer_address, blob_response.blob_hash, self.blob.blob_hash) return if response.responses: log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict()) # fire the Future with the response to our request self._response_fut.set_result(response) if response.blob_data and self.writer and not self.writer.closed(): # log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port) # write blob bytes if we're writing a blob and have blob bytes to write self._write(response.blob_data) def _write(self, data: bytes): if len(data) > (self.blob.get_length() - self._blob_bytes_received): data = data[:(self.blob.get_length() - self._blob_bytes_received)] log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port) self._blob_bytes_received += len(data) try: self.writer.write(data) except OSError as err: log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err) if self._response_fut and not self._response_fut.done(): self._response_fut.set_exception(err) except asyncio.TimeoutError as err: log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port) if self._response_fut and not self._response_fut.done(): self._response_fut.set_exception(err) async def _download_blob(self) -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]: # pylint: disable=too-many-return-statements """ :return: download success (bool), connected protocol (BlobExchangeClientProtocol) """ start_time = time.perf_counter() request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash) blob_hash = self.blob.blob_hash if not self.peer_address: addr_info = self.transport.get_extra_info('peername') self.peer_address, self.peer_port = addr_info try: msg = request.serialize() log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode()) self.transport.write(msg) if self.connection_manager: self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg)) response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout) availability_response = response.get_availability_response() price_response = response.get_price_response() blob_response = response.get_blob_response() if self.closed.is_set(): msg = f"cancelled blob request for {blob_hash} immediately after we got a response" log.warning(msg) raise asyncio.CancelledError(msg) if (not blob_response or blob_response.error) and\ (not availability_response or not availability_response.available_blobs): log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address, self.peer_port) log.warning(response.to_dict()) return self._blob_bytes_received, self.close() elif availability_response and availability_response.available_blobs and \ availability_response.available_blobs != [self.blob.blob_hash]: log.warning("blob availability response doesn't match our request from %s:%i", self.peer_address, self.peer_port) return self._blob_bytes_received, self.close() elif not availability_response: log.warning("response from %s:%i did not include an availability response (we requested %s)", self.peer_address, self.peer_port, blob_hash) return self._blob_bytes_received, self.close() if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED': log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port) return self._blob_bytes_received, self.close() if not blob_response or blob_response.error: log.warning("blob can't be downloaded from %s:%i", self.peer_address, self.peer_port) return self._blob_bytes_received, self.close() if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash: log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port) return self._blob_bytes_received, self.close() if self.blob.length is not None and self.blob.length != blob_response.length: log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port) return self._blob_bytes_received, self.close() msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \ f" timeout in {self.peer_timeout}" log.debug(msg) msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}" await asyncio.wait_for(self.writer.finished, self.peer_timeout) # wait for the io to finish await self.blob.verified.wait() log.info("%s at %fMB/s", msg, round((float(self._blob_bytes_received) / float(time.perf_counter() - start_time)) / 1000000.0, 2)) # await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed? return self._blob_bytes_received, self except asyncio.TimeoutError: return self._blob_bytes_received, self.close() except (InvalidBlobHashError, InvalidDataError): log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port) return self._blob_bytes_received, self.close() def close(self): self.closed.set() if self._response_fut and not self._response_fut.done(): self._response_fut.cancel() if self.writer and not self.writer.closed(): self.writer.close_handle() self._response_fut = None self.writer = None self.blob = None if self.transport: self.transport.close() self.transport = None self.buf = b'' async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]: self.closed.clear() blob_hash = blob.blob_hash if blob.get_is_verified() or not blob.is_writeable(): return 0, self try: self._blob_bytes_received = 0 self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port) self._response_fut = asyncio.Future() return await self._download_blob() except OSError: # i'm not sure how to fix this race condition - jack log.warning("race happened downloading %s from %s:%s", blob_hash, self.peer_address, self.peer_port) # return self._blob_bytes_received, self.transport raise except asyncio.TimeoutError: if self._response_fut and not self._response_fut.done(): self._response_fut.cancel() self.close() return self._blob_bytes_received, None except asyncio.CancelledError: self.close() raise finally: if self.writer and not self.writer.closed(): self.writer.close_handle() self.writer = None def connection_made(self, transport: asyncio.Transport): addr = transport.get_extra_info('peername') self.peer_address, self.peer_port = addr[0], addr[1] self.transport = transport if self.connection_manager: self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}") log.debug("connection made to %s:%i", self.peer_address, self.peer_port) def connection_lost(self, exc): if self.connection_manager: self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}") log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(exc), str(type(exc))) self.close() @cache_concurrent async def request_blob(loop: asyncio.AbstractEventLoop, blob: Optional['AbstractBlob'], address: str, tcp_port: int, peer_connect_timeout: float, blob_download_timeout: float, connected_protocol: Optional['BlobExchangeClientProtocol'] = None, connection_id: int = 0, connection_manager: Optional['ConnectionManager'] = None)\ -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]: """ Returns [, ] """ protocol = connected_protocol if not connected_protocol or not connected_protocol.transport or connected_protocol.transport.is_closing(): connected_protocol = None protocol = BlobExchangeClientProtocol( loop, blob_download_timeout, connection_manager ) else: log.debug("reusing connection for %s:%d", address, tcp_port) try: if not connected_protocol: await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port), peer_connect_timeout) connected_protocol = protocol if blob is None or blob.get_is_verified() or not blob.is_writeable(): # blob is None happens when we are just opening a connection # file exists but not verified means someone is writing right now, give it time, come back later return 0, connected_protocol return await connected_protocol.download_blob(blob) except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError): return 0, None ================================================ FILE: lbry/blob_exchange/downloader.py ================================================ import asyncio import typing import logging from lbry.utils import cache_concurrent from lbry.blob_exchange.client import request_blob from lbry.dht.node import get_kademlia_peers_from_hosts if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.dht.node import Node from lbry.dht.peer import KademliaPeer from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_file import AbstractBlob from lbry.blob_exchange.client import BlobExchangeClientProtocol log = logging.getLogger(__name__) class BlobDownloader: BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', peer_queue: asyncio.Queue): self.loop = loop self.config = config self.blob_manager = blob_manager self.peer_queue = peer_queue self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls self.ignored: typing.Dict['KademliaPeer', int] = {} self.scores: typing.Dict['KademliaPeer', int] = {} self.failures: typing.Dict['KademliaPeer', int] = {} self.connection_failures: typing.Set['KademliaPeer'] = set() self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {} self.is_running = asyncio.Event() def should_race_continue(self, blob: 'AbstractBlob'): max_probes = self.config.max_connections_per_download * (1 if self.connections else 10) if len(self.active_connections) >= max_probes: return False return not (blob.get_is_verified() or not blob.is_writeable()) async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0, just_probe: bool = False): if blob.get_is_verified(): return start = self.loop.time() bytes_received, protocol = await request_blob( self.loop, blob if not just_probe else None, peer.address, peer.tcp_port, self.config.peer_connect_timeout, self.config.blob_download_timeout, connected_protocol=self.connections.get(peer), connection_id=connection_id, connection_manager=self.blob_manager.connection_manager ) if not bytes_received and not protocol and peer not in self.connection_failures: self.connection_failures.add(peer) if not protocol and peer not in self.ignored: self.ignored[peer] = self.loop.time() log.debug("drop peer %s:%i", peer.address, peer.tcp_port) self.failures[peer] = self.failures.get(peer, 0) + 1 if peer in self.connections: del self.connections[peer] elif protocol: log.debug("keep peer %s:%i", peer.address, peer.tcp_port) self.failures[peer] = 0 self.connections[peer] = protocol elapsed = self.loop.time() - start self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1 async def new_peer_or_finished(self): active_tasks = list(self.active_connections.values()) + [asyncio.create_task(asyncio.sleep(1))] await asyncio.wait(active_tasks, return_when='FIRST_COMPLETED') def cleanup_active(self): if not self.active_connections and not self.connections: self.clearbanned() to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()] for peer in to_remove: del self.active_connections[peer] def clearbanned(self): now = self.loop.time() self.ignored = { peer: when for (peer, when) in self.ignored.items() if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR)) } @cache_concurrent async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None, connection_id: int = 0) -> 'AbstractBlob': blob = self.blob_manager.get_blob(blob_hash, length) if blob.get_is_verified(): return blob self.is_running.set() try: while not blob.get_is_verified() and self.is_running.is_set(): batch: typing.Set['KademliaPeer'] = set(self.connections.keys()) while not self.peer_queue.empty(): batch.update(self.peer_queue.get_nowait()) log.debug( "%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6], len(batch), len(self.ignored), len(self.active_connections), len(self.connections) ) for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True): if peer in self.ignored: continue if peer in self.active_connections or not self.should_race_continue(blob): continue log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port) t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id)) self.active_connections[peer] = t self.peer_queue.put_nowait(list(batch)) await self.new_peer_or_finished() self.cleanup_active() log.debug("downloaded %s", blob_hash[:8]) return blob finally: blob.close() if self.loop.is_running(): self.loop.call_soon(self.cleanup_active) def close(self): self.connection_failures.clear() self.scores.clear() self.ignored.clear() self.is_running.clear() for protocol in self.connections.values(): protocol.close() async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', dht_node: 'Node', blob_hash: str) -> 'AbstractBlob': search_queue = asyncio.Queue(maxsize=config.max_connections_per_download) search_queue.put_nowait(blob_hash) peer_queue, accumulate_task = dht_node.accumulate_peers(search_queue) fixed_peers = None if not config.fixed_peers else await get_kademlia_peers_from_hosts(config.fixed_peers) if fixed_peers: loop.call_later(config.fixed_peer_delay, peer_queue.put_nowait, fixed_peers) downloader = BlobDownloader(loop, config, blob_manager, peer_queue) try: return await downloader.download_blob(blob_hash) finally: if accumulate_task and not accumulate_task.done(): accumulate_task.cancel() downloader.close() ================================================ FILE: lbry/blob_exchange/serialization.py ================================================ import typing import json import logging log = logging.getLogger(__name__) class BlobMessage: key = '' def to_dict(self) -> typing.Dict: raise NotImplementedError() class BlobPriceRequest(BlobMessage): key = 'blob_data_payment_rate' def __init__(self, blob_data_payment_rate: float, **kwargs) -> None: self.blob_data_payment_rate = blob_data_payment_rate def to_dict(self) -> typing.Dict: return { self.key: self.blob_data_payment_rate } class BlobPriceResponse(BlobMessage): key = 'blob_data_payment_rate' rate_accepted = 'RATE_ACCEPTED' rate_too_low = 'RATE_TOO_LOW' rate_unset = 'RATE_UNSET' def __init__(self, blob_data_payment_rate: str, **kwargs) -> None: if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset): raise ValueError(blob_data_payment_rate) self.blob_data_payment_rate = blob_data_payment_rate def to_dict(self) -> typing.Dict: return { self.key: self.blob_data_payment_rate } class BlobAvailabilityRequest(BlobMessage): key = 'requested_blobs' def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True, **kwargs) -> None: assert len(requested_blobs) > 0 self.requested_blobs = requested_blobs self.lbrycrd_address = lbrycrd_address def to_dict(self) -> typing.Dict: return { self.key: self.requested_blobs, 'lbrycrd_address': self.lbrycrd_address } class BlobAvailabilityResponse(BlobMessage): key = 'available_blobs' def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True, **kwargs) -> None: self.available_blobs = available_blobs self.lbrycrd_address = lbrycrd_address def to_dict(self) -> typing.Dict: d = { self.key: self.available_blobs } if self.lbrycrd_address: d['lbrycrd_address'] = self.lbrycrd_address return d class BlobDownloadRequest(BlobMessage): key = 'requested_blob' def __init__(self, requested_blob: str, **kwargs) -> None: self.requested_blob = requested_blob def to_dict(self) -> typing.Dict: return { self.key: self.requested_blob } class BlobDownloadResponse(BlobMessage): key = 'incoming_blob' def __init__(self, **response: typing.Dict) -> None: incoming_blob = response[self.key] self.error = None self.incoming_blob = None if 'error' in incoming_blob: self.error = incoming_blob['error'] else: self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']} self.length = None if not self.incoming_blob else self.incoming_blob['length'] self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash'] def to_dict(self) -> typing.Dict: return { self.key if not self.error else 'error': self.incoming_blob or self.error, } class BlobPaymentAddressRequest(BlobMessage): key = 'lbrycrd_address' def __init__(self, lbrycrd_address: str, **kwargs) -> None: self.lbrycrd_address = lbrycrd_address def to_dict(self) -> typing.Dict: return { self.key: self.lbrycrd_address } class BlobPaymentAddressResponse(BlobPaymentAddressRequest): pass class BlobErrorResponse(BlobMessage): key = 'error' def __init__(self, error: str, **kwargs) -> None: self.error = error def to_dict(self) -> typing.Dict: return { self.key: self.error } blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest, # pylint: disable=invalid-name BlobPaymentAddressRequest] blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse, # pylint: disable=invalid-name BlobErrorResponse, BlobPaymentAddressResponse] def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]: # scenarios: # # # curr_pos = 0 while True: next_close_paren = response_msg.find(b'}', curr_pos) if next_close_paren == -1: return None, response_msg curr_pos = next_close_paren + 1 try: response = json.loads(response_msg[:curr_pos]) except ValueError: continue possible_response_keys = { BlobPaymentAddressResponse.key, BlobAvailabilityResponse.key, BlobPriceResponse.key, BlobDownloadResponse.key } if isinstance(response, dict) and response.keys(): if set(response.keys()).issubset(possible_response_keys): return response, response_msg[curr_pos:] return None, response_msg class BlobRequest: def __init__(self, requests: typing.List[blob_request_types]) -> None: self.requests = requests def to_dict(self): d = {} for request in self.requests: d.update(request.to_dict()) return d def _get_request(self, request_type: blob_request_types): request = tuple(filter(lambda r: type(r) == request_type, self.requests)) # pylint: disable=unidiomatic-typecheck if request: return request[0] def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]: response = self._get_request(BlobAvailabilityRequest) if response: return response def get_price_request(self) -> typing.Optional[BlobPriceRequest]: response = self._get_request(BlobPriceRequest) if response: return response def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]: response = self._get_request(BlobDownloadRequest) if response: return response def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]: response = self._get_request(BlobPaymentAddressRequest) if response: return response def serialize(self) -> bytes: return json.dumps(self.to_dict()).encode() @classmethod def deserialize(cls, data: bytes) -> 'BlobRequest': request = json.loads(data) return cls([ request_type(**request) for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest, BlobPaymentAddressRequest) if request_type.key in request ]) @classmethod def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest': return cls( [BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)] ) class BlobResponse: def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None: self.responses = responses self.blob_data = blob_data def to_dict(self): d = {} for response in self.responses: d.update(response.to_dict()) return d def _get_response(self, response_type: blob_response_types): response = tuple(filter(lambda r: type(r) == response_type, self.responses)) # pylint: disable=unidiomatic-typecheck if response: return response[0] def get_error_response(self) -> typing.Optional[BlobErrorResponse]: error = self._get_response(BlobErrorResponse) if error: log.error(error) return error def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]: response = self._get_response(BlobAvailabilityResponse) if response: return response def get_price_response(self) -> typing.Optional[BlobPriceResponse]: response = self._get_response(BlobPriceResponse) if response: return response def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]: response = self._get_response(BlobDownloadResponse) if response: return response def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]: response = self._get_response(BlobPaymentAddressResponse) if response: return response def serialize(self) -> bytes: return json.dumps(self.to_dict()).encode() @classmethod def deserialize(cls, data: bytes) -> 'BlobResponse': response, extra = _parse_blob_response(data) requests = [] if response: requests.extend([ response_type(**response) for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse, BlobErrorResponse, BlobPaymentAddressResponse) if response_type.key in response ]) return cls(requests, extra) ================================================ FILE: lbry/blob_exchange/server.py ================================================ import asyncio import binascii import logging import socket import typing from json.decoder import JSONDecodeError from lbry.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types from lbry.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \ BlobPaymentAddressResponse if typing.TYPE_CHECKING: from lbry.blob.blob_manager import BlobManager log = logging.getLogger(__name__) # a standard request will be 295 bytes MAX_REQUEST_SIZE = 1200 class BlobServerProtocol(asyncio.Protocol): def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str, idle_timeout: float = 30.0, transfer_timeout: float = 60.0): self.loop = loop self.blob_manager = blob_manager self.idle_timeout = idle_timeout self.transfer_timeout = transfer_timeout self.server_task: typing.Optional[asyncio.Task] = None self.started_listening = asyncio.Event() self.buf = b'' self.transport: typing.Optional[asyncio.Transport] = None self.lbrycrd_address = lbrycrd_address self.peer_address_and_port: typing.Optional[str] = None self.started_transfer = asyncio.Event() self.transfer_finished = asyncio.Event() self.close_on_idle_task: typing.Optional[asyncio.Task] = None async def close_on_idle(self): while self.transport: try: await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout) except asyncio.TimeoutError: log.debug("closing idle connection from %s", self.peer_address_and_port) return self.close() self.started_transfer.clear() await self.transfer_finished.wait() self.transfer_finished.clear() def close(self): if self.transport: self.transport.close() def connection_made(self, transport): self.transport = transport self.close_on_idle_task = self.loop.create_task(self.close_on_idle()) self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername') self.blob_manager.connection_manager.connection_received(self.peer_address_and_port) log.debug("received connection from %s", self.peer_address_and_port) def connection_lost(self, exc: typing.Optional[Exception]) -> None: log.debug("lost connection from %s", self.peer_address_and_port) self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port) self.transport = None if self.close_on_idle_task and not self.close_on_idle_task.done(): self.close_on_idle_task.cancel() self.close_on_idle_task = None def send_response(self, responses: typing.List[blob_response_types]): to_send = [] while responses: to_send.append(responses.pop()) serialized = BlobResponse(to_send).serialize() self.transport.write(serialized) self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized)) async def handle_request(self, request: BlobRequest): addr = self.transport.get_extra_info('peername') peer_address, peer_port = addr responses = [] address_request = request.get_address_request() if address_request: responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address)) availability_request = request.get_availability_request() if availability_request: responses.append(BlobAvailabilityResponse(available_blobs=list(set( filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes, availability_request.requested_blobs) )))) price_request = request.get_price_request() if price_request: responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED')) download_request = request.get_blob_request() if download_request: blob = self.blob_manager.get_blob(download_request.requested_blob) if blob.get_is_verified(): incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length} responses.append(BlobDownloadResponse(incoming_blob=incoming_blob)) self.send_response(responses) blob_hash = blob.blob_hash[:8] log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port) self.started_transfer.set() try: sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout) if sent and sent > 0: self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent) log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port) else: self.close() log.debug("stopped sending %s to %s:%i", blob_hash, peer_address, peer_port) return except (OSError, ValueError, asyncio.TimeoutError) as err: if isinstance(err, asyncio.TimeoutError): log.debug("timed out sending blob %s to %s", blob_hash, peer_address) else: log.warning("could not read blob %s to send %s:%i", blob_hash, peer_address, peer_port) self.close() return finally: self.transfer_finished.set() else: log.info("don't have %s to send %s:%i", blob.blob_hash[:8], peer_address, peer_port) if responses and not self.transport.is_closing(): self.send_response(responses) def data_received(self, data): request = None if len(self.buf) + len(data or b'') >= MAX_REQUEST_SIZE: log.warning("request from %s is too large", self.peer_address_and_port) self.close() return if data: self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data)) _, separator, remainder = data.rpartition(b'}') if not separator: self.buf += data return try: request = BlobRequest.deserialize(self.buf + data) self.buf = remainder except (UnicodeDecodeError, JSONDecodeError): log.error("request from %s is not valid json (%i bytes): %s", self.peer_address_and_port, len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode()) self.close() return if not request.requests: log.error("failed to decode request from %s (%i bytes): %s", self.peer_address_and_port, len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode()) self.close() return self.loop.create_task(self.handle_request(request)) class BlobServer: def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str, idle_timeout: float = 30.0, transfer_timeout: float = 60.0): self.loop = loop self.blob_manager = blob_manager self.server_task: typing.Optional[asyncio.Task] = None self.started_listening = asyncio.Event() self.lbrycrd_address = lbrycrd_address self.idle_timeout = idle_timeout self.transfer_timeout = transfer_timeout self.server_protocol_class = BlobServerProtocol def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'): if self.server_task is not None: raise Exception("already running") async def _start_server(): # checking if the port is in use # thx https://stackoverflow.com/a/52872579 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: if s.connect_ex(('localhost', port)) == 0: # the port is already in use! log.error("Failed to bind TCP %s:%d", interface, port) server = await self.loop.create_server( lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address, self.idle_timeout, self.transfer_timeout), interface, port ) self.started_listening.set() log.info("Blob server listening on TCP %s:%i", interface, port) async with server: await server.serve_forever() self.server_task = self.loop.create_task(_start_server()) def stop_server(self): if self.server_task: self.server_task.cancel() self.server_task = None log.info("Stopped blob server") ================================================ FILE: lbry/build_info.py ================================================ # don't touch this. CI server changes this during build/deployment BUILD = "dev" COMMIT_HASH = "none" DOCKER_TAG = "none" ================================================ FILE: lbry/conf.py ================================================ import os import re import sys import logging from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional from argparse import ArgumentParser from contextlib import contextmanager from appdirs import user_data_dir, user_config_dir import yaml from lbry.error import InvalidCurrencyError from lbry.dht import constants from lbry.wallet.coinselection import STRATEGIES log = logging.getLogger(__name__) NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name T = TypeVar('T') CURRENCIES = { 'BTC': {'type': 'crypto'}, 'LBC': {'type': 'crypto'}, 'USD': {'type': 'fiat'}, } class Setting(Generic[T]): def __init__(self, doc: str, default: Optional[T] = None, previous_names: Optional[List[str]] = None, metavar: Optional[str] = None): self.doc = doc self.default = default self.previous_names = previous_names or [] self.metavar = metavar def __set_name__(self, owner, name): self.name = name # pylint: disable=attribute-defined-outside-init @property def cli_name(self): return f"--{self.name.replace('_', '-')}" @property def no_cli_name(self): return f"--no-{self.name.replace('_', '-')}" def __get__(self, obj: Optional['BaseConfig'], owner) -> T: if obj is None: return self for location in obj.search_order: if self.name in location: return location[self.name] return self.default def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]): if val == NOT_SET: for location in obj.modify_order: if self.name in location: del location[self.name] else: self.validate(val) for location in obj.modify_order: location[self.name] = val def is_set(self, obj: 'BaseConfig') -> bool: for location in obj.search_order: if self.name in location: return True return False def is_set_to_default(self, obj: 'BaseConfig') -> bool: for location in obj.search_order: if self.name in location: return location[self.name] == self.default return False def validate(self, value): raise NotImplementedError() def deserialize(self, value): # pylint: disable=no-self-use return value def serialize(self, value): # pylint: disable=no-self-use return value def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, metavar=self.metavar, default=NOT_SET ) class String(Setting[str]): def validate(self, value): assert isinstance(value, str), \ f"Setting '{self.name}' must be a string." # TODO: removes this after pylint starts to understand generics def __get__(self, obj: Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation return super().__get__(obj, owner) class Integer(Setting[int]): def validate(self, value): assert isinstance(value, int), \ f"Setting '{self.name}' must be an integer." def deserialize(self, value): return int(value) class Float(Setting[float]): def validate(self, value): assert isinstance(value, float), \ f"Setting '{self.name}' must be a decimal." def deserialize(self, value): return float(value) class Toggle(Setting[bool]): def validate(self, value): assert isinstance(value, bool), \ f"Setting '{self.name}' must be a true/false value." def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, action="store_true", default=NOT_SET ) parser.add_argument( self.no_cli_name, help=f"Opposite of {self.cli_name}", dest=self.name, action="store_false", default=NOT_SET ) class Path(String): def __init__(self, doc: str, *args, default: str = '', **kwargs): super().__init__(doc, default, *args, **kwargs) def __get__(self, obj, owner) -> str: value = super().__get__(obj, owner) if isinstance(value, str): return os.path.expanduser(os.path.expandvars(value)) return value class MaxKeyFee(Setting[dict]): def validate(self, value): if value is not None: assert isinstance(value, dict) and set(value) == {'currency', 'amount'}, \ f"Setting '{self.name}' must be a dict like \"{{'amount': 50.0, 'currency': 'USD'}}\"." if value["currency"] not in CURRENCIES: raise InvalidCurrencyError(value["currency"]) @staticmethod def _parse_list(l): if l == ['null']: return None assert len(l) == 2, ( 'Max key fee is made up of either two values: ' '"AMOUNT CURRENCY", or "null" (to set no limit)' ) try: amount = float(l[0]) except ValueError: raise AssertionError('First value in max key fee is a decimal: "AMOUNT CURRENCY"') currency = str(l[1]).upper() if currency not in CURRENCIES: raise InvalidCurrencyError(currency) return {'amount': amount, 'currency': currency} def deserialize(self, value): if value is None: return if isinstance(value, dict): return { 'currency': value['currency'], 'amount': float(value['amount']), } if isinstance(value, str): value = value.split() if isinstance(value, list): return self._parse_list(value) raise AssertionError('Invalid max key fee.') def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, nargs='+', metavar=('AMOUNT', 'CURRENCY'), default=NOT_SET ) parser.add_argument( self.no_cli_name, help="Disable maximum key fee check.", dest=self.name, const=None, action="store_const", default=NOT_SET ) class StringChoice(String): def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs): super().__init__(doc, default, *args, **kwargs) if not valid_values: raise ValueError("No valid values provided") if default not in valid_values: raise ValueError(f"Default value must be one of: {', '.join(valid_values)}") self.valid_values = valid_values def validate(self, value): super().validate(value) if value not in self.valid_values: raise ValueError(f"Setting '{self.name}' value must be one of: {', '.join(self.valid_values)}") class ListSetting(Setting[list]): def validate(self, value): assert isinstance(value, (tuple, list)), \ f"Setting '{self.name}' must be a tuple or list." def contribute_to_argparse(self, parser: ArgumentParser): parser.add_argument( self.cli_name, help=self.doc, action='append' ) class Servers(ListSetting): def validate(self, value): assert isinstance(value, (tuple, list)), \ f"Setting '{self.name}' must be a tuple or list of servers." for idx, server in enumerate(value): assert isinstance(server, (tuple, list)) and len(server) == 2, \ f"Server defined '{server}' at index {idx} in setting " \ f"'{self.name}' must be a tuple or list of two items." assert isinstance(server[0], str), \ f"Server defined '{server}' at index {idx} in setting " \ f"'{self.name}' must be have hostname as string in first position." assert isinstance(server[1], int), \ f"Server defined '{server}' at index {idx} in setting " \ f"'{self.name}' must be have port as int in second position." def deserialize(self, value): servers = [] if isinstance(value, list): for server in value: if isinstance(server, str) and server.count(':') == 1: host, port = server.split(':') try: servers.append((host, int(port))) except ValueError: pass return servers def serialize(self, value): if value: return [f"{host}:{port}" for host, port in value] return value class Strings(ListSetting): def validate(self, value): assert isinstance(value, (tuple, list)), \ f"Setting '{self.name}' must be a tuple or list of strings." for idx, string in enumerate(value): assert isinstance(string, str), \ f"Value of '{string}' at index {idx} in setting " \ f"'{self.name}' must be a string." class KnownHubsList: def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'): self.file_name = file_name self.path = os.path.join(config.wallet_dir, self.file_name) if config else None self.hubs: Dict[Tuple[str, int], Dict] = {} if self.exists: self.load() @property def exists(self): return self.path and os.path.exists(self.path) @property def serialized(self) -> Dict[str, Dict]: return {f"{host}:{port}": details for (host, port), details in self.hubs.items()} def filter(self, match_none=False, **kwargs): if not kwargs: return self.hubs result = {} for hub, details in self.hubs.items(): for key, constraint in kwargs.items(): value = details.get(key) if value == constraint or (match_none and value is None): result[hub] = details break return result def load(self): if self.path: with open(self.path, 'r') as known_hubs_file: raw = known_hubs_file.read() for hub, details in yaml.safe_load(raw).items(): self.set(hub, details) def save(self): if self.path: with open(self.path, 'w') as known_hubs_file: known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False)) def set(self, hub: str, details: Dict): if hub and hub.count(':') == 1: host, port = hub.split(':') hub_parts = (host, int(port)) if hub_parts not in self.hubs: self.hubs[hub_parts] = details return hub def add_hubs(self, hubs: List[str]): added = False for hub in hubs: if self.set(hub, {}) is not None: added = True return added def items(self): return self.hubs.items() def __bool__(self): return len(self) > 0 def __len__(self): return self.hubs.__len__() def __iter__(self): return iter(self.hubs) class EnvironmentAccess: PREFIX = 'LBRY_' def __init__(self, config: 'BaseConfig', environ: dict): self.configuration = config self.data = {} if environ: self.load(environ) def load(self, environ): for setting in self.configuration.get_settings(): value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET) if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None): self.data[setting.name] = setting.deserialize(value) def __contains__(self, item: str): return item in self.data def __getitem__(self, item: str): return self.data[item] class ArgumentAccess: def __init__(self, config: 'BaseConfig', args: dict): self.configuration = config self.args = {} if args: self.load(args) def load(self, args): for setting in self.configuration.get_settings(): value = getattr(args, setting.name, NOT_SET) if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None): self.args[setting.name] = setting.deserialize(value) def __contains__(self, item: str): return item in self.args def __getitem__(self, item: str): return self.args[item] class ConfigFileAccess: def __init__(self, config: 'BaseConfig', path: str): self.configuration = config self.path = path self.data = {} if self.exists: self.load() @property def exists(self): return self.path and os.path.exists(self.path) def load(self): cls = type(self.configuration) with open(self.path, 'r') as config_file: raw = config_file.read() serialized = yaml.safe_load(raw) or {} for key, value in serialized.items(): attr = getattr(cls, key, None) if attr is None: for setting in self.configuration.settings: if key in setting.previous_names: attr = setting break if attr is not None: self.data[key] = attr.deserialize(value) def save(self): cls = type(self.configuration) serialized = {} for key, value in self.data.items(): attr = getattr(cls, key) serialized[key] = attr.serialize(value) with open(self.path, 'w') as config_file: config_file.write(yaml.safe_dump(serialized, default_flow_style=False)) def upgrade(self) -> bool: upgraded = False for key in list(self.data): for setting in self.configuration.settings: if key in setting.previous_names: self.data[setting.name] = self.data[key] del self.data[key] upgraded = True break return upgraded def __contains__(self, item: str): return item in self.data def __getitem__(self, item: str): return self.data[item] def __setitem__(self, key, value): self.data[key] = value def __delitem__(self, key): del self.data[key] TBC = TypeVar('TBC', bound='BaseConfig') class BaseConfig: config = Path("Path to configuration file.", metavar='FILE') def __init__(self, **kwargs): self.runtime = {} # set internally or by various API calls self.arguments = {} # from command line arguments self.environment = {} # from environment variables self.persisted = {} # from config file self._updating_config = False for key, value in kwargs.items(): setattr(self, key, value) @contextmanager def update_config(self): self._updating_config = True yield self self._updating_config = False if isinstance(self.persisted, ConfigFileAccess): self.persisted.save() @property def modify_order(self): locations = [self.runtime] if self._updating_config: locations.append(self.persisted) return locations @property def search_order(self): return [ self.runtime, self.arguments, self.environment, self.persisted ] @classmethod def get_settings(cls): for attr in dir(cls): setting = getattr(cls, attr) if isinstance(setting, Setting): yield setting @property def settings(self): return self.get_settings() @property def settings_dict(self): return { setting.name: getattr(self, setting.name) for setting in self.settings } @classmethod def create_from_arguments(cls, args) -> TBC: conf = cls() conf.set_arguments(args) conf.set_environment() conf.set_persisted() return conf @classmethod def contribute_to_argparse(cls, parser: ArgumentParser): for setting in cls.get_settings(): setting.contribute_to_argparse(parser) def set_arguments(self, args): self.arguments = ArgumentAccess(self, args) def set_environment(self, environ=None): self.environment = EnvironmentAccess(self, environ or os.environ) def set_persisted(self, config_file_path=None): if config_file_path is None: config_file_path = self.config if not config_file_path: return ext = os.path.splitext(config_file_path)[1] assert ext in ('.yml', '.yaml'),\ f"File extension '{ext}' is not supported, " \ f"configuration file must be in YAML (.yaml)." self.persisted = ConfigFileAccess(self, config_file_path) if self.persisted.upgrade(): self.persisted.save() class TranscodeConfig(BaseConfig): ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. ' f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}', '', previous_names=['ffmpeg_folder']) video_encoder = String('FFmpeg codec and parameters for the video encoding. ' 'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental', 'libx264 -crf 24 -preset faster -pix_fmt yuv420p') video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000) video_scaler = String('FFmpeg scaling parameters for reducing bitrate. ' 'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M', r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" ' r'-maxrate 5500K -bufsize 5000K') audio_encoder = String('FFmpeg codec and parameters for the audio encoding. ' 'Example: libopus -b:a 128k', 'aac -b:a 160k') volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '') volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240) class CLIConfig(TranscodeConfig): api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT') @property def api_connection_url(self) -> str: return f"http://{self.api}/lbryapi" @property def api_host(self): return self.api.split(':')[0] @property def api_port(self): return int(self.api.split(':')[1]) class Config(CLIConfig): jurisdiction = String("Limit interactions to wallet server in this jurisdiction.") # directories data_dir = Path("Directory path to store blobs.", metavar='DIR') download_dir = Path( "Directory path to place assembled files downloaded from LBRY.", previous_names=['download_directory'], metavar='DIR' ) wallet_dir = Path( "Directory containing a 'wallets' subdirectory with 'default_wallet' file.", previous_names=['lbryum_wallet_dir'], metavar='DIR' ) wallets = Strings( "Wallet files in 'wallet_dir' to load at startup.", ['default_wallet'] ) # network use_upnp = Toggle( "Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward" "ports or have firewall rules you likely want to disable this.", True ) udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port']) tcp_port = Integer("TCP port to listen for incoming blob requests", 4444, previous_names=['peer_port']) prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0) network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0') # routing table split_buckets_under_index = Integer( "Routing table bucket index below which we always split the bucket if given a new key to add to it and " "the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) " "will increase. This setting is used by seed nodes, you probably don't want to change it during normal " "use.", 2 ) is_bootstrap_node = Toggle( "When running as a bootstrap node, disable all logic related to balancing the routing table, so we can " "add as many peers as possible and better help first-runs.", False ) # protocol timeouts download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0) blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0) hub_timeout = Float("Timeout when making a hub request", 30.0) peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0) node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT) # blob announcement and download save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True) network_storage_limit = Integer("Disk space in MB to be allocated for helping the P2P network. 0 = disable", 0) blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0) blob_lru_cache_size = Integer( "LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when " "replying to a range request. Set to 0 to disable.", 32 ) announce_head_and_sd_only = Toggle( "Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True, previous_names=['announce_head_blobs_only'] ) concurrent_blob_announcers = Integer( "Number of blobs to iteratively announce at once, set to 0 to disable", 10, previous_names=['concurrent_announcers'] ) max_connections_per_download = Integer( "Maximum number of peers to connect to while downloading a blob", 4, previous_names=['max_connections_per_stream'] ) concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32) fixed_peer_delay = Float( "Amount of seconds before adding the reflector servers as potential peers to download from in case dht" "peers are not found or are slow", 2.0 ) max_key_fee = MaxKeyFee( "Don't download streams with fees exceeding this amount. When set to " "null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0} ) max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0") # reflector settings reflect_streams = Toggle( "Upload completed streams (published and downloaded) reflector in order to re-host them", True, previous_names=['reflect_uploads'] ) concurrent_reflector_uploads = Integer( "Maximum number of streams to upload to a reflector server at a time", 10 ) # servers reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [ ('reflector.lbry.com', 5566) ]) fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [ ('cdn.reflector.lbry.com', 5567) ]) tracker_servers = Servers("BitTorrent-compatible (BEP15) UDP trackers for helping P2P discovery", [ ('tracker.lbry.com', 9252), ('tracker.lbry.grin.io', 9252), ('tracker.lbry.pigg.es', 9252), ('tracker.lizard.technology', 9252), ('s1.lbry.network', 9252), ]) lbryum_servers = Servers("SPV wallet servers", [ ('spv11.lbry.com', 50001), ('spv12.lbry.com', 50001), ('spv13.lbry.com', 50001), ('spv14.lbry.com', 50001), ('spv15.lbry.com', 50001), ('spv16.lbry.com', 50001), ('spv17.lbry.com', 50001), ('spv18.lbry.com', 50001), ('spv19.lbry.com', 50001), ('hub.lbry.grin.io', 50001), ('hub.lizard.technology', 50001), ('s1.lbry.network', 50001), ]) known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [ ('dht.lbry.grin.io', 4444), # Grin ('dht.lbry.madiator.com', 4444), # Madiator ('dht.lbry.pigg.es', 4444), # Pigges ('lbrynet1.lbry.com', 4444), # US EAST ('lbrynet2.lbry.com', 4444), # US WEST ('lbrynet3.lbry.com', 4444), # EU ('lbrynet4.lbry.com', 4444), # ASIA ('dht.lizard.technology', 4444), # Jack ('s2.lbry.network', 4444), ]) # blockchain blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main') # daemon save_files = Toggle("Save downloaded files when calling `get` by default", False) components_to_skip = Strings("components which will be skipped during start-up of daemon", []) share_usage_data = Toggle( "Whether to share usage stats and diagnostic info with LBRY.", False, previous_names=['upload_log', 'upload_log', 'share_debug_info'] ) track_bandwidth = Toggle("Track bandwidth usage", True) allowed_origin = String( "Allowed `Origin` header value for API request (sent by browser), use * to allow " "all hosts; default is to only allow API requests with no `Origin` value.", "") # media server streaming_server = String('Host name and port to serve streaming media over range requests', 'localhost:5280', metavar='HOST:PORT') streaming_get = Toggle("Enable the /get endpoint for the streaming media server. " "Disable to prevent new streams from being added.", True) coin_selection_strategy = StringChoice( "Strategy to use when selecting UTXOs for a transaction", STRATEGIES, "prefer_confirmed" ) transaction_cache_size = Integer("Transaction cache size", 2 ** 17) save_resolved_claims = Toggle( "Save content claims to the database when they are resolved to keep file_list up to date, " "only disable this if file_x commands are not needed", True ) @property def streaming_host(self): return self.streaming_server.split(':')[0] @property def streaming_port(self): return int(self.streaming_server.split(':')[1]) def __init__(self, **kwargs): super().__init__(**kwargs) self.set_default_paths() self.known_hubs = KnownHubsList(self) def set_default_paths(self): if 'darwin' in sys.platform.lower(): get_directories = get_darwin_directories elif 'win' in sys.platform.lower(): get_directories = get_windows_directories elif 'linux' in sys.platform.lower(): get_directories = get_linux_directories else: return cls = type(self) cls.data_dir.default, cls.wallet_dir.default, cls.download_dir.default = get_directories() cls.config.default = os.path.join( self.data_dir, 'daemon_settings.yml' ) @property def log_file_path(self): return os.path.join(self.data_dir, 'lbrynet.log') def get_windows_directories() -> Tuple[str, str, str]: from lbry.winpaths import get_path, FOLDERID, UserHandle, \ PathNotFoundException # pylint: disable=import-outside-toplevel try: download_dir = get_path(FOLDERID.Downloads, UserHandle.current) except PathNotFoundException: download_dir = os.getcwd() # old appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current) data_dir = os.path.join(appdata, 'lbrynet') lbryum_dir = os.path.join(appdata, 'lbryum') if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir): return data_dir, lbryum_dir, download_dir # new data_dir = user_data_dir('lbrynet', 'lbry') lbryum_dir = user_data_dir('lbryum', 'lbry') return data_dir, lbryum_dir, download_dir def get_darwin_directories() -> Tuple[str, str, str]: data_dir = user_data_dir('LBRY') lbryum_dir = os.path.expanduser('~/.lbryum') download_dir = os.path.expanduser('~/Downloads') return data_dir, lbryum_dir, download_dir def get_linux_directories() -> Tuple[str, str, str]: try: with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg: down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()) if down_dir: down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1)) download_dir = re.sub('\"', '', down_dir) except OSError: download_dir = os.getenv('XDG_DOWNLOAD_DIR') if not download_dir: download_dir = os.path.expanduser('~/Downloads') # old data_dir = os.path.expanduser('~/.lbrynet') lbryum_dir = os.path.expanduser('~/.lbryum') if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir): return data_dir, lbryum_dir, download_dir # new return user_data_dir('lbry/lbrynet'), user_data_dir('lbry/lbryum'), download_dir ================================================ FILE: lbry/connection_manager.py ================================================ import time import asyncio import typing import collections import logging log = logging.getLogger(__name__) CONNECTED_EVENT = "connected" DISCONNECTED_EVENT = "disconnected" TRANSFERRED_EVENT = "transferred" class ConnectionManager: def __init__(self, loop: asyncio.AbstractEventLoop): self.loop = loop self.incoming_connected: typing.Set[str] = set() self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int) self.outgoing_connected: typing.Set[str] = set() self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int) self._max_incoming_mbs = 0.0 self._max_outgoing_mbs = 0.0 self._status = {} self._running = False self._task: typing.Optional[asyncio.Task] = None @property def status(self): return self._status def sent_data(self, host_and_port: str, size: int): if self._running: self.outgoing[host_and_port] += size def received_data(self, host_and_port: str, size: int): if self._running: self.incoming[host_and_port] += size def connection_made(self, host_and_port: str): if self._running: self.outgoing_connected.add(host_and_port) def connection_received(self, host_and_port: str): # self.incoming_connected.add(host_and_port) pass def outgoing_connection_lost(self, host_and_port: str): if self._running and host_and_port in self.outgoing_connected: self.outgoing_connected.remove(host_and_port) def incoming_connection_lost(self, host_and_port: str): if self._running and host_and_port in self.incoming_connected: self.incoming_connected.remove(host_and_port) async def _update(self): self._status = { 'incoming_bps': {}, 'outgoing_bps': {}, 'total_incoming_mbs': 0.0, 'total_outgoing_mbs': 0.0, 'total_sent': 0, 'total_received': 0, 'max_incoming_mbs': 0.0, 'max_outgoing_mbs': 0.0 } while True: last = time.perf_counter() await asyncio.sleep(0.1) self._status['incoming_bps'].clear() self._status['outgoing_bps'].clear() now = time.perf_counter() while self.outgoing: k, sent = self.outgoing.popitem() self._status['total_sent'] += sent self._status['outgoing_bps'][k] = sent / (now - last) while self.incoming: k, received = self.incoming.popitem() self._status['total_received'] += received self._status['incoming_bps'][k] = received / (now - last) self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values()) )) / 1000000.0 self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values()) )) / 1000000.0 self._max_incoming_mbs = max(self._max_incoming_mbs, self._status['total_incoming_mbs']) self._max_outgoing_mbs = max(self._max_outgoing_mbs, self._status['total_outgoing_mbs']) self._status['max_incoming_mbs'] = self._max_incoming_mbs self._status['max_outgoing_mbs'] = self._max_outgoing_mbs def stop(self): if self._task: self._task.cancel() self._task = None self.outgoing.clear() self.outgoing_connected.clear() self.incoming.clear() self.incoming_connected.clear() self._status.clear() self._running = False def start(self): self.stop() self._running = True self._task = self.loop.create_task(self._update()) ================================================ FILE: lbry/constants.py ================================================ CENT = 1000000 COIN = 100*CENT ================================================ FILE: lbry/crypto/__init__.py ================================================ ================================================ FILE: lbry/crypto/base58.py ================================================ from lbry.crypto.hash import double_sha256 from lbry.crypto.util import bytes_to_int, int_to_bytes class Base58Error(Exception): """ Exception used for Base58 errors. """ class Base58: """ Class providing base 58 functionality. """ chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' assert len(chars) == 58 char_map = {c: n for n, c in enumerate(chars)} @classmethod def char_value(cls, c): val = cls.char_map.get(c) if val is None: raise Base58Error(f'invalid base 58 character "{c}"') return val @classmethod def decode(cls, txt): """ Decodes txt into a big-endian bytearray. """ if isinstance(txt, memoryview): txt = str(txt) if isinstance(txt, bytes): txt = txt.decode() if not isinstance(txt, str): raise TypeError('a string is required') if not txt: raise Base58Error('string cannot be empty') value = 0 for c in txt: value = value * 58 + cls.char_value(c) result = int_to_bytes(value) # Prepend leading zero bytes if necessary count = 0 for c in txt: if c != '1': break count += 1 if count: result = bytes((0,)) * count + result return result @classmethod def encode(cls, be_bytes): """Converts a big-endian bytearray into a base58 string.""" value = bytes_to_int(be_bytes) txt = '' while value: value, mod = divmod(value, 58) txt += cls.chars[mod] for byte in be_bytes: if byte != 0: break txt += '1' return txt[::-1] @classmethod def decode_check(cls, txt, hash_fn=double_sha256): """ Decodes a Base58Check-encoded string to a payload. The version prefixes it. """ be_bytes = cls.decode(txt) result, check = be_bytes[:-4], be_bytes[-4:] if check != hash_fn(result)[:4]: raise Base58Error(f'invalid base 58 checksum for {txt}') return result @classmethod def encode_check(cls, payload, hash_fn=double_sha256): """ Encodes a payload bytearray (which includes the version byte(s)) into a Base58Check string.""" be_bytes = payload + hash_fn(payload)[:4] return cls.encode(be_bytes) ================================================ FILE: lbry/crypto/crypt.py ================================================ import os import base64 import typing from cryptography.hazmat.primitives.kdf.scrypt import Scrypt from cryptography.hazmat.primitives.ciphers import Cipher, modes from cryptography.hazmat.primitives.ciphers.algorithms import AES from cryptography.hazmat.primitives.padding import PKCS7 from cryptography.hazmat.backends import default_backend from lbry.error import InvalidPasswordError from lbry.crypto.hash import double_sha256 def aes_encrypt(secret: str, value: str, init_vector: bytes = None) -> str: if init_vector is not None: assert len(init_vector) == 16 else: init_vector = os.urandom(16) key = double_sha256(secret.encode()) encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor() padder = PKCS7(AES.block_size).padder() padded_data = padder.update(value.encode()) + padder.finalize() encrypted_data = encryptor.update(padded_data) + encryptor.finalize() return base64.b64encode(init_vector + encrypted_data).decode() def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]: try: data = base64.b64decode(value.encode()) key = double_sha256(secret.encode()) init_vector, data = data[:16], data[16:] decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor() unpadder = PKCS7(AES.block_size).unpadder() result = unpadder.update(decryptor.update(data)) + unpadder.finalize() return result.decode(), init_vector except UnicodeDecodeError: raise InvalidPasswordError() except ValueError as e: if e.args[0] == 'Invalid padding bytes.': raise InvalidPasswordError() raise def better_aes_encrypt(secret: str, value: bytes) -> bytes: init_vector = os.urandom(16) key = scrypt(secret.encode(), salt=init_vector) encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor() padder = PKCS7(AES.block_size).padder() padded_data = padder.update(value) + padder.finalize() encrypted_data = encryptor.update(padded_data) + encryptor.finalize() return base64.b64encode(b's:8192:16:1:' + init_vector + encrypted_data) def better_aes_decrypt(secret: str, value: bytes) -> bytes: try: data = base64.b64decode(value) _, scryp_n, scrypt_r, scrypt_p, data = data.split(b':', maxsplit=4) init_vector, data = data[:16], data[16:] key = scrypt(secret.encode(), init_vector, int(scryp_n), int(scrypt_r), int(scrypt_p)) decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor() unpadder = PKCS7(AES.block_size).unpadder() return unpadder.update(decryptor.update(data)) + unpadder.finalize() except ValueError as e: if e.args[0] == 'Invalid padding bytes.': raise InvalidPasswordError() raise def scrypt(passphrase, salt, scrypt_n=1<<13, scrypt_r=16, scrypt_p=1): kdf = Scrypt(salt, length=32, n=scrypt_n, r=scrypt_r, p=scrypt_p, backend=default_backend()) return kdf.derive(passphrase) ================================================ FILE: lbry/crypto/hash.py ================================================ import hashlib import hmac from binascii import hexlify, unhexlify def sha256(x): """ Simple wrapper of hashlib sha256. """ return hashlib.sha256(x).digest() def sha512(x): """ Simple wrapper of hashlib sha512. """ return hashlib.sha512(x).digest() def ripemd160(x): """ Simple wrapper of hashlib ripemd160. """ h = hashlib.new('ripemd160') h.update(x) return h.digest() def double_sha256(x): """ SHA-256 of SHA-256, as used extensively in bitcoin. """ return sha256(sha256(x)) def hmac_sha512(key, msg): """ Use SHA-512 to provide an HMAC. """ return hmac.new(key, msg, hashlib.sha512).digest() def hash160(x): """ RIPEMD-160 of SHA-256. Used to make bitcoin addresses from pubkeys. """ return ripemd160(sha256(x)) def hash_to_hex_str(x): """ Convert a big-endian binary hash to displayed hex string. Display form of a binary hash is reversed and converted to hex. """ return hexlify(reversed(x)) def hex_str_to_hash(x): """ Convert a displayed hex string to a binary hash. """ return reversed(unhexlify(x)) ================================================ FILE: lbry/crypto/util.py ================================================ from binascii import unhexlify, hexlify def bytes_to_int(be_bytes): """ Interprets a big-endian sequence of bytes as an integer. """ return int(hexlify(be_bytes), 16) def int_to_bytes(value): """ Converts an integer to a big-endian sequence of bytes. """ length = (value.bit_length() + 7) // 8 s = '%x' % value return unhexlify(('0' * (len(s) % 2) + s).zfill(length * 2)) ================================================ FILE: lbry/dht/__init__.py ================================================ ================================================ FILE: lbry/dht/blob_announcer.py ================================================ import asyncio import typing import logging from prometheus_client import Counter, Gauge if typing.TYPE_CHECKING: from lbry.dht.node import Node from lbry.extras.daemon.storage import SQLiteStorage log = logging.getLogger(__name__) class BlobAnnouncer: announcements_sent_metric = Counter( "announcements_sent", "Number of announcements sent and their respective status.", namespace="dht_node", labelnames=("peers", "error"), ) announcement_queue_size_metric = Gauge( "announcement_queue_size", "Number of hashes waiting to be announced.", namespace="dht_node", labelnames=("scope",) ) def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'): self.loop = loop self.node = node self.storage = storage self.announce_task: asyncio.Task = None self.announce_queue: typing.List[str] = [] self._done = asyncio.Event() self.announced = set() async def _run_consumer(self): while self.announce_queue: try: blob_hash = self.announce_queue.pop() peers = len(await self.node.announce_blob(blob_hash)) self.announcements_sent_metric.labels(peers=peers, error=False).inc() if peers > 4: self.announced.add(blob_hash) else: log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers) except Exception as err: self.announcements_sent_metric.labels(peers=0, error=True).inc() log.warning("error announcing %s: %s", blob_hash[:8], str(err)) async def _announce(self, batch_size: typing.Optional[int] = 10): while batch_size: if not self.node.joined.is_set(): await self.node.joined.wait() await asyncio.sleep(60) if not self.node.protocol.routing_table.get_peers(): log.warning("No peers in DHT, announce round skipped") continue self.announce_queue.extend(await self.storage.get_blobs_to_announce()) self.announcement_queue_size_metric.labels(scope="global").set(len(self.announce_queue)) log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue)) while len(self.announce_queue) > 0: log.info("%i blobs to announce", len(self.announce_queue)) await asyncio.gather(*[self._run_consumer() for _ in range(batch_size)]) announced = list(filter(None, self.announced)) if announced: await self.storage.update_last_announced_blobs(announced) log.info("announced %i blobs", len(announced)) self.announced.clear() self._done.set() self._done.clear() def start(self, batch_size: typing.Optional[int] = 10): assert not self.announce_task or self.announce_task.done(), "already running" self.announce_task = self.loop.create_task(self._announce(batch_size)) def stop(self): if self.announce_task and not self.announce_task.done(): self.announce_task.cancel() def wait(self): return self._done.wait() ================================================ FILE: lbry/dht/constants.py ================================================ import hashlib import os HASH_CLASS = hashlib.sha384 # pylint: disable=invalid-name HASH_LENGTH = HASH_CLASS().digest_size HASH_BITS = HASH_LENGTH * 8 ALPHA = 5 K = 8 SPLIT_BUCKETS_UNDER_INDEX = 1 REPLACEMENT_CACHE_SIZE = 8 RPC_TIMEOUT = 5.0 RPC_ATTEMPTS = 5 RPC_ATTEMPTS_PRUNING_WINDOW = 600 ITERATIVE_LOOKUP_DELAY = RPC_TIMEOUT / 2.0 # TODO: use config val / 2 if rpc timeout is provided REFRESH_INTERVAL = 3600 # 1 hour REPLICATE_INTERVAL = REFRESH_INTERVAL DATA_EXPIRATION = 86400 # 24 hours TOKEN_SECRET_REFRESH_INTERVAL = 300 # 5 minutes MAYBE_PING_DELAY = 300 # 5 minutes CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5 RPC_ID_LENGTH = 20 PROTOCOL_VERSION = 1 MSG_SIZE_LIMIT = 1400 def digest(data: bytes) -> bytes: h = HASH_CLASS() h.update(data) return h.digest() def generate_id(num=None) -> bytes: if num is not None: return digest(str(num).encode()) else: return digest(os.urandom(32)) def generate_rpc_id(num=None) -> bytes: return generate_id(num)[:RPC_ID_LENGTH] ================================================ FILE: lbry/dht/error.py ================================================ class BaseKademliaException(Exception): pass class DecodeError(BaseKademliaException): """ Should be raised by an C{Encoding} implementation if decode operation fails """ class BucketFull(BaseKademliaException): """ Raised when the bucket is full """ class RemoteException(BaseKademliaException): pass class TransportNotConnected(BaseKademliaException): pass ================================================ FILE: lbry/dht/node.py ================================================ import logging import asyncio import typing import socket from prometheus_client import Gauge from lbry.utils import aclosing, resolve_host from lbry.dht import constants from lbry.dht.peer import make_kademlia_peer from lbry.dht.protocol.distance import Distance from lbry.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder from lbry.dht.protocol.protocol import KademliaProtocol if typing.TYPE_CHECKING: from lbry.dht.peer import PeerManager from lbry.dht.peer import KademliaPeer log = logging.getLogger(__name__) class Node: storing_peers_metric = Gauge( "storing_peers", "Number of peers storing blobs announced to this node", namespace="dht_node", labelnames=("scope",), ) stored_blob_with_x_bytes_colliding = Gauge( "stored_blobs_x_bytes_colliding", "Number of blobs with at least X bytes colliding with this node id prefix", namespace="dht_node", labelnames=("amount",) ) def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int, internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT, split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False, storage: typing.Optional['SQLiteStorage'] = None): self.loop = loop self.internal_udp_port = internal_udp_port self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout, split_buckets_under_index, is_bootstrap_node) self.listening_port: asyncio.DatagramTransport = None self.joined = asyncio.Event() self._join_task: asyncio.Task = None self._refresh_task: asyncio.Task = None self._storage = storage @property def stored_blob_hashes(self): return self.protocol.data_store.keys() async def refresh_node(self, force_once=False): while True: # remove peers with expired blob announcements from the datastore self.protocol.data_store.removed_expired_peers() total_peers: typing.List['KademliaPeer'] = [] # add all peers in the routing table total_peers.extend(self.protocol.routing_table.get_peers()) # add all the peers who have announced blobs to us storing_peers = self.protocol.data_store.get_storing_contacts() self.storing_peers_metric.labels("global").set(len(storing_peers)) total_peers.extend(storing_peers) counts = {0: 0, 1: 0, 2: 0} node_id = self.protocol.node_id for blob_hash in self.protocol.data_store.keys(): bytes_colliding = 0 if blob_hash[0] != node_id[0] else 2 if blob_hash[1] == node_id[1] else 1 counts[bytes_colliding] += 1 self.stored_blob_with_x_bytes_colliding.labels(amount=0).set(counts[0]) self.stored_blob_with_x_bytes_colliding.labels(amount=1).set(counts[1]) self.stored_blob_with_x_bytes_colliding.labels(amount=2).set(counts[2]) # get ids falling in the midpoint of each bucket that hasn't been recently updated node_ids = self.protocol.routing_table.get_refresh_list(0, True) if self.protocol.routing_table.get_peers(): # if we have node ids to look up, perform the iterative search until we have k results while node_ids: peers = await self.peer_search(node_ids.pop()) total_peers.extend(peers) else: if force_once: break fut = asyncio.Future() self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None) await fut continue # ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True] if to_ping: self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0) if self._storage: await self._storage.save_kademlia_peers(self.protocol.routing_table.get_peers()) if force_once: break fut = asyncio.Future() self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None) await fut async def announce_blob(self, blob_hash: str) -> typing.List[bytes]: hash_value = bytes.fromhex(blob_hash) assert len(hash_value) == constants.HASH_LENGTH peers = await self.peer_search(hash_value) if not self.protocol.external_ip: raise Exception("Cannot determine external IP") log.debug("Store to %i peers", len(peers)) for peer in peers: log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port) stored_to_tup = await asyncio.gather( *(self.protocol.store_to_peer(hash_value, peer) for peer in peers) ) stored_to = [node_id for node_id, contacted in stored_to_tup if contacted] if stored_to: log.debug( "Stored %s to %i of %i attempted peers", hash_value.hex()[:8], len(stored_to), len(peers) ) else: log.debug("Failed announcing %s, stored to 0 peers", blob_hash[:8]) return stored_to def stop(self) -> None: if self.joined.is_set(): self.joined.clear() if self._join_task: self._join_task.cancel() if self._refresh_task and not (self._refresh_task.done() or self._refresh_task.cancelled()): self._refresh_task.cancel() if self.protocol and self.protocol.ping_queue.running: self.protocol.ping_queue.stop() self.protocol.stop() if self.listening_port is not None: self.listening_port.close() self._join_task = None self.listening_port = None log.info("Stopped DHT node") async def start_listening(self, interface: str = '0.0.0.0') -> None: if not self.listening_port: self.listening_port, _ = await self.loop.create_datagram_endpoint( lambda: self.protocol, (interface, self.internal_udp_port) ) log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port) self.protocol.start() else: log.warning("Already bound to port %s", self.listening_port) async def join_network(self, interface: str = '0.0.0.0', known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None): def peers_from_urls(urls: typing.Optional[typing.List[typing.Tuple[bytes, str, int, int]]]): peer_addresses = [] for node_id, address, udp_port, tcp_port in urls: if (node_id, address, udp_port, tcp_port) not in peer_addresses and \ (address, udp_port) != (self.protocol.external_ip, self.protocol.udp_port): peer_addresses.append((node_id, address, udp_port, tcp_port)) return [make_kademlia_peer(*peer_address) for peer_address in peer_addresses] if not self.listening_port: await self.start_listening(interface) self.protocol.ping_queue.start() self._refresh_task = self.loop.create_task(self.refresh_node()) while True: if self.protocol.routing_table.get_peers(): if not self.joined.is_set(): self.joined.set() log.info( "joined dht, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()), self.protocol.routing_table.buckets_with_contacts() ) else: if self.joined.is_set(): self.joined.clear() seed_peers = peers_from_urls( await self._storage.get_persisted_kademlia_peers() ) if self._storage else [] if not seed_peers: try: seed_peers.extend(peers_from_urls([ (None, await resolve_host(address, udp_port, 'udp'), udp_port, None) for address, udp_port in known_node_urls or [] ])) except socket.gaierror: await asyncio.sleep(30) continue self.protocol.peer_manager.reset() self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0) await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32) await asyncio.sleep(1) def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None): self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls)) def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None, max_results: int = constants.K) -> IterativeNodeFinder: shortlist = shortlist or self.protocol.routing_table.find_close_peers(key) return IterativeNodeFinder(self.loop, self.protocol, key, max_results, shortlist) def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None, max_results: int = -1) -> IterativeValueFinder: shortlist = shortlist or self.protocol.routing_table.find_close_peers(key) return IterativeValueFinder(self.loop, self.protocol, key, max_results, shortlist) async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2, shortlist: typing.Optional[typing.List['KademliaPeer']] = None ) -> typing.List['KademliaPeer']: peers = [] async with aclosing(self.get_iterative_node_finder( node_id, shortlist=shortlist, max_results=max_results)) as node_finder: async for iteration_peers in node_finder: peers.extend(iteration_peers) distance = Distance(node_id) peers.sort(key=lambda peer: distance(peer.node_id)) return peers[:count] async def _accumulate_peers_for_value(self, search_queue: asyncio.Queue, result_queue: asyncio.Queue): tasks = [] try: while True: blob_hash = await search_queue.get() tasks.append(self.loop.create_task(self._peers_for_value_producer(blob_hash, result_queue))) finally: for task in tasks: task.cancel() async def _peers_for_value_producer(self, blob_hash: str, result_queue: asyncio.Queue): async def put_into_result_queue_after_pong(_peer): try: await self.protocol.get_rpc_peer(_peer).ping() result_queue.put_nowait([_peer]) log.debug("pong from %s:%i for %s", _peer.address, _peer.udp_port, blob_hash) except asyncio.TimeoutError: pass # prioritize peers who reply to a dht ping first # this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers async with aclosing(self.get_iterative_value_finder(bytes.fromhex(blob_hash))) as value_finder: async for results in value_finder: to_put = [] for peer in results: if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port: continue is_good = self.protocol.peer_manager.peer_is_good(peer) if is_good: # the peer has replied recently over UDP, it can probably be reached on the TCP port to_put.append(peer) elif is_good is None: if not peer.udp_port: # TODO: use the same port for TCP and UDP # the udp port must be guessed # default to the ports being the same. if the TCP port appears to be <=0.48.0 default, # including on a network with several nodes, then assume the udp port is proportionately # based on a starting port of 4444 udp_port_to_try = peer.tcp_port if 3400 > peer.tcp_port > 3332: udp_port_to_try = (peer.tcp_port - 3333) + 4444 self.loop.create_task(put_into_result_queue_after_pong( make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port) )) else: self.loop.create_task(put_into_result_queue_after_pong(peer)) else: # the peer is known to be bad/unreachable, skip trying to connect to it over TCP log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash) if to_put: result_queue.put_nowait(to_put) def accumulate_peers(self, search_queue: asyncio.Queue, peer_queue: typing.Optional[asyncio.Queue] = None ) -> typing.Tuple[asyncio.Queue, asyncio.Task]: queue = peer_queue or asyncio.Queue() return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue)) async def get_kademlia_peers_from_hosts(peer_list: typing.List[typing.Tuple[str, int]]) -> typing.List['KademliaPeer']: peer_address_list = [(await resolve_host(url, port, proto='tcp'), port) for url, port in peer_list] kademlia_peer_list = [make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True) for address, port in peer_address_list] return kademlia_peer_list ================================================ FILE: lbry/dht/peer.py ================================================ import typing import asyncio import logging from dataclasses import dataclass, field from functools import lru_cache from prometheus_client import Gauge from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache from lbry.dht import constants from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address ALLOW_LOCALHOST = False CACHE_SIZE = 16384 log = logging.getLogger(__name__) @lru_cache(CACHE_SIZE) def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str], udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None, allow_localhost: bool = False) -> 'KademliaPeer': return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost) def is_valid_public_ipv4(address, allow_localhost: bool = False): allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST) return _is_valid_public_ipv4(address, allow_localhost) class PeerManager: peer_manager_keys_metric = Gauge( "peer_manager_keys", "Number of keys tracked by PeerManager dicts (sum)", namespace="dht_node", labelnames=("scope",) ) def __init__(self, loop: asyncio.AbstractEventLoop): self._loop = loop self._rpc_failures: typing.Dict[ typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]] ] = LRUCache(CACHE_SIZE) self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE) self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE) self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE) self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(CACHE_SIZE) self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(CACHE_SIZE) self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(CACHE_SIZE) def count_cache_keys(self): return len(self._rpc_failures) + len(self._last_replied) + len(self._last_sent) + len( self._last_requested) + len(self._node_id_mapping) + len(self._node_id_reverse_mapping) + len( self._node_tokens) def reset(self): for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested): statistic.clear() def report_failure(self, address: str, udp_port: int): now = self._loop.time() _, previous = self._rpc_failures.pop((address, udp_port), (None, None)) self._rpc_failures[(address, udp_port)] = (previous, now) def report_last_sent(self, address: str, udp_port: int): now = self._loop.time() self._last_sent[(address, udp_port)] = now def report_last_replied(self, address: str, udp_port: int): now = self._loop.time() self._last_replied[(address, udp_port)] = now def report_last_requested(self, address: str, udp_port: int): now = self._loop.time() self._last_requested[(address, udp_port)] = now def clear_token(self, node_id: bytes): self._node_tokens.pop(node_id, None) def update_token(self, node_id: bytes, token: bytes): now = self._loop.time() self._node_tokens[node_id] = (now, token) def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]: ts, token = self._node_tokens.get(node_id, (0, None)) if ts and ts > self._loop.time() - constants.TOKEN_SECRET_REFRESH_INTERVAL: return token def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]: return self._last_replied.get((address, udp_port)) def update_contact_triple(self, node_id: bytes, address: str, udp_port: int): """ Update the mapping of node_id -> address tuple and that of address tuple -> node_id This is to handle peers changing addresses and ids while assuring that the we only ever have one node id / address tuple mapped to each other """ if (address, udp_port) in self._node_id_mapping: self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port))) if node_id in self._node_id_reverse_mapping: self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id)) self._node_id_mapping[(address, udp_port)] = node_id self._node_id_reverse_mapping[node_id] = (address, udp_port) self.peer_manager_keys_metric.labels("global").set(self.count_cache_keys()) def get_node_id_for_endpoint(self, address, port): return self._node_id_mapping.get((address, port)) def prune(self): # TODO: periodically call this now = self._loop.time() to_pop = [] for (address, udp_port), (_, last_failure) in self._rpc_failures.items(): if last_failure and last_failure < now - constants.RPC_ATTEMPTS_PRUNING_WINDOW: to_pop.append((address, udp_port)) while to_pop: del self._rpc_failures[to_pop.pop()] to_pop = [] for node_id, (age, token) in self._node_tokens.items(): # pylint: disable=unused-variable if age < now - constants.TOKEN_SECRET_REFRESH_INTERVAL: to_pop.append(node_id) while to_pop: del self._node_tokens[to_pop.pop()] def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int): # pylint: disable=too-many-return-statements """ :return: False if peer is bad, None if peer is unknown, or True if peer is good """ delay = self._loop.time() - constants.CHECK_REFRESH_INTERVAL # fixme: find a way to re-enable that without breaking other parts # if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping: # return # addr_tup = (address, udp_port) # if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id: # return previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None)) last_requested = self._last_requested.get((address, udp_port)) last_replied = self._last_replied.get((address, udp_port)) if node_id is None: return None if most_recent_failure and last_replied: if delay < last_replied > most_recent_failure: return True elif last_replied > most_recent_failure: return return False elif previous_failure and most_recent_failure and most_recent_failure > delay: return False elif last_replied and last_replied > delay: return True elif last_requested and last_requested > delay: return None return def peer_is_good(self, peer: 'KademliaPeer'): return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port) def decode_tcp_peer_from_compact_address(compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use node_id, address, tcp_port = decode_compact_address(compact_address) return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port) @dataclass(unsafe_hash=True) class KademliaPeer: address: str = field(hash=True) _node_id: typing.Optional[bytes] = field(hash=True) udp_port: typing.Optional[int] = field(hash=True) tcp_port: typing.Optional[int] = field(compare=False, hash=False) protocol_version: typing.Optional[int] = field(default=1, compare=False, hash=False) allow_localhost: bool = field(default=False, compare=False, hash=False) def __post_init__(self): if self._node_id is not None: if not len(self._node_id) == constants.HASH_LENGTH: raise ValueError("invalid node_id: {}".format(self._node_id.hex())) if self.udp_port is not None and not 1024 <= self.udp_port <= 65535: raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}") if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535: raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}") if not is_valid_public_ipv4(self.address, self.allow_localhost): raise ValueError(f"invalid ip address: '{self.address}'") def update_tcp_port(self, tcp_port: int): self.tcp_port = tcp_port @property def node_id(self) -> bytes: return self._node_id def compact_address_udp(self) -> bytearray: return make_compact_address(self.node_id, self.address, self.udp_port) def compact_address_tcp(self) -> bytearray: return make_compact_address(self.node_id, self.address, self.tcp_port) def compact_ip(self): return make_compact_ip(self.address) def __str__(self): return f"{self.__class__.__name__}({self.node_id.hex()[:8]}@{self.address}:{self.udp_port}-{self.tcp_port})" ================================================ FILE: lbry/dht/protocol/__init__.py ================================================ ================================================ FILE: lbry/dht/protocol/data_store.py ================================================ import asyncio import typing from lbry.dht import constants if typing.TYPE_CHECKING: from lbry.dht.peer import KademliaPeer, PeerManager class DictDataStore: def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager'): # Dictionary format: # { : [(, ), ...] } self._data_store: typing.Dict[bytes, typing.List[typing.Tuple['KademliaPeer', float]]] = {} self.loop = loop self._peer_manager = peer_manager self.completed_blobs: typing.Set[str] = set() def keys(self): return self._data_store.keys() def __len__(self): return self._data_store.__len__() def removed_expired_peers(self): now = self.loop.time() keys = list(self._data_store.keys()) for key in keys: to_remove = [] for (peer, ts) in self._data_store[key]: if ts + constants.DATA_EXPIRATION < now or self._peer_manager.peer_is_good(peer) is False: to_remove.append((peer, ts)) for item in to_remove: self._data_store[key].remove(item) if not self._data_store[key]: del self._data_store[key] def filter_bad_and_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']: """ Returns only non-expired and unknown/good peers """ for peer in self.filter_expired_peers(key): if self._peer_manager.peer_is_good(peer) is not False: yield peer def filter_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']: """ Returns only non-expired peers """ now = self.loop.time() for (peer, ts) in self._data_store.get(key, []): if ts + constants.DATA_EXPIRATION > now: yield peer def has_peers_for_blob(self, key: bytes) -> bool: return key in self._data_store def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes) -> None: now = self.loop.time() if key in self._data_store: current = list(filter(lambda x: x[0] == contact, self._data_store[key])) if len(current) > 0: self._data_store[key][self._data_store[key].index(current[0])] = contact, now else: self._data_store[key].append((contact, now)) else: self._data_store[key] = [(contact, now)] def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']: return list(self.filter_bad_and_expired_peers(key)) def get_storing_contacts(self) -> typing.List['KademliaPeer']: peers = set() for _, stored in self._data_store.items(): peers.update(set(map(lambda tup: tup[0], stored))) return list(peers) ================================================ FILE: lbry/dht/protocol/distance.py ================================================ from lbry.dht import constants class Distance: """Calculate the XOR result between two string variables. Frequently we re-use one of the points so as an optimization we pre-calculate the value of that point. """ def __init__(self, key: bytes): if len(key) != constants.HASH_LENGTH: raise ValueError(f"invalid key length: {len(key)}") self.key = key self.val_key_one = int.from_bytes(key, 'big') def __call__(self, key_two: bytes) -> int: if len(key_two) != constants.HASH_LENGTH: raise ValueError(f"invalid length of key to compare: {len(key_two)}") val_key_two = int.from_bytes(key_two, 'big') return self.val_key_one ^ val_key_two def is_closer(self, key_a: bytes, key_b: bytes) -> bool: """Returns true is `key_a` is closer to `key` than `key_b` is""" return self(key_a) < self(key_b) ================================================ FILE: lbry/dht/protocol/iterative_find.py ================================================ import asyncio from itertools import chain from collections import defaultdict, OrderedDict from collections.abc import AsyncIterator import typing import logging from typing import TYPE_CHECKING from lbry.dht import constants from lbry.dht.error import RemoteException, TransportNotConnected from lbry.dht.protocol.distance import Distance from lbry.dht.peer import make_kademlia_peer, decode_tcp_peer_from_compact_address from lbry.dht.serialization.datagram import PAGE_KEY if TYPE_CHECKING: from lbry.dht.protocol.protocol import KademliaProtocol from lbry.dht.peer import PeerManager, KademliaPeer log = logging.getLogger(__name__) class FindResponse: @property def found(self) -> bool: raise NotImplementedError() def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]: raise NotImplementedError() def get_close_kademlia_peers(self, peer_info) -> typing.Generator[typing.Iterator['KademliaPeer'], None, None]: for contact_triple in self.get_close_triples(): node_id, address, udp_port = contact_triple try: yield make_kademlia_peer(node_id, address, udp_port) except ValueError: log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer_info.address, peer_info.udp_port, address, udp_port) class FindNodeResponse(FindResponse): def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]): self.key = key self.close_triples = close_triples @property def found(self) -> bool: return self.key in [triple[0] for triple in self.close_triples] def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]: return self.close_triples class FindValueResponse(FindResponse): def __init__(self, key: bytes, result_dict: typing.Dict): self.key = key self.token = result_dict[b'token'] self.close_triples: typing.List[typing.Tuple[bytes, bytes, int]] = result_dict.get(b'contacts', []) self.found_compact_addresses = result_dict.get(key, []) self.pages = int(result_dict.get(PAGE_KEY, 0)) @property def found(self) -> bool: return len(self.found_compact_addresses) > 0 def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]: return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples] class IterativeFinder(AsyncIterator): def __init__(self, loop: asyncio.AbstractEventLoop, protocol: 'KademliaProtocol', key: bytes, max_results: typing.Optional[int] = constants.K, shortlist: typing.Optional[typing.List['KademliaPeer']] = None): if len(key) != constants.HASH_LENGTH: raise ValueError("invalid key length: %i" % len(key)) self.loop = loop self.peer_manager = protocol.peer_manager self.protocol = protocol self.key = key self.max_results = max(constants.K, max_results) self.active: typing.Dict['KademliaPeer', int] = OrderedDict() # peer: distance, sorted self.contacted: typing.Set['KademliaPeer'] = set() self.distance = Distance(key) self.iteration_queue = asyncio.Queue() self.running_probes: typing.Dict['KademliaPeer', asyncio.Task] = {} self.iteration_count = 0 self.running = False self.tasks: typing.List[asyncio.Task] = [] for peer in shortlist: if peer.node_id: self._add_active(peer, force=True) else: # seed nodes self._schedule_probe(peer) async def send_probe(self, peer: 'KademliaPeer') -> FindResponse: """ Send the rpc request to the peer and return an object with the FindResponse interface """ raise NotImplementedError() def search_exhausted(self): """ This method ends the iterator due no more peers to contact. Override to provide last time results. """ self.iteration_queue.put_nowait(None) def check_result_ready(self, response: FindResponse): """ Called after adding peers from an rpc result to the shortlist. This method is responsible for putting a result for the generator into the Queue """ raise NotImplementedError() def get_initial_result(self) -> typing.List['KademliaPeer']: #pylint: disable=no-self-use """ Get an initial or cached result to be put into the Queue. Used for findValue requests where the blob has peers in the local data store of blobs announced to us """ return [] def _add_active(self, peer, force=False): if not force and self.peer_manager.peer_is_good(peer) is False: return if peer in self.contacted: return if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id: self.active[peer] = self.distance(peer.node_id) self.active = OrderedDict(sorted(self.active.items(), key=lambda item: item[1])) async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse): self._add_active(peer) for new_peer in response.get_close_kademlia_peers(peer): self._add_active(new_peer) self.check_result_ready(response) self._log_state(reason="check result") def _reset_closest(self, peer): if peer in self.active: del self.active[peer] async def _send_probe(self, peer: 'KademliaPeer'): try: response = await self.send_probe(peer) except asyncio.TimeoutError: self._reset_closest(peer) return except asyncio.CancelledError: log.debug("%s[%x] cancelled probe", type(self).__name__, id(self)) raise except ValueError as err: log.warning(str(err)) self._reset_closest(peer) return except TransportNotConnected: await self._aclose(reason="not connected") return except RemoteException: self._reset_closest(peer) return return await self._handle_probe_result(peer, response) def _search_round(self): """ Send up to constants.alpha (5) probes to closest active peers """ added = 0 for index, peer in enumerate(self.active.keys()): if index == 0: log.debug("%s[%x] closest to probe: %s", type(self).__name__, id(self), peer.node_id.hex()[:8]) if peer in self.contacted: continue if len(self.running_probes) >= constants.ALPHA: break if index > (constants.K + len(self.running_probes)): break origin_address = (peer.address, peer.udp_port) if peer.node_id == self.protocol.node_id: continue if origin_address == (self.protocol.external_ip, self.protocol.udp_port): continue self._schedule_probe(peer) added += 1 log.debug("%s[%x] running %d probes for key %s", type(self).__name__, id(self), len(self.running_probes), self.key.hex()[:8]) if not added and not self.running_probes: log.debug("%s[%x] search for %s exhausted", type(self).__name__, id(self), self.key.hex()[:8]) self.search_exhausted() def _schedule_probe(self, peer: 'KademliaPeer'): self.contacted.add(peer) t = self.loop.create_task(self._send_probe(peer)) def callback(_): self.running_probes.pop(peer, None) if self.running: self._search_round() t.add_done_callback(callback) self.running_probes[peer] = t def _log_state(self, reason="?"): log.debug("%s[%x] [%s] %s: %i active nodes %i contacted %i produced %i queued", type(self).__name__, id(self), self.key.hex()[:8], reason, len(self.active), len(self.contacted), self.iteration_count, self.iteration_queue.qsize()) def __aiter__(self): if self.running: raise Exception("already running") self.running = True self.loop.call_soon(self._search_round) return self async def __anext__(self) -> typing.List['KademliaPeer']: try: if self.iteration_count == 0: result = self.get_initial_result() or await self.iteration_queue.get() else: result = await self.iteration_queue.get() if not result: raise StopAsyncIteration self.iteration_count += 1 return result except asyncio.CancelledError: await self._aclose(reason="cancelled") raise except StopAsyncIteration: await self._aclose(reason="no more results") raise async def _aclose(self, reason="?"): log.debug("%s[%x] [%s] shutdown because %s: %i active nodes %i contacted %i produced %i queued", type(self).__name__, id(self), self.key.hex()[:8], reason, len(self.active), len(self.contacted), self.iteration_count, self.iteration_queue.qsize()) self.running = False self.iteration_queue.put_nowait(None) for task in chain(self.tasks, self.running_probes.values()): task.cancel() self.tasks.clear() self.running_probes.clear() async def aclose(self): if self.running: await self._aclose(reason="aclose") log.debug("%s[%x] [%s] async close completed", type(self).__name__, id(self), self.key.hex()[:8]) class IterativeNodeFinder(IterativeFinder): def __init__(self, loop: asyncio.AbstractEventLoop, protocol: 'KademliaProtocol', key: bytes, max_results: typing.Optional[int] = constants.K, shortlist: typing.Optional[typing.List['KademliaPeer']] = None): super().__init__(loop, protocol, key, max_results, shortlist) self.yielded_peers: typing.Set['KademliaPeer'] = set() async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse: log.debug("probe %s:%d (%s) for NODE %s", peer.address, peer.udp_port, peer.node_id.hex()[:8] if peer.node_id else '', self.key.hex()[:8]) response = await self.protocol.get_rpc_peer(peer).find_node(self.key) return FindNodeResponse(self.key, response) def search_exhausted(self): self.put_result(self.active.keys(), finish=True) def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False): not_yet_yielded = [ peer for peer in from_iter if peer not in self.yielded_peers and peer.node_id != self.protocol.node_id and self.peer_manager.peer_is_good(peer) is True # return only peers who answered ] not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id)) to_yield = not_yet_yielded[:max(constants.K, self.max_results)] if to_yield: self.yielded_peers.update(to_yield) self.iteration_queue.put_nowait(to_yield) if finish: self.iteration_queue.put_nowait(None) def check_result_ready(self, response: FindNodeResponse): found = response.found and self.key != self.protocol.node_id if found: log.debug("found") return self.put_result(self.active.keys(), finish=True) class IterativeValueFinder(IterativeFinder): def __init__(self, loop: asyncio.AbstractEventLoop, protocol: 'KademliaProtocol', key: bytes, max_results: typing.Optional[int] = constants.K, shortlist: typing.Optional[typing.List['KademliaPeer']] = None): super().__init__(loop, protocol, key, max_results, shortlist) self.blob_peers: typing.Set['KademliaPeer'] = set() # this tracks the index of the most recent page we requested from each peer self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int) # this tracks the set of blob peers returned by each peer self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set) async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse: log.debug("probe %s:%d (%s) for VALUE %s", peer.address, peer.udp_port, peer.node_id.hex()[:8], self.key.hex()[:8]) page = self.peer_pages[peer] response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page) parsed = FindValueResponse(self.key, response) if not parsed.found: return parsed already_known = len(self.discovered_peers[peer]) decoded_peers = set() for compact_addr in parsed.found_compact_addresses: try: decoded_peers.add(decode_tcp_peer_from_compact_address(compact_addr)) except ValueError: log.warning("misbehaving peer %s:%i returned invalid peer for blob", peer.address, peer.udp_port) self.peer_manager.report_failure(peer.address, peer.udp_port) parsed.found_compact_addresses.clear() return parsed self.discovered_peers[peer].update(decoded_peers) log.debug("probed %s:%i page %i, %i known", peer.address, peer.udp_port, page, already_known + len(parsed.found_compact_addresses)) if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses): log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port) elif len(parsed.found_compact_addresses) >= constants.K and self.peer_pages[peer] < parsed.pages: # the peer returned a full page and indicates it has more self.peer_pages[peer] += 1 if peer in self.contacted: # the peer must be removed from self.contacted so that it will be probed for the next page self.contacted.remove(peer) return parsed def check_result_ready(self, response: FindValueResponse): if response.found: blob_peers = [decode_tcp_peer_from_compact_address(compact_addr) for compact_addr in response.found_compact_addresses] to_yield = [] for blob_peer in blob_peers: if blob_peer not in self.blob_peers: self.blob_peers.add(blob_peer) to_yield.append(blob_peer) if to_yield: self.iteration_queue.put_nowait(to_yield) def get_initial_result(self) -> typing.List['KademliaPeer']: if self.protocol.data_store.has_peers_for_blob(self.key): return self.protocol.data_store.get_peers_for_blob(self.key) return [] ================================================ FILE: lbry/dht/protocol/protocol.py ================================================ import logging import socket import functools import hashlib import asyncio import time import typing import random from asyncio.protocols import DatagramProtocol from asyncio.transports import DatagramTransport from prometheus_client import Gauge, Counter, Histogram from lbry.dht import constants from lbry.dht.serialization.bencoding import DecodeError from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY from lbry.dht.error import RemoteException, TransportNotConnected from lbry.dht.protocol.routing_table import TreeRoutingTable from lbry.dht.protocol.data_store import DictDataStore from lbry.dht.peer import make_kademlia_peer if typing.TYPE_CHECKING: from lbry.dht.peer import PeerManager, KademliaPeer log = logging.getLogger(__name__) OLD_PROTOCOL_ERRORS = { "findNode() takes exactly 2 arguments (5 given)": "0.19.1", "findValue() takes exactly 2 arguments (5 given)": "0.19.1" } class KademliaRPC: stored_blob_metric = Gauge( "stored_blobs", "Number of blobs announced by other peers", namespace="dht_node", labelnames=("scope",), ) def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.AbstractEventLoop, peer_port: int = 3333): self.protocol = protocol self.loop = loop self.peer_port = peer_port self.old_token_secret: bytes = None self.token_secret = constants.generate_id() def compact_address(self): compact_ip = functools.reduce(lambda buff, x: buff + bytearray([int(x)]), self.protocol.external_ip.split('.'), bytearray()) compact_port = self.peer_port.to_bytes(2, 'big') return compact_ip + compact_port + self.protocol.node_id @staticmethod def ping(): return b'pong' def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int) -> bytes: if len(blob_hash) != constants.HASH_BITS // 8: raise ValueError(f"invalid length of blob hash: {len(blob_hash)}") if not 0 < port < 65535: raise ValueError(f"invalid tcp port: {port}") rpc_contact.update_tcp_port(port) if not self.verify_token(token, rpc_contact.compact_ip()): if self.loop.time() - self.protocol.started_listening_time < constants.TOKEN_SECRET_REFRESH_INTERVAL: pass else: raise ValueError("Invalid token") self.protocol.data_store.add_peer_to_blob( rpc_contact, blob_hash ) self.stored_blob_metric.labels("global").set(len(self.protocol.data_store)) return b'OK' def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]: if len(key) != constants.HASH_LENGTH: raise ValueError("invalid contact node_id length: %i" % len(key)) contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id) contact_triples = [] for contact in contacts[:constants.K * 2]: contact_triples.append((contact.node_id, contact.address, contact.udp_port)) return contact_triples def find_value(self, rpc_contact: 'KademliaPeer', key: bytes, page: int = 0): page = page if page > 0 else 0 if len(key) != constants.HASH_LENGTH: raise ValueError("invalid blob_exchange hash length: %i" % len(key)) response = { b'token': self.make_token(rpc_contact.compact_ip()), } if not page: response[b'contacts'] = self.find_node(rpc_contact, key)[:constants.K] if self.protocol.protocol_version: response[b'protocolVersion'] = self.protocol.protocol_version # get peers we have stored for this blob_exchange peers = [ peer.compact_address_tcp() for peer in self.protocol.data_store.get_peers_for_blob(key) if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp() ] # if we don't have k storing peers to return and we have this hash locally, include our contact information if len(peers) < constants.K and key.hex() in self.protocol.data_store.completed_blobs: peers.append(self.compact_address()) if not peers: response[PAGE_KEY] = 0 else: response[PAGE_KEY] = (len(peers) // (constants.K + 1)) + 1 # how many pages of peers we have for the blob if len(peers) > constants.K: random.Random(self.protocol.node_id).shuffle(peers) if page * constants.K < len(peers): response[key] = peers[page * constants.K:page * constants.K + constants.K] return response def refresh_token(self): # TODO: this needs to be called periodically self.old_token_secret = self.token_secret self.token_secret = constants.generate_id() def make_token(self, compact_ip): h = hashlib.new('sha384') h.update(self.token_secret + compact_ip) return h.digest() def verify_token(self, token, compact_ip): h = hashlib.new('sha384') h.update(self.token_secret + compact_ip) if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token? h = hashlib.new('sha384') h.update(self.old_token_secret + compact_ip) if not token == h.digest(): return False return True class RemoteKademliaRPC: """ Encapsulates RPC calls to remote Peers """ def __init__(self, loop: asyncio.AbstractEventLoop, peer_tracker: 'PeerManager', protocol: 'KademliaProtocol', peer: 'KademliaPeer'): self.loop = loop self.peer_tracker = peer_tracker self.protocol = protocol self.peer = peer async def ping(self) -> bytes: """ :return: b'pong' """ response = await self.protocol.send_request( self.peer, RequestDatagram.make_ping(self.protocol.node_id) ) return response.response async def store(self, blob_hash: bytes) -> bytes: """ :param blob_hash: blob hash as bytes :return: b'OK' """ if len(blob_hash) != constants.HASH_BITS // 8: raise ValueError(f"invalid length of blob hash: {len(blob_hash)}") if not self.protocol.peer_port or not 0 < self.protocol.peer_port < 65535: raise ValueError(f"invalid tcp port: {self.protocol.peer_port}") token = self.peer_tracker.get_node_token(self.peer.node_id) if not token: find_value_resp = await self.find_value(blob_hash) token = find_value_resp[b'token'] response = await self.protocol.send_request( self.peer, RequestDatagram.make_store(self.protocol.node_id, blob_hash, token, self.protocol.peer_port) ) return response.response async def find_node(self, key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]: """ :return: [(node_id, address, udp_port), ...] """ if len(key) != constants.HASH_BITS // 8: raise ValueError(f"invalid length of find node key: {len(key)}") response = await self.protocol.send_request( self.peer, RequestDatagram.make_find_node(self.protocol.node_id, key) ) return [(node_id, address.decode(), udp_port) for node_id, address, udp_port in response.response] async def find_value(self, key: bytes, page: int = 0) -> typing.Union[typing.Dict]: """ :return: { b'token': , b'contacts': [(node_id, address, udp_port), ...] : [ RemoteKademliaRPC: return RemoteKademliaRPC(self.loop, self.peer_manager, self, peer) def start(self): self.maintaing_routing_task = self.loop.create_task(self.routing_table_task()) def stop(self): if self.maintaing_routing_task: self.maintaing_routing_task.cancel() if self.transport: self.disconnect() def disconnect(self): self.transport.close() def connection_made(self, transport: DatagramTransport): self.transport = transport def connection_lost(self, exc): self.stop() @staticmethod def _migrate_incoming_rpc_args(peer: 'KademliaPeer', method: bytes, *args) -> typing.Tuple[typing.Tuple, typing.Dict]: if method == b'store' and peer.protocol_version == 0: if isinstance(args[1], dict): blob_hash = args[0] token = args[1].pop(b'token', None) port = args[1].pop(b'port', -1) original_publisher_id = args[1].pop(b'lbryid', None) age = 0 return (blob_hash, token, port, original_publisher_id, age), {} return args, {} async def _add_peer(self, peer: 'KademliaPeer'): async def probe(some_peer: 'KademliaPeer'): rpc_peer = self.get_rpc_peer(some_peer) await rpc_peer.ping() return await self.routing_table.add_peer(peer, probe) def add_peer(self, peer: 'KademliaPeer'): if peer.node_id == self.node_id: return False self._to_add.add(peer) self._wakeup_routing_task.set() def remove_peer(self, peer: 'KademliaPeer'): self._to_remove.add(peer) self._wakeup_routing_task.set() async def routing_table_task(self): while True: while self._to_remove: async with self._split_lock: peer = self._to_remove.pop() self.routing_table.remove_peer(peer) while self._to_add: async with self._split_lock: await self._add_peer(self._to_add.pop()) await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1)) self._wakeup_routing_task.clear() def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram): assert sender_contact.node_id != self.node_id, (sender_contact.node_id.hex()[:8], self.node_id.hex()[:8]) method = message.method if method not in [b'ping', b'store', b'findNode', b'findValue']: raise AttributeError('Invalid method: %s' % message.method.decode()) if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]: # args don't need reformatting args, kwargs = tuple(message.args[:-1]), message.args[-1] else: args, kwargs = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args) log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(), sender_contact.address, sender_contact.udp_port) if method == b'ping': result = self.node_rpc.ping() elif method == b'store': blob_hash, token, port, original_publisher_id, age = args[:5] # pylint: disable=unused-variable result = self.node_rpc.store(sender_contact, blob_hash, token, port) else: key = args[0] page = kwargs.get(PAGE_KEY, 0) if method == b'findNode': result = self.node_rpc.find_node(sender_contact, key) else: assert method == b'findValue' result = self.node_rpc.find_value(sender_contact, key, page) self.send_response( sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result), ) def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram): # This is an RPC method request self.received_request_metric.labels(method=request_datagram.method).inc() self.peer_manager.report_last_requested(address[0], address[1]) peer = self.routing_table.get_peer(request_datagram.node_id) if not peer: try: peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1]) except ValueError as err: log.warning("error replying to %s: %s", address[0], str(err)) return try: self._handle_rpc(peer, request_datagram) # if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it # will be added to our routing table if successful is_good = self.peer_manager.peer_is_good(peer) if is_good is None: self.ping_queue.enqueue_maybe_ping(peer) # only add a requesting contact to the routing table if it has replied to one of our requests elif is_good is True: self.add_peer(peer) except ValueError as err: log.debug("error raised handling %s request from %s:%i - %s(%s)", request_datagram.method, peer.address, peer.udp_port, str(type(err)), str(err)) self.send_error( peer, ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(), str(err).encode()) ) except Exception as err: log.warning("error raised handling %s request from %s:%i - %s(%s)", request_datagram.method, peer.address, peer.udp_port, str(type(err)), str(err)) self.send_error( peer, ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(), str(err).encode()) ) def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram): # Find the message that triggered this response if response_datagram.rpc_id in self.sent_messages: peer, future, _ = self.sent_messages[response_datagram.rpc_id] if peer.address != address[0]: future.set_exception( RemoteException(f"response from {address[0]}, expected {peer.address}") ) return # We got a result from the RPC if peer.node_id == self.node_id: future.set_exception(RemoteException("node has our node id")) return elif response_datagram.node_id == self.node_id: future.set_exception(RemoteException("incoming message is from our node id")) return peer = make_kademlia_peer(response_datagram.node_id, address[0], address[1]) self.peer_manager.report_last_replied(address[0], address[1]) self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1]) if not future.cancelled(): future.set_result(response_datagram) self.add_peer(peer) else: log.warning("%s:%i replied, but after we cancelled the request attempt", peer.address, peer.udp_port) else: # If the original message isn't found, it must have timed out # TODO: we should probably do something with this... pass def handle_error_datagram(self, address, error_datagram: ErrorDatagram): # The RPC request raised a remote exception; raise it locally remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})") if error_datagram.rpc_id in self.sent_messages: peer, future, request = self.sent_messages.pop(error_datagram.rpc_id) if (peer.address, peer.udp_port) != address: future.set_exception( RemoteException( f"response from {address[0]}:{address[1]}, " f"expected {peer.address}:{peer.udp_port}" ) ) return error_msg = f"" \ f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \ f"Args: {request.args}\n" \ f"Raised: {str(remote_exception)}" if 'Invalid token' in error_msg: log.debug(error_msg) elif error_datagram.response not in OLD_PROTOCOL_ERRORS: log.warning(error_msg) else: log.debug( "known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)", peer.address, peer.udp_port, OLD_PROTOCOL_ERRORS[error_datagram.response] ) future.set_exception(remote_exception) return else: if error_datagram.response not in OLD_PROTOCOL_ERRORS: msg = f"Received error from {address[0]}:{address[1]}, but it isn't in response to a " \ f"pending request: {str(remote_exception)}" log.warning(msg) else: log.debug( "known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)", address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response] ) def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-renamed try: message = decode_datagram(datagram) except (ValueError, TypeError, DecodeError): self.peer_manager.report_failure(address[0], address[1]) log.warning("Couldn't decode dht datagram from %s: %s", address, datagram.hex()) return if isinstance(message, RequestDatagram): self.handle_request_datagram(address, message) elif isinstance(message, ErrorDatagram): self.handle_error_datagram(address, message) else: assert isinstance(message, ResponseDatagram), "sanity" self.handle_response_datagram(address, message) async def send_request(self, peer: 'KademliaPeer', request: RequestDatagram) -> ResponseDatagram: self._send(peer, request) response_fut = self.sent_messages[request.rpc_id][1] try: self.request_sent_metric.labels(method=request.method).inc() start = time.perf_counter() response = await asyncio.wait_for(response_fut, self.rpc_timeout) self.response_time_metric.labels(method=request.method).observe(time.perf_counter() - start) self.peer_manager.report_last_replied(peer.address, peer.udp_port) self.request_success_metric.labels(method=request.method).inc() return response except asyncio.CancelledError: if not response_fut.done(): response_fut.cancel() raise except (asyncio.TimeoutError, RemoteException): self.request_error_metric.labels(method=request.method).inc() self.peer_manager.report_failure(peer.address, peer.udp_port) if self.peer_manager.peer_is_good(peer) is False: self.remove_peer(peer) raise def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram): self._send(peer, response) def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram): self._send(peer, error) def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]): if not self.transport or self.transport.is_closing(): raise TransportNotConnected() data = message.bencode() if len(data) > constants.MSG_SIZE_LIMIT: log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)", constants.MSG_SIZE_LIMIT, len(data)) log.debug("Packet is too large to send: %s", data[:3500].hex()) raise ValueError( f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)" ) if isinstance(message, (RequestDatagram, ResponseDatagram)): assert message.node_id == self.node_id, message if isinstance(message, RequestDatagram): assert self.node_id != peer.node_id def pop_from_sent_messages(_): if message.rpc_id in self.sent_messages: self.sent_messages.pop(message.rpc_id) if isinstance(message, RequestDatagram): response_fut = self.loop.create_future() response_fut.add_done_callback(pop_from_sent_messages) self.sent_messages[message.rpc_id] = (peer, response_fut, message) try: self.transport.sendto(data, (peer.address, peer.udp_port)) except OSError as err: # TODO: handle ENETUNREACH if err.errno == socket.EWOULDBLOCK: # i'm scared this may swallow important errors, but i get a million of these # on Linux and it doesn't seem to affect anything -grin log.warning("Can't send data to dht: EWOULDBLOCK") else: log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)", len(data), peer.address, peer.udp_port, str(err), err.errno) if isinstance(message, RequestDatagram): self.sent_messages[message.rpc_id][1].set_exception(err) else: raise err if isinstance(message, RequestDatagram): self.peer_manager.report_last_sent(peer.address, peer.udp_port) elif isinstance(message, ErrorDatagram): self.peer_manager.report_failure(peer.address, peer.udp_port) def change_token(self): self.old_token_secret = self.token_secret self.token_secret = constants.generate_id() def make_token(self, compact_ip): return constants.digest(self.token_secret + compact_ip) def verify_token(self, token, compact_ip): h = constants.HASH_CLASS() h.update(self.token_secret + compact_ip) if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token? h = constants.HASH_CLASS() h.update(self.old_token_secret + compact_ip) if not token == h.digest(): return False return True async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer', # pylint: disable=too-many-return-statements retry: bool = True) -> typing.Tuple[bytes, bool]: async def __store(): res = await self.get_rpc_peer(peer).store(hash_value) if res != b"OK": raise ValueError(res) log.debug("Stored %s to %s", hash_value.hex()[:8], peer) return peer.node_id, True try: return await __store() except asyncio.TimeoutError: log.debug("Timeout while storing blob_hash %s at %s", hash_value.hex()[:8], peer) return peer.node_id, False except ValueError as err: log.error("Unexpected response: %s", err) return peer.node_id, False except RemoteException as err: if 'findValue() takes exactly 2 arguments (5 given)' in str(err): log.debug("peer %s:%i is running an incompatible version of lbrynet", peer.address, peer.udp_port) return peer.node_id, False if 'Invalid token' not in str(err): log.warning("Unexpected error while storing blob_hash: %s", err) return peer.node_id, False self.peer_manager.clear_token(peer.node_id) if not retry: return peer.node_id, False return await self.store_to_peer(hash_value, peer, retry=False) ================================================ FILE: lbry/dht/protocol/routing_table.py ================================================ import asyncio import random import logging import typing import itertools from prometheus_client import Gauge from lbry import utils from lbry.dht import constants from lbry.dht.error import RemoteException from lbry.dht.protocol.distance import Distance if typing.TYPE_CHECKING: from lbry.dht.peer import KademliaPeer, PeerManager log = logging.getLogger(__name__) class KBucket: """ Kademlia K-bucket implementation. """ peer_in_routing_table_metric = Gauge( "peers_in_routing_table", "Number of peers on routing table", namespace="dht_node", labelnames=("scope",) ) peer_with_x_bit_colliding_metric = Gauge( "peer_x_bit_colliding", "Number of peers with at least X bits colliding with this node id", namespace="dht_node", labelnames=("amount",) ) def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes, capacity: int = constants.K): """ @param range_min: The lower boundary for the range in the n-bit ID space covered by this k-bucket @param range_max: The upper boundary for the range in the ID space covered by this k-bucket """ self._peer_manager = peer_manager self.range_min = range_min self.range_max = range_max self.peers: typing.List['KademliaPeer'] = [] self._node_id = node_id self._distance_to_self = Distance(node_id) self.capacity = capacity def add_peer(self, peer: 'KademliaPeer') -> bool: """ Add contact to _contact list in the right order. This will move the contact to the end of the k-bucket if it is already present. @raise kademlia.kbucket.BucketFull: Raised when the bucket is full and the contact isn't in the bucket already @param peer: The contact to add @type peer: dht.contact._Contact """ if peer in self.peers: # Move the existing contact to the end of the list # - using the new contact to allow add-on data # (e.g. optimization-specific stuff) to pe updated as well self.peers.remove(peer) self.peers.append(peer) return True else: for i, _ in enumerate(self.peers): local_peer = self.peers[i] if local_peer.node_id == peer.node_id: self.peers.remove(local_peer) self.peers.append(peer) return True if len(self.peers) < self.capacity: self.peers.append(peer) self.peer_in_routing_table_metric.labels("global").inc() bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id) self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).inc() return True else: return False def get_peer(self, node_id: bytes) -> 'KademliaPeer': for peer in self.peers: if peer.node_id == node_id: return peer def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']: """ Returns a list containing up to the first count number of contacts @param count: The amount of contacts to return (if 0 or less, return all contacts) @type count: int @param exclude_contact: A node node_id to exclude; if this contact is in the list of returned values, it will be discarded before returning. If a C{str} is passed as this argument, it must be the contact's ID. @type exclude_contact: str @param sort_distance_to: Sort distance to the node_id, defaulting to the parent node node_id. If False don't sort the contacts @raise IndexError: If the number of requested contacts is too large @return: Return up to the first count number of contacts in a list If no contacts are present an empty is returned @rtype: list """ peers = [peer for peer in self.peers if peer.node_id != exclude_contact] # Return all contacts in bucket if count <= 0: count = len(peers) # Get current contact number current_len = len(peers) # If count greater than k - return only k contacts if count > constants.K: count = constants.K if not current_len: return peers if sort_distance_to is False: pass else: sort_distance_to = sort_distance_to or self._node_id peers.sort(key=lambda c: Distance(sort_distance_to)(c.node_id)) return peers[:min(current_len, count)] def get_bad_or_unknown_peers(self) -> typing.List['KademliaPeer']: peer = self.get_peers(sort_distance_to=False) return [ peer for peer in peer if self._peer_manager.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port) is not True ] def remove_peer(self, peer: 'KademliaPeer') -> None: self.peers.remove(peer) self.peer_in_routing_table_metric.labels("global").dec() bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id) self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).dec() def key_in_range(self, key: bytes) -> bool: """ Tests whether the specified key (i.e. node ID) is in the range of the n-bit ID space covered by this k-bucket (in otherwords, it returns whether or not the specified key should be placed in this k-bucket) @param key: The key to test @type key: str or int @return: C{True} if the key is in this k-bucket's range, or C{False} if not. @rtype: bool """ return self.range_min <= self._distance_to_self(key) < self.range_max def __len__(self) -> int: return len(self.peers) def __contains__(self, item) -> bool: return item in self.peers class TreeRoutingTable: """ This class implements a routing table used by a Node class. The Kademlia routing table is a binary tree whose leaves are k-buckets, where each k-bucket contains nodes with some common prefix of their IDs. This prefix is the k-bucket's position in the binary tree; it therefore covers some range of ID values, and together all of the k-buckets cover the entire n-bit ID (or key) space (with no overlap). @note: In this implementation, nodes in the tree (the k-buckets) are added dynamically, as needed; this technique is described in the 13-page version of the Kademlia paper, in section 2.4. It does, however, use the ping RPC-based k-bucket eviction algorithm described in section 2.2 of that paper. BOOTSTRAP MODE: if set to True, we always add all peers. This is so a bootstrap node does not get a bias towards its own node id and replies are the best it can provide (joining peer knows its neighbors immediately). Over time, this will need to be optimized so we use the disk as holding everything in memory won't be feasible anymore. See: https://github.com/bittorrent/bootstrap-dht """ bucket_in_routing_table_metric = Gauge( "buckets_in_routing_table", "Number of buckets on routing table", namespace="dht_node", labelnames=("scope",) ) def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes, split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False): self._loop = loop self._peer_manager = peer_manager self._parent_node_id = parent_node_id self._split_buckets_under_index = split_buckets_under_index self.buckets: typing.List[KBucket] = [ KBucket( self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id, capacity=1 << 32 if is_bootstrap_node else constants.K ) ] def get_peers(self) -> typing.List['KademliaPeer']: return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets))) def _should_split(self, bucket_index: int, to_add: bytes) -> bool: # https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456 if bucket_index < self._split_buckets_under_index: return True contacts = self.get_peers() distance = Distance(self._parent_node_id) contacts.sort(key=lambda c: distance(c.node_id)) kth_contact = contacts[-1] if len(contacts) < constants.K else contacts[constants.K - 1] return distance(to_add) < distance(kth_contact.node_id) def find_close_peers(self, key: bytes, count: typing.Optional[int] = None, sender_node_id: typing.Optional[bytes] = None) -> typing.List['KademliaPeer']: exclude = [self._parent_node_id] if sender_node_id: exclude.append(sender_node_id) count = count or constants.K distance = Distance(key) contacts = self.get_peers() contacts = [c for c in contacts if c.node_id not in exclude] if contacts: contacts.sort(key=lambda c: distance(c.node_id)) return contacts[:min(count, len(contacts))] return [] def get_peer(self, contact_id: bytes) -> 'KademliaPeer': return self.buckets[self._kbucket_index(contact_id)].get_peer(contact_id) def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]: refresh_ids = [] for offset, _ in enumerate(self.buckets[start_index:]): refresh_ids.append(self._midpoint_id_in_bucket_range(start_index + offset)) # if we have 3 or fewer populated buckets get two random ids in the range of each to try and # populate/split the buckets further buckets_with_contacts = self.buckets_with_contacts() if buckets_with_contacts <= 3: for i in range(buckets_with_contacts): refresh_ids.append(self._random_id_in_bucket_range(i)) refresh_ids.append(self._random_id_in_bucket_range(i)) return refresh_ids def remove_peer(self, peer: 'KademliaPeer') -> None: if not peer.node_id: return bucket_index = self._kbucket_index(peer.node_id) try: self.buckets[bucket_index].remove_peer(peer) self._join_buckets() except ValueError: return def _kbucket_index(self, key: bytes) -> int: i = 0 for bucket in self.buckets: if bucket.key_in_range(key): return i else: i += 1 return i def _random_id_in_bucket_range(self, bucket_index: int) -> bytes: random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max)) return Distance( self._parent_node_id )(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big') def _midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes: half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2) return Distance(self._parent_node_id)( int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big') ).to_bytes(constants.HASH_LENGTH, 'big') def _split_bucket(self, old_bucket_index: int) -> None: """ Splits the specified k-bucket into two new buckets which together cover the same range in the key/ID space @param old_bucket_index: The index of k-bucket to split (in this table's list of k-buckets) @type old_bucket_index: int """ # Resize the range of the current (old) k-bucket old_bucket = self.buckets[old_bucket_index] split_point = old_bucket.range_max - (old_bucket.range_max - old_bucket.range_min) // 2 # Create a new k-bucket to cover the range split off from the old bucket new_bucket = KBucket(self._peer_manager, split_point, old_bucket.range_max, self._parent_node_id) old_bucket.range_max = split_point # Now, add the new bucket into the routing table tree self.buckets.insert(old_bucket_index + 1, new_bucket) # Finally, copy all nodes that belong to the new k-bucket into it... for contact in old_bucket.peers: if new_bucket.key_in_range(contact.node_id): new_bucket.add_peer(contact) # ...and remove them from the old bucket for contact in new_bucket.peers: old_bucket.remove_peer(contact) self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets)) def _join_buckets(self): if len(self.buckets) == 1: return to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0] if not to_pop: return log.info("join buckets %i", len(to_pop)) bucket_index_to_pop = to_pop[0] assert len(self.buckets[bucket_index_to_pop]) == 0 can_go_lower = bucket_index_to_pop - 1 >= 0 can_go_higher = bucket_index_to_pop + 1 < len(self.buckets) assert can_go_higher or can_go_lower bucket = self.buckets[bucket_index_to_pop] if can_go_lower and can_go_higher: midpoint = ((bucket.range_max - bucket.range_min) // 2) + bucket.range_min self.buckets[bucket_index_to_pop - 1].range_max = midpoint - 1 self.buckets[bucket_index_to_pop + 1].range_min = midpoint elif can_go_lower: self.buckets[bucket_index_to_pop - 1].range_max = bucket.range_max elif can_go_higher: self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min self.buckets.remove(bucket) self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets)) return self._join_buckets() def buckets_with_contacts(self) -> int: count = 0 for bucket in self.buckets: if len(bucket) > 0: count += 1 return count async def add_peer(self, peer: 'KademliaPeer', probe: typing.Callable[['KademliaPeer'], typing.Awaitable]): if not peer.node_id: log.warning("Tried adding a peer with no node id!") return False for my_peer in self.get_peers(): if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id: self.remove_peer(my_peer) self._join_buckets() bucket_index = self._kbucket_index(peer.node_id) if self.buckets[bucket_index].add_peer(peer): return True # The bucket is full; see if it can be split (by checking if its range includes the host node's node_id) if self._should_split(bucket_index, peer.node_id): self._split_bucket(bucket_index) # Retry the insertion attempt result = await self.add_peer(peer, probe) self._join_buckets() return result else: # We can't split the k-bucket # # The 13 page kademlia paper specifies that the least recently contacted node in the bucket # shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful # the new contact is ignored and not added to the bucket (sections 2.2 and 2.4). # # A reasonable extension to this is BEP 0005, which extends the above: # # Not all nodes that we learn about are equal. Some are "good" and some are not. # Many nodes using the DHT are able to send queries and receive responses, # but are not able to respond to queries from other nodes. It is important that # each node's routing table must contain only known good nodes. A good node is # a node has responded to one of our queries within the last 15 minutes. A node # is also good if it has ever responded to one of our queries and has sent us a # query within the last 15 minutes. After 15 minutes of inactivity, a node becomes # questionable. Nodes become bad when they fail to respond to multiple queries # in a row. Nodes that we know are good are given priority over nodes with unknown status. # # When there are bad or questionable nodes in the bucket, the least recent is selected for # potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent) # contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact # is ignored if the pinged node replies. not_good_contacts = self.buckets[bucket_index].get_bad_or_unknown_peers() not_recently_replied = [] for my_peer in not_good_contacts: last_replied = self._peer_manager.get_last_replied(my_peer.address, my_peer.udp_port) if not last_replied or last_replied + 60 < self._loop.time(): not_recently_replied.append(my_peer) if not_recently_replied: to_replace = not_recently_replied[0] else: to_replace = self.buckets[bucket_index].peers[0] last_replied = self._peer_manager.get_last_replied(to_replace.address, to_replace.udp_port) if last_replied and last_replied + 60 > self._loop.time(): return False log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port) try: await probe(to_replace) return False except (asyncio.TimeoutError, RemoteException): log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index, to_replace.address, to_replace.udp_port, peer.address, peer.udp_port) if to_replace in self.buckets[bucket_index]: self.buckets[bucket_index].remove_peer(to_replace) return await self.add_peer(peer, probe) ================================================ FILE: lbry/dht/serialization/__init__.py ================================================ ================================================ FILE: lbry/dht/serialization/bencoding.py ================================================ import typing from lbry.dht.error import DecodeError def _bencode(data: typing.Union[int, bytes, bytearray, str, list, tuple, dict]) -> bytes: if isinstance(data, int): return b'i%de' % data elif isinstance(data, (bytes, bytearray)): return b'%d:%s' % (len(data), data) elif isinstance(data, str): return b'%d:%s' % (len(data), data.encode()) elif isinstance(data, (list, tuple)): encoded_list_items = b'' for item in data: encoded_list_items += _bencode(item) return b'l%se' % encoded_list_items elif isinstance(data, dict): encoded_dict_items = b'' keys = data.keys() for key in sorted(keys): encoded_dict_items += _bencode(key) encoded_dict_items += _bencode(data[key]) return b'd%se' % encoded_dict_items else: raise TypeError(f"Cannot bencode {type(data)}") def _bdecode(data: bytes, start_index: int = 0) -> typing.Tuple[typing.Union[int, bytes, list, tuple, dict], int]: if data[start_index] == ord('i'): end_pos = data[start_index:].find(b'e') + start_index return int(data[start_index + 1:end_pos]), end_pos + 1 elif data[start_index] == ord('l'): start_index += 1 decoded_list = [] while data[start_index] != ord('e'): list_data, start_index = _bdecode(data, start_index) decoded_list.append(list_data) return decoded_list, start_index + 1 elif data[start_index] == ord('d'): start_index += 1 decoded_dict = {} while data[start_index] != ord('e'): key, start_index = _bdecode(data, start_index) value, start_index = _bdecode(data, start_index) decoded_dict[key] = value return decoded_dict, start_index else: split_pos = data[start_index:].find(b':') + start_index try: length = int(data[start_index:split_pos]) except (ValueError, TypeError) as err: raise DecodeError(err) start_index = split_pos + 1 end_pos = start_index + length return data[start_index:end_pos], end_pos def bencode(data: typing.Dict) -> bytes: if not isinstance(data, dict): raise TypeError() return _bencode(data) def bdecode(data: bytes, allow_non_dict_return: typing.Optional[bool] = False) -> typing.Dict: assert isinstance(data, bytes), DecodeError(f"invalid data type: {str(type(data))}") if len(data) == 0: raise DecodeError('Cannot decode empty string') try: result = _bdecode(data)[0] if not allow_non_dict_return and not isinstance(result, dict): raise ValueError(f'expected dict, got {type(result)}') return result except (ValueError, TypeError) as err: raise DecodeError(err) ================================================ FILE: lbry/dht/serialization/datagram.py ================================================ import typing from functools import reduce from lbry.dht import constants from lbry.dht.serialization.bencoding import bencode, bdecode REQUEST_TYPE = 0 RESPONSE_TYPE = 1 ERROR_TYPE = 2 OPTIONAL_ARG_OFFSET = 100 # bencode representation of argument keys PAGE_KEY = b'p' OPTIONAL_FIELDS = () class KademliaDatagramBase: """ field names are used to unwrap/wrap the argument names to index integers that replace them in a datagram all packets have an argument dictionary when bdecoded starting with {0: , 1: , 2: , ...} these correspond to the packet_type, rpc_id, and node_id args """ required_fields = [ 'packet_type', 'rpc_id', 'node_id' ] expected_packet_type = -1 def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes): self.packet_type = packet_type if self.expected_packet_type != packet_type: raise ValueError(f"invalid packet type: {packet_type}, expected {self.expected_packet_type}") if len(rpc_id) != constants.RPC_ID_LENGTH: raise ValueError(f"invalid rpc node_id: {len(rpc_id)} bytes (expected 20)") if not len(node_id) == constants.HASH_LENGTH: raise ValueError(f"invalid node node_id: {len(node_id)} bytes (expected 48)") self.rpc_id = rpc_id self.node_id = node_id def bencode(self) -> bytes: datagram = { i: getattr(self, k) for i, k in enumerate(self.required_fields) } for i, k in enumerate(OPTIONAL_FIELDS): value = getattr(self, k, None) if value is not None: datagram[i + OPTIONAL_ARG_OFFSET] = value return bencode(datagram) class RequestDatagram(KademliaDatagramBase): required_fields = [ 'packet_type', 'rpc_id', 'node_id', 'method', 'args' ] expected_packet_type = REQUEST_TYPE def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, method: bytes, args: typing.Optional[typing.List] = None): super().__init__(packet_type, rpc_id, node_id) self.method = method self.args = args or [] if not self.args: self.args.append({}) if isinstance(self.args[-1], dict): self.args[-1][b'protocolVersion'] = 1 else: self.args.append({b'protocolVersion': 1}) @classmethod def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram': rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH] return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping') @classmethod def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram': rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH] if len(blob_hash) != constants.HASH_BITS // 8: raise ValueError(f"invalid blob hash length: {len(blob_hash)}") if not 0 < port < 65536: raise ValueError(f"invalid port: {port}") if len(token) != constants.HASH_BITS // 8: raise ValueError(f"invalid token length: {len(token)}") store_args = [blob_hash, token, port, from_node_id, 0] return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args) @classmethod def make_find_node(cls, from_node_id: bytes, key: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram': rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH] if len(key) != constants.HASH_BITS // 8: raise ValueError(f"invalid key length: {len(key)}") return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key]) @classmethod def make_find_value(cls, from_node_id: bytes, key: bytes, rpc_id: typing.Optional[bytes] = None, page: int = 0) -> 'RequestDatagram': rpc_id = rpc_id or constants.generate_id()[:constants.RPC_ID_LENGTH] if len(key) != constants.HASH_BITS // 8: raise ValueError(f"invalid key length: {len(key)}") if page < 0: raise ValueError(f"cannot request a negative page ({page})") return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key, {PAGE_KEY: page}]) class ResponseDatagram(KademliaDatagramBase): required_fields = [ 'packet_type', 'rpc_id', 'node_id', 'response' ] expected_packet_type = RESPONSE_TYPE def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, response): super().__init__(packet_type, rpc_id, node_id) self.response = response class ErrorDatagram(KademliaDatagramBase): required_fields = [ 'packet_type', 'rpc_id', 'node_id', 'exception_type', 'response', ] expected_packet_type = ERROR_TYPE def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, exception_type: bytes, response: bytes): super().__init__(packet_type, rpc_id, node_id) self.exception_type = exception_type.decode() self.response = response.decode() def _decode_datagram(datagram: bytes): msg_types = { REQUEST_TYPE: RequestDatagram, RESPONSE_TYPE: ResponseDatagram, ERROR_TYPE: ErrorDatagram } primitive: typing.Dict = bdecode(datagram) converted = { str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items() } if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object else: raise ValueError("invalid datagram type") datagram_class = msg_types[datagram_type] decoded = { k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object for i, k in enumerate(datagram_class.required_fields) if str(i).encode() in converted # pylint: disable=unsupported-membership-test } for i, _ in enumerate(OPTIONAL_FIELDS): if str(i + OPTIONAL_ARG_OFFSET).encode() in converted: decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()] return decoded, datagram_class def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]: decoded, datagram_class = _decode_datagram(datagram) return datagram_class(**decoded) def make_compact_ip(address: str) -> bytearray: compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray()) if len(compact_ip) != 4: raise ValueError("invalid IPv4 length") return compact_ip def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray: compact_ip = make_compact_ip(address) if not 0 < port < 65536: raise ValueError(f'Invalid port: {port}') if len(node_id) != constants.HASH_BITS // 8: raise ValueError("invalid node node_id length") return compact_ip + port.to_bytes(2, 'big') + node_id def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, int]: address = "{}.{}.{}.{}".format(*compact_address[:4]) port = int.from_bytes(compact_address[4:6], 'big') node_id = compact_address[6:] if not 0 < port < 65536: raise ValueError(f'Invalid port: {port}') if len(node_id) != constants.HASH_BITS // 8: raise ValueError("invalid node node_id length") return node_id, address, port ================================================ FILE: lbry/error/Makefile ================================================ generate: python generate.py generate > __init__.py analyze: python generate.py analyze ================================================ FILE: lbry/error/README.md ================================================ # Exceptions Exceptions in LBRY are defined and generated from the Markdown table at the end of this README. ## Guidelines When possible, use [built-in Python exceptions](https://docs.python.org/3/library/exceptions.html) or `aiohttp` [general client](https://docs.aiohttp.org/en/latest/client_reference.html#client-exceptions) / [HTTP](https://docs.aiohttp.org/en/latest/web_exceptions.html) exceptions, unless: 1. You want to provide a better error message (extend the closest built-in/`aiohttp` exception in this case). 2. You need to represent a new situation. When defining your own exceptions, consider: 1. Extending a built-in Python or `aiohttp` exception. 2. Using contextual variables in the error message. ## Table Column Definitions Column | Meaning ---|--- Code | Codes are used only to define the hierarchy of exceptions and do not end up in the generated output, it is okay to re-number things as necessary at anytime to achieve the desired hierarchy. Name | Becomes the class name of the exception with "Error" appended to the end. Changing names of existing exceptions makes the API backwards incompatible. When extending other exceptions you must specify the full class name, manually adding "Error" as necessary (if extending another SDK exception). Message | User friendly error message explaining the exceptional event. Supports Python formatted strings: any variables used in the string will be generated as arguments in the `__init__` method. Use `--` to provide a doc string after the error message to be added to the class definition. ## Exceptions Table Code | Name | Message ---:|---|--- **1xx** | UserInput | User input errors. **10x** | Command | Errors preparing to execute commands. 101 | CommandDoesNotExist | Command '{command}' does not exist. 102 | CommandDeprecated | Command '{command}' is deprecated. 103 | CommandInvalidArgument | Invalid argument '{argument}' to command '{command}'. 104 | CommandTemporarilyUnavailable | Command '{command}' is temporarily unavailable. -- Such as waiting for required components to start. 105 | CommandPermanentlyUnavailable | Command '{command}' is permanently unavailable. -- such as when required component was intentionally configured not to start. **11x** | InputValue(ValueError) | Invalid argument value provided to command. 111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid. 112 | InputValueIsNone | None or null is not valid value for argument '{argument}'. 113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both. 114 | InputStringIsBlank | {argument} cannot be blank. 115 | EmptyPublishedFile | Cannot publish empty file: {file_path} 116 | MissingPublishedFile | File does not exist: {file_path} 117 | InvalidStreamURL | Invalid LBRY stream URL: '{url}' -- When an URL cannot be downloaded, such as '@Channel/' or a collection **2xx** | Configuration | Configuration errors. 201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues. 202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args. 203 | ConfigParse | Failed to parse the configuration file '{path}'. -- Includes the syntax error / line number to help user fix it. 204 | ConfigMissing | Configuration file '{path}' is missing setting that has no default / fallback. 205 | ConfigInvalid | Configuration file '{path}' has setting with invalid value. **3xx** | Network | **Networking** 301 | NoInternet | No internet connection. 302 | NoUPnPSupport | Router does not support UPnP. **4xx** | Wallet | **Wallet Errors** 401 | TransactionRejected | Transaction rejected, unknown reason. 402 | TransactionFeeTooLow | Fee too low. 403 | TransactionInvalidSignature | Invalid signature. 404 | InsufficientFunds | Not enough funds to cover this transaction. -- determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX being created and sent but then rejected by lbrycrd for unspendable utxos. 405 | ChannelKeyNotFound | Channel signing key not found. 406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key. 407 | DataDownload | Failed to download blob. *generic* 408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'. 410 | Resolve | Failed to resolve '{url}'. 411 | ResolveTimeout | Failed to resolve '{url}' within the timeout. 411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'. 420 | KeyFeeAboveMaxAllowed | {message} 421 | InvalidPassword | Password is invalid. 422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version. 423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items. 424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override. 431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round. 432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round. 433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC. 434 | WalletNotLoaded | Wallet {wallet_id} is not loaded. 435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded. 436 | WalletNotFound | Wallet not found at {wallet_path}. 437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it. **5xx** | Blob | **Blobs** 500 | BlobNotFound | Blob not found. 501 | BlobPermissionDenied | Permission denied to read blob. 502 | BlobTooBig | Blob is too big. 503 | BlobEmpty | Blob is empty. 510 | BlobFailedDecryption | Failed to decrypt blob. 511 | CorruptBlob | Blobs is corrupted. 520 | BlobFailedEncryption | Failed to encrypt blob. 531 | DownloadCancelled | Download was canceled. 532 | DownloadSDTimeout | Failed to download sd blob {download} within timeout. 533 | DownloadDataTimeout | Failed to download data blobs for sd hash {download} within timeout. 534 | InvalidStreamDescriptor | {message} 535 | InvalidData | {message} 536 | InvalidBlobHash | {message} **6xx** | Component | **Components** 601 | ComponentStartConditionNotMet | Unresolved dependencies for: {components} 602 | ComponentsNotStarted | {message} **7xx** | CurrencyExchange | **Currency Exchange** 701 | InvalidExchangeRateResponse | Failed to get exchange rate from {source}: {reason} 702 | CurrencyConversion | {message} 703 | InvalidCurrency | Invalid currency: {currency} is not a supported currency. ================================================ FILE: lbry/error/__init__.py ================================================ from .base import BaseError, claim_id class UserInputError(BaseError): """ User input errors. """ class CommandError(UserInputError): """ Errors preparing to execute commands. """ class CommandDoesNotExistError(CommandError): def __init__(self, command): self.command = command super().__init__(f"Command '{command}' does not exist.") class CommandDeprecatedError(CommandError): def __init__(self, command): self.command = command super().__init__(f"Command '{command}' is deprecated.") class CommandInvalidArgumentError(CommandError): def __init__(self, argument, command): self.argument = argument self.command = command super().__init__(f"Invalid argument '{argument}' to command '{command}'.") class CommandTemporarilyUnavailableError(CommandError): """ Such as waiting for required components to start. """ def __init__(self, command): self.command = command super().__init__(f"Command '{command}' is temporarily unavailable.") class CommandPermanentlyUnavailableError(CommandError): """ such as when required component was intentionally configured not to start. """ def __init__(self, command): self.command = command super().__init__(f"Command '{command}' is permanently unavailable.") class InputValueError(UserInputError, ValueError): """ Invalid argument value provided to command. """ class GenericInputValueError(InputValueError): def __init__(self, value, argument): self.value = value self.argument = argument super().__init__(f"The value '{value}' for argument '{argument}' is not valid.") class InputValueIsNoneError(InputValueError): def __init__(self, argument): self.argument = argument super().__init__(f"None or null is not valid value for argument '{argument}'.") class ConflictingInputValueError(InputValueError): def __init__(self, first_argument, second_argument): self.first_argument = first_argument self.second_argument = second_argument super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.") class InputStringIsBlankError(InputValueError): def __init__(self, argument): self.argument = argument super().__init__(f"{argument} cannot be blank.") class EmptyPublishedFileError(InputValueError): def __init__(self, file_path): self.file_path = file_path super().__init__(f"Cannot publish empty file: {file_path}") class MissingPublishedFileError(InputValueError): def __init__(self, file_path): self.file_path = file_path super().__init__(f"File does not exist: {file_path}") class InvalidStreamURLError(InputValueError): """ When an URL cannot be downloaded, such as '@Channel/' or a collection """ def __init__(self, url): self.url = url super().__init__(f"Invalid LBRY stream URL: '{url}'") class ConfigurationError(BaseError): """ Configuration errors. """ class ConfigWriteError(ConfigurationError): """ When writing the default config fails on startup, such as due to permission issues. """ def __init__(self, path): self.path = path super().__init__(f"Cannot write configuration file '{path}'.") class ConfigReadError(ConfigurationError): """ Can't open the config file user provided via command line args. """ def __init__(self, path): self.path = path super().__init__(f"Cannot find provided configuration file '{path}'.") class ConfigParseError(ConfigurationError): """ Includes the syntax error / line number to help user fix it. """ def __init__(self, path): self.path = path super().__init__(f"Failed to parse the configuration file '{path}'.") class ConfigMissingError(ConfigurationError): def __init__(self, path): self.path = path super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.") class ConfigInvalidError(ConfigurationError): def __init__(self, path): self.path = path super().__init__(f"Configuration file '{path}' has setting with invalid value.") class NetworkError(BaseError): """ **Networking** """ class NoInternetError(NetworkError): def __init__(self): super().__init__("No internet connection.") class NoUPnPSupportError(NetworkError): def __init__(self): super().__init__("Router does not support UPnP.") class WalletError(BaseError): """ **Wallet Errors** """ class TransactionRejectedError(WalletError): def __init__(self): super().__init__("Transaction rejected, unknown reason.") class TransactionFeeTooLowError(WalletError): def __init__(self): super().__init__("Fee too low.") class TransactionInvalidSignatureError(WalletError): def __init__(self): super().__init__("Invalid signature.") class InsufficientFundsError(WalletError): """ determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX being created and sent but then rejected by lbrycrd for unspendable utxos. """ def __init__(self): super().__init__("Not enough funds to cover this transaction.") class ChannelKeyNotFoundError(WalletError): def __init__(self): super().__init__("Channel signing key not found.") class ChannelKeyInvalidError(WalletError): """ For example, channel was updated but you don't have the updated key. """ def __init__(self): super().__init__("Channel signing key is out of date.") class DataDownloadError(WalletError): def __init__(self): super().__init__("Failed to download blob. *generic*") class PrivateKeyNotFoundError(WalletError): def __init__(self, key, value): self.key = key self.value = value super().__init__(f"Couldn't find private key for {key} '{value}'.") class ResolveError(WalletError): def __init__(self, url): self.url = url super().__init__(f"Failed to resolve '{url}'.") class ResolveTimeoutError(WalletError): def __init__(self, url): self.url = url super().__init__(f"Failed to resolve '{url}' within the timeout.") class ResolveCensoredError(WalletError): def __init__(self, url, censor_id, censor_row): self.url = url self.censor_id = censor_id self.censor_row = censor_row super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.") class KeyFeeAboveMaxAllowedError(WalletError): def __init__(self, message): self.message = message super().__init__(f"{message}") class InvalidPasswordError(WalletError): def __init__(self): super().__init__("Password is invalid.") class IncompatibleWalletServerError(WalletError): def __init__(self, server, port): self.server = server self.port = port super().__init__(f"'{server}:{port}' has an incompatibly old version.") class TooManyClaimSearchParametersError(WalletError): def __init__(self, key, limit): self.key = key self.limit = limit super().__init__(f"{key} cant have more than {limit} items.") class AlreadyPurchasedError(WalletError): """ allow-duplicate-purchase flag to override. """ def __init__(self, claim_id_hex): self.claim_id_hex = claim_id_hex super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use") class ServerPaymentInvalidAddressError(WalletError): def __init__(self, address): self.address = address super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.") class ServerPaymentWalletLockedError(WalletError): def __init__(self): super().__init__("Cannot spend funds with locked wallet, skipping payment round.") class ServerPaymentFeeAboveMaxAllowedError(WalletError): def __init__(self, daily_fee, max_fee): self.daily_fee = daily_fee self.max_fee = max_fee super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.") class WalletNotLoadedError(WalletError): def __init__(self, wallet_id): self.wallet_id = wallet_id super().__init__(f"Wallet {wallet_id} is not loaded.") class WalletAlreadyLoadedError(WalletError): def __init__(self, wallet_path): self.wallet_path = wallet_path super().__init__(f"Wallet {wallet_path} is already loaded.") class WalletNotFoundError(WalletError): def __init__(self, wallet_path): self.wallet_path = wallet_path super().__init__(f"Wallet not found at {wallet_path}.") class WalletAlreadyExistsError(WalletError): def __init__(self, wallet_path): self.wallet_path = wallet_path super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.") class BlobError(BaseError): """ **Blobs** """ class BlobNotFoundError(BlobError): def __init__(self): super().__init__("Blob not found.") class BlobPermissionDeniedError(BlobError): def __init__(self): super().__init__("Permission denied to read blob.") class BlobTooBigError(BlobError): def __init__(self): super().__init__("Blob is too big.") class BlobEmptyError(BlobError): def __init__(self): super().__init__("Blob is empty.") class BlobFailedDecryptionError(BlobError): def __init__(self): super().__init__("Failed to decrypt blob.") class CorruptBlobError(BlobError): def __init__(self): super().__init__("Blobs is corrupted.") class BlobFailedEncryptionError(BlobError): def __init__(self): super().__init__("Failed to encrypt blob.") class DownloadCancelledError(BlobError): def __init__(self): super().__init__("Download was canceled.") class DownloadSDTimeoutError(BlobError): def __init__(self, download): self.download = download super().__init__(f"Failed to download sd blob {download} within timeout.") class DownloadDataTimeoutError(BlobError): def __init__(self, download): self.download = download super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.") class InvalidStreamDescriptorError(BlobError): def __init__(self, message): self.message = message super().__init__(f"{message}") class InvalidDataError(BlobError): def __init__(self, message): self.message = message super().__init__(f"{message}") class InvalidBlobHashError(BlobError): def __init__(self, message): self.message = message super().__init__(f"{message}") class ComponentError(BaseError): """ **Components** """ class ComponentStartConditionNotMetError(ComponentError): def __init__(self, components): self.components = components super().__init__(f"Unresolved dependencies for: {components}") class ComponentsNotStartedError(ComponentError): def __init__(self, message): self.message = message super().__init__(f"{message}") class CurrencyExchangeError(BaseError): """ **Currency Exchange** """ class InvalidExchangeRateResponseError(CurrencyExchangeError): def __init__(self, source, reason): self.source = source self.reason = reason super().__init__(f"Failed to get exchange rate from {source}: {reason}") class CurrencyConversionError(CurrencyExchangeError): def __init__(self, message): self.message = message super().__init__(f"{message}") class InvalidCurrencyError(CurrencyExchangeError): def __init__(self, currency): self.currency = currency super().__init__(f"Invalid currency: {currency} is not a supported currency.") ================================================ FILE: lbry/error/base.py ================================================ from binascii import hexlify def claim_id(claim_hash): return hexlify(claim_hash[::-1]).decode() class BaseError(Exception): pass ================================================ FILE: lbry/error/generate.py ================================================ import re import sys import argparse from pathlib import Path from textwrap import fill, indent INDENT = ' ' * 4 CLASS = """ class {name}({parents}):{doc} """ INIT = """ def __init__({args}):{fields} super().__init__({format}"{message}") """ FUNCTIONS = ['claim_id'] class ErrorClass: def __init__(self, hierarchy, name, message): self.hierarchy = hierarchy.replace('**', '') self.other_parents = [] if '(' in name: assert ')' in name, f"Missing closing parenthesis in '{name}'." self.other_parents = name[name.find('(')+1:name.find(')')].split(',') name = name[:name.find('(')] self.name = name self.class_name = name+'Error' self.message = message self.comment = "" if '--' in message: self.message, self.comment = message.split('--') self.message = self.message.strip() self.comment = self.comment.strip() @property def is_leaf(self): return 'x' not in self.hierarchy @property def code(self): return self.hierarchy.replace('x', '') @property def parent_codes(self): return self.hierarchy[0:2], self.hierarchy[0] def get_arguments(self): args = ['self'] for arg in re.findall('{([a-z0-1_()]+)}', self.message): for func in FUNCTIONS: if arg.startswith(f'{func}('): arg = arg[len(f'{func}('):-1] break args.append(arg) return args @staticmethod def get_fields(args): if len(args) > 1: return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:]) return '' @staticmethod def get_doc_string(doc): if doc: return f'\n{INDENT}"""\n{indent(fill(doc, 100), INDENT)}\n{INDENT}"""' return "" def render(self, out, parent): if not parent: parents = ['BaseError'] else: parents = [parent.class_name] parents += self.other_parents args = self.get_arguments() if self.is_leaf: out.write((CLASS + INIT).format( name=self.class_name, parents=', '.join(parents), args=', '.join(args), fields=self.get_fields(args), message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else '' )) else: out.write(CLASS.format( name=self.class_name, parents=', '.join(parents), doc=self.get_doc_string(self.comment or self.message) )) def get_errors(): with open('README.md', 'r') as readme: lines = iter(readme.readlines()) for line in lines: if line.startswith('## Exceptions Table'): break for line in lines: if line.startswith('---:|'): break for line in lines: if not line: break yield ErrorClass(*[c.strip() for c in line.split('|')]) def find_parent(stack, child): for parent_code in child.parent_codes: parent = stack.get(parent_code) if parent: return parent def generate(out): out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n") stack = {} for error in get_errors(): error.render(out, find_parent(stack, error)) if not error.is_leaf: assert error.code not in stack, f"Duplicate code: {error.code}" stack[error.code] = error def analyze(): errors = {e.class_name: [] for e in get_errors() if e.is_leaf} here = Path(__file__).absolute().parents[0] module = here.parent for file_path in module.glob('**/*.py'): if here in file_path.parents: continue with open(file_path) as src_file: src = src_file.read() for error in errors.keys(): found = src.count(error) if found > 0: errors[error].append((file_path, found)) print('Unused Errors:\n') for error, used in errors.items(): if used: print(f' - {error}') for use in used: print(f' {use[0].relative_to(module.parent)} {use[1]}') print('') print('') print('Unused Errors:') for error, used in errors.items(): if not used: print(f' - {error}') def main(): parser = argparse.ArgumentParser() parser.add_argument("action", choices=['generate', 'analyze']) args = parser.parse_args() if args.action == "analyze": analyze() elif args.action == "generate": generate(sys.stdout) if __name__ == "__main__": main() ================================================ FILE: lbry/extras/__init__.py ================================================ ================================================ FILE: lbry/extras/cli.py ================================================ import os import sys import shutil import signal import pathlib import json import asyncio import argparse import logging import logging.handlers import aiohttp from aiohttp.web import GracefulExit from docopt import docopt from lbry import __version__ as lbrynet_version from lbry.extras.daemon.daemon import Daemon from lbry.conf import Config, CLIConfig log = logging.getLogger('lbry') def display(data): print(json.dumps(data, indent=2)) async def execute_command(conf, method, params, callback=display): async with aiohttp.ClientSession() as session: try: message = {'method': method, 'params': params} async with session.get(conf.api_connection_url, json=message) as resp: try: data = await resp.json() if 'result' in data: return callback(data['result']) elif 'error' in data: return callback(data['error']) except Exception as e: log.exception('Could not process response from server:', exc_info=e) except aiohttp.ClientConnectionError: print("Could not connect to daemon. Are you sure it's running?") def normalize_value(x, key=None): if not isinstance(x, str): return x if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'): return x if x.lower() == 'true': return True if x.lower() == 'false': return False if x.isdigit(): return int(x) return x def remove_brackets(key): if key.startswith("<") and key.endswith(">"): return str(key[1:-1]) return key def set_kwargs(parsed_args): kwargs = {} for key, arg in parsed_args.items(): if arg is None: continue k = None if key.startswith("--") and remove_brackets(key[2:]) not in kwargs: k = remove_brackets(key[2:]) elif remove_brackets(key) not in kwargs: k = remove_brackets(key) kwargs[k] = normalize_value(arg, k) return kwargs def split_subparser_argument(parent, original, name, condition): new_sub_parser = argparse._SubParsersAction( original.option_strings, original._prog_prefix, original._parser_class, metavar=original.metavar ) new_sub_parser._name_parser_map = original._name_parser_map new_sub_parser._choices_actions = [ a for a in original._choices_actions if condition(original._name_parser_map[a.dest]) ] group = argparse._ArgumentGroup(parent, name) group._group_actions = [new_sub_parser] return group class ArgumentParser(argparse.ArgumentParser): def __init__(self, *args, group_name=None, **kwargs): super().__init__(*args, formatter_class=HelpFormatter, add_help=False, **kwargs) self.add_argument( '--help', dest='help', action='store_true', default=False, help='Show this help message and exit.' ) self._optionals.title = 'Options' if group_name is None: self.epilog = ( "Run 'lbrynet COMMAND --help' for more information on a command or group." ) else: self.epilog = ( f"Run 'lbrynet {group_name} COMMAND --help' for more information on a command." ) self.set_defaults(group=group_name, group_parser=self) def format_help(self): formatter = self._get_formatter() formatter.add_usage( self.usage, self._actions, self._mutually_exclusive_groups ) formatter.add_text(self.description) # positionals, optionals and user-defined groups for action_group in self._granular_action_groups: formatter.start_section(action_group.title) formatter.add_text(action_group.description) formatter.add_arguments(action_group._group_actions) formatter.end_section() formatter.add_text(self.epilog) return formatter.format_help() @property def _granular_action_groups(self): if self.prog != 'lbrynet': yield from self._action_groups return yield self._optionals action: argparse._SubParsersAction = self._positionals._group_actions[0] yield split_subparser_argument( self, action, "Grouped Commands", lambda parser: 'group' in parser._defaults ) yield split_subparser_argument( self, action, "Commands", lambda parser: 'group' not in parser._defaults ) def error(self, message): self.print_help(argparse._sys.stderr) self.exit(2, f"\n{message}\n") class HelpFormatter(argparse.HelpFormatter): def add_usage(self, usage, actions, groups, prefix='Usage: '): super().add_usage( usage, [a for a in actions if a.option_strings != ['--help']], groups, prefix ) def add_command_parser(parent, command): subcommand = parent.add_parser( command['name'], help=command['doc'].strip().splitlines()[0] ) subcommand.set_defaults( api_method_name=command['api_method_name'], command=command['name'], doc=command['doc'], replaced_by=command.get('replaced_by', None) ) def get_argument_parser(): root = ArgumentParser( 'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False, ) root.add_argument( '-v', '--version', dest='cli_version', action="store_true", help='Show lbrynet CLI version and exit.' ) root.set_defaults(group=None, command=None) CLIConfig.contribute_to_argparse(root) sub = root.add_subparsers(metavar='COMMAND') start = sub.add_parser( 'start', usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...', help='Start LBRY Network interface.' ) start.add_argument( '--quiet', dest='quiet', action="store_true", help='Disable all console output.' ) start.add_argument( '--no-logging', dest='no_logging', action="store_true", help='Disable all logging of any kind.' ) start.add_argument( '--verbose', nargs="*", help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output ' 'should selectively be applied.') ) start.add_argument( '--initial-headers', dest='initial_headers', help='Specify path to initial blockchain headers, faster than downloading them on first run.' ) Config.contribute_to_argparse(start) start.set_defaults(command='start', start_parser=start, doc=start.format_help()) api = Daemon.get_api_definitions() groups = {} for group_name in sorted(api['groups']): group_parser = sub.add_parser(group_name, group_name=group_name, help=api['groups'][group_name]) groups[group_name] = group_parser.add_subparsers(metavar='COMMAND') nicer_order = ['stop', 'get', 'publish', 'resolve'] for command_name in sorted(api['commands']): if command_name not in nicer_order: nicer_order.append(command_name) for command_name in nicer_order: command = api['commands'][command_name] if command['group'] is None: add_command_parser(sub, command) else: add_command_parser(groups[command['group']], command) return root def ensure_directory_exists(path: str): if not os.path.isdir(path): pathlib.Path(path).mkdir(parents=True, exist_ok=True) use_effective_ids = os.access in os.supports_effective_ids if not os.access(path, os.W_OK, effective_ids=use_effective_ids): raise PermissionError(f"The following directory is not writable: {path}") LOG_MODULES = 'lbry', 'aioupnp' def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config): default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s") file_handler = logging.handlers.RotatingFileHandler(conf.log_file_path, maxBytes=2097152, backupCount=5) file_handler.setFormatter(default_formatter) for module_name in LOG_MODULES: logger.getChild(module_name).addHandler(file_handler) if not args.quiet: handler = logging.StreamHandler() handler.setFormatter(default_formatter) for module_name in LOG_MODULES: logger.getChild(module_name).addHandler(handler) logger.getChild('lbry').setLevel(logging.INFO) logger.getChild('aioupnp').setLevel(logging.WARNING) logger.getChild('aiohttp').setLevel(logging.CRITICAL) if args.verbose is not None: if len(args.verbose) > 0: for module in args.verbose: logger.getChild(module).setLevel(logging.DEBUG) else: logger.getChild('lbry').setLevel(logging.DEBUG) def run_daemon(args: argparse.Namespace, conf: Config): loop = asyncio.get_event_loop() if args.verbose is not None: loop.set_debug(True) if not args.no_logging: setup_logging(logging.getLogger(), args, conf) daemon = Daemon(conf) def __exit(): raise GracefulExit() try: loop.add_signal_handler(signal.SIGINT, __exit) loop.add_signal_handler(signal.SIGTERM, __exit) except NotImplementedError: pass # Not implemented on Windows try: loop.run_until_complete(daemon.start()) loop.run_forever() except (GracefulExit, KeyboardInterrupt, asyncio.CancelledError): pass finally: loop.run_until_complete(daemon.stop()) logging.shutdown() if hasattr(loop, 'shutdown_asyncgens'): loop.run_until_complete(loop.shutdown_asyncgens()) def main(argv=None): argv = argv or sys.argv[1:] parser = get_argument_parser() args, command_args = parser.parse_known_args(argv) conf = Config.create_from_arguments(args) for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir): ensure_directory_exists(directory) if args.cli_version: print(f"lbrynet {lbrynet_version}") elif args.command == 'start': if args.help: args.start_parser.print_help() else: if args.initial_headers: ledger_path = os.path.join(conf.wallet_dir, 'lbc_mainnet') ensure_directory_exists(ledger_path) current_size = 0 headers_path = os.path.join(ledger_path, 'headers') if os.path.exists(headers_path): current_size = os.stat(headers_path).st_size if os.stat(args.initial_headers).st_size > current_size: log.info('Copying header from %s to %s', args.initial_headers, headers_path) shutil.copy(args.initial_headers, headers_path) run_daemon(args, conf) elif args.command is not None: doc = args.doc api_method_name = args.api_method_name if args.replaced_by: print(f"{args.api_method_name} is deprecated, using {args.replaced_by['api_method_name']}.") doc = args.replaced_by['doc'] api_method_name = args.replaced_by['api_method_name'] if args.help: print(doc) else: parsed = docopt(doc, command_args) params = set_kwargs(parsed) asyncio.get_event_loop().run_until_complete(execute_command(conf, api_method_name, params)) elif args.group is not None: args.group_parser.print_help() else: parser.print_help() return 0 if __name__ == "__main__": sys.exit(main()) ================================================ FILE: lbry/extras/daemon/__init__.py ================================================ ================================================ FILE: lbry/extras/daemon/analytics.py ================================================ import asyncio import collections import logging import typing import aiohttp from lbry import utils from lbry.conf import Config from lbry.extras import system_info ANALYTICS_ENDPOINT = 'https://api.segment.io/v1' ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H=' # Things We Track SERVER_STARTUP = 'Server Startup' SERVER_STARTUP_SUCCESS = 'Server Startup Success' SERVER_STARTUP_ERROR = 'Server Startup Error' DOWNLOAD_STARTED = 'Download Started' DOWNLOAD_ERRORED = 'Download Errored' DOWNLOAD_FINISHED = 'Download Finished' HEARTBEAT = 'Heartbeat' DISK_SPACE = 'Disk Space' CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon NEW_CHANNEL = 'New Channel' CREDITS_SENT = 'Credits Sent' UPNP_SETUP = "UPnP Setup" BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded' TIME_TO_FIRST_BYTES = "Time To First Bytes" log = logging.getLogger(__name__) def _event_properties(installation_id: str, session_id: str, event_properties: typing.Optional[typing.Dict]) -> typing.Dict: properties = { 'lbry_id': installation_id, 'session_id': session_id, } properties.update(event_properties or {}) return properties def _download_properties(conf: Config, external_ip: str, resolve_duration: float, total_duration: typing.Optional[float], download_id: str, name: str, outpoint: str, active_peer_count: typing.Optional[int], tried_peers_count: typing.Optional[int], connection_failures_count: typing.Optional[int], added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str, sd_download_duration: typing.Optional[float] = None, head_blob_hash: typing.Optional[str] = None, head_blob_length: typing.Optional[int] = None, head_blob_download_duration: typing.Optional[float] = None, error: typing.Optional[str] = None, error_msg: typing.Optional[str] = None, wallet_server: typing.Optional[str] = None) -> typing.Dict: return { "external_ip": external_ip, "download_id": download_id, "total_duration": round(total_duration, 4), "resolve_duration": None if not resolve_duration else round(resolve_duration, 4), "error": error, "error_message": error_msg, 'name': name, "outpoint": outpoint, "node_rpc_timeout": conf.node_rpc_timeout, "peer_connect_timeout": conf.peer_connect_timeout, "blob_download_timeout": conf.blob_download_timeout, "use_fixed_peers": len(conf.fixed_peers) > 0, "fixed_peer_delay": fixed_peer_delay, "added_fixed_peers": added_fixed_peers, "active_peer_count": active_peer_count, "tried_peers_count": tried_peers_count, "sd_blob_hash": sd_hash, "sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4), "head_blob_hash": head_blob_hash, "head_blob_length": head_blob_length, "head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4), "connection_failures_count": connection_failures_count, "wallet_server": wallet_server } def _make_context(platform): # see https://segment.com/docs/spec/common/#context # they say they'll ignore fields outside the spec, but evidently they don't context = { 'app': { 'version': platform['lbrynet_version'], 'build': platform['build'], }, # TODO: expand os info to give linux/osx specific info 'os': { 'name': platform['os_system'], 'version': platform['os_release'] }, } if 'desktop' in platform and 'distro' in platform: context['os']['desktop'] = platform['desktop'] context['os']['distro'] = platform['distro'] return context class AnalyticsManager: def __init__(self, conf: Config, installation_id: str, session_id: str): self.conf = conf self.cookies = {} self.url = ANALYTICS_ENDPOINT self._write_key = utils.deobfuscate(ANALYTICS_TOKEN) self._tracked_data = collections.defaultdict(list) self.context = _make_context(system_info.get_platform()) self.installation_id = installation_id self.session_id = session_id self.task: typing.Optional[asyncio.Task] = None self.external_ip: typing.Optional[str] = None @property def enabled(self): return self.conf.share_usage_data @property def is_started(self): return self.task is not None async def start(self): if self.task is None: self.task = asyncio.create_task(self.run()) async def run(self): while True: if self.enabled: self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers) await self._send_heartbeat() await asyncio.sleep(1800) def stop(self): if self.task is not None and not self.task.done(): self.task.cancel() async def _post(self, data: typing.Dict): request_kwargs = { 'method': 'POST', 'url': self.url + '/track', 'headers': {'Connection': 'Close'}, 'auth': aiohttp.BasicAuth(self._write_key, ''), 'json': data, 'cookies': self.cookies } try: async with utils.aiohttp_request(**request_kwargs) as response: self.cookies.update(response.cookies) except Exception as e: log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e) async def track(self, event: typing.Dict): """Send a single tracking event""" if self.enabled: log.debug('Sending track event: %s', event) await self._post(event) async def send_upnp_setup_success_fail(self, success, status): await self.track( self._event(UPNP_SETUP, { 'success': success, 'status': status, }) ) async def send_disk_space_used(self, storage_used, storage_limit, is_from_network_quota): await self.track( self._event(DISK_SPACE, { 'used': storage_used, 'limit': storage_limit, 'from_network_quota': is_from_network_quota }) ) async def send_server_startup(self): await self.track(self._event(SERVER_STARTUP)) async def send_server_startup_success(self): await self.track(self._event(SERVER_STARTUP_SUCCESS)) async def send_server_startup_error(self, message): await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message})) async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float], total_duration: typing.Optional[float], download_id: str, name: str, outpoint: typing.Optional[str], found_peers_count: typing.Optional[int], tried_peers_count: typing.Optional[int], connection_failures_count: typing.Optional[int], added_fixed_peers: bool, fixed_peers_delay: float, sd_hash: str, sd_download_duration: typing.Optional[float] = None, head_blob_hash: typing.Optional[str] = None, head_blob_length: typing.Optional[int] = None, head_blob_duration: typing.Optional[int] = None, error: typing.Optional[str] = None, error_msg: typing.Optional[str] = None, wallet_server: typing.Optional[str] = None): await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties( self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint, found_peers_count, tried_peers_count, connection_failures_count, added_fixed_peers, fixed_peers_delay, sd_hash, sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error, error_msg, wallet_server ))) async def send_download_finished(self, download_id, name, sd_hash): await self.track( self._event( DOWNLOAD_FINISHED, { 'download_id': download_id, 'name': name, 'stream_info': sd_hash } ) ) async def send_claim_action(self, action): await self.track(self._event(CLAIM_ACTION, {'action': action})) async def send_new_channel(self): await self.track(self._event(NEW_CHANNEL)) async def send_credits_sent(self): await self.track(self._event(CREDITS_SENT)) async def _send_heartbeat(self): await self.track(self._event(HEARTBEAT)) def _event(self, event, properties: typing.Optional[typing.Dict] = None): return { 'userId': 'lbry', 'event': event, 'properties': _event_properties(self.installation_id, self.session_id, properties), 'context': self.context, 'timestamp': utils.isonow() } ================================================ FILE: lbry/extras/daemon/client.py ================================================ from lbry.extras.cli import execute_command from lbry.conf import Config def daemon_rpc(conf: Config, method: str, **kwargs): return execute_command(conf, method, kwargs, callback=lambda data: data) ================================================ FILE: lbry/extras/daemon/component.py ================================================ import asyncio import logging from lbry.conf import Config from lbry.extras.daemon.componentmanager import ComponentManager log = logging.getLogger(__name__) class ComponentType(type): def __new__(mcs, name, bases, newattrs): klass = type.__new__(mcs, name, bases, newattrs) if name != "Component" and newattrs['__module__'] != 'lbry.testcase': ComponentManager.default_component_classes[klass.component_name] = klass return klass class Component(metaclass=ComponentType): """ lbry-daemon component helper Inheriting classes will be automatically registered with the ComponentManager and must implement setup and stop methods """ depends_on = [] component_name = None def __init__(self, component_manager): self.conf: Config = component_manager.conf self.component_manager = component_manager self._running = False def __lt__(self, other): return self.component_name < other.component_name @property def running(self): return self._running async def get_status(self): # pylint: disable=no-self-use return async def start(self): raise NotImplementedError() async def stop(self): raise NotImplementedError() @property def component(self): raise NotImplementedError() async def _setup(self): try: result = await self.start() self._running = True return result except asyncio.CancelledError: log.info("Cancelled setup of %s component", self.__class__.__name__) raise except Exception as err: log.exception("Error setting up %s", self.component_name or self.__class__.__name__) raise err async def _stop(self): try: result = await self.stop() self._running = False return result except asyncio.CancelledError: log.info("Cancelled stop of %s component", self.__class__.__name__) raise except Exception as err: log.exception("Error stopping %s", self.__class__.__name__) raise err ================================================ FILE: lbry/extras/daemon/componentmanager.py ================================================ import logging import asyncio from lbry.conf import Config from lbry.error import ComponentStartConditionNotMetError from lbry.dht.peer import PeerManager log = logging.getLogger(__name__) class RegisteredConditions: conditions = {} class RequiredConditionType(type): def __new__(mcs, name, bases, newattrs): klass = type.__new__(mcs, name, bases, newattrs) if name != "RequiredCondition": if klass.name in RegisteredConditions.conditions: raise SyntaxError("already have a component registered for \"%s\"" % klass.name) RegisteredConditions.conditions[klass.name] = klass return klass class RequiredCondition(metaclass=RequiredConditionType): name = "" component = "" message = "" @staticmethod def evaluate(component): raise NotImplementedError() class ComponentManager: default_component_classes = {} def __init__(self, conf: Config, analytics_manager=None, skip_components=None, peer_manager=None, **override_components): self.conf = conf self.skip_components = skip_components or [] self.loop = asyncio.get_event_loop() self.analytics_manager = analytics_manager self.component_classes = {} self.components = set() self.started = asyncio.Event() self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop()) for component_name, component_class in self.default_component_classes.items(): if component_name in override_components: component_class = override_components.pop(component_name) if component_name not in self.skip_components: self.component_classes[component_name] = component_class if override_components: raise SyntaxError("unexpected components: %s" % override_components) for component_class in self.component_classes.values(): self.components.add(component_class(self)) def evaluate_condition(self, condition_name): if condition_name not in RegisteredConditions.conditions: raise NameError(condition_name) condition = RegisteredConditions.conditions[condition_name] try: component = self.get_component(condition.component) result = condition.evaluate(component) except Exception: log.exception('failed to evaluate condition:') result = False return result, "" if result else condition.message def sort_components(self, reverse=False): """ Sort components by requirements """ steps = [] staged = set() components = set(self.components) # components with no requirements step = [] for component in set(components): if not component.depends_on: step.append(component) staged.add(component.component_name) components.remove(component) if step: step.sort() steps.append(step) while components: step = [] to_stage = set() for component in set(components): reqs_met = 0 for needed in component.depends_on: if needed in staged: reqs_met += 1 if reqs_met == len(component.depends_on): step.append(component) to_stage.add(component.component_name) components.remove(component) if step: step.sort() staged.update(to_stage) steps.append(step) elif components: raise ComponentStartConditionNotMetError(components) if reverse: steps.reverse() return steps async def start(self): """ Start Components in sequence sorted by requirements """ for stage in self.sort_components(): needing_start = [ component._setup() for component in stage if not component.running ] if needing_start: await asyncio.wait(map(asyncio.create_task, needing_start)) self.started.set() async def stop(self): """ Stop Components in reversed startup order """ stages = self.sort_components(reverse=True) for stage in stages: needing_stop = [ component._stop() for component in stage if component.running ] if needing_stop: await asyncio.wait(map(asyncio.create_task, needing_stop)) def all_components_running(self, *component_names): """ Check if components are running :return: (bool) True if all specified components are running """ components = {component.component_name: component for component in self.components} for component in component_names: if component not in components: raise NameError("%s is not a known Component" % component) if not components[component].running: return False return True def get_components_status(self): """ List status of all the components, whether they are running or not :return: (dict) {(str) component_name: (bool) True is running else False} """ return { component.component_name: component.running for component in self.components } def get_actual_component(self, component_name): for component in self.components: if component.component_name == component_name: return component raise NameError(component_name) def get_component(self, component_name): return self.get_actual_component(component_name).component def has_component(self, component_name): return any(component for component in self.components if component_name == component.component_name) ================================================ FILE: lbry/extras/daemon/components.py ================================================ import math import os import asyncio import logging import binascii import typing import base58 from aioupnp import __version__ as aioupnp_version from aioupnp.upnp import UPnP from aioupnp.fault import UPnPError from lbry import utils from lbry.dht.node import Node from lbry.dht.peer import is_valid_public_ipv4 from lbry.dht.blob_announcer import BlobAnnouncer from lbry.blob.blob_manager import BlobManager from lbry.blob.disk_space_manager import DiskSpaceManager from lbry.blob_exchange.server import BlobServer from lbry.stream.background_downloader import BackgroundDownloader from lbry.stream.stream_manager import StreamManager from lbry.file.file_manager import FileManager from lbry.extras.daemon.component import Component from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.storage import SQLiteStorage from lbry.torrent.torrent_manager import TorrentManager from lbry.wallet import WalletManager from lbry.wallet.usage_payment import WalletServerPayer from lbry.torrent.tracker import TrackerClient from lbry.torrent.session import TorrentSession log = logging.getLogger(__name__) # settings must be initialized before this file is imported DATABASE_COMPONENT = "database" BLOB_COMPONENT = "blob_manager" WALLET_COMPONENT = "wallet" WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments" DHT_COMPONENT = "dht" HASH_ANNOUNCER_COMPONENT = "hash_announcer" FILE_MANAGER_COMPONENT = "file_manager" DISK_SPACE_COMPONENT = "disk_space" BACKGROUND_DOWNLOADER_COMPONENT = "background_downloader" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" UPNP_COMPONENT = "upnp" EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager" TRACKER_ANNOUNCER_COMPONENT = "tracker_announcer_component" LIBTORRENT_COMPONENT = "libtorrent_component" class DatabaseComponent(Component): component_name = DATABASE_COMPONENT def __init__(self, component_manager): super().__init__(component_manager) self.storage = None @property def component(self): return self.storage @staticmethod def get_current_db_revision(): return 15 @property def revision_filename(self): return os.path.join(self.conf.data_dir, 'db_revision') def _write_db_revision_file(self, version_num): with open(self.revision_filename, mode='w') as db_revision: db_revision.write(str(version_num)) async def start(self): # check directories exist, create them if they don't log.info("Loading databases") if not os.path.exists(self.revision_filename): log.info("db_revision file not found. Creating it") self._write_db_revision_file(self.get_current_db_revision()) # check the db migration and run any needed migrations with open(self.revision_filename, "r") as revision_read_handle: old_revision = int(revision_read_handle.read().strip()) if old_revision > self.get_current_db_revision(): raise Exception('This version of lbrynet is not compatible with the database\n' 'Your database is revision %i, expected %i' % (old_revision, self.get_current_db_revision())) if old_revision < self.get_current_db_revision(): from lbry.extras.daemon.migrator import dbmigrator # pylint: disable=import-outside-toplevel log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision()) await asyncio.get_event_loop().run_in_executor( None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision() ) self._write_db_revision_file(self.get_current_db_revision()) log.info("Finished upgrading the databases.") self.storage = SQLiteStorage( self.conf, os.path.join(self.conf.data_dir, "lbrynet.sqlite") ) await self.storage.open() async def stop(self): await self.storage.close() self.storage = None class WalletComponent(Component): component_name = WALLET_COMPONENT depends_on = [DATABASE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.wallet_manager = None @property def component(self): return self.wallet_manager async def get_status(self): if self.wallet_manager is None: return is_connected = self.wallet_manager.ledger.network.is_connected sessions = [] connected = None if is_connected: addr, port = self.wallet_manager.ledger.network.client.server connected = f"{addr}:{port}" sessions.append(self.wallet_manager.ledger.network.client) result = { 'connected': connected, 'connected_features': self.wallet_manager.ledger.network.server_features, 'servers': [ { 'host': session.server[0], 'port': session.server[1], 'latency': session.connection_latency, 'availability': session.available, } for session in sessions ], 'known_servers': len(self.wallet_manager.ledger.network.known_hubs), 'available_servers': 1 if is_connected else 0 } if self.wallet_manager.ledger.network.remote_height: local_height = self.wallet_manager.ledger.local_height_including_downloaded_height disk_height = len(self.wallet_manager.ledger.headers) remote_height = self.wallet_manager.ledger.network.remote_height download_height, target_height = local_height - disk_height, remote_height - disk_height if target_height > 0: progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100) else: progress = 100 best_hash = await self.wallet_manager.get_best_blockhash() result.update({ 'headers_synchronization_progress': progress, 'blocks': max(local_height, 0), 'blocks_behind': max(remote_height - local_height, 0), 'best_blockhash': best_hash, }) return result async def start(self): log.info("Starting wallet") self.wallet_manager = await WalletManager.from_lbrynet_config(self.conf) await self.wallet_manager.start() async def stop(self): await self.wallet_manager.stop() self.wallet_manager = None class WalletServerPaymentsComponent(Component): component_name = WALLET_SERVER_PAYMENTS_COMPONENT depends_on = [WALLET_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.usage_payment_service = WalletServerPayer( max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager, ) @property def component(self) -> typing.Optional[WalletServerPayer]: return self.usage_payment_service async def start(self): wallet_manager = self.component_manager.get_component(WALLET_COMPONENT) await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet) async def stop(self): await self.usage_payment_service.stop() async def get_status(self): return { 'max_fee': self.usage_payment_service.max_fee, 'running': self.usage_payment_service.running } class BlobComponent(Component): component_name = BLOB_COMPONENT depends_on = [DATABASE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.blob_manager: typing.Optional[BlobManager] = None @property def component(self) -> typing.Optional[BlobManager]: return self.blob_manager async def start(self): storage = self.component_manager.get_component(DATABASE_COMPONENT) data_store = None if DHT_COMPONENT not in self.component_manager.skip_components: dht_node: Node = self.component_manager.get_component(DHT_COMPONENT) if dht_node: data_store = dht_node.protocol.data_store blob_dir = os.path.join(self.conf.data_dir, 'blobfiles') if not os.path.isdir(blob_dir): os.mkdir(blob_dir) self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store) return await self.blob_manager.setup() async def stop(self): self.blob_manager.stop() async def get_status(self): count = 0 if self.blob_manager: count = len(self.blob_manager.completed_blob_hashes) return { 'finished_blobs': count, 'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status } class DHTComponent(Component): component_name = DHT_COMPONENT depends_on = [UPNP_COMPONENT, DATABASE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.dht_node: typing.Optional[Node] = None self.external_udp_port = None self.external_peer_port = None @property def component(self) -> typing.Optional[Node]: return self.dht_node async def get_status(self): return { 'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id), 'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers()) } def get_node_id(self): node_id_filename = os.path.join(self.conf.data_dir, "node_id") if os.path.isfile(node_id_filename): with open(node_id_filename, "r") as node_id_file: return base58.b58decode(str(node_id_file.read()).strip()) node_id = utils.generate_id() with open(node_id_filename, "w") as node_id_file: node_id_file.write(base58.b58encode(node_id).decode()) return node_id async def start(self): log.info("start the dht") upnp_component = self.component_manager.get_component(UPNP_COMPONENT) self.external_peer_port = upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port) self.external_udp_port = upnp_component.upnp_redirects.get("UDP", self.conf.udp_port) external_ip = upnp_component.external_ip storage = self.component_manager.get_component(DATABASE_COMPONENT) if not external_ip: external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers) if not external_ip: log.warning("failed to get external ip") self.dht_node = Node( self.component_manager.loop, self.component_manager.peer_manager, node_id=self.get_node_id(), internal_udp_port=self.conf.udp_port, udp_port=self.external_udp_port, external_ip=external_ip, peer_port=self.external_peer_port, rpc_timeout=self.conf.node_rpc_timeout, split_buckets_under_index=self.conf.split_buckets_under_index, is_bootstrap_node=self.conf.is_bootstrap_node, storage=storage ) self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes) log.info("Started the dht") async def stop(self): self.dht_node.stop() class HashAnnouncerComponent(Component): component_name = HASH_ANNOUNCER_COMPONENT depends_on = [DHT_COMPONENT, DATABASE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.hash_announcer: typing.Optional[BlobAnnouncer] = None @property def component(self) -> typing.Optional[BlobAnnouncer]: return self.hash_announcer async def start(self): storage = self.component_manager.get_component(DATABASE_COMPONENT) dht_node = self.component_manager.get_component(DHT_COMPONENT) self.hash_announcer = BlobAnnouncer(self.component_manager.loop, dht_node, storage) self.hash_announcer.start(self.conf.concurrent_blob_announcers) log.info("Started blob announcer") async def stop(self): self.hash_announcer.stop() log.info("Stopped blob announcer") async def get_status(self): return { 'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.announce_queue) } class FileManagerComponent(Component): component_name = FILE_MANAGER_COMPONENT depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.file_manager: typing.Optional[FileManager] = None @property def component(self) -> typing.Optional[FileManager]: return self.file_manager async def get_status(self): if not self.file_manager: return return { 'managed_files': len(self.file_manager.get_filtered()), } async def start(self): blob_manager = self.component_manager.get_component(BLOB_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT) wallet = self.component_manager.get_component(WALLET_COMPONENT) node = self.component_manager.get_component(DHT_COMPONENT) \ if self.component_manager.has_component(DHT_COMPONENT) else None log.info('Starting the file manager') loop = asyncio.get_event_loop() self.file_manager = FileManager( loop, self.conf, wallet, storage, self.component_manager.analytics_manager ) self.file_manager.source_managers['stream'] = StreamManager( loop, self.conf, blob_manager, wallet, storage, node, ) if self.component_manager.has_component(LIBTORRENT_COMPONENT): torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) self.file_manager.source_managers['torrent'] = TorrentManager( loop, self.conf, torrent, storage, self.component_manager.analytics_manager ) await self.file_manager.start() log.info('Done setting up file manager') async def stop(self): await self.file_manager.stop() class BackgroundDownloaderComponent(Component): MIN_PREFIX_COLLIDING_BITS = 8 component_name = BACKGROUND_DOWNLOADER_COMPONENT depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT, DISK_SPACE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.background_task: typing.Optional[asyncio.Task] = None self.download_loop_delay_seconds = 60 self.ongoing_download: typing.Optional[asyncio.Task] = None self.space_manager: typing.Optional[DiskSpaceManager] = None self.blob_manager: typing.Optional[BlobManager] = None self.background_downloader: typing.Optional[BackgroundDownloader] = None self.dht_node: typing.Optional[Node] = None self.space_available: typing.Optional[int] = None @property def is_busy(self): return bool(self.ongoing_download and not self.ongoing_download.done()) @property def component(self) -> 'BackgroundDownloaderComponent': return self async def get_status(self): return {'running': self.background_task is not None and not self.background_task.done(), 'available_free_space_mb': self.space_available, 'ongoing_download': self.is_busy} async def download_blobs_in_background(self): while True: self.space_available = await self.space_manager.get_free_space_mb(True) if not self.is_busy and self.space_available > 10: self._download_next_close_blob_hash() await asyncio.sleep(self.download_loop_delay_seconds) def _download_next_close_blob_hash(self): node_id = self.dht_node.protocol.node_id for blob_hash in self.dht_node.stored_blob_hashes: if blob_hash.hex() in self.blob_manager.completed_blob_hashes: continue if utils.get_colliding_prefix_bits(node_id, blob_hash) >= self.MIN_PREFIX_COLLIDING_BITS: self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash.hex())) return async def start(self): self.space_manager: DiskSpaceManager = self.component_manager.get_component(DISK_SPACE_COMPONENT) if not self.component_manager.has_component(DHT_COMPONENT): return self.dht_node = self.component_manager.get_component(DHT_COMPONENT) self.blob_manager = self.component_manager.get_component(BLOB_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT) self.background_downloader = BackgroundDownloader(self.conf, storage, self.blob_manager, self.dht_node) self.background_task = asyncio.create_task(self.download_blobs_in_background()) async def stop(self): if self.ongoing_download and not self.ongoing_download.done(): self.ongoing_download.cancel() if self.background_task: self.background_task.cancel() class DiskSpaceComponent(Component): component_name = DISK_SPACE_COMPONENT depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.disk_space_manager: typing.Optional[DiskSpaceManager] = None @property def component(self) -> typing.Optional[DiskSpaceManager]: return self.disk_space_manager async def get_status(self): if self.disk_space_manager: space_used = await self.disk_space_manager.get_space_used_mb(cached=True) return { 'total_used_mb': space_used['total'], 'published_blobs_storage_used_mb': space_used['private_storage'], 'content_blobs_storage_used_mb': space_used['content_storage'], 'seed_blobs_storage_used_mb': space_used['network_storage'], 'running': self.disk_space_manager.running, } return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False} async def start(self): db = self.component_manager.get_component(DATABASE_COMPONENT) blob_manager = self.component_manager.get_component(BLOB_COMPONENT) self.disk_space_manager = DiskSpaceManager( self.conf, db, blob_manager, analytics=self.component_manager.analytics_manager ) await self.disk_space_manager.start() async def stop(self): await self.disk_space_manager.stop() class TorrentComponent(Component): component_name = LIBTORRENT_COMPONENT def __init__(self, component_manager): super().__init__(component_manager) self.torrent_session = None @property def component(self) -> typing.Optional[TorrentSession]: return self.torrent_session async def get_status(self): if not self.torrent_session: return return { 'running': True, # TODO: what to return here? } async def start(self): self.torrent_session = TorrentSession(asyncio.get_event_loop(), None) await self.torrent_session.bind() # TODO: specify host/port async def stop(self): if self.torrent_session: await self.torrent_session.pause() class PeerProtocolServerComponent(Component): component_name = PEER_PROTOCOL_SERVER_COMPONENT depends_on = [UPNP_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.blob_server: typing.Optional[BlobServer] = None @property def component(self) -> typing.Optional[BlobServer]: return self.blob_server async def start(self): log.info("start blob server") blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT) wallet: WalletManager = self.component_manager.get_component(WALLET_COMPONENT) peer_port = self.conf.tcp_port address = await wallet.get_unused_address() self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address) self.blob_server.start_server(peer_port, interface=self.conf.network_interface) await self.blob_server.started_listening.wait() async def stop(self): if self.blob_server: self.blob_server.stop_server() class UPnPComponent(Component): component_name = UPNP_COMPONENT def __init__(self, component_manager): super().__init__(component_manager) self._int_peer_port = self.conf.tcp_port self._int_dht_node_port = self.conf.udp_port self.use_upnp = self.conf.use_upnp self.upnp: typing.Optional[UPnP] = None self.upnp_redirects = {} self.external_ip: typing.Optional[str] = None self._maintain_redirects_task = None @property def component(self) -> 'UPnPComponent': return self async def _repeatedly_maintain_redirects(self, now=True): while True: if now: await self._maintain_redirects() await asyncio.sleep(360) async def _maintain_redirects(self): # setup the gateway if necessary if not self.upnp: try: self.upnp = await UPnP.discover(loop=self.component_manager.loop) log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string) except Exception as err: log.warning("upnp discovery failed: %s", err) self.upnp = None # update the external ip external_ip = None if self.upnp: try: external_ip = await self.upnp.get_external_ip() if external_ip != "0.0.0.0" and not self.external_ip: log.info("got external ip from UPnP: %s", external_ip) except (asyncio.TimeoutError, UPnPError, NotImplementedError): pass if external_ip and not is_valid_public_ipv4(external_ip): log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip) external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers) if self.external_ip and self.external_ip != external_ip: log.info("external ip changed from %s to %s", self.external_ip, external_ip) if external_ip: self.external_ip = external_ip dht_component = self.component_manager.get_component(DHT_COMPONENT) if dht_component: dht_node = dht_component.component dht_node.protocol.external_ip = external_ip # assert self.external_ip is not None # TODO: handle going/starting offline if not self.upnp_redirects and self.upnp: # setup missing redirects log.info("add UPnP port mappings") upnp_redirects = {} if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: try: upnp_redirects["TCP"] = await self.upnp.get_next_mapping( self._int_peer_port, "TCP", "LBRY peer port", self._int_peer_port ) except (UPnPError, asyncio.TimeoutError, NotImplementedError): pass if DHT_COMPONENT not in self.component_manager.skip_components: try: upnp_redirects["UDP"] = await self.upnp.get_next_mapping( self._int_dht_node_port, "UDP", "LBRY DHT port", self._int_dht_node_port ) except (UPnPError, asyncio.TimeoutError, NotImplementedError): pass if upnp_redirects: log.info("set up redirects: %s", upnp_redirects) self.upnp_redirects.update(upnp_redirects) elif self.upnp: # check existing redirects are still active found = set() mappings = await self.upnp.get_redirects() for mapping in mappings: proto = mapping.protocol if proto in self.upnp_redirects and mapping.external_port == self.upnp_redirects[proto]: if mapping.lan_address == self.upnp.lan_address: found.add(proto) if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components: try: udp_port = await self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port") self.upnp_redirects['UDP'] = udp_port log.info("refreshed upnp redirect for dht port: %i", udp_port) except (asyncio.TimeoutError, UPnPError, NotImplementedError): del self.upnp_redirects['UDP'] if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: try: tcp_port = await self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port") self.upnp_redirects['TCP'] = tcp_port log.info("refreshed upnp redirect for peer port: %i", tcp_port) except (asyncio.TimeoutError, UPnPError, NotImplementedError): del self.upnp_redirects['TCP'] if ('TCP' in self.upnp_redirects and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and \ ('UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components): if self.upnp_redirects: log.debug("upnp redirects are still active") async def start(self): log.info("detecting external ip") if not self.use_upnp: self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers) return success = False await self._maintain_redirects() if self.upnp: if not self.upnp_redirects and not all( x in self.component_manager.skip_components for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT) ): log.error("failed to setup upnp") else: success = True if self.upnp_redirects: log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) else: log.error("failed to setup upnp") if not self.external_ip: self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers) if self.external_ip: log.info("detected external ip using %s fallback", probed_url) if self.component_manager.analytics_manager: self.component_manager.loop.create_task( self.component_manager.analytics_manager.send_upnp_setup_success_fail( success, await self.get_status() ) ) self._maintain_redirects_task = self.component_manager.loop.create_task( self._repeatedly_maintain_redirects(now=False) ) async def stop(self): if self.upnp_redirects: log.info("Removing upnp redirects: %s", self.upnp_redirects) await asyncio.wait([ self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items() ]) if self._maintain_redirects_task and not self._maintain_redirects_task.done(): self._maintain_redirects_task.cancel() async def get_status(self): return { 'aioupnp_version': aioupnp_version, 'redirects': self.upnp_redirects, 'gateway': 'No gateway found' if not self.upnp else self.upnp.gateway.manufacturer_string, 'dht_redirect_set': 'UDP' in self.upnp_redirects, 'peer_redirect_set': 'TCP' in self.upnp_redirects, 'external_ip': self.external_ip } class ExchangeRateManagerComponent(Component): component_name = EXCHANGE_RATE_MANAGER_COMPONENT def __init__(self, component_manager): super().__init__(component_manager) self.exchange_rate_manager = ExchangeRateManager() @property def component(self) -> ExchangeRateManager: return self.exchange_rate_manager async def start(self): self.exchange_rate_manager.start() async def stop(self): self.exchange_rate_manager.stop() class TrackerAnnouncerComponent(Component): component_name = TRACKER_ANNOUNCER_COMPONENT depends_on = [FILE_MANAGER_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.file_manager = None self.announce_task = None self.tracker_client: typing.Optional[TrackerClient] = None @property def component(self): return self.tracker_client @property def running(self): return self._running and self.announce_task and not self.announce_task.done() async def announce_forever(self): while True: sleep_seconds = 60.0 announce_sd_hashes = [] for file in self.file_manager.get_filtered(): if not file.downloader: continue announce_sd_hashes.append(bytes.fromhex(file.sd_hash)) await self.tracker_client.announce_many(*announce_sd_hashes) await asyncio.sleep(sleep_seconds) async def start(self): node = self.component_manager.get_component(DHT_COMPONENT) \ if self.component_manager.has_component(DHT_COMPONENT) else None node_id = node.protocol.node_id if node else None self.tracker_client = TrackerClient(node_id, self.conf.tcp_port, lambda: self.conf.tracker_servers) await self.tracker_client.start() self.file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT) self.announce_task = asyncio.create_task(self.announce_forever()) async def stop(self): self.file_manager = None if self.announce_task and not self.announce_task.done(): self.announce_task.cancel() self.announce_task = None self.tracker_client.stop() ================================================ FILE: lbry/extras/daemon/daemon.py ================================================ import linecache import os import re import asyncio import logging import json import time import inspect import typing import random import tracemalloc import itertools from urllib.parse import urlencode, quote from typing import Callable, Optional, List from binascii import hexlify, unhexlify from traceback import format_exc from functools import wraps, partial import base58 from aiohttp import web from prometheus_client import generate_latest as prom_generate_latest, Gauge, Histogram, Counter from google.protobuf.message import DecodeError from lbry.wallet import ( Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic, Transaction, Output, Input, Account, database ) from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES from lbry.wallet.bip32 import PrivateKey from lbry.crypto.base58 import Base58 from lbry import utils from lbry.conf import Config, Setting, NOT_SET from lbry.blob.blob_file import is_valid_blobhash, BlobBuffer from lbry.blob_exchange.downloader import download_blob from lbry.dht.peer import make_kademlia_peer from lbry.error import ( DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError, CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError, ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError, InputValueError ) from lbry.extras import system_info from lbry.extras.daemon import analytics from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT, DISK_SPACE_COMPONENT, TRACKER_ANNOUNCER_COMPONENT from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT from lbry.extras.daemon.componentmanager import RequiredCondition from lbry.extras.daemon.componentmanager import ComponentManager from lbry.extras.daemon.json_response_encoder import JSONResponseEncoder from lbry.extras.daemon.undecorated import undecorated from lbry.extras.daemon.security import ensure_request_allowed from lbry.file_analysis import VideoFileAnalyzer from lbry.schema.claim import Claim from lbry.schema.url import URL if typing.TYPE_CHECKING: from lbry.blob.blob_manager import BlobManager from lbry.dht.node import Node from lbry.extras.daemon.components import UPnPComponent, DiskSpaceManager from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.storage import SQLiteStorage from lbry.wallet import WalletManager, Ledger from lbry.file.file_manager import FileManager log = logging.getLogger(__name__) RANGE_FIELDS = { 'height', 'creation_height', 'activation_height', 'expiration_height', 'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount', 'tx_position', 'repost_count', 'limit_claims_per_channel', 'amount', 'effective_amount', 'support_amount', 'trending_score', 'censor_type', 'tx_num' } MY_RANGE_FIELDS = RANGE_FIELDS - {"limit_claims_per_channel"} REPLACEMENTS = { 'claim_name': 'normalized_name', 'name': 'normalized_name', 'txid': 'tx_id', 'nout': 'tx_nout', 'trending_group': 'trending_score', 'trending_mixed': 'trending_score', 'trending_global': 'trending_score', 'trending_local': 'trending_score', 'reposted': 'repost_count', 'stream_types': 'stream_type', 'media_types': 'media_type', 'valid_channel_signature': 'is_signature_valid' } def is_transactional_function(name): for action in ('create', 'update', 'abandon', 'send', 'fund'): if action in name: return True def requires(*components, **conditions): if conditions and ["conditions"] != list(conditions.keys()): raise SyntaxError("invalid conditions argument") condition_names = conditions.get("conditions", []) def _wrap(method): @wraps(method) def _inner(*args, **kwargs): component_manager = args[0].component_manager for condition_name in condition_names: condition_result, err_msg = component_manager.evaluate_condition(condition_name) if not condition_result: raise ComponentStartConditionNotMetError(err_msg) if not component_manager.all_components_running(*components): raise ComponentsNotStartedError( f"the following required components have not yet started: {json.dumps(components)}" ) return method(*args, **kwargs) return _inner return _wrap def deprecated(new_command=None): def _deprecated_wrapper(f): f.new_command = new_command f._deprecated = True return f return _deprecated_wrapper INITIALIZING_CODE = 'initializing' # TODO: make this consistent with the stages in Downloader.py DOWNLOAD_METADATA_CODE = 'downloading_metadata' DOWNLOAD_TIMEOUT_CODE = 'timeout' DOWNLOAD_RUNNING_CODE = 'running' DOWNLOAD_STOPPED_CODE = 'stopped' STREAM_STAGES = [ (INITIALIZING_CODE, 'Initializing'), (DOWNLOAD_METADATA_CODE, 'Downloading metadata'), (DOWNLOAD_RUNNING_CODE, 'Started %s, got %s/%s blobs, stream status: %s'), (DOWNLOAD_STOPPED_CODE, 'Paused stream'), (DOWNLOAD_TIMEOUT_CODE, 'Stream timed out') ] SHORT_ID_LEN = 20 MAX_UPDATE_FEE_ESTIMATE = 0.3 DEFAULT_PAGE_SIZE = 20 VALID_FULL_CLAIM_ID = re.compile('[0-9a-fA-F]{40}') def encode_pagination_doc(items): return { "page": "Page number of the current items.", "page_size": "Number of items to show on a page.", "total_pages": "Total number of pages.", "total_items": "Total number of items.", "items": [items], } async def paginate_rows(get_records: Callable, get_record_count: Optional[Callable], page: Optional[int], page_size: Optional[int], **constraints): page = max(1, page or 1) page_size = max(1, page_size or DEFAULT_PAGE_SIZE) constraints.update({ "offset": page_size * (page - 1), "limit": page_size }) items = await get_records(**constraints) result = {"items": items, "page": page, "page_size": page_size} if get_record_count is not None: total_items = await get_record_count(**constraints) result["total_pages"] = int((total_items + (page_size - 1)) / page_size) result["total_items"] = total_items return result def paginate_list(items: List, page: Optional[int], page_size: Optional[int]): page = max(1, page or 1) page_size = max(1, page_size or DEFAULT_PAGE_SIZE) total_items = len(items) offset = page_size * (page - 1) subitems = [] if offset <= total_items: subitems = items[offset:offset+page_size] return { "items": subitems, "total_pages": int((total_items + (page_size - 1)) / page_size), "total_items": total_items, "page": page, "page_size": page_size } DHT_HAS_CONTACTS = "dht_has_contacts" class DHTHasContacts(RequiredCondition): name = DHT_HAS_CONTACTS component = DHT_COMPONENT message = "your node is not connected to the dht" @staticmethod def evaluate(component): return len(component.contacts) > 0 class JSONRPCError: # http://www.jsonrpc.org/specification#error_object CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text. CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object. CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available. CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s). CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?) CODE_APPLICATION_ERROR = -32500 # Generic error with our app?? CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed MESSAGES = { CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.", CODE_INVALID_REQUEST: "JSON data is not a valid Request", CODE_METHOD_NOT_FOUND: "Method Not Found", CODE_INVALID_PARAMS: "Invalid Params", CODE_INTERNAL_ERROR: "Internal Error", CODE_AUTHENTICATION_ERROR: "Authentication Failed", } HTTP_CODES = { CODE_INVALID_REQUEST: 400, CODE_PARSE_ERROR: 400, CODE_INVALID_PARAMS: 400, CODE_METHOD_NOT_FOUND: 404, CODE_INTERNAL_ERROR: 500, CODE_APPLICATION_ERROR: 500, CODE_AUTHENTICATION_ERROR: 401, } def __init__(self, code: int, message: str, data: dict = None): assert code and isinstance(code, int), "'code' must be an int" assert message and isinstance(message, str), "'message' must be a string" assert data is None or isinstance(data, dict), "'data' must be None or a dict" self.code = code self.message = message self.data = data or {} def to_dict(self): return { 'code': self.code, 'message': self.message, 'data': self.data, } @staticmethod def filter_traceback(traceback): result = [] if traceback is not None: result = trace_lines = traceback.split("\n") for i, t in enumerate(trace_lines): if "--- ---" in t: if len(trace_lines) > i + 1: result = [j for j in trace_lines[i + 1:] if j] break return result @classmethod def create_command_exception(cls, command, args, kwargs, exception, traceback): if 'password' in kwargs and isinstance(kwargs['password'], str): kwargs['password'] = '*'*len(kwargs['password']) return cls( cls.CODE_APPLICATION_ERROR, str(exception), { 'name': exception.__class__.__name__, 'traceback': cls.filter_traceback(traceback), 'command': command, 'args': args, 'kwargs': kwargs, } ) class UnknownAPIMethodError(Exception): pass def jsonrpc_dumps_pretty(obj, **kwargs): if isinstance(obj, JSONRPCError): data = {"jsonrpc": "2.0", "error": obj.to_dict()} else: data = {"jsonrpc": "2.0", "result": obj} return json.dumps(data, cls=JSONResponseEncoder, sort_keys=True, indent=2, **kwargs) + "\n" def trap(err, *to_trap): err.trap(*to_trap) class JSONRPCServerType(type): def __new__(mcs, name, bases, newattrs): klass = type.__new__(mcs, name, bases, newattrs) klass.callable_methods = {} klass.deprecated_methods = {} for methodname in dir(klass): if methodname.startswith("jsonrpc_"): method = getattr(klass, methodname) if not hasattr(method, '_deprecated'): klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method}) else: klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method}) return klass HISTOGRAM_BUCKETS = ( .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') ) class Daemon(metaclass=JSONRPCServerType): """ LBRYnet daemon, a jsonrpc interface to lbry functions """ callable_methods: dict deprecated_methods: dict pending_requests_metric = Gauge( "pending_requests", "Number of running api requests", namespace="daemon_api", labelnames=("method",) ) requests_count_metric = Counter( "requests_count", "Number of requests received", namespace="daemon_api", labelnames=("method",) ) failed_request_metric = Counter( "failed_request_count", "Number of failed requests", namespace="daemon_api", labelnames=("method",) ) cancelled_request_metric = Counter( "cancelled_request_count", "Number of cancelled requests", namespace="daemon_api", labelnames=("method",) ) response_time_metric = Histogram( "response_time", "Response times", namespace="daemon_api", buckets=HISTOGRAM_BUCKETS, labelnames=("method",) ) def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None): self.conf = conf self.platform_info = system_info.get_platform() self._video_file_analyzer = VideoFileAnalyzer(conf) self._node_id = None self._installation_id = None self.session_id = base58.b58encode(utils.generate_id()).decode() self.analytics_manager = analytics.AnalyticsManager(conf, self.installation_id, self.session_id) self.component_manager = component_manager or ComponentManager( conf, analytics_manager=self.analytics_manager, skip_components=conf.components_to_skip or [] ) self.component_startup_task = None logging.getLogger('aiohttp.access').setLevel(logging.WARN) rpc_app = web.Application() rpc_app.router.add_get('/lbryapi', self.handle_old_jsonrpc) rpc_app.router.add_post('/lbryapi', self.handle_old_jsonrpc) rpc_app.router.add_post('/', self.handle_old_jsonrpc) rpc_app.router.add_options('/', self.add_cors_headers) self.rpc_runner = web.AppRunner(rpc_app) streaming_app = web.Application() streaming_app.router.add_get('/get/{claim_name}', self.handle_stream_get_request) streaming_app.router.add_get('/get/{claim_name}/{claim_id}', self.handle_stream_get_request) streaming_app.router.add_get('/stream/{sd_hash}', self.handle_stream_range_request) self.streaming_runner = web.AppRunner(streaming_app) prom_app = web.Application() prom_app.router.add_get('/metrics', self.handle_metrics_get_request) self.metrics_runner = web.AppRunner(prom_app) @property def dht_node(self) -> typing.Optional['Node']: return self.component_manager.get_component(DHT_COMPONENT) @property def wallet_manager(self) -> typing.Optional['WalletManager']: return self.component_manager.get_component(WALLET_COMPONENT) @property def storage(self) -> typing.Optional['SQLiteStorage']: return self.component_manager.get_component(DATABASE_COMPONENT) @property def file_manager(self) -> typing.Optional['FileManager']: return self.component_manager.get_component(FILE_MANAGER_COMPONENT) @property def exchange_rate_manager(self) -> typing.Optional['ExchangeRateManager']: return self.component_manager.get_component(EXCHANGE_RATE_MANAGER_COMPONENT) @property def blob_manager(self) -> typing.Optional['BlobManager']: return self.component_manager.get_component(BLOB_COMPONENT) @property def disk_space_manager(self) -> typing.Optional['DiskSpaceManager']: return self.component_manager.get_component(DISK_SPACE_COMPONENT) @property def upnp(self) -> typing.Optional['UPnPComponent']: return self.component_manager.get_component(UPNP_COMPONENT) @classmethod def get_api_definitions(cls): prefix = 'jsonrpc_' not_grouped = ['routing_table_get', 'ffmpeg_find'] api = { 'groups': { group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip() for group_name in dir(cls) if group_name.endswith('_DOC') }, 'commands': {} } for jsonrpc_method in dir(cls): if jsonrpc_method.startswith(prefix): full_name = jsonrpc_method[len(prefix):] method = getattr(cls, jsonrpc_method) if full_name in not_grouped: name_parts = [full_name] else: name_parts = full_name.split('_', 1) if len(name_parts) == 1: group = None name, = name_parts elif len(name_parts) == 2: group, name = name_parts assert group in api['groups'], \ f"Group {group} does not have doc string for command {full_name}." else: raise NameError(f'Could not parse method name: {jsonrpc_method}') api['commands'][full_name] = { 'api_method_name': full_name, 'name': name, 'group': group, 'doc': method.__doc__, 'method': method, } if hasattr(method, '_deprecated'): api['commands'][full_name]['replaced_by'] = method.new_command for command in api['commands'].values(): if 'replaced_by' in command: command['replaced_by'] = api['commands'][command['replaced_by']] return api @property def db_revision_file_path(self): return os.path.join(self.conf.data_dir, 'db_revision') @property def installation_id(self): install_id_filename = os.path.join(self.conf.data_dir, "install_id") if not self._installation_id: if os.path.isfile(install_id_filename): with open(install_id_filename, "r") as install_id_file: self._installation_id = str(install_id_file.read()).strip() if not self._installation_id: self._installation_id = base58.b58encode(utils.generate_id()).decode() with open(install_id_filename, "w") as install_id_file: install_id_file.write(self._installation_id) return self._installation_id def ensure_data_dir(self): if not os.path.isdir(self.conf.data_dir): os.makedirs(self.conf.data_dir) if not os.path.isdir(os.path.join(self.conf.data_dir, "blobfiles")): os.makedirs(os.path.join(self.conf.data_dir, "blobfiles")) return self.conf.data_dir def ensure_wallet_dir(self): if not os.path.isdir(self.conf.wallet_dir): os.makedirs(self.conf.wallet_dir) def ensure_download_dir(self): if not os.path.isdir(self.conf.download_dir): os.makedirs(self.conf.download_dir) async def start(self): log.info("Starting LBRYNet Daemon") log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2)) log.info("Platform: %s", json.dumps(self.platform_info, indent=2)) await self.analytics_manager.send_server_startup() await self.rpc_runner.setup() await self.streaming_runner.setup() await self.metrics_runner.setup() try: rpc_site = web.TCPSite(self.rpc_runner, self.conf.api_host, self.conf.api_port, shutdown_timeout=.5) await rpc_site.start() log.info('RPC server listening on TCP %s:%i', *rpc_site._server.sockets[0].getsockname()[:2]) except OSError as e: log.error('RPC server failed to bind TCP %s:%i', self.conf.api_host, self.conf.api_port) await self.analytics_manager.send_server_startup_error(str(e)) raise SystemExit() try: streaming_site = web.TCPSite(self.streaming_runner, self.conf.streaming_host, self.conf.streaming_port, shutdown_timeout=.5) await streaming_site.start() log.info('media server listening on TCP %s:%i', *streaming_site._server.sockets[0].getsockname()[:2]) except OSError as e: log.error('media server failed to bind TCP %s:%i', self.conf.streaming_host, self.conf.streaming_port) await self.analytics_manager.send_server_startup_error(str(e)) raise SystemExit() if self.conf.prometheus_port: try: prom_site = web.TCPSite(self.metrics_runner, "0.0.0.0", self.conf.prometheus_port, shutdown_timeout=.5) await prom_site.start() log.info('metrics server listening on TCP %s:%i', *prom_site._server.sockets[0].getsockname()[:2]) except OSError as e: log.error('metrics server failed to bind TCP :%i', self.conf.prometheus_port) await self.analytics_manager.send_server_startup_error(str(e)) raise SystemExit() try: await self.initialize() except asyncio.CancelledError: log.info("shutting down before finished starting") await self.analytics_manager.send_server_startup_error("shutting down before finished starting") raise except Exception as e: await self.analytics_manager.send_server_startup_error(str(e)) log.exception('Failed to start lbrynet') raise SystemExit() await self.analytics_manager.send_server_startup_success() async def initialize(self): self.ensure_data_dir() self.ensure_wallet_dir() self.ensure_download_dir() if not self.analytics_manager.is_started: await self.analytics_manager.start() self.component_startup_task = asyncio.create_task(self.component_manager.start()) await self.component_startup_task async def stop(self): if self.component_startup_task is not None: if self.component_startup_task.done(): await self.component_manager.stop() else: self.component_startup_task.cancel() # the wallet component might have not started try: wallet_component = self.component_manager.get_actual_component('wallet') except NameError: pass else: await wallet_component.stop() await self.component_manager.stop() log.info("stopped api components") await self.rpc_runner.cleanup() await self.streaming_runner.cleanup() await self.metrics_runner.cleanup() log.info("stopped api server") if self.analytics_manager.is_started: self.analytics_manager.stop() log.info("finished shutting down") async def add_cors_headers(self, request): if self.conf.allowed_origin: return web.Response( headers={ 'Access-Control-Allow-Origin': self.conf.allowed_origin, 'Access-Control-Allow-Methods': self.conf.allowed_origin, 'Access-Control-Allow-Headers': self.conf.allowed_origin, } ) return None async def handle_old_jsonrpc(self, request): ensure_request_allowed(request, self.conf) data = await request.json() params = data.get('params', {}) include_protobuf = params.pop('include_protobuf', False) if isinstance(params, dict) else False result = await self._process_rpc_call(data) ledger = None if 'wallet' in self.component_manager.get_components_status(): # self.ledger only available if wallet component is not skipped ledger = self.ledger try: encoded_result = jsonrpc_dumps_pretty( result, ledger=ledger, include_protobuf=include_protobuf) except Exception: log.exception('Failed to encode JSON RPC result:') encoded_result = jsonrpc_dumps_pretty(JSONRPCError( JSONRPCError.CODE_APPLICATION_ERROR, 'After successfully executing the command, failed to encode result for JSON RPC response.', {'traceback': format_exc()} ), ledger=ledger) headers = {} if self.conf.allowed_origin: headers.update({ 'Access-Control-Allow-Origin': self.conf.allowed_origin, 'Access-Control-Allow-Methods': self.conf.allowed_origin, 'Access-Control-Allow-Headers': self.conf.allowed_origin, }) return web.Response( text=encoded_result, headers=headers, content_type='application/json' ) @staticmethod async def handle_metrics_get_request(request: web.Request): try: return web.Response( text=prom_generate_latest().decode(), content_type='text/plain; version=0.0.4' ) except Exception: log.exception('could not generate prometheus data') raise async def handle_stream_get_request(self, request: web.Request): if not self.conf.streaming_get: log.warning("streaming_get is disabled, rejecting request") raise web.HTTPForbidden() name_and_claim_id = request.path.split("/get/")[1] if "/" not in name_and_claim_id: uri = f"lbry://{name_and_claim_id}" else: name, claim_id = name_and_claim_id.split("/") uri = f"lbry://{name}#{claim_id}" if not self.file_manager.started.is_set(): await self.file_manager.started.wait() stream = await self.jsonrpc_get(uri) if isinstance(stream, dict): raise web.HTTPServerError(text=stream['error']) raise web.HTTPFound(f"/stream/{stream.sd_hash}") async def handle_stream_range_request(self, request: web.Request): try: return await self._handle_stream_range_request(request) except web.HTTPException as err: log.warning("http code during /stream range request: %s", err) raise err except asyncio.CancelledError: # if not excepted here, it would bubble up the error to the console. every time you closed # a running tab, you'd get this error in the console log.debug("/stream range request cancelled") except Exception: log.exception("error handling /stream range request") raise finally: log.debug("finished handling /stream range request") async def _handle_stream_range_request(self, request: web.Request): sd_hash = request.path.split("/stream/")[1] if not self.file_manager.started.is_set(): await self.file_manager.started.wait() if sd_hash not in self.file_manager.streams: return web.HTTPNotFound() return await self.file_manager.stream_partial_content(request, sd_hash) async def _process_rpc_call(self, data): args = data.get('params', {}) try: function_name = data['method'] except KeyError: return JSONRPCError( JSONRPCError.CODE_METHOD_NOT_FOUND, "Missing 'method' value in request." ) try: method = self._get_jsonrpc_method(function_name) except UnknownAPIMethodError: return JSONRPCError( JSONRPCError.CODE_METHOD_NOT_FOUND, str(CommandDoesNotExistError(function_name)) ) if args in ([{}], []): _args, _kwargs = (), {} elif isinstance(args, dict): _args, _kwargs = (), args elif isinstance(args, list) and len(args) == 1 and isinstance(args[0], dict): # TODO: this is for backwards compatibility. Remove this once API and UI are updated # TODO: also delete EMPTY_PARAMS then _args, _kwargs = (), args[0] elif isinstance(args, list) and len(args) == 2 and \ isinstance(args[0], list) and isinstance(args[1], dict): _args, _kwargs = args else: return JSONRPCError( JSONRPCError.CODE_INVALID_PARAMS, f"Invalid parameters format: {args}" ) if is_transactional_function(function_name): log.info("%s %s %s", function_name, _args, _kwargs) params_error, erroneous_params = self._check_params(method, _args, _kwargs) if params_error is not None: params_error_message = '{} for {} command: {}'.format( params_error, function_name, ', '.join(erroneous_params) ) log.warning(params_error_message) return JSONRPCError( JSONRPCError.CODE_INVALID_PARAMS, params_error_message, ) self.pending_requests_metric.labels(method=function_name).inc() self.requests_count_metric.labels(method=function_name).inc() start = time.perf_counter() try: result = method(self, *_args, **_kwargs) if asyncio.iscoroutine(result): result = await result return result except asyncio.CancelledError: self.cancelled_request_metric.labels(method=function_name).inc() log.info("cancelled API call for: %s", function_name) raise except Exception as e: # pylint: disable=broad-except self.failed_request_metric.labels(method=function_name).inc() if not isinstance(e, BaseError): log.exception("error handling api request") else: log.error("error handling api request: %s", e) return JSONRPCError.create_command_exception( command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc() ) finally: self.pending_requests_metric.labels(method=function_name).dec() self.response_time_metric.labels(method=function_name).observe(time.perf_counter() - start) def _verify_method_is_callable(self, function_path): if function_path not in self.callable_methods: raise UnknownAPIMethodError(function_path) def _get_jsonrpc_method(self, function_path): if function_path in self.deprecated_methods: new_command = self.deprecated_methods[function_path].new_command log.warning('API function \"%s\" is deprecated, please update to use \"%s\"', function_path, new_command) function_path = new_command self._verify_method_is_callable(function_path) return self.callable_methods.get(function_path) @staticmethod def _check_params(function, args_tup, args_dict): argspec = inspect.getfullargspec(undecorated(function)) num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults) duplicate_params = [ duplicate_param for duplicate_param in argspec.args[1:len(args_tup) + 1] if duplicate_param in args_dict ] if duplicate_params: return 'Duplicate parameters', duplicate_params missing_required_params = [ required_param for required_param in argspec.args[len(args_tup) + 1:-num_optional_params] if required_param not in args_dict ] if len(missing_required_params) > 0: return 'Missing required parameters', missing_required_params extraneous_params = [] if argspec.varkw is not None else [ extra_param for extra_param in args_dict if extra_param not in argspec.args[1:] ] if len(extraneous_params) > 0: return 'Extraneous parameters', extraneous_params return None, None @property def ledger(self) -> Optional['Ledger']: try: return self.wallet_manager.default_account.ledger except AttributeError: return None async def get_est_cost_from_uri(self, uri: str) -> typing.Optional[float]: """ Resolve a name and return the estimated stream cost """ resolved = await self.resolve([], uri) if resolved: claim_response = resolved[uri] else: claim_response = None if claim_response and 'claim' in claim_response: if 'value' in claim_response['claim'] and claim_response['claim']['value'] is not None: claim_value = Claim.from_bytes(claim_response['claim']['value']) if not claim_value.stream.has_fee: return 0.0 return round( self.exchange_rate_manager.convert_currency( claim_value.stream.fee.currency, "LBC", claim_value.stream.fee.amount ), 5 ) else: log.warning("Failed to estimate cost for %s", uri) ############################################################################ # # # JSON-RPC API methods start here # # # ############################################################################ def jsonrpc_stop(self): # pylint: disable=no-self-use """ Stop lbrynet API server. Usage: stop Options: None Returns: (string) Shutdown message """ def shutdown(): raise web.GracefulExit() log.info("Shutting down lbrynet daemon") asyncio.get_event_loop().call_later(0, shutdown) return "Shutting down" async def jsonrpc_ffmpeg_find(self): """ Get ffmpeg installation information Usage: ffmpeg_find Options: None Returns: (dict) Dictionary of ffmpeg information { 'available': (bool) found ffmpeg, 'which': (str) path to ffmpeg, 'analyze_audio_volume': (bool) should ffmpeg analyze audio } """ return await self._video_file_analyzer.status(reset=True, recheck=True) async def jsonrpc_status(self): """ Get daemon status Usage: status Options: None Returns: (dict) lbrynet-daemon status { 'installation_id': (str) installation id - base58, 'is_running': (bool), 'skipped_components': (list) [names of skipped components (str)], 'startup_status': { Does not include components which have been skipped 'blob_manager': (bool), 'blockchain_headers': (bool), 'database': (bool), 'dht': (bool), 'exchange_rate_manager': (bool), 'hash_announcer': (bool), 'peer_protocol_server': (bool), 'file_manager': (bool), 'libtorrent_component': (bool), 'upnp': (bool), 'wallet': (bool), }, 'connection_status': { 'code': (str) connection status code, 'message': (str) connection status message }, 'blockchain_headers': { 'downloading_headers': (bool), 'download_progress': (float) 0-100.0 }, 'wallet': { 'connected': (str) host and port of the connected spv server, 'blocks': (int) local blockchain height, 'blocks_behind': (int) remote_height - local_height, 'best_blockhash': (str) block hash of most recent block, 'is_encrypted': (bool), 'is_locked': (bool), 'connected_servers': (list) [ { 'host': (str) server hostname, 'port': (int) server port, 'latency': (int) milliseconds } ], }, 'libtorrent_component': { 'running': (bool) libtorrent was detected and started successfully, }, 'dht': { 'node_id': (str) lbry dht node id - hex encoded, 'peers_in_routing_table': (int) the number of peers in the routing table, }, 'blob_manager': { 'finished_blobs': (int) number of finished blobs in the blob manager, 'connections': { 'incoming_bps': { : (int) bytes per second received, }, 'outgoing_bps': { : (int) bytes per second sent, }, 'total_outgoing_mps': (float) megabytes per second sent, 'total_incoming_mps': (float) megabytes per second received, 'max_outgoing_mbs': (float) maximum bandwidth (megabytes per second) sent, since the daemon was started 'max_incoming_mbs': (float) maximum bandwidth (megabytes per second) received, since the daemon was started 'total_sent' : (int) total number of bytes sent since the daemon was started 'total_received' : (int) total number of bytes received since the daemon was started } }, 'hash_announcer': { 'announce_queue_size': (int) number of blobs currently queued to be announced }, 'file_manager': { 'managed_files': (int) count of files in the stream manager, }, 'upnp': { 'aioupnp_version': (str), 'redirects': { : (int) external_port, }, 'gateway': (str) manufacturer and model, 'dht_redirect_set': (bool), 'peer_redirect_set': (bool), 'external_ip': (str) external ip address, } } """ ffmpeg_status = await self._video_file_analyzer.status() running_components = self.component_manager.get_components_status() response = { 'installation_id': self.installation_id, 'is_running': all(running_components.values()), 'skipped_components': self.component_manager.skip_components, 'startup_status': running_components, 'ffmpeg_status': ffmpeg_status } for component in self.component_manager.components: status = await component.get_status() if status: response[component.component_name] = status return response def jsonrpc_version(self): # pylint: disable=no-self-use """ Get lbrynet API server version information Usage: version Options: None Returns: (dict) Dictionary of lbry version information { 'processor': (str) processor type, 'python_version': (str) python version, 'platform': (str) platform string, 'os_release': (str) os release string, 'os_system': (str) os name, 'version': (str) lbrynet version, 'build': (str) "dev" | "qa" | "rc" | "release", } """ return self.platform_info @requires(WALLET_COMPONENT) async def jsonrpc_resolve(self, urls: typing.Union[str, list], wallet_id=None, **kwargs): """ Get the claim that a URL refers to. Usage: resolve ... [--wallet_id=] [--include_purchase_receipt] [--include_is_my_output] [--include_sent_supports] [--include_sent_tips] [--include_received_tips] [--new_sdk_server=] Options: --urls= : (str, list) one or more urls to resolve --wallet_id= : (str) wallet to check for claim purchase receipts --new_sdk_server= : (str) URL of the new SDK server (EXPERIMENTAL) --include_purchase_receipt : (bool) lookup and include a receipt if this wallet has purchased the claim being resolved --include_is_my_output : (bool) lookup and include a boolean indicating if claim being resolved is yours --include_sent_supports : (bool) lookup and sum the total amount of supports you've made to this claim --include_sent_tips : (bool) lookup and sum the total amount of tips you've made to this claim (only makes sense when claim is not yours) --include_received_tips : (bool) lookup and sum the total amount of tips you've received to this claim (only makes sense when claim is yours) Returns: Dictionary of results, keyed by url '': { If a resolution error occurs: 'error': Error message If the url resolves to a channel or a claim in a channel: 'certificate': { 'address': (str) claim address, 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, 'claim_sequence': (int) claim sequence number (or -1 if unknown), 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'confirmations': (int) claim depth, 'timestamp': (int) timestamp of the block that included this claim tx, 'has_signature': (bool) included if decoded_claim 'name': (str) claim name, 'permanent_url': (str) permanent url of the certificate claim, 'supports: (list) list of supports [{'txid': (str) txid, 'nout': (int) nout, 'amount': (float) amount}], 'txid': (str) claim txid, 'nout': (str) claim nout, 'signature_is_valid': (bool), included if has_signature, 'value': ClaimDict if decoded, otherwise hex string } If the url resolves to a channel: 'claims_in_channel': (int) number of claims in the channel, If the url resolves to a claim: 'claim': { 'address': (str) claim address, 'amount': (float) claim amount, 'effective_amount': (float) claim amount including supports, 'claim_id': (str) claim id, 'claim_sequence': (int) claim sequence number (or -1 if unknown), 'decoded_claim': (bool) whether or not the claim value was decoded, 'height': (int) claim height, 'depth': (int) claim depth, 'has_signature': (bool) included if decoded_claim 'name': (str) claim name, 'permanent_url': (str) permanent url of the claim, 'channel_name': (str) channel name if claim is in a channel 'supports: (list) list of supports [{'txid': (str) txid, 'nout': (int) nout, 'amount': (float) amount}] 'txid': (str) claim txid, 'nout': (str) claim nout, 'signature_is_valid': (bool), included if has_signature, 'value': ClaimDict if decoded, otherwise hex string } } """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if isinstance(urls, str): urls = [urls] results = {} valid_urls = set() for url in urls: try: URL.parse(url) valid_urls.add(url) except ValueError: results[url] = {"error": f"{url} is not a valid url"} resolved = await self.resolve(wallet.accounts, list(valid_urls), **kwargs) for resolved_uri in resolved: results[resolved_uri] = resolved[resolved_uri] if resolved[resolved_uri] is not None else \ {"error": f"{resolved_uri} did not resolve to a claim"} return results @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, FILE_MANAGER_COMPONENT) async def jsonrpc_get( self, uri, file_name=None, download_directory=None, timeout=None, save_file=None, wallet_id=None): """ Download stream from a LBRY name. Usage: get [ | --file_name=] [ | --download_directory=] [ | --timeout=] [--save_file=] [--wallet_id=] Options: --uri= : (str) uri of the content to download --file_name= : (str) specified name for the downloaded file, overrides the stream file name --download_directory= : (str) full path to the directory to download into --timeout= : (int) download timeout in number of seconds --save_file= : (bool) save the file to the downloads directory --wallet_id= : (str) wallet to check for claim purchase receipts Returns: {File} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if download_directory and not os.path.isdir(download_directory): return {"error": f"specified download directory \"{download_directory}\" does not exist"} try: stream = await self.file_manager.download_from_uri( uri, self.exchange_rate_manager, timeout, file_name, download_directory, save_file=save_file, wallet=wallet ) if not stream: raise DownloadSDTimeoutError(uri) except Exception as e: # TODO: use error from lbry.error log.warning("Error downloading %s: %s", uri, str(e)) return {"error": str(e)} return stream SETTINGS_DOC = """ Settings management. """ def jsonrpc_settings_get(self): """ Get daemon settings Usage: settings_get Options: None Returns: (dict) Dictionary of daemon settings See ADJUSTABLE_SETTINGS in lbry/conf.py for full list of settings """ return self.conf.settings_dict def jsonrpc_settings_set(self, key, value): """ Set daemon settings Usage: settings_set () () Options: None Returns: (dict) Updated dictionary of daemon settings """ with self.conf.update_config() as c: if value and isinstance(value, str) and value[0] in ('[', '{'): value = json.loads(value) attr: Setting = getattr(type(c), key) cleaned = attr.deserialize(value) setattr(c, key, cleaned) return {key: cleaned} def jsonrpc_settings_clear(self, key): """ Clear daemon settings Usage: settings_clear () Options: None Returns: (dict) Updated dictionary of daemon settings """ with self.conf.update_config() as c: setattr(c, key, NOT_SET) return {key: self.conf.settings_dict[key]} PREFERENCE_DOC = """ Preferences management. """ def jsonrpc_preference_get(self, key=None, wallet_id=None): """ Get preference value for key or all values if not key is passed in. Usage: preference_get [] [--wallet_id=] Options: --key= : (str) key associated with value --wallet_id= : (str) restrict operation to specific wallet Returns: (dict) Dictionary of preference(s) """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if key: if key in wallet.preferences: return {key: wallet.preferences[key]} return return wallet.preferences.to_dict_without_ts() def jsonrpc_preference_set(self, key, value, wallet_id=None): """ Set preferences Usage: preference_set () () [--wallet_id=] Options: --key= : (str) key associated with value --value= : (str) key associated with value --wallet_id= : (str) restrict operation to specific wallet Returns: (dict) Dictionary with key/value of new preference """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if value and isinstance(value, str) and value[0] in ('[', '{'): value = json.loads(value) wallet.preferences[key] = value wallet.save() return {key: value} WALLET_DOC = """ Create, modify and inspect wallets. """ @requires("wallet") def jsonrpc_wallet_list(self, wallet_id=None, page=None, page_size=None): """ List wallets. Usage: wallet_list [--wallet_id=] [--page=] [--page_size=] Options: --wallet_id= : (str) show specific wallet only --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination Returns: {Paginated[Wallet]} """ if wallet_id: return paginate_list([self.wallet_manager.get_wallet_or_error(wallet_id)], 1, 1) return paginate_list(self.wallet_manager.wallets, page, page_size) def jsonrpc_wallet_reconnect(self): """ Reconnects ledger network client, applying new configurations. Usage: wallet_reconnect Options: Returns: None """ return self.wallet_manager.reset() @requires("wallet") async def jsonrpc_wallet_create( self, wallet_id, skip_on_startup=False, create_account=False, single_key=False): """ Create a new wallet. Usage: wallet_create ( | --wallet_id=) [--skip_on_startup] [--create_account] [--single_key] Options: --wallet_id= : (str) wallet file name --skip_on_startup : (bool) don't add wallet to daemon_settings.yml --create_account : (bool) generates the default account --single_key : (bool) used with --create_account, creates single-key account Returns: {Wallet} """ wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id) for wallet in self.wallet_manager.wallets: if wallet.id == wallet_id: raise WalletAlreadyLoadedError(wallet_path) if os.path.exists(wallet_path): raise WalletAlreadyExistsError(wallet_path) wallet = self.wallet_manager.import_wallet(wallet_path) if not wallet.accounts and create_account: account = Account.generate( self.ledger, wallet, address_generator={ 'name': SingleKey.name if single_key else HierarchicalDeterministic.name } ) if self.ledger.network.is_connected: await self.ledger.subscribe_account(account) wallet.save() if not skip_on_startup: with self.conf.update_config() as c: c.wallets += [wallet_id] return wallet @requires("wallet") async def jsonrpc_wallet_export(self, password=None, wallet_id=None): """ Exports encrypted wallet data if password is supplied; otherwise plain JSON. Wallet must be unlocked to perform this operation. Usage: wallet_export [--password=] [--wallet_id=] Options: --password= : (str) password to encrypt outgoing data --wallet_id= : (str) wallet being exported Returns: (str) data: base64-encoded encrypted wallet, or cleartext JSON """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if password is None: return wallet.to_json() return wallet.pack(password).decode() @requires("wallet") async def jsonrpc_wallet_import(self, data, password=None, wallet_id=None, blocking=False): """ Import wallet data and merge accounts and preferences. Data is expected to be JSON if password is not supplied. Wallet must be unlocked to perform this operation. Usage: wallet_import ( | --data=) [ | --password=] [--wallet_id=] [--blocking] Options: --data= : (str) incoming wallet data --password= : (str) password to decrypt incoming data --wallet_id= : (str) wallet being merged into --blocking : (bool) wait until any new accounts have merged Returns: (str) base64-encoded encrypted wallet, or cleartext JSON """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) added_accounts, merged_accounts = wallet.merge(self.wallet_manager, password, data) for new_account in itertools.chain(added_accounts, merged_accounts): await new_account.maybe_migrate_certificates() if added_accounts and self.ledger.network.is_connected: if blocking: await asyncio.wait([ a.ledger.subscribe_account(a) for a in added_accounts ]) else: for new_account in added_accounts: asyncio.create_task(self.ledger.subscribe_account(new_account)) wallet.save() return await self.jsonrpc_wallet_export(password=password, wallet_id=wallet_id) @requires("wallet") async def jsonrpc_wallet_add(self, wallet_id): """ Add existing wallet. Usage: wallet_add ( | --wallet_id=) Options: --wallet_id= : (str) wallet file name Returns: {Wallet} """ wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id) for wallet in self.wallet_manager.wallets: if wallet.id == wallet_id: raise WalletAlreadyLoadedError(wallet_path) if not os.path.exists(wallet_path): raise WalletNotFoundError(wallet_path) wallet = self.wallet_manager.import_wallet(wallet_path) if self.ledger.network.is_connected: for account in wallet.accounts: await self.ledger.subscribe_account(account) return wallet @requires("wallet") async def jsonrpc_wallet_remove(self, wallet_id): """ Remove an existing wallet. Usage: wallet_remove ( | --wallet_id=) Options: --wallet_id= : (str) name of wallet to remove Returns: {Wallet} """ wallet = self.wallet_manager.get_wallet_or_error(wallet_id) self.wallet_manager.wallets.remove(wallet) for account in wallet.accounts: await self.ledger.unsubscribe_account(account) return wallet @requires("wallet") async def jsonrpc_wallet_balance(self, wallet_id=None, confirmations=0): """ Return the balance of a wallet Usage: wallet_balance [--wallet_id=] [--confirmations=] Options: --wallet_id= : (str) balance for specific wallet --confirmations= : (int) Only include transactions with this many confirmed blocks. Returns: (decimal) amount of lbry credits in wallet """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) balance = await self.ledger.get_detailed_balance( accounts=wallet.accounts, confirmations=confirmations ) return dict_values_to_lbc(balance) def jsonrpc_wallet_status(self, wallet_id=None): """ Status of wallet including encryption/lock state. Usage: wallet_status [ | --wallet_id=] Options: --wallet_id= : (str) status of specific wallet Returns: Dictionary of wallet status information. """ if self.wallet_manager is None: return {'is_encrypted': None, 'is_syncing': None, 'is_locked': None} wallet = self.wallet_manager.get_wallet_or_default(wallet_id) return { 'is_encrypted': wallet.is_encrypted, 'is_syncing': len(self.ledger._update_tasks) > 0, 'is_locked': wallet.is_locked } @requires(WALLET_COMPONENT) def jsonrpc_wallet_unlock(self, password, wallet_id=None): """ Unlock an encrypted wallet Usage: wallet_unlock ( | --password=) [--wallet_id=] Options: --password= : (str) password to use for unlocking --wallet_id= : (str) restrict operation to specific wallet Returns: (bool) true if wallet is unlocked, otherwise false """ return self.wallet_manager.get_wallet_or_default(wallet_id).unlock(password) @requires(WALLET_COMPONENT) def jsonrpc_wallet_lock(self, wallet_id=None): """ Lock an unlocked wallet Usage: wallet_lock [--wallet_id=] Options: --wallet_id= : (str) restrict operation to specific wallet Returns: (bool) true if wallet is locked, otherwise false """ return self.wallet_manager.get_wallet_or_default(wallet_id).lock() @requires(WALLET_COMPONENT) def jsonrpc_wallet_decrypt(self, wallet_id=None): """ Decrypt an encrypted wallet, this will remove the wallet password. The wallet must be unlocked to decrypt it Usage: wallet_decrypt [--wallet_id=] Options: --wallet_id= : (str) restrict operation to specific wallet Returns: (bool) true if wallet is decrypted, otherwise false """ return self.wallet_manager.get_wallet_or_default(wallet_id).decrypt() @requires(WALLET_COMPONENT) def jsonrpc_wallet_encrypt(self, new_password, wallet_id=None): """ Encrypt an unencrypted wallet with a password Usage: wallet_encrypt ( | --new_password=) [--wallet_id=] Options: --new_password= : (str) password to encrypt account --wallet_id= : (str) restrict operation to specific wallet Returns: (bool) true if wallet is decrypted, otherwise false """ return self.wallet_manager.get_wallet_or_default(wallet_id).encrypt(new_password) @requires(WALLET_COMPONENT) async def jsonrpc_wallet_send( self, amount, addresses, wallet_id=None, change_account_id=None, funding_account_ids=None, preview=False, blocking=True): """ Send the same number of credits to multiple addresses using all accounts in wallet to fund the transaction and the default account to receive any change. Usage: wallet_send ... [--wallet_id=] [--preview] [--change_account_id=None] [--funding_account_ids=...] [--blocking] Options: --wallet_id= : (str) restrict operation to specific wallet --change_account_id= : (str) account where change will go --funding_account_ids= : (str) accounts to fund the transaction --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until tx has synced Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." account = wallet.get_account_or_default(change_account_id) accounts = wallet.get_accounts_or_all(funding_account_ids) amount = self.get_dewies_or_error("amount", amount) if addresses and not isinstance(addresses, list): addresses = [addresses] outputs = [] for address in addresses: self.valid_address_or_error(address, allow_script_address=True) if self.ledger.is_pubkey_address(address): outputs.append( Output.pay_pubkey_hash( amount, self.ledger.address_to_hash160(address) ) ) elif self.ledger.is_script_address(address): outputs.append( Output.pay_script_hash( amount, self.ledger.address_to_hash160(address) ) ) else: raise ValueError(f"Unsupported address: '{address}'") # TODO: use error from lbry.error tx = await Transaction.create( [], outputs, accounts, account ) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_credits_sent()) else: await self.ledger.release_tx(tx) return tx ACCOUNT_DOC = """ Create, modify and inspect wallet accounts. """ @requires("wallet") async def jsonrpc_account_list( self, account_id=None, wallet_id=None, confirmations=0, include_claims=False, show_seed=False, page=None, page_size=None): """ List details of all of the accounts or a specific account. Usage: account_list [] [--wallet_id=] [--confirmations=] [--include_claims] [--show_seed] [--page=] [--page_size=] Options: --account_id= : (str) If provided only the balance for this account will be given --wallet_id= : (str) accounts in specific wallet --confirmations= : (int) required confirmations (default: 0) --include_claims : (bool) include claims, requires than a LBC account is specified (default: false) --show_seed : (bool) show the seed for the account --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination Returns: {Paginated[Account]} """ kwargs = { 'confirmations': confirmations, 'show_seed': show_seed } wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if account_id: return paginate_list([await wallet.get_account_or_error(account_id).get_details(**kwargs)], 1, 1) else: return paginate_list(await wallet.get_detailed_accounts(**kwargs), page, page_size) @requires("wallet") async def jsonrpc_account_balance(self, account_id=None, wallet_id=None, confirmations=0): """ Return the balance of an account Usage: account_balance [] [
| --address=
] [--wallet_id=] [ | --confirmations=] Options: --account_id= : (str) If provided only the balance for this account will be given. Otherwise default account. --wallet_id= : (str) balance for specific wallet --confirmations= : (int) Only include transactions with this many confirmed blocks. Returns: (decimal) amount of lbry credits in wallet """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = wallet.get_account_or_default(account_id) balance = await account.get_detailed_balance( confirmations=confirmations, read_only=True ) return dict_values_to_lbc(balance) @requires("wallet") async def jsonrpc_account_add( self, account_name, wallet_id=None, single_key=False, seed=None, private_key=None, public_key=None): """ Add a previously created account from a seed, private key or public key (read-only). Specify --single_key for single address or vanity address accounts. Usage: account_add ( | --account_name=) (--seed= | --private_key= | --public_key=) [--single_key] [--wallet_id=] Options: --account_name= : (str) name of the account to add --seed= : (str) seed to generate new account from --private_key= : (str) private key for new account --public_key= : (str) public key for new account --single_key : (bool) create single key account, default is multi-key --wallet_id= : (str) restrict operation to specific wallet Returns: {Account} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = Account.from_dict( self.ledger, wallet, { 'name': account_name, 'seed': seed, 'private_key': private_key, 'public_key': public_key, 'address_generator': { 'name': SingleKey.name if single_key else HierarchicalDeterministic.name } } ) wallet.save() if self.ledger.network.is_connected: await self.ledger.subscribe_account(account) return account @requires("wallet") async def jsonrpc_account_create(self, account_name, single_key=False, wallet_id=None): """ Create a new account. Specify --single_key if you want to use the same address for all transactions (not recommended). Usage: account_create ( | --account_name=) [--single_key] [--wallet_id=] Options: --account_name= : (str) name of the account to create --single_key : (bool) create single key account, default is multi-key --wallet_id= : (str) restrict operation to specific wallet Returns: {Account} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = Account.generate( self.ledger, wallet, account_name, { 'name': SingleKey.name if single_key else HierarchicalDeterministic.name } ) wallet.save() if self.ledger.network.is_connected: await self.ledger.subscribe_account(account) return account @requires("wallet") def jsonrpc_account_remove(self, account_id, wallet_id=None): """ Remove an existing account. Usage: account_remove ( | --account_id=) [--wallet_id=] Options: --account_id= : (str) id of the account to remove --wallet_id= : (str) restrict operation to specific wallet Returns: {Account} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = wallet.get_account_or_error(account_id) wallet.accounts.remove(account) wallet.save() return account @requires("wallet") def jsonrpc_account_set( self, account_id, wallet_id=None, default=False, new_name=None, change_gap=None, change_max_uses=None, receiving_gap=None, receiving_max_uses=None): """ Change various settings on an account. Usage: account_set ( | --account_id=) [--wallet_id=] [--default] [--new_name=] [--change_gap=] [--change_max_uses=] [--receiving_gap=] [--receiving_max_uses=] Options: --account_id= : (str) id of the account to change --wallet_id= : (str) restrict operation to specific wallet --default : (bool) make this account the default --new_name= : (str) new name for the account --receiving_gap= : (int) set the gap for receiving addresses --receiving_max_uses= : (int) set the maximum number of times to use a receiving address --change_gap= : (int) set the gap for change addresses --change_max_uses= : (int) set the maximum number of times to use a change address Returns: {Account} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = wallet.get_account_or_error(account_id) change_made = False if account.receiving.name == HierarchicalDeterministic.name: address_changes = { 'change': {'gap': change_gap, 'maximum_uses_per_address': change_max_uses}, 'receiving': {'gap': receiving_gap, 'maximum_uses_per_address': receiving_max_uses}, } for chain_name, changes in address_changes.items(): chain = getattr(account, chain_name) for attr, value in changes.items(): if value is not None: setattr(chain, attr, value) change_made = True if new_name is not None: account.name = new_name change_made = True if default and wallet.default_account != account: wallet.accounts.remove(account) wallet.accounts.insert(0, account) change_made = True if change_made: account.modified_on = int(time.time()) wallet.save() return account @requires("wallet") def jsonrpc_account_max_address_gap(self, account_id, wallet_id=None): """ Finds ranges of consecutive addresses that are unused and returns the length of the longest such range: for change and receiving address chains. This is useful to figure out ideal values to set for 'receiving_gap' and 'change_gap' account settings. Usage: account_max_address_gap ( | --account_id=) [--wallet_id=] Options: --account_id= : (str) account for which to get max gaps --wallet_id= : (str) restrict operation to specific wallet Returns: (map) maximum gap for change and receiving addresses """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) return wallet.get_account_or_error(account_id).get_max_gap() @requires("wallet") def jsonrpc_account_fund(self, to_account=None, from_account=None, amount='0.0', everything=False, outputs=1, broadcast=False, wallet_id=None): """ Transfer some amount (or --everything) to an account from another account (can be the same account). Amounts are interpreted as LBC. You can also spread the transfer across a number of --outputs (cannot be used together with --everything). Usage: account_fund [ | --to_account=] [ | --from_account=] ( | --amount= | --everything) [ | --outputs=] [--wallet_id=] [--broadcast] Options: --to_account= : (str) send to this account --from_account= : (str) spend from this account --amount= : (decimal) the amount to transfer lbc --everything : (bool) transfer everything (excluding claims), default: false. --outputs= : (int) split payment across many outputs, default: 1. --wallet_id= : (str) limit operation to specific wallet. --broadcast : (bool) actually broadcast the transaction, default: false. Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) to_account = wallet.get_account_or_default(to_account) from_account = wallet.get_account_or_default(from_account) amount = self.get_dewies_or_error('amount', amount) if amount else None if not isinstance(outputs, int): # TODO: use error from lbry.error raise ValueError("--outputs must be an integer.") if everything and outputs > 1: # TODO: use error from lbry.error raise ValueError("Using --everything along with --outputs is not supported.") return from_account.fund( to_account=to_account, amount=amount, everything=everything, outputs=outputs, broadcast=broadcast ) @requires("wallet") async def jsonrpc_account_deposit( self, txid, nout, redeem_script, private_key, to_account=None, wallet_id=None, preview=False, blocking=False ): """ Spend a time locked transaction into your account. Usage: account_deposit [ | --to_account=] [--wallet_id=] [--preview] [--blocking] Options: --txid= : (str) id of the transaction --nout= : (int) output number in the transaction --redeem_script= : (str) redeem script for output --private_key= : (str) private key to sign transaction --to_account= : (str) deposit to this account --wallet_id= : (str) limit operation to specific wallet. --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until tx has synced Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = wallet.get_account_or_default(to_account) other_tx = await self.wallet_manager.get_transaction(txid) tx = await Transaction.spend_time_lock( other_tx.outputs[nout], unhexlify(redeem_script), account ) pk = PrivateKey.from_bytes( account.ledger, Base58.decode_check(private_key)[1:-1] ) await tx.sign([account], {pk.address: pk}) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_credits_sent()) else: await self.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) def jsonrpc_account_send(self, amount, addresses, account_id=None, wallet_id=None, preview=False, blocking=False): """ Send the same number of credits to multiple addresses from a specific account (or default account). Usage: account_send ... [--account_id=] [--wallet_id=] [--preview] [--blocking] Options: --account_id= : (str) account to fund the transaction --wallet_id= : (str) restrict operation to specific wallet --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until tx has synced Returns: {Transaction} """ return self.jsonrpc_wallet_send( amount=amount, addresses=addresses, wallet_id=wallet_id, change_account_id=account_id, funding_account_ids=[account_id] if account_id else [], preview=preview, blocking=blocking ) SYNC_DOC = """ Wallet synchronization. """ @requires("wallet") def jsonrpc_sync_hash(self, wallet_id=None): """ Deterministic hash of the wallet. Usage: sync_hash [ | --wallet_id=] Options: --wallet_id= : (str) wallet for which to generate hash Returns: (str) sha256 hash of wallet """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) return hexlify(wallet.hash).decode() @requires("wallet") async def jsonrpc_sync_apply(self, password, data=None, wallet_id=None, blocking=False): """ Apply incoming synchronization data, if provided, and return a sync hash and update wallet data. Wallet must be unlocked to perform this operation. If "encrypt-on-disk" preference is True and supplied password is different from local password, or there is no local password (because local wallet was not encrypted), then the supplied password will be used for local encryption (overwriting previous local encryption password). Usage: sync_apply [--data=] [--wallet_id=] [--blocking] Options: --password= : (str) password to decrypt incoming and encrypt outgoing data --data= : (str) incoming sync data, if any --wallet_id= : (str) wallet being sync'ed --blocking : (bool) wait until any new accounts have sync'ed Returns: (map) sync hash and data """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) wallet_changed = False if data is not None: added_accounts, merged_accounts = wallet.merge(self.wallet_manager, password, data) for new_account in itertools.chain(added_accounts, merged_accounts): await new_account.maybe_migrate_certificates() if added_accounts and self.ledger.network.is_connected: if blocking: await asyncio.wait([ a.ledger.subscribe_account(a) for a in added_accounts ]) else: for new_account in added_accounts: asyncio.create_task(self.ledger.subscribe_account(new_account)) wallet_changed = True if wallet.preferences.get(ENCRYPT_ON_DISK, False) and password != wallet.encryption_password: wallet.encryption_password = password wallet_changed = True if wallet_changed: wallet.save() encrypted = wallet.pack(password) return { 'hash': self.jsonrpc_sync_hash(wallet_id), 'data': encrypted.decode() } ADDRESS_DOC = """ List, generate and verify addresses. """ @requires(WALLET_COMPONENT) async def jsonrpc_address_is_mine(self, address, account_id=None, wallet_id=None): """ Checks if an address is associated with the current wallet. Usage: address_is_mine (
| --address=
) [ | --account_id=] [--wallet_id=] Options: --address=
: (str) address to check --account_id= : (str) id of the account to use --wallet_id= : (str) restrict operation to specific wallet Returns: (bool) true, if address is associated with current wallet """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = wallet.get_account_or_default(account_id) match = await self.ledger.db.get_address(read_only=True, address=address, accounts=[account]) if match is not None: return True return False @requires(WALLET_COMPONENT) def jsonrpc_address_list(self, address=None, account_id=None, wallet_id=None, page=None, page_size=None): """ List account addresses or details of single address. Usage: address_list [--address=
] [--account_id=] [--wallet_id=] [--page=] [--page_size=] Options: --address=
: (str) just show details for single address --account_id= : (str) id of the account to use --wallet_id= : (str) restrict operation to specific wallet --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination Returns: {Paginated[Address]} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) constraints = { 'cols': ('address', 'account', 'used_times', 'pubkey', 'chain_code', 'n', 'depth') } if address: constraints['address'] = address if account_id: constraints['accounts'] = [wallet.get_account_or_error(account_id)] else: constraints['accounts'] = wallet.accounts return paginate_rows( self.ledger.get_addresses, self.ledger.get_address_count, page, page_size, read_only=True, **constraints ) @requires(WALLET_COMPONENT) def jsonrpc_address_unused(self, account_id=None, wallet_id=None): """ Return an address containing no balance, will create a new address if there is none. Usage: address_unused [--account_id=] [--wallet_id=] Options: --account_id= : (str) id of the account to use --wallet_id= : (str) restrict operation to specific wallet Returns: {Address} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) return wallet.get_account_or_default(account_id).receiving.get_or_create_usable_address() FILE_DOC = """ File management. """ @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_list(self, sort=None, reverse=False, comparison=None, wallet_id=None, page=None, page_size=None, **kwargs): """ List files limited by optional filters Usage: file_list [--sd_hash=] [--file_name=] [--stream_hash=] [--rowid=] [--added_on=] [--claim_id=] [--outpoint=] [--txid=] [--nout=] [--channel_claim_id=] [--channel_name=] [--claim_name=] [--blobs_in_stream=] [--download_path=] [--blobs_remaining=] [--uploading_to_reflector=] [--is_fully_reflected=] [--status=] [--completed=] [--sort=] [--comparison=] [--full_status=] [--reverse] [--page=] [--page_size=] [--wallet_id=] Options: --sd_hash= : (str) get file with matching sd hash --file_name= : (str) get file with matching file name in the downloads folder --stream_hash= : (str) get file with matching stream hash --rowid= : (int) get file with matching row id --added_on= : (int) get file with matching time of insertion --claim_id= : (str) get file with matching claim id(s) --outpoint= : (str) get file with matching claim outpoint(s) --txid= : (str) get file with matching claim txid --nout= : (int) get file with matching claim nout --channel_claim_id= : (str) get file with matching channel claim id(s) --channel_name= : (str) get file with matching channel name --claim_name= : (str) get file with matching claim name --blobs_in_stream= : (int) get file with matching blobs in stream --download_path= : (str) get file with matching download path --uploading_to_reflector= : (bool) get files currently uploading to reflector --is_fully_reflected= : (bool) get files that have been uploaded to reflector --status= : (str) match by status, ( running | finished | stopped ) --completed= : (bool) match only completed --blobs_remaining= : (int) amount of remaining blobs to download --sort= : (str) field to sort by (one of the above filter fields) --comparison= : (str) logical comparison, (eq | ne | g | ge | l | le | in) --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --wallet_id= : (str) add purchase receipts from this wallet Returns: {Paginated[File]} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) sort = sort or 'rowid' comparison = comparison or 'eq' paginated = paginate_list( self.file_manager.get_filtered(sort, reverse, comparison, **kwargs), page, page_size ) if paginated['items']: receipts = { txo.purchased_claim_id: txo for txo in await self.ledger.db.get_purchases( accounts=wallet.accounts, purchased_claim_id__in=[s.claim_id for s in paginated['items']] ) } for stream in paginated['items']: stream.purchase_receipt = receipts.get(stream.claim_id) return paginated @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_set_status(self, status, **kwargs): """ Start or stop downloading a file Usage: file_set_status ( | --status=) [--sd_hash=] [--file_name=] [--stream_hash=] [--rowid=] Options: --status= : (str) one of "start" or "stop" --sd_hash= : (str) set status of file with matching sd hash --file_name= : (str) set status of file with matching file name in the downloads folder --stream_hash= : (str) set status of file with matching stream hash --rowid= : (int) set status of file with matching row id Returns: (str) Confirmation message """ if status not in ['start', 'stop']: # TODO: use error from lbry.error raise Exception('Status must be "start" or "stop".') streams = self.file_manager.get_filtered(**kwargs) if not streams: # TODO: use error from lbry.error raise Exception(f'Unable to find a file for {kwargs}') stream = streams[0] if status == 'start' and not stream.running: if not hasattr(stream, 'bt_infohash') and 'dht' not in self.conf.components_to_skip: stream.downloader.node = self.dht_node await stream.save_file() msg = "Resumed download" elif status == 'stop' and stream.running: await stream.stop() msg = "Stopped download" else: msg = ( "File was already being downloaded" if status == 'start' else "File was already stopped" ) return msg @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_delete(self, delete_from_download_dir=False, delete_all=False, **kwargs): """ Delete a LBRY file Usage: file_delete [--delete_from_download_dir] [--delete_all] [--sd_hash=] [--file_name=] [--stream_hash=] [--rowid=] [--claim_id=] [--txid=] [--nout=] [--claim_name=] [--channel_claim_id=] [--channel_name=] Options: --delete_from_download_dir : (bool) delete file from download directory, instead of just deleting blobs --delete_all : (bool) if there are multiple matching files, allow the deletion of multiple files. Otherwise do not delete anything. --sd_hash= : (str) delete by file sd hash --file_name= : (str) delete by file name in downloads folder --stream_hash= : (str) delete by file stream hash --rowid= : (int) delete by file row id --claim_id= : (str) delete by file claim id --txid= : (str) delete by file claim txid --nout= : (int) delete by file claim nout --claim_name= : (str) delete by file claim name --channel_claim_id= : (str) delete by file channel claim id --channel_name= : (str) delete by file channel claim name Returns: (bool) true if deletion was successful """ streams = self.file_manager.get_filtered(**kwargs) if len(streams) > 1: if not delete_all: log.warning("There are %i files to delete, use narrower filters to select one", len(streams)) return False else: log.warning("Deleting %i files", len(streams)) if not streams: log.warning("There is no file to delete") return False else: for stream in streams: message = f"Deleted file {stream.file_name}" await self.file_manager.delete(stream, delete_file=delete_from_download_dir) log.info(message) result = True return result @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_save(self, file_name=None, download_directory=None, **kwargs): """ Start saving a file to disk. Usage: file_save [--file_name=] [--download_directory=] [--sd_hash=] [--stream_hash=] [--rowid=] [--claim_id=] [--txid=] [--nout=] [--claim_name=] [--channel_claim_id=] [--channel_name=] Options: --file_name= : (str) file name to save to --download_directory= : (str) directory to save into --sd_hash= : (str) save file with matching sd hash --stream_hash= : (str) save file with matching stream hash --rowid= : (int) save file with matching row id --claim_id= : (str) save file with matching claim id --txid= : (str) save file with matching claim txid --nout= : (int) save file with matching claim nout --claim_name= : (str) save file with matching claim name --channel_claim_id= : (str) save file with matching channel claim id --channel_name= : (str) save file with matching channel claim name Returns: {File} """ streams = self.file_manager.get_filtered(**kwargs) if len(streams) > 1: log.warning("There are %i matching files, use narrower filters to select one", len(streams)) return False if not streams: log.warning("There is no file to save") return False stream = streams[0] if not hasattr(stream, 'bt_infohash') and 'dht' not in self.conf.components_to_skip: stream.downloader.node = self.dht_node await stream.save_file(file_name, download_directory) return stream PURCHASE_DOC = """ List and make purchases of claims. """ @requires(WALLET_COMPONENT) def jsonrpc_purchase_list( self, claim_id=None, resolve=False, account_id=None, wallet_id=None, page=None, page_size=None): """ List my claim purchases. Usage: purchase_list [ | --claim_id=] [--resolve] [--account_id=] [--wallet_id=] [--page=] [--page_size=] Options: --claim_id= : (str) purchases for specific claim --resolve : (str) include resolved claim information --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination Returns: {Paginated[Output]} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) constraints = { "wallet": wallet, "accounts": [wallet.get_account_or_error(account_id)] if account_id else wallet.accounts, "resolve": resolve, } if claim_id: constraints["purchased_claim_id"] = claim_id return paginate_rows( self.ledger.get_purchases, self.ledger.get_purchase_count, page, page_size, **constraints ) @requires(WALLET_COMPONENT) async def jsonrpc_purchase_create( self, claim_id=None, url=None, wallet_id=None, funding_account_ids=None, allow_duplicate_purchase=False, override_max_key_fee=False, preview=False, blocking=False): """ Purchase a claim. Usage: purchase_create (--claim_id= | --url=) [--wallet_id=] [--funding_account_ids=...] [--allow_duplicate_purchase] [--override_max_key_fee] [--preview] [--blocking] Options: --claim_id= : (str) claim id of claim to purchase --url= : (str) lookup claim to purchase by url --wallet_id= : (str) restrict operation to specific wallet --funding_account_ids=: (list) ids of accounts to fund this transaction --allow_duplicate_purchase : (bool) allow purchasing claim_id you already own --override_max_key_fee : (bool) ignore max key fee for this purchase --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." accounts = wallet.get_accounts_or_all(funding_account_ids) txo = None if claim_id: txo = await self.ledger.get_claim_by_claim_id(claim_id, accounts, include_purchase_receipt=True) if not isinstance(txo, Output) or not txo.is_claim: # TODO: use error from lbry.error raise Exception(f"Could not find claim with claim_id '{claim_id}'.") elif url: txo = (await self.ledger.resolve(accounts, [url], include_purchase_receipt=True))[url] if not isinstance(txo, Output) or not txo.is_claim: # TODO: use error from lbry.error raise Exception(f"Could not find claim with url '{url}'.") else: # TODO: use error from lbry.error raise Exception("Missing argument claim_id or url.") if not allow_duplicate_purchase and txo.purchase_receipt: raise AlreadyPurchasedError(claim_id) claim = txo.claim if not claim.is_stream or not claim.stream.has_fee: # TODO: use error from lbry.error raise Exception(f"Claim '{claim_id}' does not have a purchase price.") tx = await self.wallet_manager.create_purchase_transaction( accounts, txo, self.exchange_rate_manager, override_max_key_fee ) if not preview: await self.broadcast_or_release(tx, blocking) else: await self.ledger.release_tx(tx) return tx CLAIM_DOC = """ List and search all types of claims. """ @requires(WALLET_COMPONENT) def jsonrpc_claim_list(self, claim_type=None, **kwargs): """ List my stream and channel claims. Usage: claim_list [--claim_type=...] [--claim_id=...] [--name=...] [--is_spent] [--reposted_claim_id=...] [--channel_id=...] [--account_id=] [--wallet_id=] [--has_source | --has_no_source] [--page=] [--page_size=] [--resolve] [--order_by=] [--no_totals] [--include_received_tips] Options: --claim_type= : (str or list) claim type: channel, stream, repost, collection --claim_id= : (str or list) claim id --channel_id= : (str or list) streams in this channel --name= : (str or list) claim name --is_spent : (bool) shows previous claim updates and abandons --reposted_claim_id= : (str or list) reposted claim id --account_id= : (str) id of the account to query --wallet_id= : (str) restrict results to specific wallet --has_source : (bool) list claims containing a source field --has_no_source : (bool) list claims not containing a source field --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata --order_by= : (str) field to order by: 'name', 'height', 'amount' --no_totals : (bool) do not calculate the total number of pages and items in result set (significant performance boost) --include_received_tips : (bool) calculate the amount of tips received for claim outputs Returns: {Paginated[Output]} """ kwargs['type'] = claim_type or CLAIM_TYPE_NAMES if not kwargs.get('is_spent', False): kwargs['is_not_spent'] = True return self.jsonrpc_txo_list(**kwargs) async def jsonrpc_support_sum(self, claim_id, new_sdk_server, include_channel_content=False, **kwargs): """ List total staked supports for a claim, grouped by the channel that signed the support. If claim_id is a channel claim, you can use --include_channel_content to also include supports for content claims in the channel. !!!! NOTE: PAGINATION DOES NOT DO ANYTHING AT THE MOMENT !!!!! Usage: support_sum [--include_channel_content] [--page=] [--page_size=] Options: --claim_id= : (str) claim id --new_sdk_server= : (str) URL of the new SDK server (EXPERIMENTAL) --include_channel_content : (bool) if claim_id is for a channel, include supports for claims in that channel --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination Returns: {Paginated[Dict]} """ page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50) kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size}) support_sums = await self.ledger.sum_supports( new_sdk_server, claim_id=claim_id, include_channel_content=include_channel_content, **kwargs ) return { "items": support_sums, "page": page_num, "page_size": page_size } @requires(WALLET_COMPONENT) async def jsonrpc_claim_search(self, **kwargs): """ Search for stream and channel claims on the blockchain. Arguments marked with "supports equality constraints" allow prepending the value with an equality constraint such as '>', '>=', '<' and '<=' eg. --height=">400000" would limit results to only claims above 400k block height. They also support multiple constraints passed as a list of the args described above. eg. --release_time=[">1000000", "<2000000"] Usage: claim_search [ | --name=] [--text=] [--txid=] [--nout=] [--claim_id= | --claim_ids=...] [--channel= | [[--channel_ids=...] [--not_channel_ids=...]]] [--has_channel_signature] [--valid_channel_signature | --invalid_channel_signature] [--limit_claims_per_channel=] [--is_controlling] [--release_time=] [--public_key_id=] [--timestamp=] [--creation_timestamp=] [--height=] [--creation_height=] [--activation_height=] [--expiration_height=] [--amount=] [--effective_amount=] [--support_amount=] [--trending_group=] [--trending_mixed=] [--trending_local=] [--trending_global=] [--reposted=] [--claim_type=] [--stream_types=...] [--media_types=...] [--fee_currency=] [--fee_amount=] [--duration=] [--any_tags=...] [--all_tags=...] [--not_tags=...] [--any_languages=...] [--all_languages=...] [--not_languages=...] [--any_locations=...] [--all_locations=...] [--not_locations=...] [--order_by=...] [--no_totals] [--page=] [--page_size=] [--wallet_id=] [--include_purchase_receipt] [--include_is_my_output] [--remove_duplicates] [--has_source | --has_no_source] [--sd_hash=] [--new_sdk_server=] Options: --name= : (str) claim name (normalized) --text= : (str) full text search --claim_id= : (str) full or partial claim id --claim_ids= : (list) list of full claim ids --txid= : (str) transaction id --nout= : (str) position in the transaction --channel= : (str) claims signed by this channel (argument is a URL which automatically gets resolved), see --channel_ids if you need to filter by multiple channels at the same time, includes claims with invalid signatures, use in conjunction with --valid_channel_signature --channel_ids= : (list) claims signed by any of these channels (arguments must be claim ids of the channels), includes claims with invalid signatures, implies --has_channel_signature, use in conjunction with --valid_channel_signature --not_channel_ids=: (list) exclude claims signed by any of these channels (arguments must be claim ids of the channels) --has_channel_signature : (bool) claims with a channel signature (valid or invalid) --valid_channel_signature : (bool) claims with a valid channel signature or no signature, use in conjunction with --has_channel_signature to only get claims with valid signatures --invalid_channel_signature : (bool) claims with invalid channel signature or no signature, use in conjunction with --has_channel_signature to only get claims with invalid signatures --limit_claims_per_channel=: (int) only return up to the specified number of claims per channel --is_controlling : (bool) winning claims of their respective name --public_key_id= : (str) only return channels having this public key id, this is the same key as used in the wallet file to map channel certificate private keys: {'public_key_id': 'private key'} --height= : (int) last updated block height (supports equality constraints) --timestamp= : (int) last updated timestamp (supports equality constraints) --creation_height= : (int) created at block height (supports equality constraints) --creation_timestamp=: (int) created at timestamp (supports equality constraints) --activation_height= : (int) height at which claim starts competing for name (supports equality constraints) --expiration_height= : (int) height at which claim will expire (supports equality constraints) --release_time= : (int) limit to claims self-described as having been released to the public on or after this UTC timestamp, when claim does not provide a release time the publish time is used instead (supports equality constraints) --amount= : (int) limit by claim value (supports equality constraints) --support_amount=: (int) limit by supports and tips received (supports equality constraints) --effective_amount=: (int) limit by total value (initial claim value plus all tips and supports received), this amount is blank until claim has reached activation height (supports equality constraints) --trending_score=: (int) limit by trending score (supports equality constraints) --trending_group=: (int) DEPRECATED - instead please use trending_score --trending_mixed=: (int) DEPRECATED - instead please use trending_score --trending_local=: (int) DEPRECATED - instead please use trending_score --trending_global=: (int) DEPRECATED - instead please use trending_score --reposted_claim_id=: (str) all reposts of the specified original claim id --reposted= : (int) claims reposted this many times (supports equality constraints) --claim_type= : (str) filter by 'channel', 'stream', 'repost' or 'collection' --stream_types= : (list) filter by 'video', 'image', 'document', etc --media_types= : (list) filter by 'video/mp4', 'image/png', etc --fee_currency= : (string) specify fee currency: LBC, BTC, USD --fee_amount= : (decimal) content download fee (supports equality constraints) --duration= : (int) duration of video or audio in seconds (supports equality constraints) --any_tags= : (list) find claims containing any of the tags --all_tags= : (list) find claims containing every tag --not_tags= : (list) find claims not containing any of these tags --any_languages= : (list) find claims containing any of the languages --all_languages= : (list) find claims containing every language --not_languages= : (list) find claims not containing any of these languages --any_locations= : (list) find claims containing any of the locations --all_locations= : (list) find claims containing every location --not_locations= : (list) find claims not containing any of these locations --page= : (int) page to return during paginating --page_size= : (int) number of items on page during pagination --order_by= : (list) field to order by, default is descending order, to do an ascending order prepend ^ to the field name, eg. '^amount' available fields: 'name', 'height', 'release_time', 'publish_time', 'amount', 'effective_amount', 'support_amount', 'trending_group', 'trending_mixed', 'trending_local', 'trending_global', 'activation_height' --no_totals : (bool) do not calculate the total number of pages and items in result set (significant performance boost) --wallet_id= : (str) wallet to check for claim purchase receipts --include_purchase_receipt : (bool) lookup and include a receipt if this wallet has purchased the claim --include_is_my_output : (bool) lookup and include a boolean indicating if claim being resolved is yours --remove_duplicates : (bool) removes duplicated content from search by picking either the original claim or the oldest matching repost --has_source : (bool) find claims containing a source field --sd_hash= : (str) find claims where the source stream descriptor hash matches (partially or completely) the given hexadecimal string --has_no_source : (bool) find claims not containing a source field --new_sdk_server= : (str) URL of the new SDK server (EXPERIMENTAL) Returns: {Paginated[Output]} """ if "claim_ids" in kwargs and not kwargs["claim_ids"]: kwargs.pop("claim_ids") if {'claim_id', 'claim_ids'}.issubset(kwargs): raise ConflictingInputValueError('claim_id', 'claim_ids') if kwargs.pop('valid_channel_signature', False): kwargs['signature_valid'] = 1 if kwargs.pop('invalid_channel_signature', False): kwargs['signature_valid'] = 0 if 'has_no_source' in kwargs: kwargs['has_source'] = not kwargs.pop('has_no_source') if 'order_by' in kwargs: # TODO: remove this after removing support for old trending args from the api value = kwargs.pop('order_by') value = value if isinstance(value, list) else [value] new_value = [] for new_v in value: migrated = new_v if new_v not in ( 'trending_mixed', 'trending_local', 'trending_global', 'trending_group' ) else 'trending_score' if migrated not in new_value: new_value.append(migrated) kwargs['order_by'] = new_value page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50) wallet = self.wallet_manager.get_wallet_or_default(kwargs.pop('wallet_id', None)) kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size}) txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **kwargs) result = { "items": txos, "blocked": blocked, "page": page_num, "page_size": page_size } if not kwargs.pop('no_totals', False): result['total_pages'] = int((total + (page_size - 1)) / page_size) result['total_items'] = total return result CHANNEL_DOC = """ Create, update, abandon and list your channel claims. """ @deprecated('channel_create') def jsonrpc_channel_new(self): """ deprecated """ @requires(WALLET_COMPONENT) async def jsonrpc_channel_create( self, name, bid, allow_duplicate_name=False, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False, **kwargs): """ Create a new channel by generating a channel private key and establishing an '@' prefixed claim. Usage: channel_create ( | --name=) ( | --bid=) [--allow_duplicate_name=] [--title=] [--description=<description>] [--email=<email>] [--website_url=<website_url>] [--featured=<featured>...] [--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...] [--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] Options: --name=<name> : (str) name of the channel prefixed with '@' --bid=<bid> : (decimal) amount to back the claim --allow_duplicate_name=<allow_duplicate_name> : (bool) create new channel even if one already exists with given name. default: false. --title=<title> : (str) title of the publication --description=<description> : (str) description of the publication --email=<email> : (str) email of channel owner --website_url=<website_url> : (str) website url --featured=<featured> : (list) claim_ids of featured content in channel --tags=<tags> : (list) content tags --languages=<languages> : (list) languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant` --locations=<locations> : (list) locations of the channel, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... "locations": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: "COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE" making sure to include colon for blank values, for example to provide only the city: ... --locations="::Manchester" with all values set: ... --locations="US:NH:Manchester:03101:42.990605:-71.460989" optionally, you can just pass the "LATITUDE:LONGITUDE": ... --locations="42.990605:-71.460989" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations="{'country': 'US', 'state': 'NH'}" --thumbnail_url=<thumbnail_url>: (str) thumbnail url --cover_url=<cover_url> : (str) url of cover image --account_id=<account_id> : (str) account to use for holding the transaction --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the channel is sent to, if not specified it will be determined automatically from the account --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." account = wallet.get_account_or_default(account_id) funding_accounts = wallet.get_accounts_or_all(funding_account_ids) self.valid_channel_name_or_error(name) amount = self.get_dewies_or_error('bid', bid, positive_value=True) claim_address = await self.get_receiving_address(claim_address, account) existing_channels = await self.ledger.get_channels(accounts=wallet.accounts, claim_name=name) if len(existing_channels) > 0: if not allow_duplicate_name: # TODO: use error from lbry.error raise Exception( f"You already have a channel under the name '{name}'. " f"Use --allow-duplicate-name flag to override." ) claim = Claim() claim.channel.update(**kwargs) tx = await Transaction.claim_create( name, claim, amount, claim_address, funding_accounts, funding_accounts[0] ) txo = tx.outputs[0] txo.set_channel_private_key( await funding_accounts[0].generate_channel_private_key() ) await tx.sign(funding_accounts) if not preview: wallet.save() await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info( tx, txo, claim_address, claim, name )])) self.component_manager.loop.create_task(self.analytics_manager.send_new_channel()) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) async def jsonrpc_channel_update( self, claim_id, bid=None, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, new_signing_key=False, preview=False, blocking=False, replace=False, **kwargs): """ Update an existing channel claim. Usage: channel_update (<claim_id> | --claim_id=<claim_id>) [<bid> | --bid=<bid>] [--title=<title>] [--description=<description>] [--email=<email>] [--website_url=<website_url>] [--featured=<featured>...] [--clear_featured] [--tags=<tags>...] [--clear_tags] [--languages=<languages>...] [--clear_languages] [--locations=<locations>...] [--clear_locations] [--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--new_signing_key] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] [--replace] Options: --claim_id=<claim_id> : (str) claim_id of the channel to update --bid=<bid> : (decimal) amount to back the claim --title=<title> : (str) title of the publication --description=<description> : (str) description of the publication --email=<email> : (str) email of channel owner --website_url=<website_url> : (str) website url --featured=<featured> : (list) claim_ids of featured content in channel --clear_featured : (bool) clear existing featured content (prior to adding new ones) --tags=<tags> : (list) add content tags --clear_tags : (bool) clear existing tags (prior to adding new ones) --languages=<languages> : (list) languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant` --clear_languages : (bool) clear existing languages (prior to adding new ones) --locations=<locations> : (list) locations of the channel, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... "locations": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: "COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE" making sure to include colon for blank values, for example to provide only the city: ... --locations="::Manchester" with all values set: ... --locations="US:NH:Manchester:03101:42.990605:-71.460989" optionally, you can just pass the "LATITUDE:LONGITUDE": ... --locations="42.990605:-71.460989" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations="{'country': 'US', 'state': 'NH'}" --clear_locations : (bool) clear existing locations (prior to adding new ones) --thumbnail_url=<thumbnail_url>: (str) thumbnail url --cover_url=<cover_url> : (str) url of cover image --account_id=<account_id> : (str) account in which to look for channel (default: all) --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the channel is sent --new_signing_key : (bool) generate a new signing key, will invalidate all previous publishes --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool --replace : (bool) instead of modifying specific values on the channel, this will clear all existing values and only save passed in values, useful for form submissions where all values are always set Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." funding_accounts = wallet.get_accounts_or_all(funding_account_ids) if account_id: account = wallet.get_account_or_error(account_id) accounts = [account] else: account = wallet.default_account accounts = wallet.accounts existing_channels = await self.ledger.get_claims( wallet=wallet, accounts=accounts, claim_id=claim_id ) if len(existing_channels) != 1: account_ids = ', '.join(f"'{account.id}'" for account in accounts) # TODO: use error from lbry.error raise Exception( f"Can't find the channel '{claim_id}' in account(s) {account_ids}." ) old_txo = existing_channels[0] if not old_txo.claim.is_channel: # TODO: use error from lbry.error raise Exception( f"A claim with id '{claim_id}' was found but it is not a channel." ) if bid is not None: amount = self.get_dewies_or_error('bid', bid, positive_value=True) else: amount = old_txo.amount if claim_address is not None: self.valid_address_or_error(claim_address) else: claim_address = old_txo.get_address(account.ledger) if replace: claim = Claim() claim.channel.public_key_bytes = old_txo.claim.channel.public_key_bytes else: claim = Claim.from_bytes(old_txo.claim.to_bytes()) claim.channel.update(**kwargs) tx = await Transaction.claim_update( old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0] ) new_txo = tx.outputs[0] if new_signing_key: new_txo.set_channel_private_key( await funding_accounts[0].generate_channel_private_key() ) else: new_txo.private_key = old_txo.private_key new_txo.script.generate() await tx.sign(funding_accounts) if not preview: wallet.save() await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info( tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name )])) self.component_manager.loop.create_task(self.analytics_manager.send_new_channel()) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) async def jsonrpc_channel_sign( self, channel_name=None, channel_id=None, hexdata=None, salt=None, channel_account_id=None, wallet_id=None): """ Signs data using the specified channel signing key. Usage: channel_sign [<channel_name> | --channel_name=<channel_name>] [<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>] [<salt> | --salt=<salt>] [--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>] Options: --channel_name=<channel_name> : (str) name of channel used to sign (or use channel id) --channel_id=<channel_id> : (str) claim id of channel used to sign (or use channel name) --hexdata=<hexdata> : (str) data to sign, encoded as hexadecimal --salt=<salt> : (str) salt to use for signing, default is to use timestamp --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --wallet_id=<wallet_id> : (str) restrict operation to specific wallet Returns: (dict) Signature if successfully made, (None) or an error otherwise { "signature": (str) The signature of the comment, "signing_ts": (str) The timestamp used to sign the comment, } """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." signing_channel = await self.get_channel_or_error( wallet, channel_account_id, channel_id, channel_name, for_signing=True ) if salt is None: salt = str(int(time.time())) signature = signing_channel.sign_data(unhexlify(str(hexdata)), salt) return { 'signature': signature, 'signing_ts': salt, # DEPRECATED 'salt': salt, } @requires(WALLET_COMPONENT) async def jsonrpc_channel_abandon( self, claim_id=None, txid=None, nout=None, account_id=None, wallet_id=None, preview=False, blocking=True): """ Abandon one of my channel claims. Usage: channel_abandon [<claim_id> | --claim_id=<claim_id>] [<txid> | --txid=<txid>] [<nout> | --nout=<nout>] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--preview] [--blocking] Options: --claim_id=<claim_id> : (str) claim_id of the claim to abandon --txid=<txid> : (str) txid of the claim to abandon --nout=<nout> : (int) nout of the claim to abandon --account_id=<account_id> : (str) id of the account to use --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until abandon is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." if account_id: account = wallet.get_account_or_error(account_id) accounts = [account] else: account = wallet.default_account accounts = wallet.accounts if txid is not None and nout is not None: claims = await self.ledger.get_claims( wallet=wallet, accounts=accounts, **{'txo.txid': txid, 'txo.position': nout} ) elif claim_id is not None: claims = await self.ledger.get_claims( wallet=wallet, accounts=accounts, claim_id=claim_id ) else: # TODO: use error from lbry.error raise Exception('Must specify claim_id, or txid and nout') if not claims: # TODO: use error from lbry.error raise Exception('No claim found for the specified claim_id or txid:nout') tx = await Transaction.create( [Input.spend(txo) for txo in claims], [], [account], account ) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('abandon')) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) def jsonrpc_channel_list(self, *args, **kwargs): """ List my channel claims. Usage: channel_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>] [--name=<name>...] [--claim_id=<claim_id>...] [--is_spent] [--page=<page>] [--page_size=<page_size>] [--resolve] [--no_totals] Options: --name=<name> : (str or list) channel name --claim_id=<claim_id> : (str or list) channel id --is_spent : (bool) shows previous channel updates and abandons --account_id=<account_id> : (str) id of the account to use --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination --resolve : (bool) resolves each channel to provide additional metadata --no_totals : (bool) do not calculate the total number of pages and items in result set (significant performance boost) Returns: {Paginated[Output]} """ kwargs['type'] = 'channel' if 'is_spent' not in kwargs or not kwargs['is_spent']: kwargs['is_not_spent'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) async def jsonrpc_channel_export(self, channel_id=None, channel_name=None, account_id=None, wallet_id=None): """ Export channel private key. Usage: channel_export (<channel_id> | --channel_id=<channel_id> | --channel_name=<channel_name>) [--account_id=<account_id>...] [--wallet_id=<wallet_id>] Options: --channel_id=<channel_id> : (str) claim id of channel to export --channel_name=<channel_name> : (str) name of channel to export --account_id=<account_id> : (str) one or more account ids for accounts to look in for channels, defaults to all accounts. --wallet_id=<wallet_id> : (str) restrict operation to specific wallet Returns: (str) serialized channel private key """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) channel = await self.get_channel_or_error(wallet, account_id, channel_id, channel_name, for_signing=True) address = channel.get_address(self.ledger) public_key = await self.ledger.get_public_key_for_address(wallet, address) if not public_key: # TODO: use error from lbry.error raise Exception("Can't find public key for address holding the channel.") export = { 'name': channel.claim_name, 'channel_id': channel.claim_id, 'holding_address': address, 'holding_public_key': public_key.extended_key_string(), 'signing_private_key': channel.private_key.signing_key.to_pem().decode() } return base58.b58encode(json.dumps(export, separators=(',', ':'))) @requires(WALLET_COMPONENT) async def jsonrpc_channel_import(self, channel_data, wallet_id=None): """ Import serialized channel private key (to allow signing new streams to the channel) Usage: channel_import (<channel_data> | --channel_data=<channel_data>) [--wallet_id=<wallet_id>] Options: --channel_data=<channel_data> : (str) serialized channel, as exported by channel export --wallet_id=<wallet_id> : (str) import into specific wallet Returns: (dict) Result dictionary """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) decoded = base58.b58decode(channel_data) data = json.loads(decoded) channel_private_key = PrivateKey.from_pem( self.ledger, data['signing_private_key'] ) # check that the holding_address hasn't changed since the export was made holding_address = data['holding_address'] channels, _, _, _ = await self.ledger.claim_search( wallet.accounts, public_key_id=channel_private_key.address ) if channels and channels[0].get_address(self.ledger) != holding_address: holding_address = channels[0].get_address(self.ledger) account = await self.ledger.get_account_for_address(wallet, holding_address) if account: # Case 1: channel holding address is in one of the accounts we already have # simply add the certificate to existing account pass else: # Case 2: channel holding address hasn't changed and thus is in the bundled read-only account # create a single-address holding account to manage the channel if holding_address == data['holding_address']: account = Account.from_dict(self.ledger, wallet, { 'name': f"Holding Account For Channel {data['name']}", 'public_key': data['holding_public_key'], 'address_generator': {'name': 'single-address'} }) if self.ledger.network.is_connected: await self.ledger.subscribe_account(account) await self.ledger._update_tasks.done.wait() # Case 3: the holding address has changed and we can't create or find an account for it else: # TODO: use error from lbry.error raise Exception( "Channel owning account has changed since the channel was exported and " "it is not an account to which you have access." ) account.add_channel_private_key(channel_private_key) wallet.save() return f"Added channel signing key for {data['name']}." STREAM_DOC = """ Create, update, abandon, list and inspect your stream claims. """ @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_publish(self, name, **kwargs): """ Create or replace a stream claim at a given name (use 'stream create/update' for more control). Usage: publish (<name> | --name=<name>) [--bid=<bid>] [--file_path=<file_path>] [--file_name=<file_name>] [--file_hash=<file_hash>] [--validate_file] [--optimize_file] [--fee_currency=<fee_currency>] [--fee_amount=<fee_amount>] [--fee_address=<fee_address>] [--title=<title>] [--description=<description>] [--author=<author>] [--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...] [--license=<license>] [--license_url=<license_url>] [--thumbnail_url=<thumbnail_url>] [--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>] [--sd_hash=<sd_hash>] [--channel_id=<channel_id> | --channel_name=<channel_name>] [--channel_account_id=<channel_account_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] Options: --name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash)) --bid=<bid> : (decimal) amount to back the claim --file_path=<file_path> : (str) path to file to be associated with name. --file_name=<file_name> : (str) name of file to be associated with stream. --file_hash=<file_hash> : (str) hash of file to be associated with stream. --validate_file : (bool) validate that the video container and encodings match common web browser support or that optimization succeeds if specified. FFmpeg is required --optimize_file : (bool) transcode the video & audio if necessary to ensure common web browser support. FFmpeg is required --fee_currency=<fee_currency> : (string) specify fee currency --fee_amount=<fee_amount> : (decimal) content download fee --fee_address=<fee_address> : (str) address where to send fee payments, will use value from --claim_address if not provided --title=<title> : (str) title of the publication --description=<description> : (str) description of the publication --author=<author> : (str) author of the publication. The usage for this field is not the same as for channels. The author field is used to credit an author who is not the publisher and is not represented by the channel. For example, a pdf file of 'The Odyssey' has an author of 'Homer' but may by published to a channel such as '@classics', or to no channel at all --tags=<tags> : (list) add content tags --languages=<languages> : (list) languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant` --locations=<locations> : (list) locations relevant to the stream, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... "locations": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: "COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE" making sure to include colon for blank values, for example to provide only the city: ... --locations="::Manchester" with all values set: ... --locations="US:NH:Manchester:03101:42.990605:-71.460989" optionally, you can just pass the "LATITUDE:LONGITUDE": ... --locations="42.990605:-71.460989" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations="{'country': 'US', 'state': 'NH'}" --license=<license> : (str) publication license --license_url=<license_url> : (str) publication license url --thumbnail_url=<thumbnail_url>: (str) thumbnail url --release_time=<release_time> : (int) original public release of content, seconds since UNIX epoch --width=<width> : (int) image/video width, automatically calculated from media file --height=<height> : (int) image/video height, automatically calculated from media file --duration=<duration> : (int) audio/video duration in seconds, automatically calculated --sd_hash=<sd_hash> : (str) sd_hash of stream --channel_id=<channel_id> : (str) claim id of the publisher channel --channel_name=<channel_name> : (str) name of publisher channel --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --account_id=<account_id> : (str) account to use for holding the transaction --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the claim is sent to, if not specified it will be determined automatically from the account --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool Returns: {Transaction} """ self.valid_stream_name_or_error(name) wallet = self.wallet_manager.get_wallet_or_default(kwargs.get('wallet_id')) if kwargs.get('account_id'): accounts = [wallet.get_account_or_error(kwargs.get('account_id'))] else: accounts = wallet.accounts claims = await self.ledger.get_claims( wallet=wallet, accounts=accounts, claim_name=name ) if len(claims) == 0: if 'bid' not in kwargs: # TODO: use error from lbry.error raise Exception("'bid' is a required argument for new publishes.") return await self.jsonrpc_stream_create(name, **kwargs) elif len(claims) == 1: assert claims[0].claim.is_stream, f"Claim at name '{name}' is not a stream claim." return await self.jsonrpc_stream_update(claims[0].claim_id, replace=True, **kwargs) # TODO: use error from lbry.error raise Exception( f"There are {len(claims)} claims for '{name}', please use 'stream update' command " f"to update a specific stream claim." ) @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_repost( self, name, bid, claim_id, allow_duplicate_name=False, channel_id=None, channel_name=None, channel_account_id=None, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False, **kwargs): """ Creates a claim that references an existing stream by its claim id. Usage: stream_repost (<name> | --name=<name>) (<bid> | --bid=<bid>) (<claim_id> | --claim_id=<claim_id>) [--allow_duplicate_name=<allow_duplicate_name>] [--title=<title>] [--description=<description>] [--tags=<tags>...] [--channel_id=<channel_id> | --channel_name=<channel_name>] [--channel_account_id=<channel_account_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] Options: --name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash)) --bid=<bid> : (decimal) amount to back the claim --claim_id=<claim_id> : (str) id of the claim being reposted --allow_duplicate_name=<allow_duplicate_name> : (bool) create new claim even if one already exists with given name. default: false. --title=<title> : (str) title of the repost --description=<description> : (str) description of the repost --tags=<tags> : (list) add repost tags --channel_id=<channel_id> : (str) claim id of the publisher channel --channel_name=<channel_name> : (str) name of the publisher channel --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --account_id=<account_id> : (str) account to use for holding the transaction --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the claim is sent to, if not specified it will be determined automatically from the account --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) self.valid_stream_name_or_error(name) account = wallet.get_account_or_default(account_id) funding_accounts = wallet.get_accounts_or_all(funding_account_ids) channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True) amount = self.get_dewies_or_error('bid', bid, positive_value=True) claim_address = await self.get_receiving_address(claim_address, account) claims = await account.get_claims(claim_name=name) if len(claims) > 0: if not allow_duplicate_name: # TODO: use error from lbry.error raise Exception( f"You already have a stream claim published under the name '{name}'. " f"Use --allow-duplicate-name flag to override." ) if not VALID_FULL_CLAIM_ID.fullmatch(claim_id): # TODO: use error from lbry.error raise Exception('Invalid claim id. It is expected to be a 40 characters long hexadecimal string.') claim = Claim() claim.repost.update(**kwargs) claim.repost.reference.claim_id = claim_id tx = await Transaction.claim_create( name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel ) new_txo = tx.outputs[0] if channel: new_txo.sign(channel) await tx.sign(funding_accounts) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish')) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_create( self, name, bid, file_path=None, allow_duplicate_name=False, channel_id=None, channel_name=None, channel_account_id=None, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False, validate_file=False, optimize_file=False, **kwargs): """ Make a new stream claim and announce the associated file to lbrynet. Usage: stream_create (<name> | --name=<name>) (<bid> | --bid=<bid>) [<file_path> | --file_path=<file_path>] [--file_name=<file_name>] [--file_hash=<file_hash>] [--validate_file] [--optimize_file] [--allow_duplicate_name=<allow_duplicate_name>] [--fee_currency=<fee_currency>] [--fee_amount=<fee_amount>] [--fee_address=<fee_address>] [--title=<title>] [--description=<description>] [--author=<author>] [--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...] [--license=<license>] [--license_url=<license_url>] [--thumbnail_url=<thumbnail_url>] [--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>] [--sd_hash=<sd_hash>] [--channel_id=<channel_id> | --channel_name=<channel_name>] [--channel_account_id=<channel_account_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] Options: --name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash)) --bid=<bid> : (decimal) amount to back the claim --file_path=<file_path> : (str) path to file to be associated with name. --file_name=<file_name> : (str) name of file to be associated with stream. --file_hash=<file_hash> : (str) hash of file to be associated with stream. --validate_file : (bool) validate that the video container and encodings match common web browser support or that optimization succeeds if specified. FFmpeg is required --optimize_file : (bool) transcode the video & audio if necessary to ensure common web browser support. FFmpeg is required --allow_duplicate_name=<allow_duplicate_name> : (bool) create new claim even if one already exists with given name. default: false. --fee_currency=<fee_currency> : (string) specify fee currency --fee_amount=<fee_amount> : (decimal) content download fee --fee_address=<fee_address> : (str) address where to send fee payments, will use value from --claim_address if not provided --title=<title> : (str) title of the publication --description=<description> : (str) description of the publication --author=<author> : (str) author of the publication. The usage for this field is not the same as for channels. The author field is used to credit an author who is not the publisher and is not represented by the channel. For example, a pdf file of 'The Odyssey' has an author of 'Homer' but may by published to a channel such as '@classics', or to no channel at all --tags=<tags> : (list) add content tags --languages=<languages> : (list) languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant` --locations=<locations> : (list) locations relevant to the stream, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... "locations": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: "COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE" making sure to include colon for blank values, for example to provide only the city: ... --locations="::Manchester" with all values set: ... --locations="US:NH:Manchester:03101:42.990605:-71.460989" optionally, you can just pass the "LATITUDE:LONGITUDE": ... --locations="42.990605:-71.460989" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations="{'country': 'US', 'state': 'NH'}" --license=<license> : (str) publication license --license_url=<license_url> : (str) publication license url --thumbnail_url=<thumbnail_url>: (str) thumbnail url --release_time=<release_time> : (int) original public release of content, seconds since UNIX epoch --width=<width> : (int) image/video width, automatically calculated from media file --height=<height> : (int) image/video height, automatically calculated from media file --duration=<duration> : (int) audio/video duration in seconds, automatically calculated --sd_hash=<sd_hash> : (str) sd_hash of stream --channel_id=<channel_id> : (str) claim id of the publisher channel --channel_name=<channel_name> : (str) name of the publisher channel --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --account_id=<account_id> : (str) account to use for holding the transaction --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the claim is sent to, if not specified it will be determined automatically from the account --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." self.valid_stream_name_or_error(name) account = wallet.get_account_or_default(account_id) funding_accounts = wallet.get_accounts_or_all(funding_account_ids) channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True) amount = self.get_dewies_or_error('bid', bid, positive_value=True) claim_address = await self.get_receiving_address(claim_address, account) kwargs['fee_address'] = self.get_fee_address(kwargs, claim_address) claims = await account.get_claims(claim_name=name) if len(claims) > 0: if not allow_duplicate_name: # TODO: use error from lbry.error raise Exception( f"You already have a stream claim published under the name '{name}'. " f"Use --allow-duplicate-name flag to override." ) if file_path is not None: file_path, spec = await self._video_file_analyzer.verify_or_repair( validate_file, optimize_file, file_path, ignore_non_video=True ) kwargs.update(spec) claim = Claim() if file_path is not None: claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs) else: claim.stream.update(**kwargs) tx = await Transaction.claim_create( name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel ) new_txo = tx.outputs[0] file_stream = None if not preview and file_path is not None: file_stream = await self.file_manager.create_stream(file_path) claim.stream.source.sd_hash = file_stream.sd_hash new_txo.script.generate() if channel: new_txo.sign(channel) await tx.sign(funding_accounts) if not preview: await self.broadcast_or_release(tx, blocking) async def save_claims(): await self.storage.save_claims([self._old_get_temp_claim_info( tx, new_txo, claim_address, claim, name )]) if file_path is not None: await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id) self.component_manager.loop.create_task(save_claims()) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish')) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_update( self, claim_id, bid=None, file_path=None, channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False, replace=False, validate_file=False, optimize_file=False, **kwargs): """ Update an existing stream claim and if a new file is provided announce it to lbrynet. Usage: stream_update (<claim_id> | --claim_id=<claim_id>) [--bid=<bid>] [--file_path=<file_path>] [--validate_file] [--optimize_file] [--file_name=<file_name>] [--file_size=<file_size>] [--file_hash=<file_hash>] [--fee_currency=<fee_currency>] [--fee_amount=<fee_amount>] [--fee_address=<fee_address>] [--clear_fee] [--title=<title>] [--description=<description>] [--author=<author>] [--tags=<tags>...] [--clear_tags] [--languages=<languages>...] [--clear_languages] [--locations=<locations>...] [--clear_locations] [--license=<license>] [--license_url=<license_url>] [--thumbnail_url=<thumbnail_url>] [--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>] [--sd_hash=<sd_hash>] [--channel_id=<channel_id> | --channel_name=<channel_name> | --clear_channel] [--channel_account_id=<channel_account_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] [--replace] Options: --claim_id=<claim_id> : (str) id of the stream claim to update --bid=<bid> : (decimal) amount to back the claim --file_path=<file_path> : (str) path to file to be associated with name. --validate_file : (bool) validate that the video container and encodings match common web browser support or that optimization succeeds if specified. FFmpeg is required and file_path must be specified. --optimize_file : (bool) transcode the video & audio if necessary to ensure common web browser support. FFmpeg is required and file_path must be specified. --file_name=<file_name> : (str) override file name, defaults to name from file_path. --file_size=<file_size> : (str) override file size, otherwise automatically computed. --file_hash=<file_hash> : (str) override file hash, otherwise automatically computed. --fee_currency=<fee_currency> : (string) specify fee currency --fee_amount=<fee_amount> : (decimal) content download fee --fee_address=<fee_address> : (str) address where to send fee payments, will use value from --claim_address if not provided --clear_fee : (bool) clear previously set fee --title=<title> : (str) title of the publication --description=<description> : (str) description of the publication --author=<author> : (str) author of the publication. The usage for this field is not the same as for channels. The author field is used to credit an author who is not the publisher and is not represented by the channel. For example, a pdf file of 'The Odyssey' has an author of 'Homer' but may by published to a channel such as '@classics', or to no channel at all --tags=<tags> : (list) add content tags --clear_tags : (bool) clear existing tags (prior to adding new ones) --languages=<languages> : (list) languages used by the channel, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant` --clear_languages : (bool) clear existing languages (prior to adding new ones) --locations=<locations> : (list) locations relevant to the stream, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... "locations": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: "COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE" making sure to include colon for blank values, for example to provide only the city: ... --locations="::Manchester" with all values set: ... --locations="US:NH:Manchester:03101:42.990605:-71.460989" optionally, you can just pass the "LATITUDE:LONGITUDE": ... --locations="42.990605:-71.460989" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations="{'country': 'US', 'state': 'NH'}" --clear_locations : (bool) clear existing locations (prior to adding new ones) --license=<license> : (str) publication license --license_url=<license_url> : (str) publication license url --thumbnail_url=<thumbnail_url>: (str) thumbnail url --release_time=<release_time> : (int) original public release of content, seconds since UNIX epoch --width=<width> : (int) image/video width, automatically calculated from media file --height=<height> : (int) image/video height, automatically calculated from media file --duration=<duration> : (int) audio/video duration in seconds, automatically calculated --sd_hash=<sd_hash> : (str) sd_hash of stream --channel_id=<channel_id> : (str) claim id of the publisher channel --channel_name=<channel_name> : (str) name of the publisher channel --clear_channel : (bool) remove channel signature --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --account_id=<account_id> : (str) account in which to look for stream (default: all) --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the claim is sent to, if not specified it will be determined automatically from the account --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool --replace : (bool) instead of modifying specific values on the stream, this will clear all existing values and only save passed in values, useful for form submissions where all values are always set Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." funding_accounts = wallet.get_accounts_or_all(funding_account_ids) if account_id: account = wallet.get_account_or_error(account_id) accounts = [account] else: account = wallet.default_account accounts = wallet.accounts existing_claims = await self.ledger.get_claims( wallet=wallet, accounts=accounts, claim_id=claim_id ) if len(existing_claims) != 1: account_ids = ', '.join(f"'{account.id}'" for account in accounts) raise InputValueError( f"Can't find the stream '{claim_id}' in account(s) {account_ids}." ) old_txo = existing_claims[0] if not old_txo.claim.is_stream and not old_txo.claim.is_repost: # in principle it should work with any type of claim, but its safer to # limit it to ones we know won't be broken. in the future we can expand # this if we have a test case for e.g. channel or support claims raise InputValueError( f"A claim with id '{claim_id}' was found but it is not a stream or repost claim." ) if bid is not None: amount = self.get_dewies_or_error('bid', bid, positive_value=True) else: amount = old_txo.amount if claim_address is not None: self.valid_address_or_error(claim_address) else: claim_address = old_txo.get_address(account.ledger) channel = None if not clear_channel and (channel_id or channel_name): channel = await self.get_channel_or_error( wallet, channel_account_id, channel_id, channel_name, for_signing=True) elif old_txo.claim.is_signed and not clear_channel and not replace: channel = old_txo.channel fee_address = self.get_fee_address(kwargs, claim_address) if fee_address: kwargs['fee_address'] = fee_address file_path, spec = await self._video_file_analyzer.verify_or_repair( validate_file, optimize_file, file_path, ignore_non_video=True ) kwargs.update(spec) if replace: claim = Claim() if old_txo.claim.is_stream: if old_txo.claim.stream.has_source: claim.stream.message.source.CopyFrom( old_txo.claim.stream.message.source ) stream_type = old_txo.claim.stream.stream_type if stream_type: old_stream_type = getattr(old_txo.claim.stream.message, stream_type) new_stream_type = getattr(claim.stream.message, stream_type) new_stream_type.CopyFrom(old_stream_type) else: claim = Claim.from_bytes(old_txo.claim.to_bytes()) if old_txo.claim.is_stream: claim.stream.update(file_path=file_path, **kwargs) elif old_txo.claim.is_repost: claim.repost.update(**kwargs) if clear_channel: claim.clear_signature() tx = await Transaction.claim_update( old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel if not clear_channel else None ) new_txo = tx.outputs[0] stream_hash = None if not preview and old_txo.claim.is_stream: old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash) old_stream = old_stream[0] if old_stream else None if file_path is not None: if old_stream: await self.file_manager.delete(old_stream, delete_file=False) file_stream = await self.file_manager.create_stream(file_path) new_txo.claim.stream.source.sd_hash = file_stream.sd_hash new_txo.script.generate() stream_hash = file_stream.stream_hash elif old_stream: stream_hash = old_stream.stream_hash if channel: new_txo.sign(channel) await tx.sign(funding_accounts) if not preview: await self.broadcast_or_release(tx, blocking) async def save_claims(): await self.storage.save_claims([self._old_get_temp_claim_info( tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name )]) if stream_hash: await self.storage.save_content_claim(stream_hash, new_txo.id) self.component_manager.loop.create_task(save_claims()) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish')) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) async def jsonrpc_stream_abandon( self, claim_id=None, txid=None, nout=None, account_id=None, wallet_id=None, preview=False, blocking=False): """ Abandon one of my stream claims. Usage: stream_abandon [<claim_id> | --claim_id=<claim_id>] [<txid> | --txid=<txid>] [<nout> | --nout=<nout>] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--preview] [--blocking] Options: --claim_id=<claim_id> : (str) claim_id of the claim to abandon --txid=<txid> : (str) txid of the claim to abandon --nout=<nout> : (int) nout of the claim to abandon --account_id=<account_id> : (str) id of the account to use --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until abandon is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." if account_id: account = wallet.get_account_or_error(account_id) accounts = [account] else: account = wallet.default_account accounts = wallet.accounts if txid is not None and nout is not None: claims = await self.ledger.get_claims( wallet=wallet, accounts=accounts, **{'txo.txid': txid, 'txo.position': nout} ) elif claim_id is not None: claims = await self.ledger.get_claims( wallet=wallet, accounts=accounts, claim_id=claim_id ) else: # TODO: use error from lbry.error raise Exception('Must specify claim_id, or txid and nout') if not claims: # TODO: use error from lbry.error raise Exception('No claim found for the specified claim_id or txid:nout') tx = await Transaction.create( [Input.spend(txo) for txo in claims], [], accounts, account ) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('abandon')) else: await self.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) def jsonrpc_stream_list(self, *args, **kwargs): """ List my stream claims. Usage: stream_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>] [--name=<name>...] [--claim_id=<claim_id>...] [--is_spent] [--page=<page>] [--page_size=<page_size>] [--resolve] [--no_totals] Options: --name=<name> : (str or list) stream name --claim_id=<claim_id> : (str or list) stream id --is_spent : (bool) shows previous stream updates and abandons --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination --resolve : (bool) resolves each stream to provide additional metadata --no_totals : (bool) do not calculate the total number of pages and items in result set (significant performance boost) Returns: {Paginated[Output]} """ kwargs['type'] = 'stream' if 'is_spent' not in kwargs: kwargs['is_not_spent'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DHT_COMPONENT, DATABASE_COMPONENT) def jsonrpc_stream_cost_estimate(self, uri): """ Get estimated cost for a lbry stream Usage: stream_cost_estimate (<uri> | --uri=<uri>) Options: --uri=<uri> : (str) uri to use Returns: (float) Estimated cost in lbry credits, returns None if uri is not resolvable """ return self.get_est_cost_from_uri(uri) COLLECTION_DOC = """ Create, update, list, resolve, and abandon collections. """ @requires(WALLET_COMPONENT) async def jsonrpc_collection_create( self, name, bid, claims, allow_duplicate_name=False, channel_id=None, channel_name=None, channel_account_id=None, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False, **kwargs): """ Create a new collection. Usage: collection_create (<name> | --name=<name>) (<bid> | --bid=<bid>) (--claims=<claims>...) [--allow_duplicate_name] [--title=<title>] [--description=<description>] [--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...] [--thumbnail_url=<thumbnail_url>] [--channel_id=<channel_id> | --channel_name=<channel_name>] [--channel_account_id=<channel_account_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] Options: --name=<name> : (str) name of the collection --bid=<bid> : (decimal) amount to back the claim --claims=<claims> : (list) claim ids to be included in the collection --allow_duplicate_name : (bool) create new collection even if one already exists with given name. default: false. --title=<title> : (str) title of the collection --description=<description> : (str) description of the collection --tags=<tags> : (list) content tags --clear_languages : (bool) clear existing languages (prior to adding new ones) --languages=<languages> : (list) languages used by the collection, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant` --locations=<locations> : (list) locations of the collection, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... "locations": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: "COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE" making sure to include colon for blank values, for example to provide only the city: ... --locations="::Manchester" with all values set: ... --locations="US:NH:Manchester:03101:42.990605:-71.460989" optionally, you can just pass the "LATITUDE:LONGITUDE": ... --locations="42.990605:-71.460989" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations="{'country': 'US', 'state': 'NH'}" --thumbnail_url=<thumbnail_url>: (str) thumbnail url --channel_id=<channel_id> : (str) claim id of the publisher channel --channel_name=<channel_name> : (str) name of the publisher channel --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --account_id=<account_id> : (str) account to use for holding the transaction --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the collection is sent to, if not specified it will be determined automatically from the account --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) account = wallet.get_account_or_default(account_id) funding_accounts = wallet.get_accounts_or_all(funding_account_ids) self.valid_collection_name_or_error(name) channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True) amount = self.get_dewies_or_error('bid', bid, positive_value=True) claim_address = await self.get_receiving_address(claim_address, account) existing_collections = await self.ledger.get_collections(accounts=wallet.accounts, claim_name=name) if len(existing_collections) > 0: if not allow_duplicate_name: # TODO: use error from lbry.error raise Exception( f"You already have a collection under the name '{name}'. " f"Use --allow-duplicate-name flag to override." ) claim = Claim() claim.collection.update(claims=claims, **kwargs) tx = await Transaction.claim_create( name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel ) new_txo = tx.outputs[0] if channel: new_txo.sign(channel) await tx.sign(funding_accounts) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish')) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) async def jsonrpc_collection_update( self, claim_id, bid=None, channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False, replace=False, **kwargs): """ Update an existing collection claim. Usage: collection_update (<claim_id> | --claim_id=<claim_id>) [--bid=<bid>] [--claims=<claims>...] [--clear_claims] [--title=<title>] [--description=<description>] [--tags=<tags>...] [--clear_tags] [--languages=<languages>...] [--clear_languages] [--locations=<locations>...] [--clear_locations] [--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>] [--channel_id=<channel_id> | --channel_name=<channel_name>] [--channel_account_id=<channel_account_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...] [--preview] [--blocking] [--replace] Options: --claim_id=<claim_id> : (str) claim_id of the collection to update --bid=<bid> : (decimal) amount to back the claim --claims=<claims> : (list) claim ids --clear_claims : (bool) clear existing claim references (prior to adding new ones) --title=<title> : (str) title of the collection --description=<description> : (str) description of the collection --tags=<tags> : (list) add content tags --clear_tags : (bool) clear existing tags (prior to adding new ones) --languages=<languages> : (list) languages used by the collection, using RFC 5646 format, eg: for English `--languages=en` for Spanish (Spain) `--languages=es-ES` for Spanish (Mexican) `--languages=es-MX` for Chinese (Simplified) `--languages=zh-Hans` for Chinese (Traditional) `--languages=zh-Hant` --clear_languages : (bool) clear existing languages (prior to adding new ones) --locations=<locations> : (list) locations of the collection, consisting of 2 letter `country` code and a `state`, `city` and a postal `code` along with a `latitude` and `longitude`. for JSON RPC: pass a dictionary with aforementioned attributes as keys, eg: ... "locations": [{'country': 'US', 'state': 'NH'}] ... for command line: pass a colon delimited list with values in the following order: "COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE" making sure to include colon for blank values, for example to provide only the city: ... --locations="::Manchester" with all values set: ... --locations="US:NH:Manchester:03101:42.990605:-71.460989" optionally, you can just pass the "LATITUDE:LONGITUDE": ... --locations="42.990605:-71.460989" finally, you can also pass JSON string of dictionary on the command line as you would via JSON RPC ... --locations="{'country': 'US', 'state': 'NH'}" --clear_locations : (bool) clear existing locations (prior to adding new ones) --thumbnail_url=<thumbnail_url>: (str) thumbnail url --channel_id=<channel_id> : (str) claim id of the publisher channel --channel_name=<channel_name> : (str) name of the publisher channel --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --account_id=<account_id> : (str) account in which to look for collection (default: all) --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --claim_address=<claim_address>: (str) address where the collection is sent --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool --replace : (bool) instead of modifying specific values on the collection, this will clear all existing values and only save passed in values, useful for form submissions where all values are always set Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) funding_accounts = wallet.get_accounts_or_all(funding_account_ids) if account_id: account = wallet.get_account_or_error(account_id) accounts = [account] else: account = wallet.default_account accounts = wallet.accounts existing_collections = await self.ledger.get_collections( wallet=wallet, accounts=accounts, claim_id=claim_id ) if len(existing_collections) != 1: account_ids = ', '.join(f"'{account.id}'" for account in accounts) # TODO: use error from lbry.error raise Exception( f"Can't find the collection '{claim_id}' in account(s) {account_ids}." ) old_txo = existing_collections[0] if not old_txo.claim.is_collection: # TODO: use error from lbry.error raise Exception( f"A claim with id '{claim_id}' was found but it is not a collection." ) if bid is not None: amount = self.get_dewies_or_error('bid', bid, positive_value=True) else: amount = old_txo.amount if claim_address is not None: self.valid_address_or_error(claim_address) else: claim_address = old_txo.get_address(account.ledger) channel = None if channel_id or channel_name: channel = await self.get_channel_or_error( wallet, channel_account_id, channel_id, channel_name, for_signing=True) elif old_txo.claim.is_signed and not clear_channel and not replace: channel = old_txo.channel if replace: claim = Claim() claim.collection.update(**kwargs) else: claim = Claim.from_bytes(old_txo.claim.to_bytes()) claim.collection.update(**kwargs) tx = await Transaction.claim_update( old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel ) new_txo = tx.outputs[0] if channel: new_txo.sign(channel) await tx.sign(funding_accounts) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish')) else: await account.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) async def jsonrpc_collection_abandon(self, *args, **kwargs): """ Abandon one of my collection claims. Usage: collection_abandon [<claim_id> | --claim_id=<claim_id>] [<txid> | --txid=<txid>] [<nout> | --nout=<nout>] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--preview] [--blocking] Options: --claim_id=<claim_id> : (str) claim_id of the claim to abandon --txid=<txid> : (str) txid of the claim to abandon --nout=<nout> : (int) nout of the claim to abandon --account_id=<account_id> : (str) id of the account to use --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until abandon is in mempool Returns: {Transaction} """ return await self.jsonrpc_stream_abandon(*args, **kwargs) @requires(WALLET_COMPONENT) def jsonrpc_collection_list( self, resolve_claims=0, resolve=False, account_id=None, wallet_id=None, page=None, page_size=None): """ List my collection claims. Usage: collection_list [--resolve_claims=<resolve_claims>] [--resolve] [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>] Options: --resolve : (bool) resolve collection claim --resolve_claims=<resolve_claims> : (int) resolve every claim --account_id=<account_id> : (str) id of the account to use --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination Returns: {Paginated[Output]} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if account_id: account = wallet.get_account_or_error(account_id) collections = account.get_collections collection_count = account.get_collection_count else: collections = partial(self.ledger.get_collections, wallet=wallet, accounts=wallet.accounts) collection_count = partial(self.ledger.get_collection_count, wallet=wallet, accounts=wallet.accounts) return paginate_rows( collections, collection_count, page, page_size, resolve=resolve, resolve_claims=resolve_claims ) async def jsonrpc_collection_resolve( self, claim_id=None, url=None, wallet_id=None, page=1, page_size=DEFAULT_PAGE_SIZE): """ Resolve claims in the collection. Usage: collection_resolve (--claim_id=<claim_id> | --url=<url>) [--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>] Options: --claim_id=<claim_id> : (str) claim id of the collection --url=<url> : (str) url of the collection --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination Returns: {Paginated[Output]} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if claim_id: txo = await self.ledger.get_claim_by_claim_id(claim_id, wallet.accounts) if not isinstance(txo, Output) or not txo.is_claim: # TODO: use error from lbry.error raise Exception(f"Could not find collection with claim_id '{claim_id}'.") elif url: txo = (await self.ledger.resolve(wallet.accounts, [url]))[url] if not isinstance(txo, Output) or not txo.is_claim: # TODO: use error from lbry.error raise Exception(f"Could not find collection with url '{url}'.") else: # TODO: use error from lbry.error raise Exception("Missing argument claim_id or url.") page_num, page_size = abs(page), min(abs(page_size), 50) items = await self.ledger.resolve_collection(txo, page_size * (page_num - 1), page_size) total_items = len(txo.claim.collection.claims.ids) return { "items": items, "total_pages": int((total_items + (page_size - 1)) / page_size), "total_items": total_items, "page_size": page_size, "page": page } SUPPORT_DOC = """ Create, list and abandon all types of supports. """ @requires(WALLET_COMPONENT) async def jsonrpc_support_create( self, claim_id, amount, tip=False, channel_id=None, channel_name=None, channel_account_id=None, account_id=None, wallet_id=None, funding_account_ids=None, comment=None, preview=False, blocking=False): """ Create a support or a tip for name claim. Usage: support_create (<claim_id> | --claim_id=<claim_id>) (<amount> | --amount=<amount>) [--tip] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--channel_id=<channel_id> | --channel_name=<channel_name>] [--channel_account_id=<channel_account_id>...] [--comment=<comment>] [--preview] [--blocking] [--funding_account_ids=<funding_account_ids>...] Options: --claim_id=<claim_id> : (str) claim_id of the claim to support --amount=<amount> : (decimal) amount of support --tip : (bool) send support to claim owner, default: false. --channel_id=<channel_id> : (str) claim id of the supporters identity channel --channel_name=<channel_name> : (str) name of the supporters identity channel --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in for channel certificates, defaults to all accounts. --account_id=<account_id> : (str) account to use for holding the transaction --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction --comment=<comment> : (str) add a comment to the support --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until transaction is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." funding_accounts = wallet.get_accounts_or_all(funding_account_ids) channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True) amount = self.get_dewies_or_error("amount", amount) claim = await self.ledger.get_claim_by_claim_id(claim_id) claim_address = claim.get_address(self.ledger) if not tip: account = wallet.get_account_or_default(account_id) claim_address = await account.receiving.get_or_create_usable_address() tx = await Transaction.support( claim.claim_name, claim_id, amount, claim_address, funding_accounts, funding_accounts[0], channel, comment=comment ) new_txo = tx.outputs[0] if channel: new_txo.sign(channel) await tx.sign(funding_accounts) if not preview: await self.broadcast_or_release(tx, blocking) await self.storage.save_supports({claim_id: [{ 'txid': tx.id, 'nout': tx.position, 'address': claim_address, 'claim_id': claim_id, 'amount': dewies_to_lbc(new_txo.amount) }]}) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('new_support')) else: await self.ledger.release_tx(tx) return tx @requires(WALLET_COMPONENT) def jsonrpc_support_list(self, *args, received=False, sent=False, staked=False, **kwargs): """ List staked supports and sent/received tips. Usage: support_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>] [--name=<name>...] [--claim_id=<claim_id>...] [--received | --sent | --staked] [--is_spent] [--page=<page>] [--page_size=<page_size>] [--no_totals] Options: --name=<name> : (str or list) claim name --claim_id=<claim_id> : (str or list) claim id --received : (bool) only show received (tips) --sent : (bool) only show sent (tips) --staked : (bool) only show my staked supports --is_spent : (bool) show abandoned supports --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination --no_totals : (bool) do not calculate the total number of pages and items in result set (significant performance boost) Returns: {Paginated[Output]} """ kwargs['type'] = 'support' if 'is_spent' not in kwargs: kwargs['is_not_spent'] = True if received: kwargs['is_not_my_input'] = True kwargs['is_my_output'] = True elif sent: kwargs['is_my_input'] = True kwargs['is_not_my_output'] = True # spent for not my outputs is undetermined kwargs.pop('is_spent', None) kwargs.pop('is_not_spent', None) elif staked: kwargs['is_my_input'] = True kwargs['is_my_output'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) async def jsonrpc_support_abandon( self, claim_id=None, txid=None, nout=None, keep=None, account_id=None, wallet_id=None, preview=False, blocking=False): """ Abandon supports, including tips, of a specific claim, optionally keeping some amount as supports. Usage: support_abandon [--claim_id=<claim_id>] [(--txid=<txid> --nout=<nout>)] [--keep=<keep>] [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--preview] [--blocking] Options: --claim_id=<claim_id> : (str) claim_id of the support to abandon --txid=<txid> : (str) txid of the claim to abandon --nout=<nout> : (int) nout of the claim to abandon --keep=<keep> : (decimal) amount of lbc to keep as support --account_id=<account_id> : (str) id of the account to use --wallet_id=<wallet_id> : (str) restrict operation to specific wallet --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until abandon is in mempool Returns: {Transaction} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first." if account_id: account = wallet.get_account_or_error(account_id) accounts = [account] else: account = wallet.default_account accounts = wallet.accounts if txid is not None and nout is not None: supports = await self.ledger.get_supports( wallet=wallet, accounts=accounts, **{'txo.txid': txid, 'txo.position': nout} ) elif claim_id is not None: supports = await self.ledger.get_supports( wallet=wallet, accounts=accounts, claim_id=claim_id ) else: # TODO: use error from lbry.error raise Exception('Must specify claim_id, or txid and nout') if not supports: # TODO: use error from lbry.error raise Exception('No supports found for the specified claim_id or txid:nout') if keep is not None: keep = self.get_dewies_or_error('keep', keep) else: keep = 0 outputs = [] if keep > 0: outputs = [ Output.pay_support_pubkey_hash( keep, supports[0].claim_name, supports[0].claim_id, supports[0].pubkey_hash ) ] tx = await Transaction.create( [Input.spend(txo) for txo in supports], outputs, accounts, account ) if not preview: await self.broadcast_or_release(tx, blocking) self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('abandon')) else: await self.ledger.release_tx(tx) return tx TRANSACTION_DOC = """ Transaction management. """ @requires(WALLET_COMPONENT) def jsonrpc_transaction_list(self, account_id=None, wallet_id=None, page=None, page_size=None): """ List transactions belonging to wallet Usage: transaction_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>] Options: --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination Returns: (list) List of transactions { "claim_info": (list) claim info if in txn [{ "address": (str) address of claim, "balance_delta": (float) bid amount, "amount": (float) claim amount, "claim_id": (str) claim id, "claim_name": (str) claim name, "nout": (int) nout }], "abandon_info": (list) abandon info if in txn [{ "address": (str) address of abandoned claim, "balance_delta": (float) returned amount, "amount": (float) claim amount, "claim_id": (str) claim id, "claim_name": (str) claim name, "nout": (int) nout }], "confirmations": (int) number of confirmations for the txn, "date": (str) date and time of txn, "fee": (float) txn fee, "support_info": (list) support info if in txn [{ "address": (str) address of support, "balance_delta": (float) support amount, "amount": (float) support amount, "claim_id": (str) claim id, "claim_name": (str) claim name, "is_tip": (bool), "nout": (int) nout }], "timestamp": (int) timestamp, "txid": (str) txn id, "update_info": (list) update info if in txn [{ "address": (str) address of claim, "balance_delta": (float) credited/debited "amount": (float) absolute amount, "claim_id": (str) claim id, "claim_name": (str) claim name, "nout": (int) nout }], "value": (float) value of txn } """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if account_id: account = wallet.get_account_or_error(account_id) transactions = account.get_transaction_history transaction_count = account.get_transaction_history_count else: transactions = partial( self.ledger.get_transaction_history, wallet=wallet, accounts=wallet.accounts) transaction_count = partial( self.ledger.get_transaction_history_count, wallet=wallet, accounts=wallet.accounts) return paginate_rows(transactions, transaction_count, page, page_size, read_only=True) @requires(WALLET_COMPONENT) def jsonrpc_transaction_show(self, txid): """ Get a decoded transaction from a txid Usage: transaction_show (<txid> | --txid=<txid>) Options: --txid=<txid> : (str) txid of the transaction Returns: {Transaction} """ return self.wallet_manager.get_transaction(txid) TXO_DOC = """ List and sum transaction outputs. """ @staticmethod def _constrain_txo_from_kwargs( constraints, type=None, txid=None, # pylint: disable=redefined-builtin claim_id=None, channel_id=None, not_channel_id=None, name=None, reposted_claim_id=None, is_spent=False, is_not_spent=False, has_source=None, has_no_source=None, is_my_input_or_output=None, exclude_internal_transfers=False, is_my_output=None, is_not_my_output=None, is_my_input=None, is_not_my_input=None): if is_spent: constraints['is_spent'] = True elif is_not_spent: constraints['is_spent'] = False if has_source: constraints['has_source'] = True elif has_no_source: constraints['has_source'] = False constraints['exclude_internal_transfers'] = exclude_internal_transfers if is_my_input_or_output is True: constraints['is_my_input_or_output'] = True else: if is_my_input is True: constraints['is_my_input'] = True elif is_not_my_input is True: constraints['is_my_input'] = False if is_my_output is True: constraints['is_my_output'] = True elif is_not_my_output is True: constraints['is_my_output'] = False database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x]) database.constrain_single_or_list(constraints, 'channel_id', channel_id) database.constrain_single_or_list(constraints, 'channel_id', not_channel_id, negate=True) database.constrain_single_or_list(constraints, 'claim_id', claim_id) database.constrain_single_or_list(constraints, 'claim_name', name) database.constrain_single_or_list(constraints, 'txid', txid) database.constrain_single_or_list(constraints, 'reposted_claim_id', reposted_claim_id) return constraints @requires(WALLET_COMPONENT) def jsonrpc_txo_list( self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False, order_by=None, no_totals=False, include_received_tips=False, **kwargs): """ List my transaction outputs. Usage: txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...] [--name=<name>...] [--is_spent | --is_not_spent] [--is_my_input_or_output | [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] ] [--exclude_internal_transfers] [--include_received_tips] [--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>] [--resolve] [--order_by=<order_by>][--no_totals] Options: --type=<type> : (str or list) claim type: stream, channel, support, purchase, collection, repost, other --txid=<txid> : (str or list) transaction id of outputs --claim_id=<claim_id> : (str or list) claim id --channel_id=<channel_id> : (str or list) claims in this channel --not_channel_id=<not_channel_id>: (str or list) claims not in this channel --name=<name> : (str or list) claim name --is_spent : (bool) only show spent txos --is_not_spent : (bool) only show not spent txos --is_my_input_or_output : (bool) txos which have your inputs or your outputs, if using this flag the other related flags are ignored (--is_my_output, --is_my_input, etc) --is_my_output : (bool) show outputs controlled by you --is_not_my_output : (bool) show outputs not controlled by you --is_my_input : (bool) show outputs created by you --is_not_my_input : (bool) show outputs not created by you --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: "--is_my_input --is_my_output --type=other" this allows to exclude "change" payments, this flag can be used in combination with any of the other flags --include_received_tips : (bool) calculate the amount of tips received for claim outputs --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination --resolve : (bool) resolves each claim to provide additional metadata --order_by=<order_by> : (str) field to order by: 'name', 'height', 'amount' and 'none' --no_totals : (bool) do not calculate the total number of pages and items in result set (significant performance boost) Returns: {Paginated[Output]} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if account_id: account = wallet.get_account_or_error(account_id) claims = account.get_txos claim_count = account.get_txo_count else: claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts, read_only=True) claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts, read_only=True) constraints = { 'resolve': resolve, 'include_is_spent': True, 'include_is_my_input': True, 'include_is_my_output': True, 'include_received_tips': include_received_tips } if order_by is not None: if order_by == 'name': constraints['order_by'] = 'txo.claim_name' elif order_by in ('height', 'amount', 'none'): constraints['order_by'] = order_by else: # TODO: use error from lbry.error raise ValueError(f"'{order_by}' is not a valid --order_by value.") self._constrain_txo_from_kwargs(constraints, **kwargs) return paginate_rows(claims, None if no_totals else claim_count, page, page_size, **constraints) @requires(WALLET_COMPONENT) async def jsonrpc_txo_spend( self, account_id=None, wallet_id=None, batch_size=100, include_full_tx=False, preview=False, blocking=False, **kwargs): """ Spend transaction outputs, batching into multiple transactions as necessary. Usage: txo_spend [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...] [--name=<name>...] [--is_my_input | --is_not_my_input] [--exclude_internal_transfers] [--wallet_id=<wallet_id>] [--preview] [--blocking] [--batch_size=<batch_size>] [--include_full_tx] Options: --type=<type> : (str or list) claim type: stream, channel, support, purchase, collection, repost, other --txid=<txid> : (str or list) transaction id of outputs --claim_id=<claim_id> : (str or list) claim id --channel_id=<channel_id> : (str or list) claims in this channel --not_channel_id=<not_channel_id>: (str or list) claims not in this channel --name=<name> : (str or list) claim name --is_my_input : (bool) show outputs created by you --is_not_my_input : (bool) show outputs not created by you --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: "--is_my_input --is_my_output --type=other" this allows to exclude "change" payments, this flag can be used in combination with any of the other flags --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet --preview : (bool) do not broadcast the transaction --blocking : (bool) wait until abandon is in mempool --batch_size=<batch_size> : (int) number of txos to spend per transactions --include_full_tx : (bool) include entire tx in output and not just the txid Returns: {List[Transaction]} """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) accounts = [wallet.get_account_or_error(account_id)] if account_id else wallet.accounts txos = await self.ledger.get_txos( wallet=wallet, accounts=accounts, read_only=True, no_tx=True, no_channel_info=True, **self._constrain_txo_from_kwargs( {}, is_not_spent=True, is_my_output=True, **kwargs ) ) txs = [] while txos: txs.append( await Transaction.create( [Input.spend(txos.pop()) for _ in range(min(len(txos), batch_size))], [], accounts, accounts[0] ) ) if not preview: for tx in txs: await self.broadcast_or_release(tx, blocking) if include_full_tx: return txs return [{'txid': tx.id} for tx in txs] @requires(WALLET_COMPONENT) def jsonrpc_txo_sum(self, account_id=None, wallet_id=None, **kwargs): """ Sum of transaction outputs. Usage: txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...] [--claim_id=<claim_id>...] [--name=<name>...] [--is_spent] [--is_not_spent] [--is_my_input_or_output | [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] ] [--exclude_internal_transfers] [--wallet_id=<wallet_id>] Options: --type=<type> : (str or list) claim type: stream, channel, support, purchase, collection, repost, other --txid=<txid> : (str or list) transaction id of outputs --claim_id=<claim_id> : (str or list) claim id --name=<name> : (str or list) claim name --channel_id=<channel_id> : (str or list) claims in this channel --not_channel_id=<not_channel_id>: (str or list) claims not in this channel --is_spent : (bool) only show spent txos --is_not_spent : (bool) only show not spent txos --is_my_input_or_output : (bool) txos which have your inputs or your outputs, if using this flag the other related flags are ignored (--is_my_output, --is_my_input, etc) --is_my_output : (bool) show outputs controlled by you --is_not_my_output : (bool) show outputs not controlled by you --is_my_input : (bool) show outputs created by you --is_not_my_input : (bool) show outputs not created by you --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: "--is_my_input --is_my_output --type=other" this allows to exclude "change" payments, this flag can be used in combination with any of the other flags --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet Returns: int """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) return self.ledger.get_txo_sum( wallet=wallet, accounts=[wallet.get_account_or_error(account_id)] if account_id else wallet.accounts, read_only=True, **self._constrain_txo_from_kwargs({}, **kwargs) ) @requires(WALLET_COMPONENT) async def jsonrpc_txo_plot( self, account_id=None, wallet_id=None, days_back=0, start_day=None, days_after=None, end_day=None, **kwargs): """ Plot transaction output sum over days. Usage: txo_plot [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...] [--name=<name>...] [--is_spent] [--is_not_spent] [--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...] [--is_my_input_or_output | [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] ] [--exclude_internal_transfers] [--wallet_id=<wallet_id>] [--days_back=<days_back> | [--start_day=<start_day> [--days_after=<days_after> | --end_day=<end_day>]] ] Options: --type=<type> : (str or list) claim type: stream, channel, support, purchase, collection, repost, other --txid=<txid> : (str or list) transaction id of outputs --claim_id=<claim_id> : (str or list) claim id --name=<name> : (str or list) claim name --channel_id=<channel_id> : (str or list) claims in this channel --not_channel_id=<not_channel_id>: (str or list) claims not in this channel --is_spent : (bool) only show spent txos --is_not_spent : (bool) only show not spent txos --is_my_input_or_output : (bool) txos which have your inputs or your outputs, if using this flag the other related flags are ignored (--is_my_output, --is_my_input, etc) --is_my_output : (bool) show outputs controlled by you --is_not_my_output : (bool) show outputs not controlled by you --is_my_input : (bool) show outputs created by you --is_not_my_input : (bool) show outputs not created by you --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: "--is_my_input --is_my_output --type=other" this allows to exclude "change" payments, this flag can be used in combination with any of the other flags --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet --days_back=<days_back> : (int) number of days back from today (not compatible with --start_day, --days_after, --end_day) --start_day=<start_day> : (date) start on specific date (YYYY-MM-DD) (instead of --days_back) --days_after=<days_after> : (int) end number of days after --start_day (instead of --end_day) --end_day=<end_day> : (date) end on specific date (YYYY-MM-DD) (instead of --days_after) Returns: List[Dict] """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) plot = await self.ledger.get_txo_plot( wallet=wallet, accounts=[wallet.get_account_or_error(account_id)] if account_id else wallet.accounts, read_only=True, days_back=days_back, start_day=start_day, days_after=days_after, end_day=end_day, **self._constrain_txo_from_kwargs({}, **kwargs) ) for row in plot: row['total'] = dewies_to_lbc(row['total']) return plot UTXO_DOC = """ Unspent transaction management. """ @requires(WALLET_COMPONENT) def jsonrpc_utxo_list(self, *args, **kwargs): """ List unspent transaction outputs Usage: utxo_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>] Options: --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict results to specific wallet --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination Returns: {Paginated[Output]} """ kwargs['type'] = ['other', 'purchase'] kwargs['is_not_spent'] = True return self.jsonrpc_txo_list(*args, **kwargs) @requires(WALLET_COMPONENT) async def jsonrpc_utxo_release(self, account_id=None, wallet_id=None): """ When spending a UTXO it is locally locked to prevent double spends; occasionally this can result in a UTXO being locked which ultimately did not get spent (failed to broadcast, spend transaction was not accepted by blockchain node, etc). This command releases the lock on all UTXOs in your account. Usage: utxo_release [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>] Options: --account_id=<account_id> : (str) id of the account to query --wallet_id=<wallet_id> : (str) restrict operation to specific wallet Returns: None """ wallet = self.wallet_manager.get_wallet_or_default(wallet_id) if account_id is not None: await wallet.get_account_or_error(account_id).release_all_outputs() else: for account in wallet.accounts: await account.release_all_outputs() BLOB_DOC = """ Blob management. """ @requires(WALLET_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT) async def jsonrpc_blob_get(self, blob_hash, timeout=None, read=False): """ Download and return a blob Usage: blob_get (<blob_hash> | --blob_hash=<blob_hash>) [--timeout=<timeout>] [--read] Options: --blob_hash=<blob_hash> : (str) blob hash of the blob to get --timeout=<timeout> : (int) timeout in number of seconds Returns: (str) Success/Fail message or (dict) decoded data """ blob = await download_blob(asyncio.get_event_loop(), self.conf, self.blob_manager, self.dht_node, blob_hash) if read: with blob.reader_context() as handle: return handle.read().decode() elif isinstance(blob, BlobBuffer): log.warning("manually downloaded blob buffer could have missed garbage collection, clearing it") blob.delete() return "Downloaded blob %s" % blob_hash @requires(BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_blob_delete(self, blob_hash): """ Delete a blob Usage: blob_delete (<blob_hash> | --blob_hash=<blob_hash>) Options: --blob_hash=<blob_hash> : (str) blob hash of the blob to delete Returns: (str) Success/fail message """ if not blob_hash or not is_valid_blobhash(blob_hash): return f"Invalid blob hash to delete '{blob_hash}'" streams = self.file_manager.get_filtered(sd_hash=blob_hash) if streams: await self.file_manager.delete(streams[0]) else: await self.blob_manager.delete_blobs([blob_hash]) return "Deleted %s" % blob_hash PEER_DOC = """ DHT / Blob Exchange peer commands. """ async def jsonrpc_peer_list(self, blob_hash, page=None, page_size=None): """ Get peers for blob hash Usage: peer_list (<blob_hash> | --blob_hash=<blob_hash>) [--page=<page>] [--page_size=<page_size>] Options: --blob_hash=<blob_hash> : (str) find available peers for this blob hash --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination Returns: (list) List of contact dictionaries {'address': <peer ip>, 'udp_port': <dht port>, 'tcp_port': <peer port>, 'node_id': <peer node id>} """ if not is_valid_blobhash(blob_hash): # TODO: use error from lbry.error raise Exception("invalid blob hash") peer_q = asyncio.Queue(loop=self.component_manager.loop) if self.component_manager.has_component(TRACKER_ANNOUNCER_COMPONENT): tracker = self.component_manager.get_component(TRACKER_ANNOUNCER_COMPONENT) tracker_peers = await tracker.get_kademlia_peer_list(bytes.fromhex(blob_hash)) log.info("Found %d peers for %s from trackers.", len(tracker_peers), blob_hash[:8]) peer_q.put_nowait(tracker_peers) elif not self.component_manager.has_component(DHT_COMPONENT): raise Exception("Peer list needs, at least, either a DHT component or a Tracker component for discovery.") peers = [] if self.component_manager.has_component(DHT_COMPONENT): await self.dht_node._peers_for_value_producer(blob_hash, peer_q) while not peer_q.empty(): peers.extend(peer_q.get_nowait()) results = { (peer.address, peer.tcp_port): { "node_id": hexlify(peer.node_id).decode() if peer.node_id else None, "address": peer.address, "udp_port": peer.udp_port, "tcp_port": peer.tcp_port, } for peer in peers } return paginate_list(list(results.values()), page, page_size) @requires(DATABASE_COMPONENT) async def jsonrpc_blob_announce(self, blob_hash=None, stream_hash=None, sd_hash=None): """ Announce blobs to the DHT Usage: blob_announce (<blob_hash> | --blob_hash=<blob_hash> | --stream_hash=<stream_hash> | --sd_hash=<sd_hash>) Options: --blob_hash=<blob_hash> : (str) announce a blob, specified by blob_hash --stream_hash=<stream_hash> : (str) announce all blobs associated with stream_hash --sd_hash=<sd_hash> : (str) announce all blobs associated with sd_hash and the sd_hash itself Returns: (bool) true if successful """ blob_hashes = [] if blob_hash: blob_hashes.append(blob_hash) elif stream_hash or sd_hash: if sd_hash and stream_hash: # TODO: use error from lbry.error raise Exception("either the sd hash or the stream hash should be provided, not both") if sd_hash: stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash) blobs = await self.storage.get_blobs_for_stream(stream_hash, only_completed=True) blob_hashes.extend(blob.blob_hash for blob in blobs if blob.blob_hash is not None) else: # TODO: use error from lbry.error raise Exception('single argument must be specified') await self.storage.should_single_announce_blobs(blob_hashes, immediate=True) return True @requires(BLOB_COMPONENT, WALLET_COMPONENT) async def jsonrpc_blob_list(self, uri=None, stream_hash=None, sd_hash=None, needed=None, finished=None, page=None, page_size=None): """ Returns blob hashes. If not given filters, returns all blobs known by the blob manager Usage: blob_list [--needed] [--finished] [<uri> | --uri=<uri>] [<stream_hash> | --stream_hash=<stream_hash>] [<sd_hash> | --sd_hash=<sd_hash>] [--page=<page>] [--page_size=<page_size>] Options: --needed : (bool) only return needed blobs --finished : (bool) only return finished blobs --uri=<uri> : (str) filter blobs by stream in a uri --stream_hash=<stream_hash> : (str) filter blobs by stream hash --sd_hash=<sd_hash> : (str) filter blobs in a stream by sd hash, ie the hash of the stream descriptor blob for a stream that has been downloaded --page=<page> : (int) page to return during paginating --page_size=<page_size> : (int) number of items on page during pagination Returns: (list) List of blob hashes """ if uri or stream_hash or sd_hash: if uri: metadata = (await self.resolve([], uri))[uri] sd_hash = utils.get_sd_hash(metadata) stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash) elif stream_hash: sd_hash = await self.storage.get_sd_blob_hash_for_stream(stream_hash) elif sd_hash: stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash) sd_hash = await self.storage.get_sd_blob_hash_for_stream(stream_hash) if sd_hash: blobs = [sd_hash] else: blobs = [] if stream_hash: blobs.extend([b.blob_hash for b in (await self.storage.get_blobs_for_stream(stream_hash))[:-1]]) else: blobs = list(self.blob_manager.completed_blob_hashes) if needed: blobs = [blob_hash for blob_hash in blobs if not self.blob_manager.is_blob_verified(blob_hash)] if finished: blobs = [blob_hash for blob_hash in blobs if self.blob_manager.is_blob_verified(blob_hash)] return paginate_list(blobs, page, page_size) @requires(BLOB_COMPONENT) async def jsonrpc_blob_reflect(self, blob_hashes, reflector_server=None): """ Reflects specified blobs Usage: blob_reflect (<blob_hashes>...) [--reflector_server=<reflector_server>] Options: --reflector_server=<reflector_server> : (str) reflector address Returns: (list) reflected blob hashes """ raise NotImplementedError() @requires(BLOB_COMPONENT) async def jsonrpc_blob_reflect_all(self): """ Reflects all saved blobs Usage: blob_reflect_all Options: None Returns: (bool) true if successful """ raise NotImplementedError() @requires(DISK_SPACE_COMPONENT) async def jsonrpc_blob_clean(self): """ Deletes blobs to cleanup disk space Usage: blob_clean Options: None Returns: (bool) true if successful """ return await self.disk_space_manager.clean() @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_reflect(self, **kwargs): """ Reflect all the blobs in a file matching the filter criteria Usage: file_reflect [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>] [--rowid=<rowid>] [--reflector=<reflector>] Options: --sd_hash=<sd_hash> : (str) get file with matching sd hash --file_name=<file_name> : (str) get file with matching file name in the downloads folder --stream_hash=<stream_hash> : (str) get file with matching stream hash --rowid=<rowid> : (int) get file with matching row id --reflector=<reflector> : (str) reflector server, ip address or url by default choose a server from the config Returns: (list) list of blobs reflected """ server, port = kwargs.get('server'), kwargs.get('port') if server and port: port = int(port) else: server, port = random.choice(self.conf.reflector_servers) reflected = await asyncio.gather(*[ self.file_manager.source_managers['stream'].reflect_stream(stream, server, port) for stream in self.file_manager.get_filtered(**kwargs) ]) total = [] for reflected_for_stream in reflected: total.extend(reflected_for_stream) return total @requires(DHT_COMPONENT) async def jsonrpc_peer_ping(self, node_id, address, port): """ Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged, if not provided the peer is located first. Usage: peer_ping (<node_id> | --node_id=<node_id>) (<address> | --address=<address>) (<port> | --port=<port>) Options: None Returns: (str) pong, or {'error': <error message>} if an error is encountered """ peer = None if node_id and address and port: peer = make_kademlia_peer(unhexlify(node_id), address, udp_port=int(port)) try: return await self.dht_node.protocol.get_rpc_peer(peer).ping() except asyncio.TimeoutError: return {'error': 'timeout'} if not peer: return {'error': 'peer not found'} @requires(DHT_COMPONENT) def jsonrpc_routing_table_get(self): """ Get DHT routing information Usage: routing_table_get Options: None Returns: (dict) dictionary containing routing and peer information { "buckets": { <bucket index>: [ { "address": (str) peer address, "udp_port": (int) peer udp port, "tcp_port": (int) peer tcp port, "node_id": (str) peer node id, } ] }, "node_id": (str) the local dht node id "prefix_neighbors_count": (int) the amount of peers sharing the same byte prefix of the local node id } """ result = { 'buckets': {}, 'prefix_neighbors_count': 0 } for i, _ in enumerate(self.dht_node.protocol.routing_table.buckets): result['buckets'][i] = [] for peer in self.dht_node.protocol.routing_table.buckets[i].peers: host = { "address": peer.address, "udp_port": peer.udp_port, "tcp_port": peer.tcp_port, "node_id": hexlify(peer.node_id).decode(), } result['buckets'][i].append(host) result['prefix_neighbors_count'] += 1 if peer.node_id[0] == self.dht_node.protocol.node_id[0] else 0 result['node_id'] = hexlify(self.dht_node.protocol.node_id).decode() return result TRACEMALLOC_DOC = """ Controls and queries tracemalloc memory tracing tools for troubleshooting. """ def jsonrpc_tracemalloc_enable(self): # pylint: disable=no-self-use """ Enable tracemalloc memory tracing Usage: jsonrpc_tracemalloc_enable Options: None Returns: (bool) is it tracing? """ tracemalloc.start() return tracemalloc.is_tracing() def jsonrpc_tracemalloc_disable(self): # pylint: disable=no-self-use """ Disable tracemalloc memory tracing Usage: jsonrpc_tracemalloc_disable Options: None Returns: (bool) is it tracing? """ tracemalloc.stop() return tracemalloc.is_tracing() def jsonrpc_tracemalloc_top(self, items: int = 10): # pylint: disable=no-self-use """ Show most common objects, the place that created them and their size. Usage: jsonrpc_tracemalloc_top [(<items> | --items=<items>)] Options: --items=<items> : (int) maximum items to return, from the most common Returns: (dict) dictionary containing most common objects in memory { "line": (str) filename and line number where it was created, "code": (str) code that created it, "size": (int) size in bytes, for each "memory block", "count" (int) number of memory blocks } """ if not tracemalloc.is_tracing(): # TODO: use error from lbry.error raise Exception("Enable tracemalloc first! See 'tracemalloc set' command.") stats = tracemalloc.take_snapshot().filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), # tracemalloc and linecache here use some memory, but thats not relevant tracemalloc.Filter(False, tracemalloc.__file__), tracemalloc.Filter(False, linecache.__file__), )).statistics('lineno', True) results = [] for stat in stats: frame = stat.traceback[0] filename = os.sep.join(frame.filename.split(os.sep)[-2:]) line = linecache.getline(frame.filename, frame.lineno).strip() results.append({ "line": f"{filename}:{frame.lineno}", "code": line, "size": stat.size, "count": stat.count }) if len(results) == items: break return results async def broadcast_or_release(self, tx, blocking=False): await self.wallet_manager.broadcast_or_release(tx, blocking) def valid_address_or_error(self, address, allow_script_address=False): try: assert self.ledger.is_pubkey_address(address) or ( allow_script_address and self.ledger.is_script_address(address) ) except: # TODO: use error from lbry.error raise Exception(f"'{address}' is not a valid address") @staticmethod def valid_stream_name_or_error(name: str): try: if not name: raise InputStringIsBlankError('Stream name') parsed = URL.parse(name) if parsed.has_channel: # TODO: use error from lbry.error raise Exception( "Stream names cannot start with '@' symbol. This is reserved for channels claims." ) if not parsed.has_stream or parsed.stream.name != name: # TODO: use error from lbry.error raise Exception('Stream name has invalid characters.') except (TypeError, ValueError): # TODO: use error from lbry.error raise Exception("Invalid stream name.") @staticmethod def valid_collection_name_or_error(name: str): try: if not name: # TODO: use error from lbry.error raise Exception('Collection name cannot be blank.') parsed = URL.parse(name) if parsed.has_channel: # TODO: use error from lbry.error raise Exception( "Collection names cannot start with '@' symbol. This is reserved for channels claims." ) if not parsed.has_stream or parsed.stream.name != name: # TODO: use error from lbry.error raise Exception('Collection name has invalid characters.') except (TypeError, ValueError): # TODO: use error from lbry.error raise Exception("Invalid collection name.") @staticmethod def valid_channel_name_or_error(name: str): try: if not name: # TODO: use error from lbry.error raise Exception( "Channel name cannot be blank." ) parsed = URL.parse(name) if not parsed.has_channel: # TODO: use error from lbry.error raise Exception("Channel names must start with '@' symbol.") if parsed.channel.name != name: # TODO: use error from lbry.error raise Exception("Channel name has invalid character") except (TypeError, ValueError): # TODO: use error from lbry.error raise Exception("Invalid channel name.") def get_fee_address(self, kwargs: dict, claim_address: str) -> str: if 'fee_address' in kwargs: self.valid_address_or_error(kwargs['fee_address']) return kwargs['fee_address'] if 'fee_currency' in kwargs or 'fee_amount' in kwargs: return claim_address async def get_receiving_address(self, address: str, account: Optional[Account]) -> str: if address is None and account is not None: return await account.receiving.get_or_create_usable_address() self.valid_address_or_error(address) return address async def get_channel_or_none( self, wallet: Wallet, account_ids: List[str], channel_id: str = None, channel_name: str = None, for_signing: bool = False) -> Output: if channel_id is not None or channel_name is not None: return await self.get_channel_or_error( wallet, account_ids, channel_id, channel_name, for_signing ) async def get_channel_or_error( self, wallet: Wallet, account_ids: List[str], channel_id: str = None, channel_name: str = None, for_signing: bool = False) -> Output: if channel_id: key, value = 'id', channel_id elif channel_name: key, value = 'name', channel_name else: # TODO: use error from lbry.error raise ValueError("Couldn't find channel because a channel_id or channel_name was not provided.") channels = await self.ledger.get_channels( wallet=wallet, accounts=wallet.get_accounts_or_all(account_ids), **{f'claim_{key}': value} ) if len(channels) == 1: if for_signing and not channels[0].has_private_key: # TODO: use error from lbry.error raise PrivateKeyNotFoundError(key, value) return channels[0] elif len(channels) > 1: # TODO: use error from lbry.error raise ValueError( f"Multiple channels found with channel_{key} '{value}', " f"pass a channel_id to narrow it down." ) # TODO: use error from lbry.error raise ValueError(f"Couldn't find channel with channel_{key} '{value}'.") @staticmethod def get_dewies_or_error(argument: str, lbc: str, positive_value=False): try: dewies = lbc_to_dewies(lbc) if positive_value and dewies <= 0: # TODO: use error from lbry.error raise ValueError(f"'{argument}' value must be greater than 0.0") return dewies except ValueError as e: # TODO: use error from lbry.error raise ValueError(f"Invalid value for '{argument}': {e.args[0]}") async def resolve(self, accounts, urls, **kwargs): results = await self.ledger.resolve(accounts, urls, **kwargs) if self.conf.save_resolved_claims and results: try: await self.storage.save_claim_from_output( self.ledger, *(result for result in results.values() if isinstance(result, Output)) ) except DecodeError: pass return results @staticmethod def _old_get_temp_claim_info(tx, txo, address, claim_dict, name): return { "claim_id": txo.claim_id, "name": name, "amount": dewies_to_lbc(txo.amount), "address": address, "txid": tx.id, "nout": txo.position, "value": claim_dict, "height": -1, "claim_sequence": -1, } def loggly_time_string(date): formatted_dt = date.strftime("%Y-%m-%dT%H:%M:%S") milliseconds = str(round(date.microsecond * (10.0 ** -5), 3)) return quote(formatted_dt + milliseconds + "Z") def get_loggly_query_string(installation_id): base_loggly_search_url = "https://lbry.loggly.com/search#" now = utils.now() yesterday = now - utils.timedelta(days=1) params = { 'terms': f'json.installation_id:{installation_id[:SHORT_ID_LEN]}*', 'from': loggly_time_string(yesterday), 'to': loggly_time_string(now) } data = urlencode(params) return base_loggly_search_url + data ================================================ FILE: lbry/extras/daemon/exchange_rate_manager.py ================================================ import json import time import asyncio import logging from statistics import median from decimal import Decimal from typing import Optional, Iterable, Type from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError from lbry.utils import aiohttp_request from lbry.wallet.dewies import lbc_to_dewies log = logging.getLogger(__name__) class ExchangeRate: def __init__(self, market, spot, ts): if not int(time.time()) - ts < 600: raise ValueError('The timestamp is too dated.') if not spot > 0: raise ValueError('Spot must be greater than 0.') self.currency_pair = (market[0:3], market[3:6]) self.spot = spot self.ts = ts def __repr__(self): return f"Currency pair:{self.currency_pair}, spot:{self.spot}, ts:{self.ts}" def as_dict(self): return {'spot': self.spot, 'ts': self.ts} class MarketFeed: name: str = "" market: str = "" url: str = "" params = {} fee = 0 update_interval = 300 request_timeout = 50 def __init__(self): self.rate: Optional[float] = None self.last_check = 0 self._last_response = None self._task: Optional[asyncio.Task] = None self.event = asyncio.Event() @property def has_rate(self): return self.rate is not None @property def is_online(self): return self.last_check+self.update_interval+self.request_timeout > time.time() def get_rate_from_response(self, json_response): raise NotImplementedError() async def get_response(self): async with aiohttp_request( 'get', self.url, params=self.params, timeout=self.request_timeout, headers={"User-Agent": "lbrynet"} ) as response: try: self._last_response = await response.json(content_type=None) except ContentTypeError as e: self._last_response = {} log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message) log.debug(await response.text()) return self._last_response async def get_rate(self): try: data = await self.get_response() rate = self.get_rate_from_response(data) rate = rate / (1.0 - self.fee) log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name) self.rate = ExchangeRate(self.market, rate, int(time.time())) self.last_check = time.time() return self.rate except asyncio.TimeoutError: log.warning("Timed out fetching exchange rate from %s.", self.name) except json.JSONDecodeError as e: msg = e.doc if '<html>' not in e.doc else 'unexpected content type.' log.warning("Could not parse exchange rate response from %s: %s", self.name, msg) log.debug(e.doc) except InvalidExchangeRateResponseError as e: log.warning(str(e)) except ClientConnectionError as e: log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e)) except Exception as e: log.exception("Exchange rate error (%s from %s):", self.market, self.name) finally: self.event.set() async def keep_updated(self): while True: await self.get_rate() await asyncio.sleep(self.update_interval) def start(self): if not self._task: self._task = asyncio.create_task(self.keep_updated()) def stop(self): if self._task and not self._task.done(): self._task.cancel() self._task = None self.event.clear() class BaseBittrexFeed(MarketFeed): name = "Bittrex" market = None url = None fee = 0.0025 def get_rate_from_response(self, json_response): if 'lastTradeRate' not in json_response: raise InvalidExchangeRateResponseError(self.name, 'result not found') return 1.0 / float(json_response['lastTradeRate']) class BittrexBTCFeed(BaseBittrexFeed): market = "BTCLBC" url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker" class BittrexUSDFeed(BaseBittrexFeed): market = "USDLBC" url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker" class BaseCoinExFeed(MarketFeed): name = "CoinEx" market = None url = None def get_rate_from_response(self, json_response): if 'data' not in json_response or \ 'ticker' not in json_response['data'] or \ 'last' not in json_response['data']['ticker']: raise InvalidExchangeRateResponseError(self.name, 'result not found') return 1.0 / float(json_response['data']['ticker']['last']) class CoinExBTCFeed(BaseCoinExFeed): market = "BTCLBC" url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC" class CoinExUSDFeed(BaseCoinExFeed): market = "USDLBC" url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT" class BaseHotbitFeed(MarketFeed): name = "hotbit" market = None url = "https://api.hotbit.io/api/v1/market.last" def get_rate_from_response(self, json_response): if 'result' not in json_response: raise InvalidExchangeRateResponseError(self.name, 'result not found') return 1.0 / float(json_response['result']) class HotbitBTCFeed(BaseHotbitFeed): market = "BTCLBC" params = {"market": "LBC/BTC"} class HotbitUSDFeed(BaseHotbitFeed): market = "USDLBC" params = {"market": "LBC/USDT"} class UPbitBTCFeed(MarketFeed): name = "UPbit" market = "BTCLBC" url = "https://api.upbit.com/v1/ticker" params = {"markets": "BTC-LBC"} def get_rate_from_response(self, json_response): if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]: raise InvalidExchangeRateResponseError(self.name, 'result not found') return 1.0 / float(json_response[0]['trade_price']) FEEDS: Iterable[Type[MarketFeed]] = ( BittrexBTCFeed, BittrexUSDFeed, CoinExBTCFeed, CoinExUSDFeed, # HotbitBTCFeed, # HotbitUSDFeed, # UPbitBTCFeed, ) class ExchangeRateManager: def __init__(self, feeds=FEEDS): self.market_feeds = [Feed() for Feed in feeds] def wait(self): return asyncio.wait( [feed.event.wait() for feed in self.market_feeds], ) def start(self): log.info("Starting exchange rate manager") for feed in self.market_feeds: feed.start() def stop(self): log.info("Stopping exchange rate manager") for source in self.market_feeds: source.stop() def convert_currency(self, from_currency, to_currency, amount): log.debug( "Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, [market.rate for market in self.market_feeds] ) if from_currency == to_currency: return round(amount, 8) rates = [] for market in self.market_feeds: if (market.has_rate and market.is_online and market.rate.currency_pair == (from_currency, to_currency)): rates.append(market.rate.spot) if rates: return round(amount * Decimal(median(rates)), 8) raise CurrencyConversionError( f'Unable to convert {amount} from {from_currency} to {to_currency}') def to_dewies(self, currency, amount) -> int: converted = self.convert_currency(currency, "LBC", amount) return lbc_to_dewies(str(converted)) def fee_dict(self): return {market: market.rate.as_dict() for market in self.market_feeds} ================================================ FILE: lbry/extras/daemon/json_response_encoder.py ================================================ import logging from decimal import Decimal from binascii import hexlify, unhexlify from datetime import datetime from json import JSONEncoder from google.protobuf.message import DecodeError from lbry.schema.claim import Claim from lbry.schema.support import Support from lbry.torrent.torrent_manager import TorrentSource from lbry.wallet import Wallet, Ledger, Account, Transaction, Output from lbry.wallet.bip32 import PublicKey from lbry.wallet.dewies import dewies_to_lbc from lbry.stream.managed_stream import ManagedStream log = logging.getLogger(__name__) def encode_txo_doc(): return { 'txid': "hash of transaction in hex", 'nout': "position in the transaction", 'height': "block where transaction was recorded", 'amount': "value of the txo as a decimal", 'address': "address of who can spend the txo", 'confirmations': "number of confirmed blocks", 'is_change': "payment to change address, only available when it can be determined", 'is_received': "true if txo was sent from external account to this account", 'is_spent': "true if txo is spent", 'is_mine': "payment to one of your accounts, only available when it can be determined", 'type': "one of 'claim', 'support' or 'purchase'", 'name': "when type is 'claim' or 'support', this is the claim name", 'claim_id': "when type is 'claim', 'support' or 'purchase', this is the claim id", 'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'", 'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload", 'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc", 'protobuf': "hex encoded raw protobuf version of 'value' field", 'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL", 'claim': "for purchase outputs only, metadata of purchased claim", 'reposted_claim': "for repost claims only, metadata of claim being reposted", 'signing_channel': "for signed claims only, metadata of signing channel", 'is_channel_signature_valid': "for signed claims only, whether signature is valid", 'purchase_receipt': "metadata for the purchase transaction associated with this claim" } def encode_tx_doc(): return { 'txid': "hash of transaction in hex", 'height': "block where transaction was recorded", 'inputs': [encode_txo_doc()], 'outputs': [encode_txo_doc()], 'total_input': "sum of inputs as a decimal", 'total_output': "sum of outputs, sans fee, as a decimal", 'total_fee': "fee amount", 'hex': "entire transaction encoded in hex", } def encode_account_doc(): return { 'id': 'account_id', 'is_default': 'this account is used by default', 'ledger': 'name of crypto currency and network', 'name': 'optional account name', 'seed': 'human friendly words from which account can be recreated', 'encrypted': 'if account is encrypted', 'private_key': 'extended private key', 'public_key': 'extended public key', 'address_generator': 'settings for generating addresses', 'modified_on': 'date of last modification to account settings' } def encode_wallet_doc(): return { 'id': 'wallet_id', 'name': 'optional wallet name', } def encode_file_doc(): return { 'streaming_url': '(str) url to stream the file using range requests', 'completed': '(bool) true if download is completed', 'file_name': '(str) name of file', 'download_directory': '(str) download directory', 'points_paid': '(float) credit paid to download file', 'stopped': '(bool) true if download is stopped', 'stream_hash': '(str) stream hash of file', 'stream_name': '(str) stream name', 'suggested_file_name': '(str) suggested file name', 'sd_hash': '(str) sd hash of file', 'download_path': '(str) download path of file', 'mime_type': '(str) mime type of file', 'key': '(str) key attached to file', 'total_bytes_lower_bound': '(int) lower bound file size in bytes', 'total_bytes': '(int) file upper bound size in bytes', 'written_bytes': '(int) written size in bytes', 'blobs_completed': '(int) number of fully downloaded blobs', 'blobs_in_stream': '(int) total blobs on stream', 'blobs_remaining': '(int) total blobs remaining to download', 'status': '(str) downloader status', 'claim_id': '(str) None if claim is not found else the claim id', 'txid': '(str) None if claim is not found else the transaction id', 'nout': '(int) None if claim is not found else the transaction output index', 'outpoint': '(str) None if claim is not found else the tx and output', 'metadata': '(dict) None if claim is not found else the claim metadata', 'channel_claim_id': '(str) None if claim is not found or not signed', 'channel_name': '(str) None if claim is not found or not signed', 'claim_name': '(str) None if claim is not found else the claim name', 'reflector_progress': '(int) reflector upload progress, 0 to 100', 'uploading_to_reflector': '(bool) set to True when currently uploading to reflector' } class JSONResponseEncoder(JSONEncoder): def __init__(self, *args, ledger: Ledger, include_protobuf=False, **kwargs): super().__init__(*args, **kwargs) self.ledger = ledger self.include_protobuf = include_protobuf def default(self, obj): # pylint: disable=method-hidden,arguments-renamed,too-many-return-statements if isinstance(obj, Account): return self.encode_account(obj) if isinstance(obj, Wallet): return self.encode_wallet(obj) if isinstance(obj, (ManagedStream, TorrentSource)): return self.encode_file(obj) if isinstance(obj, Transaction): return self.encode_transaction(obj) if isinstance(obj, Output): return self.encode_output(obj) if isinstance(obj, Claim): return self.encode_claim(obj) if isinstance(obj, Support): return obj.to_dict() if isinstance(obj, PublicKey): return obj.extended_key_string() if isinstance(obj, datetime): return obj.strftime("%Y%m%dT%H:%M:%S") if isinstance(obj, Decimal): return float(obj) if isinstance(obj, bytes): return obj.decode() return super().default(obj) def encode_transaction(self, tx): return { 'txid': tx.id, 'height': tx.height, 'inputs': [self.encode_input(txo) for txo in tx.inputs], 'outputs': [self.encode_output(txo) for txo in tx.outputs], 'total_input': dewies_to_lbc(tx.input_sum), 'total_output': dewies_to_lbc(tx.input_sum - tx.fee), 'total_fee': dewies_to_lbc(tx.fee), 'hex': hexlify(tx.raw).decode(), } def encode_output(self, txo, check_signature=True): if not txo: return tx_height = txo.tx_ref.height best_height = self.ledger.headers.height output = { 'txid': txo.tx_ref.id, 'nout': txo.position, 'height': tx_height, 'amount': dewies_to_lbc(txo.amount), 'address': txo.get_address(self.ledger) if txo.has_address else None, 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, 'timestamp': self.ledger.headers.estimated_timestamp(tx_height) } if txo.is_spent is not None: output['is_spent'] = txo.is_spent if txo.is_my_output is not None: output['is_my_output'] = txo.is_my_output if txo.is_my_input is not None: output['is_my_input'] = txo.is_my_input if txo.sent_supports is not None: output['sent_supports'] = dewies_to_lbc(txo.sent_supports) if txo.sent_tips is not None: output['sent_tips'] = dewies_to_lbc(txo.sent_tips) if txo.received_tips is not None: output['received_tips'] = dewies_to_lbc(txo.received_tips) if txo.is_internal_transfer is not None: output['is_internal_transfer'] = txo.is_internal_transfer if txo.script.is_claim_name: output['type'] = 'claim' output['claim_op'] = 'create' elif txo.script.is_update_claim: output['type'] = 'claim' output['claim_op'] = 'update' elif txo.script.is_support_claim: output['type'] = 'support' elif txo.script.is_return_data: output['type'] = 'data' elif txo.purchase is not None: output['type'] = 'purchase' output['claim_id'] = txo.purchased_claim_id if txo.purchased_claim is not None: output['claim'] = self.encode_output(txo.purchased_claim) else: output['type'] = 'payment' if txo.script.is_claim_involved: output.update({ 'name': txo.claim_name, 'normalized_name': txo.normalized_name, 'claim_id': txo.claim_id, 'permanent_url': txo.permanent_url, 'meta': self.encode_claim_meta(txo.meta.copy()) }) if 'short_url' in output['meta']: output['short_url'] = output['meta'].pop('short_url') if 'canonical_url' in output['meta']: output['canonical_url'] = output['meta'].pop('canonical_url') if txo.claims is not None: output['claims'] = [self.encode_output(o) for o in txo.claims] if txo.reposted_claim is not None: output['reposted_claim'] = self.encode_output(txo.reposted_claim) if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data: try: output['value'] = txo.signable if self.include_protobuf: output['protobuf'] = hexlify(txo.signable.to_bytes()) if txo.purchase_receipt is not None: output['purchase_receipt'] = self.encode_output(txo.purchase_receipt) if txo.script.is_claim_name or txo.script.is_update_claim: output['value_type'] = txo.claim.claim_type if txo.claim.is_channel: output['has_signing_key'] = txo.has_private_key if check_signature and txo.signable.is_signed: if txo.channel is not None: output['signing_channel'] = self.encode_output(txo.channel) output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger) else: output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id} output['is_channel_signature_valid'] = False except DecodeError: pass return output def encode_claim_meta(self, meta): for key, value in meta.items(): if key.endswith('_amount'): if isinstance(value, int): meta[key] = dewies_to_lbc(value) if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height: meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height']) return meta def encode_input(self, txi): return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else { 'txid': txi.txo_ref.tx_ref.id, 'nout': txi.txo_ref.position } def encode_account(self, account): result = account.to_dict() result['id'] = account.id result.pop('certificates', None) result['is_default'] = self.ledger.accounts[0] == account return result @staticmethod def encode_wallet(wallet): return { 'id': wallet.id, 'name': wallet.name } def encode_file(self, managed_stream): output_exists = managed_stream.output_file_exists tx_height = managed_stream.stream_claim_info.height best_height = self.ledger.headers.height is_stream = hasattr(managed_stream, 'stream_hash') if is_stream: total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length() total_bytes = managed_stream.descriptor.upper_bound_decrypted_length() else: total_bytes_lower_bound = total_bytes = managed_stream.torrent_length result = { 'streaming_url': None, 'completed': managed_stream.completed, 'file_name': None, 'download_directory': None, 'download_path': None, 'points_paid': 0.0, 'stopped': not managed_stream.running, 'stream_hash': None, 'stream_name': None, 'suggested_file_name': None, 'sd_hash': None, 'mime_type': None, 'key': None, 'total_bytes_lower_bound': total_bytes_lower_bound, 'total_bytes': total_bytes, 'written_bytes': managed_stream.written_bytes, 'blobs_completed': None, 'blobs_in_stream': None, 'blobs_remaining': None, 'status': managed_stream.status, 'claim_id': managed_stream.claim_id, 'txid': managed_stream.txid, 'nout': managed_stream.nout, 'outpoint': managed_stream.outpoint, 'metadata': managed_stream.metadata, 'protobuf': managed_stream.metadata_protobuf, 'channel_claim_id': managed_stream.channel_claim_id, 'channel_name': managed_stream.channel_name, 'claim_name': managed_stream.claim_name, 'content_fee': managed_stream.content_fee, 'purchase_receipt': self.encode_output(managed_stream.purchase_receipt), 'added_on': managed_stream.added_on, 'height': tx_height, 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, 'timestamp': self.ledger.headers.estimated_timestamp(tx_height), 'is_fully_reflected': False, 'reflector_progress': False, 'uploading_to_reflector': False } if is_stream: result.update({ 'streaming_url': managed_stream.stream_url, 'stream_hash': managed_stream.stream_hash, 'stream_name': managed_stream.stream_name, 'suggested_file_name': managed_stream.suggested_file_name, 'sd_hash': managed_stream.descriptor.sd_hash, 'mime_type': managed_stream.mime_type, 'key': managed_stream.descriptor.key, 'blobs_completed': managed_stream.blobs_completed, 'blobs_in_stream': managed_stream.blobs_in_stream, 'blobs_remaining': managed_stream.blobs_remaining, 'is_fully_reflected': managed_stream.is_fully_reflected, 'reflector_progress': managed_stream.reflector_progress, 'uploading_to_reflector': managed_stream.uploading_to_reflector }) else: result.update({ 'streaming_url': f'file://{managed_stream.full_path}', }) if output_exists: result.update({ 'file_name': managed_stream.file_name, 'download_directory': managed_stream.download_directory, 'download_path': managed_stream.full_path, }) return result def encode_claim(self, claim): encoded = getattr(claim, claim.claim_type).to_dict() if 'public_key' in encoded: encoded['public_key_id'] = self.ledger.public_key_to_address( unhexlify(encoded['public_key']) ) return encoded ================================================ FILE: lbry/extras/daemon/migrator/__init__.py ================================================ ================================================ FILE: lbry/extras/daemon/migrator/dbmigrator.py ================================================ # pylint: skip-file import os import sys import logging log = logging.getLogger(__name__) def migrate_db(conf, start, end): current = start while current < end: if current == 1: from .migrate1to2 import do_migration elif current == 2: from .migrate2to3 import do_migration elif current == 3: from .migrate3to4 import do_migration elif current == 4: from .migrate4to5 import do_migration elif current == 5: from .migrate5to6 import do_migration elif current == 6: from .migrate6to7 import do_migration elif current == 7: from .migrate7to8 import do_migration elif current == 8: from .migrate8to9 import do_migration elif current == 9: from .migrate9to10 import do_migration elif current == 10: from .migrate10to11 import do_migration elif current == 11: from .migrate11to12 import do_migration elif current == 12: from .migrate12to13 import do_migration elif current == 13: from .migrate13to14 import do_migration elif current == 14: from .migrate14to15 import do_migration elif current == 15: from .migrate15to16 import do_migration else: raise Exception(f"DB migration of version {current} to {current+1} is not available") try: do_migration(conf) except Exception: log.exception("failed to migrate database") if os.path.exists(os.path.join(conf.data_dir, "lbrynet.sqlite")): backup_name = f"rev_{current}_unmigrated_database" count = 0 while os.path.exists(os.path.join(conf.data_dir, backup_name + ".sqlite")): count += 1 backup_name = f"rev_{current}_unmigrated_database_{count}" backup_path = os.path.join(conf.data_dir, backup_name + ".sqlite") os.rename(os.path.join(conf.data_dir, "lbrynet.sqlite"), backup_path) log.info("made a backup of the unmigrated database: %s", backup_path) if os.path.isfile(os.path.join(conf.data_dir, "db_revision")): os.remove(os.path.join(conf.data_dir, "db_revision")) return None current += 1 log.info("successfully migrated the database from revision %i to %i", current - 1, current) return None def run_migration_script(): log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s" logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log") sys.stdout = open("migrator.out.log", 'w') sys.stderr = open("migrator.err.log", 'w') migrate_db(sys.argv[1], int(sys.argv[2]), int(sys.argv[3])) if __name__ == "__main__": run_migration_script() ================================================ FILE: lbry/extras/daemon/migrator/migrate10to11.py ================================================ import sqlite3 import os import binascii def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() current_columns = [] for col_info in cursor.execute("pragma table_info('file');").fetchall(): current_columns.append(col_info[1]) if 'content_fee' in current_columns or 'saved_file' in current_columns: connection.close() print("already migrated") return cursor.execute( "pragma foreign_keys=off;" ) cursor.execute(""" create table if not exists new_file ( stream_hash text primary key not null references stream, file_name text, download_directory text, blob_data_rate real not null, status text not null, saved_file integer not null, content_fee text ); """) for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall(): saved_file = 0 if download_dir != '{stream}' and file_name != '{stream}': try: if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(), binascii.unhexlify(file_name).decode())): saved_file = 1 else: download_dir, file_name = None, None except Exception: download_dir, file_name = None, None else: download_dir, file_name = None, None cursor.execute( "insert into new_file values (?, ?, ?, ?, ?, ?, NULL)", (stream_hash, file_name, download_dir, data_rate, status, saved_file) ) cursor.execute("drop table file") cursor.execute("alter table new_file rename to file") connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate11to12.py ================================================ import sqlite3 import os import time def do_migration(conf): db_path = os.path.join(conf.data_dir, 'lbrynet.sqlite') connection = sqlite3.connect(db_path) connection.row_factory = sqlite3.Row cursor = connection.cursor() current_columns = [] for col_info in cursor.execute("pragma table_info('file');").fetchall(): current_columns.append(col_info[1]) if 'added_on' in current_columns: connection.close() print('already migrated') return # follow 12 step schema change procedure cursor.execute("pragma foreign_keys=off") # we don't have any indexes, views or triggers, so step 3 is skipped. cursor.execute("drop table if exists new_file") cursor.execute(""" create table if not exists new_file ( stream_hash text not null primary key references stream, file_name text, download_directory text, blob_data_rate text not null, status text not null, saved_file integer not null, content_fee text, added_on integer not null ); """) # step 5: transfer content from old to new select = "select * from file" for (stream_hash, file_name, download_dir, blob_rate, status, saved_file, fee) \ in cursor.execute(select).fetchall(): added_on = int(time.time()) cursor.execute( "insert into new_file values (?, ?, ?, ?, ?, ?, ?, ?)", (stream_hash, file_name, download_dir, blob_rate, status, saved_file, fee, added_on) ) # step 6: drop old table cursor.execute("drop table file") # step 7: rename new table to old table cursor.execute("alter table new_file rename to file") # step 8: we aren't using indexes, views or triggers so skip # step 9: no views so skip # step 10: foreign key check cursor.execute("pragma foreign_key_check;") # step 11: commit transaction connection.commit() # step 12: re-enable foreign keys connection.execute("pragma foreign_keys=on;") # done :) connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate12to13.py ================================================ import os import sqlite3 def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() current_columns = [] for col_info in cursor.execute("pragma table_info('file');").fetchall(): current_columns.append(col_info[1]) if 'bt_infohash' in current_columns: connection.close() print("already migrated") return cursor.executescript(""" pragma foreign_keys=off; create table if not exists torrent ( bt_infohash char(20) not null primary key, tracker text, length integer not null, name text not null ); create table if not exists torrent_node ( -- BEP-0005 bt_infohash char(20) not null references torrent, host text not null, port integer not null ); create table if not exists torrent_tracker ( -- BEP-0012 bt_infohash char(20) not null references torrent, tracker text not null ); create table if not exists torrent_http_seed ( -- BEP-0017 bt_infohash char(20) not null references torrent, http_seed text not null ); create table if not exists new_file ( stream_hash char(96) references stream, bt_infohash char(20) references torrent, file_name text, download_directory text, blob_data_rate real not null, status text not null, saved_file integer not null, content_fee text, added_on integer not null ); create table if not exists new_content_claim ( stream_hash char(96) references stream, bt_infohash char(20) references torrent, claim_outpoint text unique not null references claim ); insert into new_file (stream_hash, bt_infohash, file_name, download_directory, blob_data_rate, status, saved_file, content_fee, added_on) select stream_hash, NULL, file_name, download_directory, blob_data_rate, status, saved_file, content_fee, added_on from file; insert or ignore into new_content_claim (stream_hash, bt_infohash, claim_outpoint) select stream_hash, NULL, claim_outpoint from content_claim; drop table file; drop table content_claim; alter table new_file rename to file; alter table new_content_claim rename to content_claim; pragma foreign_keys=on; """) connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate13to14.py ================================================ import os import sqlite3 def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() cursor.executescript(""" create table if not exists peer ( node_id char(96) not null primary key, address text not null, udp_port integer not null, tcp_port integer, unique (address, udp_port) ); """) connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate14to15.py ================================================ import os import sqlite3 def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() cursor.executescript(""" alter table blob add column added_on integer not null default 0; alter table blob add column is_mine integer not null default 1; """) connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate15to16.py ================================================ import os import sqlite3 def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() cursor.executescript(""" update blob set should_announce=0 where should_announce=1 and blob.blob_hash in (select stream_blob.blob_hash from stream_blob where position=0); """) connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate1to2.py ================================================ import sqlite3 import os import logging log = logging.getLogger(__name__) UNSET_NOUT = -1 def do_migration(conf): log.info("Doing the migration") migrate_blockchainname_db(conf.data_dir) log.info("Migration succeeded") def migrate_blockchainname_db(db_dir): blockchainname_db = os.path.join(db_dir, "blockchainname.db") # skip migration on fresh installs if not os.path.isfile(blockchainname_db): return temp_db = sqlite3.connect(":memory:") db_file = sqlite3.connect(blockchainname_db) file_cursor = db_file.cursor() mem_cursor = temp_db.cursor() mem_cursor.execute("create table if not exists name_metadata (" " name text, " " txid text, " " n integer, " " sd_hash text)") mem_cursor.execute("create table if not exists claim_ids (" " claimId text, " " name text, " " txid text, " " n integer)") temp_db.commit() name_metadata = file_cursor.execute("select * from name_metadata").fetchall() claim_metadata = file_cursor.execute("select * from claim_ids").fetchall() # fill n as V1_UNSET_NOUT, Wallet.py will be responsible for filling in correct n for name, txid, sd_hash in name_metadata: mem_cursor.execute( "insert into name_metadata values (?, ?, ?, ?) ", (name, txid, UNSET_NOUT, sd_hash)) for claim_id, name, txid in claim_metadata: mem_cursor.execute( "insert into claim_ids values (?, ?, ?, ?)", (claim_id, name, txid, UNSET_NOUT)) temp_db.commit() new_name_metadata = mem_cursor.execute("select * from name_metadata").fetchall() new_claim_metadata = mem_cursor.execute("select * from claim_ids").fetchall() file_cursor.execute("drop table name_metadata") file_cursor.execute("create table name_metadata (" " name text, " " txid text, " " n integer, " " sd_hash text)") for name, txid, n, sd_hash in new_name_metadata: file_cursor.execute( "insert into name_metadata values (?, ?, ?, ?) ", (name, txid, n, sd_hash)) file_cursor.execute("drop table claim_ids") file_cursor.execute("create table claim_ids (" " claimId text, " " name text, " " txid text, " " n integer)") for claim_id, name, txid, n in new_claim_metadata: file_cursor.execute("insert into claim_ids values (?, ?, ?, ?)", (claim_id, name, txid, n)) db_file.commit() db_file.close() temp_db.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate2to3.py ================================================ import sqlite3 import os import logging log = logging.getLogger(__name__) def do_migration(conf): log.info("Doing the migration") migrate_blockchainname_db(conf.data_dir) log.info("Migration succeeded") def migrate_blockchainname_db(db_dir): blockchainname_db = os.path.join(db_dir, "blockchainname.db") # skip migration on fresh installs if not os.path.isfile(blockchainname_db): return db_file = sqlite3.connect(blockchainname_db) file_cursor = db_file.cursor() tables = file_cursor.execute("SELECT tbl_name FROM sqlite_master " "WHERE type='table'").fetchall() if 'tmp_name_metadata_table' in tables and 'name_metadata' not in tables: file_cursor.execute("ALTER TABLE tmp_name_metadata_table RENAME TO name_metadata") else: file_cursor.executescript( "CREATE TABLE IF NOT EXISTS tmp_name_metadata_table " " (name TEXT UNIQUE NOT NULL, " " txid TEXT NOT NULL, " " n INTEGER NOT NULL, " " sd_hash TEXT NOT NULL); " "INSERT OR IGNORE INTO tmp_name_metadata_table " " (name, txid, n, sd_hash) " " SELECT name, txid, n, sd_hash FROM name_metadata; " "DROP TABLE name_metadata; " "ALTER TABLE tmp_name_metadata_table RENAME TO name_metadata;" ) db_file.commit() db_file.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate3to4.py ================================================ import sqlite3 import os import logging log = logging.getLogger(__name__) def do_migration(conf): log.info("Doing the migration") migrate_blobs_db(conf.data_dir) log.info("Migration succeeded") def migrate_blobs_db(db_dir): """ We migrate the blobs.db used in BlobManager to have a "should_announce" column, and set this to True for blobs that are sd_hash's or head blobs (first blob in stream) """ blobs_db = os.path.join(db_dir, "blobs.db") lbryfile_info_db = os.path.join(db_dir, 'lbryfile_info.db') # skip migration on fresh installs if not os.path.isfile(blobs_db) and not os.path.isfile(lbryfile_info_db): return # if blobs.db doesn't exist, skip migration if not os.path.isfile(blobs_db): log.info("blobs.db was not found but lbryfile_info.db was found, skipping migration") return blobs_db_file = sqlite3.connect(blobs_db) blobs_db_cursor = blobs_db_file.cursor() # check if new columns exist (it shouldn't) and create it try: blobs_db_cursor.execute("SELECT should_announce FROM blobs") except sqlite3.OperationalError: blobs_db_cursor.execute( "ALTER TABLE blobs ADD COLUMN should_announce integer NOT NULL DEFAULT 0") else: log.warning("should_announce already exists somehow, proceeding anyways") # if lbryfile_info.db doesn't exist, skip marking blobs as should_announce = True if not os.path.isfile(lbryfile_info_db): log.error("lbryfile_info.db was not found, skipping check for should_announce") return lbryfile_info_file = sqlite3.connect(lbryfile_info_db) lbryfile_info_cursor = lbryfile_info_file.cursor() # find blobs that are stream descriptors lbryfile_info_cursor.execute('SELECT * FROM lbry_file_descriptors') descriptors = lbryfile_info_cursor.fetchall() should_announce_blob_hashes = [] for d in descriptors: sd_blob_hash = (d[0],) should_announce_blob_hashes.append(sd_blob_hash) # find blobs that are the first blob in a stream lbryfile_info_cursor.execute('SELECT * FROM lbry_file_blobs WHERE position = 0') blobs = lbryfile_info_cursor.fetchall() head_blob_hashes = [] for b in blobs: blob_hash = (b[0],) should_announce_blob_hashes.append(blob_hash) # now mark them as should_announce = True blobs_db_cursor.executemany('UPDATE blobs SET should_announce=1 WHERE blob_hash=?', should_announce_blob_hashes) # Now run some final checks here to make sure migration succeeded try: blobs_db_cursor.execute("SELECT should_announce FROM blobs") except sqlite3.OperationalError: raise Exception('Migration failed, cannot find should_announce') blobs_db_cursor.execute("SELECT * FROM blobs WHERE should_announce=1") blobs = blobs_db_cursor.fetchall() if len(blobs) != len(should_announce_blob_hashes): log.error("Some how not all blobs were marked as announceable") blobs_db_file.commit() blobs_db_file.close() lbryfile_info_file.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate4to5.py ================================================ import sqlite3 import os import logging log = logging.getLogger(__name__) def do_migration(conf): log.info("Doing the migration") add_lbry_file_metadata(conf.data_dir) log.info("Migration succeeded") def add_lbry_file_metadata(db_dir): """ We migrate the blobs.db used in BlobManager to have a "should_announce" column, and set this to True for blobs that are sd_hash's or head blobs (first blob in stream) """ name_metadata = os.path.join(db_dir, "blockchainname.db") lbryfile_info_db = os.path.join(db_dir, 'lbryfile_info.db') if not os.path.isfile(name_metadata) and not os.path.isfile(lbryfile_info_db): return if not os.path.isfile(lbryfile_info_db): log.info("blockchainname.db was not found but lbryfile_info.db was found, skipping migration") return name_metadata_db = sqlite3.connect(name_metadata) lbryfile_db = sqlite3.connect(lbryfile_info_db) name_metadata_cursor = name_metadata_db.cursor() lbryfile_cursor = lbryfile_db.cursor() lbryfile_db.executescript( "create table if not exists lbry_file_metadata (" + " lbry_file integer primary key, " + " txid text, " + " n integer, " + " foreign key(lbry_file) references lbry_files(rowid)" ")") _files = lbryfile_cursor.execute("select rowid, stream_hash from lbry_files").fetchall() lbry_files = {x[1]: x[0] for x in _files} for (sd_hash, stream_hash) in lbryfile_cursor.execute("select * " "from lbry_file_descriptors").fetchall(): lbry_file_id = lbry_files[stream_hash] outpoint = name_metadata_cursor.execute("select txid, n from name_metadata " "where sd_hash=?", (sd_hash,)).fetchall() if outpoint: txid, nout = outpoint[0] lbryfile_cursor.execute("insert into lbry_file_metadata values (?, ?, ?)", (lbry_file_id, txid, nout)) else: lbryfile_cursor.execute("insert into lbry_file_metadata values (?, ?, ?)", (lbry_file_id, None, None)) lbryfile_db.commit() lbryfile_db.close() name_metadata_db.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate5to6.py ================================================ import sqlite3 import os import json import logging from binascii import hexlify from lbry.schema.claim import Claim log = logging.getLogger(__name__) CREATE_TABLES_QUERY = """ pragma foreign_keys=on; pragma journal_mode=WAL; create table if not exists blob ( blob_hash char(96) primary key not null, blob_length integer not null, next_announce_time integer not null, should_announce integer not null default 0, status text not null ); create table if not exists stream ( stream_hash char(96) not null primary key, sd_hash char(96) not null references blob, stream_key text not null, stream_name text not null, suggested_filename text not null ); create table if not exists stream_blob ( stream_hash char(96) not null references stream, blob_hash char(96) references blob, position integer not null, iv char(32) not null, primary key (stream_hash, blob_hash) ); create table if not exists claim ( claim_outpoint text not null primary key, claim_id char(40) not null, claim_name text not null, amount integer not null, height integer not null, serialized_metadata blob not null, channel_claim_id text, address text not null, claim_sequence integer not null ); create table if not exists file ( stream_hash text primary key not null references stream, file_name text not null, download_directory text not null, blob_data_rate real not null, status text not null ); create table if not exists content_claim ( stream_hash text unique not null references file, claim_outpoint text not null references claim, primary key (stream_hash, claim_outpoint) ); create table if not exists support ( support_outpoint text not null primary key, claim_id text not null, amount integer not null, address text not null ); """ def run_operation(db): def _decorate(fn): def _wrapper(*args): cursor = db.cursor() try: result = fn(cursor, *args) db.commit() return result except sqlite3.IntegrityError: db.rollback() raise return _wrapper return _decorate def verify_sd_blob(sd_hash, blob_dir): with open(os.path.join(blob_dir, sd_hash), "r") as sd_file: data = sd_file.read() sd_length = len(data) decoded = json.loads(data) assert set(decoded.keys()) == { 'stream_name', 'blobs', 'stream_type', 'key', 'suggested_file_name', 'stream_hash' }, "invalid sd blob" for blob in sorted(decoded['blobs'], key=lambda x: int(x['blob_num']), reverse=True): if blob['blob_num'] == len(decoded['blobs']) - 1: assert {'length', 'blob_num', 'iv'} == set(blob.keys()), 'invalid stream terminator' assert blob['length'] == 0, 'non zero length stream terminator' else: assert {'blob_hash', 'length', 'blob_num', 'iv'} == set(blob.keys()), 'invalid stream blob' assert blob['length'] > 0, 'zero length stream blob' return decoded, sd_length def do_migration(conf): new_db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(new_db_path) metadata_db = sqlite3.connect(os.path.join(conf.data_dir, "blockchainname.db")) lbryfile_db = sqlite3.connect(os.path.join(conf.data_dir, 'lbryfile_info.db')) blobs_db = sqlite3.connect(os.path.join(conf.data_dir, 'blobs.db')) name_metadata_cursor = metadata_db.cursor() lbryfile_cursor = lbryfile_db.cursor() blobs_db_cursor = blobs_db.cursor() old_rowid_to_outpoint = { rowid: (txid, nout) for (rowid, txid, nout) in lbryfile_cursor.execute("select * from lbry_file_metadata").fetchall() } old_sd_hash_to_outpoint = { sd_hash: (txid, nout) for (txid, nout, sd_hash) in name_metadata_cursor.execute("select txid, n, sd_hash from name_metadata").fetchall() } sd_hash_to_stream_hash = dict( lbryfile_cursor.execute("select sd_blob_hash, stream_hash from lbry_file_descriptors").fetchall() ) stream_hash_to_stream_blobs = {} for (blob_hash, stream_hash, position, iv, length) in lbryfile_db.execute( "select * from lbry_file_blobs").fetchall(): stream_blobs = stream_hash_to_stream_blobs.get(stream_hash, []) stream_blobs.append((blob_hash, length, position, iv)) stream_hash_to_stream_blobs[stream_hash] = stream_blobs claim_outpoint_queries = {} for claim_query in metadata_db.execute( "select distinct c.txid, c.n, c.claimId, c.name, claim_cache.claim_sequence, claim_cache.claim_address, " "claim_cache.height, claim_cache.amount, claim_cache.claim_pb " "from claim_cache inner join claim_ids c on claim_cache.claim_id=c.claimId"): txid, nout = claim_query[0], claim_query[1] if (txid, nout) in claim_outpoint_queries: continue claim_outpoint_queries[(txid, nout)] = claim_query @run_operation(connection) def _populate_blobs(transaction, blob_infos): transaction.executemany( "insert into blob values (?, ?, ?, ?, ?)", [(blob_hash, blob_length, int(next_announce_time), should_announce, "finished") for (blob_hash, blob_length, _, next_announce_time, should_announce) in blob_infos] ) @run_operation(connection) def _import_file(transaction, sd_hash, stream_hash, key, stream_name, suggested_file_name, data_rate, status, stream_blobs): try: transaction.execute( "insert or ignore into stream values (?, ?, ?, ?, ?)", (stream_hash, sd_hash, key, stream_name, suggested_file_name) ) except sqlite3.IntegrityError: # failed because the sd isn't a known blob, we'll try to read the blob file and recover it return sd_hash # insert any stream blobs that were missing from the blobs table transaction.executemany( "insert or ignore into blob values (?, ?, ?, ?, ?)", [ (blob_hash, length, 0, 0, "pending") for (blob_hash, length, position, iv) in stream_blobs ] ) # insert the stream blobs for blob_hash, length, position, iv in stream_blobs: transaction.execute( "insert or ignore into stream_blob values (?, ?, ?, ?)", (stream_hash, blob_hash, position, iv) ) download_dir = conf.download_dir if not isinstance(download_dir, bytes): download_dir = download_dir.encode() # insert the file transaction.execute( "insert or ignore into file values (?, ?, ?, ?, ?)", (stream_hash, stream_name, hexlify(download_dir), data_rate, status) ) @run_operation(connection) def _add_recovered_blobs(transaction, blob_infos, sd_hash, sd_length): transaction.execute( "insert or replace into blob values (?, ?, ?, ?, ?)", (sd_hash, sd_length, 0, 1, "finished") ) for blob in sorted(blob_infos, key=lambda x: x['blob_num'], reverse=True): if blob['blob_num'] < len(blob_infos) - 1: transaction.execute( "insert or ignore into blob values (?, ?, ?, ?, ?)", (blob['blob_hash'], blob['length'], 0, 0, "pending") ) @run_operation(connection) def _make_db(new_db): # create the new tables new_db.executescript(CREATE_TABLES_QUERY) # first migrate the blobs blobs = blobs_db_cursor.execute("select * from blobs").fetchall() _populate_blobs(blobs) # pylint: disable=no-value-for-parameter log.info("migrated %i blobs", new_db.execute("select count(*) from blob").fetchone()[0]) # used to store the query arguments if we need to try re-importing the lbry file later file_args = {} # <sd_hash>: args tuple file_outpoints = {} # <outpoint tuple>: sd_hash # get the file and stream queries ready for (rowid, sd_hash, stream_hash, key, stream_name, suggested_file_name, data_rate, status) in \ lbryfile_db.execute( "select distinct lbry_files.rowid, d.sd_blob_hash, lbry_files.*, o.blob_data_rate, o.status " "from lbry_files " "inner join lbry_file_descriptors d on lbry_files.stream_hash=d.stream_hash " "inner join lbry_file_options o on lbry_files.stream_hash=o.stream_hash"): # this is try to link the file to a content claim after we've imported all the files if rowid in old_rowid_to_outpoint: file_outpoints[old_rowid_to_outpoint[rowid]] = sd_hash elif sd_hash in old_sd_hash_to_outpoint: file_outpoints[old_sd_hash_to_outpoint[sd_hash]] = sd_hash sd_hash_to_stream_hash[sd_hash] = stream_hash if stream_hash in stream_hash_to_stream_blobs: file_args[sd_hash] = ( sd_hash, stream_hash, key, stream_name, suggested_file_name, data_rate or 0.0, status, stream_hash_to_stream_blobs.pop(stream_hash) ) # used to store the query arguments if we need to try re-importing the claim claim_queries = {} # <sd_hash>: claim query tuple # get the claim queries ready, only keep those with associated files for outpoint, sd_hash in file_outpoints.items(): if outpoint in claim_outpoint_queries: claim_queries[sd_hash] = claim_outpoint_queries[outpoint] # insert the claims new_db.executemany( "insert or ignore into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)", [ ( "%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3], claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8], Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4] ) for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup ] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized) ) log.info("migrated %i claims", new_db.execute("select count(*) from claim").fetchone()[0]) damaged_stream_sds = [] # import the files and get sd hashes of streams to attempt recovering for sd_hash, file_query in file_args.items(): failed_sd = _import_file(*file_query) if failed_sd: damaged_stream_sds.append(failed_sd) # recover damaged streams if damaged_stream_sds: blob_dir = os.path.join(conf.data_dir, "blobfiles") damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir) if p in damaged_stream_sds}) for damaged_sd in damaged_sds_on_disk: try: decoded, sd_length = verify_sd_blob(damaged_sd, blob_dir) blobs = decoded['blobs'] _add_recovered_blobs(blobs, damaged_sd, sd_length) # pylint: disable=no-value-for-parameter _import_file(*file_args[damaged_sd]) damaged_stream_sds.remove(damaged_sd) except (OSError, ValueError, TypeError, AssertionError, sqlite3.IntegrityError): continue log.info("migrated %i files", new_db.execute("select count(*) from file").fetchone()[0]) # associate the content claims to their respective files for claim_arg_tup in claim_queries.values(): if claim_arg_tup and (claim_arg_tup[0], claim_arg_tup[1]) in file_outpoints \ and file_outpoints[(claim_arg_tup[0], claim_arg_tup[1])] in sd_hash_to_stream_hash: try: new_db.execute( "insert or ignore into content_claim values (?, ?)", ( sd_hash_to_stream_hash.get(file_outpoints.get((claim_arg_tup[0], claim_arg_tup[1]))), "%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]) ) ) except sqlite3.IntegrityError: continue log.info("migrated %i content claims", new_db.execute("select count(*) from content_claim").fetchone()[0]) try: _make_db() # pylint: disable=no-value-for-parameter except sqlite3.OperationalError as err: if err.message == "table blob has 7 columns but 5 values were supplied": log.warning("detected a failed previous migration to revision 6, repairing it") connection.close() os.remove(new_db_path) return do_migration(conf) raise err connection.close() blobs_db.close() lbryfile_db.close() metadata_db.close() # os.remove(os.path.join(db_dir, "blockchainname.db")) # os.remove(os.path.join(db_dir, 'lbryfile_info.db')) # os.remove(os.path.join(db_dir, 'blobs.db')) ================================================ FILE: lbry/extras/daemon/migrator/migrate6to7.py ================================================ import sqlite3 import os def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() cursor.executescript("alter table blob add last_announced_time integer;") cursor.executescript("alter table blob add single_announce integer;") cursor.execute("update blob set next_announce_time=0") connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate7to8.py ================================================ import sqlite3 import os def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() cursor.executescript( """ create table reflected_stream ( sd_hash text not null, reflector_address text not null, timestamp integer, primary key (sd_hash, reflector_address) ); """ ) connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/migrator/migrate8to9.py ================================================ import sqlite3 import logging import os from lbry.blob.blob_info import BlobInfo from lbry.stream.descriptor import StreamDescriptor log = logging.getLogger(__name__) def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") blob_dir = os.path.join(conf.data_dir, "blobfiles") connection = sqlite3.connect(db_path) cursor = connection.cursor() query = "select stream_name, stream_key, suggested_filename, sd_hash, stream_hash from stream" streams = cursor.execute(query).fetchall() blobs = cursor.execute("select s.stream_hash, s.position, s.iv, b.blob_hash, b.blob_length from stream_blob s " "left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall() blobs_by_stream = {} for stream_hash, position, iv, blob_hash, blob_length in blobs: blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, 0, blob_hash)) for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams: sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename, blobs_by_stream[stream_hash], stream_hash, sd_hash) if sd_hash != sd.calculate_sd_hash(): log.info("Stream for descriptor %s is invalid, cleaning it up", sd_hash) blob_hashes = [blob.blob_hash for blob in blobs_by_stream[stream_hash]] delete_stream(cursor, stream_hash, sd_hash, blob_hashes, blob_dir) connection.commit() connection.close() def delete_stream(transaction, stream_hash, sd_hash, blob_hashes, blob_dir): transaction.execute("delete from content_claim where stream_hash=? ", (stream_hash,)) transaction.execute("delete from file where stream_hash=? ", (stream_hash, )) transaction.execute("delete from stream_blob where stream_hash=?", (stream_hash, )) transaction.execute("delete from stream where stream_hash=? ", (stream_hash, )) transaction.execute("delete from blob where blob_hash=?", (sd_hash, )) for blob_hash in blob_hashes: transaction.execute("delete from blob where blob_hash=?", (blob_hash, )) file_path = os.path.join(blob_dir, blob_hash) if os.path.isfile(file_path): os.unlink(file_path) ================================================ FILE: lbry/extras/daemon/migrator/migrate9to10.py ================================================ import sqlite3 import os def do_migration(conf): db_path = os.path.join(conf.data_dir, "lbrynet.sqlite") connection = sqlite3.connect(db_path) cursor = connection.cursor() query = "select stream_hash, sd_hash from main.stream" for stream_hash, sd_hash in cursor.execute(query).fetchall(): head_blob_hash = cursor.execute( "select blob_hash from stream_blob where position = 0 and stream_hash = ?", (stream_hash,) ).fetchone() if not head_blob_hash: continue cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],)) connection.commit() connection.close() ================================================ FILE: lbry/extras/daemon/security.py ================================================ import logging from aiohttp import web log = logging.getLogger(__name__) def ensure_request_allowed(request, conf): if is_request_allowed(request, conf): return if conf.allowed_origin: log.warning( "API requests with Origin '%s' are not allowed, " "configuration 'allowed_origin' limits requests to: '%s'", request.headers.get('Origin'), conf.allowed_origin ) else: log.warning( "API requests with Origin '%s' are not allowed, " "update configuration 'allowed_origin' to enable this origin.", request.headers.get('Origin') ) raise web.HTTPForbidden() def is_request_allowed(request, conf) -> bool: origin = request.headers.get('Origin') return ( origin is None or origin == conf.allowed_origin or conf.allowed_origin == '*' ) ================================================ FILE: lbry/extras/daemon/storage.py ================================================ import os import logging import sqlite3 import typing import asyncio import binascii import time from typing import Optional from lbry.wallet import SQLiteMixin from lbry.conf import Config from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies from lbry.wallet.transaction import Transaction, Output from lbry.schema.claim import Claim from lbry.dht.constants import DATA_EXPIRATION from lbry.blob.blob_info import BlobInfo if typing.TYPE_CHECKING: from lbry.blob.blob_file import BlobFile from lbry.stream.descriptor import StreamDescriptor log = logging.getLogger(__name__) def calculate_effective_amount(amount: str, supports: typing.Optional[typing.List[typing.Dict]] = None) -> str: return dewies_to_lbc( lbc_to_dewies(amount) + sum([lbc_to_dewies(support['amount']) for support in supports]) ) class StoredContentClaim: def __init__(self, outpoint: Optional[str] = None, claim_id: Optional[str] = None, name: Optional[str] = None, amount: Optional[int] = None, height: Optional[int] = None, serialized: Optional[str] = None, channel_claim_id: Optional[str] = None, address: Optional[str] = None, claim_sequence: Optional[int] = None, channel_name: Optional[str] = None): self.claim_id = claim_id self.outpoint = outpoint self.claim_name = name self.amount = amount self.height = height self.claim: typing.Optional[Claim] = None if not serialized else Claim.from_bytes( binascii.unhexlify(serialized) ) self.claim_address = address self.claim_sequence = claim_sequence self.channel_claim_id = channel_claim_id self.channel_name = channel_name @property def txid(self) -> typing.Optional[str]: return None if not self.outpoint else self.outpoint.split(":")[0] @property def nout(self) -> typing.Optional[int]: return None if not self.outpoint else int(self.outpoint.split(":")[1]) def as_dict(self) -> typing.Dict: return { "name": self.claim_name, "claim_id": self.claim_id, "address": self.claim_address, "claim_sequence": self.claim_sequence, "value": self.claim, "height": self.height, "amount": dewies_to_lbc(self.amount), "nout": self.nout, "txid": self.txid, "channel_claim_id": self.channel_claim_id, "channel_name": self.channel_name } def _get_content_claims(transaction: sqlite3.Connection, query: str, source_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]: claims = {} for claim_info in _batched_select(transaction, query, source_hashes): claims[claim_info[0]] = StoredContentClaim(*claim_info[1:]) return claims def get_claims_from_stream_hashes(transaction: sqlite3.Connection, stream_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]: query = ( "select content_claim.stream_hash, c.*, case when c.channel_claim_id is not null then " " (select claim_name from claim where claim_id==c.channel_claim_id) " " else null end as channel_name " " from content_claim " " inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.stream_hash in {}" " order by c.rowid desc" ) return _get_content_claims(transaction, query, stream_hashes) def get_claims_from_torrent_info_hashes(transaction: sqlite3.Connection, info_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]: query = ( "select content_claim.bt_infohash, c.*, case when c.channel_claim_id is not null then " " (select claim_name from claim where claim_id==c.channel_claim_id) " " else null end as channel_name " " from content_claim " " inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.bt_infohash in {}" " order by c.rowid desc" ) return _get_content_claims(transaction, query, info_hashes) def _batched_select(transaction, query, parameters, batch_size=900): for start_index in range(0, len(parameters), batch_size): current_batch = parameters[start_index:start_index+batch_size] bind = "({})".format(','.join(['?'] * len(current_batch))) yield from transaction.execute(query.format(bind), current_batch) def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status, sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file, raw_content_fee, fully_reflected): return { "rowid": rowid, "added_on": added_on, "stream_hash": stream_hash, "file_name": file_name, # hex "download_directory": download_dir, # hex "blob_data_rate": data_rate, "status": status, "sd_hash": sd_hash, "key": stream_key, "stream_name": stream_name, # hex "suggested_file_name": suggested_file_name, # hex "claim": claim, "saved_file": bool(saved_file), "content_fee": None if not raw_content_fee else Transaction( binascii.unhexlify(raw_content_fee) ), "fully_reflected": fully_reflected } def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]: files = [] signed_claims = {} for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute( "select file.rowid, file.*, stream.*, c.*, " " case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) " " is null then 0 else 1 end as fully_reflected " "from file inner join stream on file.stream_hash=stream.stream_hash " "inner join content_claim cc on file.stream_hash=cc.stream_hash " "inner join claim c on cc.claim_outpoint=c.claim_outpoint " "order by c.rowid desc").fetchall(): claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1] claim = StoredContentClaim(*claim_args) if claim.channel_claim_id: if claim.channel_claim_id not in signed_claims: signed_claims[claim.channel_claim_id] = [] signed_claims[claim.channel_claim_id].append(claim) files.append( _get_lbry_file_stream_dict( rowid, added_on, stream_hash, file_name, download_dir, data_rate, status, sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file, raw_content_fee, fully_reflected ) ) for claim_name, claim_id in _batched_select( transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}", tuple(signed_claims.keys())): for claim in signed_claims[claim_id]: claim.channel_name = claim_name return files def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'): # add all blobs, except the last one, which is empty transaction.executemany( "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)", ((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0, blob.added_on, blob.is_mine) for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]) ).fetchall() # associate the blobs to the stream transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)", (descriptor.stream_hash, sd_blob.blob_hash, descriptor.key, binascii.hexlify(descriptor.stream_name.encode()).decode(), binascii.hexlify(descriptor.suggested_file_name.encode()).decode())).fetchall() # add the stream transaction.executemany( "insert or ignore into stream_blob values (?, ?, ?, ?)", ((descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv) for blob in descriptor.blobs) ).fetchall() # ensure should_announce is set regardless if insert was ignored transaction.execute( "update blob set should_announce=1 where blob_hash in (?)", (sd_blob.blob_hash,) ).fetchall() def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'): blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]] blob_hashes.append((descriptor.sd_hash, )) transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)).fetchall() transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)).fetchall() transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)).fetchall() transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)).fetchall() transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall() def delete_torrent(transaction: sqlite3.Connection, bt_infohash: str): transaction.execute("delete from content_claim where bt_infohash=?", (bt_infohash, )).fetchall() transaction.execute("delete from torrent_tracker where bt_infohash=?", (bt_infohash,)).fetchall() transaction.execute("delete from torrent_node where bt_infohash=?", (bt_infohash,)).fetchall() transaction.execute("delete from torrent_http_seed where bt_infohash=?", (bt_infohash,)).fetchall() transaction.execute("delete from file where bt_infohash=?", (bt_infohash,)).fetchall() transaction.execute("delete from torrent where bt_infohash=?", (bt_infohash,)).fetchall() def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str], download_directory: typing.Optional[str], data_payment_rate: float, status: str, content_fee: typing.Optional[Transaction], added_on: typing.Optional[int] = None) -> int: if not file_name and not download_directory: encoded_file_name, encoded_download_dir = None, None else: encoded_file_name = binascii.hexlify(file_name.encode()).decode() encoded_download_dir = binascii.hexlify(download_directory.encode()).decode() time_added = added_on or int(time.time()) transaction.execute( "insert or replace into file values (?, NULL, ?, ?, ?, ?, ?, ?, ?)", (stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status, 1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0, None if not content_fee else binascii.hexlify(content_fee.raw).decode(), time_added) ).fetchall() return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0] class SQLiteStorage(SQLiteMixin): CREATE_TABLES_QUERY = """ pragma foreign_keys=on; pragma journal_mode=WAL; create table if not exists blob ( blob_hash char(96) primary key not null, blob_length integer not null, next_announce_time integer not null, should_announce integer not null default 0, status text not null, last_announced_time integer, single_announce integer, added_on integer not null, is_mine integer not null default 0 ); create table if not exists stream ( stream_hash char(96) not null primary key, sd_hash char(96) not null references blob, stream_key text not null, stream_name text not null, suggested_filename text not null ); create table if not exists stream_blob ( stream_hash char(96) not null references stream, blob_hash char(96) references blob, position integer not null, iv char(32) not null, primary key (stream_hash, blob_hash) ); create table if not exists claim ( claim_outpoint text not null primary key, claim_id char(40) not null, claim_name text not null, amount integer not null, height integer not null, serialized_metadata blob not null, channel_claim_id text, address text not null, claim_sequence integer not null ); create table if not exists torrent ( bt_infohash char(20) not null primary key, tracker text, length integer not null, name text not null ); create table if not exists torrent_node ( -- BEP-0005 bt_infohash char(20) not null references torrent, host text not null, port integer not null ); create table if not exists torrent_tracker ( -- BEP-0012 bt_infohash char(20) not null references torrent, tracker text not null ); create table if not exists torrent_http_seed ( -- BEP-0017 bt_infohash char(20) not null references torrent, http_seed text not null ); create table if not exists file ( stream_hash char(96) references stream, bt_infohash char(20) references torrent, file_name text, download_directory text, blob_data_rate real not null, status text not null, saved_file integer not null, content_fee text, added_on integer not null ); create table if not exists content_claim ( stream_hash char(96) references stream, bt_infohash char(20) references torrent, claim_outpoint text unique not null references claim ); create table if not exists support ( support_outpoint text not null primary key, claim_id text not null, amount integer not null, address text not null ); create table if not exists reflected_stream ( sd_hash text not null, reflector_address text not null, timestamp integer, primary key (sd_hash, reflector_address) ); create table if not exists peer ( node_id char(96) not null primary key, address text not null, udp_port integer not null, tcp_port integer, unique (address, udp_port) ); create index if not exists blob_data on blob(blob_hash, blob_length, is_mine); """ def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None): super().__init__(path) self.conf = conf self.content_claim_callbacks = {} self.loop = loop or asyncio.get_event_loop() self.time_getter = time_getter or time.time async def run_and_return_one_or_none(self, query, *args): for row in await self.db.execute_fetchall(query, args): if len(row) == 1: return row[0] return row async def run_and_return_list(self, query, *args): rows = list(await self.db.execute_fetchall(query, args)) return [col[0] for col in rows] if rows else [] # # # # # # # # # blob functions # # # # # # # # # async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int, int, int], finished=False): def _add_blobs(transaction: sqlite3.Connection): transaction.executemany( "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)", ( (blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0, added_on, is_mine) for blob_hash, length, added_on, is_mine in blob_hashes_and_lengths ) ).fetchall() if finished: transaction.executemany( "update blob set status='finished' where blob.blob_hash=?", ( (blob_hash, ) for blob_hash, _, _, _ in blob_hashes_and_lengths ) ).fetchall() return await self.db.run(_add_blobs) def get_blob_status(self, blob_hash: str): return self.run_and_return_one_or_none( "select status from blob where blob_hash=?", blob_hash ) def set_announce(self, *blob_hashes): return self.db.execute_fetchall( "update blob set should_announce=1 where blob_hash in (?, ?)", blob_hashes ) def update_last_announced_blobs(self, blob_hashes: typing.List[str]): def _update_last_announced_blobs(transaction: sqlite3.Connection): last_announced = self.time_getter() return transaction.executemany( "update blob set next_announce_time=?, last_announced_time=?, single_announce=0 " "where blob_hash=?", ((int(last_announced + (DATA_EXPIRATION / 2)), int(last_announced), blob_hash) for blob_hash in blob_hashes) ).fetchall() return self.db.run(_update_last_announced_blobs) def should_single_announce_blobs(self, blob_hashes, immediate=False): def set_single_announce(transaction): now = int(self.time_getter()) for blob_hash in blob_hashes: if immediate: transaction.execute( "update blob set single_announce=1, next_announce_time=? " "where blob_hash=? and status='finished'", (int(now), blob_hash) ).fetchall() else: transaction.execute( "update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash,) ).fetchall() return self.db.run(set_single_announce) def get_blobs_to_announce(self): def get_and_update(transaction): timestamp = int(self.time_getter()) if self.conf.announce_head_and_sd_only: r = transaction.execute( "select blob_hash from blob " "where blob_hash is not null and " "(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' " "order by next_announce_time asc limit ?", (timestamp, int(self.conf.concurrent_blob_announcers * 10)) ).fetchall() else: r = transaction.execute( "select blob_hash from blob where blob_hash is not null " "and next_announce_time<? and status='finished' " "order by next_announce_time asc limit ?", (timestamp, int(self.conf.concurrent_blob_announcers * 10)) ).fetchall() return [b[0] for b in r] return self.db.run(get_and_update) def delete_blobs_from_db(self, blob_hashes): def delete_blobs(transaction): transaction.executemany( "delete from blob where blob_hash=?;", ((blob_hash,) for blob_hash in blob_hashes) ).fetchall() return self.db.run_with_foreign_keys_disabled(delete_blobs) def get_all_blob_hashes(self): return self.run_and_return_list("select blob_hash from blob") async def get_stored_blobs(self, is_mine: bool, is_network_blob=False): is_mine = 1 if is_mine else 0 if is_network_blob: return await self.db.execute_fetchall( "select blob.blob_hash, blob.blob_length, blob.added_on " "from blob left join stream_blob using (blob_hash) " "where stream_blob.stream_hash is null and blob.is_mine=? and blob.status='finished'" "order by blob.blob_length desc, blob.added_on asc", (is_mine,) ) sd_blobs = await self.db.execute_fetchall( "select blob.blob_hash, blob.blob_length, blob.added_on " "from blob join stream on blob.blob_hash=stream.sd_hash join file using (stream_hash) " "where blob.is_mine=? order by blob.added_on asc", (is_mine,) ) content_blobs = await self.db.execute_fetchall( "select blob.blob_hash, blob.blob_length, blob.added_on " "from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)" "cross join file using (stream_hash)" "where blob.is_mine=? and blob.status='finished' order by blob.added_on asc, blob.blob_length asc", (is_mine,) ) return content_blobs + sd_blobs async def get_stored_blob_disk_usage(self): total, network_size, content_size, private_size = await self.db.execute_fetchone(""" select coalesce(sum(blob_length), 0) as total, coalesce(sum(case when stream_blob.stream_hash is null then blob_length else 0 end), 0) as network_storage, coalesce(sum(case when stream_blob.blob_hash is not null and is_mine=0 then blob_length else 0 end), 0) as content_storage, coalesce(sum(case when is_mine=1 then blob_length else 0 end), 0) as private_storage from blob left join stream_blob using (blob_hash) where blob_hash not in (select sd_hash from stream) and blob.status="finished" """) return { 'network_storage': network_size, 'content_storage': content_size, 'private_storage': private_size, 'total': total } async def update_blob_ownership(self, sd_hash, is_mine: bool): is_mine = 1 if is_mine else 0 await self.db.execute_fetchall( "update blob set is_mine = ? where blob_hash in (" " select blob_hash from blob natural join stream_blob natural join stream where sd_hash = ?" ") OR blob_hash = ?", (is_mine, sd_hash, sd_hash) ) def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]: def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]: finished_blob_hashes = tuple( blob_hash for (blob_hash, ) in transaction.execute( "select blob_hash from blob where status='finished'" ).fetchall() ) finished_blobs_set = set(finished_blob_hashes) to_update_set = finished_blobs_set.difference(blob_files) transaction.executemany( "update blob set status='pending' where blob_hash=?", ((blob_hash, ) for blob_hash in to_update_set) ).fetchall() return blob_files.intersection(finished_blobs_set) return self.db.run(_sync_blobs) # # # # # # # # # stream functions # # # # # # # # # async def stream_exists(self, sd_hash: str) -> bool: streams = await self.run_and_return_one_or_none("select stream_hash from stream where sd_hash=?", sd_hash) return streams is not None async def file_exists(self, sd_hash: str) -> bool: streams = await self.run_and_return_one_or_none("select f.stream_hash from file f " "inner join stream s on " "s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash) return streams is not None def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'): return self.db.run(store_stream, sd_blob, descriptor) def get_blobs_for_stream(self, stream_hash, only_completed=False) -> typing.Awaitable[typing.List[BlobInfo]]: def _get_blobs_for_stream(transaction): crypt_blob_infos = [] stream_blobs = transaction.execute( "select s.blob_hash, s.position, s.iv, b.added_on " "from stream_blob s left outer join blob b on b.blob_hash=s.blob_hash where stream_hash=? " "order by position asc", (stream_hash, ) ).fetchall() if only_completed: lengths = transaction.execute( "select b.blob_hash, b.blob_length from blob b " "inner join stream_blob s ON b.blob_hash=s.blob_hash and b.status='finished' and s.stream_hash=?", (stream_hash, ) ).fetchall() else: lengths = transaction.execute( "select b.blob_hash, b.blob_length from blob b " "inner join stream_blob s ON b.blob_hash=s.blob_hash and s.stream_hash=?", (stream_hash, ) ).fetchall() blob_length_dict = {} for blob_hash, length in lengths: blob_length_dict[blob_hash] = length current_time = time.time() for blob_hash, position, iv, added_on in stream_blobs: blob_length = blob_length_dict.get(blob_hash, 0) crypt_blob_infos.append(BlobInfo(position, blob_length, iv, added_on or current_time, blob_hash)) if not blob_hash: break return crypt_blob_infos return self.db.run(_get_blobs_for_stream) def get_sd_blob_hash_for_stream(self, stream_hash): return self.run_and_return_one_or_none( "select sd_hash from stream where stream_hash=?", stream_hash ) def get_stream_hash_for_sd_hash(self, sd_blob_hash): return self.run_and_return_one_or_none( "select stream_hash from stream where sd_hash = ?", sd_blob_hash ) def delete_stream(self, descriptor: 'StreamDescriptor'): return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor) async def delete_torrent(self, bt_infohash: str): return await self.db.run(delete_torrent, bt_infohash) # # # # # # # # # file stuff # # # # # # # # # def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str], download_directory: typing.Optional[str], data_payment_rate: float, content_fee: typing.Optional[Transaction] = None, added_on: typing.Optional[int] = None) -> typing.Awaitable[int]: return self.save_published_file( stream_hash, file_name, download_directory, data_payment_rate, status="running", content_fee=content_fee, added_on=added_on ) def save_published_file(self, stream_hash: str, file_name: typing.Optional[str], download_directory: typing.Optional[str], data_payment_rate: float, status: str = "finished", content_fee: typing.Optional[Transaction] = None, added_on: typing.Optional[int] = None) -> typing.Awaitable[int]: return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status, content_fee, added_on) async def update_manually_removed_files_since_last_run(self): """ Update files that have been removed from the downloads directory since the last run """ def update_manually_removed_files(transaction: sqlite3.Connection): files = {} query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \ "and stream_hash is not null" for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall(): if download_directory and file_name: files[stream_hash] = download_directory, file_name return files def detect_removed(files): return [ stream_hash for stream_hash, (download_directory, file_name) in files.items() if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(), binascii.unhexlify(file_name).decode())) ] def update_db_removed(transaction: sqlite3.Connection, removed): query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}" for cur in _batched_select(transaction, query, removed): cur.fetchall() stream_and_file = await self.db.run(update_manually_removed_files) removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file) if removed: await self.db.run(update_db_removed, removed) def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]: return self.db.run(get_all_lbry_files) def change_file_status(self, stream_hash: str, new_status: str): log.debug("update file status %s -> %s", stream_hash, new_status) return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash)) def stop_all_files(self): log.debug("stopping all files") return self.db.execute_fetchall("update file set status=?", ("stopped",)) async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str], file_name: typing.Optional[str]): if not file_name or not download_dir: encoded_file_name, encoded_download_dir = None, None else: encoded_file_name = binascii.hexlify(file_name.encode()).decode() encoded_download_dir = binascii.hexlify(download_dir.encode()).decode() return await self.db.execute_fetchall("update file set download_directory=?, file_name=? where stream_hash=?", ( encoded_download_dir, encoded_file_name, stream_hash, )) async def save_content_fee(self, stream_hash: str, content_fee: Transaction): return await self.db.execute_fetchall("update file set content_fee=? where stream_hash=?", ( binascii.hexlify(content_fee.raw), stream_hash, )) async def set_saved_file(self, stream_hash: str): return await self.db.execute_fetchall("update file set saved_file=1 where stream_hash=?", ( stream_hash, )) async def clear_saved_file(self, stream_hash: str): return await self.db.execute_fetchall("update file set saved_file=0 where stream_hash=?", ( stream_hash, )) async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile', typing.Optional[Transaction]]], download_directory: str): def _recover(transaction: sqlite3.Connection): stream_hashes = [x[0].stream_hash for x in descriptors_and_sds] for descriptor, sd_blob, content_fee in descriptors_and_sds: content_claim = transaction.execute( "select * from content_claim where stream_hash=?", (descriptor.stream_hash, ) ).fetchone() delete_stream(transaction, descriptor) # this will also delete the content claim store_stream(transaction, sd_blob, descriptor) store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name), download_directory, 0.0, 'stopped', content_fee=content_fee) if content_claim: transaction.execute("insert or ignore into content_claim values (?, ?, ?)", content_claim) transaction.executemany( "update file set status='stopped' where stream_hash=?", ((stream_hash, ) for stream_hash in stream_hashes) ).fetchall() download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode() transaction.executemany( "update file set download_directory=? where stream_hash=?", ((download_dir, stream_hash) for stream_hash in stream_hashes) ).fetchall() await self.db.run_with_foreign_keys_disabled(_recover) def get_all_stream_hashes(self): return self.run_and_return_list("select stream_hash from stream") # # # # # # # # # support functions # # # # # # # # # def save_supports(self, claim_id_to_supports: dict): # TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server def _save_support(transaction): bind = "({})".format(','.join(['?'] * len(claim_id_to_supports))) transaction.execute( f"delete from support where claim_id in {bind}", tuple(claim_id_to_supports.keys()) ).fetchall() for claim_id, supports in claim_id_to_supports.items(): for support in supports: transaction.execute( "insert into support values (?, ?, ?, ?)", ("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']), support.get('address', "")) ).fetchall() return self.db.run(_save_support) def get_supports(self, *claim_ids): def _format_support(outpoint, supported_id, amount, address): return { "txid": outpoint.split(":")[0], "nout": int(outpoint.split(":")[1]), "claim_id": supported_id, "amount": dewies_to_lbc(amount), "address": address, } def _get_supports(transaction): return [ _format_support(*support_info) for support_info in _batched_select( transaction, "select * from support where claim_id in {}", claim_ids ) ] return self.db.run(_get_supports) # # # # # # # # # claim functions # # # # # # # # # async def save_claims(self, claim_infos): claim_id_to_supports = {} update_file_callbacks = [] def _save_claims(transaction): content_claims_to_update = [] for claim_info in claim_infos: outpoint = "%s:%i" % (claim_info['txid'], claim_info['nout']) claim_id = claim_info['claim_id'] name = claim_info['name'] amount = lbc_to_dewies(claim_info['amount']) height = claim_info['height'] address = claim_info['address'] sequence = claim_info['claim_sequence'] certificate_id = claim_info['value'].signing_channel_id try: source_hash = claim_info['value'].stream.source.sd_hash except (AttributeError, ValueError): source_hash = None serialized = binascii.hexlify(claim_info['value'].to_bytes()) transaction.execute( "insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)", (outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence) ).fetchall() # if this response doesn't have support info don't overwrite the existing # support info if 'supports' in claim_info: claim_id_to_supports[claim_id] = claim_info['supports'] if not source_hash: continue stream_hash = transaction.execute( "select file.stream_hash from stream " "inner join file on file.stream_hash=stream.stream_hash where sd_hash=?", (source_hash,) ).fetchone() if not stream_hash: continue stream_hash = stream_hash[0] known_outpoint = transaction.execute( "select claim_outpoint from content_claim where stream_hash=?", (stream_hash,) ).fetchone() known_claim_id = transaction.execute( "select claim_id from claim " "inner join content_claim c3 ON claim.claim_outpoint=c3.claim_outpoint " "where c3.stream_hash=?", (stream_hash,) ).fetchone() if not known_claim_id: content_claims_to_update.append((stream_hash, outpoint)) elif known_outpoint != outpoint: content_claims_to_update.append((stream_hash, outpoint)) for stream_hash, outpoint in content_claims_to_update: self._save_content_claim(transaction, outpoint, stream_hash) if stream_hash in self.content_claim_callbacks: update_file_callbacks.append(self.content_claim_callbacks[stream_hash]()) await self.db.run(_save_claims) if update_file_callbacks: await asyncio.wait(map(asyncio.create_task, update_file_callbacks)) if claim_id_to_supports: await self.save_supports(claim_id_to_supports) def save_claim_from_output(self, ledger, *outputs: Output): return self.save_claims([{ "claim_id": output.claim_id, "name": output.claim_name, "amount": dewies_to_lbc(output.amount), "address": output.get_address(ledger), "txid": output.tx_ref.id, "nout": output.position, "value": output.claim, "height": output.tx_ref.height, "claim_sequence": -1, } for output in outputs]) def save_claims_for_resolve(self, claim_infos): to_save = {} for info in claim_infos: if 'value' in info: if info['value']: to_save[info['claim_id']] = info else: for key in ('certificate', 'claim'): if info.get(key, {}).get('value'): to_save[info[key]['claim_id']] = info[key] return self.save_claims(to_save.values()) @staticmethod def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None): assert stream_hash or bt_infohash # get the claim id and serialized metadata claim_info = transaction.execute( "select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,) ).fetchone() if not claim_info: raise Exception("claim not found") new_claim_id, claim = claim_info[0], Claim.from_bytes(binascii.unhexlify(claim_info[1])) # certificate claims should not be in the content_claim table if not claim.is_stream: raise Exception("claim does not contain a stream") # get the known sd hash for this stream known_sd_hash = transaction.execute( "select sd_hash from stream where stream_hash=?", (stream_hash,) ).fetchone() if not known_sd_hash: raise Exception("stream not found") # check the claim contains the same sd hash if known_sd_hash[0] != claim.stream.source.sd_hash: raise Exception("stream mismatch") # if there is a current claim associated to the file, check that the new claim is an update to it current_associated_content = transaction.execute( "select claim_outpoint from content_claim where stream_hash=?", (stream_hash,) ).fetchone() if current_associated_content: current_associated_claim_id = transaction.execute( "select claim_id from claim where claim_outpoint=?", current_associated_content ).fetchone()[0] if current_associated_claim_id != new_claim_id: raise Exception( f"mismatching claim ids when updating stream {current_associated_claim_id} vs {new_claim_id}" ) # update the claim associated to the file transaction.execute("delete from content_claim where stream_hash=?", (stream_hash, )).fetchall() transaction.execute( "insert into content_claim values (?, NULL, ?)", (stream_hash, claim_outpoint) ).fetchall() async def save_content_claim(self, stream_hash, claim_outpoint): await self.db.run(self._save_content_claim, claim_outpoint, stream_hash) # update corresponding ManagedEncryptedFileDownloader object if stream_hash in self.content_claim_callbacks: await self.content_claim_callbacks[stream_hash]() async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name): def _save_torrent(transaction): transaction.execute( "insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name) ).fetchall() transaction.execute( "insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint) ).fetchall() await self.db.run(_save_torrent) # update corresponding ManagedEncryptedFileDownloader object if bt_infohash in self.content_claim_callbacks: await self.content_claim_callbacks[bt_infohash]() async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict: claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash]) claim = None if claims: claim = claims[stream_hash].as_dict() if include_supports: supports = await self.get_supports(claim['claim_id']) claim['supports'] = supports claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports) return claim async def get_content_claim_for_torrent(self, bt_infohash): claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash]) return claims[bt_infohash].as_dict() if claims else None # # # # # # # # # reflector functions # # # # # # # # # def update_reflected_stream(self, sd_hash, reflector_address, success=True): if success: return self.db.execute_fetchall( "insert or replace into reflected_stream values (?, ?, ?)", (sd_hash, reflector_address, self.time_getter()) ) return self.db.execute_fetchall( "delete from reflected_stream where sd_hash=? and reflector_address=?", (sd_hash, reflector_address) ) def get_streams_to_re_reflect(self): return self.run_and_return_list( "select s.sd_hash from stream s " "left outer join reflected_stream r on s.sd_hash=r.sd_hash " "where r.timestamp is null or r.timestamp < ?", int(self.time_getter()) - 86400 ) # # # # # # # # # # dht functions # # # # # # # # # # # async def get_persisted_kademlia_peers(self) -> typing.List[typing.Tuple[bytes, str, int, int]]: query = 'select node_id, address, udp_port, tcp_port from peer' return [(binascii.unhexlify(n), a, u, t) for n, a, u, t in await self.db.execute_fetchall(query)] async def save_kademlia_peers(self, peers: typing.List['KademliaPeer']): def _save_kademlia_peers(transaction: sqlite3.Connection): transaction.execute('delete from peer').fetchall() transaction.executemany( 'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)', ((binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers) ).fetchall() return await self.db.run(_save_kademlia_peers) ================================================ FILE: lbry/extras/daemon/undecorated.py ================================================ # Copyright 2016-2017 Ionuț Arțăriși <ionut@artarisi.eu> # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This came from https://github.com/mapleoin/undecorated from inspect import isfunction, ismethod, isclass __version__ = '0.3.0' def undecorated(o): """Remove all decorators from a function, method or class""" # class decorator if isinstance(o, type): return o try: # python2 closure = o.func_closure except AttributeError: pass try: # python3 closure = o.__closure__ except AttributeError: return if closure: for cell in closure: # avoid infinite recursion if cell.cell_contents is o: continue # check if the contents looks like a decorator; in that case # we need to go one level down into the dream, otherwise it # might just be a different closed-over variable, which we # can ignore. # Note: this favors supporting decorators defined without # @wraps to the detriment of function/method/class closures if looks_like_a_decorator(cell.cell_contents): undecd = undecorated(cell.cell_contents) if undecd: return undecd return o def looks_like_a_decorator(a): return isfunction(a) or ismethod(a) or isclass(a) ================================================ FILE: lbry/extras/system_info.py ================================================ import platform import os import logging.handlers from lbry import build_info, __version__ as lbrynet_version log = logging.getLogger(__name__) def get_platform() -> dict: os_system = platform.system() if os.environ and 'ANDROID_ARGUMENT' in os.environ: os_system = 'android' d = { "processor": platform.processor(), "python_version": platform.python_version(), "platform": platform.platform(), "os_release": platform.release(), "os_system": os_system, "lbrynet_version": lbrynet_version, "version": lbrynet_version, "build": build_info.BUILD, # CI server sets this during build step } if d["os_system"] == "Linux": import distro # pylint: disable=import-outside-toplevel d["distro"] = distro.info() d["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown') return d ================================================ FILE: lbry/file/__init__.py ================================================ ================================================ FILE: lbry/file/file_manager.py ================================================ import asyncio import logging import typing from typing import Optional from aiohttp.web import Request from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError from lbry.error import InvalidStreamURLError from lbry.stream.managed_stream import ManagedStream from lbry.torrent.torrent_manager import TorrentSource from lbry.utils import cache_concurrent from lbry.schema.url import URL from lbry.wallet.dewies import dewies_to_lbc from lbry.file.source_manager import SourceManager from lbry.file.source import ManagedDownloadSource from lbry.extras.daemon.storage import StoredContentClaim if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage from lbry.wallet import WalletManager from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager log = logging.getLogger(__name__) class FileManager: def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager', storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None): self.loop = loop self.config = config self.wallet_manager = wallet_manager self.storage = storage self.analytics_manager = analytics_manager self.source_managers: typing.Dict[str, SourceManager] = {} self.started = asyncio.Event() @property def streams(self): return self.source_managers['stream']._sources async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource: if 'stream' in self.source_managers: return await self.source_managers['stream'].create(file_path, key, **kwargs) raise NotImplementedError async def start(self): await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values())) for manager in self.source_managers.values(): await manager.started.wait() self.started.set() async def stop(self): for manager in self.source_managers.values(): # fixme: pop or not? await manager.stop() self.started.clear() @cache_concurrent async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager', timeout: Optional[float] = None, file_name: Optional[str] = None, download_directory: Optional[str] = None, save_file: Optional[bool] = None, resolve_timeout: float = 3.0, wallet: Optional['Wallet'] = None) -> ManagedDownloadSource: wallet = wallet or self.wallet_manager.default_wallet timeout = timeout or self.config.download_timeout start_time = self.loop.time() resolved_time = None stream = None claim = None error = None outpoint = None if save_file is None: save_file = self.config.save_files if file_name and not save_file: save_file = True if save_file: download_directory = download_directory or self.config.download_dir else: download_directory = None payment = None try: # resolve the claim try: if not URL.parse(uri).has_stream: raise InvalidStreamURLError(uri) except ValueError: raise InvalidStreamURLError(uri) try: resolved_result = await asyncio.wait_for( self.wallet_manager.ledger.resolve( wallet.accounts, [uri], include_purchase_receipt=True, include_is_my_output=True ), resolve_timeout ) except asyncio.TimeoutError: raise ResolveTimeoutError(uri) except Exception as err: log.exception("Unexpected error resolving stream:") raise ResolveError(f"Unexpected error resolving stream: {str(err)}") if 'error' in resolved_result: raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}") if not resolved_result or uri not in resolved_result: raise ResolveError(f"Failed to resolve stream at '{uri}'") txo = resolved_result[uri] if isinstance(txo, dict): raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}") claim = txo.claim outpoint = f"{txo.tx_ref.id}:{txo.position}" resolved_time = self.loop.time() - start_time await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo) #################### # update or replace #################### if claim.stream.source.bt_infohash: source_manager = self.source_managers['torrent'] existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash) elif claim.stream.source.sd_hash: source_manager = self.source_managers['stream'] existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash) else: raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset") # resume or update an existing stream, if the stream changed: download it and delete the old one after to_replace, updated_stream = None, None if existing and existing[0].claim_id != txo.claim_id: raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}") if existing: log.info("claim contains a metadata only update to a stream we have") if claim.stream.source.bt_infohash: await self.storage.save_torrent_content_claim( existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name ) claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier) existing[0].set_claim(claim_info, claim) else: await self.storage.save_content_claim( existing[0].stream_hash, outpoint ) await source_manager._update_content_claim(existing[0]) updated_stream = existing[0] else: existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id) if existing_for_claim_id: log.info("claim contains an update to a stream we have, downloading it") if save_file and existing_for_claim_id[0].output_file_exists: save_file = False if not claim.stream.source.bt_infohash: existing_for_claim_id[0].downloader.node = source_manager.node await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file) if not existing_for_claim_id[0].output_file_exists and ( save_file or file_name or download_directory): await existing_for_claim_id[0].save_file( file_name=file_name, download_directory=download_directory ) to_replace = existing_for_claim_id[0] # resume or update an existing stream, if the stream changed: download it and delete the old one after if updated_stream: log.info("already have stream for %s", uri) if save_file and updated_stream.output_file_exists: save_file = False if not claim.stream.source.bt_infohash: updated_stream.downloader.node = source_manager.node await updated_stream.start(timeout=timeout, save_now=save_file) if not updated_stream.output_file_exists and (save_file or file_name or download_directory): await updated_stream.save_file( file_name=file_name, download_directory=download_directory ) return updated_stream #################### # pay fee #################### needs_purchasing = ( not to_replace and not txo.is_my_output and txo.has_price and not txo.purchase_receipt ) if needs_purchasing: payment = await self.wallet_manager.create_purchase_transaction( wallet.accounts, txo, exchange_rate_manager ) #################### # make downloader and wait for start #################### # temporary with fields we know so downloader can start. Missing fields are populated later. stored_claim = StoredContentClaim(outpoint=outpoint, claim_id=txo.claim_id, name=txo.claim_name, amount=txo.amount, height=txo.tx_ref.height, serialized=claim.to_bytes().hex()) if not claim.stream.source.bt_infohash: # fixme: this shouldnt be here stream = ManagedStream( self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash, download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, analytics_manager=self.analytics_manager, claim=stored_claim ) stream.downloader.node = source_manager.node else: stream = TorrentSource( self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash, file_name=file_name, download_directory=download_directory or self.config.download_dir, status=ManagedStream.STATUS_RUNNING, claim=stored_claim, analytics_manager=self.analytics_manager, torrent_session=source_manager.torrent_session ) log.info("starting download for %s", uri) before_download = self.loop.time() await stream.start(timeout, save_file) #################### # success case: delete to_replace if applicable, broadcast fee payment #################### if to_replace: # delete old stream now that the replacement has started downloading await source_manager.delete(to_replace) if payment is not None: await self.wallet_manager.broadcast_or_release(payment) payment = None # to avoid releasing in `finally` later log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri) await self.storage.save_content_fee(stream.stream_hash, stream.content_fee) source_manager.add(stream) if not claim.stream.source.bt_infohash: await self.storage.save_content_claim(stream.stream_hash, outpoint) else: await self.storage.save_torrent_content_claim( stream.identifier, outpoint, stream.torrent_length, stream.torrent_name ) claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier) stream.set_claim(claim_info, claim) if save_file: await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download)) return stream except asyncio.TimeoutError: error = DownloadDataTimeoutError(stream.sd_hash) raise error except (Exception, asyncio.CancelledError) as err: # forgive data timeout, don't delete stream expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError, KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError) if isinstance(err, expected): log.warning("Failed to download %s: %s", uri, str(err)) elif isinstance(err, asyncio.CancelledError): pass else: log.exception("Unexpected error downloading stream:") error = err raise finally: if payment is not None: # payment is set to None after broadcasting, if we're here an exception probably happened await self.wallet_manager.ledger.release_tx(payment) if self.analytics_manager and claim and claim.stream.source.bt_infohash: # TODO: analytics for torrents pass elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or stream.downloader.time_to_first_bytes))): server = self.wallet_manager.ledger.network.client.server self.loop.create_task( self.analytics_manager.send_time_to_first_bytes( resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id, uri, outpoint, None if not stream else len(stream.downloader.blob_downloader.active_connections), None if not stream else len(stream.downloader.blob_downloader.scores), None if not stream else len(stream.downloader.blob_downloader.connection_failures), False if not stream else stream.downloader.added_fixed_peers, self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay, None if not stream else stream.sd_hash, None if not stream else stream.downloader.time_to_descriptor, None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash, None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length, None if not stream else stream.downloader.time_to_first_bytes, None if not error else error.__class__.__name__, None if not error else str(error), None if not server else f"{server[0]}:{server[1]}" ) ) async def stream_partial_content(self, request: Request, sd_hash: str): return await self.source_managers['stream'].stream_partial_content(request, sd_hash) def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]: """ Get a list of filtered and sorted ManagedStream objects :param sort_by: field to sort by :param reverse: reverse sorting :param comparison: comparison operator used for filtering :param search_by: fields and values to filter by """ return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), []) async def delete(self, source: ManagedDownloadSource, delete_file=False): for manager in self.source_managers.values(): await manager.delete(source, delete_file) ================================================ FILE: lbry/file/source.py ================================================ import os import asyncio import typing import logging import binascii from typing import Optional from lbry.utils import generate_id from lbry.extras.daemon.storage import StoredContentClaim if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.extras.daemon.analytics import AnalyticsManager from lbry.wallet.transaction import Transaction from lbry.extras.daemon.storage import SQLiteStorage log = logging.getLogger(__name__) class ManagedDownloadSource: STATUS_RUNNING = "running" STATUS_STOPPED = "stopped" STATUS_FINISHED = "finished" SAVING_ID = 1 STREAMING_ID = 2 def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str, file_name: Optional[str] = None, download_directory: Optional[str] = None, status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None, download_id: Optional[str] = None, rowid: Optional[int] = None, content_fee: Optional['Transaction'] = None, analytics_manager: Optional['AnalyticsManager'] = None, added_on: Optional[int] = None): self.loop = loop self.storage = storage self.config = config self.identifier = identifier self.download_directory = download_directory self._file_name = file_name self._status = status self.stream_claim_info = claim self.download_id = download_id or binascii.hexlify(generate_id()).decode() self.rowid = rowid self.content_fee = content_fee self.purchase_receipt = None self._added_on = added_on self.analytics_manager = analytics_manager self.downloader = None self.saving = asyncio.Event() self.finished_writing = asyncio.Event() self.started_writing = asyncio.Event() self.finished_write_attempt = asyncio.Event() # @classmethod # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str, # key: Optional[bytes] = None, # iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource': # raise NotImplementedError() async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False): raise NotImplementedError() async def stop(self, finished: bool = False): raise NotImplementedError() async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): raise NotImplementedError() async def stop_tasks(self): raise NotImplementedError() def set_claim(self, claim_info: typing.Dict, claim: 'Claim'): self.stream_claim_info = StoredContentClaim( f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'], claim_info['name'], claim_info['amount'], claim_info['height'], binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'], claim_info['claim_sequence'], claim_info.get('channel_name') ) # async def update_content_claim(self, claim_info: Optional[typing.Dict] = None): # if not claim_info: # claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash) # self.set_claim(claim_info, claim_info['value']) @property def file_name(self) -> Optional[str]: return self._file_name @property def added_on(self) -> Optional[int]: return self._added_on @property def status(self) -> str: return self._status @property def completed(self): raise NotImplementedError() # @property # def stream_url(self): # return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash} @property def finished(self) -> bool: return self.status == self.STATUS_FINISHED @property def running(self) -> bool: return self.status == self.STATUS_RUNNING @property def claim_id(self) -> Optional[str]: return None if not self.stream_claim_info else self.stream_claim_info.claim_id @property def txid(self) -> Optional[str]: return None if not self.stream_claim_info else self.stream_claim_info.txid @property def nout(self) -> Optional[int]: return None if not self.stream_claim_info else self.stream_claim_info.nout @property def outpoint(self) -> Optional[str]: return None if not self.stream_claim_info else self.stream_claim_info.outpoint @property def claim_height(self) -> Optional[int]: return None if not self.stream_claim_info else self.stream_claim_info.height @property def channel_claim_id(self) -> Optional[str]: return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id @property def channel_name(self) -> Optional[str]: return None if not self.stream_claim_info else self.stream_claim_info.channel_name @property def claim_name(self) -> Optional[str]: return None if not self.stream_claim_info else self.stream_claim_info.claim_name @property def metadata(self) -> Optional[typing.Dict]: return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict() @property def metadata_protobuf(self) -> bytes: if self.stream_claim_info: return binascii.hexlify(self.stream_claim_info.claim.to_bytes()) @property def full_path(self) -> Optional[str]: return os.path.join(self.download_directory, os.path.basename(self.file_name)) \ if self.file_name and self.download_directory else None @property def output_file_exists(self): return os.path.isfile(self.full_path) if self.full_path else False ================================================ FILE: lbry/file/source_manager.py ================================================ import os import asyncio import logging import typing from typing import Optional from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage log = logging.getLogger(__name__) COMPARISON_OPERATORS = { 'eq': lambda a, b: a == b, 'ne': lambda a, b: a != b, 'g': lambda a, b: a > b, 'l': lambda a, b: a < b, 'ge': lambda a, b: a >= b, 'le': lambda a, b: a <= b, } class SourceManager: filter_fields = { 'rowid', 'status', 'file_name', 'added_on', 'download_path', 'claim_name', 'claim_height', 'claim_id', 'outpoint', 'txid', 'nout', 'channel_claim_id', 'channel_name', 'completed' } set_filter_fields = { "claim_ids": "claim_id", "channel_claim_ids": "channel_claim_id", "outpoints": "outpoint" } source_class = ManagedDownloadSource def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None): self.loop = loop self.config = config self.storage = storage self.analytics_manager = analytics_manager self._sources: typing.Dict[str, ManagedDownloadSource] = {} self.started = asyncio.Event() def add(self, source: ManagedDownloadSource): self._sources[source.identifier] = source async def remove(self, source: ManagedDownloadSource): if source.identifier not in self._sources: return self._sources.pop(source.identifier) await source.stop_tasks() async def initialize_from_database(self): raise NotImplementedError() async def start(self): await self.initialize_from_database() self.started.set() async def stop(self): while self._sources: _, source = self._sources.popitem() await source.stop_tasks() self.started.clear() async def create(self, file_path: str, key: Optional[bytes] = None, iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource: raise NotImplementedError() async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): await self.remove(source) if delete_file and source.output_file_exists: os.remove(source.full_path) def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False, comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]: """ Get a list of filtered and sorted ManagedStream objects :param sort_by: field to sort by :param reverse: reverse sorting :param comparison: comparison operator used for filtering :param search_by: fields and values to filter by """ if sort_by and sort_by not in self.filter_fields: raise ValueError(f"'{sort_by}' is not a valid field to sort by") if comparison and comparison not in COMPARISON_OPERATORS: raise ValueError(f"'{comparison}' is not a valid comparison") if 'full_status' in search_by: del search_by['full_status'] for search in search_by: if search not in self.filter_fields: raise ValueError(f"'{search}' is not a valid search operation") compare_sets = {} if isinstance(search_by.get('claim_id'), list): compare_sets['claim_ids'] = search_by.pop('claim_id') if isinstance(search_by.get('outpoint'), list): compare_sets['outpoints'] = search_by.pop('outpoint') if isinstance(search_by.get('channel_claim_id'), list): compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id') if search_by or compare_sets: comparison = comparison or 'eq' streams = [] for stream in self._sources.values(): if compare_sets and not all( getattr(stream, self.set_filter_fields[set_search]) in val for set_search, val in compare_sets.items()): continue if search_by and not all( COMPARISON_OPERATORS[comparison](getattr(stream, search), val) for search, val in search_by.items()): continue streams.append(stream) else: streams = list(self._sources.values()) if sort_by: streams.sort(key=lambda s: getattr(s, sort_by) or "") if reverse: streams.reverse() return streams ================================================ FILE: lbry/file_analysis.py ================================================ import asyncio import json import logging import os import pathlib import platform import re import shlex import shutil import subprocess from math import ceil import lbry.utils from lbry.conf import TranscodeConfig log = logging.getLogger(__name__) class VideoFileAnalyzer: def _replace_or_pop_env(self, variable): if variable + '_ORIG' in self._env_copy: self._env_copy[variable] = self._env_copy[variable + '_ORIG'] else: self._env_copy.pop(variable, None) def __init__(self, conf: TranscodeConfig): self._conf = conf self._available_encoders = "" self._ffmpeg_installed = None self._which_ffmpeg = None self._which_ffprobe = None self._env_copy = dict(os.environ) self._checked_ffmpeg = False if lbry.utils.is_running_from_bundle(): # handle the situation where PyInstaller overrides our runtime environment: self._replace_or_pop_env('LD_LIBRARY_PATH') @staticmethod def _execute(command, environment): # log.debug("Executing: %s", command) try: with subprocess.Popen( shlex.split(command) if platform.system() != 'Windows' else command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment ) as process: (stdout, stderr) = process.communicate() # blocks until the process exits return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode except subprocess.SubprocessError as e: return str(e), -1 # This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here. # The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7. # We work around that issue here by using run_in_executor. Check it again in Python 3.8. async def _execute_ffmpeg(self, arguments): arguments = self._which_ffmpeg + " " + arguments return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy) async def _execute_ffprobe(self, arguments): arguments = self._which_ffprobe + " " + arguments return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy) async def _verify_executables(self): try: await self._execute_ffprobe("-version") version, code = await self._execute_ffmpeg("-version") except Exception as e: code = -1 version = str(e) if code != 0 or not version.startswith("ffmpeg"): log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version) raise FileNotFoundError("Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg " "and ensure that it is callable via PATH or conf.ffmpeg_path") log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg) return version @staticmethod def _which_ffmpeg_and_ffmprobe(path): return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path) async def _verify_ffmpeg_installed(self): if self._ffmpeg_installed: return self._ffmpeg_installed = False path = self._conf.ffmpeg_path if hasattr(self._conf, "data_dir"): path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin") path += os.path.pathsep + self._env_copy.get("PATH", "") self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor( None, self._which_ffmpeg_and_ffmprobe, path ) if not self._which_ffmpeg: log.warning("Unable to locate ffmpeg executable. Path: %s", path) raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}") if not self._which_ffprobe: log.warning("Unable to locate ffprobe executable. Path: %s", path) raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}") if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe): log.warning("ffmpeg and ffprobe are in different folders!") await self._verify_executables() self._ffmpeg_installed = True async def status(self, reset=False, recheck=False): if reset: self._available_encoders = "" self._ffmpeg_installed = None if self._checked_ffmpeg and not recheck: pass elif self._ffmpeg_installed is None: try: await self._verify_ffmpeg_installed() except FileNotFoundError: pass self._checked_ffmpeg = True return { "available": self._ffmpeg_installed, "which": self._which_ffmpeg, "analyze_audio_volume": int(self._conf.volume_analysis_time) > 0 } @staticmethod def _verify_container(scan_data: json): container = scan_data["format"]["format_name"] log.debug(" Detected container is %s", container) splits = container.split(",") if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits): return "Container format is not in the approved list of WebM, MP4. " \ f"Actual: {container} [{scan_data['format']['format_long_name']}]" if "matroska" in splits: for stream in scan_data["streams"]: if stream["codec_type"] == "video": codec = stream["codec_name"] if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")): return "WebM format requires VP8/9 or AV1 video. " \ f"Actual: {codec} [{stream['codec_long_name']}]" elif stream["codec_type"] == "audio": codec = stream["codec_name"] if not {"vorbis", "opus"}.intersection(codec.split(",")): return "WebM format requires Vorbis or Opus audio. " \ f"Actual: {codec} [{stream['codec_long_name']}]" return "" @staticmethod def _verify_video_encoding(scan_data: json): for stream in scan_data["streams"]: if stream["codec_type"] != "video": continue codec = stream["codec_name"] log.debug(" Detected video codec is %s, format is %s", codec, stream["pix_fmt"]) if not {"h264", "vp8", "vp9", "av1", "theora"}.intersection(codec.split(",")): return "Video codec is not in the approved list of H264, VP8, VP9, AV1, Theora. " \ f"Actual: {codec} [{stream['codec_long_name']}]" if "h264" in codec.split(",") and stream["pix_fmt"] != "yuv420p": return "Video codec is H264, but its pixel format does not match the approved yuv420p. " \ f"Actual: {stream['pix_fmt']}" return "" def _verify_bitrate(self, scan_data: json, file_path): bit_rate_max = float(self._conf.video_bitrate_maximum) if bit_rate_max <= 0: return "" if "bit_rate" in scan_data["format"]: bit_rate = float(scan_data["format"]["bit_rate"]) else: bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"]) log.debug(" Detected bitrate is %s Mbps. Allowed max: %s Mbps", str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0)) if bit_rate > bit_rate_max: return "The bit rate is above the configured maximum. Actual: " \ f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps" return "" async def _verify_fast_start(self, scan_data: json, video_file): container = scan_data["format"]["format_name"] if {"webm", "ogg"}.intersection(container.split(",")): return "" result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"') match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result) if match and int(match.group(1)) != 0: return "Video stream descriptors are not at the start of the file (the faststart flag was not used)." return "" @staticmethod def _verify_audio_encoding(scan_data: json): for stream in scan_data["streams"]: if stream["codec_type"] != "audio": continue codec = stream["codec_name"] log.debug(" Detected audio codec is %s", codec) if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")): return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \ f"Actual: {codec} [{stream['codec_long_name']}]" if int(stream['sample_rate']) > 48000: return "Sample rate out of range" return "" async def _verify_audio_volume(self, seconds, video_file): try: validate_volume = int(seconds) > 0 except ValueError: validate_volume = False if not validate_volume: return "" result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} ' f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"') try: mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1)) max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1)) except Exception as e: log.debug(" Failure in volume analysis. Message: %s", str(e)) return "" if max_volume < -5.0 and mean_volume < -22.0: return "Audio is at least five dB lower than prime. " \ f"Actual max: {max_volume}, mean: {mean_volume}" log.debug(" Detected audio volume has mean, max of %f, %f dB", mean_volume, max_volume) return "" @staticmethod def _compute_crf(scan_data): height = 240.0 for stream in scan_data["streams"]: if stream["codec_type"] == "video": height = max(height, float(stream["height"])) # https://developers.google.com/media/vp9/settings/vod/ return int(-0.011 * height + 40) def _get_video_scaler(self): return self._conf.video_scaler async def _get_video_encoder(self, scan_data): # use what the user said if it's there: # if it's not there, use h264 if we can because it's way faster than the others # if we don't have h264 use vp9; it's fairly compatible even though it's slow if not self._available_encoders: self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet") encoder = self._conf.video_encoder.split(" ", 1)[0] if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE): return self._conf.video_encoder if re.search(r"^\s*V..... libx264 ", self._available_encoders, re.MULTILINE): if encoder: log.warning(" Using libx264 since the requested encoder was unavailable. Requested: %s", encoder) return 'libx264 -crf 19 -vf "format=yuv420p"' if not encoder: encoder = "libx264" if re.search(r"^\s*V..... libvpx-vp9 ", self._available_encoders, re.MULTILINE): log.warning(" Using libvpx-vp9 since the requested encoder was unavailable. Requested: %s", encoder) crf = self._compute_crf(scan_data) return f"libvpx-vp9 -crf {crf} -b:v 0" if re.search(r"^\s*V..... libtheora", self._available_encoders, re.MULTILINE): log.warning(" Using libtheora since the requested encoder was unavailable. Requested: %s", encoder) return "libtheora -q:v 7" raise Exception(f"The video encoder is not available. Requested: {encoder}") async def _get_audio_encoder(self, extension): # if the video encoding is theora or av1/vp8/vp9 use opus (or fallback to vorbis) # or we don't have a video encoding but we have an ogg or webm container use opus # if we need to use opus/vorbis see if the conf file has it else use our own params # else use the user-set value if it exists # else use aac wants_opus = extension != "mp4" if not self._available_encoders: self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet") encoder = self._conf.audio_encoder.split(" ", 1)[0] if wants_opus and 'opus' in encoder: return self._conf.audio_encoder if wants_opus and re.search(r"^\s*A..... libopus ", self._available_encoders, re.MULTILINE): return "libopus -b:a 160k" if wants_opus and 'vorbis' in encoder: return self._conf.audio_encoder if wants_opus and re.search(r"^\s*A..... libvorbis ", self._available_encoders, re.MULTILINE): return "libvorbis -q:a 6" if re.search(fr"^\s*A..... {encoder} ", self._available_encoders, re.MULTILINE): return self._conf.audio_encoder if re.search(r"^\s*A..... aac ", self._available_encoders, re.MULTILINE): return "aac -b:a 192k" raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}") @staticmethod def _get_best_container_extension(scan_data, video_encoder): # the container is chosen by the video format # if we are theora-encoded, we want ogg # if we are vp8/vp9/av1 we want webm # use mp4 for anything else if video_encoder: # not re-encoding video if "theora" in video_encoder: return "ogv" if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]): return "webm" return "mp4" for stream in scan_data["streams"]: if stream["codec_type"] != "video": continue codec = stream["codec_name"].split(",") if "theora" in codec: return "ogv" if {"vp8", "vp9", "av1"}.intersection(codec): return "webm" return "mp4" async def _get_scan_data(self, validate, file_path): arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"' result, _ = await self._execute_ffprobe(arguments) try: scan_data = json.loads(result) except Exception as e: log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e)) raise ValueError(f'Absent or unreadable video file: {file_path}') if "format" not in scan_data or "duration" not in scan_data["format"]: log.debug("Format data is missing from ffprobe results for: %s", file_path) raise ValueError(f'Media file does not appear to contain video content: {file_path}') if float(scan_data["format"]["duration"]) < 0.1: log.debug("Media file appears to be an image: %s", file_path) raise ValueError(f'Assuming image file at: {file_path}') return scan_data @staticmethod def _build_spec(scan_data): assert scan_data duration = ceil(float(scan_data["format"]["duration"])) # existence verified when scan_data made width = -1 height = -1 for stream in scan_data["streams"]: if stream["codec_type"] != "video": continue width = max(width, int(stream["width"])) height = max(height, int(stream["height"])) log.debug(" Detected duration: %d sec. with resolution: %d x %d", duration, width, height) spec = {"duration": duration} if height >= 0: spec["height"] = height if width >= 0: spec["width"] = width return spec async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False): if not validate and not repair: return file_path, {} if ignore_non_video and not file_path: return file_path, {} await self._verify_ffmpeg_installed() try: scan_data = await self._get_scan_data(validate, file_path) except ValueError: if ignore_non_video: return file_path, {} raise fast_start_msg = await self._verify_fast_start(scan_data, file_path) log.debug("Analyzing %s:", file_path) spec = self._build_spec(scan_data) log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true") container_msg = self._verify_container(scan_data) bitrate_msg = self._verify_bitrate(scan_data, file_path) video_msg = self._verify_video_encoding(scan_data) audio_msg = self._verify_audio_encoding(scan_data) volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path) messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg] if not any(messages): return file_path, spec if not repair: errors = ["Streamability verification failed:"] errors.extend(filter(None, messages)) raise Exception("\n ".join(errors)) # the plan for transcoding: # we have to re-encode the video if it is in a nonstandard format # we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg) # we also re-encode if our bitrate or sample rate is too high try: transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v'] video_encoder = "" if video_msg or bitrate_msg: video_encoder = await self._get_video_encoder(scan_data) transcode_command.append(video_encoder) # could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway... transcode_command.append(self._get_video_scaler()) else: transcode_command.append("copy") transcode_command.append("-movflags +faststart -c:a") extension = self._get_best_container_extension(scan_data, video_encoder) if audio_msg or volume_msg: audio_encoder = await self._get_audio_encoder(extension) transcode_command.append(audio_encoder) if volume_msg and self._conf.volume_filter: transcode_command.append(self._conf.volume_filter) if audio_msg == "Sample rate out of range": transcode_command.append(" -ar 48000 ") else: transcode_command.append("copy") # TODO: put it in a temp folder and delete it after we upload? path = pathlib.Path(file_path) output = path.parent / f"{path.stem}_fixed.{extension}" transcode_command.append(f'"{output}"') ffmpeg_command = " ".join(transcode_command) log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command) result, code = await self._execute_ffmpeg(ffmpeg_command) if code != 0: raise Exception(f"Failure to complete the transcode command. Output: {result}") except Exception as e: if validate: raise log.info("Unable to transcode %s . Message: %s", file_path, str(e)) # TODO: delete partial output file here if it exists? return file_path, spec return str(output), spec ================================================ FILE: lbry/prometheus.py ================================================ import time import logging import asyncio import asyncio.tasks from aiohttp import web from prometheus_client import generate_latest as prom_generate_latest from prometheus_client import Counter, Histogram, Gauge PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio') PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio') PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio') TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio') def get_loop_metrics(delay=1): loop = asyncio.get_event_loop() def callback(started): PROBE_TIMES.observe(time.perf_counter() - started - delay) PROBES_FINISHED.inc() async def monitor_loop_responsiveness(): while True: now = time.perf_counter() loop.call_later(delay, callback, now) PROBES_IN_FLIGHT.inc() TASK_COUNT.set(len(asyncio.tasks._all_tasks)) await asyncio.sleep(delay) return loop.create_task(monitor_loop_responsiveness()) class PrometheusServer: def __init__(self, logger=None): self.runner = None self.logger = logger or logging.getLogger(__name__) self._monitor_loop_task = None async def start(self, interface: str, port: int): self.logger.info("start prometheus metrics") prom_app = web.Application() prom_app.router.add_get('/metrics', self.handle_metrics_get_request) self.runner = web.AppRunner(prom_app) await self.runner.setup() metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5) await metrics_site.start() self.logger.info( 'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2] ) self._monitor_loop_task = get_loop_metrics() async def handle_metrics_get_request(self, request: web.Request): try: return web.Response( text=prom_generate_latest().decode(), content_type='text/plain; version=0.0.4' ) except Exception: self.logger.exception('could not generate prometheus data') raise async def stop(self): if self._monitor_loop_task and not self._monitor_loop_task.done(): self._monitor_loop_task.cancel() self._monitor_loop_task = None await self.runner.cleanup() ================================================ FILE: lbry/schema/Makefile ================================================ build: rm types/v2/* -rf touch types/v2/__init__.py cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto cd types/v2/ && cp ../../../../../types/jsonschema/* ./ sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py ================================================ FILE: lbry/schema/README.md ================================================ Schema ===== Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like: ``` repos/ - lbry-sdk/ - types/ ``` Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is: ```bash cd ~/.local/bin wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d.. ``` Finally, `make` should update everything in place. ### Why protoc 3.2.0? Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!). ================================================ FILE: lbry/schema/__init__.py ================================================ from .claim import Claim ================================================ FILE: lbry/schema/attrs.py ================================================ import json import logging import os.path import hashlib from typing import Tuple, List from string import ascii_letters from decimal import Decimal, ROUND_UP from binascii import hexlify, unhexlify from google.protobuf.json_format import MessageToDict from lbry.crypto.base58 import Base58 from lbry.constants import COIN from lbry.error import MissingPublishedFileError, EmptyPublishedFileError from lbry.schema.mime_types import guess_media_type from lbry.schema.base import Metadata, BaseMessageList from lbry.schema.tags import clean_tags, normalize_tag from lbry.schema.types.v2.claim_pb2 import ( Fee as FeeMessage, Location as LocationMessage, Language as LanguageMessage ) log = logging.getLogger(__name__) def calculate_sha384_file_hash(file_path): sha384 = hashlib.sha384() with open(file_path, 'rb') as f: for chunk in iter(lambda: f.read(128 * sha384.block_size), b''): sha384.update(chunk) return sha384.digest() def country_int_to_str(country: int) -> str: r = LocationMessage.Country.Name(country) return r[1:] if r.startswith('R') else r def country_str_to_int(country: str) -> int: if len(country) == 3: country = 'R' + country return LocationMessage.Country.Value(country) class Dimmensional(Metadata): __slots__ = () @property def width(self) -> int: return self.message.width @width.setter def width(self, width: int): self.message.width = width @property def height(self) -> int: return self.message.height @height.setter def height(self, height: int): self.message.height = height @property def dimensions(self) -> Tuple[int, int]: return self.width, self.height @dimensions.setter def dimensions(self, dimensions: Tuple[int, int]): self.message.width, self.message.height = dimensions def _extract(self, file_metadata, field): try: setattr(self, field, file_metadata.getValues(field)[0]) except: log.exception(f'Could not extract {field} from file metadata.') def update(self, file_metadata=None, height=None, width=None): if height is not None: self.height = height elif file_metadata: self._extract(file_metadata, 'height') if width is not None: self.width = width elif file_metadata: self._extract(file_metadata, 'width') class Playable(Metadata): __slots__ = () @property def duration(self) -> int: return self.message.duration @duration.setter def duration(self, duration: int): self.message.duration = duration def update(self, file_metadata=None, duration=None): if duration is not None: self.duration = duration elif file_metadata: try: self.duration = file_metadata.getValues('duration')[0].seconds except: log.exception('Could not extract duration from file metadata.') class Image(Dimmensional): __slots__ = () class Audio(Playable): __slots__ = () class Video(Dimmensional, Playable): __slots__ = () def update(self, file_metadata=None, height=None, width=None, duration=None): Dimmensional.update(self, file_metadata, height, width) Playable.update(self, file_metadata, duration) class Source(Metadata): __slots__ = () def update(self, file_path=None): if file_path is not None: self.name = os.path.basename(file_path) self.media_type, stream_type = guess_media_type(file_path) if not os.path.isfile(file_path): raise MissingPublishedFileError(file_path) self.size = os.path.getsize(file_path) if self.size == 0: raise EmptyPublishedFileError(file_path) self.file_hash_bytes = calculate_sha384_file_hash(file_path) return stream_type @property def name(self) -> str: return self.message.name @name.setter def name(self, name: str): self.message.name = name @property def size(self) -> int: return self.message.size @size.setter def size(self, size: int): self.message.size = size @property def media_type(self) -> str: return self.message.media_type @media_type.setter def media_type(self, media_type: str): self.message.media_type = media_type @property def file_hash(self) -> str: return hexlify(self.message.hash).decode() @file_hash.setter def file_hash(self, file_hash: str): self.message.hash = unhexlify(file_hash.encode()) @property def file_hash_bytes(self) -> bytes: return self.message.hash @file_hash_bytes.setter def file_hash_bytes(self, file_hash_bytes: bytes): self.message.hash = file_hash_bytes @property def sd_hash(self) -> str: return hexlify(self.message.sd_hash).decode() @sd_hash.setter def sd_hash(self, sd_hash: str): self.message.sd_hash = unhexlify(sd_hash.encode()) @property def sd_hash_bytes(self) -> bytes: return self.message.sd_hash @sd_hash_bytes.setter def sd_hash_bytes(self, sd_hash: bytes): self.message.sd_hash = sd_hash @property def bt_infohash(self) -> str: return hexlify(self.message.bt_infohash).decode() @bt_infohash.setter def bt_infohash(self, bt_infohash: str): self.message.bt_infohash = unhexlify(bt_infohash.encode()) @property def bt_infohash_bytes(self) -> bytes: return self.message.bt_infohash.decode() @bt_infohash_bytes.setter def bt_infohash_bytes(self, bt_infohash: bytes): self.message.bt_infohash = bt_infohash @property def url(self) -> str: return self.message.url @url.setter def url(self, url: str): self.message.url = url class Fee(Metadata): __slots__ = () def update(self, address: str = None, currency: str = None, amount=None): if amount: currency = (currency or self.currency or '').lower() if not currency: raise Exception('In order to set a fee amount, please specify a fee currency.') if currency not in ('lbc', 'btc', 'usd'): raise Exception(f'Missing or unknown currency provided: {currency}') setattr(self, currency, Decimal(amount)) elif currency: raise Exception('In order to set a fee currency, please specify a fee amount.') if address: if not self.currency: raise Exception('In order to set a fee address, please specify a fee amount and currency.') self.address = address @property def currency(self) -> str: if self.message.currency: return FeeMessage.Currency.Name(self.message.currency) @property def address(self) -> str: if self.address_bytes: return Base58.encode(self.address_bytes) @address.setter def address(self, address: str): self.address_bytes = Base58.decode(address) @property def address_bytes(self) -> bytes: return self.message.address @address_bytes.setter def address_bytes(self, address: bytes): self.message.address = address @property def amount(self) -> Decimal: if self.currency == 'LBC': return self.lbc if self.currency == 'BTC': return self.btc if self.currency == 'USD': return self.usd DEWIES = Decimal(COIN) @property def lbc(self) -> Decimal: if self.message.currency != FeeMessage.LBC: raise ValueError('LBC can only be returned for LBC fees.') return Decimal(self.message.amount / self.DEWIES) @lbc.setter def lbc(self, amount: Decimal): self.dewies = int(amount * self.DEWIES) @property def dewies(self) -> int: if self.message.currency != FeeMessage.LBC: raise ValueError('Dewies can only be returned for LBC fees.') return self.message.amount @dewies.setter def dewies(self, amount: int): self.message.amount = amount self.message.currency = FeeMessage.LBC SATOSHIES = Decimal(COIN) @property def btc(self) -> Decimal: if self.message.currency != FeeMessage.BTC: raise ValueError('BTC can only be returned for BTC fees.') return Decimal(self.message.amount / self.SATOSHIES) @btc.setter def btc(self, amount: Decimal): self.satoshis = int(amount * self.SATOSHIES) @property def satoshis(self) -> int: if self.message.currency != FeeMessage.BTC: raise ValueError('Satoshies can only be returned for BTC fees.') return self.message.amount @satoshis.setter def satoshis(self, amount: int): self.message.amount = amount self.message.currency = FeeMessage.BTC PENNIES = Decimal('100.0') PENNY = Decimal('0.01') @property def usd(self) -> Decimal: if self.message.currency != FeeMessage.USD: raise ValueError('USD can only be returned for USD fees.') return Decimal(self.message.amount / self.PENNIES) @usd.setter def usd(self, amount: Decimal): self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES) @property def pennies(self) -> int: if self.message.currency != FeeMessage.USD: raise ValueError('Pennies can only be returned for USD fees.') return self.message.amount @pennies.setter def pennies(self, amount: int): self.message.amount = amount self.message.currency = FeeMessage.USD class ClaimReference(Metadata): __slots__ = () @property def claim_id(self) -> str: return hexlify(self.claim_hash[::-1]).decode() @claim_id.setter def claim_id(self, claim_id: str): self.claim_hash = unhexlify(claim_id)[::-1] @property def claim_hash(self) -> bytes: return self.message.claim_hash @claim_hash.setter def claim_hash(self, claim_hash: bytes): self.message.claim_hash = claim_hash class ClaimList(BaseMessageList[ClaimReference]): __slots__ = () item_class = ClaimReference @property def _message(self): return self.message.claim_references def append(self, value): self.add().claim_id = value @property def ids(self) -> List[str]: return [c.claim_id for c in self] class Language(Metadata): __slots__ = () @property def langtag(self) -> str: langtag = [] if self.language: langtag.append(self.language) if self.script: langtag.append(self.script) if self.region: langtag.append(self.region) return '-'.join(langtag) @langtag.setter def langtag(self, langtag: str): parts = langtag.split('-') self.language = parts.pop(0) if parts and len(parts[0]) == 4: self.script = parts.pop(0) if parts and len(parts[0]) == 2 and parts[0].isalpha(): self.region = parts.pop(0) if parts and len(parts[0]) == 3 and parts[0].isdigit(): self.region = parts.pop(0) assert not parts, f"Failed to parse language tag: {langtag}" @property def language(self) -> str: if self.message.language: return LanguageMessage.Language.Name(self.message.language) @language.setter def language(self, language: str): self.message.language = LanguageMessage.Language.Value(language) @property def script(self) -> str: if self.message.script: return LanguageMessage.Script.Name(self.message.script) @script.setter def script(self, script: str): self.message.script = LanguageMessage.Script.Value(script) @property def region(self) -> str: if self.message.region: return country_int_to_str(self.message.region) @region.setter def region(self, region: str): self.message.region = country_str_to_int(region) class LanguageList(BaseMessageList[Language]): __slots__ = () item_class = Language def append(self, value: str): self.add().langtag = value class Location(Metadata): __slots__ = () def from_value(self, value): if isinstance(value, str) and value.startswith('{'): value = json.loads(value) if isinstance(value, dict): for key, val in value.items(): setattr(self, key, val) elif isinstance(value, str): parts = value.split(':') if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters): country = parts and parts.pop(0) if country: self.country = country state = parts and parts.pop(0) if state: self.state = state city = parts and parts.pop(0) if city: self.city = city code = parts and parts.pop(0) if code: self.code = code latitude = parts and parts.pop(0) if latitude: self.latitude = latitude longitude = parts and parts.pop(0) if longitude: self.longitude = longitude else: raise ValueError(f'Could not parse country value: {value}') def to_dict(self): d = MessageToDict(self.message) if self.message.longitude: d['longitude'] = self.longitude if self.message.latitude: d['latitude'] = self.latitude return d @property def country(self) -> str: if self.message.country: return LocationMessage.Country.Name(self.message.country) @country.setter def country(self, country: str): self.message.country = LocationMessage.Country.Value(country) @property def state(self) -> str: return self.message.state @state.setter def state(self, state: str): self.message.state = state @property def city(self) -> str: return self.message.city @city.setter def city(self, city: str): self.message.city = city @property def code(self) -> str: return self.message.code @code.setter def code(self, code: str): self.message.code = code GPS_PRECISION = Decimal('10000000') @property def latitude(self) -> str: if self.message.latitude: return str(Decimal(self.message.latitude) / self.GPS_PRECISION) @latitude.setter def latitude(self, latitude: str): latitude = Decimal(latitude) assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees." self.message.latitude = int(latitude * self.GPS_PRECISION) @property def longitude(self) -> str: if self.message.longitude: return str(Decimal(self.message.longitude) / self.GPS_PRECISION) @longitude.setter def longitude(self, longitude: str): longitude = Decimal(longitude) assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees." self.message.longitude = int(longitude * self.GPS_PRECISION) class LocationList(BaseMessageList[Location]): __slots__ = () item_class = Location def append(self, value): self.add().from_value(value) class TagList(BaseMessageList[str]): __slots__ = () item_class = str def append(self, tag: str): tag = normalize_tag(tag) if tag and tag not in self.message: self.message.append(tag) ================================================ FILE: lbry/schema/base.py ================================================ from binascii import hexlify, unhexlify from typing import List, Iterator, TypeVar, Generic from google.protobuf.message import DecodeError from google.protobuf.json_format import MessageToDict class Signable: __slots__ = ( 'message', 'version', 'signature', 'signature_type', 'unsigned_payload', 'signing_channel_hash' ) message_class = None def __init__(self, message=None): self.message = message or self.message_class() self.version = 2 self.signature = None self.signature_type = 'SECP256k1' self.unsigned_payload = None self.signing_channel_hash = None def clear_signature(self): self.signature = None self.unsigned_payload = None self.signing_channel_hash = None @property def signing_channel_id(self): return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None @signing_channel_id.setter def signing_channel_id(self, channel_id: str): self.signing_channel_hash = unhexlify(channel_id)[::-1] @property def is_signed(self): return self.signature is not None def to_dict(self): return MessageToDict(self.message) def to_message_bytes(self) -> bytes: return self.message.SerializeToString() def to_bytes(self) -> bytes: pieces = bytearray() if self.is_signed: pieces.append(1) pieces.extend(self.signing_channel_hash) pieces.extend(self.signature) else: pieces.append(0) pieces.extend(self.to_message_bytes()) return bytes(pieces) @classmethod def from_bytes(cls, data: bytes): signable = cls() if data[0] == 0: signable.message.ParseFromString(data[1:]) elif data[0] == 1: signable.signing_channel_hash = data[1:21] signable.signature = data[21:85] signable.message.ParseFromString(data[85:]) else: raise DecodeError('Could not determine message format version.') return signable def __len__(self): return len(self.to_bytes()) def __bytes__(self): return self.to_bytes() class Metadata: __slots__ = 'message', def __init__(self, message): self.message = message I = TypeVar('I') class BaseMessageList(Metadata, Generic[I]): __slots__ = () item_class = None @property def _message(self): return self.message def add(self) -> I: return self.item_class(self._message.add()) def extend(self, values: List[str]): for value in values: self.append(value) def append(self, value: str): raise NotImplemented def __len__(self): return len(self._message) def __iter__(self) -> Iterator[I]: for item in self._message: yield self.item_class(item) def __getitem__(self, item) -> I: return self.item_class(self._message[item]) def __delitem__(self, key): del self._message[key] def __eq__(self, other) -> bool: return self._message == other ================================================ FILE: lbry/schema/claim.py ================================================ import logging from typing import List from binascii import hexlify, unhexlify from asn1crypto.keys import PublicKeyInfo from coincurve import PublicKey as cPublicKey from google.protobuf.json_format import MessageToDict from google.protobuf.message import DecodeError from hachoir.core.log import log as hachoir_log from hachoir.parser import createParser as binary_file_parser from hachoir.metadata import extractMetadata as binary_file_metadata from lbry.schema import compat from lbry.schema.base import Signable from lbry.schema.mime_types import guess_media_type, guess_stream_type from lbry.schema.attrs import ( Source, Playable, Dimmensional, Fee, Image, Video, Audio, LanguageList, LocationList, ClaimList, ClaimReference, TagList ) from lbry.schema.types.v2.claim_pb2 import Claim as ClaimMessage from lbry.error import InputValueIsNoneError hachoir_log.use_print = False log = logging.getLogger(__name__) class Claim(Signable): STREAM = 'stream' CHANNEL = 'channel' COLLECTION = 'collection' REPOST = 'repost' __slots__ = () message_class = ClaimMessage @property def claim_type(self) -> str: return self.message.WhichOneof('type') def get_message(self, type_name): message = getattr(self.message, type_name) if self.claim_type is None: message.SetInParent() if self.claim_type != type_name: raise ValueError(f'Claim is not a {type_name}.') return message @property def is_stream(self): return self.claim_type == self.STREAM @property def stream(self) -> 'Stream': return Stream(self) @property def is_channel(self): return self.claim_type == self.CHANNEL @property def channel(self) -> 'Channel': return Channel(self) @property def is_repost(self): return self.claim_type == self.REPOST @property def repost(self) -> 'Repost': return Repost(self) @property def is_collection(self): return self.claim_type == self.COLLECTION @property def collection(self) -> 'Collection': return Collection(self) @classmethod def from_bytes(cls, data: bytes) -> 'Claim': try: return super().from_bytes(data) except DecodeError: claim = cls() if data[0] == ord('{'): claim.version = 0 compat.from_old_json_schema(claim, data) elif data[0] not in (0, 1): claim.version = 1 compat.from_types_v1(claim, data) else: raise return claim class BaseClaim: __slots__ = 'claim', 'message' claim_type = None object_fields = 'thumbnail', repeat_fields = 'tags', 'languages', 'locations' def __init__(self, claim: Claim = None): self.claim = claim or Claim() self.message = self.claim.get_message(self.claim_type) def to_dict(self): claim = MessageToDict(self.claim.message, preserving_proto_field_name=True) claim.update(claim.pop(self.claim_type)) if 'languages' in claim: claim['languages'] = self.langtags if 'locations' in claim: claim['locations'] = [l.to_dict() for l in self.locations] return claim def none_check(self, kwargs): for key, value in kwargs.items(): if value is None: raise InputValueIsNoneError(key) def update(self, **kwargs): self.none_check(kwargs) for key in list(kwargs): for field in self.object_fields: if key.startswith(f'{field}_'): attr = getattr(self, field) setattr(attr, key[len(f'{field}_'):], kwargs.pop(key)) continue for l in self.repeat_fields: field = getattr(self, l) if kwargs.pop(f'clear_{l}', False): del field[:] items = kwargs.pop(l, None) if items is not None: if isinstance(items, str): field.append(items) elif isinstance(items, list): field.extend(items) else: raise ValueError(f"Unknown {l} value: {items}") for key, value in kwargs.items(): setattr(self, key, value) @property def title(self) -> str: return self.claim.message.title @title.setter def title(self, title: str): self.claim.message.title = title @property def description(self) -> str: return self.claim.message.description @description.setter def description(self, description: str): self.claim.message.description = description @property def thumbnail(self) -> Source: return Source(self.claim.message.thumbnail) @property def tags(self) -> List[str]: return TagList(self.claim.message.tags) @property def languages(self) -> LanguageList: return LanguageList(self.claim.message.languages) @property def langtags(self) -> List[str]: return [l.langtag for l in self.languages] @property def locations(self) -> LocationList: return LocationList(self.claim.message.locations) class Stream(BaseClaim): __slots__ = () claim_type = Claim.STREAM object_fields = BaseClaim.object_fields + ('source',) def to_dict(self): claim = super().to_dict() if 'source' in claim: if 'hash' in claim['source']: claim['source']['hash'] = self.source.file_hash if 'sd_hash' in claim['source']: claim['source']['sd_hash'] = self.source.sd_hash elif 'bt_infohash' in claim['source']: claim['source']['bt_infohash'] = self.source.bt_infohash if 'media_type' in claim['source']: claim['stream_type'] = guess_stream_type(claim['source']['media_type']) fee = claim.get('fee', {}) if 'address' in fee: fee['address'] = self.fee.address if 'amount' in fee: fee['amount'] = str(self.fee.amount) return claim def update(self, file_path=None, height=None, width=None, duration=None, **kwargs): if kwargs.pop('clear_fee', False): self.message.ClearField('fee') else: self.fee.update( kwargs.pop('fee_address', None), kwargs.pop('fee_currency', None), kwargs.pop('fee_amount', None) ) self.none_check(kwargs) if 'sd_hash' in kwargs: self.source.sd_hash = kwargs.pop('sd_hash') elif 'bt_infohash' in kwargs: self.source.bt_infohash = kwargs.pop('bt_infohash') if 'file_name' in kwargs: self.source.name = kwargs.pop('file_name') if 'file_hash' in kwargs: self.source.file_hash = kwargs.pop('file_hash') stream_type = None if file_path is not None: stream_type = self.source.update(file_path=file_path) elif self.source.name: self.source.media_type, stream_type = guess_media_type(self.source.name) elif self.source.media_type: stream_type = guess_stream_type(self.source.media_type) if 'file_size' in kwargs: self.source.size = kwargs.pop('file_size') if self.stream_type is not None and self.stream_type != stream_type: self.message.ClearField(self.stream_type) if stream_type in ('image', 'video', 'audio'): media = getattr(self, stream_type) media_args = {'file_metadata': None} if file_path is not None and not all((duration, width, height)): try: media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path)) except: log.exception('Could not read file metadata.') if isinstance(media, Playable): media_args['duration'] = duration if isinstance(media, Dimmensional): media_args['height'] = height media_args['width'] = width media.update(**media_args) super().update(**kwargs) @property def author(self) -> str: return self.message.author @author.setter def author(self, author: str): self.message.author = author @property def license(self) -> str: return self.message.license @license.setter def license(self, license: str): self.message.license = license @property def license_url(self) -> str: return self.message.license_url @license_url.setter def license_url(self, license_url: str): self.message.license_url = license_url @property def release_time(self) -> int: return self.message.release_time @release_time.setter def release_time(self, release_time: int): self.message.release_time = release_time @property def fee(self) -> Fee: return Fee(self.message.fee) @property def has_fee(self) -> bool: return self.message.HasField('fee') @property def has_source(self) -> bool: return self.message.HasField('source') @property def source(self) -> Source: return Source(self.message.source) @property def stream_type(self) -> str: return self.message.WhichOneof('type') @property def image(self) -> Image: return Image(self.message.image) @property def video(self) -> Video: return Video(self.message.video) @property def audio(self) -> Audio: return Audio(self.message.audio) class Channel(BaseClaim): __slots__ = () claim_type = Claim.CHANNEL object_fields = BaseClaim.object_fields + ('cover',) repeat_fields = BaseClaim.repeat_fields + ('featured',) def to_dict(self): claim = super().to_dict() claim['public_key'] = self.public_key if 'featured' in claim: claim['featured'] = self.featured.ids return claim @property def public_key(self) -> str: return hexlify(self.public_key_bytes).decode() @public_key.setter def public_key(self, sd_public_key: str): self.message.public_key = unhexlify(sd_public_key.encode()) @property def public_key_bytes(self) -> bytes: if len(self.message.public_key) == 33: return self.message.public_key public_key_info = PublicKeyInfo.load(self.message.public_key) public_key = cPublicKey(public_key_info.native['public_key']) return public_key.format(compressed=True) @public_key_bytes.setter def public_key_bytes(self, public_key: bytes): self.message.public_key = public_key @property def email(self) -> str: return self.message.email @email.setter def email(self, email: str): self.message.email = email @property def website_url(self) -> str: return self.message.website_url @website_url.setter def website_url(self, website_url: str): self.message.website_url = website_url @property def cover(self) -> Source: return Source(self.message.cover) @property def featured(self) -> ClaimList: return ClaimList(self.message.featured) class Repost(BaseClaim): __slots__ = () claim_type = Claim.REPOST def to_dict(self): claim = super().to_dict() if claim.pop('claim_hash', None): claim['claim_id'] = self.reference.claim_id return claim @property def reference(self) -> ClaimReference: return ClaimReference(self.message) class Collection(BaseClaim): __slots__ = () claim_type = Claim.COLLECTION repeat_fields = BaseClaim.repeat_fields + ('claims',) def to_dict(self): claim = super().to_dict() if claim.pop('claim_references', None): claim['claims'] = self.claims.ids return claim @property def claims(self) -> ClaimList: return ClaimList(self.message) ================================================ FILE: lbry/schema/compat.py ================================================ import json from decimal import Decimal from google.protobuf.message import DecodeError from lbry.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage from lbry.schema.types.v1.certificate_pb2 import KeyType from lbry.schema.types.v1.fee_pb2 import Fee as FeeMessage def from_old_json_schema(claim, payload: bytes): try: value = json.loads(payload) except: raise DecodeError('Could not parse JSON.') stream = claim.stream stream.source.sd_hash = value['sources']['lbry_sd_hash'] stream.source.media_type = ( value.get('content_type', value.get('content-type')) or 'application/octet-stream' ) stream.title = value.get('title', '') stream.description = value.get('description', '') if value.get('thumbnail', ''): stream.thumbnail.url = value.get('thumbnail', '') stream.author = value.get('author', '') stream.license = value.get('license', '') stream.license_url = value.get('license_url', '') language = value.get('language', '') if language: if language.lower() == 'english': language = 'en' try: stream.languages.append(language) except: pass if value.get('nsfw', False): stream.tags.append('mature') if "fee" in value and isinstance(value['fee'], dict): fee = value["fee"] currency = list(fee.keys())[0] if currency == 'LBC': stream.fee.lbc = Decimal(fee[currency]['amount']) elif currency == 'USD': stream.fee.usd = Decimal(fee[currency]['amount']) elif currency == 'BTC': stream.fee.btc = Decimal(fee[currency]['amount']) else: raise DecodeError(f'Unknown currency: {currency}') stream.fee.address = fee[currency]['address'] return claim def from_types_v1(claim, payload: bytes): old = OldClaimMessage() old.ParseFromString(payload) if old.claimType == 2: channel = claim.channel channel.public_key_bytes = old.certificate.publicKey else: stream = claim.stream stream.title = old.stream.metadata.title stream.description = old.stream.metadata.description stream.author = old.stream.metadata.author stream.license = old.stream.metadata.license stream.license_url = old.stream.metadata.licenseUrl stream.thumbnail.url = old.stream.metadata.thumbnail if old.stream.metadata.HasField('language'): stream.languages.add().message.language = old.stream.metadata.language stream.source.media_type = old.stream.source.contentType stream.source.sd_hash_bytes = old.stream.source.source if old.stream.metadata.nsfw: stream.tags.append('mature') if old.stream.metadata.HasField('fee'): fee = old.stream.metadata.fee stream.fee.address_bytes = fee.address currency = FeeMessage.Currency.Name(fee.currency) if currency == 'LBC': stream.fee.lbc = Decimal(fee.amount) elif currency == 'USD': stream.fee.usd = Decimal(fee.amount) elif currency == 'BTC': stream.fee.btc = Decimal(fee.amount) else: raise DecodeError(f'Unsupported currency: {currency}') if old.HasField('publisherSignature'): sig = old.publisherSignature claim.signature = sig.signature claim.signature_type = KeyType.Name(sig.signatureType) claim.signing_channel_hash = sig.certificateId[::-1] old.ClearField("publisherSignature") claim.unsigned_payload = old.SerializeToString() return claim ================================================ FILE: lbry/schema/mime_types.py ================================================ import os import filetype import logging types_map = { # http://www.iana.org/assignments/media-types # Type mapping for automated metadata extraction (video, audio, image, document, binary, model) '.a': ('application/octet-stream', 'binary'), '.ai': ('application/postscript', 'image'), '.aif': ('audio/x-aiff', 'audio'), '.aifc': ('audio/x-aiff', 'audio'), '.aiff': ('audio/x-aiff', 'audio'), '.au': ('audio/basic', 'audio'), '.avi': ('video/x-msvideo', 'video'), '.bat': ('text/plain', 'document'), '.bcpio': ('application/x-bcpio', 'binary'), '.bin': ('application/octet-stream', 'binary'), '.bmp': ('image/bmp', 'image'), '.c': ('text/plain', 'document'), '.cdf': ('application/x-netcdf', 'binary'), '.cpio': ('application/x-cpio', 'binary'), '.csh': ('application/x-csh', 'binary'), '.css': ('text/css', 'document'), '.csv': ('text/csv', 'document'), '.dll': ('application/octet-stream', 'binary'), '.doc': ('application/msword', 'document'), '.dot': ('application/msword', 'document'), '.dvi': ('application/x-dvi', 'binary'), '.eml': ('message/rfc822', 'document'), '.eps': ('application/postscript', 'document'), '.epub': ('application/epub+zip', 'document'), '.etx': ('text/x-setext', 'document'), '.exe': ('application/octet-stream', 'binary'), '.gif': ('image/gif', 'image'), '.gtar': ('application/x-gtar', 'binary'), '.h': ('text/plain', 'document'), '.hdf': ('application/x-hdf', 'binary'), '.htm': ('text/html', 'document'), '.html': ('text/html', 'document'), '.ico': ('image/vnd.microsoft.icon', 'image'), '.ief': ('image/ief', 'image'), '.iges': ('model/iges', 'model'), '.jpe': ('image/jpeg', 'image'), '.jpeg': ('image/jpeg', 'image'), '.jpg': ('image/jpeg', 'image'), '.js': ('application/javascript', 'document'), '.json': ('application/json', 'document'), '.ksh': ('text/plain', 'document'), '.latex': ('application/x-latex', 'binary'), '.m1v': ('video/mpeg', 'video'), '.m3u': ('application/x-mpegurl', 'audio'), '.m3u8': ('application/x-mpegurl', 'video'), '.man': ('application/x-troff-man', 'document'), '.markdown': ('text/markdown', 'document'), '.md': ('text/markdown', 'document'), '.me': ('application/x-troff-me', 'binary'), '.mht': ('message/rfc822', 'document'), '.mhtml': ('message/rfc822', 'document'), '.mif': ('application/x-mif', 'binary'), '.mov': ('video/quicktime', 'video'), '.movie': ('video/x-sgi-movie', 'video'), '.mp2': ('audio/mpeg', 'audio'), '.mp3': ('audio/mpeg', 'audio'), '.mp4': ('video/mp4', 'video'), '.mpa': ('video/mpeg', 'video'), '.mpd': ('application/dash+xml', 'video'), '.mpe': ('video/mpeg', 'video'), '.mpeg': ('video/mpeg', 'video'), '.mpg': ('video/mpeg', 'video'), '.ms': ('application/x-troff-ms', 'binary'), '.m4s': ('video/iso.segment', 'binary'), '.nc': ('application/x-netcdf', 'binary'), '.nws': ('message/rfc822', 'document'), '.o': ('application/octet-stream', 'binary'), '.obj': ('application/octet-stream', 'model'), '.oda': ('application/oda', 'binary'), '.p12': ('application/x-pkcs12', 'binary'), '.p7c': ('application/pkcs7-mime', 'binary'), '.pbm': ('image/x-portable-bitmap', 'image'), '.pdf': ('application/pdf', 'document'), '.pfx': ('application/x-pkcs12', 'binary'), '.pgm': ('image/x-portable-graymap', 'image'), '.pl': ('text/plain', 'document'), '.png': ('image/png', 'image'), '.pnm': ('image/x-portable-anymap', 'image'), '.pot': ('application/vnd.ms-powerpoint', 'document'), '.ppa': ('application/vnd.ms-powerpoint', 'document'), '.ppm': ('image/x-portable-pixmap', 'image'), '.pps': ('application/vnd.ms-powerpoint', 'document'), '.ppt': ('application/vnd.ms-powerpoint', 'document'), '.ps': ('application/postscript', 'document'), '.pwz': ('application/vnd.ms-powerpoint', 'document'), '.py': ('text/x-python', 'document'), '.pyc': ('application/x-python-code', 'binary'), '.pyo': ('application/x-python-code', 'binary'), '.qt': ('video/quicktime', 'video'), '.ra': ('audio/x-pn-realaudio', 'audio'), '.ram': ('application/x-pn-realaudio', 'audio'), '.ras': ('image/x-cmu-raster', 'image'), '.rdf': ('application/xml', 'binary'), '.rgb': ('image/x-rgb', 'image'), '.roff': ('application/x-troff', 'binary'), '.rtx': ('text/richtext', 'document'), '.sgm': ('text/x-sgml', 'document'), '.sgml': ('text/x-sgml', 'document'), '.sh': ('application/x-sh', 'document'), '.shar': ('application/x-shar', 'binary'), '.snd': ('audio/basic', 'audio'), '.so': ('application/octet-stream', 'binary'), '.src': ('application/x-wais-source', 'binary'), '.stl': ('model/stl', 'model'), '.sv4cpio': ('application/x-sv4cpio', 'binary'), '.sv4crc': ('application/x-sv4crc', 'binary'), '.svg': ('image/svg+xml', 'image'), '.swf': ('application/x-shockwave-flash', 'binary'), '.t': ('application/x-troff', 'binary'), '.tar': ('application/x-tar', 'binary'), '.tcl': ('application/x-tcl', 'binary'), '.tex': ('application/x-tex', 'binary'), '.texi': ('application/x-texinfo', 'binary'), '.texinfo': ('application/x-texinfo', 'binary'), '.tif': ('image/tiff', 'image'), '.tiff': ('image/tiff', 'image'), '.tr': ('application/x-troff', 'binary'), '.ts': ('video/mp2t', 'video'), '.tsv': ('text/tab-separated-values', 'document'), '.txt': ('text/plain', 'document'), '.ustar': ('application/x-ustar', 'binary'), '.vcf': ('text/x-vcard', 'document'), '.vtt': ('text/vtt', 'document'), '.wav': ('audio/x-wav', 'audio'), '.webm': ('video/webm', 'video'), '.wiz': ('application/msword', 'document'), '.wsdl': ('application/xml', 'document'), '.xbm': ('image/x-xbitmap', 'image'), '.xlb': ('application/vnd.ms-excel', 'document'), '.xls': ('application/vnd.ms-excel', 'document'), '.xml': ('text/xml', 'document'), '.xpdl': ('application/xml', 'document'), '.xpm': ('image/x-xpixmap', 'image'), '.xsl': ('application/xml', 'document'), '.xwd': ('image/x-xwindowdump', 'image'), '.zip': ('application/zip', 'binary'), # These are non-standard types, commonly found in the wild. '.cbr': ('application/vnd.comicbook-rar', 'document'), '.cbz': ('application/vnd.comicbook+zip', 'document'), '.flac': ('audio/flac', 'audio'), '.lbry': ('application/x-ext-lbry', 'document'), '.m4a': ('audio/mp4', 'audio'), '.m4v': ('video/m4v', 'video'), '.mid': ('audio/midi', 'audio'), '.midi': ('audio/midi', 'audio'), '.mkv': ('video/x-matroska', 'video'), '.mobi': ('application/x-mobipocket-ebook', 'document'), '.oga': ('audio/ogg', 'audio'), '.ogv': ('video/ogg', 'video'), '.ogg': ('video/ogg', 'video'), '.pct': ('image/pict', 'image'), '.pic': ('image/pict', 'image'), '.pict': ('image/pict', 'image'), '.prc': ('application/x-mobipocket-ebook', 'document'), '.rtf': ('application/rtf', 'document'), '.xul': ('text/xul', 'document'), # microsoft is special and has its own 'standard' # https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions '.wmv': ('video/x-ms-wmv', 'video') } # maps detected extensions to the possible analogs # i.e. .cbz file is actually a .zip synonyms_map = { '.zip': ['.cbz'], '.rar': ['.cbr'], '.ar': ['.a'] } log = logging.getLogger(__name__) def guess_media_type(path): _, ext = os.path.splitext(path) extension = ext.strip().lower() try: kind = filetype.guess(path) if kind: real_extension = f".{kind.extension}" if extension != real_extension: if extension: log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}") else: log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}") if extension not in synonyms_map.get(real_extension, []): extension = real_extension except OSError as error: pass if extension[1:]: if extension in types_map: return types_map[extension] return f'application/x-ext-{extension[1:]}', 'binary' return 'application/octet-stream', 'binary' def guess_stream_type(media_type): for media, stream in types_map.values(): if media == media_type: return stream return 'binary' ================================================ FILE: lbry/schema/purchase.py ================================================ from google.protobuf.message import DecodeError from google.protobuf.json_format import MessageToDict from lbry.schema.types.v2.purchase_pb2 import Purchase as PurchaseMessage from .attrs import ClaimReference class Purchase(ClaimReference): START_BYTE = ord('P') __slots__ = () def __init__(self, claim_id=None): super().__init__(PurchaseMessage()) if claim_id is not None: self.claim_id = claim_id def to_dict(self): return MessageToDict(self.message) def to_message_bytes(self) -> bytes: return self.message.SerializeToString() def to_bytes(self) -> bytes: pieces = bytearray() pieces.append(self.START_BYTE) pieces.extend(self.to_message_bytes()) return bytes(pieces) @classmethod def has_start_byte(cls, data: bytes): return data and data[0] == cls.START_BYTE @classmethod def from_bytes(cls, data: bytes): purchase = cls() if purchase.has_start_byte(data): purchase.message.ParseFromString(data[1:]) else: raise DecodeError('Message does not start with correct byte.') return purchase def __len__(self): return len(self.to_bytes()) def __bytes__(self): return self.to_bytes() ================================================ FILE: lbry/schema/result.py ================================================ import base64 from typing import List, Union, Optional, NamedTuple from binascii import hexlify from itertools import chain from lbry.error import ResolveCensoredError from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage from lbry.schema.types.v2.result_pb2 import Error as ErrorMessage INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID) NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND) BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED) def set_reference(reference, claim_hash, rows): if claim_hash: for txo in rows: if claim_hash == txo.claim_hash: reference.tx_hash = txo.tx_hash reference.nout = txo.position reference.height = txo.height return class ResolveResult(NamedTuple): name: str normalized_name: str claim_hash: bytes tx_num: int position: int tx_hash: bytes height: int amount: int short_url: str is_controlling: bool canonical_url: str creation_height: int activation_height: int expiration_height: int effective_amount: int support_amount: int reposted: int last_takeover_height: Optional[int] claims_in_channel: Optional[int] channel_hash: Optional[bytes] reposted_claim_hash: Optional[bytes] signature_valid: Optional[bool] class Censor: NOT_CENSORED = 0 SEARCH = 1 RESOLVE = 2 __slots__ = 'censor_type', 'censored' def __init__(self, censor_type): self.censor_type = censor_type self.censored = {} def is_censored(self, row): return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type def apply(self, rows): return [row for row in rows if not self.censor(row)] def censor(self, row) -> Optional[bytes]: if self.is_censored(row): censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1] self.censored.setdefault(censoring_channel_hash, set()) self.censored[censoring_channel_hash].add(row['tx_hash']) return censoring_channel_hash return None def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict): for censoring_channel_hash, count in self.censored.items(): blocked = outputs.blocked.add() blocked.count = len(count) set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows) outputs.blocked_total += len(count) class Outputs: __slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total', 'blocked', 'blocked_total' def __init__(self, txos: List, extra_txos: List, txs: set, offset: int, total: int, blocked: List, blocked_total: int): self.txos = txos self.txs = txs self.extra_txos = extra_txos self.offset = offset self.total = total self.blocked = blocked self.blocked_total = blocked_total def inflate(self, txs): tx_map = {tx.hash: tx for tx in txs} for txo_message in self.extra_txos: self.message_to_txo(txo_message, tx_map) txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos] return txos, self.inflate_blocked(tx_map) def inflate_blocked(self, tx_map): return { "total": self.blocked_total, "channels": [{ 'channel': self.message_to_txo(blocked.channel, tx_map), 'blocked': blocked.count } for blocked in self.blocked] } def message_to_txo(self, txo_message, tx_map): if txo_message.WhichOneof('meta') == 'error': error = { 'error': { 'name': txo_message.error.Code.Name(txo_message.error.code), 'text': txo_message.error.text, } } if error['error']['name'] == BLOCKED: error['error']['censor'] = self.message_to_txo( txo_message.error.blocked.channel, tx_map ) return error tx = tx_map.get(txo_message.tx_hash) if not tx: return txo = tx.outputs[txo_message.nout] if txo_message.WhichOneof('meta') == 'claim': claim = txo_message.claim txo.meta = { 'short_url': f'lbry://{claim.short_url}', 'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}', 'reposted': claim.reposted, 'is_controlling': claim.is_controlling, 'take_over_height': claim.take_over_height, 'creation_height': claim.creation_height, 'activation_height': claim.activation_height, 'expiration_height': claim.expiration_height, 'effective_amount': claim.effective_amount, 'support_amount': claim.support_amount, # 'trending_group': claim.trending_group, # 'trending_mixed': claim.trending_mixed, # 'trending_local': claim.trending_local, # 'trending_global': claim.trending_global, } if claim.HasField('channel'): txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout] if claim.HasField('repost'): txo.reposted_claim = tx_map[claim.repost.tx_hash].outputs[claim.repost.nout] try: if txo.claim.is_channel: txo.meta['claims_in_channel'] = claim.claims_in_channel except: pass return txo @classmethod def from_base64(cls, data: str) -> 'Outputs': return cls.from_bytes(base64.b64decode(data)) @classmethod def from_bytes(cls, data: bytes) -> 'Outputs': outputs = OutputsMessage() outputs.ParseFromString(data) txs = set() for txo_message in chain(outputs.txos, outputs.extra_txos): if txo_message.WhichOneof('meta') == 'error': continue txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height)) return cls( outputs.txos, outputs.extra_txos, txs, outputs.offset, outputs.total, outputs.blocked, outputs.blocked_total ) @classmethod def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str: return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode() @classmethod def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes: page = OutputsMessage() page.offset = offset if total is not None: page.total = total if blocked is not None: blocked.to_message(page, extra_txo_rows) for row in extra_txo_rows: txo_message: 'OutputsMessage' = page.extra_txos.add() if not isinstance(row, Exception): if row.channel_hash: set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows) if row.reposted_claim_hash: set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows) cls.encode_txo(txo_message, row) for row in txo_rows: # cls.row_to_message(row, page.txos.add(), extra_txo_rows) txo_message: 'OutputsMessage' = page.txos.add() cls.encode_txo(txo_message, row) if not isinstance(row, Exception): if row.channel_hash: set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows) if row.reposted_claim_hash: set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows) elif isinstance(row, ResolveCensoredError): set_reference(txo_message.error.blocked.channel, row.censor_id, extra_txo_rows) return page.SerializeToString() @classmethod def encode_txo(cls, txo_message, resolve_result: Union['ResolveResult', Exception]): if isinstance(resolve_result, Exception): txo_message.error.text = resolve_result.args[0] if isinstance(resolve_result, ValueError): txo_message.error.code = ErrorMessage.INVALID elif isinstance(resolve_result, LookupError): txo_message.error.code = ErrorMessage.NOT_FOUND elif isinstance(resolve_result, ResolveCensoredError): txo_message.error.code = ErrorMessage.BLOCKED return txo_message.tx_hash = resolve_result.tx_hash txo_message.nout = resolve_result.position txo_message.height = resolve_result.height txo_message.claim.short_url = resolve_result.short_url txo_message.claim.reposted = resolve_result.reposted txo_message.claim.is_controlling = resolve_result.is_controlling txo_message.claim.creation_height = resolve_result.creation_height txo_message.claim.activation_height = resolve_result.activation_height txo_message.claim.expiration_height = resolve_result.expiration_height txo_message.claim.effective_amount = resolve_result.effective_amount txo_message.claim.support_amount = resolve_result.support_amount if resolve_result.canonical_url is not None: txo_message.claim.canonical_url = resolve_result.canonical_url if resolve_result.last_takeover_height is not None: txo_message.claim.take_over_height = resolve_result.last_takeover_height if resolve_result.claims_in_channel is not None: txo_message.claim.claims_in_channel = resolve_result.claims_in_channel ================================================ FILE: lbry/schema/support.py ================================================ from lbry.schema.base import Signable from lbry.schema.types.v2.support_pb2 import Support as SupportMessage class Support(Signable): __slots__ = () message_class = SupportMessage @property def emoji(self) -> str: return self.message.emoji @emoji.setter def emoji(self, emoji: str): self.message.emoji = emoji @property def comment(self) -> str: return self.message.comment @comment.setter def comment(self, comment: str): self.message.comment = comment ================================================ FILE: lbry/schema/tags.py ================================================ from typing import List import re MULTI_SPACE_RE = re.compile(r"\s{2,}") WEIRD_CHARS_RE = re.compile(r"[#!~]") def normalize_tag(tag: str): return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip() def clean_tags(tags: List[str]): return [tag for tag in {normalize_tag(tag) for tag in tags} if tag] ================================================ FILE: lbry/schema/types/__init__.py ================================================ ================================================ FILE: lbry/schema/types/v1/__init__.py ================================================ ================================================ FILE: lbry/schema/types/v1/certificate_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: certificate.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='certificate.proto', package='legacy_pb', syntax='proto2', serialized_options=None, serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03') ) _KEYTYPE = _descriptor.EnumDescriptor( name='KeyType', full_name='legacy_pb.KeyType', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_PUBLIC_KEY_TYPE', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='NIST256p', index=1, number=1, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='NIST384p', index=2, number=2, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='SECP256k1', index=3, number=3, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=197, serialized_end=278, ) _sym_db.RegisterEnumDescriptor(_KEYTYPE) KeyType = enum_type_wrapper.EnumTypeWrapper(_KEYTYPE) UNKNOWN_PUBLIC_KEY_TYPE = 0 NIST256p = 1 NIST384p = 2 SECP256k1 = 3 _CERTIFICATE_VERSION = _descriptor.EnumDescriptor( name='Version', full_name='legacy_pb.Certificate.Version', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_VERSION', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_1', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=153, serialized_end=195, ) _sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION) _CERTIFICATE = _descriptor.Descriptor( name='Certificate', full_name='legacy_pb.Certificate', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='version', full_name='legacy_pb.Certificate.version', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='keyType', full_name='legacy_pb.Certificate.keyType', index=1, number=2, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2, number=4, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ _CERTIFICATE_VERSION, ], serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=33, serialized_end=195, ) _CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION _CERTIFICATE.fields_by_name['keyType'].enum_type = _KEYTYPE _CERTIFICATE_VERSION.containing_type = _CERTIFICATE DESCRIPTOR.message_types_by_name['Certificate'] = _CERTIFICATE DESCRIPTOR.enum_types_by_name['KeyType'] = _KEYTYPE _sym_db.RegisterFileDescriptor(DESCRIPTOR) Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict( DESCRIPTOR = _CERTIFICATE, __module__ = 'certificate_pb2' # @@protoc_insertion_point(class_scope:legacy_pb.Certificate) )) _sym_db.RegisterMessage(Certificate) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v1/fee_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: fee.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='fee.proto', package='legacy_pb', syntax='proto2', serialized_options=None, serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03') ) _FEE_VERSION = _descriptor.EnumDescriptor( name='Version', full_name='legacy_pb.Fee.Version', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_VERSION', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_1', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=149, serialized_end=191, ) _sym_db.RegisterEnumDescriptor(_FEE_VERSION) _FEE_CURRENCY = _descriptor.EnumDescriptor( name='Currency', full_name='legacy_pb.Fee.Currency', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_CURRENCY', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='LBC', index=1, number=1, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='BTC', index=2, number=2, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='USD', index=3, number=3, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=193, serialized_end=252, ) _sym_db.RegisterEnumDescriptor(_FEE_CURRENCY) _FEE = _descriptor.Descriptor( name='Fee', full_name='legacy_pb.Fee', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='version', full_name='legacy_pb.Fee.version', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='currency', full_name='legacy_pb.Fee.currency', index=1, number=2, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='address', full_name='legacy_pb.Fee.address', index=2, number=3, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='amount', full_name='legacy_pb.Fee.amount', index=3, number=4, type=2, cpp_type=6, label=2, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ _FEE_VERSION, _FEE_CURRENCY, ], serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=25, serialized_end=252, ) _FEE.fields_by_name['version'].enum_type = _FEE_VERSION _FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY _FEE_VERSION.containing_type = _FEE _FEE_CURRENCY.containing_type = _FEE DESCRIPTOR.message_types_by_name['Fee'] = _FEE _sym_db.RegisterFileDescriptor(DESCRIPTOR) Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict( DESCRIPTOR = _FEE, __module__ = 'fee_pb2' # @@protoc_insertion_point(class_scope:legacy_pb.Fee) )) _sym_db.RegisterMessage(Fee) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v1/legacy_claim_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: legacy_claim.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from . import stream_pb2 as stream__pb2 from . import certificate_pb2 as certificate__pb2 from . import signature_pb2 as signature__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='legacy_claim.proto', package='legacy_pb', syntax='proto2', serialized_options=None, serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02') , dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,]) _CLAIM_VERSION = _descriptor.EnumDescriptor( name='Version', full_name='legacy_pb.Claim.Version', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_VERSION', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_1', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=313, serialized_end=355, ) _sym_db.RegisterEnumDescriptor(_CLAIM_VERSION) _CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor( name='ClaimType', full_name='legacy_pb.Claim.ClaimType', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_CLAIM_TYPE', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='streamType', index=1, number=1, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='certificateType', index=2, number=2, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=357, serialized_end=429, ) _sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE) _CLAIM = _descriptor.Descriptor( name='Claim', full_name='legacy_pb.Claim', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='version', full_name='legacy_pb.Claim.version', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='claimType', full_name='legacy_pb.Claim.claimType', index=1, number=2, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='stream', full_name='legacy_pb.Claim.stream', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='certificate', full_name='legacy_pb.Claim.certificate', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ _CLAIM_VERSION, _CLAIM_CLAIMTYPE, ], serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=84, serialized_end=429, ) _CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION _CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE _CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM _CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE _CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE _CLAIM_VERSION.containing_type = _CLAIM _CLAIM_CLAIMTYPE.containing_type = _CLAIM DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM _sym_db.RegisterFileDescriptor(DESCRIPTOR) Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict( DESCRIPTOR = _CLAIM, __module__ = 'legacy_claim_pb2' # @@protoc_insertion_point(class_scope:legacy_pb.Claim) )) _sym_db.RegisterMessage(Claim) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v1/metadata_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: metadata.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from . import fee_pb2 as fee__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='metadata.proto', package='legacy_pb', syntax='proto2', serialized_options=None, serialized_pb=_b('\n\x0emetadata.proto\x12\tlegacy_pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12,\n\x07version\x18\x01 \x02(\x0e\x32\x1b.legacy_pb.Metadata.Version\x12.\n\x08language\x18\x02 \x02(\x0e\x32\x1c.legacy_pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x1b\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x0e.legacy_pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x10\xa9\x01\x12\x07\n\x02ug\x10\xaa\x01\x12\x07\n\x02uk\x10\xab\x01\x12\x07\n\x02ur\x10\xac\x01\x12\x07\n\x02uz\x10\xad\x01\x12\x07\n\x02ve\x10\xae\x01\x12\x07\n\x02vi\x10\xaf\x01\x12\x07\n\x02vo\x10\xb0\x01\x12\x07\n\x02wa\x10\xb1\x01\x12\x07\n\x02wo\x10\xb2\x01\x12\x07\n\x02xh\x10\xb3\x01\x12\x07\n\x02yi\x10\xb4\x01\x12\x07\n\x02yo\x10\xb5\x01\x12\x07\n\x02za\x10\xb6\x01\x12\x07\n\x02zh\x10\xb7\x01\x12\x07\n\x02zu\x10\xb8\x01') , dependencies=[fee__pb2.DESCRIPTOR,]) _METADATA_VERSION = _descriptor.EnumDescriptor( name='Version', full_name='legacy_pb.Metadata.Version', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_VERSION', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_1', index=1, number=1, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_2', index=2, number=2, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_3', index=3, number=3, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_1_0', index=4, number=4, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=315, serialized_end=393, ) _sym_db.RegisterEnumDescriptor(_METADATA_VERSION) _METADATA_LANGUAGE = _descriptor.EnumDescriptor( name='Language', full_name='legacy_pb.Metadata.Language', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_LANGUAGE', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='en', index=1, number=1, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='aa', index=2, number=2, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ab', index=3, number=3, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ae', index=4, number=4, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='af', index=5, number=5, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ak', index=6, number=6, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='am', index=7, number=7, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='an', index=8, number=8, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ar', index=9, number=9, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='as', index=10, number=10, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='av', index=11, number=11, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ay', index=12, number=12, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='az', index=13, number=13, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ba', index=14, number=14, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='be', index=15, number=15, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='bg', index=16, number=16, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='bh', index=17, number=17, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='bi', index=18, number=18, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='bm', index=19, number=19, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='bn', index=20, number=20, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='bo', index=21, number=21, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='br', index=22, number=22, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='bs', index=23, number=23, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ca', index=24, number=24, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ce', index=25, number=25, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ch', index=26, number=26, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='co', index=27, number=27, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='cr', index=28, number=28, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='cs', index=29, number=29, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='cu', index=30, number=30, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='cv', index=31, number=31, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='cy', index=32, number=32, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='da', index=33, number=33, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='de', index=34, number=34, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='dv', index=35, number=35, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='dz', index=36, number=36, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ee', index=37, number=37, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='el', index=38, number=38, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='eo', index=39, number=39, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='es', index=40, number=40, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='et', index=41, number=41, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='eu', index=42, number=42, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='fa', index=43, number=43, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ff', index=44, number=44, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='fi', index=45, number=45, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='fj', index=46, number=46, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='fo', index=47, number=47, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='fr', index=48, number=48, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='fy', index=49, number=49, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ga', index=50, number=50, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='gd', index=51, number=51, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='gl', index=52, number=52, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='gn', index=53, number=53, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='gu', index=54, number=54, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='gv', index=55, number=55, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ha', index=56, number=56, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='he', index=57, number=57, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='hi', index=58, number=58, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ho', index=59, number=59, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='hr', index=60, number=60, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ht', index=61, number=61, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='hu', index=62, number=62, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='hy', index=63, number=63, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='hz', index=64, number=64, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ia', index=65, number=65, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='id', index=66, number=66, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ie', index=67, number=67, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ig', index=68, number=68, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ii', index=69, number=69, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ik', index=70, number=70, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='io', index=71, number=71, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='is', index=72, number=72, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='it', index=73, number=73, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='iu', index=74, number=74, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ja', index=75, number=75, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='jv', index=76, number=76, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ka', index=77, number=77, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kg', index=78, number=78, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ki', index=79, number=79, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kj', index=80, number=80, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kk', index=81, number=81, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kl', index=82, number=82, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='km', index=83, number=83, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kn', index=84, number=84, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ko', index=85, number=85, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kr', index=86, number=86, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ks', index=87, number=87, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ku', index=88, number=88, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kv', index=89, number=89, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='kw', index=90, number=90, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ky', index=91, number=91, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='la', index=92, number=92, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='lb', index=93, number=93, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='lg', index=94, number=94, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='li', index=95, number=95, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ln', index=96, number=96, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='lo', index=97, number=97, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='lt', index=98, number=98, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='lu', index=99, number=99, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='lv', index=100, number=100, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='mg', index=101, number=101, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='mh', index=102, number=102, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='mi', index=103, number=103, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='mk', index=104, number=104, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ml', index=105, number=105, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='mn', index=106, number=106, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='mr', index=107, number=107, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ms', index=108, number=108, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='mt', index=109, number=109, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='my', index=110, number=110, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='na', index=111, number=111, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='nb', index=112, number=112, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='nd', index=113, number=113, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ne', index=114, number=114, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ng', index=115, number=115, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='nl', index=116, number=116, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='nn', index=117, number=117, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='no', index=118, number=118, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='nr', index=119, number=119, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='nv', index=120, number=120, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ny', index=121, number=121, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='oc', index=122, number=122, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='oj', index=123, number=123, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='om', index=124, number=124, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='or', index=125, number=125, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='os', index=126, number=126, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='pa', index=127, number=127, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='pi', index=128, number=128, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='pl', index=129, number=129, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ps', index=130, number=130, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='pt', index=131, number=131, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='qu', index=132, number=132, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='rm', index=133, number=133, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='rn', index=134, number=134, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ro', index=135, number=135, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ru', index=136, number=136, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='rw', index=137, number=137, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sa', index=138, number=138, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sc', index=139, number=139, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sd', index=140, number=140, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='se', index=141, number=141, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sg', index=142, number=142, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='si', index=143, number=143, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sk', index=144, number=144, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sl', index=145, number=145, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sm', index=146, number=146, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sn', index=147, number=147, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='so', index=148, number=148, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sq', index=149, number=149, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sr', index=150, number=150, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ss', index=151, number=151, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='st', index=152, number=152, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='su', index=153, number=153, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sv', index=154, number=154, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='sw', index=155, number=155, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ta', index=156, number=156, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='te', index=157, number=157, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='tg', index=158, number=158, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='th', index=159, number=159, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ti', index=160, number=160, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='tk', index=161, number=161, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='tl', index=162, number=162, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='tn', index=163, number=163, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='to', index=164, number=164, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='tr', index=165, number=165, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ts', index=166, number=166, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='tt', index=167, number=167, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='tw', index=168, number=168, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ty', index=169, number=169, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ug', index=170, number=170, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='uk', index=171, number=171, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ur', index=172, number=172, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='uz', index=173, number=173, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ve', index=174, number=174, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='vi', index=175, number=175, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='vo', index=176, number=176, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='wa', index=177, number=177, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='wo', index=178, number=178, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='xh', index=179, number=179, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='yi', index=180, number=180, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='yo', index=181, number=181, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='za', index=182, number=182, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='zh', index=183, number=183, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='zu', index=184, number=184, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=396, serialized_end=1957, ) _sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE) _METADATA = _descriptor.Descriptor( name='Metadata', full_name='legacy_pb.Metadata', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='version', full_name='legacy_pb.Metadata.version', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='language', full_name='legacy_pb.Metadata.language', index=1, number=2, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='title', full_name='legacy_pb.Metadata.title', index=2, number=3, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='description', full_name='legacy_pb.Metadata.description', index=3, number=4, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='author', full_name='legacy_pb.Metadata.author', index=4, number=5, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='license', full_name='legacy_pb.Metadata.license', index=5, number=6, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='nsfw', full_name='legacy_pb.Metadata.nsfw', index=6, number=7, type=8, cpp_type=7, label=2, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='fee', full_name='legacy_pb.Metadata.fee', index=7, number=8, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='thumbnail', full_name='legacy_pb.Metadata.thumbnail', index=8, number=9, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='preview', full_name='legacy_pb.Metadata.preview', index=9, number=10, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='licenseUrl', full_name='legacy_pb.Metadata.licenseUrl', index=10, number=11, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ _METADATA_VERSION, _METADATA_LANGUAGE, ], serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=41, serialized_end=1957, ) _METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION _METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE _METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE _METADATA_VERSION.containing_type = _METADATA _METADATA_LANGUAGE.containing_type = _METADATA DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA _sym_db.RegisterFileDescriptor(DESCRIPTOR) Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict( DESCRIPTOR = _METADATA, __module__ = 'metadata_pb2' # @@protoc_insertion_point(class_scope:legacy_pb.Metadata) )) _sym_db.RegisterMessage(Metadata) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v1/signature_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: signature.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from . import certificate_pb2 as certificate__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='signature.proto', package='legacy_pb', syntax='proto2', serialized_options=None, serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01') , dependencies=[certificate__pb2.DESCRIPTOR,]) _SIGNATURE_VERSION = _descriptor.EnumDescriptor( name='Version', full_name='legacy_pb.Signature.Version', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_VERSION', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_1', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=195, serialized_end=237, ) _sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION) _SIGNATURE = _descriptor.Descriptor( name='Signature', full_name='legacy_pb.Signature', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='version', full_name='legacy_pb.Signature.version', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1, number=2, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='signature', full_name='legacy_pb.Signature.signature', index=2, number=3, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3, number=4, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ _SIGNATURE_VERSION, ], serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=50, serialized_end=237, ) _SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION _SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE _SIGNATURE_VERSION.containing_type = _SIGNATURE DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE _sym_db.RegisterFileDescriptor(DESCRIPTOR) Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict( DESCRIPTOR = _SIGNATURE, __module__ = 'signature_pb2' # @@protoc_insertion_point(class_scope:legacy_pb.Signature) )) _sym_db.RegisterMessage(Signature) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v1/source_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: source.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='source.proto', package='legacy_pb', syntax='proto2', serialized_options=None, serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01') ) _SOURCE_VERSION = _descriptor.EnumDescriptor( name='Version', full_name='legacy_pb.Source.Version', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_VERSION', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_1', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=170, serialized_end=212, ) _sym_db.RegisterEnumDescriptor(_SOURCE_VERSION) _SOURCE_SOURCETYPES = _descriptor.EnumDescriptor( name='SourceTypes', full_name='legacy_pb.Source.SourceTypes', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_SOURCE_TYPE', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='lbry_sd_hash', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=214, serialized_end=270, ) _sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES) _SOURCE = _descriptor.Descriptor( name='Source', full_name='legacy_pb.Source', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='version', full_name='legacy_pb.Source.version', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='sourceType', full_name='legacy_pb.Source.sourceType', index=1, number=2, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='source', full_name='legacy_pb.Source.source', index=2, number=3, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='contentType', full_name='legacy_pb.Source.contentType', index=3, number=4, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ _SOURCE_VERSION, _SOURCE_SOURCETYPES, ], serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=28, serialized_end=270, ) _SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION _SOURCE.fields_by_name['sourceType'].enum_type = _SOURCE_SOURCETYPES _SOURCE_VERSION.containing_type = _SOURCE _SOURCE_SOURCETYPES.containing_type = _SOURCE DESCRIPTOR.message_types_by_name['Source'] = _SOURCE _sym_db.RegisterFileDescriptor(DESCRIPTOR) Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict( DESCRIPTOR = _SOURCE, __module__ = 'source_pb2' # @@protoc_insertion_point(class_scope:legacy_pb.Source) )) _sym_db.RegisterMessage(Source) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v1/stream_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: stream.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from . import metadata_pb2 as metadata__pb2 from . import source_pb2 as source__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='stream.proto', package='legacy_pb', syntax='proto2', serialized_options=None, serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01') , dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,]) _STREAM_VERSION = _descriptor.EnumDescriptor( name='Version', full_name='legacy_pb.Stream.Version', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_VERSION', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='_0_0_1', index=1, number=1, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=186, serialized_end=228, ) _sym_db.RegisterEnumDescriptor(_STREAM_VERSION) _STREAM = _descriptor.Descriptor( name='Stream', full_name='legacy_pb.Stream', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='version', full_name='legacy_pb.Stream.version', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='metadata', full_name='legacy_pb.Stream.metadata', index=1, number=2, type=11, cpp_type=10, label=2, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='source', full_name='legacy_pb.Stream.source', index=2, number=3, type=11, cpp_type=10, label=2, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ _STREAM_VERSION, ], serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=58, serialized_end=228, ) _STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION _STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA _STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE _STREAM_VERSION.containing_type = _STREAM DESCRIPTOR.message_types_by_name['Stream'] = _STREAM _sym_db.RegisterFileDescriptor(DESCRIPTOR) Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict( DESCRIPTOR = _STREAM, __module__ = 'stream_pb2' # @@protoc_insertion_point(class_scope:legacy_pb.Stream) )) _sym_db.RegisterMessage(Stream) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v2/__init__.py ================================================ ================================================ FILE: lbry/schema/types/v2/claim_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: claim.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='claim.proto', package='pb', syntax='proto3', serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"\xab\x02\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x12#\n\ncollection\x18\x03 \x01(\x0b\x32\r.pb.ClaimListH\x00\x12$\n\x06repost\x18\x04 \x01(\x0b\x32\x12.pb.ClaimReferenceH\x00\x12\r\n\x05title\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x1d\n\tthumbnail\x18\n \x01(\x0b\x32\n.pb.Source\x12\x0c\n\x04tags\x18\x0b \x03(\t\x12\x1f\n\tlanguages\x18\x0c \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\r \x03(\x0b\x32\x0c.pb.LocationB\x06\n\x04type\"\x84\x02\n\x06Stream\x12\x1a\n\x06source\x18\x01 \x01(\x0b\x32\n.pb.Source\x12\x0e\n\x06\x61uthor\x18\x02 \x01(\t\x12\x0f\n\x07license\x18\x03 \x01(\t\x12\x13\n\x0blicense_url\x18\x04 \x01(\t\x12\x14\n\x0crelease_time\x18\x05 \x01(\x03\x12\x14\n\x03\x66\x65\x65\x18\x06 \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\n \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x0b \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x0c \x01(\x0b\x32\t.pb.AudioH\x00\x12 \n\x08software\x18\r \x01(\x0b\x32\x0c.pb.SoftwareH\x00\x42\x06\n\x04type\"}\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x13\n\x0bwebsite_url\x18\x03 \x01(\t\x12\x19\n\x05\x63over\x18\x04 \x01(\x0b\x32\n.pb.Source\x12\x1f\n\x08\x66\x65\x61tured\x18\x05 \x01(\x0b\x32\r.pb.ClaimList\"$\n\x0e\x43laimReference\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\"\x90\x01\n\tClaimList\x12)\n\tlist_type\x18\x01 \x01(\x0e\x32\x16.pb.ClaimList.ListType\x12,\n\x10\x63laim_references\x18\x02 \x03(\x0b\x32\x12.pb.ClaimReference\"*\n\x08ListType\x12\x0e\n\nCOLLECTION\x10\x00\x12\x0e\n\nDERIVATION\x10\x02\"y\n\x06Source\x12\x0c\n\x04hash\x18\x01 \x01(\x0c\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x12\n\nmedia_type\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07sd_hash\x18\x06 \x01(\x0c\x12\x13\n\x0b\x62t_infohash\x18\x07 \x01(\x0c\"\x87\x01\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"R\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\x12\x18\n\x05\x61udio\x18\x0f \x01(\x0b\x32\t.pb.Audio\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"l\n\x08Software\x12\n\n\x02os\x18\x01 \x01(\t\"T\n\x02OS\x12\x0e\n\nUNKNOWN_OS\x10\x00\x12\x07\n\x03\x41NY\x10\x01\x12\t\n\x05LINUX\x10\x02\x12\x0b\n\x07WINDOWS\x10\x03\x12\x07\n\x03MAC\x10\x04\x12\x0b\n\x07\x41NDROID\x10\x05\x12\x07\n\x03IOS\x10\x06\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x10\xa9\x01\x12\x07\n\x02ug\x10\xaa\x01\x12\x07\n\x02uk\x10\xab\x01\x12\x07\n\x02ur\x10\xac\x01\x12\x07\n\x02uz\x10\xad\x01\x12\x07\n\x02ve\x10\xae\x01\x12\x07\n\x02vi\x10\xaf\x01\x12\x07\n\x02vo\x10\xb0\x01\x12\x07\n\x02wa\x10\xb1\x01\x12\x07\n\x02wo\x10\xb2\x01\x12\x07\n\x02xh\x10\xb3\x01\x12\x07\n\x02yi\x10\xb4\x01\x12\x07\n\x02yo\x10\xb5\x01\x12\x07\n\x02za\x10\xb6\x01\x12\x07\n\x02zh\x10\xb7\x01\x12\x07\n\x02zu\x10\xb8\x01\"\xaa\x10\n\x06Script\x12\x12\n\x0eUNKNOWN_SCRIPT\x10\x00\x12\x08\n\x04\x41\x64lm\x10\x01\x12\x08\n\x04\x41\x66\x61k\x10\x02\x12\x08\n\x04\x41ghb\x10\x03\x12\x08\n\x04\x41hom\x10\x04\x12\x08\n\x04\x41rab\x10\x05\x12\x08\n\x04\x41ran\x10\x06\x12\x08\n\x04\x41rmi\x10\x07\x12\x08\n\x04\x41rmn\x10\x08\x12\x08\n\x04\x41vst\x10\t\x12\x08\n\x04\x42\x61li\x10\n\x12\x08\n\x04\x42\x61mu\x10\x0b\x12\x08\n\x04\x42\x61ss\x10\x0c\x12\x08\n\x04\x42\x61tk\x10\r\x12\x08\n\x04\x42\x65ng\x10\x0e\x12\x08\n\x04\x42hks\x10\x0f\x12\x08\n\x04\x42lis\x10\x10\x12\x08\n\x04\x42opo\x10\x11\x12\x08\n\x04\x42rah\x10\x12\x12\x08\n\x04\x42rai\x10\x13\x12\x08\n\x04\x42ugi\x10\x14\x12\x08\n\x04\x42uhd\x10\x15\x12\x08\n\x04\x43\x61km\x10\x16\x12\x08\n\x04\x43\x61ns\x10\x17\x12\x08\n\x04\x43\x61ri\x10\x18\x12\x08\n\x04\x43ham\x10\x19\x12\x08\n\x04\x43her\x10\x1a\x12\x08\n\x04\x43irt\x10\x1b\x12\x08\n\x04\x43opt\x10\x1c\x12\x08\n\x04\x43pmn\x10\x1d\x12\x08\n\x04\x43prt\x10\x1e\x12\x08\n\x04\x43yrl\x10\x1f\x12\x08\n\x04\x43yrs\x10 \x12\x08\n\x04\x44\x65va\x10!\x12\x08\n\x04\x44ogr\x10\"\x12\x08\n\x04\x44srt\x10#\x12\x08\n\x04\x44upl\x10$\x12\x08\n\x04\x45gyd\x10%\x12\x08\n\x04\x45gyh\x10&\x12\x08\n\x04\x45gyp\x10\'\x12\x08\n\x04\x45lba\x10(\x12\x08\n\x04\x45lym\x10)\x12\x08\n\x04\x45thi\x10*\x12\x08\n\x04Geok\x10+\x12\x08\n\x04Geor\x10,\x12\x08\n\x04Glag\x10-\x12\x08\n\x04Gong\x10.\x12\x08\n\x04Gonm\x10/\x12\x08\n\x04Goth\x10\x30\x12\x08\n\x04Gran\x10\x31\x12\x08\n\x04Grek\x10\x32\x12\x08\n\x04Gujr\x10\x33\x12\x08\n\x04Guru\x10\x34\x12\x08\n\x04Hanb\x10\x35\x12\x08\n\x04Hang\x10\x36\x12\x08\n\x04Hani\x10\x37\x12\x08\n\x04Hano\x10\x38\x12\x08\n\x04Hans\x10\x39\x12\x08\n\x04Hant\x10:\x12\x08\n\x04Hatr\x10;\x12\x08\n\x04Hebr\x10<\x12\x08\n\x04Hira\x10=\x12\x08\n\x04Hluw\x10>\x12\x08\n\x04Hmng\x10?\x12\x08\n\x04Hmnp\x10@\x12\x08\n\x04Hrkt\x10\x41\x12\x08\n\x04Hung\x10\x42\x12\x08\n\x04Inds\x10\x43\x12\x08\n\x04Ital\x10\x44\x12\x08\n\x04Jamo\x10\x45\x12\x08\n\x04Java\x10\x46\x12\x08\n\x04Jpan\x10G\x12\x08\n\x04Jurc\x10H\x12\x08\n\x04Kali\x10I\x12\x08\n\x04Kana\x10J\x12\x08\n\x04Khar\x10K\x12\x08\n\x04Khmr\x10L\x12\x08\n\x04Khoj\x10M\x12\x08\n\x04Kitl\x10N\x12\x08\n\x04Kits\x10O\x12\x08\n\x04Knda\x10P\x12\x08\n\x04Kore\x10Q\x12\x08\n\x04Kpel\x10R\x12\x08\n\x04Kthi\x10S\x12\x08\n\x04Lana\x10T\x12\x08\n\x04Laoo\x10U\x12\x08\n\x04Latf\x10V\x12\x08\n\x04Latg\x10W\x12\x08\n\x04Latn\x10X\x12\x08\n\x04Leke\x10Y\x12\x08\n\x04Lepc\x10Z\x12\x08\n\x04Limb\x10[\x12\x08\n\x04Lina\x10\\\x12\x08\n\x04Linb\x10]\x12\x08\n\x04Lisu\x10^\x12\x08\n\x04Loma\x10_\x12\x08\n\x04Lyci\x10`\x12\x08\n\x04Lydi\x10\x61\x12\x08\n\x04Mahj\x10\x62\x12\x08\n\x04Maka\x10\x63\x12\x08\n\x04Mand\x10\x64\x12\x08\n\x04Mani\x10\x65\x12\x08\n\x04Marc\x10\x66\x12\x08\n\x04Maya\x10g\x12\x08\n\x04Medf\x10h\x12\x08\n\x04Mend\x10i\x12\x08\n\x04Merc\x10j\x12\x08\n\x04Mero\x10k\x12\x08\n\x04Mlym\x10l\x12\x08\n\x04Modi\x10m\x12\x08\n\x04Mong\x10n\x12\x08\n\x04Moon\x10o\x12\x08\n\x04Mroo\x10p\x12\x08\n\x04Mtei\x10q\x12\x08\n\x04Mult\x10r\x12\x08\n\x04Mymr\x10s\x12\x08\n\x04Nand\x10t\x12\x08\n\x04Narb\x10u\x12\x08\n\x04Nbat\x10v\x12\x08\n\x04Newa\x10w\x12\x08\n\x04Nkdb\x10x\x12\x08\n\x04Nkgb\x10y\x12\x08\n\x04Nkoo\x10z\x12\x08\n\x04Nshu\x10{\x12\x08\n\x04Ogam\x10|\x12\x08\n\x04Olck\x10}\x12\x08\n\x04Orkh\x10~\x12\x08\n\x04Orya\x10\x7f\x12\t\n\x04Osge\x10\x80\x01\x12\t\n\x04Osma\x10\x81\x01\x12\t\n\x04Palm\x10\x82\x01\x12\t\n\x04Pauc\x10\x83\x01\x12\t\n\x04Perm\x10\x84\x01\x12\t\n\x04Phag\x10\x85\x01\x12\t\n\x04Phli\x10\x86\x01\x12\t\n\x04Phlp\x10\x87\x01\x12\t\n\x04Phlv\x10\x88\x01\x12\t\n\x04Phnx\x10\x89\x01\x12\t\n\x04Plrd\x10\x8a\x01\x12\t\n\x04Piqd\x10\x8b\x01\x12\t\n\x04Prti\x10\x8c\x01\x12\t\n\x04Qaaa\x10\x8d\x01\x12\t\n\x04Qabx\x10\x8e\x01\x12\t\n\x04Rjng\x10\x8f\x01\x12\t\n\x04Rohg\x10\x90\x01\x12\t\n\x04Roro\x10\x91\x01\x12\t\n\x04Runr\x10\x92\x01\x12\t\n\x04Samr\x10\x93\x01\x12\t\n\x04Sara\x10\x94\x01\x12\t\n\x04Sarb\x10\x95\x01\x12\t\n\x04Saur\x10\x96\x01\x12\t\n\x04Sgnw\x10\x97\x01\x12\t\n\x04Shaw\x10\x98\x01\x12\t\n\x04Shrd\x10\x99\x01\x12\t\n\x04Shui\x10\x9a\x01\x12\t\n\x04Sidd\x10\x9b\x01\x12\t\n\x04Sind\x10\x9c\x01\x12\t\n\x04Sinh\x10\x9d\x01\x12\t\n\x04Sogd\x10\x9e\x01\x12\t\n\x04Sogo\x10\x9f\x01\x12\t\n\x04Sora\x10\xa0\x01\x12\t\n\x04Soyo\x10\xa1\x01\x12\t\n\x04Sund\x10\xa2\x01\x12\t\n\x04Sylo\x10\xa3\x01\x12\t\n\x04Syrc\x10\xa4\x01\x12\t\n\x04Syre\x10\xa5\x01\x12\t\n\x04Syrj\x10\xa6\x01\x12\t\n\x04Syrn\x10\xa7\x01\x12\t\n\x04Tagb\x10\xa8\x01\x12\t\n\x04Takr\x10\xa9\x01\x12\t\n\x04Tale\x10\xaa\x01\x12\t\n\x04Talu\x10\xab\x01\x12\t\n\x04Taml\x10\xac\x01\x12\t\n\x04Tang\x10\xad\x01\x12\t\n\x04Tavt\x10\xae\x01\x12\t\n\x04Telu\x10\xaf\x01\x12\t\n\x04Teng\x10\xb0\x01\x12\t\n\x04Tfng\x10\xb1\x01\x12\t\n\x04Tglg\x10\xb2\x01\x12\t\n\x04Thaa\x10\xb3\x01\x12\t\n\x04Thai\x10\xb4\x01\x12\t\n\x04Tibt\x10\xb5\x01\x12\t\n\x04Tirh\x10\xb6\x01\x12\t\n\x04Ugar\x10\xb7\x01\x12\t\n\x04Vaii\x10\xb8\x01\x12\t\n\x04Visp\x10\xb9\x01\x12\t\n\x04Wara\x10\xba\x01\x12\t\n\x04Wcho\x10\xbb\x01\x12\t\n\x04Wole\x10\xbc\x01\x12\t\n\x04Xpeo\x10\xbd\x01\x12\t\n\x04Xsux\x10\xbe\x01\x12\t\n\x04Yiii\x10\xbf\x01\x12\t\n\x04Zanb\x10\xc0\x01\x12\t\n\x04Zinh\x10\xc1\x01\x12\t\n\x04Zmth\x10\xc2\x01\x12\t\n\x04Zsye\x10\xc3\x01\x12\t\n\x04Zsym\x10\xc4\x01\x12\t\n\x04Zxxx\x10\xc5\x01\x12\t\n\x04Zyyy\x10\xc6\x01\x12\t\n\x04Zzzz\x10\xc7\x01\"\xec)\n\x08Location\x12%\n\x07\x63ountry\x18\x01 \x01(\x0e\x32\x14.pb.Location.Country\x12\r\n\x05state\x18\x02 \x01(\t\x12\x0c\n\x04\x63ity\x18\x03 \x01(\t\x12\x0c\n\x04\x63ode\x18\x04 \x01(\t\x12\x10\n\x08latitude\x18\x05 \x01(\x11\x12\x11\n\tlongitude\x18\x06 \x01(\x11\"\xe8(\n\x07\x43ountry\x12\x13\n\x0fUNKNOWN_COUNTRY\x10\x00\x12\x06\n\x02\x41\x46\x10\x01\x12\x06\n\x02\x41X\x10\x02\x12\x06\n\x02\x41L\x10\x03\x12\x06\n\x02\x44Z\x10\x04\x12\x06\n\x02\x41S\x10\x05\x12\x06\n\x02\x41\x44\x10\x06\x12\x06\n\x02\x41O\x10\x07\x12\x06\n\x02\x41I\x10\x08\x12\x06\n\x02\x41Q\x10\t\x12\x06\n\x02\x41G\x10\n\x12\x06\n\x02\x41R\x10\x0b\x12\x06\n\x02\x41M\x10\x0c\x12\x06\n\x02\x41W\x10\r\x12\x06\n\x02\x41U\x10\x0e\x12\x06\n\x02\x41T\x10\x0f\x12\x06\n\x02\x41Z\x10\x10\x12\x06\n\x02\x42S\x10\x11\x12\x06\n\x02\x42H\x10\x12\x12\x06\n\x02\x42\x44\x10\x13\x12\x06\n\x02\x42\x42\x10\x14\x12\x06\n\x02\x42Y\x10\x15\x12\x06\n\x02\x42\x45\x10\x16\x12\x06\n\x02\x42Z\x10\x17\x12\x06\n\x02\x42J\x10\x18\x12\x06\n\x02\x42M\x10\x19\x12\x06\n\x02\x42T\x10\x1a\x12\x06\n\x02\x42O\x10\x1b\x12\x06\n\x02\x42Q\x10\x1c\x12\x06\n\x02\x42\x41\x10\x1d\x12\x06\n\x02\x42W\x10\x1e\x12\x06\n\x02\x42V\x10\x1f\x12\x06\n\x02\x42R\x10 \x12\x06\n\x02IO\x10!\x12\x06\n\x02\x42N\x10\"\x12\x06\n\x02\x42G\x10#\x12\x06\n\x02\x42\x46\x10$\x12\x06\n\x02\x42I\x10%\x12\x06\n\x02KH\x10&\x12\x06\n\x02\x43M\x10\'\x12\x06\n\x02\x43\x41\x10(\x12\x06\n\x02\x43V\x10)\x12\x06\n\x02KY\x10*\x12\x06\n\x02\x43\x46\x10+\x12\x06\n\x02TD\x10,\x12\x06\n\x02\x43L\x10-\x12\x06\n\x02\x43N\x10.\x12\x06\n\x02\x43X\x10/\x12\x06\n\x02\x43\x43\x10\x30\x12\x06\n\x02\x43O\x10\x31\x12\x06\n\x02KM\x10\x32\x12\x06\n\x02\x43G\x10\x33\x12\x06\n\x02\x43\x44\x10\x34\x12\x06\n\x02\x43K\x10\x35\x12\x06\n\x02\x43R\x10\x36\x12\x06\n\x02\x43I\x10\x37\x12\x06\n\x02HR\x10\x38\x12\x06\n\x02\x43U\x10\x39\x12\x06\n\x02\x43W\x10:\x12\x06\n\x02\x43Y\x10;\x12\x06\n\x02\x43Z\x10<\x12\x06\n\x02\x44K\x10=\x12\x06\n\x02\x44J\x10>\x12\x06\n\x02\x44M\x10?\x12\x06\n\x02\x44O\x10@\x12\x06\n\x02\x45\x43\x10\x41\x12\x06\n\x02\x45G\x10\x42\x12\x06\n\x02SV\x10\x43\x12\x06\n\x02GQ\x10\x44\x12\x06\n\x02\x45R\x10\x45\x12\x06\n\x02\x45\x45\x10\x46\x12\x06\n\x02\x45T\x10G\x12\x06\n\x02\x46K\x10H\x12\x06\n\x02\x46O\x10I\x12\x06\n\x02\x46J\x10J\x12\x06\n\x02\x46I\x10K\x12\x06\n\x02\x46R\x10L\x12\x06\n\x02GF\x10M\x12\x06\n\x02PF\x10N\x12\x06\n\x02TF\x10O\x12\x06\n\x02GA\x10P\x12\x06\n\x02GM\x10Q\x12\x06\n\x02GE\x10R\x12\x06\n\x02\x44\x45\x10S\x12\x06\n\x02GH\x10T\x12\x06\n\x02GI\x10U\x12\x06\n\x02GR\x10V\x12\x06\n\x02GL\x10W\x12\x06\n\x02GD\x10X\x12\x06\n\x02GP\x10Y\x12\x06\n\x02GU\x10Z\x12\x06\n\x02GT\x10[\x12\x06\n\x02GG\x10\\\x12\x06\n\x02GN\x10]\x12\x06\n\x02GW\x10^\x12\x06\n\x02GY\x10_\x12\x06\n\x02HT\x10`\x12\x06\n\x02HM\x10\x61\x12\x06\n\x02VA\x10\x62\x12\x06\n\x02HN\x10\x63\x12\x06\n\x02HK\x10\x64\x12\x06\n\x02HU\x10\x65\x12\x06\n\x02IS\x10\x66\x12\x06\n\x02IN\x10g\x12\x06\n\x02ID\x10h\x12\x06\n\x02IR\x10i\x12\x06\n\x02IQ\x10j\x12\x06\n\x02IE\x10k\x12\x06\n\x02IM\x10l\x12\x06\n\x02IL\x10m\x12\x06\n\x02IT\x10n\x12\x06\n\x02JM\x10o\x12\x06\n\x02JP\x10p\x12\x06\n\x02JE\x10q\x12\x06\n\x02JO\x10r\x12\x06\n\x02KZ\x10s\x12\x06\n\x02KE\x10t\x12\x06\n\x02KI\x10u\x12\x06\n\x02KP\x10v\x12\x06\n\x02KR\x10w\x12\x06\n\x02KW\x10x\x12\x06\n\x02KG\x10y\x12\x06\n\x02LA\x10z\x12\x06\n\x02LV\x10{\x12\x06\n\x02LB\x10|\x12\x06\n\x02LS\x10}\x12\x06\n\x02LR\x10~\x12\x06\n\x02LY\x10\x7f\x12\x07\n\x02LI\x10\x80\x01\x12\x07\n\x02LT\x10\x81\x01\x12\x07\n\x02LU\x10\x82\x01\x12\x07\n\x02MO\x10\x83\x01\x12\x07\n\x02MK\x10\x84\x01\x12\x07\n\x02MG\x10\x85\x01\x12\x07\n\x02MW\x10\x86\x01\x12\x07\n\x02MY\x10\x87\x01\x12\x07\n\x02MV\x10\x88\x01\x12\x07\n\x02ML\x10\x89\x01\x12\x07\n\x02MT\x10\x8a\x01\x12\x07\n\x02MH\x10\x8b\x01\x12\x07\n\x02MQ\x10\x8c\x01\x12\x07\n\x02MR\x10\x8d\x01\x12\x07\n\x02MU\x10\x8e\x01\x12\x07\n\x02YT\x10\x8f\x01\x12\x07\n\x02MX\x10\x90\x01\x12\x07\n\x02\x46M\x10\x91\x01\x12\x07\n\x02MD\x10\x92\x01\x12\x07\n\x02MC\x10\x93\x01\x12\x07\n\x02MN\x10\x94\x01\x12\x07\n\x02ME\x10\x95\x01\x12\x07\n\x02MS\x10\x96\x01\x12\x07\n\x02MA\x10\x97\x01\x12\x07\n\x02MZ\x10\x98\x01\x12\x07\n\x02MM\x10\x99\x01\x12\x07\n\x02NA\x10\x9a\x01\x12\x07\n\x02NR\x10\x9b\x01\x12\x07\n\x02NP\x10\x9c\x01\x12\x07\n\x02NL\x10\x9d\x01\x12\x07\n\x02NC\x10\x9e\x01\x12\x07\n\x02NZ\x10\x9f\x01\x12\x07\n\x02NI\x10\xa0\x01\x12\x07\n\x02NE\x10\xa1\x01\x12\x07\n\x02NG\x10\xa2\x01\x12\x07\n\x02NU\x10\xa3\x01\x12\x07\n\x02NF\x10\xa4\x01\x12\x07\n\x02MP\x10\xa5\x01\x12\x07\n\x02NO\x10\xa6\x01\x12\x07\n\x02OM\x10\xa7\x01\x12\x07\n\x02PK\x10\xa8\x01\x12\x07\n\x02PW\x10\xa9\x01\x12\x07\n\x02PS\x10\xaa\x01\x12\x07\n\x02PA\x10\xab\x01\x12\x07\n\x02PG\x10\xac\x01\x12\x07\n\x02PY\x10\xad\x01\x12\x07\n\x02PE\x10\xae\x01\x12\x07\n\x02PH\x10\xaf\x01\x12\x07\n\x02PN\x10\xb0\x01\x12\x07\n\x02PL\x10\xb1\x01\x12\x07\n\x02PT\x10\xb2\x01\x12\x07\n\x02PR\x10\xb3\x01\x12\x07\n\x02QA\x10\xb4\x01\x12\x07\n\x02RE\x10\xb5\x01\x12\x07\n\x02RO\x10\xb6\x01\x12\x07\n\x02RU\x10\xb7\x01\x12\x07\n\x02RW\x10\xb8\x01\x12\x07\n\x02\x42L\x10\xb9\x01\x12\x07\n\x02SH\x10\xba\x01\x12\x07\n\x02KN\x10\xbb\x01\x12\x07\n\x02LC\x10\xbc\x01\x12\x07\n\x02MF\x10\xbd\x01\x12\x07\n\x02PM\x10\xbe\x01\x12\x07\n\x02VC\x10\xbf\x01\x12\x07\n\x02WS\x10\xc0\x01\x12\x07\n\x02SM\x10\xc1\x01\x12\x07\n\x02ST\x10\xc2\x01\x12\x07\n\x02SA\x10\xc3\x01\x12\x07\n\x02SN\x10\xc4\x01\x12\x07\n\x02RS\x10\xc5\x01\x12\x07\n\x02SC\x10\xc6\x01\x12\x07\n\x02SL\x10\xc7\x01\x12\x07\n\x02SG\x10\xc8\x01\x12\x07\n\x02SX\x10\xc9\x01\x12\x07\n\x02SK\x10\xca\x01\x12\x07\n\x02SI\x10\xcb\x01\x12\x07\n\x02SB\x10\xcc\x01\x12\x07\n\x02SO\x10\xcd\x01\x12\x07\n\x02ZA\x10\xce\x01\x12\x07\n\x02GS\x10\xcf\x01\x12\x07\n\x02SS\x10\xd0\x01\x12\x07\n\x02\x45S\x10\xd1\x01\x12\x07\n\x02LK\x10\xd2\x01\x12\x07\n\x02SD\x10\xd3\x01\x12\x07\n\x02SR\x10\xd4\x01\x12\x07\n\x02SJ\x10\xd5\x01\x12\x07\n\x02SZ\x10\xd6\x01\x12\x07\n\x02SE\x10\xd7\x01\x12\x07\n\x02\x43H\x10\xd8\x01\x12\x07\n\x02SY\x10\xd9\x01\x12\x07\n\x02TW\x10\xda\x01\x12\x07\n\x02TJ\x10\xdb\x01\x12\x07\n\x02TZ\x10\xdc\x01\x12\x07\n\x02TH\x10\xdd\x01\x12\x07\n\x02TL\x10\xde\x01\x12\x07\n\x02TG\x10\xdf\x01\x12\x07\n\x02TK\x10\xe0\x01\x12\x07\n\x02TO\x10\xe1\x01\x12\x07\n\x02TT\x10\xe2\x01\x12\x07\n\x02TN\x10\xe3\x01\x12\x07\n\x02TR\x10\xe4\x01\x12\x07\n\x02TM\x10\xe5\x01\x12\x07\n\x02TC\x10\xe6\x01\x12\x07\n\x02TV\x10\xe7\x01\x12\x07\n\x02UG\x10\xe8\x01\x12\x07\n\x02UA\x10\xe9\x01\x12\x07\n\x02\x41\x45\x10\xea\x01\x12\x07\n\x02GB\x10\xeb\x01\x12\x07\n\x02US\x10\xec\x01\x12\x07\n\x02UM\x10\xed\x01\x12\x07\n\x02UY\x10\xee\x01\x12\x07\n\x02UZ\x10\xef\x01\x12\x07\n\x02VU\x10\xf0\x01\x12\x07\n\x02VE\x10\xf1\x01\x12\x07\n\x02VN\x10\xf2\x01\x12\x07\n\x02VG\x10\xf3\x01\x12\x07\n\x02VI\x10\xf4\x01\x12\x07\n\x02WF\x10\xf5\x01\x12\x07\n\x02\x45H\x10\xf6\x01\x12\x07\n\x02YE\x10\xf7\x01\x12\x07\n\x02ZM\x10\xf8\x01\x12\x07\n\x02ZW\x10\xf9\x01\x12\t\n\x04R001\x10\xfa\x01\x12\t\n\x04R002\x10\xfb\x01\x12\t\n\x04R015\x10\xfc\x01\x12\t\n\x04R012\x10\xfd\x01\x12\t\n\x04R818\x10\xfe\x01\x12\t\n\x04R434\x10\xff\x01\x12\t\n\x04R504\x10\x80\x02\x12\t\n\x04R729\x10\x81\x02\x12\t\n\x04R788\x10\x82\x02\x12\t\n\x04R732\x10\x83\x02\x12\t\n\x04R202\x10\x84\x02\x12\t\n\x04R014\x10\x85\x02\x12\t\n\x04R086\x10\x86\x02\x12\t\n\x04R108\x10\x87\x02\x12\t\n\x04R174\x10\x88\x02\x12\t\n\x04R262\x10\x89\x02\x12\t\n\x04R232\x10\x8a\x02\x12\t\n\x04R231\x10\x8b\x02\x12\t\n\x04R260\x10\x8c\x02\x12\t\n\x04R404\x10\x8d\x02\x12\t\n\x04R450\x10\x8e\x02\x12\t\n\x04R454\x10\x8f\x02\x12\t\n\x04R480\x10\x90\x02\x12\t\n\x04R175\x10\x91\x02\x12\t\n\x04R508\x10\x92\x02\x12\t\n\x04R638\x10\x93\x02\x12\t\n\x04R646\x10\x94\x02\x12\t\n\x04R690\x10\x95\x02\x12\t\n\x04R706\x10\x96\x02\x12\t\n\x04R728\x10\x97\x02\x12\t\n\x04R800\x10\x98\x02\x12\t\n\x04R834\x10\x99\x02\x12\t\n\x04R894\x10\x9a\x02\x12\t\n\x04R716\x10\x9b\x02\x12\t\n\x04R017\x10\x9c\x02\x12\t\n\x04R024\x10\x9d\x02\x12\t\n\x04R120\x10\x9e\x02\x12\t\n\x04R140\x10\x9f\x02\x12\t\n\x04R148\x10\xa0\x02\x12\t\n\x04R178\x10\xa1\x02\x12\t\n\x04R180\x10\xa2\x02\x12\t\n\x04R226\x10\xa3\x02\x12\t\n\x04R266\x10\xa4\x02\x12\t\n\x04R678\x10\xa5\x02\x12\t\n\x04R018\x10\xa6\x02\x12\t\n\x04R072\x10\xa7\x02\x12\t\n\x04R748\x10\xa8\x02\x12\t\n\x04R426\x10\xa9\x02\x12\t\n\x04R516\x10\xaa\x02\x12\t\n\x04R710\x10\xab\x02\x12\t\n\x04R011\x10\xac\x02\x12\t\n\x04R204\x10\xad\x02\x12\t\n\x04R854\x10\xae\x02\x12\t\n\x04R132\x10\xaf\x02\x12\t\n\x04R384\x10\xb0\x02\x12\t\n\x04R270\x10\xb1\x02\x12\t\n\x04R288\x10\xb2\x02\x12\t\n\x04R324\x10\xb3\x02\x12\t\n\x04R624\x10\xb4\x02\x12\t\n\x04R430\x10\xb5\x02\x12\t\n\x04R466\x10\xb6\x02\x12\t\n\x04R478\x10\xb7\x02\x12\t\n\x04R562\x10\xb8\x02\x12\t\n\x04R566\x10\xb9\x02\x12\t\n\x04R654\x10\xba\x02\x12\t\n\x04R686\x10\xbb\x02\x12\t\n\x04R694\x10\xbc\x02\x12\t\n\x04R768\x10\xbd\x02\x12\t\n\x04R019\x10\xbe\x02\x12\t\n\x04R419\x10\xbf\x02\x12\t\n\x04R029\x10\xc0\x02\x12\t\n\x04R660\x10\xc1\x02\x12\t\n\x04R028\x10\xc2\x02\x12\t\n\x04R533\x10\xc3\x02\x12\t\n\x04R044\x10\xc4\x02\x12\t\n\x04R052\x10\xc5\x02\x12\t\n\x04R535\x10\xc6\x02\x12\t\n\x04R092\x10\xc7\x02\x12\t\n\x04R136\x10\xc8\x02\x12\t\n\x04R192\x10\xc9\x02\x12\t\n\x04R531\x10\xca\x02\x12\t\n\x04R212\x10\xcb\x02\x12\t\n\x04R214\x10\xcc\x02\x12\t\n\x04R308\x10\xcd\x02\x12\t\n\x04R312\x10\xce\x02\x12\t\n\x04R332\x10\xcf\x02\x12\t\n\x04R388\x10\xd0\x02\x12\t\n\x04R474\x10\xd1\x02\x12\t\n\x04R500\x10\xd2\x02\x12\t\n\x04R630\x10\xd3\x02\x12\t\n\x04R652\x10\xd4\x02\x12\t\n\x04R659\x10\xd5\x02\x12\t\n\x04R662\x10\xd6\x02\x12\t\n\x04R663\x10\xd7\x02\x12\t\n\x04R670\x10\xd8\x02\x12\t\n\x04R534\x10\xd9\x02\x12\t\n\x04R780\x10\xda\x02\x12\t\n\x04R796\x10\xdb\x02\x12\t\n\x04R850\x10\xdc\x02\x12\t\n\x04R013\x10\xdd\x02\x12\t\n\x04R084\x10\xde\x02\x12\t\n\x04R188\x10\xdf\x02\x12\t\n\x04R222\x10\xe0\x02\x12\t\n\x04R320\x10\xe1\x02\x12\t\n\x04R340\x10\xe2\x02\x12\t\n\x04R484\x10\xe3\x02\x12\t\n\x04R558\x10\xe4\x02\x12\t\n\x04R591\x10\xe5\x02\x12\t\n\x04R005\x10\xe6\x02\x12\t\n\x04R032\x10\xe7\x02\x12\t\n\x04R068\x10\xe8\x02\x12\t\n\x04R074\x10\xe9\x02\x12\t\n\x04R076\x10\xea\x02\x12\t\n\x04R152\x10\xeb\x02\x12\t\n\x04R170\x10\xec\x02\x12\t\n\x04R218\x10\xed\x02\x12\t\n\x04R238\x10\xee\x02\x12\t\n\x04R254\x10\xef\x02\x12\t\n\x04R328\x10\xf0\x02\x12\t\n\x04R600\x10\xf1\x02\x12\t\n\x04R604\x10\xf2\x02\x12\t\n\x04R239\x10\xf3\x02\x12\t\n\x04R740\x10\xf4\x02\x12\t\n\x04R858\x10\xf5\x02\x12\t\n\x04R862\x10\xf6\x02\x12\t\n\x04R021\x10\xf7\x02\x12\t\n\x04R060\x10\xf8\x02\x12\t\n\x04R124\x10\xf9\x02\x12\t\n\x04R304\x10\xfa\x02\x12\t\n\x04R666\x10\xfb\x02\x12\t\n\x04R840\x10\xfc\x02\x12\t\n\x04R010\x10\xfd\x02\x12\t\n\x04R142\x10\xfe\x02\x12\t\n\x04R143\x10\xff\x02\x12\t\n\x04R398\x10\x80\x03\x12\t\n\x04R417\x10\x81\x03\x12\t\n\x04R762\x10\x82\x03\x12\t\n\x04R795\x10\x83\x03\x12\t\n\x04R860\x10\x84\x03\x12\t\n\x04R030\x10\x85\x03\x12\t\n\x04R156\x10\x86\x03\x12\t\n\x04R344\x10\x87\x03\x12\t\n\x04R446\x10\x88\x03\x12\t\n\x04R408\x10\x89\x03\x12\t\n\x04R392\x10\x8a\x03\x12\t\n\x04R496\x10\x8b\x03\x12\t\n\x04R410\x10\x8c\x03\x12\t\n\x04R035\x10\x8d\x03\x12\t\n\x04R096\x10\x8e\x03\x12\t\n\x04R116\x10\x8f\x03\x12\t\n\x04R360\x10\x90\x03\x12\t\n\x04R418\x10\x91\x03\x12\t\n\x04R458\x10\x92\x03\x12\t\n\x04R104\x10\x93\x03\x12\t\n\x04R608\x10\x94\x03\x12\t\n\x04R702\x10\x95\x03\x12\t\n\x04R764\x10\x96\x03\x12\t\n\x04R626\x10\x97\x03\x12\t\n\x04R704\x10\x98\x03\x12\t\n\x04R034\x10\x99\x03\x12\t\n\x04R004\x10\x9a\x03\x12\t\n\x04R050\x10\x9b\x03\x12\t\n\x04R064\x10\x9c\x03\x12\t\n\x04R356\x10\x9d\x03\x12\t\n\x04R364\x10\x9e\x03\x12\t\n\x04R462\x10\x9f\x03\x12\t\n\x04R524\x10\xa0\x03\x12\t\n\x04R586\x10\xa1\x03\x12\t\n\x04R144\x10\xa2\x03\x12\t\n\x04R145\x10\xa3\x03\x12\t\n\x04R051\x10\xa4\x03\x12\t\n\x04R031\x10\xa5\x03\x12\t\n\x04R048\x10\xa6\x03\x12\t\n\x04R196\x10\xa7\x03\x12\t\n\x04R268\x10\xa8\x03\x12\t\n\x04R368\x10\xa9\x03\x12\t\n\x04R376\x10\xaa\x03\x12\t\n\x04R400\x10\xab\x03\x12\t\n\x04R414\x10\xac\x03\x12\t\n\x04R422\x10\xad\x03\x12\t\n\x04R512\x10\xae\x03\x12\t\n\x04R634\x10\xaf\x03\x12\t\n\x04R682\x10\xb0\x03\x12\t\n\x04R275\x10\xb1\x03\x12\t\n\x04R760\x10\xb2\x03\x12\t\n\x04R792\x10\xb3\x03\x12\t\n\x04R784\x10\xb4\x03\x12\t\n\x04R887\x10\xb5\x03\x12\t\n\x04R150\x10\xb6\x03\x12\t\n\x04R151\x10\xb7\x03\x12\t\n\x04R112\x10\xb8\x03\x12\t\n\x04R100\x10\xb9\x03\x12\t\n\x04R203\x10\xba\x03\x12\t\n\x04R348\x10\xbb\x03\x12\t\n\x04R616\x10\xbc\x03\x12\t\n\x04R498\x10\xbd\x03\x12\t\n\x04R642\x10\xbe\x03\x12\t\n\x04R643\x10\xbf\x03\x12\t\n\x04R703\x10\xc0\x03\x12\t\n\x04R804\x10\xc1\x03\x12\t\n\x04R154\x10\xc2\x03\x12\t\n\x04R248\x10\xc3\x03\x12\t\n\x04R830\x10\xc4\x03\x12\t\n\x04R831\x10\xc5\x03\x12\t\n\x04R832\x10\xc6\x03\x12\t\n\x04R680\x10\xc7\x03\x12\t\n\x04R208\x10\xc8\x03\x12\t\n\x04R233\x10\xc9\x03\x12\t\n\x04R234\x10\xca\x03\x12\t\n\x04R246\x10\xcb\x03\x12\t\n\x04R352\x10\xcc\x03\x12\t\n\x04R372\x10\xcd\x03\x12\t\n\x04R833\x10\xce\x03\x12\t\n\x04R428\x10\xcf\x03\x12\t\n\x04R440\x10\xd0\x03\x12\t\n\x04R578\x10\xd1\x03\x12\t\n\x04R744\x10\xd2\x03\x12\t\n\x04R752\x10\xd3\x03\x12\t\n\x04R826\x10\xd4\x03\x12\t\n\x04R039\x10\xd5\x03\x12\t\n\x04R008\x10\xd6\x03\x12\t\n\x04R020\x10\xd7\x03\x12\t\n\x04R070\x10\xd8\x03\x12\t\n\x04R191\x10\xd9\x03\x12\t\n\x04R292\x10\xda\x03\x12\t\n\x04R300\x10\xdb\x03\x12\t\n\x04R336\x10\xdc\x03\x12\t\n\x04R380\x10\xdd\x03\x12\t\n\x04R470\x10\xde\x03\x12\t\n\x04R499\x10\xdf\x03\x12\t\n\x04R807\x10\xe0\x03\x12\t\n\x04R620\x10\xe1\x03\x12\t\n\x04R674\x10\xe2\x03\x12\t\n\x04R688\x10\xe3\x03\x12\t\n\x04R705\x10\xe4\x03\x12\t\n\x04R724\x10\xe5\x03\x12\t\n\x04R155\x10\xe6\x03\x12\t\n\x04R040\x10\xe7\x03\x12\t\n\x04R056\x10\xe8\x03\x12\t\n\x04R250\x10\xe9\x03\x12\t\n\x04R276\x10\xea\x03\x12\t\n\x04R438\x10\xeb\x03\x12\t\n\x04R442\x10\xec\x03\x12\t\n\x04R492\x10\xed\x03\x12\t\n\x04R528\x10\xee\x03\x12\t\n\x04R756\x10\xef\x03\x12\t\n\x04R009\x10\xf0\x03\x12\t\n\x04R053\x10\xf1\x03\x12\t\n\x04R036\x10\xf2\x03\x12\t\n\x04R162\x10\xf3\x03\x12\t\n\x04R166\x10\xf4\x03\x12\t\n\x04R334\x10\xf5\x03\x12\t\n\x04R554\x10\xf6\x03\x12\t\n\x04R574\x10\xf7\x03\x12\t\n\x04R054\x10\xf8\x03\x12\t\n\x04R242\x10\xf9\x03\x12\t\n\x04R540\x10\xfa\x03\x12\t\n\x04R598\x10\xfb\x03\x12\t\n\x04R090\x10\xfc\x03\x12\t\n\x04R548\x10\xfd\x03\x12\t\n\x04R057\x10\xfe\x03\x12\t\n\x04R316\x10\xff\x03\x12\t\n\x04R296\x10\x80\x04\x12\t\n\x04R584\x10\x81\x04\x12\t\n\x04R583\x10\x82\x04\x12\t\n\x04R520\x10\x83\x04\x12\t\n\x04R580\x10\x84\x04\x12\t\n\x04R585\x10\x85\x04\x12\t\n\x04R581\x10\x86\x04\x12\t\n\x04R061\x10\x87\x04\x12\t\n\x04R016\x10\x88\x04\x12\t\n\x04R184\x10\x89\x04\x12\t\n\x04R258\x10\x8a\x04\x12\t\n\x04R570\x10\x8b\x04\x12\t\n\x04R612\x10\x8c\x04\x12\t\n\x04R882\x10\x8d\x04\x12\t\n\x04R772\x10\x8e\x04\x12\t\n\x04R776\x10\x8f\x04\x12\t\n\x04R798\x10\x90\x04\x12\t\n\x04R876\x10\x91\x04\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _CLAIMLIST_LISTTYPE = _descriptor.EnumDescriptor( name='ListType', full_name='pb.ClaimList.ListType', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='COLLECTION', index=0, number=0, options=None, type=None), _descriptor.EnumValueDescriptor( name='DERIVATION', index=1, number=2, options=None, type=None), ], containing_type=None, options=None, serialized_start=852, serialized_end=894, ) _sym_db.RegisterEnumDescriptor(_CLAIMLIST_LISTTYPE) _FEE_CURRENCY = _descriptor.EnumDescriptor( name='Currency', full_name='pb.Fee.Currency', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_CURRENCY', index=0, number=0, options=None, type=None), _descriptor.EnumValueDescriptor( name='LBC', index=1, number=1, options=None, type=None), _descriptor.EnumValueDescriptor( name='BTC', index=2, number=2, options=None, type=None), _descriptor.EnumValueDescriptor( name='USD', index=3, number=3, options=None, type=None), ], containing_type=None, options=None, serialized_start=1096, serialized_end=1155, ) _sym_db.RegisterEnumDescriptor(_FEE_CURRENCY) _SOFTWARE_OS = _descriptor.EnumDescriptor( name='OS', full_name='pb.Software.OS', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_OS', index=0, number=0, options=None, type=None), _descriptor.EnumValueDescriptor( name='ANY', index=1, number=1, options=None, type=None), _descriptor.EnumValueDescriptor( name='LINUX', index=2, number=2, options=None, type=None), _descriptor.EnumValueDescriptor( name='WINDOWS', index=3, number=3, options=None, type=None), _descriptor.EnumValueDescriptor( name='MAC', index=4, number=4, options=None, type=None), _descriptor.EnumValueDescriptor( name='ANDROID', index=5, number=5, options=None, type=None), _descriptor.EnumValueDescriptor( name='IOS', index=6, number=6, options=None, type=None), ], containing_type=None, options=None, serialized_start=1332, serialized_end=1416, ) _sym_db.RegisterEnumDescriptor(_SOFTWARE_OS) _LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor( name='Language', full_name='pb.Language.Language', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_LANGUAGE', index=0, number=0, options=None, type=None), _descriptor.EnumValueDescriptor( name='en', index=1, number=1, options=None, type=None), _descriptor.EnumValueDescriptor( name='aa', index=2, number=2, options=None, type=None), _descriptor.EnumValueDescriptor( name='ab', index=3, number=3, options=None, type=None), _descriptor.EnumValueDescriptor( name='ae', index=4, number=4, options=None, type=None), _descriptor.EnumValueDescriptor( name='af', index=5, number=5, options=None, type=None), _descriptor.EnumValueDescriptor( name='ak', index=6, number=6, options=None, type=None), _descriptor.EnumValueDescriptor( name='am', index=7, number=7, options=None, type=None), _descriptor.EnumValueDescriptor( name='an', index=8, number=8, options=None, type=None), _descriptor.EnumValueDescriptor( name='ar', index=9, number=9, options=None, type=None), _descriptor.EnumValueDescriptor( name='as', index=10, number=10, options=None, type=None), _descriptor.EnumValueDescriptor( name='av', index=11, number=11, options=None, type=None), _descriptor.EnumValueDescriptor( name='ay', index=12, number=12, options=None, type=None), _descriptor.EnumValueDescriptor( name='az', index=13, number=13, options=None, type=None), _descriptor.EnumValueDescriptor( name='ba', index=14, number=14, options=None, type=None), _descriptor.EnumValueDescriptor( name='be', index=15, number=15, options=None, type=None), _descriptor.EnumValueDescriptor( name='bg', index=16, number=16, options=None, type=None), _descriptor.EnumValueDescriptor( name='bh', index=17, number=17, options=None, type=None), _descriptor.EnumValueDescriptor( name='bi', index=18, number=18, options=None, type=None), _descriptor.EnumValueDescriptor( name='bm', index=19, number=19, options=None, type=None), _descriptor.EnumValueDescriptor( name='bn', index=20, number=20, options=None, type=None), _descriptor.EnumValueDescriptor( name='bo', index=21, number=21, options=None, type=None), _descriptor.EnumValueDescriptor( name='br', index=22, number=22, options=None, type=None), _descriptor.EnumValueDescriptor( name='bs', index=23, number=23, options=None, type=None), _descriptor.EnumValueDescriptor( name='ca', index=24, number=24, options=None, type=None), _descriptor.EnumValueDescriptor( name='ce', index=25, number=25, options=None, type=None), _descriptor.EnumValueDescriptor( name='ch', index=26, number=26, options=None, type=None), _descriptor.EnumValueDescriptor( name='co', index=27, number=27, options=None, type=None), _descriptor.EnumValueDescriptor( name='cr', index=28, number=28, options=None, type=None), _descriptor.EnumValueDescriptor( name='cs', index=29, number=29, options=None, type=None), _descriptor.EnumValueDescriptor( name='cu', index=30, number=30, options=None, type=None), _descriptor.EnumValueDescriptor( name='cv', index=31, number=31, options=None, type=None), _descriptor.EnumValueDescriptor( name='cy', index=32, number=32, options=None, type=None), _descriptor.EnumValueDescriptor( name='da', index=33, number=33, options=None, type=None), _descriptor.EnumValueDescriptor( name='de', index=34, number=34, options=None, type=None), _descriptor.EnumValueDescriptor( name='dv', index=35, number=35, options=None, type=None), _descriptor.EnumValueDescriptor( name='dz', index=36, number=36, options=None, type=None), _descriptor.EnumValueDescriptor( name='ee', index=37, number=37, options=None, type=None), _descriptor.EnumValueDescriptor( name='el', index=38, number=38, options=None, type=None), _descriptor.EnumValueDescriptor( name='eo', index=39, number=39, options=None, type=None), _descriptor.EnumValueDescriptor( name='es', index=40, number=40, options=None, type=None), _descriptor.EnumValueDescriptor( name='et', index=41, number=41, options=None, type=None), _descriptor.EnumValueDescriptor( name='eu', index=42, number=42, options=None, type=None), _descriptor.EnumValueDescriptor( name='fa', index=43, number=43, options=None, type=None), _descriptor.EnumValueDescriptor( name='ff', index=44, number=44, options=None, type=None), _descriptor.EnumValueDescriptor( name='fi', index=45, number=45, options=None, type=None), _descriptor.EnumValueDescriptor( name='fj', index=46, number=46, options=None, type=None), _descriptor.EnumValueDescriptor( name='fo', index=47, number=47, options=None, type=None), _descriptor.EnumValueDescriptor( name='fr', index=48, number=48, options=None, type=None), _descriptor.EnumValueDescriptor( name='fy', index=49, number=49, options=None, type=None), _descriptor.EnumValueDescriptor( name='ga', index=50, number=50, options=None, type=None), _descriptor.EnumValueDescriptor( name='gd', index=51, number=51, options=None, type=None), _descriptor.EnumValueDescriptor( name='gl', index=52, number=52, options=None, type=None), _descriptor.EnumValueDescriptor( name='gn', index=53, number=53, options=None, type=None), _descriptor.EnumValueDescriptor( name='gu', index=54, number=54, options=None, type=None), _descriptor.EnumValueDescriptor( name='gv', index=55, number=55, options=None, type=None), _descriptor.EnumValueDescriptor( name='ha', index=56, number=56, options=None, type=None), _descriptor.EnumValueDescriptor( name='he', index=57, number=57, options=None, type=None), _descriptor.EnumValueDescriptor( name='hi', index=58, number=58, options=None, type=None), _descriptor.EnumValueDescriptor( name='ho', index=59, number=59, options=None, type=None), _descriptor.EnumValueDescriptor( name='hr', index=60, number=60, options=None, type=None), _descriptor.EnumValueDescriptor( name='ht', index=61, number=61, options=None, type=None), _descriptor.EnumValueDescriptor( name='hu', index=62, number=62, options=None, type=None), _descriptor.EnumValueDescriptor( name='hy', index=63, number=63, options=None, type=None), _descriptor.EnumValueDescriptor( name='hz', index=64, number=64, options=None, type=None), _descriptor.EnumValueDescriptor( name='ia', index=65, number=65, options=None, type=None), _descriptor.EnumValueDescriptor( name='id', index=66, number=66, options=None, type=None), _descriptor.EnumValueDescriptor( name='ie', index=67, number=67, options=None, type=None), _descriptor.EnumValueDescriptor( name='ig', index=68, number=68, options=None, type=None), _descriptor.EnumValueDescriptor( name='ii', index=69, number=69, options=None, type=None), _descriptor.EnumValueDescriptor( name='ik', index=70, number=70, options=None, type=None), _descriptor.EnumValueDescriptor( name='io', index=71, number=71, options=None, type=None), _descriptor.EnumValueDescriptor( name='is', index=72, number=72, options=None, type=None), _descriptor.EnumValueDescriptor( name='it', index=73, number=73, options=None, type=None), _descriptor.EnumValueDescriptor( name='iu', index=74, number=74, options=None, type=None), _descriptor.EnumValueDescriptor( name='ja', index=75, number=75, options=None, type=None), _descriptor.EnumValueDescriptor( name='jv', index=76, number=76, options=None, type=None), _descriptor.EnumValueDescriptor( name='ka', index=77, number=77, options=None, type=None), _descriptor.EnumValueDescriptor( name='kg', index=78, number=78, options=None, type=None), _descriptor.EnumValueDescriptor( name='ki', index=79, number=79, options=None, type=None), _descriptor.EnumValueDescriptor( name='kj', index=80, number=80, options=None, type=None), _descriptor.EnumValueDescriptor( name='kk', index=81, number=81, options=None, type=None), _descriptor.EnumValueDescriptor( name='kl', index=82, number=82, options=None, type=None), _descriptor.EnumValueDescriptor( name='km', index=83, number=83, options=None, type=None), _descriptor.EnumValueDescriptor( name='kn', index=84, number=84, options=None, type=None), _descriptor.EnumValueDescriptor( name='ko', index=85, number=85, options=None, type=None), _descriptor.EnumValueDescriptor( name='kr', index=86, number=86, options=None, type=None), _descriptor.EnumValueDescriptor( name='ks', index=87, number=87, options=None, type=None), _descriptor.EnumValueDescriptor( name='ku', index=88, number=88, options=None, type=None), _descriptor.EnumValueDescriptor( name='kv', index=89, number=89, options=None, type=None), _descriptor.EnumValueDescriptor( name='kw', index=90, number=90, options=None, type=None), _descriptor.EnumValueDescriptor( name='ky', index=91, number=91, options=None, type=None), _descriptor.EnumValueDescriptor( name='la', index=92, number=92, options=None, type=None), _descriptor.EnumValueDescriptor( name='lb', index=93, number=93, options=None, type=None), _descriptor.EnumValueDescriptor( name='lg', index=94, number=94, options=None, type=None), _descriptor.EnumValueDescriptor( name='li', index=95, number=95, options=None, type=None), _descriptor.EnumValueDescriptor( name='ln', index=96, number=96, options=None, type=None), _descriptor.EnumValueDescriptor( name='lo', index=97, number=97, options=None, type=None), _descriptor.EnumValueDescriptor( name='lt', index=98, number=98, options=None, type=None), _descriptor.EnumValueDescriptor( name='lu', index=99, number=99, options=None, type=None), _descriptor.EnumValueDescriptor( name='lv', index=100, number=100, options=None, type=None), _descriptor.EnumValueDescriptor( name='mg', index=101, number=101, options=None, type=None), _descriptor.EnumValueDescriptor( name='mh', index=102, number=102, options=None, type=None), _descriptor.EnumValueDescriptor( name='mi', index=103, number=103, options=None, type=None), _descriptor.EnumValueDescriptor( name='mk', index=104, number=104, options=None, type=None), _descriptor.EnumValueDescriptor( name='ml', index=105, number=105, options=None, type=None), _descriptor.EnumValueDescriptor( name='mn', index=106, number=106, options=None, type=None), _descriptor.EnumValueDescriptor( name='mr', index=107, number=107, options=None, type=None), _descriptor.EnumValueDescriptor( name='ms', index=108, number=108, options=None, type=None), _descriptor.EnumValueDescriptor( name='mt', index=109, number=109, options=None, type=None), _descriptor.EnumValueDescriptor( name='my', index=110, number=110, options=None, type=None), _descriptor.EnumValueDescriptor( name='na', index=111, number=111, options=None, type=None), _descriptor.EnumValueDescriptor( name='nb', index=112, number=112, options=None, type=None), _descriptor.EnumValueDescriptor( name='nd', index=113, number=113, options=None, type=None), _descriptor.EnumValueDescriptor( name='ne', index=114, number=114, options=None, type=None), _descriptor.EnumValueDescriptor( name='ng', index=115, number=115, options=None, type=None), _descriptor.EnumValueDescriptor( name='nl', index=116, number=116, options=None, type=None), _descriptor.EnumValueDescriptor( name='nn', index=117, number=117, options=None, type=None), _descriptor.EnumValueDescriptor( name='no', index=118, number=118, options=None, type=None), _descriptor.EnumValueDescriptor( name='nr', index=119, number=119, options=None, type=None), _descriptor.EnumValueDescriptor( name='nv', index=120, number=120, options=None, type=None), _descriptor.EnumValueDescriptor( name='ny', index=121, number=121, options=None, type=None), _descriptor.EnumValueDescriptor( name='oc', index=122, number=122, options=None, type=None), _descriptor.EnumValueDescriptor( name='oj', index=123, number=123, options=None, type=None), _descriptor.EnumValueDescriptor( name='om', index=124, number=124, options=None, type=None), _descriptor.EnumValueDescriptor( name='or', index=125, number=125, options=None, type=None), _descriptor.EnumValueDescriptor( name='os', index=126, number=126, options=None, type=None), _descriptor.EnumValueDescriptor( name='pa', index=127, number=127, options=None, type=None), _descriptor.EnumValueDescriptor( name='pi', index=128, number=128, options=None, type=None), _descriptor.EnumValueDescriptor( name='pl', index=129, number=129, options=None, type=None), _descriptor.EnumValueDescriptor( name='ps', index=130, number=130, options=None, type=None), _descriptor.EnumValueDescriptor( name='pt', index=131, number=131, options=None, type=None), _descriptor.EnumValueDescriptor( name='qu', index=132, number=132, options=None, type=None), _descriptor.EnumValueDescriptor( name='rm', index=133, number=133, options=None, type=None), _descriptor.EnumValueDescriptor( name='rn', index=134, number=134, options=None, type=None), _descriptor.EnumValueDescriptor( name='ro', index=135, number=135, options=None, type=None), _descriptor.EnumValueDescriptor( name='ru', index=136, number=136, options=None, type=None), _descriptor.EnumValueDescriptor( name='rw', index=137, number=137, options=None, type=None), _descriptor.EnumValueDescriptor( name='sa', index=138, number=138, options=None, type=None), _descriptor.EnumValueDescriptor( name='sc', index=139, number=139, options=None, type=None), _descriptor.EnumValueDescriptor( name='sd', index=140, number=140, options=None, type=None), _descriptor.EnumValueDescriptor( name='se', index=141, number=141, options=None, type=None), _descriptor.EnumValueDescriptor( name='sg', index=142, number=142, options=None, type=None), _descriptor.EnumValueDescriptor( name='si', index=143, number=143, options=None, type=None), _descriptor.EnumValueDescriptor( name='sk', index=144, number=144, options=None, type=None), _descriptor.EnumValueDescriptor( name='sl', index=145, number=145, options=None, type=None), _descriptor.EnumValueDescriptor( name='sm', index=146, number=146, options=None, type=None), _descriptor.EnumValueDescriptor( name='sn', index=147, number=147, options=None, type=None), _descriptor.EnumValueDescriptor( name='so', index=148, number=148, options=None, type=None), _descriptor.EnumValueDescriptor( name='sq', index=149, number=149, options=None, type=None), _descriptor.EnumValueDescriptor( name='sr', index=150, number=150, options=None, type=None), _descriptor.EnumValueDescriptor( name='ss', index=151, number=151, options=None, type=None), _descriptor.EnumValueDescriptor( name='st', index=152, number=152, options=None, type=None), _descriptor.EnumValueDescriptor( name='su', index=153, number=153, options=None, type=None), _descriptor.EnumValueDescriptor( name='sv', index=154, number=154, options=None, type=None), _descriptor.EnumValueDescriptor( name='sw', index=155, number=155, options=None, type=None), _descriptor.EnumValueDescriptor( name='ta', index=156, number=156, options=None, type=None), _descriptor.EnumValueDescriptor( name='te', index=157, number=157, options=None, type=None), _descriptor.EnumValueDescriptor( name='tg', index=158, number=158, options=None, type=None), _descriptor.EnumValueDescriptor( name='th', index=159, number=159, options=None, type=None), _descriptor.EnumValueDescriptor( name='ti', index=160, number=160, options=None, type=None), _descriptor.EnumValueDescriptor( name='tk', index=161, number=161, options=None, type=None), _descriptor.EnumValueDescriptor( name='tl', index=162, number=162, options=None, type=None), _descriptor.EnumValueDescriptor( name='tn', index=163, number=163, options=None, type=None), _descriptor.EnumValueDescriptor( name='to', index=164, number=164, options=None, type=None), _descriptor.EnumValueDescriptor( name='tr', index=165, number=165, options=None, type=None), _descriptor.EnumValueDescriptor( name='ts', index=166, number=166, options=None, type=None), _descriptor.EnumValueDescriptor( name='tt', index=167, number=167, options=None, type=None), _descriptor.EnumValueDescriptor( name='tw', index=168, number=168, options=None, type=None), _descriptor.EnumValueDescriptor( name='ty', index=169, number=169, options=None, type=None), _descriptor.EnumValueDescriptor( name='ug', index=170, number=170, options=None, type=None), _descriptor.EnumValueDescriptor( name='uk', index=171, number=171, options=None, type=None), _descriptor.EnumValueDescriptor( name='ur', index=172, number=172, options=None, type=None), _descriptor.EnumValueDescriptor( name='uz', index=173, number=173, options=None, type=None), _descriptor.EnumValueDescriptor( name='ve', index=174, number=174, options=None, type=None), _descriptor.EnumValueDescriptor( name='vi', index=175, number=175, options=None, type=None), _descriptor.EnumValueDescriptor( name='vo', index=176, number=176, options=None, type=None), _descriptor.EnumValueDescriptor( name='wa', index=177, number=177, options=None, type=None), _descriptor.EnumValueDescriptor( name='wo', index=178, number=178, options=None, type=None), _descriptor.EnumValueDescriptor( name='xh', index=179, number=179, options=None, type=None), _descriptor.EnumValueDescriptor( name='yi', index=180, number=180, options=None, type=None), _descriptor.EnumValueDescriptor( name='yo', index=181, number=181, options=None, type=None), _descriptor.EnumValueDescriptor( name='za', index=182, number=182, options=None, type=None), _descriptor.EnumValueDescriptor( name='zh', index=183, number=183, options=None, type=None), _descriptor.EnumValueDescriptor( name='zu', index=184, number=184, options=None, type=None), ], containing_type=None, options=None, serialized_start=1548, serialized_end=3109, ) _sym_db.RegisterEnumDescriptor(_LANGUAGE_LANGUAGE) _LANGUAGE_SCRIPT = _descriptor.EnumDescriptor( name='Script', full_name='pb.Language.Script', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_SCRIPT', index=0, number=0, options=None, type=None), _descriptor.EnumValueDescriptor( name='Adlm', index=1, number=1, options=None, type=None), _descriptor.EnumValueDescriptor( name='Afak', index=2, number=2, options=None, type=None), _descriptor.EnumValueDescriptor( name='Aghb', index=3, number=3, options=None, type=None), _descriptor.EnumValueDescriptor( name='Ahom', index=4, number=4, options=None, type=None), _descriptor.EnumValueDescriptor( name='Arab', index=5, number=5, options=None, type=None), _descriptor.EnumValueDescriptor( name='Aran', index=6, number=6, options=None, type=None), _descriptor.EnumValueDescriptor( name='Armi', index=7, number=7, options=None, type=None), _descriptor.EnumValueDescriptor( name='Armn', index=8, number=8, options=None, type=None), _descriptor.EnumValueDescriptor( name='Avst', index=9, number=9, options=None, type=None), _descriptor.EnumValueDescriptor( name='Bali', index=10, number=10, options=None, type=None), _descriptor.EnumValueDescriptor( name='Bamu', index=11, number=11, options=None, type=None), _descriptor.EnumValueDescriptor( name='Bass', index=12, number=12, options=None, type=None), _descriptor.EnumValueDescriptor( name='Batk', index=13, number=13, options=None, type=None), _descriptor.EnumValueDescriptor( name='Beng', index=14, number=14, options=None, type=None), _descriptor.EnumValueDescriptor( name='Bhks', index=15, number=15, options=None, type=None), _descriptor.EnumValueDescriptor( name='Blis', index=16, number=16, options=None, type=None), _descriptor.EnumValueDescriptor( name='Bopo', index=17, number=17, options=None, type=None), _descriptor.EnumValueDescriptor( name='Brah', index=18, number=18, options=None, type=None), _descriptor.EnumValueDescriptor( name='Brai', index=19, number=19, options=None, type=None), _descriptor.EnumValueDescriptor( name='Bugi', index=20, number=20, options=None, type=None), _descriptor.EnumValueDescriptor( name='Buhd', index=21, number=21, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cakm', index=22, number=22, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cans', index=23, number=23, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cari', index=24, number=24, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cham', index=25, number=25, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cher', index=26, number=26, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cirt', index=27, number=27, options=None, type=None), _descriptor.EnumValueDescriptor( name='Copt', index=28, number=28, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cpmn', index=29, number=29, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cprt', index=30, number=30, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cyrl', index=31, number=31, options=None, type=None), _descriptor.EnumValueDescriptor( name='Cyrs', index=32, number=32, options=None, type=None), _descriptor.EnumValueDescriptor( name='Deva', index=33, number=33, options=None, type=None), _descriptor.EnumValueDescriptor( name='Dogr', index=34, number=34, options=None, type=None), _descriptor.EnumValueDescriptor( name='Dsrt', index=35, number=35, options=None, type=None), _descriptor.EnumValueDescriptor( name='Dupl', index=36, number=36, options=None, type=None), _descriptor.EnumValueDescriptor( name='Egyd', index=37, number=37, options=None, type=None), _descriptor.EnumValueDescriptor( name='Egyh', index=38, number=38, options=None, type=None), _descriptor.EnumValueDescriptor( name='Egyp', index=39, number=39, options=None, type=None), _descriptor.EnumValueDescriptor( name='Elba', index=40, number=40, options=None, type=None), _descriptor.EnumValueDescriptor( name='Elym', index=41, number=41, options=None, type=None), _descriptor.EnumValueDescriptor( name='Ethi', index=42, number=42, options=None, type=None), _descriptor.EnumValueDescriptor( name='Geok', index=43, number=43, options=None, type=None), _descriptor.EnumValueDescriptor( name='Geor', index=44, number=44, options=None, type=None), _descriptor.EnumValueDescriptor( name='Glag', index=45, number=45, options=None, type=None), _descriptor.EnumValueDescriptor( name='Gong', index=46, number=46, options=None, type=None), _descriptor.EnumValueDescriptor( name='Gonm', index=47, number=47, options=None, type=None), _descriptor.EnumValueDescriptor( name='Goth', index=48, number=48, options=None, type=None), _descriptor.EnumValueDescriptor( name='Gran', index=49, number=49, options=None, type=None), _descriptor.EnumValueDescriptor( name='Grek', index=50, number=50, options=None, type=None), _descriptor.EnumValueDescriptor( name='Gujr', index=51, number=51, options=None, type=None), _descriptor.EnumValueDescriptor( name='Guru', index=52, number=52, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hanb', index=53, number=53, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hang', index=54, number=54, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hani', index=55, number=55, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hano', index=56, number=56, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hans', index=57, number=57, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hant', index=58, number=58, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hatr', index=59, number=59, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hebr', index=60, number=60, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hira', index=61, number=61, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hluw', index=62, number=62, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hmng', index=63, number=63, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hmnp', index=64, number=64, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hrkt', index=65, number=65, options=None, type=None), _descriptor.EnumValueDescriptor( name='Hung', index=66, number=66, options=None, type=None), _descriptor.EnumValueDescriptor( name='Inds', index=67, number=67, options=None, type=None), _descriptor.EnumValueDescriptor( name='Ital', index=68, number=68, options=None, type=None), _descriptor.EnumValueDescriptor( name='Jamo', index=69, number=69, options=None, type=None), _descriptor.EnumValueDescriptor( name='Java', index=70, number=70, options=None, type=None), _descriptor.EnumValueDescriptor( name='Jpan', index=71, number=71, options=None, type=None), _descriptor.EnumValueDescriptor( name='Jurc', index=72, number=72, options=None, type=None), _descriptor.EnumValueDescriptor( name='Kali', index=73, number=73, options=None, type=None), _descriptor.EnumValueDescriptor( name='Kana', index=74, number=74, options=None, type=None), _descriptor.EnumValueDescriptor( name='Khar', index=75, number=75, options=None, type=None), _descriptor.EnumValueDescriptor( name='Khmr', index=76, number=76, options=None, type=None), _descriptor.EnumValueDescriptor( name='Khoj', index=77, number=77, options=None, type=None), _descriptor.EnumValueDescriptor( name='Kitl', index=78, number=78, options=None, type=None), _descriptor.EnumValueDescriptor( name='Kits', index=79, number=79, options=None, type=None), _descriptor.EnumValueDescriptor( name='Knda', index=80, number=80, options=None, type=None), _descriptor.EnumValueDescriptor( name='Kore', index=81, number=81, options=None, type=None), _descriptor.EnumValueDescriptor( name='Kpel', index=82, number=82, options=None, type=None), _descriptor.EnumValueDescriptor( name='Kthi', index=83, number=83, options=None, type=None), _descriptor.EnumValueDescriptor( name='Lana', index=84, number=84, options=None, type=None), _descriptor.EnumValueDescriptor( name='Laoo', index=85, number=85, options=None, type=None), _descriptor.EnumValueDescriptor( name='Latf', index=86, number=86, options=None, type=None), _descriptor.EnumValueDescriptor( name='Latg', index=87, number=87, options=None, type=None), _descriptor.EnumValueDescriptor( name='Latn', index=88, number=88, options=None, type=None), _descriptor.EnumValueDescriptor( name='Leke', index=89, number=89, options=None, type=None), _descriptor.EnumValueDescriptor( name='Lepc', index=90, number=90, options=None, type=None), _descriptor.EnumValueDescriptor( name='Limb', index=91, number=91, options=None, type=None), _descriptor.EnumValueDescriptor( name='Lina', index=92, number=92, options=None, type=None), _descriptor.EnumValueDescriptor( name='Linb', index=93, number=93, options=None, type=None), _descriptor.EnumValueDescriptor( name='Lisu', index=94, number=94, options=None, type=None), _descriptor.EnumValueDescriptor( name='Loma', index=95, number=95, options=None, type=None), _descriptor.EnumValueDescriptor( name='Lyci', index=96, number=96, options=None, type=None), _descriptor.EnumValueDescriptor( name='Lydi', index=97, number=97, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mahj', index=98, number=98, options=None, type=None), _descriptor.EnumValueDescriptor( name='Maka', index=99, number=99, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mand', index=100, number=100, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mani', index=101, number=101, options=None, type=None), _descriptor.EnumValueDescriptor( name='Marc', index=102, number=102, options=None, type=None), _descriptor.EnumValueDescriptor( name='Maya', index=103, number=103, options=None, type=None), _descriptor.EnumValueDescriptor( name='Medf', index=104, number=104, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mend', index=105, number=105, options=None, type=None), _descriptor.EnumValueDescriptor( name='Merc', index=106, number=106, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mero', index=107, number=107, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mlym', index=108, number=108, options=None, type=None), _descriptor.EnumValueDescriptor( name='Modi', index=109, number=109, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mong', index=110, number=110, options=None, type=None), _descriptor.EnumValueDescriptor( name='Moon', index=111, number=111, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mroo', index=112, number=112, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mtei', index=113, number=113, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mult', index=114, number=114, options=None, type=None), _descriptor.EnumValueDescriptor( name='Mymr', index=115, number=115, options=None, type=None), _descriptor.EnumValueDescriptor( name='Nand', index=116, number=116, options=None, type=None), _descriptor.EnumValueDescriptor( name='Narb', index=117, number=117, options=None, type=None), _descriptor.EnumValueDescriptor( name='Nbat', index=118, number=118, options=None, type=None), _descriptor.EnumValueDescriptor( name='Newa', index=119, number=119, options=None, type=None), _descriptor.EnumValueDescriptor( name='Nkdb', index=120, number=120, options=None, type=None), _descriptor.EnumValueDescriptor( name='Nkgb', index=121, number=121, options=None, type=None), _descriptor.EnumValueDescriptor( name='Nkoo', index=122, number=122, options=None, type=None), _descriptor.EnumValueDescriptor( name='Nshu', index=123, number=123, options=None, type=None), _descriptor.EnumValueDescriptor( name='Ogam', index=124, number=124, options=None, type=None), _descriptor.EnumValueDescriptor( name='Olck', index=125, number=125, options=None, type=None), _descriptor.EnumValueDescriptor( name='Orkh', index=126, number=126, options=None, type=None), _descriptor.EnumValueDescriptor( name='Orya', index=127, number=127, options=None, type=None), _descriptor.EnumValueDescriptor( name='Osge', index=128, number=128, options=None, type=None), _descriptor.EnumValueDescriptor( name='Osma', index=129, number=129, options=None, type=None), _descriptor.EnumValueDescriptor( name='Palm', index=130, number=130, options=None, type=None), _descriptor.EnumValueDescriptor( name='Pauc', index=131, number=131, options=None, type=None), _descriptor.EnumValueDescriptor( name='Perm', index=132, number=132, options=None, type=None), _descriptor.EnumValueDescriptor( name='Phag', index=133, number=133, options=None, type=None), _descriptor.EnumValueDescriptor( name='Phli', index=134, number=134, options=None, type=None), _descriptor.EnumValueDescriptor( name='Phlp', index=135, number=135, options=None, type=None), _descriptor.EnumValueDescriptor( name='Phlv', index=136, number=136, options=None, type=None), _descriptor.EnumValueDescriptor( name='Phnx', index=137, number=137, options=None, type=None), _descriptor.EnumValueDescriptor( name='Plrd', index=138, number=138, options=None, type=None), _descriptor.EnumValueDescriptor( name='Piqd', index=139, number=139, options=None, type=None), _descriptor.EnumValueDescriptor( name='Prti', index=140, number=140, options=None, type=None), _descriptor.EnumValueDescriptor( name='Qaaa', index=141, number=141, options=None, type=None), _descriptor.EnumValueDescriptor( name='Qabx', index=142, number=142, options=None, type=None), _descriptor.EnumValueDescriptor( name='Rjng', index=143, number=143, options=None, type=None), _descriptor.EnumValueDescriptor( name='Rohg', index=144, number=144, options=None, type=None), _descriptor.EnumValueDescriptor( name='Roro', index=145, number=145, options=None, type=None), _descriptor.EnumValueDescriptor( name='Runr', index=146, number=146, options=None, type=None), _descriptor.EnumValueDescriptor( name='Samr', index=147, number=147, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sara', index=148, number=148, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sarb', index=149, number=149, options=None, type=None), _descriptor.EnumValueDescriptor( name='Saur', index=150, number=150, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sgnw', index=151, number=151, options=None, type=None), _descriptor.EnumValueDescriptor( name='Shaw', index=152, number=152, options=None, type=None), _descriptor.EnumValueDescriptor( name='Shrd', index=153, number=153, options=None, type=None), _descriptor.EnumValueDescriptor( name='Shui', index=154, number=154, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sidd', index=155, number=155, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sind', index=156, number=156, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sinh', index=157, number=157, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sogd', index=158, number=158, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sogo', index=159, number=159, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sora', index=160, number=160, options=None, type=None), _descriptor.EnumValueDescriptor( name='Soyo', index=161, number=161, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sund', index=162, number=162, options=None, type=None), _descriptor.EnumValueDescriptor( name='Sylo', index=163, number=163, options=None, type=None), _descriptor.EnumValueDescriptor( name='Syrc', index=164, number=164, options=None, type=None), _descriptor.EnumValueDescriptor( name='Syre', index=165, number=165, options=None, type=None), _descriptor.EnumValueDescriptor( name='Syrj', index=166, number=166, options=None, type=None), _descriptor.EnumValueDescriptor( name='Syrn', index=167, number=167, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tagb', index=168, number=168, options=None, type=None), _descriptor.EnumValueDescriptor( name='Takr', index=169, number=169, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tale', index=170, number=170, options=None, type=None), _descriptor.EnumValueDescriptor( name='Talu', index=171, number=171, options=None, type=None), _descriptor.EnumValueDescriptor( name='Taml', index=172, number=172, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tang', index=173, number=173, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tavt', index=174, number=174, options=None, type=None), _descriptor.EnumValueDescriptor( name='Telu', index=175, number=175, options=None, type=None), _descriptor.EnumValueDescriptor( name='Teng', index=176, number=176, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tfng', index=177, number=177, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tglg', index=178, number=178, options=None, type=None), _descriptor.EnumValueDescriptor( name='Thaa', index=179, number=179, options=None, type=None), _descriptor.EnumValueDescriptor( name='Thai', index=180, number=180, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tibt', index=181, number=181, options=None, type=None), _descriptor.EnumValueDescriptor( name='Tirh', index=182, number=182, options=None, type=None), _descriptor.EnumValueDescriptor( name='Ugar', index=183, number=183, options=None, type=None), _descriptor.EnumValueDescriptor( name='Vaii', index=184, number=184, options=None, type=None), _descriptor.EnumValueDescriptor( name='Visp', index=185, number=185, options=None, type=None), _descriptor.EnumValueDescriptor( name='Wara', index=186, number=186, options=None, type=None), _descriptor.EnumValueDescriptor( name='Wcho', index=187, number=187, options=None, type=None), _descriptor.EnumValueDescriptor( name='Wole', index=188, number=188, options=None, type=None), _descriptor.EnumValueDescriptor( name='Xpeo', index=189, number=189, options=None, type=None), _descriptor.EnumValueDescriptor( name='Xsux', index=190, number=190, options=None, type=None), _descriptor.EnumValueDescriptor( name='Yiii', index=191, number=191, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zanb', index=192, number=192, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zinh', index=193, number=193, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zmth', index=194, number=194, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zsye', index=195, number=195, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zsym', index=196, number=196, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zxxx', index=197, number=197, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zyyy', index=198, number=198, options=None, type=None), _descriptor.EnumValueDescriptor( name='Zzzz', index=199, number=199, options=None, type=None), ], containing_type=None, options=None, serialized_start=3112, serialized_end=5202, ) _sym_db.RegisterEnumDescriptor(_LANGUAGE_SCRIPT) _LOCATION_COUNTRY = _descriptor.EnumDescriptor( name='Country', full_name='pb.Location.Country', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_COUNTRY', index=0, number=0, options=None, type=None), _descriptor.EnumValueDescriptor( name='AF', index=1, number=1, options=None, type=None), _descriptor.EnumValueDescriptor( name='AX', index=2, number=2, options=None, type=None), _descriptor.EnumValueDescriptor( name='AL', index=3, number=3, options=None, type=None), _descriptor.EnumValueDescriptor( name='DZ', index=4, number=4, options=None, type=None), _descriptor.EnumValueDescriptor( name='AS', index=5, number=5, options=None, type=None), _descriptor.EnumValueDescriptor( name='AD', index=6, number=6, options=None, type=None), _descriptor.EnumValueDescriptor( name='AO', index=7, number=7, options=None, type=None), _descriptor.EnumValueDescriptor( name='AI', index=8, number=8, options=None, type=None), _descriptor.EnumValueDescriptor( name='AQ', index=9, number=9, options=None, type=None), _descriptor.EnumValueDescriptor( name='AG', index=10, number=10, options=None, type=None), _descriptor.EnumValueDescriptor( name='AR', index=11, number=11, options=None, type=None), _descriptor.EnumValueDescriptor( name='AM', index=12, number=12, options=None, type=None), _descriptor.EnumValueDescriptor( name='AW', index=13, number=13, options=None, type=None), _descriptor.EnumValueDescriptor( name='AU', index=14, number=14, options=None, type=None), _descriptor.EnumValueDescriptor( name='AT', index=15, number=15, options=None, type=None), _descriptor.EnumValueDescriptor( name='AZ', index=16, number=16, options=None, type=None), _descriptor.EnumValueDescriptor( name='BS', index=17, number=17, options=None, type=None), _descriptor.EnumValueDescriptor( name='BH', index=18, number=18, options=None, type=None), _descriptor.EnumValueDescriptor( name='BD', index=19, number=19, options=None, type=None), _descriptor.EnumValueDescriptor( name='BB', index=20, number=20, options=None, type=None), _descriptor.EnumValueDescriptor( name='BY', index=21, number=21, options=None, type=None), _descriptor.EnumValueDescriptor( name='BE', index=22, number=22, options=None, type=None), _descriptor.EnumValueDescriptor( name='BZ', index=23, number=23, options=None, type=None), _descriptor.EnumValueDescriptor( name='BJ', index=24, number=24, options=None, type=None), _descriptor.EnumValueDescriptor( name='BM', index=25, number=25, options=None, type=None), _descriptor.EnumValueDescriptor( name='BT', index=26, number=26, options=None, type=None), _descriptor.EnumValueDescriptor( name='BO', index=27, number=27, options=None, type=None), _descriptor.EnumValueDescriptor( name='BQ', index=28, number=28, options=None, type=None), _descriptor.EnumValueDescriptor( name='BA', index=29, number=29, options=None, type=None), _descriptor.EnumValueDescriptor( name='BW', index=30, number=30, options=None, type=None), _descriptor.EnumValueDescriptor( name='BV', index=31, number=31, options=None, type=None), _descriptor.EnumValueDescriptor( name='BR', index=32, number=32, options=None, type=None), _descriptor.EnumValueDescriptor( name='IO', index=33, number=33, options=None, type=None), _descriptor.EnumValueDescriptor( name='BN', index=34, number=34, options=None, type=None), _descriptor.EnumValueDescriptor( name='BG', index=35, number=35, options=None, type=None), _descriptor.EnumValueDescriptor( name='BF', index=36, number=36, options=None, type=None), _descriptor.EnumValueDescriptor( name='BI', index=37, number=37, options=None, type=None), _descriptor.EnumValueDescriptor( name='KH', index=38, number=38, options=None, type=None), _descriptor.EnumValueDescriptor( name='CM', index=39, number=39, options=None, type=None), _descriptor.EnumValueDescriptor( name='CA', index=40, number=40, options=None, type=None), _descriptor.EnumValueDescriptor( name='CV', index=41, number=41, options=None, type=None), _descriptor.EnumValueDescriptor( name='KY', index=42, number=42, options=None, type=None), _descriptor.EnumValueDescriptor( name='CF', index=43, number=43, options=None, type=None), _descriptor.EnumValueDescriptor( name='TD', index=44, number=44, options=None, type=None), _descriptor.EnumValueDescriptor( name='CL', index=45, number=45, options=None, type=None), _descriptor.EnumValueDescriptor( name='CN', index=46, number=46, options=None, type=None), _descriptor.EnumValueDescriptor( name='CX', index=47, number=47, options=None, type=None), _descriptor.EnumValueDescriptor( name='CC', index=48, number=48, options=None, type=None), _descriptor.EnumValueDescriptor( name='CO', index=49, number=49, options=None, type=None), _descriptor.EnumValueDescriptor( name='KM', index=50, number=50, options=None, type=None), _descriptor.EnumValueDescriptor( name='CG', index=51, number=51, options=None, type=None), _descriptor.EnumValueDescriptor( name='CD', index=52, number=52, options=None, type=None), _descriptor.EnumValueDescriptor( name='CK', index=53, number=53, options=None, type=None), _descriptor.EnumValueDescriptor( name='CR', index=54, number=54, options=None, type=None), _descriptor.EnumValueDescriptor( name='CI', index=55, number=55, options=None, type=None), _descriptor.EnumValueDescriptor( name='HR', index=56, number=56, options=None, type=None), _descriptor.EnumValueDescriptor( name='CU', index=57, number=57, options=None, type=None), _descriptor.EnumValueDescriptor( name='CW', index=58, number=58, options=None, type=None), _descriptor.EnumValueDescriptor( name='CY', index=59, number=59, options=None, type=None), _descriptor.EnumValueDescriptor( name='CZ', index=60, number=60, options=None, type=None), _descriptor.EnumValueDescriptor( name='DK', index=61, number=61, options=None, type=None), _descriptor.EnumValueDescriptor( name='DJ', index=62, number=62, options=None, type=None), _descriptor.EnumValueDescriptor( name='DM', index=63, number=63, options=None, type=None), _descriptor.EnumValueDescriptor( name='DO', index=64, number=64, options=None, type=None), _descriptor.EnumValueDescriptor( name='EC', index=65, number=65, options=None, type=None), _descriptor.EnumValueDescriptor( name='EG', index=66, number=66, options=None, type=None), _descriptor.EnumValueDescriptor( name='SV', index=67, number=67, options=None, type=None), _descriptor.EnumValueDescriptor( name='GQ', index=68, number=68, options=None, type=None), _descriptor.EnumValueDescriptor( name='ER', index=69, number=69, options=None, type=None), _descriptor.EnumValueDescriptor( name='EE', index=70, number=70, options=None, type=None), _descriptor.EnumValueDescriptor( name='ET', index=71, number=71, options=None, type=None), _descriptor.EnumValueDescriptor( name='FK', index=72, number=72, options=None, type=None), _descriptor.EnumValueDescriptor( name='FO', index=73, number=73, options=None, type=None), _descriptor.EnumValueDescriptor( name='FJ', index=74, number=74, options=None, type=None), _descriptor.EnumValueDescriptor( name='FI', index=75, number=75, options=None, type=None), _descriptor.EnumValueDescriptor( name='FR', index=76, number=76, options=None, type=None), _descriptor.EnumValueDescriptor( name='GF', index=77, number=77, options=None, type=None), _descriptor.EnumValueDescriptor( name='PF', index=78, number=78, options=None, type=None), _descriptor.EnumValueDescriptor( name='TF', index=79, number=79, options=None, type=None), _descriptor.EnumValueDescriptor( name='GA', index=80, number=80, options=None, type=None), _descriptor.EnumValueDescriptor( name='GM', index=81, number=81, options=None, type=None), _descriptor.EnumValueDescriptor( name='GE', index=82, number=82, options=None, type=None), _descriptor.EnumValueDescriptor( name='DE', index=83, number=83, options=None, type=None), _descriptor.EnumValueDescriptor( name='GH', index=84, number=84, options=None, type=None), _descriptor.EnumValueDescriptor( name='GI', index=85, number=85, options=None, type=None), _descriptor.EnumValueDescriptor( name='GR', index=86, number=86, options=None, type=None), _descriptor.EnumValueDescriptor( name='GL', index=87, number=87, options=None, type=None), _descriptor.EnumValueDescriptor( name='GD', index=88, number=88, options=None, type=None), _descriptor.EnumValueDescriptor( name='GP', index=89, number=89, options=None, type=None), _descriptor.EnumValueDescriptor( name='GU', index=90, number=90, options=None, type=None), _descriptor.EnumValueDescriptor( name='GT', index=91, number=91, options=None, type=None), _descriptor.EnumValueDescriptor( name='GG', index=92, number=92, options=None, type=None), _descriptor.EnumValueDescriptor( name='GN', index=93, number=93, options=None, type=None), _descriptor.EnumValueDescriptor( name='GW', index=94, number=94, options=None, type=None), _descriptor.EnumValueDescriptor( name='GY', index=95, number=95, options=None, type=None), _descriptor.EnumValueDescriptor( name='HT', index=96, number=96, options=None, type=None), _descriptor.EnumValueDescriptor( name='HM', index=97, number=97, options=None, type=None), _descriptor.EnumValueDescriptor( name='VA', index=98, number=98, options=None, type=None), _descriptor.EnumValueDescriptor( name='HN', index=99, number=99, options=None, type=None), _descriptor.EnumValueDescriptor( name='HK', index=100, number=100, options=None, type=None), _descriptor.EnumValueDescriptor( name='HU', index=101, number=101, options=None, type=None), _descriptor.EnumValueDescriptor( name='IS', index=102, number=102, options=None, type=None), _descriptor.EnumValueDescriptor( name='IN', index=103, number=103, options=None, type=None), _descriptor.EnumValueDescriptor( name='ID', index=104, number=104, options=None, type=None), _descriptor.EnumValueDescriptor( name='IR', index=105, number=105, options=None, type=None), _descriptor.EnumValueDescriptor( name='IQ', index=106, number=106, options=None, type=None), _descriptor.EnumValueDescriptor( name='IE', index=107, number=107, options=None, type=None), _descriptor.EnumValueDescriptor( name='IM', index=108, number=108, options=None, type=None), _descriptor.EnumValueDescriptor( name='IL', index=109, number=109, options=None, type=None), _descriptor.EnumValueDescriptor( name='IT', index=110, number=110, options=None, type=None), _descriptor.EnumValueDescriptor( name='JM', index=111, number=111, options=None, type=None), _descriptor.EnumValueDescriptor( name='JP', index=112, number=112, options=None, type=None), _descriptor.EnumValueDescriptor( name='JE', index=113, number=113, options=None, type=None), _descriptor.EnumValueDescriptor( name='JO', index=114, number=114, options=None, type=None), _descriptor.EnumValueDescriptor( name='KZ', index=115, number=115, options=None, type=None), _descriptor.EnumValueDescriptor( name='KE', index=116, number=116, options=None, type=None), _descriptor.EnumValueDescriptor( name='KI', index=117, number=117, options=None, type=None), _descriptor.EnumValueDescriptor( name='KP', index=118, number=118, options=None, type=None), _descriptor.EnumValueDescriptor( name='KR', index=119, number=119, options=None, type=None), _descriptor.EnumValueDescriptor( name='KW', index=120, number=120, options=None, type=None), _descriptor.EnumValueDescriptor( name='KG', index=121, number=121, options=None, type=None), _descriptor.EnumValueDescriptor( name='LA', index=122, number=122, options=None, type=None), _descriptor.EnumValueDescriptor( name='LV', index=123, number=123, options=None, type=None), _descriptor.EnumValueDescriptor( name='LB', index=124, number=124, options=None, type=None), _descriptor.EnumValueDescriptor( name='LS', index=125, number=125, options=None, type=None), _descriptor.EnumValueDescriptor( name='LR', index=126, number=126, options=None, type=None), _descriptor.EnumValueDescriptor( name='LY', index=127, number=127, options=None, type=None), _descriptor.EnumValueDescriptor( name='LI', index=128, number=128, options=None, type=None), _descriptor.EnumValueDescriptor( name='LT', index=129, number=129, options=None, type=None), _descriptor.EnumValueDescriptor( name='LU', index=130, number=130, options=None, type=None), _descriptor.EnumValueDescriptor( name='MO', index=131, number=131, options=None, type=None), _descriptor.EnumValueDescriptor( name='MK', index=132, number=132, options=None, type=None), _descriptor.EnumValueDescriptor( name='MG', index=133, number=133, options=None, type=None), _descriptor.EnumValueDescriptor( name='MW', index=134, number=134, options=None, type=None), _descriptor.EnumValueDescriptor( name='MY', index=135, number=135, options=None, type=None), _descriptor.EnumValueDescriptor( name='MV', index=136, number=136, options=None, type=None), _descriptor.EnumValueDescriptor( name='ML', index=137, number=137, options=None, type=None), _descriptor.EnumValueDescriptor( name='MT', index=138, number=138, options=None, type=None), _descriptor.EnumValueDescriptor( name='MH', index=139, number=139, options=None, type=None), _descriptor.EnumValueDescriptor( name='MQ', index=140, number=140, options=None, type=None), _descriptor.EnumValueDescriptor( name='MR', index=141, number=141, options=None, type=None), _descriptor.EnumValueDescriptor( name='MU', index=142, number=142, options=None, type=None), _descriptor.EnumValueDescriptor( name='YT', index=143, number=143, options=None, type=None), _descriptor.EnumValueDescriptor( name='MX', index=144, number=144, options=None, type=None), _descriptor.EnumValueDescriptor( name='FM', index=145, number=145, options=None, type=None), _descriptor.EnumValueDescriptor( name='MD', index=146, number=146, options=None, type=None), _descriptor.EnumValueDescriptor( name='MC', index=147, number=147, options=None, type=None), _descriptor.EnumValueDescriptor( name='MN', index=148, number=148, options=None, type=None), _descriptor.EnumValueDescriptor( name='ME', index=149, number=149, options=None, type=None), _descriptor.EnumValueDescriptor( name='MS', index=150, number=150, options=None, type=None), _descriptor.EnumValueDescriptor( name='MA', index=151, number=151, options=None, type=None), _descriptor.EnumValueDescriptor( name='MZ', index=152, number=152, options=None, type=None), _descriptor.EnumValueDescriptor( name='MM', index=153, number=153, options=None, type=None), _descriptor.EnumValueDescriptor( name='NA', index=154, number=154, options=None, type=None), _descriptor.EnumValueDescriptor( name='NR', index=155, number=155, options=None, type=None), _descriptor.EnumValueDescriptor( name='NP', index=156, number=156, options=None, type=None), _descriptor.EnumValueDescriptor( name='NL', index=157, number=157, options=None, type=None), _descriptor.EnumValueDescriptor( name='NC', index=158, number=158, options=None, type=None), _descriptor.EnumValueDescriptor( name='NZ', index=159, number=159, options=None, type=None), _descriptor.EnumValueDescriptor( name='NI', index=160, number=160, options=None, type=None), _descriptor.EnumValueDescriptor( name='NE', index=161, number=161, options=None, type=None), _descriptor.EnumValueDescriptor( name='NG', index=162, number=162, options=None, type=None), _descriptor.EnumValueDescriptor( name='NU', index=163, number=163, options=None, type=None), _descriptor.EnumValueDescriptor( name='NF', index=164, number=164, options=None, type=None), _descriptor.EnumValueDescriptor( name='MP', index=165, number=165, options=None, type=None), _descriptor.EnumValueDescriptor( name='NO', index=166, number=166, options=None, type=None), _descriptor.EnumValueDescriptor( name='OM', index=167, number=167, options=None, type=None), _descriptor.EnumValueDescriptor( name='PK', index=168, number=168, options=None, type=None), _descriptor.EnumValueDescriptor( name='PW', index=169, number=169, options=None, type=None), _descriptor.EnumValueDescriptor( name='PS', index=170, number=170, options=None, type=None), _descriptor.EnumValueDescriptor( name='PA', index=171, number=171, options=None, type=None), _descriptor.EnumValueDescriptor( name='PG', index=172, number=172, options=None, type=None), _descriptor.EnumValueDescriptor( name='PY', index=173, number=173, options=None, type=None), _descriptor.EnumValueDescriptor( name='PE', index=174, number=174, options=None, type=None), _descriptor.EnumValueDescriptor( name='PH', index=175, number=175, options=None, type=None), _descriptor.EnumValueDescriptor( name='PN', index=176, number=176, options=None, type=None), _descriptor.EnumValueDescriptor( name='PL', index=177, number=177, options=None, type=None), _descriptor.EnumValueDescriptor( name='PT', index=178, number=178, options=None, type=None), _descriptor.EnumValueDescriptor( name='PR', index=179, number=179, options=None, type=None), _descriptor.EnumValueDescriptor( name='QA', index=180, number=180, options=None, type=None), _descriptor.EnumValueDescriptor( name='RE', index=181, number=181, options=None, type=None), _descriptor.EnumValueDescriptor( name='RO', index=182, number=182, options=None, type=None), _descriptor.EnumValueDescriptor( name='RU', index=183, number=183, options=None, type=None), _descriptor.EnumValueDescriptor( name='RW', index=184, number=184, options=None, type=None), _descriptor.EnumValueDescriptor( name='BL', index=185, number=185, options=None, type=None), _descriptor.EnumValueDescriptor( name='SH', index=186, number=186, options=None, type=None), _descriptor.EnumValueDescriptor( name='KN', index=187, number=187, options=None, type=None), _descriptor.EnumValueDescriptor( name='LC', index=188, number=188, options=None, type=None), _descriptor.EnumValueDescriptor( name='MF', index=189, number=189, options=None, type=None), _descriptor.EnumValueDescriptor( name='PM', index=190, number=190, options=None, type=None), _descriptor.EnumValueDescriptor( name='VC', index=191, number=191, options=None, type=None), _descriptor.EnumValueDescriptor( name='WS', index=192, number=192, options=None, type=None), _descriptor.EnumValueDescriptor( name='SM', index=193, number=193, options=None, type=None), _descriptor.EnumValueDescriptor( name='ST', index=194, number=194, options=None, type=None), _descriptor.EnumValueDescriptor( name='SA', index=195, number=195, options=None, type=None), _descriptor.EnumValueDescriptor( name='SN', index=196, number=196, options=None, type=None), _descriptor.EnumValueDescriptor( name='RS', index=197, number=197, options=None, type=None), _descriptor.EnumValueDescriptor( name='SC', index=198, number=198, options=None, type=None), _descriptor.EnumValueDescriptor( name='SL', index=199, number=199, options=None, type=None), _descriptor.EnumValueDescriptor( name='SG', index=200, number=200, options=None, type=None), _descriptor.EnumValueDescriptor( name='SX', index=201, number=201, options=None, type=None), _descriptor.EnumValueDescriptor( name='SK', index=202, number=202, options=None, type=None), _descriptor.EnumValueDescriptor( name='SI', index=203, number=203, options=None, type=None), _descriptor.EnumValueDescriptor( name='SB', index=204, number=204, options=None, type=None), _descriptor.EnumValueDescriptor( name='SO', index=205, number=205, options=None, type=None), _descriptor.EnumValueDescriptor( name='ZA', index=206, number=206, options=None, type=None), _descriptor.EnumValueDescriptor( name='GS', index=207, number=207, options=None, type=None), _descriptor.EnumValueDescriptor( name='SS', index=208, number=208, options=None, type=None), _descriptor.EnumValueDescriptor( name='ES', index=209, number=209, options=None, type=None), _descriptor.EnumValueDescriptor( name='LK', index=210, number=210, options=None, type=None), _descriptor.EnumValueDescriptor( name='SD', index=211, number=211, options=None, type=None), _descriptor.EnumValueDescriptor( name='SR', index=212, number=212, options=None, type=None), _descriptor.EnumValueDescriptor( name='SJ', index=213, number=213, options=None, type=None), _descriptor.EnumValueDescriptor( name='SZ', index=214, number=214, options=None, type=None), _descriptor.EnumValueDescriptor( name='SE', index=215, number=215, options=None, type=None), _descriptor.EnumValueDescriptor( name='CH', index=216, number=216, options=None, type=None), _descriptor.EnumValueDescriptor( name='SY', index=217, number=217, options=None, type=None), _descriptor.EnumValueDescriptor( name='TW', index=218, number=218, options=None, type=None), _descriptor.EnumValueDescriptor( name='TJ', index=219, number=219, options=None, type=None), _descriptor.EnumValueDescriptor( name='TZ', index=220, number=220, options=None, type=None), _descriptor.EnumValueDescriptor( name='TH', index=221, number=221, options=None, type=None), _descriptor.EnumValueDescriptor( name='TL', index=222, number=222, options=None, type=None), _descriptor.EnumValueDescriptor( name='TG', index=223, number=223, options=None, type=None), _descriptor.EnumValueDescriptor( name='TK', index=224, number=224, options=None, type=None), _descriptor.EnumValueDescriptor( name='TO', index=225, number=225, options=None, type=None), _descriptor.EnumValueDescriptor( name='TT', index=226, number=226, options=None, type=None), _descriptor.EnumValueDescriptor( name='TN', index=227, number=227, options=None, type=None), _descriptor.EnumValueDescriptor( name='TR', index=228, number=228, options=None, type=None), _descriptor.EnumValueDescriptor( name='TM', index=229, number=229, options=None, type=None), _descriptor.EnumValueDescriptor( name='TC', index=230, number=230, options=None, type=None), _descriptor.EnumValueDescriptor( name='TV', index=231, number=231, options=None, type=None), _descriptor.EnumValueDescriptor( name='UG', index=232, number=232, options=None, type=None), _descriptor.EnumValueDescriptor( name='UA', index=233, number=233, options=None, type=None), _descriptor.EnumValueDescriptor( name='AE', index=234, number=234, options=None, type=None), _descriptor.EnumValueDescriptor( name='GB', index=235, number=235, options=None, type=None), _descriptor.EnumValueDescriptor( name='US', index=236, number=236, options=None, type=None), _descriptor.EnumValueDescriptor( name='UM', index=237, number=237, options=None, type=None), _descriptor.EnumValueDescriptor( name='UY', index=238, number=238, options=None, type=None), _descriptor.EnumValueDescriptor( name='UZ', index=239, number=239, options=None, type=None), _descriptor.EnumValueDescriptor( name='VU', index=240, number=240, options=None, type=None), _descriptor.EnumValueDescriptor( name='VE', index=241, number=241, options=None, type=None), _descriptor.EnumValueDescriptor( name='VN', index=242, number=242, options=None, type=None), _descriptor.EnumValueDescriptor( name='VG', index=243, number=243, options=None, type=None), _descriptor.EnumValueDescriptor( name='VI', index=244, number=244, options=None, type=None), _descriptor.EnumValueDescriptor( name='WF', index=245, number=245, options=None, type=None), _descriptor.EnumValueDescriptor( name='EH', index=246, number=246, options=None, type=None), _descriptor.EnumValueDescriptor( name='YE', index=247, number=247, options=None, type=None), _descriptor.EnumValueDescriptor( name='ZM', index=248, number=248, options=None, type=None), _descriptor.EnumValueDescriptor( name='ZW', index=249, number=249, options=None, type=None), _descriptor.EnumValueDescriptor( name='R001', index=250, number=250, options=None, type=None), _descriptor.EnumValueDescriptor( name='R002', index=251, number=251, options=None, type=None), _descriptor.EnumValueDescriptor( name='R015', index=252, number=252, options=None, type=None), _descriptor.EnumValueDescriptor( name='R012', index=253, number=253, options=None, type=None), _descriptor.EnumValueDescriptor( name='R818', index=254, number=254, options=None, type=None), _descriptor.EnumValueDescriptor( name='R434', index=255, number=255, options=None, type=None), _descriptor.EnumValueDescriptor( name='R504', index=256, number=256, options=None, type=None), _descriptor.EnumValueDescriptor( name='R729', index=257, number=257, options=None, type=None), _descriptor.EnumValueDescriptor( name='R788', index=258, number=258, options=None, type=None), _descriptor.EnumValueDescriptor( name='R732', index=259, number=259, options=None, type=None), _descriptor.EnumValueDescriptor( name='R202', index=260, number=260, options=None, type=None), _descriptor.EnumValueDescriptor( name='R014', index=261, number=261, options=None, type=None), _descriptor.EnumValueDescriptor( name='R086', index=262, number=262, options=None, type=None), _descriptor.EnumValueDescriptor( name='R108', index=263, number=263, options=None, type=None), _descriptor.EnumValueDescriptor( name='R174', index=264, number=264, options=None, type=None), _descriptor.EnumValueDescriptor( name='R262', index=265, number=265, options=None, type=None), _descriptor.EnumValueDescriptor( name='R232', index=266, number=266, options=None, type=None), _descriptor.EnumValueDescriptor( name='R231', index=267, number=267, options=None, type=None), _descriptor.EnumValueDescriptor( name='R260', index=268, number=268, options=None, type=None), _descriptor.EnumValueDescriptor( name='R404', index=269, number=269, options=None, type=None), _descriptor.EnumValueDescriptor( name='R450', index=270, number=270, options=None, type=None), _descriptor.EnumValueDescriptor( name='R454', index=271, number=271, options=None, type=None), _descriptor.EnumValueDescriptor( name='R480', index=272, number=272, options=None, type=None), _descriptor.EnumValueDescriptor( name='R175', index=273, number=273, options=None, type=None), _descriptor.EnumValueDescriptor( name='R508', index=274, number=274, options=None, type=None), _descriptor.EnumValueDescriptor( name='R638', index=275, number=275, options=None, type=None), _descriptor.EnumValueDescriptor( name='R646', index=276, number=276, options=None, type=None), _descriptor.EnumValueDescriptor( name='R690', index=277, number=277, options=None, type=None), _descriptor.EnumValueDescriptor( name='R706', index=278, number=278, options=None, type=None), _descriptor.EnumValueDescriptor( name='R728', index=279, number=279, options=None, type=None), _descriptor.EnumValueDescriptor( name='R800', index=280, number=280, options=None, type=None), _descriptor.EnumValueDescriptor( name='R834', index=281, number=281, options=None, type=None), _descriptor.EnumValueDescriptor( name='R894', index=282, number=282, options=None, type=None), _descriptor.EnumValueDescriptor( name='R716', index=283, number=283, options=None, type=None), _descriptor.EnumValueDescriptor( name='R017', index=284, number=284, options=None, type=None), _descriptor.EnumValueDescriptor( name='R024', index=285, number=285, options=None, type=None), _descriptor.EnumValueDescriptor( name='R120', index=286, number=286, options=None, type=None), _descriptor.EnumValueDescriptor( name='R140', index=287, number=287, options=None, type=None), _descriptor.EnumValueDescriptor( name='R148', index=288, number=288, options=None, type=None), _descriptor.EnumValueDescriptor( name='R178', index=289, number=289, options=None, type=None), _descriptor.EnumValueDescriptor( name='R180', index=290, number=290, options=None, type=None), _descriptor.EnumValueDescriptor( name='R226', index=291, number=291, options=None, type=None), _descriptor.EnumValueDescriptor( name='R266', index=292, number=292, options=None, type=None), _descriptor.EnumValueDescriptor( name='R678', index=293, number=293, options=None, type=None), _descriptor.EnumValueDescriptor( name='R018', index=294, number=294, options=None, type=None), _descriptor.EnumValueDescriptor( name='R072', index=295, number=295, options=None, type=None), _descriptor.EnumValueDescriptor( name='R748', index=296, number=296, options=None, type=None), _descriptor.EnumValueDescriptor( name='R426', index=297, number=297, options=None, type=None), _descriptor.EnumValueDescriptor( name='R516', index=298, number=298, options=None, type=None), _descriptor.EnumValueDescriptor( name='R710', index=299, number=299, options=None, type=None), _descriptor.EnumValueDescriptor( name='R011', index=300, number=300, options=None, type=None), _descriptor.EnumValueDescriptor( name='R204', index=301, number=301, options=None, type=None), _descriptor.EnumValueDescriptor( name='R854', index=302, number=302, options=None, type=None), _descriptor.EnumValueDescriptor( name='R132', index=303, number=303, options=None, type=None), _descriptor.EnumValueDescriptor( name='R384', index=304, number=304, options=None, type=None), _descriptor.EnumValueDescriptor( name='R270', index=305, number=305, options=None, type=None), _descriptor.EnumValueDescriptor( name='R288', index=306, number=306, options=None, type=None), _descriptor.EnumValueDescriptor( name='R324', index=307, number=307, options=None, type=None), _descriptor.EnumValueDescriptor( name='R624', index=308, number=308, options=None, type=None), _descriptor.EnumValueDescriptor( name='R430', index=309, number=309, options=None, type=None), _descriptor.EnumValueDescriptor( name='R466', index=310, number=310, options=None, type=None), _descriptor.EnumValueDescriptor( name='R478', index=311, number=311, options=None, type=None), _descriptor.EnumValueDescriptor( name='R562', index=312, number=312, options=None, type=None), _descriptor.EnumValueDescriptor( name='R566', index=313, number=313, options=None, type=None), _descriptor.EnumValueDescriptor( name='R654', index=314, number=314, options=None, type=None), _descriptor.EnumValueDescriptor( name='R686', index=315, number=315, options=None, type=None), _descriptor.EnumValueDescriptor( name='R694', index=316, number=316, options=None, type=None), _descriptor.EnumValueDescriptor( name='R768', index=317, number=317, options=None, type=None), _descriptor.EnumValueDescriptor( name='R019', index=318, number=318, options=None, type=None), _descriptor.EnumValueDescriptor( name='R419', index=319, number=319, options=None, type=None), _descriptor.EnumValueDescriptor( name='R029', index=320, number=320, options=None, type=None), _descriptor.EnumValueDescriptor( name='R660', index=321, number=321, options=None, type=None), _descriptor.EnumValueDescriptor( name='R028', index=322, number=322, options=None, type=None), _descriptor.EnumValueDescriptor( name='R533', index=323, number=323, options=None, type=None), _descriptor.EnumValueDescriptor( name='R044', index=324, number=324, options=None, type=None), _descriptor.EnumValueDescriptor( name='R052', index=325, number=325, options=None, type=None), _descriptor.EnumValueDescriptor( name='R535', index=326, number=326, options=None, type=None), _descriptor.EnumValueDescriptor( name='R092', index=327, number=327, options=None, type=None), _descriptor.EnumValueDescriptor( name='R136', index=328, number=328, options=None, type=None), _descriptor.EnumValueDescriptor( name='R192', index=329, number=329, options=None, type=None), _descriptor.EnumValueDescriptor( name='R531', index=330, number=330, options=None, type=None), _descriptor.EnumValueDescriptor( name='R212', index=331, number=331, options=None, type=None), _descriptor.EnumValueDescriptor( name='R214', index=332, number=332, options=None, type=None), _descriptor.EnumValueDescriptor( name='R308', index=333, number=333, options=None, type=None), _descriptor.EnumValueDescriptor( name='R312', index=334, number=334, options=None, type=None), _descriptor.EnumValueDescriptor( name='R332', index=335, number=335, options=None, type=None), _descriptor.EnumValueDescriptor( name='R388', index=336, number=336, options=None, type=None), _descriptor.EnumValueDescriptor( name='R474', index=337, number=337, options=None, type=None), _descriptor.EnumValueDescriptor( name='R500', index=338, number=338, options=None, type=None), _descriptor.EnumValueDescriptor( name='R630', index=339, number=339, options=None, type=None), _descriptor.EnumValueDescriptor( name='R652', index=340, number=340, options=None, type=None), _descriptor.EnumValueDescriptor( name='R659', index=341, number=341, options=None, type=None), _descriptor.EnumValueDescriptor( name='R662', index=342, number=342, options=None, type=None), _descriptor.EnumValueDescriptor( name='R663', index=343, number=343, options=None, type=None), _descriptor.EnumValueDescriptor( name='R670', index=344, number=344, options=None, type=None), _descriptor.EnumValueDescriptor( name='R534', index=345, number=345, options=None, type=None), _descriptor.EnumValueDescriptor( name='R780', index=346, number=346, options=None, type=None), _descriptor.EnumValueDescriptor( name='R796', index=347, number=347, options=None, type=None), _descriptor.EnumValueDescriptor( name='R850', index=348, number=348, options=None, type=None), _descriptor.EnumValueDescriptor( name='R013', index=349, number=349, options=None, type=None), _descriptor.EnumValueDescriptor( name='R084', index=350, number=350, options=None, type=None), _descriptor.EnumValueDescriptor( name='R188', index=351, number=351, options=None, type=None), _descriptor.EnumValueDescriptor( name='R222', index=352, number=352, options=None, type=None), _descriptor.EnumValueDescriptor( name='R320', index=353, number=353, options=None, type=None), _descriptor.EnumValueDescriptor( name='R340', index=354, number=354, options=None, type=None), _descriptor.EnumValueDescriptor( name='R484', index=355, number=355, options=None, type=None), _descriptor.EnumValueDescriptor( name='R558', index=356, number=356, options=None, type=None), _descriptor.EnumValueDescriptor( name='R591', index=357, number=357, options=None, type=None), _descriptor.EnumValueDescriptor( name='R005', index=358, number=358, options=None, type=None), _descriptor.EnumValueDescriptor( name='R032', index=359, number=359, options=None, type=None), _descriptor.EnumValueDescriptor( name='R068', index=360, number=360, options=None, type=None), _descriptor.EnumValueDescriptor( name='R074', index=361, number=361, options=None, type=None), _descriptor.EnumValueDescriptor( name='R076', index=362, number=362, options=None, type=None), _descriptor.EnumValueDescriptor( name='R152', index=363, number=363, options=None, type=None), _descriptor.EnumValueDescriptor( name='R170', index=364, number=364, options=None, type=None), _descriptor.EnumValueDescriptor( name='R218', index=365, number=365, options=None, type=None), _descriptor.EnumValueDescriptor( name='R238', index=366, number=366, options=None, type=None), _descriptor.EnumValueDescriptor( name='R254', index=367, number=367, options=None, type=None), _descriptor.EnumValueDescriptor( name='R328', index=368, number=368, options=None, type=None), _descriptor.EnumValueDescriptor( name='R600', index=369, number=369, options=None, type=None), _descriptor.EnumValueDescriptor( name='R604', index=370, number=370, options=None, type=None), _descriptor.EnumValueDescriptor( name='R239', index=371, number=371, options=None, type=None), _descriptor.EnumValueDescriptor( name='R740', index=372, number=372, options=None, type=None), _descriptor.EnumValueDescriptor( name='R858', index=373, number=373, options=None, type=None), _descriptor.EnumValueDescriptor( name='R862', index=374, number=374, options=None, type=None), _descriptor.EnumValueDescriptor( name='R021', index=375, number=375, options=None, type=None), _descriptor.EnumValueDescriptor( name='R060', index=376, number=376, options=None, type=None), _descriptor.EnumValueDescriptor( name='R124', index=377, number=377, options=None, type=None), _descriptor.EnumValueDescriptor( name='R304', index=378, number=378, options=None, type=None), _descriptor.EnumValueDescriptor( name='R666', index=379, number=379, options=None, type=None), _descriptor.EnumValueDescriptor( name='R840', index=380, number=380, options=None, type=None), _descriptor.EnumValueDescriptor( name='R010', index=381, number=381, options=None, type=None), _descriptor.EnumValueDescriptor( name='R142', index=382, number=382, options=None, type=None), _descriptor.EnumValueDescriptor( name='R143', index=383, number=383, options=None, type=None), _descriptor.EnumValueDescriptor( name='R398', index=384, number=384, options=None, type=None), _descriptor.EnumValueDescriptor( name='R417', index=385, number=385, options=None, type=None), _descriptor.EnumValueDescriptor( name='R762', index=386, number=386, options=None, type=None), _descriptor.EnumValueDescriptor( name='R795', index=387, number=387, options=None, type=None), _descriptor.EnumValueDescriptor( name='R860', index=388, number=388, options=None, type=None), _descriptor.EnumValueDescriptor( name='R030', index=389, number=389, options=None, type=None), _descriptor.EnumValueDescriptor( name='R156', index=390, number=390, options=None, type=None), _descriptor.EnumValueDescriptor( name='R344', index=391, number=391, options=None, type=None), _descriptor.EnumValueDescriptor( name='R446', index=392, number=392, options=None, type=None), _descriptor.EnumValueDescriptor( name='R408', index=393, number=393, options=None, type=None), _descriptor.EnumValueDescriptor( name='R392', index=394, number=394, options=None, type=None), _descriptor.EnumValueDescriptor( name='R496', index=395, number=395, options=None, type=None), _descriptor.EnumValueDescriptor( name='R410', index=396, number=396, options=None, type=None), _descriptor.EnumValueDescriptor( name='R035', index=397, number=397, options=None, type=None), _descriptor.EnumValueDescriptor( name='R096', index=398, number=398, options=None, type=None), _descriptor.EnumValueDescriptor( name='R116', index=399, number=399, options=None, type=None), _descriptor.EnumValueDescriptor( name='R360', index=400, number=400, options=None, type=None), _descriptor.EnumValueDescriptor( name='R418', index=401, number=401, options=None, type=None), _descriptor.EnumValueDescriptor( name='R458', index=402, number=402, options=None, type=None), _descriptor.EnumValueDescriptor( name='R104', index=403, number=403, options=None, type=None), _descriptor.EnumValueDescriptor( name='R608', index=404, number=404, options=None, type=None), _descriptor.EnumValueDescriptor( name='R702', index=405, number=405, options=None, type=None), _descriptor.EnumValueDescriptor( name='R764', index=406, number=406, options=None, type=None), _descriptor.EnumValueDescriptor( name='R626', index=407, number=407, options=None, type=None), _descriptor.EnumValueDescriptor( name='R704', index=408, number=408, options=None, type=None), _descriptor.EnumValueDescriptor( name='R034', index=409, number=409, options=None, type=None), _descriptor.EnumValueDescriptor( name='R004', index=410, number=410, options=None, type=None), _descriptor.EnumValueDescriptor( name='R050', index=411, number=411, options=None, type=None), _descriptor.EnumValueDescriptor( name='R064', index=412, number=412, options=None, type=None), _descriptor.EnumValueDescriptor( name='R356', index=413, number=413, options=None, type=None), _descriptor.EnumValueDescriptor( name='R364', index=414, number=414, options=None, type=None), _descriptor.EnumValueDescriptor( name='R462', index=415, number=415, options=None, type=None), _descriptor.EnumValueDescriptor( name='R524', index=416, number=416, options=None, type=None), _descriptor.EnumValueDescriptor( name='R586', index=417, number=417, options=None, type=None), _descriptor.EnumValueDescriptor( name='R144', index=418, number=418, options=None, type=None), _descriptor.EnumValueDescriptor( name='R145', index=419, number=419, options=None, type=None), _descriptor.EnumValueDescriptor( name='R051', index=420, number=420, options=None, type=None), _descriptor.EnumValueDescriptor( name='R031', index=421, number=421, options=None, type=None), _descriptor.EnumValueDescriptor( name='R048', index=422, number=422, options=None, type=None), _descriptor.EnumValueDescriptor( name='R196', index=423, number=423, options=None, type=None), _descriptor.EnumValueDescriptor( name='R268', index=424, number=424, options=None, type=None), _descriptor.EnumValueDescriptor( name='R368', index=425, number=425, options=None, type=None), _descriptor.EnumValueDescriptor( name='R376', index=426, number=426, options=None, type=None), _descriptor.EnumValueDescriptor( name='R400', index=427, number=427, options=None, type=None), _descriptor.EnumValueDescriptor( name='R414', index=428, number=428, options=None, type=None), _descriptor.EnumValueDescriptor( name='R422', index=429, number=429, options=None, type=None), _descriptor.EnumValueDescriptor( name='R512', index=430, number=430, options=None, type=None), _descriptor.EnumValueDescriptor( name='R634', index=431, number=431, options=None, type=None), _descriptor.EnumValueDescriptor( name='R682', index=432, number=432, options=None, type=None), _descriptor.EnumValueDescriptor( name='R275', index=433, number=433, options=None, type=None), _descriptor.EnumValueDescriptor( name='R760', index=434, number=434, options=None, type=None), _descriptor.EnumValueDescriptor( name='R792', index=435, number=435, options=None, type=None), _descriptor.EnumValueDescriptor( name='R784', index=436, number=436, options=None, type=None), _descriptor.EnumValueDescriptor( name='R887', index=437, number=437, options=None, type=None), _descriptor.EnumValueDescriptor( name='R150', index=438, number=438, options=None, type=None), _descriptor.EnumValueDescriptor( name='R151', index=439, number=439, options=None, type=None), _descriptor.EnumValueDescriptor( name='R112', index=440, number=440, options=None, type=None), _descriptor.EnumValueDescriptor( name='R100', index=441, number=441, options=None, type=None), _descriptor.EnumValueDescriptor( name='R203', index=442, number=442, options=None, type=None), _descriptor.EnumValueDescriptor( name='R348', index=443, number=443, options=None, type=None), _descriptor.EnumValueDescriptor( name='R616', index=444, number=444, options=None, type=None), _descriptor.EnumValueDescriptor( name='R498', index=445, number=445, options=None, type=None), _descriptor.EnumValueDescriptor( name='R642', index=446, number=446, options=None, type=None), _descriptor.EnumValueDescriptor( name='R643', index=447, number=447, options=None, type=None), _descriptor.EnumValueDescriptor( name='R703', index=448, number=448, options=None, type=None), _descriptor.EnumValueDescriptor( name='R804', index=449, number=449, options=None, type=None), _descriptor.EnumValueDescriptor( name='R154', index=450, number=450, options=None, type=None), _descriptor.EnumValueDescriptor( name='R248', index=451, number=451, options=None, type=None), _descriptor.EnumValueDescriptor( name='R830', index=452, number=452, options=None, type=None), _descriptor.EnumValueDescriptor( name='R831', index=453, number=453, options=None, type=None), _descriptor.EnumValueDescriptor( name='R832', index=454, number=454, options=None, type=None), _descriptor.EnumValueDescriptor( name='R680', index=455, number=455, options=None, type=None), _descriptor.EnumValueDescriptor( name='R208', index=456, number=456, options=None, type=None), _descriptor.EnumValueDescriptor( name='R233', index=457, number=457, options=None, type=None), _descriptor.EnumValueDescriptor( name='R234', index=458, number=458, options=None, type=None), _descriptor.EnumValueDescriptor( name='R246', index=459, number=459, options=None, type=None), _descriptor.EnumValueDescriptor( name='R352', index=460, number=460, options=None, type=None), _descriptor.EnumValueDescriptor( name='R372', index=461, number=461, options=None, type=None), _descriptor.EnumValueDescriptor( name='R833', index=462, number=462, options=None, type=None), _descriptor.EnumValueDescriptor( name='R428', index=463, number=463, options=None, type=None), _descriptor.EnumValueDescriptor( name='R440', index=464, number=464, options=None, type=None), _descriptor.EnumValueDescriptor( name='R578', index=465, number=465, options=None, type=None), _descriptor.EnumValueDescriptor( name='R744', index=466, number=466, options=None, type=None), _descriptor.EnumValueDescriptor( name='R752', index=467, number=467, options=None, type=None), _descriptor.EnumValueDescriptor( name='R826', index=468, number=468, options=None, type=None), _descriptor.EnumValueDescriptor( name='R039', index=469, number=469, options=None, type=None), _descriptor.EnumValueDescriptor( name='R008', index=470, number=470, options=None, type=None), _descriptor.EnumValueDescriptor( name='R020', index=471, number=471, options=None, type=None), _descriptor.EnumValueDescriptor( name='R070', index=472, number=472, options=None, type=None), _descriptor.EnumValueDescriptor( name='R191', index=473, number=473, options=None, type=None), _descriptor.EnumValueDescriptor( name='R292', index=474, number=474, options=None, type=None), _descriptor.EnumValueDescriptor( name='R300', index=475, number=475, options=None, type=None), _descriptor.EnumValueDescriptor( name='R336', index=476, number=476, options=None, type=None), _descriptor.EnumValueDescriptor( name='R380', index=477, number=477, options=None, type=None), _descriptor.EnumValueDescriptor( name='R470', index=478, number=478, options=None, type=None), _descriptor.EnumValueDescriptor( name='R499', index=479, number=479, options=None, type=None), _descriptor.EnumValueDescriptor( name='R807', index=480, number=480, options=None, type=None), _descriptor.EnumValueDescriptor( name='R620', index=481, number=481, options=None, type=None), _descriptor.EnumValueDescriptor( name='R674', index=482, number=482, options=None, type=None), _descriptor.EnumValueDescriptor( name='R688', index=483, number=483, options=None, type=None), _descriptor.EnumValueDescriptor( name='R705', index=484, number=484, options=None, type=None), _descriptor.EnumValueDescriptor( name='R724', index=485, number=485, options=None, type=None), _descriptor.EnumValueDescriptor( name='R155', index=486, number=486, options=None, type=None), _descriptor.EnumValueDescriptor( name='R040', index=487, number=487, options=None, type=None), _descriptor.EnumValueDescriptor( name='R056', index=488, number=488, options=None, type=None), _descriptor.EnumValueDescriptor( name='R250', index=489, number=489, options=None, type=None), _descriptor.EnumValueDescriptor( name='R276', index=490, number=490, options=None, type=None), _descriptor.EnumValueDescriptor( name='R438', index=491, number=491, options=None, type=None), _descriptor.EnumValueDescriptor( name='R442', index=492, number=492, options=None, type=None), _descriptor.EnumValueDescriptor( name='R492', index=493, number=493, options=None, type=None), _descriptor.EnumValueDescriptor( name='R528', index=494, number=494, options=None, type=None), _descriptor.EnumValueDescriptor( name='R756', index=495, number=495, options=None, type=None), _descriptor.EnumValueDescriptor( name='R009', index=496, number=496, options=None, type=None), _descriptor.EnumValueDescriptor( name='R053', index=497, number=497, options=None, type=None), _descriptor.EnumValueDescriptor( name='R036', index=498, number=498, options=None, type=None), _descriptor.EnumValueDescriptor( name='R162', index=499, number=499, options=None, type=None), _descriptor.EnumValueDescriptor( name='R166', index=500, number=500, options=None, type=None), _descriptor.EnumValueDescriptor( name='R334', index=501, number=501, options=None, type=None), _descriptor.EnumValueDescriptor( name='R554', index=502, number=502, options=None, type=None), _descriptor.EnumValueDescriptor( name='R574', index=503, number=503, options=None, type=None), _descriptor.EnumValueDescriptor( name='R054', index=504, number=504, options=None, type=None), _descriptor.EnumValueDescriptor( name='R242', index=505, number=505, options=None, type=None), _descriptor.EnumValueDescriptor( name='R540', index=506, number=506, options=None, type=None), _descriptor.EnumValueDescriptor( name='R598', index=507, number=507, options=None, type=None), _descriptor.EnumValueDescriptor( name='R090', index=508, number=508, options=None, type=None), _descriptor.EnumValueDescriptor( name='R548', index=509, number=509, options=None, type=None), _descriptor.EnumValueDescriptor( name='R057', index=510, number=510, options=None, type=None), _descriptor.EnumValueDescriptor( name='R316', index=511, number=511, options=None, type=None), _descriptor.EnumValueDescriptor( name='R296', index=512, number=512, options=None, type=None), _descriptor.EnumValueDescriptor( name='R584', index=513, number=513, options=None, type=None), _descriptor.EnumValueDescriptor( name='R583', index=514, number=514, options=None, type=None), _descriptor.EnumValueDescriptor( name='R520', index=515, number=515, options=None, type=None), _descriptor.EnumValueDescriptor( name='R580', index=516, number=516, options=None, type=None), _descriptor.EnumValueDescriptor( name='R585', index=517, number=517, options=None, type=None), _descriptor.EnumValueDescriptor( name='R581', index=518, number=518, options=None, type=None), _descriptor.EnumValueDescriptor( name='R061', index=519, number=519, options=None, type=None), _descriptor.EnumValueDescriptor( name='R016', index=520, number=520, options=None, type=None), _descriptor.EnumValueDescriptor( name='R184', index=521, number=521, options=None, type=None), _descriptor.EnumValueDescriptor( name='R258', index=522, number=522, options=None, type=None), _descriptor.EnumValueDescriptor( name='R570', index=523, number=523, options=None, type=None), _descriptor.EnumValueDescriptor( name='R612', index=524, number=524, options=None, type=None), _descriptor.EnumValueDescriptor( name='R882', index=525, number=525, options=None, type=None), _descriptor.EnumValueDescriptor( name='R772', index=526, number=526, options=None, type=None), _descriptor.EnumValueDescriptor( name='R776', index=527, number=527, options=None, type=None), _descriptor.EnumValueDescriptor( name='R798', index=528, number=528, options=None, type=None), _descriptor.EnumValueDescriptor( name='R876', index=529, number=529, options=None, type=None), ], containing_type=None, options=None, serialized_start=5337, serialized_end=10561, ) _sym_db.RegisterEnumDescriptor(_LOCATION_COUNTRY) _CLAIM = _descriptor.Descriptor( name='Claim', full_name='pb.Claim', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='stream', full_name='pb.Claim.stream', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='channel', full_name='pb.Claim.channel', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='collection', full_name='pb.Claim.collection', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='repost', full_name='pb.Claim.repost', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='title', full_name='pb.Claim.title', index=4, number=8, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='description', full_name='pb.Claim.description', index=5, number=9, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='thumbnail', full_name='pb.Claim.thumbnail', index=6, number=10, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='tags', full_name='pb.Claim.tags', index=7, number=11, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='languages', full_name='pb.Claim.languages', index=8, number=12, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='locations', full_name='pb.Claim.locations', index=9, number=13, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ _descriptor.OneofDescriptor( name='type', full_name='pb.Claim.type', index=0, containing_type=None, fields=[]), ], serialized_start=20, serialized_end=319, ) _STREAM = _descriptor.Descriptor( name='Stream', full_name='pb.Stream', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='source', full_name='pb.Stream.source', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='author', full_name='pb.Stream.author', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='license', full_name='pb.Stream.license', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='license_url', full_name='pb.Stream.license_url', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='release_time', full_name='pb.Stream.release_time', index=4, number=5, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='fee', full_name='pb.Stream.fee', index=5, number=6, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='image', full_name='pb.Stream.image', index=6, number=10, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='video', full_name='pb.Stream.video', index=7, number=11, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='audio', full_name='pb.Stream.audio', index=8, number=12, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='software', full_name='pb.Stream.software', index=9, number=13, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ _descriptor.OneofDescriptor( name='type', full_name='pb.Stream.type', index=0, containing_type=None, fields=[]), ], serialized_start=322, serialized_end=582, ) _CHANNEL = _descriptor.Descriptor( name='Channel', full_name='pb.Channel', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='public_key', full_name='pb.Channel.public_key', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='email', full_name='pb.Channel.email', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='website_url', full_name='pb.Channel.website_url', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='cover', full_name='pb.Channel.cover', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='featured', full_name='pb.Channel.featured', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=584, serialized_end=709, ) _CLAIMREFERENCE = _descriptor.Descriptor( name='ClaimReference', full_name='pb.ClaimReference', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='claim_hash', full_name='pb.ClaimReference.claim_hash', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=711, serialized_end=747, ) _CLAIMLIST = _descriptor.Descriptor( name='ClaimList', full_name='pb.ClaimList', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='list_type', full_name='pb.ClaimList.list_type', index=0, number=1, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='claim_references', full_name='pb.ClaimList.claim_references', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ _CLAIMLIST_LISTTYPE, ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=750, serialized_end=894, ) _SOURCE = _descriptor.Descriptor( name='Source', full_name='pb.Source', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='hash', full_name='pb.Source.hash', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='name', full_name='pb.Source.name', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='size', full_name='pb.Source.size', index=2, number=3, type=4, cpp_type=4, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='media_type', full_name='pb.Source.media_type', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='url', full_name='pb.Source.url', index=4, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='sd_hash', full_name='pb.Source.sd_hash', index=5, number=6, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='bt_infohash', full_name='pb.Source.bt_infohash', index=6, number=7, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=896, serialized_end=1017, ) _FEE = _descriptor.Descriptor( name='Fee', full_name='pb.Fee', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='currency', full_name='pb.Fee.currency', index=0, number=1, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='address', full_name='pb.Fee.address', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='amount', full_name='pb.Fee.amount', index=2, number=3, type=4, cpp_type=4, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ _FEE_CURRENCY, ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=1020, serialized_end=1155, ) _IMAGE = _descriptor.Descriptor( name='Image', full_name='pb.Image', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='width', full_name='pb.Image.width', index=0, number=1, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='height', full_name='pb.Image.height', index=1, number=2, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=1157, serialized_end=1195, ) _VIDEO = _descriptor.Descriptor( name='Video', full_name='pb.Video', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='width', full_name='pb.Video.width', index=0, number=1, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='height', full_name='pb.Video.height', index=1, number=2, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='duration', full_name='pb.Video.duration', index=2, number=3, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='audio', full_name='pb.Video.audio', index=3, number=15, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=1197, serialized_end=1279, ) _AUDIO = _descriptor.Descriptor( name='Audio', full_name='pb.Audio', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='duration', full_name='pb.Audio.duration', index=0, number=1, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=1281, serialized_end=1306, ) _SOFTWARE = _descriptor.Descriptor( name='Software', full_name='pb.Software', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='os', full_name='pb.Software.os', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ _SOFTWARE_OS, ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=1308, serialized_end=1416, ) _LANGUAGE = _descriptor.Descriptor( name='Language', full_name='pb.Language', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='language', full_name='pb.Language.language', index=0, number=1, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='script', full_name='pb.Language.script', index=1, number=2, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='region', full_name='pb.Language.region', index=2, number=3, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ _LANGUAGE_LANGUAGE, _LANGUAGE_SCRIPT, ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=1419, serialized_end=5202, ) _LOCATION = _descriptor.Descriptor( name='Location', full_name='pb.Location', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='country', full_name='pb.Location.country', index=0, number=1, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='state', full_name='pb.Location.state', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='city', full_name='pb.Location.city', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='code', full_name='pb.Location.code', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='latitude', full_name='pb.Location.latitude', index=4, number=5, type=17, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='longitude', full_name='pb.Location.longitude', index=5, number=6, type=17, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ _LOCATION_COUNTRY, ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=5205, serialized_end=10561, ) _CLAIM.fields_by_name['stream'].message_type = _STREAM _CLAIM.fields_by_name['channel'].message_type = _CHANNEL _CLAIM.fields_by_name['collection'].message_type = _CLAIMLIST _CLAIM.fields_by_name['repost'].message_type = _CLAIMREFERENCE _CLAIM.fields_by_name['thumbnail'].message_type = _SOURCE _CLAIM.fields_by_name['languages'].message_type = _LANGUAGE _CLAIM.fields_by_name['locations'].message_type = _LOCATION _CLAIM.oneofs_by_name['type'].fields.append( _CLAIM.fields_by_name['stream']) _CLAIM.fields_by_name['stream'].containing_oneof = _CLAIM.oneofs_by_name['type'] _CLAIM.oneofs_by_name['type'].fields.append( _CLAIM.fields_by_name['channel']) _CLAIM.fields_by_name['channel'].containing_oneof = _CLAIM.oneofs_by_name['type'] _CLAIM.oneofs_by_name['type'].fields.append( _CLAIM.fields_by_name['collection']) _CLAIM.fields_by_name['collection'].containing_oneof = _CLAIM.oneofs_by_name['type'] _CLAIM.oneofs_by_name['type'].fields.append( _CLAIM.fields_by_name['repost']) _CLAIM.fields_by_name['repost'].containing_oneof = _CLAIM.oneofs_by_name['type'] _STREAM.fields_by_name['source'].message_type = _SOURCE _STREAM.fields_by_name['fee'].message_type = _FEE _STREAM.fields_by_name['image'].message_type = _IMAGE _STREAM.fields_by_name['video'].message_type = _VIDEO _STREAM.fields_by_name['audio'].message_type = _AUDIO _STREAM.fields_by_name['software'].message_type = _SOFTWARE _STREAM.oneofs_by_name['type'].fields.append( _STREAM.fields_by_name['image']) _STREAM.fields_by_name['image'].containing_oneof = _STREAM.oneofs_by_name['type'] _STREAM.oneofs_by_name['type'].fields.append( _STREAM.fields_by_name['video']) _STREAM.fields_by_name['video'].containing_oneof = _STREAM.oneofs_by_name['type'] _STREAM.oneofs_by_name['type'].fields.append( _STREAM.fields_by_name['audio']) _STREAM.fields_by_name['audio'].containing_oneof = _STREAM.oneofs_by_name['type'] _STREAM.oneofs_by_name['type'].fields.append( _STREAM.fields_by_name['software']) _STREAM.fields_by_name['software'].containing_oneof = _STREAM.oneofs_by_name['type'] _CHANNEL.fields_by_name['cover'].message_type = _SOURCE _CHANNEL.fields_by_name['featured'].message_type = _CLAIMLIST _CLAIMLIST.fields_by_name['list_type'].enum_type = _CLAIMLIST_LISTTYPE _CLAIMLIST.fields_by_name['claim_references'].message_type = _CLAIMREFERENCE _CLAIMLIST_LISTTYPE.containing_type = _CLAIMLIST _FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY _FEE_CURRENCY.containing_type = _FEE _VIDEO.fields_by_name['audio'].message_type = _AUDIO _SOFTWARE_OS.containing_type = _SOFTWARE _LANGUAGE.fields_by_name['language'].enum_type = _LANGUAGE_LANGUAGE _LANGUAGE.fields_by_name['script'].enum_type = _LANGUAGE_SCRIPT _LANGUAGE.fields_by_name['region'].enum_type = _LOCATION_COUNTRY _LANGUAGE_LANGUAGE.containing_type = _LANGUAGE _LANGUAGE_SCRIPT.containing_type = _LANGUAGE _LOCATION.fields_by_name['country'].enum_type = _LOCATION_COUNTRY _LOCATION_COUNTRY.containing_type = _LOCATION DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM DESCRIPTOR.message_types_by_name['Stream'] = _STREAM DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL DESCRIPTOR.message_types_by_name['ClaimReference'] = _CLAIMREFERENCE DESCRIPTOR.message_types_by_name['ClaimList'] = _CLAIMLIST DESCRIPTOR.message_types_by_name['Source'] = _SOURCE DESCRIPTOR.message_types_by_name['Fee'] = _FEE DESCRIPTOR.message_types_by_name['Image'] = _IMAGE DESCRIPTOR.message_types_by_name['Video'] = _VIDEO DESCRIPTOR.message_types_by_name['Audio'] = _AUDIO DESCRIPTOR.message_types_by_name['Software'] = _SOFTWARE DESCRIPTOR.message_types_by_name['Language'] = _LANGUAGE DESCRIPTOR.message_types_by_name['Location'] = _LOCATION Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict( DESCRIPTOR = _CLAIM, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Claim) )) _sym_db.RegisterMessage(Claim) Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict( DESCRIPTOR = _STREAM, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Stream) )) _sym_db.RegisterMessage(Stream) Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict( DESCRIPTOR = _CHANNEL, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Channel) )) _sym_db.RegisterMessage(Channel) ClaimReference = _reflection.GeneratedProtocolMessageType('ClaimReference', (_message.Message,), dict( DESCRIPTOR = _CLAIMREFERENCE, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.ClaimReference) )) _sym_db.RegisterMessage(ClaimReference) ClaimList = _reflection.GeneratedProtocolMessageType('ClaimList', (_message.Message,), dict( DESCRIPTOR = _CLAIMLIST, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.ClaimList) )) _sym_db.RegisterMessage(ClaimList) Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict( DESCRIPTOR = _SOURCE, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Source) )) _sym_db.RegisterMessage(Source) Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict( DESCRIPTOR = _FEE, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Fee) )) _sym_db.RegisterMessage(Fee) Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict( DESCRIPTOR = _IMAGE, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Image) )) _sym_db.RegisterMessage(Image) Video = _reflection.GeneratedProtocolMessageType('Video', (_message.Message,), dict( DESCRIPTOR = _VIDEO, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Video) )) _sym_db.RegisterMessage(Video) Audio = _reflection.GeneratedProtocolMessageType('Audio', (_message.Message,), dict( DESCRIPTOR = _AUDIO, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Audio) )) _sym_db.RegisterMessage(Audio) Software = _reflection.GeneratedProtocolMessageType('Software', (_message.Message,), dict( DESCRIPTOR = _SOFTWARE, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Software) )) _sym_db.RegisterMessage(Software) Language = _reflection.GeneratedProtocolMessageType('Language', (_message.Message,), dict( DESCRIPTOR = _LANGUAGE, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Language) )) _sym_db.RegisterMessage(Language) Location = _reflection.GeneratedProtocolMessageType('Location', (_message.Message,), dict( DESCRIPTOR = _LOCATION, __module__ = 'claim_pb2' # @@protoc_insertion_point(class_scope:pb.Location) )) _sym_db.RegisterMessage(Location) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v2/purchase_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: purchase.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='purchase.proto', package='pb', syntax='proto3', serialized_pb=_b('\n\x0epurchase.proto\x12\x02pb\"\x1e\n\x08Purchase\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _PURCHASE = _descriptor.Descriptor( name='Purchase', full_name='pb.Purchase', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='claim_hash', full_name='pb.Purchase.claim_hash', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=22, serialized_end=52, ) DESCRIPTOR.message_types_by_name['Purchase'] = _PURCHASE Purchase = _reflection.GeneratedProtocolMessageType('Purchase', (_message.Message,), dict( DESCRIPTOR = _PURCHASE, __module__ = 'purchase_pb2' # @@protoc_insertion_point(class_scope:pb.Purchase) )) _sym_db.RegisterMessage(Purchase) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v2/result_pb2.py ================================================ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: result.proto """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='result.proto', package='pb', syntax='proto3', serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb', create_key=_descriptor._internal_create_key, serialized_pb=b'\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe6\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_score\x18\x16 \x01(\x01\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.OutputB&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3' ) _ERROR_CODE = _descriptor.EnumDescriptor( name='Code', full_name='pb.Error.Code', filename=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key, values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_CODE', index=0, number=0, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), _descriptor.EnumValueDescriptor( name='NOT_FOUND', index=1, number=1, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), _descriptor.EnumValueDescriptor( name='INVALID', index=2, number=2, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), _descriptor.EnumValueDescriptor( name='BLOCKED', index=3, number=3, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), ], containing_type=None, serialized_options=None, serialized_start=744, serialized_end=809, ) _sym_db.RegisterEnumDescriptor(_ERROR_CODE) _OUTPUTS = _descriptor.Descriptor( name='Outputs', full_name='pb.Outputs', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='txos', full_name='pb.Outputs.txos', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='extra_txos', full_name='pb.Outputs.extra_txos', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='total', full_name='pb.Outputs.total', index=2, number=3, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='offset', full_name='pb.Outputs.offset', index=3, number=4, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='blocked', full_name='pb.Outputs.blocked', index=4, number=5, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='blocked_total', full_name='pb.Outputs.blocked_total', index=5, number=6, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=21, serialized_end=172, ) _OUTPUT = _descriptor.Descriptor( name='Output', full_name='pb.Output', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='tx_hash', full_name='pb.Output.tx_hash', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='nout', full_name='pb.Output.nout', index=1, number=2, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='height', full_name='pb.Output.height', index=2, number=3, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='claim', full_name='pb.Output.claim', index=3, number=7, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='error', full_name='pb.Output.error', index=4, number=15, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ _descriptor.OneofDescriptor( name='meta', full_name='pb.Output.meta', index=0, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), ], serialized_start=174, serialized_end=297, ) _CLAIMMETA = _descriptor.Descriptor( name='ClaimMeta', full_name='pb.ClaimMeta', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='channel', full_name='pb.ClaimMeta.channel', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='repost', full_name='pb.ClaimMeta.repost', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='short_url', full_name='pb.ClaimMeta.short_url', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4, number=5, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5, number=6, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6, number=7, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7, number=8, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8, number=9, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9, number=10, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='reposted', full_name='pb.ClaimMeta.reposted', index=10, number=11, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11, number=20, type=4, cpp_type=4, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12, number=21, type=4, cpp_type=4, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='trending_score', full_name='pb.ClaimMeta.trending_score', index=13, number=22, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=300, serialized_end=658, ) _ERROR = _descriptor.Descriptor( name='Error', full_name='pb.Error', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='code', full_name='pb.Error.code', index=0, number=1, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='text', full_name='pb.Error.text', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='blocked', full_name='pb.Error.blocked', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], nested_types=[], enum_types=[ _ERROR_CODE, ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=661, serialized_end=809, ) _BLOCKED = _descriptor.Descriptor( name='Blocked', full_name='pb.Blocked', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='count', full_name='pb.Blocked.count', index=0, number=1, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='channel', full_name='pb.Blocked.channel', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=811, serialized_end=864, ) _OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT _OUTPUTS.fields_by_name['extra_txos'].message_type = _OUTPUT _OUTPUTS.fields_by_name['blocked'].message_type = _BLOCKED _OUTPUT.fields_by_name['claim'].message_type = _CLAIMMETA _OUTPUT.fields_by_name['error'].message_type = _ERROR _OUTPUT.oneofs_by_name['meta'].fields.append( _OUTPUT.fields_by_name['claim']) _OUTPUT.fields_by_name['claim'].containing_oneof = _OUTPUT.oneofs_by_name['meta'] _OUTPUT.oneofs_by_name['meta'].fields.append( _OUTPUT.fields_by_name['error']) _OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta'] _CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT _CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT _ERROR.fields_by_name['code'].enum_type = _ERROR_CODE _ERROR.fields_by_name['blocked'].message_type = _BLOCKED _ERROR_CODE.containing_type = _ERROR _BLOCKED.fields_by_name['channel'].message_type = _OUTPUT DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA DESCRIPTOR.message_types_by_name['Error'] = _ERROR DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED _sym_db.RegisterFileDescriptor(DESCRIPTOR) Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), { 'DESCRIPTOR' : _OUTPUTS, '__module__' : 'result_pb2' # @@protoc_insertion_point(class_scope:pb.Outputs) }) _sym_db.RegisterMessage(Outputs) Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), { 'DESCRIPTOR' : _OUTPUT, '__module__' : 'result_pb2' # @@protoc_insertion_point(class_scope:pb.Output) }) _sym_db.RegisterMessage(Output) ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), { 'DESCRIPTOR' : _CLAIMMETA, '__module__' : 'result_pb2' # @@protoc_insertion_point(class_scope:pb.ClaimMeta) }) _sym_db.RegisterMessage(ClaimMeta) Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), { 'DESCRIPTOR' : _ERROR, '__module__' : 'result_pb2' # @@protoc_insertion_point(class_scope:pb.Error) }) _sym_db.RegisterMessage(Error) Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), { 'DESCRIPTOR' : _BLOCKED, '__module__' : 'result_pb2' # @@protoc_insertion_point(class_scope:pb.Blocked) }) _sym_db.RegisterMessage(Blocked) DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v2/support_pb2.py ================================================ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: support.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='support.proto', package='pb', syntax='proto3', serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _SUPPORT = _descriptor.Descriptor( name='Support', full_name='pb.Support', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='emoji', full_name='pb.Support.emoji', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='comment', full_name='pb.Support.comment', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=21, serialized_end=62, ) DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict( DESCRIPTOR = _SUPPORT, __module__ = 'support_pb2' # @@protoc_insertion_point(class_scope:pb.Support) )) _sym_db.RegisterMessage(Support) # @@protoc_insertion_point(module_scope) ================================================ FILE: lbry/schema/types/v2/wallet.json ================================================ { "title": "Wallet", "description": "An LBC wallet", "type": "object", "required": ["name", "version", "accounts", "preferences"], "additionalProperties": false, "properties": { "name": { "description": "Human readable name for this wallet", "type": "string" }, "version": { "description": "Wallet spec version", "type": "integer", "$comment": "Should this be a string? We may need some sort of decimal type if we want exact decimal versions." }, "accounts": { "description": "Accounts associated with this wallet", "type": "array", "items": { "type": "object", "required": ["address_generator", "certificates", "encrypted", "ledger", "modified_on", "name", "private_key", "public_key", "seed"], "additionalProperties": false, "properties": { "address_generator": { "description": "Higher level manager of either singular or deterministically generated addresses", "type": "object", "oneOf": [ { "required": ["name", "change", "receiving"], "additionalProperties": false, "properties": { "name": { "description": "type of address generator: a deterministic chain of addresses", "enum": ["deterministic-chain"], "type": "string" }, "change": { "$ref": "#/$defs/address_manager", "description": "Manager for deterministically generated change address (not used for single address)" }, "receiving": { "$ref": "#/$defs/address_manager", "description": "Manager for deterministically generated receiving address (not used for single address)" } } }, { "required": ["name"], "additionalProperties": false, "properties": { "name": { "description": "type of address generator: a single address", "enum": ["single-address"], "type": "string" } } } ] }, "certificates": { "type": "object", "description": "Channel keys. Mapping from public key address to pem-formatted private key.", "additionalProperties": {"type": "string"} }, "encrypted": { "type": "boolean", "description": "Whether private key and seed are encrypted with a password" }, "ledger": { "description": "Which network to use", "type": "string", "examples": [ "lbc_mainnet", "lbc_testnet" ] }, "modified_on": { "description": "last modified time in Unix Time", "type": "integer" }, "name": { "description": "Name for account, possibly human readable", "type": "string" }, "private_key": { "description": "Private key for address if `address_generator` is a single address. Root of chain of private keys for addresses if `address_generator` is a deterministic chain of addresses. Encrypted if `encrypted` is true.", "type": "string" }, "public_key": { "description": "Public key for address if `address_generator` is a single address. Root of chain of public keys for addresses if `address_generator` is a deterministic chain of addresses.", "type": "string" }, "seed": { "description": "Human readable representation of `private_key`. encrypted if `encrypted` is set to `true`", "type": "string" } } } }, "preferences": { "description": "Timestamped application-level preferences. Values can be objects or of a primitive type.", "$comment": "enable-sync is seen in example wallet. encrypt-on-disk is seen in example wallet. they both have a boolean `value` field. Do we want them explicitly defined here? local and shared seem to have at least a similar structure (type, value [yes, again], version), value being the free-form part. Should we define those here? Or can there be any key under preferences, and `value` be literally be anything in any form?", "type": "object", "additionalProperties": { "type": "object", "required": ["ts", "value"], "additionalProperties": false, "properties": { "ts": { "type": "number", "description": "When the item was set, in Unix time format.", "$comment": "Do we want a string (decimal)?" }, "value": { "$comment": "Sometimes this has been an object, sometimes just a boolean. I don't want to prescribe anything." } } } } }, "$defs": { "address_manager": { "description": "Manager for deterministically generated addresses", "type": "object", "required": ["gap", "maximum_uses_per_address"], "additionalProperties": false, "properties": { "gap": { "description": "Maximum allowed consecutive generated addresses with no transactions", "type": "integer" }, "maximum_uses_per_address": { "description": "Maximum number of uses for each generated address", "type": "integer" } } } } } ================================================ FILE: lbry/schema/url.py ================================================ import re import unicodedata from typing import NamedTuple, Tuple def _create_url_regex(): # see https://spec.lbry.com/ and test_url.py invalid_names_regex = \ r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \ r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+" def _named(name, regex): return "(?P<" + name + ">" + regex + ")" def _group(regex): return "(?:" + regex + ")" def _oneof(*choices): return _group('|'.join(choices)) def _claim(name, prefix=""): return _group( _named(name+"_name", prefix + invalid_names_regex) + _oneof( _group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")), _group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*')) ) + '?' ) return ( '^' + _named("scheme", "lbry://") + '?' + _oneof( _group(_claim("channel_with_stream", "@") + "/" + _claim("stream_in_channel")), _claim("channel", "@"), _claim("stream") ) + '$' ) URL_REGEX = _create_url_regex() def normalize_name(name): return unicodedata.normalize('NFD', name).casefold() class PathSegment(NamedTuple): name: str claim_id: str = None amount_order: int = None @property def normalized(self): return normalize_name(self.name) @property def is_shortid(self): return self.claim_id is not None and len(self.claim_id) < 40 @property def is_fullid(self): return self.claim_id is not None and len(self.claim_id) == 40 def to_dict(self): q = {'name': self.name} if self.claim_id is not None: q['claim_id'] = self.claim_id if self.amount_order is not None: q['amount_order'] = self.amount_order return q def __str__(self): if self.claim_id is not None: return f"{self.name}:{self.claim_id}" elif self.amount_order is not None: return f"{self.name}${self.amount_order}" return self.name class URL(NamedTuple): stream: PathSegment channel: PathSegment @property def has_channel(self): return self.channel is not None @property def has_stream(self): return self.stream is not None @property def has_stream_in_channel(self): return self.has_channel and self.has_stream @property def parts(self) -> Tuple: if self.has_stream_in_channel: return self.channel, self.stream if self.has_channel: return self.channel, return self.stream, def __str__(self): return f"lbry://{'/'.join(str(p) for p in self.parts)}" @classmethod def parse(cls, url): match = re.match(URL_REGEX, url) if match is None: raise ValueError('Invalid LBRY URL') segments = {} parts = match.groupdict() for segment in ('channel', 'stream', 'channel_with_stream', 'stream_in_channel'): if parts[f'{segment}_name'] is not None: segments[segment] = PathSegment( parts[f'{segment}_name'], parts[f'{segment}_claim_id'], parts[f'{segment}_amount_order'] ) if 'channel_with_stream' in segments: segments['channel'] = segments['channel_with_stream'] segments['stream'] = segments['stream_in_channel'] return cls(segments.get('stream', None), segments.get('channel', None)) ================================================ FILE: lbry/stream/__init__.py ================================================ ================================================ FILE: lbry/stream/background_downloader.py ================================================ import asyncio import logging from lbry.stream.downloader import StreamDownloader log = logging.getLogger(__name__) class BackgroundDownloader: def __init__(self, conf, storage, blob_manager, dht_node=None): self.storage = storage self.blob_manager = blob_manager self.node = dht_node self.conf = conf async def download_blobs(self, sd_hash): downloader = StreamDownloader(asyncio.get_running_loop(), self.conf, self.blob_manager, sd_hash) try: await downloader.start(self.node, save_stream=False) for blob_info in downloader.descriptor.blobs[:-1]: await downloader.download_stream_blob(blob_info) except ValueError: return except asyncio.CancelledError: log.debug("Cancelled background downloader") raise except Exception: log.error("Unexpected download error on background downloader") finally: downloader.stop() ================================================ FILE: lbry/stream/descriptor.py ================================================ import os import json import binascii import logging import typing import asyncio import time import re from collections import OrderedDict from cryptography.hazmat.primitives.ciphers.algorithms import AES from lbry.blob import MAX_BLOB_SIZE from lbry.blob.blob_info import BlobInfo from lbry.blob.blob_file import AbstractBlob, BlobFile from lbry.utils import get_lbry_hash_obj from lbry.error import InvalidStreamDescriptorError log = logging.getLogger(__name__) RE_ILLEGAL_FILENAME_CHARS = re.compile( r'(' r'[<>:"/\\|?*]+|' # Illegal characters r'[\x00-\x1F]+|' # All characters in range 0-31 r'[ \t]*(\.)+[ \t]*$|' # Dots at the end r'(^[ \t]+|[ \t]+$)|' # Leading and trailing whitespace r'^CON$|^PRN$|^AUX$|' # Illegal names r'^NUL$|^COM[1-9]$|^LPT[1-9]$' # ... r')' ) def format_sd_info(stream_name: str, key: str, suggested_file_name: str, stream_hash: str, blobs: typing.List[typing.Dict]) -> typing.Dict: return { "stream_type": "lbryfile", "stream_name": stream_name, "key": key, "suggested_file_name": suggested_file_name, "stream_hash": stream_hash, "blobs": blobs } def random_iv_generator() -> typing.Generator[bytes, None, None]: while 1: yield os.urandom(AES.block_size // 8) def read_bytes(file_path: str, offset: int, to_read: int): with open(file_path, 'rb') as f: f.seek(offset) return f.read(to_read) async def file_reader(file_path: str): length = int(os.stat(file_path).st_size) offset = 0 while offset < length: bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1) if not bytes_to_read: break blob_bytes = await asyncio.get_event_loop().run_in_executor( None, read_bytes, file_path, offset, bytes_to_read ) yield blob_bytes offset += bytes_to_read def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'): file_name, ext = os.path.splitext(dirty_name) file_name = re.sub(RE_ILLEGAL_FILENAME_CHARS, '', file_name) ext = re.sub(RE_ILLEGAL_FILENAME_CHARS, '', ext) if not file_name: log.warning('Unable to sanitize file name for %s, returning default value %s', dirty_name, default_file_name) file_name = default_file_name if len(ext) > 1: file_name += ext return file_name class StreamDescriptor: __slots__ = [ 'loop', 'blob_dir', 'stream_name', 'key', 'suggested_file_name', 'blobs', 'stream_hash', 'sd_hash' ] def __init__(self, loop: asyncio.AbstractEventLoop, blob_dir: str, stream_name: str, key: str, suggested_file_name: str, blobs: typing.List[BlobInfo], stream_hash: typing.Optional[str] = None, sd_hash: typing.Optional[str] = None): self.loop = loop self.blob_dir = blob_dir self.stream_name = stream_name self.key = key self.suggested_file_name = suggested_file_name self.blobs = blobs self.stream_hash = stream_hash or self.get_stream_hash() self.sd_hash = sd_hash @property def length(self) -> int: return len(self.as_json()) def get_stream_hash(self) -> str: return self.calculate_stream_hash( binascii.hexlify(self.stream_name.encode()), self.key.encode(), binascii.hexlify(self.suggested_file_name.encode()), [blob_info.as_dict() for blob_info in self.blobs] ) def calculate_sd_hash(self) -> str: h = get_lbry_hash_obj() h.update(self.as_json()) return h.hexdigest() def as_json(self) -> bytes: return json.dumps( format_sd_info(binascii.hexlify(self.stream_name.encode()).decode(), self.key, binascii.hexlify(self.suggested_file_name.encode()).decode(), self.stream_hash, [blob_info.as_dict() for blob_info in self.blobs]), sort_keys=True ).encode() def old_sort_json(self) -> bytes: blobs = [] for blob in self.blobs: blobs.append(OrderedDict( [('length', blob.length), ('blob_num', blob.blob_num), ('iv', blob.iv)] if not blob.blob_hash else [('length', blob.length), ('blob_num', blob.blob_num), ('blob_hash', blob.blob_hash), ('iv', blob.iv)] )) if not blob.blob_hash: break return json.dumps( OrderedDict([ ('stream_name', binascii.hexlify(self.stream_name.encode()).decode()), ('blobs', blobs), ('stream_type', 'lbryfile'), ('key', self.key), ('suggested_file_name', binascii.hexlify(self.suggested_file_name.encode()).decode()), ('stream_hash', self.stream_hash), ]) ).encode() def calculate_old_sort_sd_hash(self) -> str: h = get_lbry_hash_obj() h.update(self.old_sort_json()) return h.hexdigest() async def make_sd_blob( self, blob_file_obj: typing.Optional[AbstractBlob] = None, old_sort: typing.Optional[bool] = False, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None, added_on: float = None, is_mine: bool = False ): sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash() if not old_sort: sd_data = self.as_json() else: sd_data = self.old_sort_json() sd_blob = blob_file_obj or BlobFile( self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir, added_on, is_mine ) if blob_file_obj: blob_file_obj.set_length(len(sd_data)) if not sd_blob.get_is_verified(): writer = sd_blob.get_blob_writer() writer.write(sd_data) await sd_blob.verified.wait() sd_blob.close() return sd_blob @classmethod def _from_stream_descriptor_blob(cls, loop: asyncio.AbstractEventLoop, blob_dir: str, blob: AbstractBlob) -> 'StreamDescriptor': with blob.reader_context() as blob_reader: json_bytes = blob_reader.read() try: decoded = json.loads(json_bytes.decode()) except json.JSONDecodeError: blob.delete() raise InvalidStreamDescriptorError("Does not decode as valid JSON") if decoded['blobs'][-1]['length'] != 0: raise InvalidStreamDescriptorError("Does not end with a zero-length blob.") if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]): raise InvalidStreamDescriptorError("Contains zero-length data blob") if 'blob_hash' in decoded['blobs'][-1]: raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash") if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])): raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs") added_on = time.time() descriptor = cls( loop, blob_dir, binascii.unhexlify(decoded['stream_name']).decode(), decoded['key'], binascii.unhexlify(decoded['suggested_file_name']).decode(), [BlobInfo(info['blob_num'], info['length'], info['iv'], added_on, info.get('blob_hash')) for info in decoded['blobs']], decoded['stream_hash'], blob.blob_hash ) if descriptor.get_stream_hash() != decoded['stream_hash']: raise InvalidStreamDescriptorError("Stream hash does not match stream metadata") return descriptor @classmethod async def from_stream_descriptor_blob(cls, loop: asyncio.AbstractEventLoop, blob_dir: str, blob: AbstractBlob) -> 'StreamDescriptor': if not blob.is_readable(): raise InvalidStreamDescriptorError(f"unreadable/missing blob: {blob.blob_hash}") return await loop.run_in_executor(None, cls._from_stream_descriptor_blob, loop, blob_dir, blob) @staticmethod def get_blob_hashsum(blob_dict: typing.Dict): length = blob_dict['length'] if length != 0: blob_hash = blob_dict['blob_hash'] else: blob_hash = None blob_num = blob_dict['blob_num'] iv = blob_dict['iv'] blob_hashsum = get_lbry_hash_obj() if length != 0: blob_hashsum.update(blob_hash.encode()) blob_hashsum.update(str(blob_num).encode()) blob_hashsum.update(iv.encode()) blob_hashsum.update(str(length).encode()) return blob_hashsum.digest() @staticmethod def calculate_stream_hash(hex_stream_name: bytes, key: bytes, hex_suggested_file_name: bytes, blob_infos: typing.List[typing.Dict]) -> str: h = get_lbry_hash_obj() h.update(hex_stream_name) h.update(key) h.update(hex_suggested_file_name) blobs_hashsum = get_lbry_hash_obj() for blob in blob_infos: blobs_hashsum.update(StreamDescriptor.get_blob_hashsum(blob)) h.update(blobs_hashsum.digest()) return h.hexdigest() @classmethod async def create_stream( cls, loop: asyncio.AbstractEventLoop, blob_dir: str, file_path: str, key: typing.Optional[bytes] = None, iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None, old_sort: bool = False, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None) -> 'StreamDescriptor': blobs: typing.List[BlobInfo] = [] iv_generator = iv_generator or random_iv_generator() key = key or os.urandom(AES.block_size // 8) blob_num = -1 added_on = time.time() async for blob_bytes in file_reader(file_path): blob_num += 1 blob_info = await BlobFile.create_from_unencrypted( loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, added_on, True, blob_completed_callback ) blobs.append(blob_info) blobs.append( # add the stream terminator BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), added_on, None, True) ) file_name = os.path.basename(file_path) suggested_file_name = sanitize_file_name(file_name) descriptor = cls( loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs ) sd_blob = await descriptor.make_sd_blob( old_sort=old_sort, blob_completed_callback=blob_completed_callback, added_on=added_on, is_mine=True ) descriptor.sd_hash = sd_blob.blob_hash return descriptor def lower_bound_decrypted_length(self) -> int: length = sum(blob.length - 1 for blob in self.blobs[:-2]) return length + self.blobs[-2].length - (AES.block_size // 8) def upper_bound_decrypted_length(self) -> int: return self.lower_bound_decrypted_length() + (AES.block_size // 8) @classmethod async def recover(cls, blob_dir: str, sd_blob: 'AbstractBlob', stream_hash: str, stream_name: str, suggested_file_name: str, key: str, blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']: descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name, blobs, stream_hash, sd_blob.blob_hash) if descriptor.calculate_sd_hash() == sd_blob.blob_hash: # first check for a normal valid sd old_sort = False elif descriptor.calculate_old_sort_sd_hash() == sd_blob.blob_hash: # check if old field sorting works old_sort = True else: return await descriptor.make_sd_blob(sd_blob, old_sort) return descriptor ================================================ FILE: lbry/stream/downloader.py ================================================ import asyncio import typing import logging import binascii from lbry.dht.node import get_kademlia_peers_from_hosts from lbry.error import DownloadSDTimeoutError from lbry.utils import lru_cache_concurrent from lbry.stream.descriptor import StreamDescriptor from lbry.blob_exchange.downloader import BlobDownloader from lbry.torrent.tracker import enqueue_tracker_search if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.dht.node import Node from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_file import AbstractBlob from lbry.blob.blob_info import BlobInfo log = logging.getLogger(__name__) class StreamDownloader: def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', sd_hash: str, descriptor: typing.Optional[StreamDescriptor] = None): self.loop = loop self.config = config self.blob_manager = blob_manager self.sd_hash = sd_hash self.search_queue = asyncio.Queue() # blob hashes to feed into the iterative finder self.peer_queue = asyncio.Queue() # new peers to try self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue) self.descriptor: typing.Optional[StreamDescriptor] = descriptor self.node: typing.Optional['Node'] = None self.accumulate_task: typing.Optional[asyncio.Task] = None self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None self.fixed_peers_delay: typing.Optional[float] = None self.added_fixed_peers = False self.time_to_descriptor: typing.Optional[float] = None self.time_to_first_bytes: typing.Optional[float] = None async def cached_read_blob(blob_info: 'BlobInfo') -> bytes: return await self.read_blob(blob_info, 2) if self.blob_manager.decrypted_blob_lru_cache is not None: cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)( cached_read_blob ) self.cached_read_blob = cached_read_blob async def add_fixed_peers(self): def _add_fixed_peers(fixed_peers): self.peer_queue.put_nowait(fixed_peers) self.added_fixed_peers = True if not self.config.fixed_peers: return if 'dht' in self.config.components_to_skip or not self.node or not \ len(self.node.protocol.routing_table.get_peers()) > 0: self.fixed_peers_delay = 0.0 else: self.fixed_peers_delay = self.config.fixed_peer_delay fixed_peers = await get_kademlia_peers_from_hosts(self.config.fixed_peers) self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _add_fixed_peers, fixed_peers) async def load_descriptor(self, connection_id: int = 0): # download or get the sd blob sd_blob = self.blob_manager.get_blob(self.sd_hash) if not sd_blob.get_is_verified(): try: now = self.loop.time() sd_blob = await asyncio.wait_for( self.blob_downloader.download_blob(self.sd_hash, connection_id), self.config.blob_download_timeout ) log.info("downloaded sd blob %s", self.sd_hash) self.time_to_descriptor = self.loop.time() - now except asyncio.TimeoutError: raise DownloadSDTimeoutError(self.sd_hash) # parse the descriptor self.descriptor = await StreamDescriptor.from_stream_descriptor_blob( self.loop, self.blob_manager.blob_dir, sd_blob ) log.info("loaded stream manifest %s", self.sd_hash) async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0, save_stream=True): # set up peer accumulation self.node = node or self.node # fixme: this shouldnt be set here! if self.node: if self.accumulate_task and not self.accumulate_task.done(): self.accumulate_task.cancel() _, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue) await self.add_fixed_peers() enqueue_tracker_search(bytes.fromhex(self.sd_hash), self.peer_queue) # start searching for peers for the sd hash self.search_queue.put_nowait(self.sd_hash) log.info("searching for peers for stream %s", self.sd_hash) if not self.descriptor: await self.load_descriptor(connection_id) if not await self.blob_manager.storage.stream_exists(self.sd_hash) and save_stream: await self.blob_manager.storage.store_stream( self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor ) async def download_stream_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> 'AbstractBlob': if not filter(lambda b: b.blob_hash == blob_info.blob_hash, self.descriptor.blobs[:-1]): raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}") blob = await asyncio.wait_for( self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id), self.config.blob_download_timeout * 10 ) return blob def decrypt_blob(self, blob_info: 'BlobInfo', blob: 'AbstractBlob') -> bytes: return blob.decrypt( binascii.unhexlify(self.descriptor.key.encode()), binascii.unhexlify(blob_info.iv.encode()) ) async def read_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> bytes: start = None if self.time_to_first_bytes is None: start = self.loop.time() blob = await self.download_stream_blob(blob_info, connection_id) decrypted = self.decrypt_blob(blob_info, blob) if start: self.time_to_first_bytes = self.loop.time() - start return decrypted def stop(self): if self.accumulate_task: self.accumulate_task.cancel() self.accumulate_task = None if self.fixed_peers_handle: self.fixed_peers_handle.cancel() self.fixed_peers_handle = None self.blob_downloader.close() ================================================ FILE: lbry/stream/managed_stream.py ================================================ import os import asyncio import time import typing import logging from typing import Optional from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable from lbry.error import DownloadSDTimeoutError from lbry.schema.mime_types import guess_media_type from lbry.stream.downloader import StreamDownloader from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name from lbry.stream.reflector.client import StreamReflectorClient from lbry.extras.daemon.storage import StoredContentClaim from lbry.blob import MAX_BLOB_SIZE from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_info import BlobInfo from lbry.extras.daemon.analytics import AnalyticsManager from lbry.wallet.transaction import Transaction log = logging.getLogger(__name__) def _get_next_available_file_name(download_directory: str, file_name: str) -> str: base_name, ext = os.path.splitext(os.path.basename(file_name)) i = 0 while os.path.isfile(os.path.join(download_directory, file_name)): i += 1 file_name = "%s_%i%s" % (base_name, i, ext) return file_name async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download_directory: str, file_name: str) -> str: return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name) class ManagedStream(ManagedDownloadSource): def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None, status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED, claim: Optional[StoredContentClaim] = None, download_id: Optional[str] = None, rowid: Optional[int] = None, descriptor: Optional[StreamDescriptor] = None, content_fee: Optional['Transaction'] = None, analytics_manager: Optional['AnalyticsManager'] = None, added_on: Optional[int] = None): super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim, download_id, rowid, content_fee, analytics_manager, added_on) self.blob_manager = blob_manager self.purchase_receipt = None self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor) self.analytics_manager = analytics_manager self.reflector_progress = 0 self.uploading_to_reflector = False self.file_output_task: typing.Optional[asyncio.Task] = None self.delayed_stop_task: typing.Optional[asyncio.Task] = None self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = [] self.fully_reflected = asyncio.Event() self.streaming = asyncio.Event() self._running = asyncio.Event() @property def sd_hash(self) -> str: return self.identifier @property def is_fully_reflected(self) -> bool: return self.fully_reflected.is_set() @property def descriptor(self) -> StreamDescriptor: return self.downloader.descriptor @property def stream_hash(self) -> str: return self.descriptor.stream_hash @property def file_name(self) -> Optional[str]: return self._file_name or self.suggested_file_name @property def suggested_file_name(self) -> Optional[str]: first_option = ((self.descriptor and self.descriptor.suggested_file_name) or '').strip() return sanitize_file_name(first_option or (self.stream_claim_info and self.stream_claim_info.claim and self.stream_claim_info.claim.stream.source.name)) @property def stream_name(self) -> Optional[str]: first_option = ((self.descriptor and self.descriptor.stream_name) or '').strip() return first_option or (self.stream_claim_info and self.stream_claim_info.claim and self.stream_claim_info.claim.stream.source.name) @property def written_bytes(self) -> int: return 0 if not self.output_file_exists else os.stat(self.full_path).st_size @property def completed(self): return self.written_bytes >= self.descriptor.lower_bound_decrypted_length() @property def stream_url(self): return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}" async def update_status(self, status: str): assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED] self._status = status await self.blob_manager.storage.change_file_status(self.stream_hash, status) @property def blobs_completed(self) -> int: return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0 for b in self.descriptor.blobs[:-1]]) @property def blobs_in_stream(self) -> int: return len(self.descriptor.blobs) - 1 @property def blobs_remaining(self) -> int: return self.blobs_in_stream - self.blobs_completed @property def mime_type(self): return guess_media_type(os.path.basename(self.suggested_file_name))[0] @property def download_path(self): return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None # @classmethod # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', # file_path: str, key: Optional[bytes] = None, # iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource': # """ # Generate a stream from a file and save it to the db # """ # descriptor = await StreamDescriptor.create_stream( # loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator, # blob_completed_callback=blob_manager.blob_completed # ) # await blob_manager.storage.store_stream( # blob_manager.get_blob(descriptor.sd_hash), descriptor # ) # row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path), # os.path.dirname(file_path), 0) # return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path), # os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor) async def start(self, timeout: Optional[float] = None, save_now: bool = False): timeout = timeout or self.config.download_timeout if self._running.is_set(): return log.info("start downloader for stream (sd hash: %s)", self.sd_hash) self._running.set() try: await asyncio.wait_for(self.downloader.start(), timeout) except asyncio.TimeoutError: self._running.clear() raise DownloadSDTimeoutError(self.sd_hash) if self.delayed_stop_task and not self.delayed_stop_task.done(): self.delayed_stop_task.cancel() self.delayed_stop_task = self.loop.create_task(self._delayed_stop()) if not await self.blob_manager.storage.file_exists(self.sd_hash): if save_now: if not self._file_name: self._file_name = await get_next_available_file_name( self.loop, self.download_directory, self._file_name or sanitize_file_name(self.suggested_file_name) ) file_name, download_dir = self._file_name, self.download_directory else: file_name, download_dir = None, None self._added_on = int(time.time()) self.rowid = await self.blob_manager.storage.save_downloaded_file( self.stream_hash, file_name, download_dir, 0.0, added_on=self._added_on ) if self.status != self.STATUS_RUNNING: await self.update_status(self.STATUS_RUNNING) async def stop(self, finished: bool = False): """ Stop any running save/stream tasks as well as the downloader and update the status in the database """ await self.stop_tasks() if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING: await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED) async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\ -> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]: if start_blob_num >= len(self.descriptor.blobs[:-1]): raise IndexError(start_blob_num) for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]): assert i + start_blob_num == blob_info.blob_num if connection_id == self.STREAMING_ID: decrypted = await self.downloader.cached_read_blob(blob_info) else: decrypted = await self.downloader.read_blob(blob_info, connection_id) yield (blob_info, decrypted) async def stream_file(self, request: Request) -> StreamResponse: log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id, self.sd_hash[:6]) headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers( request.headers.get('range', 'bytes=0-') ) await self.start() response = StreamResponse( status=206, headers=headers ) await response.prepare(request) self.streaming_responses.append((request, response)) self.streaming.set() wrote = 0 try: async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=self.STREAMING_ID): if not wrote: decrypted = decrypted[first_blob_start_offset:] if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size): decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * (MAX_BLOB_SIZE - 1)))) log.debug("sending browser final blob (%i/%i)", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1) await response.write_eof(decrypted) else: log.debug("sending browser blob (%i/%i)", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1) await response.write(decrypted) wrote += len(decrypted) log.info("sent browser %sblob %i/%i", "(final) " if response._eof_sent else "", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1) if response._eof_sent: break return response except ConnectionResetError: log.warning("connection was reset after sending browser %i blob bytes", wrote) raise asyncio.CancelledError("range request transport was reset") finally: response.force_close() if (request, response) in self.streaming_responses: self.streaming_responses.remove((request, response)) if not self.streaming_responses: self.streaming.clear() @staticmethod def _write_decrypted_blob(output_path: str, data: bytes): with open(output_path, 'ab') as handle: handle.write(data) handle.flush() async def _save_file(self, output_path: str): log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6], output_path) self.saving.set() self.finished_write_attempt.clear() self.finished_writing.clear() self.started_writing.clear() try: open(output_path, 'wb').close() # pylint: disable=consider-using-with async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID): log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1) await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted) if not self.started_writing.is_set(): self.started_writing.set() await self.update_status(ManagedStream.STATUS_FINISHED) if self.analytics_manager: self.loop.create_task(self.analytics_manager.send_download_finished( self.download_id, self.claim_name, self.sd_hash )) self.finished_writing.set() log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6], self.full_path) await self.blob_manager.storage.set_saved_file(self.stream_hash) except (Exception, asyncio.CancelledError) as err: if os.path.isfile(output_path): log.warning("removing incomplete download %s for %s", output_path, self.sd_hash) os.remove(output_path) if isinstance(err, asyncio.TimeoutError): self.downloader.stop() await self.blob_manager.storage.change_file_download_dir_and_file_name( self.stream_hash, None, None ) self._file_name, self.download_directory = None, None await self.blob_manager.storage.clear_saved_file(self.stream_hash) await self.update_status(self.STATUS_STOPPED) return elif not isinstance(err, asyncio.CancelledError): log.exception("unexpected error encountered writing file for stream %s", self.sd_hash) raise err finally: self.saving.clear() self.finished_write_attempt.set() async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): await self.start() if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task self.file_output_task.cancel() self.download_directory = download_directory or self.download_directory or self.config.download_dir if not self.download_directory: raise ValueError("no directory to download to") if not (file_name or self._file_name or self.suggested_file_name): raise ValueError("no file name to download to") if not os.path.isdir(self.download_directory): log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory) os.mkdir(self.download_directory) self._file_name = await get_next_available_file_name( self.loop, self.download_directory, file_name or self._file_name or sanitize_file_name(self.suggested_file_name) ) await self.blob_manager.storage.change_file_download_dir_and_file_name( self.stream_hash, self.download_directory, self.file_name ) await self.update_status(ManagedStream.STATUS_RUNNING) self.file_output_task = self.loop.create_task(self._save_file(self.full_path)) try: await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout) except asyncio.TimeoutError: log.warning("timeout starting to write data for lbry://%s#%s", self.claim_name, self.claim_id) await self.stop_tasks() await self.update_status(ManagedStream.STATUS_STOPPED) async def stop_tasks(self): if self.file_output_task and not self.file_output_task.done(): self.file_output_task.cancel() await asyncio.gather(self.file_output_task, return_exceptions=True) self.file_output_task = None while self.streaming_responses: req, response = self.streaming_responses.pop() response.force_close() req.transport.close() self.downloader.stop() self._running.clear() async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]: sent = [] protocol = StreamReflectorClient(self.blob_manager, self.descriptor) try: self.uploading_to_reflector = True await self.loop.create_connection(lambda: protocol, host, port) await protocol.send_handshake() sent_sd, needed = await protocol.send_descriptor() if sent_sd: # reflector needed the sd blob sent.append(self.sd_hash) if not sent_sd and not needed: # reflector already has the stream if not self.fully_reflected.is_set(): self.fully_reflected.set() await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}") return [] we_have = [ blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes ] log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed), self.claim_name, self.claim_id) for i, blob_hash in enumerate(we_have): await protocol.send_blob(blob_hash) sent.append(blob_hash) self.reflector_progress = int((i + 1) / len(we_have) * 100) except (asyncio.TimeoutError, ValueError): return sent except ConnectionError: return sent except (OSError, Exception, asyncio.CancelledError) as err: if isinstance(err, asyncio.CancelledError): log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id) elif isinstance(err, OSError): log.warning( "stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name, self.claim_id ) else: log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id) return sent finally: if protocol.transport: protocol.transport.close() self.uploading_to_reflector = False return sent async def update_content_claim(self, claim_info: Optional[typing.Dict] = None): if not claim_info: claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash) self.set_claim(claim_info, claim_info['value']) async def _delayed_stop(self): stalled_count = 0 while self._running.is_set(): if self.saving.is_set() or self.streaming.is_set(): stalled_count = 0 else: stalled_count += 1 if stalled_count > 1: log.info("stopping inactive download for lbry://%s#%s (%s...)", self.claim_name, self.claim_id, self.sd_hash[:6]) await self.stop() return await asyncio.sleep(1) def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]: if '=' in get_range: get_range = get_range.split('=')[1] start, end = get_range.split('-') size = 0 for blob in self.descriptor.blobs[:-1]: size += blob.length - 1 if self.stream_claim_info and self.stream_claim_info.claim.stream.source.size: size_from_claim = int(self.stream_claim_info.claim.stream.source.size) if not size_from_claim <= size <= size_from_claim + 16: raise ValueError("claim contains implausible stream size") log.debug("using stream size from claim") size = size_from_claim elif self.stream_claim_info: log.debug("estimating stream size") start = int(start) if not 0 <= start < size: raise HTTPRequestRangeNotSatisfiable() end = int(end) if end else size - 1 if end >= size: raise HTTPRequestRangeNotSatisfiable() skip_blobs = start // (MAX_BLOB_SIZE - 2) # -2 because ... dont remember skip = skip_blobs * (MAX_BLOB_SIZE - 1) # -1 because skip_first_blob = start - skip start = skip_first_blob + skip final_size = end - start + 1 headers = { 'Accept-Ranges': 'bytes', 'Content-Range': f'bytes {start}-{end}/{size}', 'Content-Length': str(final_size), 'Content-Type': self.mime_type } return headers, size, skip_blobs, skip_first_blob ================================================ FILE: lbry/stream/reflector/__init__.py ================================================ ================================================ FILE: lbry/stream/reflector/client.py ================================================ import asyncio import json import logging import typing if typing.TYPE_CHECKING: from lbry.blob.blob_manager import BlobManager from lbry.stream.descriptor import StreamDescriptor REFLECTOR_V1 = 0 REFLECTOR_V2 = 1 MAX_RESPONSE_SIZE = 2000000 log = logging.getLogger(__name__) class StreamReflectorClient(asyncio.Protocol): def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'): self.loop = asyncio.get_event_loop() self.transport: typing.Optional[asyncio.WriteTransport] = None self.blob_manager = blob_manager self.descriptor = descriptor self.response_buff = b'' self.reflected_blobs = [] self.connected = asyncio.Event() self.response_queue = asyncio.Queue(maxsize=1) self.pending_request: typing.Optional[asyncio.Task] = None def connection_made(self, transport): self.transport = transport log.debug("Connected to reflector") self.connected.set() def connection_lost(self, exc: typing.Optional[Exception]): self.transport = None self.connected.clear() if self.pending_request: self.pending_request.cancel() if self.reflected_blobs: log.info("Finished sending reflector %i blobs", len(self.reflected_blobs)) def data_received(self, data): if len(self.response_buff + (data or b'')) > MAX_RESPONSE_SIZE: log.warning("response message to large from reflector server: %i bytes", len(self.response_buff + (data or b''))) self.response_buff = b'' self.transport.close() return self.response_buff += (data or b'') try: response = json.loads(self.response_buff.decode()) self.response_buff = b'' self.response_queue.put_nowait(response) except ValueError: if not data: log.warning("got undecodable response from reflector server") self.response_buff = b'' return async def send_request(self, request_dict: typing.Dict, timeout: int = 180): msg = json.dumps(request_dict, sort_keys=True) try: self.transport.write(msg.encode()) self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout)) return await self.pending_request except (AttributeError, asyncio.CancelledError) as err: # attribute error happens when we transport.write after disconnect # cancelled error happens when the pending_request task is cancelled by a disconnect if self.transport: self.transport.close() raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError() finally: self.pending_request = None async def send_handshake(self) -> None: response_dict = await self.send_request({'version': REFLECTOR_V2}) if 'version' not in response_dict: raise ValueError("Need protocol version number!") server_version = int(response_dict['version']) if server_version != REFLECTOR_V2: raise ValueError(f"I can't handle protocol version {server_version}!") return async def send_descriptor(self) -> typing.Tuple[bool, typing.List[str]]: # returns a list of needed blob hashes sd_blob = self.blob_manager.get_blob(self.descriptor.sd_hash) assert self.blob_manager.is_blob_verified(self.descriptor.sd_hash), "need to have sd blob to send at this point" response = await self.send_request({ 'sd_blob_hash': sd_blob.blob_hash, 'sd_blob_size': sd_blob.length }) if 'send_sd_blob' not in response: raise ValueError("I don't know whether to send the sd blob or not!") needed = response.get('needed_blobs', []) sent_sd = False if response['send_sd_blob']: try: sent = await sd_blob.sendfile(self) if sent == -1: log.warning("failed to send sd blob") raise asyncio.CancelledError() received = await asyncio.wait_for(self.response_queue.get(), 30) except asyncio.CancelledError as err: if self.transport: self.transport.close() raise err if received.get('received_sd_blob'): sent_sd = True if not needed: for blob in self.descriptor.blobs[:-1]: if self.blob_manager.is_blob_verified(blob.blob_hash, blob.length): needed.append(blob.blob_hash) log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8]) self.reflected_blobs.append(sd_blob.blob_hash) else: log.warning("Reflector failed to receive descriptor %s", sd_blob.blob_hash[:8]) return sent_sd, needed async def send_blob(self, blob_hash: str): assert self.blob_manager.is_blob_verified(blob_hash), "need to have a blob to send at this point" blob = self.blob_manager.get_blob(blob_hash) response = await self.send_request({ 'blob_hash': blob.blob_hash, 'blob_size': blob.length }) if 'send_blob' not in response: raise ValueError("I don't know whether to send the blob or not!") if response['send_blob']: try: sent = await blob.sendfile(self) if sent == -1: log.warning("failed to send blob") raise asyncio.CancelledError() received = await asyncio.wait_for(self.response_queue.get(), 30) except asyncio.CancelledError as err: if self.transport: self.transport.close() raise err if received.get('received_blob'): self.reflected_blobs.append(blob.blob_hash) log.info("Sent reflector blob %s", blob.blob_hash[:8]) else: log.warning("Reflector failed to receive blob %s", blob.blob_hash[:8]) ================================================ FILE: lbry/stream/reflector/server.py ================================================ import asyncio import logging import typing import json from json.decoder import JSONDecodeError from lbry.stream.descriptor import StreamDescriptor if typing.TYPE_CHECKING: from lbry.blob.blob_file import BlobFile from lbry.blob.blob_manager import BlobManager from lbry.blob.writer import HashBlobWriter log = logging.getLogger(__name__) class ReflectorServerProtocol(asyncio.Protocol): def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000, stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None, not_incoming_event: asyncio.Event = None, partial_event: asyncio.Event = None): self.loop = asyncio.get_event_loop() self.blob_manager = blob_manager self.server_task: asyncio.Task = None self.started_listening = asyncio.Event() self.buf = b'' self.transport: asyncio.StreamWriter = None self.writer: typing.Optional['HashBlobWriter'] = None self.client_version: typing.Optional[int] = None self.descriptor: typing.Optional['StreamDescriptor'] = None self.sd_blob: typing.Optional['BlobFile'] = None self.received = [] self.incoming = incoming_event or asyncio.Event() self.not_incoming = not_incoming_event or asyncio.Event() self.stop_event = stop_event or asyncio.Event() self.chunk_size = response_chunk_size self.wait_for_stop_task: typing.Optional[asyncio.Task] = None self.partial_event = partial_event async def wait_for_stop(self): await self.stop_event.wait() if self.transport: self.transport.close() def connection_made(self, transport): self.transport = transport self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop()) def connection_lost(self, exc): if self.wait_for_stop_task: self.wait_for_stop_task.cancel() self.wait_for_stop_task = None def data_received(self, data: bytes): if self.incoming.is_set(): try: self.writer.write(data) except OSError as err: log.error("error receiving blob: %s", err) self.transport.close() return try: request = json.loads(data.decode()) except (ValueError, JSONDecodeError): return self.loop.create_task(self.handle_request(request)) def send_response(self, response: typing.Dict): def chunk_response(remaining: bytes): f = self.loop.create_future() f.add_done_callback(lambda _: self.transport.write(remaining[:self.chunk_size])) if len(remaining) > self.chunk_size: f.add_done_callback(lambda _: self.loop.call_soon(chunk_response, remaining[self.chunk_size:])) self.loop.call_soon(f.set_result, None) response_bytes = json.dumps(response).encode() chunk_response(response_bytes) async def handle_request(self, request: typing.Dict): # pylint: disable=too-many-return-statements if self.client_version is None: if 'version' not in request: self.transport.close() return self.client_version = request['version'] self.send_response({'version': 1}) return if not self.sd_blob: if 'sd_blob_hash' not in request: self.transport.close() return self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size']) if not self.sd_blob.get_is_verified(): self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername')) self.not_incoming.clear() self.incoming.set() self.send_response({"send_sd_blob": True}) try: await asyncio.wait_for(self.sd_blob.verified.wait(), 30) self.descriptor = await StreamDescriptor.from_stream_descriptor_blob( self.loop, self.blob_manager.blob_dir, self.sd_blob ) self.send_response({"received_sd_blob": True}) except asyncio.TimeoutError: self.send_response({"received_sd_blob": False}) self.transport.close() finally: self.incoming.clear() self.not_incoming.set() self.writer.close_handle() self.writer = None else: self.descriptor = await StreamDescriptor.from_stream_descriptor_blob( self.loop, self.blob_manager.blob_dir, self.sd_blob ) self.incoming.clear() self.not_incoming.set() if self.writer: self.writer.close_handle() self.writer = None needs = [blob.blob_hash for blob in self.descriptor.blobs[:-1] if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()] if needs and not self.partial_event.is_set(): needs = needs[:3] self.partial_event.set() self.send_response({"send_sd_blob": False, 'needed_blobs': needs}) return return elif self.descriptor: if 'blob_hash' not in request: self.transport.close() return if request['blob_hash'] not in map(lambda b: b.blob_hash, self.descriptor.blobs[:-1]): self.send_response({"send_blob": False}) return blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size']) if not blob.get_is_verified(): self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername')) self.not_incoming.clear() self.incoming.set() self.send_response({"send_blob": True}) try: await asyncio.wait_for(blob.verified.wait(), 30) self.send_response({"received_blob": True}) except asyncio.TimeoutError: self.send_response({"received_blob": False}) self.incoming.clear() self.not_incoming.set() self.writer.close_handle() self.writer = None else: self.send_response({"send_blob": False}) return else: self.transport.close() class ReflectorServer: def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000, stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None, not_incoming_event: asyncio.Event = None, partial_needs=False): self.loop = asyncio.get_event_loop() self.blob_manager = blob_manager self.server_task: typing.Optional[asyncio.Task] = None self.started_listening = asyncio.Event() self.stopped_listening = asyncio.Event() self.incoming_event = incoming_event or asyncio.Event() self.not_incoming_event = not_incoming_event or asyncio.Event() self.response_chunk_size = response_chunk_size self.stop_event = stop_event self.partial_needs = partial_needs # for testing cases where it doesn't know what it wants def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'): if self.server_task is not None: raise Exception("already running") async def _start_server(): partial_event = asyncio.Event() if not self.partial_needs: partial_event.set() server = await self.loop.create_server(lambda: ReflectorServerProtocol( self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event, self.not_incoming_event, partial_event), interface, port) self.started_listening.set() self.stopped_listening.clear() log.info("Reflector server listening on TCP %s:%i", interface, port) try: async with server: await server.serve_forever() finally: self.stopped_listening.set() self.server_task = self.loop.create_task(_start_server()) def stop_server(self): if self.server_task: self.server_task.cancel() self.server_task = None log.info("Stopped reflector server") ================================================ FILE: lbry/stream/stream_manager.py ================================================ import os import asyncio import binascii import logging import random import typing from typing import Optional from aiohttp.web import Request from lbry.error import InvalidStreamDescriptorError from lbry.file.source_manager import SourceManager from lbry.stream.descriptor import StreamDescriptor from lbry.stream.managed_stream import ManagedStream from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.blob.blob_manager import BlobManager from lbry.dht.node import Node from lbry.wallet.wallet import WalletManager from lbry.wallet.transaction import Transaction from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim log = logging.getLogger(__name__) def path_or_none(encoded_path) -> Optional[str]: if not encoded_path: return return binascii.unhexlify(encoded_path).decode() class StreamManager(SourceManager): _sources: typing.Dict[str, ManagedStream] filter_fields = SourceManager.filter_fields filter_fields.update({ 'sd_hash', 'stream_hash', 'full_status', # TODO: remove 'blobs_remaining', 'blobs_in_stream', 'uploading_to_reflector', 'is_fully_reflected' }) def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'], analytics_manager: Optional['AnalyticsManager'] = None): super().__init__(loop, config, storage, analytics_manager) self.blob_manager = blob_manager self.wallet_manager = wallet_manager self.node = node self.resume_saving_task: Optional[asyncio.Task] = None self.re_reflect_task: Optional[asyncio.Task] = None self.update_stream_finished_futs: typing.List[asyncio.Future] = [] self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {} self.started = asyncio.Event() @property def streams(self): return self._sources def add(self, source: ManagedStream): super().add(source) self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source) async def _update_content_claim(self, stream: ManagedStream): claim_info = await self.storage.get_content_claim(stream.stream_hash) self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value']) async def recover_streams(self, file_infos: typing.List[typing.Dict]): to_restore = [] to_check = [] async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str, suggested_file_name: str, key: str, content_fee: Optional['Transaction']) -> Optional[StreamDescriptor]: sd_blob = self.blob_manager.get_blob(sd_hash) blobs = await self.storage.get_blobs_for_stream(stream_hash) descriptor = await StreamDescriptor.recover( self.blob_manager.blob_dir, sd_blob, stream_hash, stream_name, suggested_file_name, key, blobs ) if not descriptor: return to_restore.append((descriptor, sd_blob, content_fee)) to_check.extend([sd_blob.blob_hash] + [blob.blob_hash for blob in descriptor.blobs[:-1]]) await asyncio.gather(*[ recover_stream( file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(), binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key'], file_info['content_fee'] ) for file_info in file_infos ]) if to_restore: await self.storage.recover_streams(to_restore, self.config.download_dir) if to_check: await self.blob_manager.ensure_completed_blobs_status(to_check) # if self.blob_manager._save_blobs: # log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos)) async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str], download_directory: Optional[str], status: str, claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'], added_on: Optional[int], fully_reflected: Optional[bool]): try: descriptor = await self.blob_manager.get_stream_descriptor(sd_hash) except InvalidStreamDescriptorError as err: log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err)) return stream = ManagedStream( self.loop, self.config, self.blob_manager, descriptor.sd_hash, download_directory, file_name, status, claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor, analytics_manager=self.analytics_manager, added_on=added_on ) if fully_reflected: stream.fully_reflected.set() self.add(stream) async def initialize_from_database(self): to_recover = [] to_start = [] await self.storage.update_manually_removed_files_since_last_run() for file_info in await self.storage.get_all_lbry_files(): # if the sd blob is not verified, try to reconstruct it from the database # this could either be because the blob files were deleted manually or save_blobs was not true when # the stream was downloaded if not self.blob_manager.is_blob_verified(file_info['sd_hash']): to_recover.append(file_info) to_start.append(file_info) if to_recover: await self.recover_streams(to_recover) log.info("Initializing %i files", len(to_start)) to_resume_saving = [] add_stream_tasks = [] for file_info in to_start: file_name = path_or_none(file_info['file_name']) download_directory = path_or_none(file_info['download_directory']) if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running': to_resume_saving.append((file_name, download_directory, file_info['sd_hash'])) add_stream_tasks.append(self.loop.create_task(self._load_stream( file_info['rowid'], file_info['sd_hash'], file_name, download_directory, file_info['status'], file_info['claim'], file_info['content_fee'], file_info['added_on'], file_info['fully_reflected'] ))) if add_stream_tasks: await asyncio.gather(*add_stream_tasks) log.info("Started stream manager with %i files", len(self._sources)) if not self.node: log.info("no DHT node given, resuming downloads trusting that we can contact reflector") if to_resume_saving: log.info("Resuming saving %i files", len(to_resume_saving)) self.resume_saving_task = asyncio.ensure_future(asyncio.gather( *(self._sources[sd_hash].save_file(file_name, download_directory) for (file_name, download_directory, sd_hash) in to_resume_saving), )) async def reflect_streams(self): try: return await self._reflect_streams() except Exception: log.exception("reflector task encountered an unexpected error!") async def _reflect_streams(self): # todo: those debug statements are temporary for #2987 - remove them if its closed while True: if self.config.reflect_streams and self.config.reflector_servers: log.debug("collecting streams to reflect") sd_hashes = await self.storage.get_streams_to_re_reflect() sd_hashes = [sd for sd in sd_hashes if sd in self._sources] batch = [] while sd_hashes: stream = self.streams[sd_hashes.pop()] if self.blob_manager.is_blob_verified(stream.sd_hash) and stream.blobs_completed and \ stream.sd_hash not in self.running_reflector_uploads and not \ stream.fully_reflected.is_set(): batch.append(self.reflect_stream(stream)) if len(batch) >= self.config.concurrent_reflector_uploads: log.debug("waiting for batch of %s reflecting streams", len(batch)) await asyncio.gather(*batch) log.debug("done processing %s streams", len(batch)) batch = [] if batch: log.debug("waiting for batch of %s reflecting streams", len(batch)) await asyncio.gather(*batch) log.debug("done processing %s streams", len(batch)) await asyncio.sleep(300) async def start(self): await super().start() self.re_reflect_task = self.loop.create_task(self.reflect_streams()) async def stop(self): await super().stop() if self.resume_saving_task and not self.resume_saving_task.done(): self.resume_saving_task.cancel() if self.re_reflect_task and not self.re_reflect_task.done(): self.re_reflect_task.cancel() while self.update_stream_finished_futs: self.update_stream_finished_futs.pop().cancel() while self.running_reflector_uploads: _, t = self.running_reflector_uploads.popitem() t.cancel() self.started.clear() log.info("finished stopping the stream manager") def reflect_stream(self, stream: ManagedStream, server: Optional[str] = None, port: Optional[int] = None) -> asyncio.Task: if not server or not port: server, port = random.choice(self.config.reflector_servers) if stream.sd_hash in self.running_reflector_uploads: return self.running_reflector_uploads[stream.sd_hash] task = self.loop.create_task(self._retriable_reflect_stream(stream, server, port)) self.running_reflector_uploads[stream.sd_hash] = task task.add_done_callback( lambda _: None if stream.sd_hash not in self.running_reflector_uploads else self.running_reflector_uploads.pop(stream.sd_hash) ) return task @staticmethod async def _retriable_reflect_stream(stream, host, port): sent = await stream.upload_to_reflector(host, port) while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0: stream.reflector_progress = 0 sent = await stream.upload_to_reflector(host, port) return sent async def create(self, file_path: str, key: Optional[bytes] = None, iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream: descriptor = await StreamDescriptor.create_stream( self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator, blob_completed_callback=self.blob_manager.blob_completed ) await self.storage.store_stream( self.blob_manager.get_blob(descriptor.sd_hash, is_mine=True), descriptor ) row_id = await self.storage.save_published_file( descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0 ) stream = ManagedStream( self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path), os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED, rowid=row_id, descriptor=descriptor ) self.streams[stream.sd_hash] = stream self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) if self.config.reflect_streams and self.config.reflector_servers: self.reflect_stream(stream) return stream async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): if not isinstance(source, ManagedStream): return if source.identifier in self.running_reflector_uploads: self.running_reflector_uploads[source.identifier].cancel() await source.stop_tasks() if source.identifier in self.streams: del self.streams[source.identifier] blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]] await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) await self.storage.delete_stream(source.descriptor) if delete_file and source.output_file_exists: os.remove(source.full_path) async def stream_partial_content(self, request: Request, sd_hash: str): stream = self._sources[sd_hash] if not stream.downloader.node: stream.downloader.node = self.node return await stream.stream_file(request) ================================================ FILE: lbry/testcase.py ================================================ import os import sys import json import shutil import logging import tempfile import functools import asyncio from asyncio.runners import _cancel_all_tasks # type: ignore import unittest from unittest.case import _Outcome from typing import Optional from time import time from binascii import unhexlify from functools import partial from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction from lbry.conf import Config from lbry.wallet.util import satoshis_to_coins from lbry.wallet.dewies import lbc_to_dewies from lbry.wallet.orchstr8 import Conductor from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode from lbry.schema.claim import Claim from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty from lbry.extras.daemon.components import Component, WalletComponent from lbry.extras.daemon.components import ( DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT ) from lbry.extras.daemon.componentmanager import ComponentManager from lbry.extras.daemon.exchange_rate_manager import ( ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed ) from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_manager import BlobManager from lbry.stream.reflector.server import ReflectorServer from lbry.blob_exchange.server import BlobServer class ColorHandler(logging.StreamHandler): level_color = { logging.DEBUG: "black", logging.INFO: "light_gray", logging.WARNING: "yellow", logging.ERROR: "red" } color_code = dict( black=30, red=31, green=32, yellow=33, blue=34, magenta=35, cyan=36, white=37, light_gray='0;37', dark_gray='1;30' ) def emit(self, record): try: msg = self.format(record) color_name = self.level_color.get(record.levelno, "black") color_code = self.color_code[color_name] stream = self.stream stream.write(f'\x1b[{color_code}m{msg}\x1b[0m') stream.write(self.terminator) self.flush() except Exception: self.handleError(record) HANDLER = ColorHandler(sys.stdout) HANDLER.setFormatter( logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ) logging.getLogger().addHandler(HANDLER) class AsyncioTestCase(unittest.TestCase): # Implementation inspired by discussion: # https://bugs.python.org/issue32972 LOOP_SLOW_CALLBACK_DURATION = 0.2 TIMEOUT = 120.0 maxDiff = None async def asyncSetUp(self): # pylint: disable=C0103 pass async def asyncTearDown(self): # pylint: disable=C0103 pass def run(self, result=None): # pylint: disable=R0915 orig_result = result if result is None: result = self.defaultTestResult() startTestRun = getattr(result, 'startTestRun', None) # pylint: disable=C0103 if startTestRun is not None: startTestRun() result.startTest(self) testMethod = getattr(self, self._testMethodName) # pylint: disable=C0103 if (getattr(self.__class__, "__unittest_skip__", False) or getattr(testMethod, "__unittest_skip__", False)): # If the class or method was skipped. try: skip_why = (getattr(self.__class__, '__unittest_skip_why__', '') or getattr(testMethod, '__unittest_skip_why__', '')) self._addSkip(result, self, skip_why) finally: result.stopTest(self) return expecting_failure_method = getattr(testMethod, "__unittest_expecting_failure__", False) expecting_failure_class = getattr(self, "__unittest_expecting_failure__", False) expecting_failure = expecting_failure_class or expecting_failure_method outcome = _Outcome(result) self.loop = asyncio.new_event_loop() # pylint: disable=W0201 asyncio.set_event_loop(self.loop) self.loop.set_debug(True) self.loop.slow_callback_duration = self.LOOP_SLOW_CALLBACK_DURATION try: self._outcome = outcome with outcome.testPartExecutor(self): self.setUp() self.add_timeout() self.loop.run_until_complete(self.asyncSetUp()) if outcome.success: outcome.expecting_failure = expecting_failure with outcome.testPartExecutor(self, isTest=True): maybe_coroutine = testMethod() if asyncio.iscoroutine(maybe_coroutine): self.add_timeout() self.loop.run_until_complete(maybe_coroutine) outcome.expecting_failure = False with outcome.testPartExecutor(self): self.add_timeout() self.loop.run_until_complete(self.asyncTearDown()) self.tearDown() self.doAsyncCleanups() try: _cancel_all_tasks(self.loop) self.loop.run_until_complete(self.loop.shutdown_asyncgens()) finally: asyncio.set_event_loop(None) self.loop.close() for test, reason in outcome.skipped: self._addSkip(result, test, reason) self._feedErrorsToResult(result, outcome.errors) if outcome.success: if expecting_failure: if outcome.expectedFailure: self._addExpectedFailure(result, outcome.expectedFailure) else: self._addUnexpectedSuccess(result) else: result.addSuccess(self) return result finally: result.stopTest(self) if orig_result is None: stopTestRun = getattr(result, 'stopTestRun', None) # pylint: disable=C0103 if stopTestRun is not None: stopTestRun() # pylint: disable=E1102 # explicitly break reference cycles: # outcome.errors -> frame -> outcome -> outcome.errors # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure outcome.errors.clear() outcome.expectedFailure = None # clear the outcome, no more needed self._outcome = None def doAsyncCleanups(self): # pylint: disable=C0103 outcome = self._outcome or _Outcome() while self._cleanups: function, args, kwargs = self._cleanups.pop() with outcome.testPartExecutor(self): maybe_coroutine = function(*args, **kwargs) if asyncio.iscoroutine(maybe_coroutine): self.add_timeout() self.loop.run_until_complete(maybe_coroutine) def cancel(self): for task in asyncio.all_tasks(self.loop): if not task.done(): task.print_stack() task.cancel() def add_timeout(self): if self.TIMEOUT: self.loop.call_later(self.TIMEOUT, self.check_timeout, time()) def check_timeout(self, started): if time() - started >= self.TIMEOUT: self.cancel() else: self.loop.call_later(self.TIMEOUT, self.check_timeout, started) class AdvanceTimeTestCase(AsyncioTestCase): async def asyncSetUp(self): self._time = 0 # pylint: disable=W0201 self.loop.time = functools.wraps(self.loop.time)(lambda: self._time) await super().asyncSetUp() async def advance(self, seconds): while self.loop._ready: await asyncio.sleep(0) self._time += seconds await asyncio.sleep(0) while self.loop._ready: await asyncio.sleep(0) class IntegrationTestCase(AsyncioTestCase): SEED = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.conductor: Optional[Conductor] = None self.blockchain: Optional[LBCWalletNode] = None self.wallet_node: Optional[WalletNode] = None self.manager: Optional[WalletManager] = None self.ledger: Optional[Ledger] = None self.wallet: Optional[Wallet] = None self.account: Optional[Account] = None async def asyncSetUp(self): self.conductor = Conductor(seed=self.SEED) await self.conductor.start_lbcd() self.addCleanup(self.conductor.stop_lbcd) await self.conductor.start_lbcwallet() self.addCleanup(self.conductor.stop_lbcwallet) await self.conductor.start_spv() self.addCleanup(self.conductor.stop_spv) await self.conductor.start_wallet() self.addCleanup(self.conductor.stop_wallet) self.blockchain = self.conductor.lbcwallet_node self.wallet_node = self.conductor.wallet_node self.manager = self.wallet_node.manager self.ledger = self.wallet_node.ledger self.wallet = self.wallet_node.wallet self.account = self.wallet_node.wallet.default_account async def assertBalance(self, account, expected_balance: str): # pylint: disable=C0103 balance = await account.get_balance() self.assertEqual(satoshis_to_coins(balance), expected_balance) def broadcast(self, tx): return self.ledger.broadcast(tx) async def broadcast_and_confirm(self, tx, ledger=None): ledger = ledger or self.ledger notifications = asyncio.create_task(ledger.wait(tx)) await ledger.broadcast(tx) await notifications await self.generate_and_wait(1, [tx.id], ledger) async def on_header(self, height): if self.ledger.headers.height < height: await self.ledger.on_header.where( lambda e: e.height == height ) return True async def send_to_address_and_wait(self, address, amount, blocks_to_generate=0, ledger=None): tx_watch = [] txid = None done = False watcher = (ledger or self.ledger).on_transaction.where( lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id) ) txid = await self.blockchain.send_to_address(address, amount) done = txid in tx_watch await watcher await self.generate_and_wait(blocks_to_generate, [txid], ledger) return txid async def generate_and_wait(self, blocks_to_generate, txids, ledger=None): if blocks_to_generate > 0: watcher = (ledger or self.ledger).on_transaction.where( lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1] # multi-statement lambda ) await self.generate(blocks_to_generate) await watcher def on_address_update(self, address): return self.ledger.on_transaction.where( lambda e: e.address == address ) def on_transaction_address(self, tx, address): return self.ledger.on_transaction.where( lambda e: e.tx.id == tx.id and e.address == address ) async def generate(self, blocks): """ Ask lbrycrd to generate some blocks and wait until ledger has them. """ prepare = self.ledger.on_header.where(self.blockchain.is_expected_block) self.conductor.spv_node.server.synchronized.clear() await self.blockchain.generate(blocks) height = self.blockchain.block_expected await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate while True: await self.conductor.spv_node.server.synchronized.wait() self.conductor.spv_node.server.synchronized.clear() if self.conductor.spv_node.server.db.db_height < height: continue if self.conductor.spv_node.server._es_height < height: continue break class FakeExchangeRateManager(ExchangeRateManager): def __init__(self, market_feeds, rates): # pylint: disable=super-init-not-called self.market_feeds = market_feeds for feed in self.market_feeds: feed.last_check = time() feed.rate = ExchangeRate(feed.market, rates[feed.market], time()) def start(self): pass def stop(self): pass def get_fake_exchange_rate_manager(rates=None): return FakeExchangeRateManager( [BittrexBTCFeed(), BittrexUSDFeed()], rates or {'BTCLBC': 3.0, 'USDLBC': 2.0} ) class ExchangeRateManagerComponent(Component): component_name = EXCHANGE_RATE_MANAGER_COMPONENT def __init__(self, component_manager, rates=None): super().__init__(component_manager) self.exchange_rate_manager = get_fake_exchange_rate_manager(rates) @property def component(self) -> ExchangeRateManager: return self.exchange_rate_manager async def start(self): self.exchange_rate_manager.start() async def stop(self): self.exchange_rate_manager.stop() class CommandTestCase(IntegrationTestCase): VERBOSITY = logging.WARN blob_lru_cache_size = 0 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.daemon = None self.daemons = [] self.server_config = None self.server_storage = None self.extra_wallet_nodes = [] self.extra_wallet_node_port = 5280 self.server_blob_manager = None self.server = None self.reflector = None self.skip_libtorrent = True async def asyncSetUp(self): logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY) logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY) logging.getLogger('lbry.stream').setLevel(self.VERBOSITY) logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY) await super().asyncSetUp() self.daemon = await self.add_daemon(self.wallet_node) await self.account.ensure_address_gap() address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0] await self.send_to_address_and_wait(address, 10, 6) server_tmp_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, server_tmp_dir) self.server_config = Config( data_dir=server_tmp_dir, wallet_dir=server_tmp_dir, save_files=True, download_dir=server_tmp_dir ) self.server_config.transaction_cache_size = 10000 self.server_storage = SQLiteStorage(self.server_config, ':memory:') await self.server_storage.open() self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_config) self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP') self.server.start_server(5567, '127.0.0.1') await self.server.started_listening.wait() self.reflector = ReflectorServer(self.server_blob_manager) self.reflector.start_server(5566, '127.0.0.1') await self.reflector.started_listening.wait() self.addCleanup(self.reflector.stop_server) async def asyncTearDown(self): await super().asyncTearDown() for wallet_node in self.extra_wallet_nodes: await wallet_node.stop(cleanup=True) for daemon in self.daemons: daemon.component_manager.get_component('wallet')._running = False await daemon.stop() async def add_daemon(self, wallet_node=None, seed=None): start_wallet_node = False if wallet_node is None: wallet_node = WalletNode( self.wallet_node.manager_class, self.wallet_node.ledger_class, port=self.extra_wallet_node_port ) self.extra_wallet_node_port += 1 start_wallet_node = True upload_dir = os.path.join(wallet_node.data_path, 'uploads') os.mkdir(upload_dir) conf = Config( # needed during instantiation to access known_hubs path data_dir=wallet_node.data_path, wallet_dir=wallet_node.data_path, save_files=True, download_dir=wallet_node.data_path ) conf.upload_dir = upload_dir # not a real conf setting conf.share_usage_data = False conf.use_upnp = False conf.reflect_streams = True conf.blockchain_name = 'lbrycrd_regtest' conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)] conf.reflector_servers = [('127.0.0.1', 5566)] conf.fixed_peers = [('127.0.0.1', 5567)] conf.known_dht_nodes = [] conf.blob_lru_cache_size = self.blob_lru_cache_size conf.transaction_cache_size = 10000 conf.components_to_skip = [ DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT ] if self.skip_libtorrent: conf.components_to_skip.append(LIBTORRENT_COMPONENT) if start_wallet_node: await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf) self.extra_wallet_nodes.append(wallet_node) else: wallet_node.manager.config = conf wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs def wallet_maker(component_manager): wallet_component = WalletComponent(component_manager) wallet_component.wallet_manager = wallet_node.manager wallet_component._running = True return wallet_component daemon = Daemon(conf, ComponentManager( conf, skip_components=conf.components_to_skip, wallet=wallet_maker, exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={ 'BTCLBC': 1.0, 'USDLBC': 2.0 }) )) await daemon.initialize() self.daemons.append(daemon) wallet_node.manager.old_db = daemon.storage return daemon async def confirm_tx(self, txid, ledger=None): """ Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """ # await (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid) on_tx = (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid) await asyncio.wait([self.generate(1), on_tx], timeout=5) # # actually, if it's in the mempool or in the block we're fine # await self.generate_and_wait(1, [txid], ledger=ledger) # return txid return txid async def on_transaction_dict(self, tx): await self.ledger.wait(Transaction(unhexlify(tx['hex']))) @staticmethod def get_all_addresses(tx): addresses = set() for txi in tx['inputs']: addresses.add(txi['address']) for txo in tx['outputs']: addresses.add(txo['address']) return list(addresses) async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True): txid = await self.blockchain._cli_cmnd('claimname', name, value, amount) if confirm: await self.generate(1) return txid async def blockchain_update_name(self, txid: str, value: str, amount: str, confirm=True): txid = await self.blockchain._cli_cmnd('updateclaim', txid, value, amount) if confirm: await self.generate(1) return txid async def out(self, awaitable): """ Serializes lbrynet API results to JSON then loads and returns it as dictionary. """ return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result'] def sout(self, value): """ Synchronous version of `out` method. """ return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result'] async def confirm_and_render(self, awaitable, confirm, return_tx=False) -> Transaction: tx = await awaitable if confirm: await self.ledger.wait(tx) await self.generate(1) await self.ledger.wait(tx, self.blockchain.block_expected) if not return_tx: return self.sout(tx) return tx async def create_nondeterministic_channel(self, name, price, pubkey_bytes, daemon=None, blocking=False): account = (daemon or self.daemon).wallet_manager.default_account claim_address = await account.receiving.get_or_create_usable_address() claim = Claim() claim.channel.public_key_bytes = pubkey_bytes tx = await Transaction.claim_create( name, claim, lbc_to_dewies(price), claim_address, [self.account], self.account ) await tx.sign([self.account]) await (daemon or self.daemon).broadcast_or_release(tx, blocking) return self.sout(tx) def create_upload_file(self, data, prefix=None, suffix=None): file_path = tempfile.mktemp(prefix=prefix or "tmp", suffix=suffix or "", dir=self.daemon.conf.upload_dir) with open(file_path, 'w+b') as file: file.write(data) file.flush() return file.name async def stream_create( self, name='hovercraft', bid='1.0', file_path=None, data=b'hi!', confirm=True, prefix=None, suffix=None, return_tx=False, **kwargs): if file_path is None and data is not None: file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix) return await self.confirm_and_render( self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm, return_tx ) async def stream_update( self, claim_id, data=None, prefix=None, suffix=None, confirm=True, return_tx=False, **kwargs): if data is not None: file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix) return await self.confirm_and_render( self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm, return_tx ) return await self.confirm_and_render( self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm ) async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm ) async def stream_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm ) async def purchase_create(self, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm ) async def publish(self, name, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm ) async def channel_create(self, name='@arena', bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_channel_create(name, bid, **kwargs), confirm ) async def channel_update(self, claim_id, confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_channel_update(claim_id, **kwargs), confirm ) async def channel_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.daemon.jsonrpc_channel_abandon(*args, **kwargs), confirm ) async def collection_create( self, name='firstcollection', bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_collection_create(name, bid, **kwargs), confirm ) async def collection_update( self, claim_id, confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_collection_update(claim_id, **kwargs), confirm ) async def collection_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm ) async def support_create(self, claim_id, bid='1.0', confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm ) async def support_abandon(self, *args, confirm=True, **kwargs): if 'blocking' not in kwargs: kwargs['blocking'] = False return await self.confirm_and_render( self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm ) async def account_send(self, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_account_send(*args, **kwargs), confirm ) async def wallet_send(self, *args, confirm=True, **kwargs): return await self.confirm_and_render( self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm ) async def txo_spend(self, *args, confirm=True, **kwargs): txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs) if confirm: await asyncio.wait([self.ledger.wait(tx) for tx in txs]) await self.generate(1) await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs]) return self.sout(txs) async def blob_clean(self): return await self.out(self.daemon.jsonrpc_blob_clean()) async def status(self): return await self.out(self.daemon.jsonrpc_status()) async def resolve(self, uri, **kwargs): return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri] async def claim_search(self, **kwargs): return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items'] async def get_claim_by_claim_id(self, claim_id): return await self.out(self.ledger.get_claim_by_claim_id(claim_id)) async def file_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items'] async def txo_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items'] async def txo_sum(self, *args, **kwargs): return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs)) async def txo_plot(self, *args, **kwargs): return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs)) async def claim_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items'] async def stream_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_stream_list(*args, **kwargs)))['items'] async def channel_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items'] async def transaction_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items'] async def blob_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_blob_list(*args, **kwargs)))['items'] @staticmethod def get_claim_id(tx): return tx['outputs'][0]['claim_id'] def assertItemCount(self, result, count): # pylint: disable=invalid-name self.assertEqual(count, result['total_items']) ================================================ FILE: lbry/torrent/__init__.py ================================================ ================================================ FILE: lbry/torrent/session.py ================================================ import asyncio import binascii import os import logging import random from hashlib import sha1 from tempfile import mkdtemp from typing import Optional import libtorrent log = logging.getLogger(__name__) DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted? libtorrent.add_torrent_params_flags_t.flag_auto_managed | libtorrent.add_torrent_params_flags_t.flag_update_subscribe ) class TorrentHandle: def __init__(self, loop, executor, handle): self._loop = loop self._executor = executor self._handle: libtorrent.torrent_handle = handle self.started = asyncio.Event(loop=loop) self.finished = asyncio.Event(loop=loop) self.metadata_completed = asyncio.Event(loop=loop) self.size = 0 self.total_wanted_done = 0 self.name = '' self.tasks = [] self.torrent_file: Optional[libtorrent.file_storage] = None self._base_path = None self._handle.set_sequential_download(1) @property def largest_file(self) -> Optional[str]: if not self.torrent_file: return None index = self.largest_file_index return os.path.join(self._base_path, self.torrent_file.at(index).path) @property def largest_file_index(self): largest_size, index = 0, 0 for file_num in range(self.torrent_file.num_files()): if self.torrent_file.file_size(file_num) > largest_size: largest_size = self.torrent_file.file_size(file_num) index = file_num return index def stop_tasks(self): while self.tasks: self.tasks.pop().cancel() def _show_status(self): # fixme: cleanup if not self._handle.is_valid(): return status = self._handle.status() if status.has_metadata: self.size = status.total_wanted self.total_wanted_done = status.total_wanted_done self.name = status.name if not self.metadata_completed.is_set(): self.metadata_completed.set() log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name) self.torrent_file = self._handle.get_torrent_info().files() self._base_path = status.save_path first_piece = self.torrent_file.at(self.largest_file_index).offset if not self.started.is_set(): if self._handle.have_piece(first_piece): self.started.set() else: # prioritize it self._handle.set_piece_deadline(first_piece, 100) if not status.is_seeding: log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s', status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, status.num_peers, status.num_seeds, status.state, status.save_path) elif not self.finished.is_set(): self.finished.set() log.info("Torrent finished: %s", self.name) async def status_loop(self): while True: self._show_status() if self.finished.is_set(): break await asyncio.sleep(0.1) async def pause(self): await self._loop.run_in_executor( self._executor, self._handle.pause ) async def resume(self): await self._loop.run_in_executor( self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda ) class TorrentSession: def __init__(self, loop, executor): self._loop = loop self._executor = executor self._session: Optional[libtorrent.session] = None self._handles = {} self.tasks = [] self.wait_start = True async def add_fake_torrent(self): tmpdir = mkdtemp() info, btih = _create_fake_torrent(tmpdir) flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode handle = self._session.add_torrent({ 'ti': info, 'save_path': tmpdir, 'flags': flags }) self._handles[btih] = TorrentHandle(self._loop, self._executor, handle) return btih async def bind(self, interface: str = '0.0.0.0', port: int = 10889): settings = { 'listen_interfaces': f"{interface}:{port}", 'enable_natpmp': False, 'enable_upnp': False } self._session = await self._loop.run_in_executor( self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member ) self.tasks.append(self._loop.create_task(self.process_alerts())) def stop(self): while self.tasks: self.tasks.pop().cancel() self._session.save_state() self._session.pause() self._session.stop_dht() self._session.stop_lsd() self._session.stop_natpmp() self._session.stop_upnp() self._session = None def _pop_alerts(self): for alert in self._session.pop_alerts(): log.info("torrent alert: %s", alert) async def process_alerts(self): while True: await self._loop.run_in_executor( self._executor, self._pop_alerts ) await asyncio.sleep(1) async def pause(self): await self._loop.run_in_executor( self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda ) await self._loop.run_in_executor( self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda ) async def resume(self): await self._loop.run_in_executor( self._executor, self._session.resume ) def _add_torrent(self, btih: str, download_directory: Optional[str]): params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS} if download_directory: params['save_path'] = download_directory handle = self._session.add_torrent(params) handle.force_dht_announce() self._handles[btih] = TorrentHandle(self._loop, self._executor, handle) def full_path(self, btih): return self._handles[btih].largest_file async def add_torrent(self, btih, download_path): await self._loop.run_in_executor( self._executor, self._add_torrent, btih, download_path ) self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop())) await self._handles[btih].metadata_completed.wait() if self.wait_start: # fixme: temporary until we add streaming support, otherwise playback fails! await self._handles[btih].started.wait() def remove_torrent(self, btih, remove_files=False): if btih in self._handles: handle = self._handles[btih] handle.stop_tasks() self._session.remove_torrent(handle._handle, 1 if remove_files else 0) self._handles.pop(btih) async def save_file(self, btih, download_directory): handle = self._handles[btih] await handle.resume() def get_size(self, btih): return self._handles[btih].size def get_name(self, btih): return self._handles[btih].name def get_downloaded(self, btih): return self._handles[btih].total_wanted_done def is_completed(self, btih): return self._handles[btih].finished.is_set() def get_magnet_uri(btih): return f"magnet:?xt=urn:btih:{btih}" def _create_fake_torrent(tmpdir): # beware, that's just for testing path = os.path.join(tmpdir, 'tmp') with open(path, 'wb') as myfile: size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024) file_storage = libtorrent.file_storage() file_storage.add_file('tmp', size) t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024) libtorrent.set_piece_hashes(t, tmpdir) info = libtorrent.torrent_info(t.generate()) btih = sha1(info.metadata()).hexdigest() return info, btih async def main(): if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"): os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent") if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"): os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso") btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c" executor = None session = TorrentSession(asyncio.get_event_loop(), executor) session2 = TorrentSession(asyncio.get_event_loop(), executor) await session.bind('localhost', port=4040) await session2.bind('localhost', port=4041) btih = await session.add_fake_torrent() session2._session.add_dht_node(('localhost', 4040)) await session2.add_torrent(btih, "/tmp/down") while True: await asyncio.sleep(100) await session.pause() executor.shutdown() if __name__ == "__main__": asyncio.run(main()) ================================================ FILE: lbry/torrent/torrent.py ================================================ import asyncio import logging import typing log = logging.getLogger(__name__) class TorrentInfo: __slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size') def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]], http_seeds: typing.Tuple[typing.Dict[str, typing.Any]], trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int): self.dht_seeds = dht_seeds self.http_seeds = http_seeds self.trackers = trackers self.total_size = total_size @classmethod def from_libtorrent_info(cls, torrent_info): return cls( torrent_info.nodes(), tuple( { 'url': web_seed['url'], 'type': web_seed['type'], 'auth': web_seed['auth'] } for web_seed in torrent_info.web_seeds() ), tuple( (tracker.url, tracker.tier) for tracker in torrent_info.trackers() ), torrent_info.total_size() ) class Torrent: def __init__(self, loop, handle): self._loop = loop self._handle = handle self.finished = asyncio.Event() def _threaded_update_status(self): status = self._handle.status() if not status.is_seeding: log.info( '%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s', status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, status.num_peers, status.state ) elif not self.finished.is_set(): self.finished.set() async def wait_for_finished(self): while True: await self._loop.run_in_executor( None, self._threaded_update_status ) if self.finished.is_set(): log.info("finished downloading torrent!") await self.pause() break await asyncio.sleep(1) async def pause(self): log.info("pause torrent") await self._loop.run_in_executor( None, self._handle.pause ) async def resume(self): await self._loop.run_in_executor( None, self._handle.resume ) ================================================ FILE: lbry/torrent/torrent_manager.py ================================================ import asyncio import binascii import logging import os import typing from typing import Optional from aiohttp.web import Request from lbry.file.source_manager import SourceManager from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.torrent.session import TorrentSession from lbry.conf import Config from lbry.wallet.transaction import Transaction from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim from lbry.extras.daemon.storage import StoredContentClaim log = logging.getLogger(__name__) def path_or_none(encoded_path) -> Optional[str]: if not encoded_path: return return binascii.unhexlify(encoded_path).decode() class TorrentSource(ManagedDownloadSource): STATUS_STOPPED = "stopped" filter_fields = SourceManager.filter_fields filter_fields.update({ 'bt_infohash' }) def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str, file_name: Optional[str] = None, download_directory: Optional[str] = None, status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None, download_id: Optional[str] = None, rowid: Optional[int] = None, content_fee: Optional['Transaction'] = None, analytics_manager: Optional['AnalyticsManager'] = None, added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None): super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id, rowid, content_fee, analytics_manager, added_on) self.torrent_session = torrent_session @property def full_path(self) -> Optional[str]: full_path = self.torrent_session.full_path(self.identifier) self.download_directory = os.path.dirname(full_path) return full_path async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False): await self.torrent_session.add_torrent(self.identifier, self.download_directory) async def stop(self, finished: bool = False): await self.torrent_session.remove_torrent(self.identifier) async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): await self.torrent_session.save_file(self.identifier, download_directory) @property def torrent_length(self): return self.torrent_session.get_size(self.identifier) @property def written_bytes(self): return self.torrent_session.get_downloaded(self.identifier) @property def torrent_name(self): return self.torrent_session.get_name(self.identifier) @property def bt_infohash(self): return self.identifier async def stop_tasks(self): pass @property def completed(self): return self.torrent_session.is_completed(self.identifier) class TorrentManager(SourceManager): _sources: typing.Dict[str, ManagedDownloadSource] filter_fields = set(SourceManager.filter_fields) filter_fields.update({ 'bt_infohash', 'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept 'blobs_in_stream' }) def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession', storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None): super().__init__(loop, config, storage, analytics_manager) self.torrent_session: 'TorrentSession' = torrent_session async def recover_streams(self, file_infos: typing.List[typing.Dict]): raise NotImplementedError async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str], download_directory: Optional[str], status: str, claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'], added_on: Optional[int]): stream = TorrentSource( self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name, download_directory=download_directory, status=status, claim=claim, rowid=rowid, content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on, torrent_session=self.torrent_session ) self.add(stream) async def initialize_from_database(self): pass async def start(self): await super().start() async def stop(self): await super().stop() log.info("finished stopping the torrent manager") async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): await super().delete(source, delete_file) self.torrent_session.remove_torrent(source.identifier, delete_file) async def create(self, file_path: str, key: Optional[bytes] = None, iv_generator: Optional[typing.Generator[bytes, None, None]] = None): raise NotImplementedError async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): raise NotImplementedError # blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]] # await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) # await self.storage.delete_stream(source.descriptor) async def stream_partial_content(self, request: Request, sd_hash: str): raise NotImplementedError ================================================ FILE: lbry/torrent/tracker.py ================================================ import random import socket import string import struct import asyncio import logging import time import ipaddress from collections import namedtuple from functools import reduce from typing import Optional from lbry.dht.node import get_kademlia_peers_from_hosts from lbry.utils import resolve_host, async_timed_cache, cache_concurrent from lbry.wallet.stream import StreamController from lbry import version log = logging.getLogger(__name__) CONNECTION_EXPIRES_AFTER_SECONDS = 50 PREFIX = 'LB' # todo: PR BEP20 to add ourselves DEFAULT_TIMEOUT_SECONDS = 10.0 DEFAULT_CONCURRENCY_LIMIT = 100 # see: http://bittorrent.org/beps/bep_0015.html and http://xbtt.sourceforge.net/udp_tracker_protocol.html ConnectRequest = namedtuple("ConnectRequest", ["connection_id", "action", "transaction_id"]) ConnectResponse = namedtuple("ConnectResponse", ["action", "transaction_id", "connection_id"]) AnnounceRequest = namedtuple("AnnounceRequest", ["connection_id", "action", "transaction_id", "info_hash", "peer_id", "downloaded", "left", "uploaded", "event", "ip_addr", "key", "num_want", "port"]) AnnounceResponse = namedtuple("AnnounceResponse", ["action", "transaction_id", "interval", "leechers", "seeders", "peers"]) CompactIPv4Peer = namedtuple("CompactPeer", ["address", "port"]) ScrapeRequest = namedtuple("ScrapeRequest", ["connection_id", "action", "transaction_id", "infohashes"]) ScrapeResponse = namedtuple("ScrapeResponse", ["action", "transaction_id", "items"]) ScrapeResponseItem = namedtuple("ScrapeResponseItem", ["seeders", "completed", "leechers"]) ErrorResponse = namedtuple("ErrorResponse", ["action", "transaction_id", "message"]) structs = { ConnectRequest: struct.Struct(">QII"), ConnectResponse: struct.Struct(">IIQ"), AnnounceRequest: struct.Struct(">QII20s20sQQQIIIiH"), AnnounceResponse: struct.Struct(">IIIII"), CompactIPv4Peer: struct.Struct(">IH"), ScrapeRequest: struct.Struct(">QII"), ScrapeResponse: struct.Struct(">II"), ScrapeResponseItem: struct.Struct(">III"), ErrorResponse: struct.Struct(">II") } def decode(cls, data, offset=0): decoder = structs[cls] if cls is AnnounceResponse: return AnnounceResponse(*decoder.unpack_from(data, offset), peers=[decode(CompactIPv4Peer, data, index) for index in range(20, len(data), 6)]) elif cls is ScrapeResponse: return ScrapeResponse(*decoder.unpack_from(data, offset), items=[decode(ScrapeResponseItem, data, index) for index in range(8, len(data), 12)]) elif cls is ErrorResponse: return ErrorResponse(*decoder.unpack_from(data, offset), data[decoder.size:]) return cls(*decoder.unpack_from(data, offset)) def encode(obj): if isinstance(obj, ScrapeRequest): return structs[ScrapeRequest].pack(*obj[:-1]) + b''.join(obj.infohashes) elif isinstance(obj, ErrorResponse): return structs[ErrorResponse].pack(*obj[:-1]) + obj.message elif isinstance(obj, AnnounceResponse): return structs[AnnounceResponse].pack(*obj[:-1]) + b''.join([encode(peer) for peer in obj.peers]) return structs[type(obj)].pack(*obj) def make_peer_id(random_part: Optional[str] = None) -> bytes: # see https://wiki.theory.org/BitTorrentSpecification#peer_id and https://www.bittorrent.org/beps/bep_0020.html # not to confuse with node id; peer id identifies uniquely the software, version and instance random_part = random_part or ''.join(random.choice(string.ascii_letters) for _ in range(20)) return f"{PREFIX}-{'-'.join(map(str, version))}-{random_part}"[:20].encode() class UDPTrackerClientProtocol(asyncio.DatagramProtocol): def __init__(self, timeout: float = DEFAULT_TIMEOUT_SECONDS): self.transport = None self.data_queue = {} self.timeout = timeout self.semaphore = asyncio.Semaphore(DEFAULT_CONCURRENCY_LIMIT) def connection_made(self, transport: asyncio.DatagramTransport) -> None: self.transport = transport async def request(self, obj, tracker_ip, tracker_port): self.data_queue[obj.transaction_id] = asyncio.get_running_loop().create_future() try: async with self.semaphore: self.transport.sendto(encode(obj), (tracker_ip, tracker_port)) return await asyncio.wait_for(self.data_queue[obj.transaction_id], self.timeout) finally: self.data_queue.pop(obj.transaction_id, None) async def connect(self, tracker_ip, tracker_port): transaction_id = random.getrandbits(32) return decode(ConnectResponse, await self.request(ConnectRequest(0x41727101980, 0, transaction_id), tracker_ip, tracker_port)) @cache_concurrent @async_timed_cache(CONNECTION_EXPIRES_AFTER_SECONDS) async def ensure_connection_id(self, peer_id, tracker_ip, tracker_port): # peer_id is just to ensure cache coherency return (await self.connect(tracker_ip, tracker_port)).connection_id async def announce(self, info_hash, peer_id, port, tracker_ip, tracker_port, stopped=False): connection_id = await self.ensure_connection_id(peer_id, tracker_ip, tracker_port) # this should make the key deterministic but unique per info hash + peer id key = int.from_bytes(info_hash[:4], "big") ^ int.from_bytes(peer_id[:4], "big") ^ port transaction_id = random.getrandbits(32) req = AnnounceRequest( connection_id, 1, transaction_id, info_hash, peer_id, 0, 0, 0, 3 if stopped else 1, 0, key, -1, port) return decode(AnnounceResponse, await self.request(req, tracker_ip, tracker_port)) async def scrape(self, infohashes, tracker_ip, tracker_port, connection_id=None): connection_id = await self.ensure_connection_id(None, tracker_ip, tracker_port) transaction_id = random.getrandbits(32) reply = await self.request( ScrapeRequest(connection_id, 2, transaction_id, infohashes), tracker_ip, tracker_port) return decode(ScrapeResponse, reply), connection_id def datagram_received(self, data: bytes, addr: (str, int)) -> None: if len(data) < 8: return transaction_id = int.from_bytes(data[4:8], byteorder="big", signed=False) if transaction_id in self.data_queue: if not self.data_queue[transaction_id].done(): if data[3] == 3: return self.data_queue[transaction_id].set_exception(Exception(decode(ErrorResponse, data).message)) return self.data_queue[transaction_id].set_result(data) log.debug("unexpected packet (can be a response for a previously timed out request): %s", data.hex()) def connection_lost(self, exc: Exception = None) -> None: self.transport = None class TrackerClient: event_controller = StreamController() def __init__(self, node_id, announce_port, get_servers, timeout=10.0): self.client = UDPTrackerClientProtocol(timeout=timeout) self.transport = None self.peer_id = make_peer_id(node_id.hex() if node_id else None) self.announce_port = announce_port self._get_servers = get_servers self.results = {} # we can't probe the server before the interval, so we keep the result here until it expires self.tasks = {} async def start(self): self.transport, _ = await asyncio.get_running_loop().create_datagram_endpoint( lambda: self.client, local_addr=("0.0.0.0", 0)) self.event_controller.stream.listen( lambda request: self.on_hash(request[1], request[2]) if request[0] == 'search' else None) def stop(self): while self.tasks: self.tasks.popitem()[1].cancel() if self.transport is not None: self.transport.close() self.client = None self.transport = None self.event_controller.close() def on_hash(self, info_hash, on_announcement=None): if info_hash not in self.tasks: task = asyncio.create_task(self.get_peer_list(info_hash, on_announcement=on_announcement)) task.add_done_callback(lambda *_: self.tasks.pop(info_hash, None)) self.tasks[info_hash] = task async def announce_many(self, *info_hashes, stopped=False): await asyncio.gather( *[self._announce_many(server, info_hashes, stopped=stopped) for server in self._get_servers()], return_exceptions=True) async def _announce_many(self, server, info_hashes, stopped=False): tracker_ip = await resolve_host(*server, 'udp') still_good_info_hashes = { info_hash for (info_hash, (next_announcement, _)) in self.results.get(tracker_ip, {}).items() if time.time() < next_announcement } results = await asyncio.gather( *[self._probe_server(info_hash, tracker_ip, server[1], stopped=stopped) for info_hash in info_hashes if info_hash not in still_good_info_hashes], return_exceptions=True) if results: errors = sum([1 for result in results if result is None or isinstance(result, Exception)]) log.info("Tracker: finished announcing %d files to %s:%d, %d errors", len(results), *server, errors) async def get_peer_list(self, info_hash, stopped=False, on_announcement=None, no_port=False): found = [] probes = [self._probe_server(info_hash, *server, stopped, no_port) for server in self._get_servers()] for done in asyncio.as_completed(probes): result = await done if result is not None: await asyncio.gather(*filter(asyncio.iscoroutine, [on_announcement(result)] if on_announcement else [])) found.append(result) return found async def get_kademlia_peer_list(self, info_hash): responses = await self.get_peer_list(info_hash, no_port=True) return await announcement_to_kademlia_peers(*responses) async def _probe_server(self, info_hash, tracker_host, tracker_port, stopped=False, no_port=False): result = None try: tracker_host = await resolve_host(tracker_host, tracker_port, 'udp') except socket.error: log.warning("DNS failure while resolving tracker host: %s, skipping.", tracker_host) return self.results.setdefault(tracker_host, {}) if info_hash in self.results[tracker_host]: next_announcement, result = self.results[tracker_host][info_hash] if time.time() < next_announcement: return result try: result = await self.client.announce( info_hash, self.peer_id, 0 if no_port else self.announce_port, tracker_host, tracker_port, stopped) self.results[tracker_host][info_hash] = (time.time() + result.interval, result) except asyncio.TimeoutError: # todo: this is UDP, timeout is common, we need a better metric for failures self.results[tracker_host][info_hash] = (time.time() + 60.0, result) log.debug("Tracker timed out: %s:%d", tracker_host, tracker_port) return None log.debug("Announced: %s found %d peers for %s", tracker_host, len(result.peers), info_hash.hex()[:8]) return result def enqueue_tracker_search(info_hash: bytes, peer_q: asyncio.Queue): async def on_announcement(announcement: AnnounceResponse): peers = await announcement_to_kademlia_peers(announcement) log.info("Found %d peers from tracker for %s", len(peers), info_hash.hex()[:8]) peer_q.put_nowait(peers) TrackerClient.event_controller.add(('search', info_hash, on_announcement)) def announcement_to_kademlia_peers(*announcements: AnnounceResponse): peers = [ (str(ipaddress.ip_address(peer.address)), peer.port) for announcement in announcements for peer in announcement.peers if peer.port > 1024 # no privileged or 0 ] return get_kademlia_peers_from_hosts(peers) class UDPTrackerServerProtocol(asyncio.DatagramProtocol): # for testing. Not suitable for production def __init__(self): self.transport = None self.known_conns = set() self.peers = {} def connection_made(self, transport: asyncio.DatagramTransport) -> None: self.transport = transport def add_peer(self, info_hash, ip_address: str, port: int): self.peers.setdefault(info_hash, []) self.peers[info_hash].append(encode_peer(ip_address, port)) def datagram_received(self, data: bytes, addr: (str, int)) -> None: if len(data) < 16: return action = int.from_bytes(data[8:12], "big", signed=False) if action == 0: req = decode(ConnectRequest, data) connection_id = random.getrandbits(32) self.known_conns.add(connection_id) return self.transport.sendto(encode(ConnectResponse(0, req.transaction_id, connection_id)), addr) elif action == 1: req = decode(AnnounceRequest, data) if req.connection_id not in self.known_conns: resp = encode(ErrorResponse(3, req.transaction_id, b'Connection ID missmatch.\x00')) else: compact_address = encode_peer(addr[0], req.port) if req.event != 3: self.add_peer(req.info_hash, addr[0], req.port) elif compact_address in self.peers.get(req.info_hash, []): self.peers[req.info_hash].remove(compact_address) peers = [decode(CompactIPv4Peer, peer) for peer in self.peers[req.info_hash]] resp = encode(AnnounceResponse(1, req.transaction_id, 1700, 0, len(peers), peers)) return self.transport.sendto(resp, addr) def encode_peer(ip_address: str, port: int): compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), ip_address.split('.'), bytearray()) return compact_ip + port.to_bytes(2, "big", signed=False) ================================================ FILE: lbry/utils.py ================================================ import base64 import codecs import datetime import random import socket import time import string import sys import json import typing import asyncio import ssl import logging import ipaddress import contextlib import functools import collections import hashlib import pkg_resources import certifi import aiohttp from prometheus_client import Counter from lbry.schema.claim import Claim log = logging.getLogger(__name__) # defining these time functions here allows for easier overriding in testing def now(): return datetime.datetime.now() def utcnow(): return datetime.datetime.utcnow() def isonow(): """Return utc now in isoformat with timezone""" return utcnow().isoformat() + 'Z' def today(): return datetime.datetime.today() def timedelta(**kwargs): return datetime.timedelta(**kwargs) def datetime_obj(*args, **kwargs): return datetime.datetime(*args, **kwargs) def get_lbry_hash_obj(): return hashlib.sha384() def generate_id(num=None): h = get_lbry_hash_obj() if num is not None: h.update(str(num).encode()) else: h.update(str(random.getrandbits(512)).encode()) return h.digest() def version_is_greater_than(version_a, version_b): """Returns True if version a is more recent than version b""" return pkg_resources.parse_version(version_a) > pkg_resources.parse_version(version_b) def rot13(some_str): return codecs.encode(some_str, 'rot_13') def deobfuscate(obfustacated): return base64.b64decode(rot13(obfustacated)).decode() def obfuscate(plain): return rot13(base64.b64encode(plain).decode()) def check_connection(server="lbry.com", port=80, timeout=5) -> bool: """Attempts to open a socket to server:port and returns True if successful.""" log.debug('Checking connection to %s:%s', server, port) try: server = socket.gethostbyname(server) socket.create_connection((server, port), timeout).close() return True except (socket.gaierror, socket.herror): log.debug("Failed to connect to %s:%s. Unable to resolve domain. Trying to bypass DNS", server, port) try: server = "8.8.8.8" port = 53 socket.create_connection((server, port), timeout).close() return True except OSError: return False except OSError: return False def random_string(length=10, chars=string.ascii_lowercase): return ''.join([random.choice(chars) for _ in range(length)]) def short_hash(hash_str): return hash_str[:6] def get_sd_hash(stream_info): if not stream_info: return None if isinstance(stream_info, Claim): return stream_info.stream.source.sd_hash result = stream_info.get('claim', {}).\ get('value', {}).\ get('stream', {}).\ get('source', {}).\ get('source') if not result: log.warning("Unable to get sd_hash") return result def json_dumps_pretty(obj, **kwargs): return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs) try: # the standard contextlib.aclosing() is available in 3.10+ from contextlib import aclosing # pylint: disable=unused-import except ImportError: @contextlib.asynccontextmanager async def aclosing(thing): try: yield thing finally: await thing.aclose() def async_timed_cache(duration: int): def wrapper(func): cache: typing.Dict[typing.Tuple, typing.Tuple[typing.Any, float]] = {} @functools.wraps(func) async def _inner(*args, **kwargs) -> typing.Any: loop = asyncio.get_running_loop() time_now = loop.time() key = (args, tuple(kwargs.items())) if key in cache and (time_now - cache[key][1] < duration): return cache[key][0] to_cache = await func(*args, **kwargs) cache[key] = to_cache, time_now return to_cache return _inner return wrapper def cache_concurrent(async_fn): """ When the decorated function has concurrent calls made to it with the same arguments, only run it once """ cache: typing.Dict = {} @functools.wraps(async_fn) async def wrapper(*args, **kwargs): key = (args, tuple(kwargs.items())) cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs)) try: return await cache[key] finally: cache.pop(key, None) return wrapper @async_timed_cache(300) async def resolve_host(url: str, port: int, proto: str) -> str: if proto not in ['udp', 'tcp']: raise Exception("invalid protocol") if url.lower() == 'localhost': return '127.0.0.1' try: if ipaddress.ip_address(url): return url except ValueError: pass loop = asyncio.get_running_loop() return (await loop.getaddrinfo( url, port, proto=socket.IPPROTO_TCP if proto == 'tcp' else socket.IPPROTO_UDP, type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM, family=socket.AF_INET ))[0][4][0] class LRUCacheWithMetrics: __slots__ = [ 'capacity', 'cache', '_track_metrics', 'hits', 'misses' ] def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"): self.capacity = capacity self.cache = collections.OrderedDict() if metric_name is None: self._track_metrics = False self.hits = self.misses = None else: self._track_metrics = True try: self.hits = Counter( f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace ) self.misses = Counter( f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace ) except ValueError as err: log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err) self._track_metrics = False self.hits = self.misses = None def get(self, key, default=None): try: value = self.cache.pop(key) if self._track_metrics: self.hits.inc() except KeyError: if self._track_metrics: self.misses.inc() return default self.cache[key] = value return value def set(self, key, value): try: self.cache.pop(key) except KeyError: if len(self.cache) >= self.capacity: self.cache.popitem(last=False) self.cache[key] = value def clear(self): self.cache.clear() def pop(self, key): return self.cache.pop(key) def __setitem__(self, key, value): return self.set(key, value) def __getitem__(self, item): return self.get(item) def __contains__(self, item) -> bool: return item in self.cache def __len__(self): return len(self.cache) def __delitem__(self, key): self.cache.pop(key) def __del__(self): self.clear() class LRUCache: __slots__ = [ 'capacity', 'cache' ] def __init__(self, capacity: int): self.capacity = capacity self.cache = collections.OrderedDict() def get(self, key, default=None): try: value = self.cache.pop(key) except KeyError: return default self.cache[key] = value return value def set(self, key, value): try: self.cache.pop(key) except KeyError: if len(self.cache) >= self.capacity: self.cache.popitem(last=False) self.cache[key] = value def items(self): return self.cache.items() def clear(self): self.cache.clear() def pop(self, key, default=None): return self.cache.pop(key, default) def __setitem__(self, key, value): return self.set(key, value) def __getitem__(self, item): return self.get(item) def __contains__(self, item) -> bool: return item in self.cache def __len__(self): return len(self.cache) def __delitem__(self, key): self.cache.pop(key) def __del__(self): self.clear() def lru_cache_concurrent(cache_size: typing.Optional[int] = None, override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None): if not cache_size and override_lru_cache is None: raise ValueError("invalid cache size") concurrent_cache = {} lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size) def wrapper(async_fn): @functools.wraps(async_fn) async def _inner(*args, **kwargs): key = (args, tuple(kwargs.items())) if key in lru_cache: return lru_cache.get(key) concurrent_cache[key] = concurrent_cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs)) try: result = await concurrent_cache[key] lru_cache.set(key, result) return result finally: concurrent_cache.pop(key, None) return _inner return wrapper def get_ssl_context() -> ssl.SSLContext: return ssl.create_default_context( purpose=ssl.Purpose.CLIENT_AUTH, capath=certifi.where() ) @contextlib.asynccontextmanager async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]: async with aiohttp.ClientSession() as session: async with session.request(method, url, **kwargs) as response: yield response # the ipaddress module does not show these subnets as reserved CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10') IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24') def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False): try: parsed_ip = ipaddress.ip_address(address) if parsed_ip.is_loopback and allow_localhost: return True if allow_lan and parsed_ip.is_private: return True if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)): return False else: return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")), IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")))) except (ipaddress.AddressValueError, ValueError): return False async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection try: async with aiohttp_request("get", "https://api.lbry.com/ip") as resp: response = await resp.json() if response['success']: return response['data']['ip'], None except Exception: return None, None async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]: # used if upnp is disabled or non-functioning from lbry.wallet.udp import SPVStatusClientProtocol # pylint: disable=C0415 hostname_to_ip = {} ip_to_hostnames = collections.defaultdict(list) async def resolve_spv(server, port): try: server_addr = await resolve_host(server, port, 'udp') hostname_to_ip[server] = (server_addr, port) ip_to_hostnames[(server_addr, port)].append(server) except Exception: log.exception("error looking up dns for spv servers") # accumulate the dns results await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers)) loop = asyncio.get_event_loop() pong_responses = asyncio.Queue() connection = SPVStatusClientProtocol(pong_responses) try: await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0)) # could raise OSError if it cant bind randomized_servers = list(ip_to_hostnames.keys()) random.shuffle(randomized_servers) for server in randomized_servers: connection.ping(server) try: _, pong = await asyncio.wait_for(pong_responses.get(), 1) if is_valid_public_ipv4(pong.ip_address): return pong.ip_address, ip_to_hostnames[server][0] except asyncio.TimeoutError: pass return None, None finally: connection.close() async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]: ip_from_spv_servers = await _get_external_ip(default_servers) if not ip_from_spv_servers[1]: return await fallback_get_external_ip() return ip_from_spv_servers def is_running_from_bundle(): # see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') class LockWithMetrics(asyncio.Lock): def __init__(self, acquire_metric, held_time_metric): super().__init__() self._acquire_metric = acquire_metric self._lock_held_time_metric = held_time_metric self._lock_acquired_time = None async def acquire(self): start = time.perf_counter() try: return await super().acquire() finally: self._lock_acquired_time = time.perf_counter() self._acquire_metric.observe(self._lock_acquired_time - start) def release(self): try: return super().release() finally: self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time) def get_colliding_prefix_bits(first_value: bytes, second_value: bytes): """ Calculates the amount of colliding prefix bits between <first_value> and <second_value>. This is given by the amount of bits that are the same until the first different one (via XOR), starting from the most significant bit to the least significant bit. :param first_value: first value to compare, bigger than size. :param second_value: second value to compare, bigger than size. :return: amount of prefix colliding bits. """ assert len(first_value) == len(second_value), "length should be the same" size = len(first_value) * 8 first_value, second_value = int.from_bytes(first_value, "big"), int.from_bytes(second_value, "big") return size - (first_value ^ second_value).bit_length() ================================================ FILE: lbry/wallet/__init__.py ================================================ __lbcd__ = 'lbcd' __lbcctl__ = 'lbcctl' __lbcwallet__ = 'lbcwallet' __lbcd_url__ = ( 'https://github.com/lbryio/lbcd/releases/download/' + 'v0.22.100-rc.0/lbcd_0.22.100-rc.0_TARGET_PLATFORM.tar.gz' ) __lbcwallet_url__ = ( 'https://github.com/lbryio/lbcwallet/releases/download/' + 'v0.13.100-alpha.0/lbcwallet_0.13.100-alpha.0_TARGET_PLATFORM.tar.gz' ) __spvserver__ = 'lbry.wallet.server.coin.LBCRegTest' from lbry.wallet.wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DISK from lbry.wallet.manager import WalletManager from lbry.wallet.network import Network from lbry.wallet.ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent from lbry.wallet.account import Account, AddressManager, SingleKey, HierarchicalDeterministic, \ DeterministicChannelKeyManager from lbry.wallet.transaction import Transaction, Output, Input from lbry.wallet.script import OutputScript, InputScript from lbry.wallet.database import SQLiteMixin, Database from lbry.wallet.header import Headers ================================================ FILE: lbry/wallet/account.py ================================================ import os import time import json import logging import typing import asyncio import random from hashlib import sha256 from string import hexdigits from typing import Type, Dict, Tuple, Optional, Any, List from lbry.error import InvalidPasswordError from lbry.crypto.crypt import aes_encrypt, aes_decrypt from .bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string from .mnemonic import Mnemonic from .constants import COIN, TXO_TYPES from .transaction import Transaction, Input, Output if typing.TYPE_CHECKING: from .ledger import Ledger from .wallet import Wallet log = logging.getLogger(__name__) def validate_claim_id(claim_id): if not len(claim_id) == 40: raise Exception("Incorrect claimid length: %i" % len(claim_id)) if isinstance(claim_id, bytes): claim_id = claim_id.decode('utf-8') if set(claim_id).difference(hexdigits): raise Exception("Claim id is not hex encoded") class DeterministicChannelKeyManager: def __init__(self, account: 'Account'): self.account = account self.last_known = 0 self.cache = {} self._private_key: Optional[PrivateKey] = None @property def private_key(self): if self._private_key is None: if self.account.private_key is not None: self._private_key = self.account.private_key.child(KeyPath.CHANNEL) return self._private_key def maybe_generate_deterministic_key_for_channel(self, txo): if self.private_key is None: return next_private_key = self.private_key.child(self.last_known) public_key = next_private_key.public_key public_key_bytes = public_key.pubkey_bytes if txo.claim.channel.public_key_bytes == public_key_bytes: self.cache[public_key.address] = next_private_key self.last_known += 1 async def ensure_cache_primed(self): if self.private_key is not None: await self.generate_next_key() async def generate_next_key(self) -> PrivateKey: db = self.account.ledger.db while True: next_private_key = self.private_key.child(self.last_known) public_key = next_private_key.public_key self.cache[public_key.address] = next_private_key if not await db.is_channel_key_used(self.account, public_key): return next_private_key self.last_known += 1 def get_private_key_from_pubkey_hash(self, pubkey_hash) -> PrivateKey: return self.cache.get(pubkey_hash) class AddressManager: name: str __slots__ = 'account', 'public_key', 'chain_number', 'address_generator_lock' def __init__(self, account, public_key, chain_number): self.account = account self.public_key = public_key self.chain_number = chain_number self.address_generator_lock = asyncio.Lock() @classmethod def from_dict(cls, account: 'Account', d: dict) \ -> Tuple['AddressManager', 'AddressManager']: raise NotImplementedError @classmethod def to_dict(cls, receiving: 'AddressManager', change: 'AddressManager') -> Dict: d: Dict[str, Any] = {'name': cls.name} receiving_dict = receiving.to_dict_instance() if receiving_dict: d['receiving'] = receiving_dict change_dict = change.to_dict_instance() if change_dict: d['change'] = change_dict return d def merge(self, d: dict): pass def to_dict_instance(self) -> Optional[dict]: raise NotImplementedError def _query_addresses(self, **constraints): return self.account.ledger.db.get_addresses( read_only=constraints.pop("read_only", False), accounts=[self.account], chain=self.chain_number, **constraints ) def get_private_key(self, index: int) -> PrivateKey: raise NotImplementedError def get_public_key(self, index: int) -> PublicKey: raise NotImplementedError async def get_max_gap(self): raise NotImplementedError async def ensure_address_gap(self): raise NotImplementedError def get_address_records(self, only_usable: bool = False, **constraints): raise NotImplementedError async def get_addresses(self, only_usable: bool = False, **constraints) -> List[str]: records = await self.get_address_records(only_usable=only_usable, **constraints) return [r['address'] for r in records] async def get_or_create_usable_address(self) -> str: async with self.address_generator_lock: addresses = await self.get_addresses(only_usable=True, limit=10) if addresses: return random.choice(addresses) addresses = await self.ensure_address_gap() return addresses[0] class HierarchicalDeterministic(AddressManager): """ Implements simple version of Bitcoin Hierarchical Deterministic key management. """ name: str = "deterministic-chain" __slots__ = 'gap', 'maximum_uses_per_address' def __init__(self, account: 'Account', chain: int, gap: int, maximum_uses_per_address: int) -> None: super().__init__(account, account.public_key.child(chain), chain) self.gap = gap self.maximum_uses_per_address = maximum_uses_per_address @classmethod def from_dict(cls, account: 'Account', d: dict) -> Tuple[AddressManager, AddressManager]: return ( cls(account, KeyPath.RECEIVE, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})), cls(account, KeyPath.CHANGE, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1})) ) def merge(self, d: dict): self.gap = d.get('gap', self.gap) self.maximum_uses_per_address = d.get('maximum_uses_per_address', self.maximum_uses_per_address) def to_dict_instance(self): return {'gap': self.gap, 'maximum_uses_per_address': self.maximum_uses_per_address} def get_private_key(self, index: int) -> PrivateKey: return self.account.private_key.child(self.chain_number).child(index) def get_public_key(self, index: int) -> PublicKey: return self.account.public_key.child(self.chain_number).child(index) async def get_max_gap(self) -> int: addresses = await self._query_addresses(order_by="n asc") max_gap = 0 current_gap = 0 for address in addresses: if address['used_times'] == 0: current_gap += 1 else: max_gap = max(max_gap, current_gap) current_gap = 0 return max_gap async def ensure_address_gap(self) -> List[str]: async with self.address_generator_lock: addresses = await self._query_addresses(limit=self.gap, order_by="n desc") existing_gap = 0 for address in addresses: if address['used_times'] == 0: existing_gap += 1 else: break if existing_gap == self.gap: return [] start = addresses[0]['pubkey'].n+1 if addresses else 0 end = start + (self.gap - existing_gap) new_keys = await self._generate_keys(start, end-1) await self.account.ledger.announce_addresses(self, new_keys) return new_keys async def _generate_keys(self, start: int, end: int) -> List[str]: if not self.address_generator_lock.locked(): raise RuntimeError('Should not be called outside of address_generator_lock.') keys = [self.public_key.child(index) for index in range(start, end+1)] await self.account.ledger.db.add_keys(self.account, self.chain_number, keys) return [key.address for key in keys] def get_address_records(self, only_usable: bool = False, **constraints): if only_usable: constraints['used_times__lt'] = self.maximum_uses_per_address if 'order_by' not in constraints: constraints['order_by'] = "used_times asc, n asc" return self._query_addresses(**constraints) class SingleKey(AddressManager): """ Single Key address manager always returns the same address for all operations. """ name: str = "single-address" __slots__ = () @classmethod def from_dict(cls, account: 'Account', d: dict) \ -> Tuple[AddressManager, AddressManager]: same_address_manager = cls(account, account.public_key, KeyPath.RECEIVE) return same_address_manager, same_address_manager def to_dict_instance(self): return None def get_private_key(self, index: int) -> PrivateKey: return self.account.private_key def get_public_key(self, index: int) -> PublicKey: return self.account.public_key async def get_max_gap(self) -> int: return 0 async def ensure_address_gap(self) -> List[str]: async with self.address_generator_lock: exists = await self.get_address_records() if not exists: await self.account.ledger.db.add_keys(self.account, self.chain_number, [self.public_key]) new_keys = [self.public_key.address] await self.account.ledger.announce_addresses(self, new_keys) return new_keys return [] def get_address_records(self, only_usable: bool = False, **constraints): return self._query_addresses(**constraints) class Account: address_generators: Dict[str, Type[AddressManager]] = { SingleKey.name: SingleKey, HierarchicalDeterministic.name: HierarchicalDeterministic, } def __init__(self, ledger: 'Ledger', wallet: 'Wallet', name: str, seed: str, private_key_string: str, encrypted: bool, private_key: Optional[PrivateKey], public_key: PublicKey, address_generator: dict, modified_on: float, channel_keys: dict) -> None: self.ledger = ledger self.wallet = wallet self.id = public_key.address self.name = name self.seed = seed self.modified_on = modified_on self.private_key_string = private_key_string self.init_vectors: Dict[str, bytes] = {} self.encrypted = encrypted self.private_key: Optional[PrivateKey] = private_key self.public_key: PublicKey = public_key generator_name = address_generator.get('name', HierarchicalDeterministic.name) self.address_generator = self.address_generators[generator_name] self.receiving, self.change = self.address_generator.from_dict(self, address_generator) self.address_managers = {am.chain_number: am for am in (self.receiving, self.change)} self.channel_keys = channel_keys self.deterministic_channel_keys = DeterministicChannelKeyManager(self) ledger.add_account(self) wallet.add_account(self) def get_init_vector(self, key) -> Optional[bytes]: init_vector = self.init_vectors.get(key, None) if init_vector is None: init_vector = self.init_vectors[key] = os.urandom(16) return init_vector @classmethod def generate(cls, ledger: 'Ledger', wallet: 'Wallet', name: str = None, address_generator: dict = None): return cls.from_dict(ledger, wallet, { 'name': name, 'seed': Mnemonic().make_seed(), 'address_generator': address_generator or {} }) @classmethod def get_private_key_from_seed(cls, ledger: 'Ledger', seed: str, password: str): return PrivateKey.from_seed( ledger, Mnemonic.mnemonic_to_seed(seed, password or 'lbryum') ) @classmethod def keys_from_dict(cls, ledger: 'Ledger', d: dict) \ -> Tuple[str, Optional[PrivateKey], PublicKey]: seed = d.get('seed', '') private_key_string = d.get('private_key', '') private_key = None public_key = None encrypted = d.get('encrypted', False) if not encrypted: if seed: private_key = cls.get_private_key_from_seed(ledger, seed, '') public_key = private_key.public_key elif private_key_string: private_key = from_extended_key_string(ledger, private_key_string) public_key = private_key.public_key if public_key is None: public_key = from_extended_key_string(ledger, d['public_key']) return seed, private_key, public_key @classmethod def from_dict(cls, ledger: 'Ledger', wallet: 'Wallet', d: dict): seed, private_key, public_key = cls.keys_from_dict(ledger, d) name = d.get('name') if not name: name = f'Account #{public_key.address}' return cls( ledger=ledger, wallet=wallet, name=name, seed=seed, private_key_string=d.get('private_key', ''), encrypted=d.get('encrypted', False), private_key=private_key, public_key=public_key, address_generator=d.get('address_generator', {}), modified_on=int(d.get('modified_on', time.time())), channel_keys=d.get('certificates', {}) ) def to_dict(self, encrypt_password: str = None, include_channel_keys: bool = True): private_key_string, seed = self.private_key_string, self.seed if not self.encrypted and self.private_key: private_key_string = self.private_key.extended_key_string() if not self.encrypted and encrypt_password: if private_key_string: private_key_string = aes_encrypt( encrypt_password, private_key_string, self.get_init_vector('private_key') ) if seed: seed = aes_encrypt(encrypt_password, self.seed, self.get_init_vector('seed')) d = { 'ledger': self.ledger.get_id(), 'name': self.name, 'seed': seed, 'encrypted': bool(self.encrypted or encrypt_password), 'private_key': private_key_string, 'public_key': self.public_key.extended_key_string(), 'address_generator': self.address_generator.to_dict(self.receiving, self.change), 'modified_on': self.modified_on } if include_channel_keys: d['certificates'] = self.channel_keys return d def merge(self, d: dict): if d.get('modified_on', 0) > self.modified_on: self.name = d['name'] self.modified_on = int(d.get('modified_on', time.time())) assert self.address_generator.name == d['address_generator']['name'] for chain_name in ('change', 'receiving'): if chain_name in d['address_generator']: chain_object = getattr(self, chain_name) chain_object.merge(d['address_generator'][chain_name]) self.channel_keys.update(d.get('certificates', {})) @property def hash(self) -> bytes: assert not self.encrypted, "Cannot hash an encrypted account." h = sha256(json.dumps(self.to_dict(include_channel_keys=False)).encode()) for cert in sorted(self.channel_keys.keys()): h.update(cert.encode()) return h.digest() async def get_details(self, show_seed=False, **kwargs): satoshis = await self.get_balance(**kwargs) details = { 'id': self.id, 'name': self.name, 'ledger': self.ledger.get_id(), 'coins': round(satoshis/COIN, 2), 'satoshis': satoshis, 'encrypted': self.encrypted, 'public_key': self.public_key.extended_key_string(), 'address_generator': self.address_generator.to_dict(self.receiving, self.change) } if show_seed: details['seed'] = self.seed details['certificates'] = len(self.channel_keys) return details def decrypt(self, password: str) -> bool: assert self.encrypted, "Key is not encrypted." try: seed = self._decrypt_seed(password) except (ValueError, InvalidPasswordError): return False try: private_key = self._decrypt_private_key_string(password) except (TypeError, ValueError, InvalidPasswordError): return False self.seed = seed self.private_key = private_key self.private_key_string = "" self.encrypted = False return True def _decrypt_private_key_string(self, password: str) -> Optional[PrivateKey]: if not self.private_key_string: return None private_key_string, self.init_vectors['private_key'] = aes_decrypt(password, self.private_key_string) if not private_key_string: return None return from_extended_key_string( self.ledger, private_key_string ) def _decrypt_seed(self, password: str) -> str: if not self.seed: return "" seed, self.init_vectors['seed'] = aes_decrypt(password, self.seed) if not seed: return "" try: Mnemonic().mnemonic_decode(seed) except IndexError: # failed to decode the seed, this either means it decrypted and is invalid # or that we hit an edge case where an incorrect password gave valid padding raise ValueError("Failed to decode seed.") return seed def encrypt(self, password: str) -> bool: assert not self.encrypted, "Key is already encrypted." if self.seed: self.seed = aes_encrypt(password, self.seed, self.get_init_vector('seed')) if isinstance(self.private_key, PrivateKey): self.private_key_string = aes_encrypt( password, self.private_key.extended_key_string(), self.get_init_vector('private_key') ) self.private_key = None self.encrypted = True return True async def ensure_address_gap(self): addresses = [] for address_manager in self.address_managers.values(): new_addresses = await address_manager.ensure_address_gap() addresses.extend(new_addresses) return addresses async def get_addresses(self, read_only=False, **constraints) -> List[str]: rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints) return [r['address'] for r in rows] def get_address_records(self, **constraints): return self.ledger.db.get_addresses(accounts=[self], **constraints) def get_address_count(self, **constraints): return self.ledger.db.get_address_count(accounts=[self], **constraints) def get_private_key(self, chain: int, index: int) -> PrivateKey: assert not self.encrypted, "Cannot get private key on encrypted wallet account." return self.address_managers[chain].get_private_key(index) def get_public_key(self, chain: int, index: int) -> PublicKey: return self.address_managers[chain].get_public_key(index) def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints): if not include_claims: constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])}) if confirmations > 0: height = self.ledger.headers.height - (confirmations-1) constraints.update({'height__lte': height, 'height__gt': 0}) return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints) async def get_max_gap(self): change_gap = await self.change.get_max_gap() receiving_gap = await self.receiving.get_max_gap() return { 'max_change_gap': change_gap, 'max_receiving_gap': receiving_gap, } def get_txos(self, **constraints): return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints) def get_txo_count(self, **constraints): return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints) def get_utxos(self, **constraints): return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints) def get_utxo_count(self, **constraints): return self.ledger.get_utxo_count(wallet=self.wallet, accounts=[self], **constraints) def get_transactions(self, **constraints): return self.ledger.get_transactions(wallet=self.wallet, accounts=[self], **constraints) def get_transaction_count(self, **constraints): return self.ledger.get_transaction_count(wallet=self.wallet, accounts=[self], **constraints) async def fund(self, to_account, amount=None, everything=False, outputs=1, broadcast=False, **constraints): assert self.ledger == to_account.ledger, 'Can only transfer between accounts of the same ledger.' if everything: utxos = await self.get_utxos(**constraints) await self.ledger.reserve_outputs(utxos) tx = await Transaction.create( inputs=[Input.spend(txo) for txo in utxos], outputs=[], funding_accounts=[self], change_account=to_account ) elif amount > 0: to_address = await to_account.change.get_or_create_usable_address() to_hash160 = to_account.ledger.address_to_hash160(to_address) tx = await Transaction.create( inputs=[], outputs=[ Output.pay_pubkey_hash(amount//outputs, to_hash160) for _ in range(outputs) ], funding_accounts=[self], change_account=self ) else: raise ValueError('An amount is required.') if broadcast: await self.ledger.broadcast(tx) else: await self.ledger.release_tx(tx) return tx async def generate_channel_private_key(self): return await self.deterministic_channel_keys.generate_next_key() def add_channel_private_key(self, private_key: PrivateKey): self.channel_keys[private_key.address] = private_key.to_pem().decode() async def get_channel_private_key(self, public_key_bytes) -> PrivateKey: channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes) private_key_pem = self.channel_keys.get(channel_pubkey_hash) if private_key_pem: return PrivateKey.from_pem(self.ledger, private_key_pem) return self.deterministic_channel_keys.get_private_key_from_pubkey_hash(channel_pubkey_hash) async def maybe_migrate_certificates(self): if not self.channel_keys: return channel_keys = {} for private_key_pem in self.channel_keys.values(): if not isinstance(private_key_pem, str): continue if not private_key_pem.startswith("-----BEGIN"): continue private_key = PrivateKey.from_pem(self.ledger, private_key_pem) channel_keys[private_key.address] = private_key_pem if self.channel_keys != channel_keys: self.channel_keys = channel_keys self.wallet.save() async def save_max_gap(self): if issubclass(self.address_generator, HierarchicalDeterministic): gap = await self.get_max_gap() gap_changed = False new_receiving_gap = max(20, gap['max_receiving_gap'] + 1) if self.receiving.gap != new_receiving_gap: self.receiving.gap = new_receiving_gap gap_changed = True new_change_gap = max(6, gap['max_change_gap'] + 1) if self.change.gap != new_change_gap: self.change.gap = new_change_gap gap_changed = True if gap_changed: self.wallet.save() async def get_detailed_balance(self, confirmations=0, read_only=False): constraints = {} if confirmations > 0: height = self.ledger.headers.height - (confirmations-1) constraints.update({'height__lte': height, 'height__gt': 0}) return await self.ledger.db.get_detailed_balance( accounts=[self], read_only=read_only, **constraints ) def get_transaction_history(self, read_only=False, **constraints): return self.ledger.get_transaction_history( read_only=read_only, wallet=self.wallet, accounts=[self], **constraints ) def get_transaction_history_count(self, read_only=False, **constraints): return self.ledger.get_transaction_history_count( read_only=read_only, wallet=self.wallet, accounts=[self], **constraints ) def get_claims(self, **constraints): return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints) def get_claim_count(self, **constraints): return self.ledger.get_claim_count(wallet=self.wallet, accounts=[self], **constraints) def get_streams(self, **constraints): return self.ledger.get_streams(wallet=self.wallet, accounts=[self], **constraints) def get_stream_count(self, **constraints): return self.ledger.get_stream_count(wallet=self.wallet, accounts=[self], **constraints) def get_channels(self, **constraints): return self.ledger.get_channels(wallet=self.wallet, accounts=[self], **constraints) def get_channel_count(self, **constraints): return self.ledger.get_channel_count(wallet=self.wallet, accounts=[self], **constraints) def get_collections(self, **constraints): return self.ledger.get_collections(wallet=self.wallet, accounts=[self], **constraints) def get_collection_count(self, **constraints): return self.ledger.get_collection_count(wallet=self.wallet, accounts=[self], **constraints) def get_supports(self, **constraints): return self.ledger.get_supports(wallet=self.wallet, accounts=[self], **constraints) def get_support_count(self, **constraints): return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints) def get_support_summary(self): return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self]) async def release_all_outputs(self): await self.ledger.db.release_all_outputs(self) ================================================ FILE: lbry/wallet/bcd_data_stream.py ================================================ import struct from io import BytesIO class BCDataStream: def __init__(self, data=None): self.data = BytesIO(data) def reset(self): self.data.seek(0) def get_bytes(self): return self.data.getvalue() def read(self, size): return self.data.read(size) def write(self, data): self.data.write(data) def write_many(self, many): self.data.writelines(many) def read_string(self): return self.read(self.read_compact_size()) def write_string(self, s): self.write_compact_size(len(s)) self.write(s) def read_compact_size(self): size = self.read_uint8() if size < 253: return size if size == 253: return self.read_uint16() if size == 254: return self.read_uint32() if size == 255: return self.read_uint64() def write_compact_size(self, size): if size < 253: self.write_uint8(size) elif size <= 0xFFFF: self.write_uint8(253) self.write_uint16(size) elif size <= 0xFFFFFFFF: self.write_uint8(254) self.write_uint32(size) else: self.write_uint8(255) self.write_uint64(size) def read_boolean(self): return self.read_uint8() != 0 def write_boolean(self, val): return self.write_uint8(1 if val else 0) int8 = struct.Struct('b') uint8 = struct.Struct('B') int16 = struct.Struct('<h') uint16 = struct.Struct('<H') int32 = struct.Struct('<i') uint32 = struct.Struct('<I') int64 = struct.Struct('<q') uint64 = struct.Struct('<Q') def _read_struct(self, fmt): value = self.read(fmt.size) if value: return fmt.unpack(value)[0] def read_int8(self): return self._read_struct(self.int8) def read_uint8(self): return self._read_struct(self.uint8) def read_int16(self): return self._read_struct(self.int16) def read_uint16(self): return self._read_struct(self.uint16) def read_int32(self): return self._read_struct(self.int32) def read_uint32(self): return self._read_struct(self.uint32) def read_int64(self): return self._read_struct(self.int64) def read_uint64(self): return self._read_struct(self.uint64) def write_int8(self, val): self.write(self.int8.pack(val)) def write_uint8(self, val): self.write(self.uint8.pack(val)) def write_int16(self, val): self.write(self.int16.pack(val)) def write_uint16(self, val): self.write(self.uint16.pack(val)) def write_int32(self, val): self.write(self.int32.pack(val)) def write_uint32(self, val): self.write(self.uint32.pack(val)) def write_int64(self, val): self.write(self.int64.pack(val)) def write_uint64(self, val): self.write(self.uint64.pack(val)) ================================================ FILE: lbry/wallet/bip32.py ================================================ from asn1crypto.keys import PrivateKeyInfo, ECPrivateKey from coincurve import PublicKey as cPublicKey, PrivateKey as cPrivateKey from coincurve.utils import ( pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi ) from coincurve.ecdsa import CDATA_SIG_LENGTH from lbry.crypto.hash import hmac_sha512, hash160, double_sha256 from lbry.crypto.base58 import Base58 from .util import cachedproperty class KeyPath: RECEIVE = 0 CHANGE = 1 CHANNEL = 2 class DerivationError(Exception): """ Raised when an invalid derivation occurs. """ class _KeyBase: """ A BIP32 Key, public or private. """ def __init__(self, ledger, chain_code, n, depth, parent): if not isinstance(chain_code, (bytes, bytearray)): raise TypeError('chain code must be raw bytes') if len(chain_code) != 32: raise ValueError('invalid chain code') if not 0 <= n < 1 << 32: raise ValueError('invalid child number') if not 0 <= depth < 256: raise ValueError('invalid depth') if parent is not None: if not isinstance(parent, type(self)): raise TypeError('parent key has bad type') self.ledger = ledger self.chain_code = chain_code self.n = n self.depth = depth self.parent = parent def _hmac_sha512(self, msg): """ Use SHA-512 to provide an HMAC, returned as a pair of 32-byte objects. """ hmac = hmac_sha512(self.chain_code, msg) return hmac[:32], hmac[32:] def _extended_key(self, ver_bytes, raw_serkey): """ Return the 78-byte extended key given prefix version bytes and serialized key bytes. """ if not isinstance(ver_bytes, (bytes, bytearray)): raise TypeError('ver_bytes must be raw bytes') if len(ver_bytes) != 4: raise ValueError('ver_bytes must have length 4') if not isinstance(raw_serkey, (bytes, bytearray)): raise TypeError('raw_serkey must be raw bytes') if len(raw_serkey) != 33: raise ValueError('raw_serkey must have length 33') return ( ver_bytes + bytes((self.depth,)) + self.parent_fingerprint() + self.n.to_bytes(4, 'big') + self.chain_code + raw_serkey ) def identifier(self): raise NotImplementedError def extended_key(self): raise NotImplementedError def fingerprint(self): """ Return the key's fingerprint as 4 bytes. """ return self.identifier()[:4] def parent_fingerprint(self): """ Return the parent key's fingerprint as 4 bytes. """ return self.parent.fingerprint() if self.parent else bytes((0,)*4) def extended_key_string(self): """ Return an extended key as a base58 string. """ return Base58.encode_check(self.extended_key()) class PublicKey(_KeyBase): """ A BIP32 public key. """ def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None): super().__init__(ledger, chain_code, n, depth, parent) if isinstance(pubkey, cPublicKey): self.verifying_key = pubkey else: self.verifying_key = self._verifying_key_from_pubkey(pubkey) @classmethod def from_compressed(cls, public_key_bytes, ledger=None) -> 'PublicKey': return cls(ledger, public_key_bytes, bytes((0,)*32), 0, 0) @classmethod def _verifying_key_from_pubkey(cls, pubkey): """ Converts a 33-byte compressed pubkey into an coincurve.PublicKey object. """ if not isinstance(pubkey, (bytes, bytearray)): raise TypeError('pubkey must be raw bytes') if len(pubkey) != 33: raise ValueError('pubkey must be 33 bytes') if pubkey[0] not in (2, 3): raise ValueError('invalid pubkey prefix byte') return cPublicKey(pubkey) @cachedproperty def pubkey_bytes(self): """ Return the compressed public key as 33 bytes. """ return self.verifying_key.format(True) @cachedproperty def address(self): """ The public key as a P2PKH address. """ return self.ledger.public_key_to_address(self.pubkey_bytes) def ec_point(self): return self.verifying_key.point() def child(self, n: int) -> 'PublicKey': """ Return the derived child extended pubkey at index N. """ if not 0 <= n < (1 << 31): raise ValueError('invalid BIP32 public key child number') msg = self.pubkey_bytes + n.to_bytes(4, 'big') L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name derived_key = self.verifying_key.add(L_b) return PublicKey(self.ledger, derived_key, R_b, n, self.depth + 1, self) def identifier(self): """ Return the key's identifier as 20 bytes. """ return hash160(self.pubkey_bytes) def extended_key(self): """ Return a raw extended public key. """ return self._extended_key( self.ledger.extended_public_key_prefix, self.pubkey_bytes ) def verify(self, signature, digest) -> bool: """ Verify that a signature is valid for a 32 byte digest. """ if len(signature) != 64: raise ValueError('Signature must be 64 bytes long.') if len(digest) != 32: raise ValueError('Digest must be 32 bytes long.') key = self.verifying_key raw_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *') parsed = libsecp256k1.secp256k1_ecdsa_signature_parse_compact( key.context.ctx, raw_signature, signature ) assert parsed == 1 normalized_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *') libsecp256k1.secp256k1_ecdsa_signature_normalize( key.context.ctx, normalized_signature, raw_signature ) verified = libsecp256k1.secp256k1_ecdsa_verify( key.context.ctx, normalized_signature, digest, key.public_key ) return bool(verified) class PrivateKey(_KeyBase): """A BIP32 private key.""" HARDENED = 1 << 31 def __init__(self, ledger, privkey, chain_code, n, depth, parent=None): super().__init__(ledger, chain_code, n, depth, parent) if isinstance(privkey, cPrivateKey): self.signing_key = privkey else: self.signing_key = self._signing_key_from_privkey(privkey) @classmethod def _signing_key_from_privkey(cls, private_key): """ Converts a 32-byte private key into an coincurve.PrivateKey object. """ return cPrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key)) @classmethod def _private_key_secret_exponent(cls, private_key): """ Return the private key as a secret exponent if it is a valid private key. """ if not isinstance(private_key, (bytes, bytearray)): raise TypeError('private key must be raw bytes') if len(private_key) != 32: raise ValueError('private key must be 32 bytes') return int.from_bytes(private_key, 'big') @classmethod def from_seed(cls, ledger, seed) -> 'PrivateKey': # This hard-coded message string seems to be coin-independent... hmac = hmac_sha512(b'Bitcoin seed', seed) privkey, chain_code = hmac[:32], hmac[32:] return cls(ledger, privkey, chain_code, 0, 0) @classmethod def from_pem(cls, ledger, pem) -> 'PrivateKey': der = pem_to_der(pem.encode()) try: key_int = ECPrivateKey.load(der).native['private_key'] except ValueError: key_int = PrivateKeyInfo.load(der).native['private_key']['private_key'] private_key = cPrivateKey.from_int(key_int) return cls(ledger, private_key, bytes((0,)*32), 0, 0) @classmethod def from_bytes(cls, ledger, key_bytes) -> 'PrivateKey': return cls(ledger, cPrivateKey(key_bytes), bytes((0,)*32), 0, 0) @cachedproperty def private_key_bytes(self): """ Return the serialized private key (no leading zero byte). """ return self.signing_key.secret @cachedproperty def public_key(self) -> PublicKey: """ Return the corresponding extended public key. """ verifying_key = self.signing_key.public_key parent_pubkey = self.parent.public_key if self.parent else None return PublicKey( self.ledger, verifying_key, self.chain_code, self.n, self.depth, parent_pubkey ) def ec_point(self): return self.public_key.ec_point() def secret_exponent(self): """ Return the private key as a secret exponent. """ return self.signing_key.to_int() def wif(self): """ Return the private key encoded in Wallet Import Format. """ return self.ledger.private_key_to_wif(self.private_key_bytes) @property def address(self): """ The public key as a P2PKH address. """ return self.public_key.address def child(self, n) -> 'PrivateKey': """ Return the derived child extended private key at index N.""" if not 0 <= n < (1 << 32): raise ValueError('invalid BIP32 private key child number') if n >= self.HARDENED: serkey = b'\0' + self.private_key_bytes else: serkey = self.public_key.pubkey_bytes msg = serkey + n.to_bytes(4, 'big') L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name derived_key = self.signing_key.add(L_b) return PrivateKey(self.ledger, derived_key, R_b, n, self.depth + 1, self) def sign(self, data): """ Produce a signature for piece of data by double hashing it and signing the hash. """ return self.signing_key.sign(data, hasher=double_sha256) def sign_compact(self, digest): """ Produce a compact signature. """ key = self.signing_key signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *') signed = libsecp256k1.secp256k1_ecdsa_sign( key.context.ctx, signature, digest, key.secret, libsecp256k1_ffi.NULL, libsecp256k1_ffi.NULL ) if not signed: raise ValueError('The private key was invalid.') serialized = libsecp256k1_ffi.new('unsigned char[%d]' % CDATA_SIG_LENGTH) compacted = libsecp256k1.secp256k1_ecdsa_signature_serialize_compact( key.context.ctx, serialized, signature ) if compacted != 1: raise ValueError('The signature could not be compacted.') return bytes(libsecp256k1_ffi.buffer(serialized, CDATA_SIG_LENGTH)) def identifier(self): """Return the key's identifier as 20 bytes.""" return self.public_key.identifier() def extended_key(self): """Return a raw extended private key.""" return self._extended_key( self.ledger.extended_private_key_prefix, b'\0' + self.private_key_bytes ) def to_pem(self): return self.signing_key.to_pem() def _from_extended_key(ledger, ekey): """Return a PublicKey or PrivateKey from an extended key raw bytes.""" if not isinstance(ekey, (bytes, bytearray)): raise TypeError('extended key must be raw bytes') if len(ekey) != 78: raise ValueError('extended key must have length 78') depth = ekey[4] n = int.from_bytes(ekey[9:13], 'big') chain_code = ekey[13:45] if ekey[:4] == ledger.extended_public_key_prefix: pubkey = ekey[45:] key = PublicKey(ledger, pubkey, chain_code, n, depth) elif ekey[:4] == ledger.extended_private_key_prefix: if ekey[45] != 0: raise ValueError('invalid extended private key prefix byte') privkey = ekey[46:] key = PrivateKey(ledger, privkey, chain_code, n, depth) else: raise ValueError('version bytes unrecognised') return key def from_extended_key_string(ledger, ekey_str): """Given an extended key string, such as xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd 3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL return a PublicKey or PrivateKey. """ return _from_extended_key(ledger, Base58.decode_check(ekey_str)) ================================================ FILE: lbry/wallet/checkpoints.py ================================================ HASHES = { 0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2', 1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9', 2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2', 3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e', 4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071', 5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff', 6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8', 7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e', 8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe', 9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0', 10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4', 11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145', 12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909', 13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf', 14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17', 15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103', 16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9', 17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50', 18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d', 19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3', 20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6', 21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed', 22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5', 23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026', 24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe', 25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585', 26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b', 27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5', 28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f', 29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66', 30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a', 31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f', 32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde', 33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f', 34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309', 35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374', 36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da', 37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832', 38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa', 39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f', 40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49', 41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363', 42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4', 43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf', 44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd', 45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b', 46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c', 47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a', 48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c', 49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f', 50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a', 51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482', 52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011', 53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3', 54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849', 55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55', 56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926', 57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2', 58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b', 59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7', 60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754', 61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76', 62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6', 63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9', 64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3', 65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb', 66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b', 67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41', 68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73', 69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546', 70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47', 71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000', 72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2', 73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95', 74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20', 75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58', 76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0', 77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573', 78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0', 79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f', 80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8', 81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab', 82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1', 83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3', 84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b', 85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8', 86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158', 87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868', 88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561', 89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57', 90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7', 91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7', 92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398', 93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02', 94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307', 95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6', 96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e', 97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394', 98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86', 99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7', 100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce', 101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a', 102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482', 103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922', 104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad', 105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9', 106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a', 107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144', 108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704', 109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca', 110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456', 111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18', 112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b', 113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972', 114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744', 115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5', 116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067', 117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498', 118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1', 119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4', 120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792', 121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc', 122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef', 123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00', 124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3', 125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a', 126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf', 127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0', 128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d', 129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e', 130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0', 131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072', 132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8', 133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0', 134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890', 135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4', 136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663', 137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864', 138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e', 139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b', 140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041', 141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92', 142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea', 143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097', 144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254', 145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009', 146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176', 147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863', 148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7', 149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4', 150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c', 151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7', 152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0', 153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3', 154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20', 155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e', 156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278', 157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19', 158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d', 159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d', 160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51', 161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f', 162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582', 163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db', 164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb', 165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616', 166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25', 167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557', 168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff', 169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998', 170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb', 171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103', 172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7', 173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d', 174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152', 175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550', 176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957', 177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5', 178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb', 179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86', 180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a', 181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff', 182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0', 183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977', 184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837', 185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a', 186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4', 187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a', 188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0', 189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6', 190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b', 191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d', 192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c', 193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407', 194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254', 195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af', 196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8', 197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401', 198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530', 199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70', 200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7', 201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c', 202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1', 203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142', 204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f', 205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256', 206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0', 207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64', 208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0', 209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1', 210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee', 211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db', 212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637', 213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb', 214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7', 215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a', 216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e', 217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d', 218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392', 219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98', 220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163', 221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6', 222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008', 223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d', 224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f', 225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0', 226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28', 227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c', 228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d', 229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149', 230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6', 231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f', 232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f', 233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b', 234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c', 235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a', 236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598', 237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d', 238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d', 239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f', 240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c', 241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a', 242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd', 243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f', 244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393', 245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206', 246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7', 247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128', 248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34', 249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1', 250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000', 251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4', 252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756', 253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92', 254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454', 255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51', 256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c', 257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2', 258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64', 259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13', 260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2', 261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5', 262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c', 263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a', 264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8', 265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826', 266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d', 267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507', 268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c', 269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1', 270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4', 271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f', 272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a', 273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b', 274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564', 275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a', 276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993', 277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca', 278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240', 279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2', 280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f', 281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3', 282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e', 283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42', 284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1', 285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37', 286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3', 287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f', 288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a', 289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9', 290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5', 291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9', 292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4', 293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767', 294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604', 295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2', 296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756', 297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8', 298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c', 299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07', 300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7', 301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63', 302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8', 303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156', 304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e', 305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a', 306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888', 307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb', 308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1', 309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f', 310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812', 311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a', 312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315', 313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e', 314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45', 315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743', 316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271', 317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4', 318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481', 319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d', 320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b', 321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19', 322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2', 323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8', 324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540', 325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75', 326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59', 327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca', 328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e', 329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461', 330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5', 331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d', 332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2', 333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6', 334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0', 335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac', 336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b', 337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc', 338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574', 339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827', 340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad', 341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d', 342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf', 343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a', 344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3', 345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770', 346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c', 347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b', 348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86', 349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670', 350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9', 351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e', 352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54', 353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232', 354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8', 355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516', 356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e', 357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861', 358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc', 359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd', 360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301', 361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29', 362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1', 363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867', 364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0', 365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507', 366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba', 367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa', 368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136', 369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e', 370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43', 371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110', 372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2', 373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4', 374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260', 375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f', 376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391', 377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa', 378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d', 379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442', 380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32', 381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454', 382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7', 383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb', 384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2', 385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2', 386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837', 387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703', 388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c', 389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4', 390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67', 391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c', 392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3', 393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f', 394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60', 395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e', 396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9', 397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d', 398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93', 399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea', 400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752', 401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807', 402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31', 403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2', 404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e', 405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5', 406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544', 407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda', 408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8', 409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480', 410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6', 411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924', 412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388', 413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a', 414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0', 415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23', 416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6', 417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f', 418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8', 419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac', 420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba', 421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229', 422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df', 423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd', 424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116', 425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54', 426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5', 427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0', 428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13', 429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51', 430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5', 431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8', 432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7', 433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f', 434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365', 435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168', 436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10', 437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d', 438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec', 439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9', 440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94', 441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a', 442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0', 443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad', 444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8', 445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17', 446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40', 447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb', 448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce', 449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65', 450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356', 451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71', 452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5', 453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e', 454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb', 455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5', 456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4', 457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c', 458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655', 459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8', 460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a', 461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33', 462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315', 463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d', 464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f', 465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778', 466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f', 467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a', 468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7', 469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af', 470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc', 471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4', 472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27', 473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6', 474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9', 475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff', 476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6', 477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368', 478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9', 479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f', 480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a', 481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886', 482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9', 483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8', 484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80', 485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2', 486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43', 487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b', 488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda', 489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08', 490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df', 491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b', 492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6', 493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0', 494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b', 495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54', 496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e', 497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51', 498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0', 499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538', 500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145', 501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105', 502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492', 503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500', 504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6', 505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33', 506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558', 507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9', 508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902', 509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0', 510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a', 511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de', 512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09', 513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff', 514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5', 515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57', 516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0', 517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3', 518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4', 519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7', 520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97', 521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7', 522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f', 523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34', 524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc', 525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c', 526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a', 527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01', 528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c', 529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75', 530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7', 531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5', 532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1', 533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52', 534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd', 535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc', 536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010', 537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3', 538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c', 539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb', 540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f', 541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125', 542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980', 543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00', 544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe', 545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e', 546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23', 547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f', 548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6', 549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e', 550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29', 551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94', 552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc', 553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2', 554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea', 555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e', 556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c', 557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6', 558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be', 559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce', 560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff', 561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422', 562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb', 563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2', 564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5', 565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235', 566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d', 567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa', 568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e', 569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc', 570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9', 571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260', 572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2', 573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef', 574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6', 575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84', 576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044', 577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794', 578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0', 579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54', 580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a', 581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40', 582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56', 583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35', 584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391', 585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce', 586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90', 587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff', 588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992', 589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd', 590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39', 591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc', 592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d', 593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526', 594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb', 595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5', 596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de', 597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41', 598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b', 599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e', 600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508', 601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4', 602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567', 603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368', 604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988', 605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33', 606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4', 607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299', 608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776', 609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596', 610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a', 611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14', 612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2', 613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21', 614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c', 615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725', 616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a', 617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544', 618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c', 619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332', 620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446', 621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec', 622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e', 623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51', 624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167', 625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3', 626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b', 627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9', 628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742', 629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568', 630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7', 631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4', 632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3', 633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820', 634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0', 635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5', 636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6', 637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c', 638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81', 639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c', 640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a', 641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655', 642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f', 643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf', 644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d', 645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949', 646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833', 647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353', 648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6', 649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d', 650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9', 651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11', 652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118', 653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470', 654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b', 655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593', 656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd', 657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825', 658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451', 659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6', 660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb', 661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2', 662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21', 663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1', 664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8', 665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369', 666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e', 667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e', 668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe', 669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692', 670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33', 671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6', 672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc', 673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af', 674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6', 675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df', 676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8', 677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8', 678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976', 679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8', 680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a', 681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9', 682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4', 683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5', 684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143', 685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd', 686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec', 687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a', 688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3', 689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3', 690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19', 691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084', 692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2', 693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1', 694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8', 695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d', 696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140', 697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0', 698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f', 699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234', 700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5', 701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7', 702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3', 703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7', 704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c', 705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d', 706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62', 707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746', 708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514', 709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04', 710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084', 711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6', 712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860', 713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf', 714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799', 715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387', 716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073', 717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77', 718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a', 719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a', 720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4', 721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902', 722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a', 723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928', 724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49', 725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693', 726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0', 727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7', 728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca', 729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491', 730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669', 731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97', 732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c', 733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630', 734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f', 735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355', 736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687', 737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a', 738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb', 739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04', 740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37', 741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458', 742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646', 743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181', 744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013', 745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff', 746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54', 747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633', 748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde', 749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c', 750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26', 751000: 'cedaf12415dac1314942e58ced80830b92fbfabc41f42a0b0f054f0672ef9822', 752000: '293606bcd9fbbee5584724301b2cf86bb69204820023e1fb46c238ddfbc660ab', 753000: 'f4d43cbb38b7d97919dedc0f5a6dc8007896c4f443b76f3e5693e25bc46760cf', 754000: 'fcaad22fd815311280fe451086516375d1d9d92b2990c7c351407df5aa19011e', 755000: 'b9276f10d1844cb5b0308766c8db960490ac34a73c4653d0a91202789a6ccb9b', 756000: '2fe5581f1110c1c8dcea46cad647551bd6bd640cb37738d863e189bd8f368347', 757000: 'b9d915f366f0b010429a52245b0fb02774157eb9fd8f66bce32dcd3acc71c2a1', 758000: '62d1854fc15db56b5d0e05ceeb54c1297966bf9dc7f7a0a14b42c059fc485d1b', 759000: 'f4ca9f69d16d092f4a0ea5102e6343b21204c4ea9cd9b22cddd77dbb5d68ade3', 760000: 'df3bb86641330d8cc7f55a2fd0da28251219e95babe960a308b18e08a7d88fc8', 761000: 'a93029475de4bc7569b6ae802d658cd91c84cc253772712a279f140a6c3b91b1', 762000: '307e289dc6ec8bcd62ca8831e4159d5edd780f2fae55ba55dd446225450f46f8', 763000: '293f73514abca24f374473bd0394179812952a04ea13dc60ef5ada5331fa274f', 764000: 'dd8b082db9281e3d9bacf15d6b352fda186d2d2923c7731844d0d4764dd71db8', 765000: '201239e562d2571bf47347b3522fff89632aecea3b2d8cef05151f88b2b0bcdb', 766000: '4a55a538b51b5650979e64521998cd5c5ad055ba9f3ac0e3e2a28febc6cc2798', 767000: '3916666f2adbb05ea98ec1961f9546b9afa0f6910ec95e42ce37267f2ae4f79c', 768000: 'dc0ad881eedcb5fd4954238f462080d6e7636b058d481698ed1c077e0ce2207e', 769000: 'eaf10a1e1ec6e129289b8479a05df03e0808f1f0946f1995de6524e9ebe7a461', 770000: '7200c64f22e32de7f999583361c933680fc9a2ffcb9a5ab73d3076fd49ec7537', 771000: 'd883111a2eeacff80ce31df35ab6c943805b9e48877b413fccf371e5dbfa7fb2', 772000: '3977d3c60edb9c80c97bb2b759b1659cbb650ad2d3a6f61d2caec83f1b2ae84c', 773000: '9c7175fb8646a1a82383b4c534fd01bcf92d65c43d87ae854d51a784b04dc77e', 774000: 'e0e92485f86e5fffa87b3497424e43b02a37710517d9d3f272392e8cdc56e5e9', 775000: '6395229113d3aa2105afbaeb8b59621a536fc61fe272314b2fc3bdda98dd66cc', 776000: 'b4b00207328b5f032bd4f0b634f91323ff520ada8c8bfec241b23c8e4bfd5a4e', 777000: '14cdc6f5f7b4bd5bad745dfe6fcd114e9194026412a2e1b3f345be2eef433d16', 778000: 'd3cd7b68be504c32117b670d38d59d44b02dcf3d65811efc2ca5531d902623cc', 779000: 'afcd220e4040cb5f92d4b38fc204e59822df2218f767f2c4b33597b238a35f77', 780000: '78252a9cfc289a70192ed8dd3dddeb1b9a4f9b8eff9a5d0ac259b3254472cf68', 781000: '02ebc3f17d947481a311b4771c254f1e002b6a9198d4a5258ce6c13165aadddc', 782000: '8dd9f1f372ee6d688a0bcdc3b342c77804ba5a646a218be4bc2aa02d846206c0', 783000: 'e46b0d02ec2ef488fae455665e107520e1bd2b4f35ca52af7ad8addd2f72fa73', 784000: '9ee8a8de94231e3ae3a610b82fdbca48dc14d9b80791d20af6c365a31822df6f', 785000: '21e1cc12def8173a50158b2833bd91a62140c61646f5e08aecaee3e6da20735e', 786000: 'b3e659f84d73de42888cc0f2b69bae71dd5fa6756a437a4b21958b182faa316e', 787000: 'a9be7ba00ea6a9ea6bd03d8412ec014ca7e8cda6bdc33382f165e702811b8836', 788000: 'a4c14729f8a68c03f5a0ccd890ac6a92b39c143f1f752fe81ad051eb52d8dce0', 789000: '5cf66d224e5645097efc9c3c0392b51c8ca8ea1295151921a7912a2f04ee1274', 790000: '676769ade71c33bc102bce416e66eb2c6794b03d7b8f5a590c87c380da463775', 791000: '0228e074451797bf6bfbc941bcafcbadc972d32e4e1e0c5da015513f65714217', 792000: '0fa3d00a1f19c5ac060e10a410cf7cea18eac5f89018d79ce51ac3fc66bbb365', 793000: '5f68d0868b424e32f5ce3d8e7d9f18979da7b831b8ef4e3974d62fb20ff53a97', 794000: '34508c56423739c00a837801b654b07decb274d02b383eff396d23c4d64bc0e9', 795000: '7f70910c855d1fd88cd7f9be8a3b94314ee408a31a2da6301404bf8deb07c12c', 796000: 'b74ab8813b1d2a0967fea0e66597572e5f0b5a285e21f5150fcc9d5f757de130', 797000: 'bba27b1491d907ab1baa456cb651dc5b071231b1b6ad27b62d351ca12c25dbfd', 798000: 'e75dcb15b2fc91f02e75e600dde9f6f46c09672533bc82a5d6916c4a2cd8613a', 799000: 'adf62c826a3e0b33af439a7881918ae4ce19c5fb2ca37d21243415f7d716aa65', 800000: 'd8f0ca13a8c8a19c254a3a6ba15150a34711dca96f2d877162cc44aa2acfb268', 801000: '2a8c7104c4040a2bc31913ae25e9361df5bac9477368c708f86c1ca640480887', 802000: '1f3b09d3561c4a8a056b263289bd492dc6c0d604c3fa195935e735d1c0ddc40e', 803000: '037769628c40a701fdb4b16d79084b8fbb319fde79770a7ac842f3cdc813099e', 804000: 'a0c6a089e5fa1e3589ca282085fe7201a5705776d81b257ffd252b2947fa6428', 805000: 'b2ac99bfc4a488e7b7624b31ee061991a6dd0881bb005cd13f3dd2e66a08fe19', 806000: 'ffe63cb999a278280b80a667d2dcb60c40e43a53f733914d8bec808b694ebf83', 807000: 'eddb09fc6c4869a59b520d0befb1fb6ac952333f3cc5de086539c85ea8558778', 808000: '0f4fb3f9172e52897ea992d9f3a2024126c4d2e63e9888739f11fb1f5e4c1f46', 809000: '9641dd720d23ced2f1cb6e5cf46ac4e547afb9f56263c4cf58e3b19d407cf401', 810000: 'de6dc953acd7e5ef213b3aaf1c4a9ee1d5b756bfce5525ee105214647e243a85', 811000: 'c52c83712ca12b24b2db1b4a575e7f352b1d560cbf702e121a03bdca9e8be23d', 812000: '83143734bb965318a53a38a7e403dcdb3e3fadedb01ab12c370417fc2a0655c0', 813000: 'e480deff10c5a84fc957e3aed936690e24b74dd08fa8858a8a953c2f7383b914', 814000: '810d33afcee07b9abe16c6cdc3a041038daa131c476b0daf48a080007f08b490', 815000: 'b4aeb9e16fddd27844b2d56bc2b221134039bb5642c9e9ba88372afbdeac3972', 816000: '86e73b67aae3d248011b8f66ed414cb8a9ba4b2a3cf7e32773cfbff055d719b7', 817000: '3ebb8b83752b48242016cb682f0f6bd14e15371bf1163a5933193eaa0edeb351', 818000: '4d925e17f642f220bbf317d3d5355d2f41fbce325f190f8c3b32dc0b337d24d6', 819000: 'b9cc126d620f6b99d90a00d35957b0e428aaaa7c986bc9e816a60e4334572961', 820000: '9c2f8c142bed1f94dca29276f7c83958be8cfe11773bb9b56c808fbcf7d3b1f8', 821000: 'e5509eb98895cfa12a8da5d54c1df3f52472ffcbdf707adbf84a4a9c5d356203', 822000: '764aada4802ebfe4ef935ab50af06a4f83aa556c49fdde3d9e12e1abd230c16b', 823000: '1dbd745c2e96a365d865f990d109137d32d42977f503af55d8c00b109d31d3c3', 824000: '954304a0b0c8f549c3bffd5ff46b5b8f05b0f0fde2a36f24fd5af9d774fb3079', 825000: '17808b14f2056c1a5d46cb7617e9de9be6a1a6084edbc1bdb778586467a72297', 826000: '3ca1167d4cac8b187829b23001b438617c43704b42462c4eb001b0d434cb9651', 827000: '246d1607245e4a202f420393ac2e30e9cbf5eb5570dc997073b897f6d8643023', 828000: '1764730a8dc3e89d02d168ff6bb54e8c903820b74711af6ff27bd0c8545577e7', 829000: 'd9f3ab0cd823c6305bd8b95a96188bb4f2ca90b4d66c5d12293e8b6192bac0f2', 830000: 'd4ff51f0092b04aedf8d39937680d8e8309b1be21d36e7833ed36f8e30aad6ea', 831000: '3e92e76721b962396dce52993fa7606552f0907b38f7b2bd7b21ada98c145f47', 832000: 'df12fcdb4cbe53ba627ace6de898298de175f8671d3d90170732d110fcdc34b8', 833000: '25167ff38ae4a5964b618cabe0a12d4de62ac7a4c47448cdb4499e09e108d5b9', 834000: 'd31f5309ea179a1e386e835fc372e47dcda6871a3a239abfba50c4f368994f13', 835000: 'aff7e8dd3e55ea807fcbe284014075f420b3a23f1b0eb47bacdc1c91d2899813', 836000: '3b5ac6d64c470739bb17d1544a285affb40f2d33e92687e5ba7c5ac602e0d72a', 837000: 'd5619cbfe4f27c55f2bf9351b4891636cf64fef88212a5eeeae7bd3de47fe0bd', 838000: '1f9102a49c6ac470cb5d0050e5300b1443840d6d65719b835e3bea484aafb2ec', 839000: '3f63e391f0fbc5787fbe4ace3bada3816261294ea1c6ee435001801023682f90', 840000: '777894fd12bd0d6dee7bcde2995c68e55e7094e3122da38571e4b6c4304b75e0', 841000: 'ceb0c598c788e25e43e25aa4beff5c7377035824844cf1675eaea537074df028', 842000: '8661cf2065dc713d2ba043f0b81f0effcc940eeb3e91906a21ff22c210561dcd', 843000: '0dc2766f90415009d0c86bedffee6ebcf58042eb08262c0c67c4e9ed86b2aec8', 844000: '26d072da864cab268a12794977b04ec44fb69ef3978e2342e82225974dac54dd', 845000: '95e93bb60be8d5f07a1f4d26290c914957a82fc9d26ae8a3f20082eda27406ff', 846000: 'f1bdc39af7705e58ab8b6c31dc70dce1e115db1cfd8cc9b037949dfbec82a59a', 847000: 'f5f10f06396ecf2765d8a081141d489737c1d8d57c281f28f57c4cb2f90db883', 848000: '331b8ef08605bae8d749893af9ed54f0df4f07a5a002108a2a0aea82d0360979', 849000: '75b5f6233ab9a1bbc3c8b2893e5b22a0aa98e7ea635261255dc3c281f67d2260', 850000: '5d7e6fe83e0ea1910a54a00090704737671d6f44df4228e21440ad1fc15e595f', 851000: '7822db25d3ff0f6695ee38bad91edf317b5c6611673d28f1d22053110bb558be', 852000: '2f0effad83a3561fc1a2806a562786a641d9ddb18d16bb9308006e7d324a21e9', 853000: 'f603b2eaff11d5296377d990651317d40a1b2599ad2c5250eab131090f4b9458', 854000: '34d59b26a50f18a9f250736d0f2e69d28b7e196fbef9b8a26c6b0b75c16aa194', 855000: '76dd1ffff3946c0878969886fcf177ce5ab5560df19ddf006f9bcb02ae3e4e4f', 856000: '74ff0b6f64e9dd5802fec2aac1d3ae194d28b9264114adaf0a882b46c8c918fe', 857000: '7b5badfa2e4f40aa597a504d7ebe83c3705a2c6169a8c168ce293db223bc2d32', 858000: '2bb0767a0f72b20d45ecfc3e34517dbda16d85758e040cf0e147f4cbd0cc57ac', 859000: '3d741b9c365a91ed76f85824b94d19ec19b608d232660840ba59c7aa4b2cb67f', 860000: 'd481a5a117878c0e3acd1f5844e150fb30e617577947d9846b1d214d703b71b0', 861000: '54033424e488a3f1ad6946d4a6d9acb48465d6b1dbe8e1c2504a54cc84d7cad4', 862000: '464bc3820a8cc8844dc9e26c388009e9982c656d46ef4b4fd0a2cb0e4eea0aaa', 863000: 'd1aa94be2174f66780c4f226b9da3f6712b0f37af8dec33360bea83ca261b342', 864000: '8c16008f11de5bc395d88cd802514ff647450f1bc136724b9aaf2ccce10a494f', 865000: '3dae86012e97a201e2e1a47c899001ac00f78dc108026ed7c4194858c6c6dd5a', 866000: 'afe5b0ccab995e1a1fa25fbc24c1d4b1a92c43042d03395f8743dcd806e72fd8', 867000: 'c83716ac171aa9ab0d414833db340fa30e82bfda6cc616d3038529caab9b5600', 868000: '8c409fe03cd35ef2d8e366818788b40eaeb4c8f6ae91450d75f4a66ca5f69cad', 869000: '1d47909ceba790b8e1ce2e9902ee2775ea99e58efdb95668f9803a8ccf95f286', 870000: '9adf5da1476388f053aa42de636da169d1cf1c9652cdf7cd9ad4fb18a0eb3388', 871000: '8ad57fb1e74bcba0b5614fbac003be2bb32275dd85b38f2d28a0585005a99cfc', 872000: '84a32e92012a356106e9657da8dab1a5491ea588fc29d411c69b20680c666420', 873000: 'adf5921bbbfaa43929f67e6a070975313b77b456e262c700a27be611fceb17ae', 874000: '09eaa7c4b18c79a46a2895190333f72336826223d5c986849a06f5153f49f2a5', 875000: '235d7e4f31966507312149ea4c5e294aa84c695cf840117f0ef5963be7a0bda1', 876000: '9aa9cb806ccbec0475ac330b496c5b2edeba38ba3f1e13ddd54a01457634a288', 877000: 'c1e7f9b2b20bb1c4c0deadbc786d31fdf36f262325342aa23d1a66e2846b22bc', 878000: 'ee0d2b20ac28ce23ab38698a57c6beff14f12b7af9d027c05cc92f652695f46b', 879000: '0eb0810f4b81d1845b0a88f05449408df2e45715c9210a656f45278c5fdf7956', 880000: 'e7d613027e3b4ca38d09bbef07998b57db237c6d67f1e8ea50024d2e0d9a1a72', 881000: '21af4d355d8756b8bf0369b2d79b5c824148ae069026ba5c14f9dd6b7555e1db', 882000: 'bc26f028e547ec44fc3864925bd1493211773b5cb9a9583ba4c1909b89fe0d33', 883000: '170a624f4be04cd2fd435cfb6ba1f31b9ef5d7b084a25dfa23cd118c2752029e', 884000: '46cccb7a12b4d01d07c211b7b8db41321cd73f30069df27bcdb3bb600c0272b0', 885000: '7c27f79d5a99baf0f81f2b09eb5c1bf905976a0f872e02bd4ca9e82f0ed50cb0', 886000: '256e3e00cecc72dbbfef5cea627ecf1d43b56edd5fd1642a2bc4e97c17056f34', 887000: '658ebac7dfa62bc7a22b1a9ba4e5b425a866f7550a6b40fd07de47119fd1f7e8', 888000: '497a9d02868605b9ff6e7f15948a83a7e07606829107e63c2e091c90c7a7b4d4', 889000: '561daaa7ebc87e586d37a96ecfbc72484d7eb602824f38f484ed333e78208e9e', 890000: 'ab5a8cb625b28343f8fac858eab6576c856dab88bde8cda02b80b3edfd307d71', 891000: '2e81d9fc885ddc09222b298ac9efbb73638a5721802b9256de6505ecf122dbaa', 892000: '73be08881b8832e986c0bb9a06c70fff346edb2afaf69630e47e4a4a90c5fece', 893000: 'd39079dcaa4d8af1c26f0edf7e16df43cd857a31e0aa4c4123226793f1ab497f', 894000: '0a3b677d72c590d4b1ff7a9b4098d6b52d0dc10d64c30c2766d18e6eb02872cd', 895000: 'a3bbba831f48c5b68e494ee63015b487782c64c5c24bb29436283360c28fd1e0', 896000: '20af178a192ca43975ab6c838fe97ca42ba6c682682eddbc6481efd153ecb0a2', 897000: '8d0ee14b9fdb853a09ab2951d26b8f7cb8bc8038b09513bd330ee4b0bdcc4780', 898000: 'c97fbb70f804408b131a98f9fb4c04cdf2df1655d3e8ff2e0d58ed8537349f4e', 899000: 'eba2be80478e8dec2d66ca40b853580c5dad040351c64c177e3d8c25aff6c1b6', 900000: 'c4dc344a993558418b93b3f60aaef0030e2a4116086577fbf1e2f544bdbddae1', 901000: '36d84229afa63045875fc8fea0c55de8eb90694b3a37cceb825c87abf1fea998', 902000: '8ca4890ecfc5e3f9d767e4fcdf318a1e3e3597675bbcfe534d64e76bc4e8fbf4', 903000: '8b9f6a7514033c57668ca94fb3758cc6d1ef37ac982c2ff5a9f0f206fcd8d0a8', 904000: 'e9ae813991f35ca89af2fe1f1b6adf9e93c6b1dd6a74f003ebbe699a30b252ea', 905000: 'd426489d01d4f4c829f2eb68a67721d2c0e1c71e8c33ef9253593447e8603462', 906000: '63000bbed97451e68d64485c02c1c3d90b4156237dac315f4e012ffb538e375b', 907000: '96759653a4e514541effa7ef86d9f22a272ddde7b069149d17e9d9203a1edafb', 908000: 'eec6477d2f3b71bde76dc2380d6e06aa8aa306ca56ba1dd15a31c22ae0db501b', 909000: 'd5c2984cf130335aa29296ba5b17672d00360fe0ec73977326180014908c0b55', 910000: '7b99cb1c94144f606937903e173bd9ef63bfffd3db8110693fa4c2caa0abc21f', 911000: '95eed0d9dd9869ac6f83fa67863e77f24df69bcb90fef70918f30b2400e24ea8', 912000: '34c3c8780c54ecced50f0a6b394309d09ee6ce37cd98794699c63771d1d91144', 913000: '536052ddcd445702160288ef3f669ce56868c085315556c9f5ca081ef0c0b9e1', 914000: '1bcd1fe9632f93a0a1fe7d8a1891a4fc6ef1be40ccf887524a9095ed7aa9fa44', 915000: '139bad9fa12ec72a37b62ad8511300ebfda89330fa5d5a83861f864b6adeae67', 916000: '81d15282214ff83e2a034212eb58abeafcb5664d3734bff13b22b4c093b20fea', 917000: 'f31081031cebe450e4450ef397d91790fc0068e98e6746cd0aab86d17e4448f5', 918000: '4af8eb28616ef0e859b5471650c7f8e910cd692a6b4ff3a7171a709db2f18e4e', 919000: '78a197b5f9733e9e4dc9820e1c79bd335beb19f6b87056e48e8e21fbe27d83d6', 920000: '33d20f86d1367f07d6731e1e2cc9305252b281b1b092403133924cc1052f501d', 921000: '6926f1e31e7fe9b8f7a81efa73d5635f8f28c1db1708e4d57f6e7ead951a4beb', 922000: '811e2335798eb54696a4b11ca3a44b9d79486262119383d542491afa9ae80204', 923000: '8f47ac365bc380885db809f2818ffc7dd2076aaa0f9bf6c180df1b4358dc842e', 924000: '535e79802c10630c17fb8fddec3ba2bf85eedbc0c076f3575f8189fe887ba993', 925000: 'ca43bd24d17d75d55e72e45549384b395c62e1daf0d3f58f296e18168b918fbf', 926000: '9a03be89e0725877d42296e6c995d9c48bb5f4bbd971f5a9add191af2d1c144b', 927000: 'a14e0ef6bd1bc221dbba99031c16ddbbd76394186677c29bdf07b89fa2a6efac', 928000: 'b16931bd7392e9db26be975b072024210fb5fe6ee22fc0809d51980aa8068a98', 929000: '4da56a2e66fcd98a70039d9061ea5eb0fb6d9460b437d2191e47441182419a04', 930000: '87e820e2237a54c4ea100bdd0145598f05add92185cd3d0929aa2d5099f4d5e0', 931000: '515b22c91172157c443a47cf213014aff144181a77e276e291535ab3762bb1ae', 932000: 'e130c6a9eb416f96256d1f90256a148957daa32f56af228d2d9ce6ff27ce2011', 933000: '30c992ec7a9a320fb4db260373121efc7b5e7fc744f4b31defbe6a7608e0749e', 934000: 'ec490fa0de6b1d78a4121a5044f501bbb3bd9e448c18121cea87eb8e3cadba41', 935000: '603e4ae6a6d936c79b3f1c9f9e88305930953b9b390dac442976a6e8395fc520', 936000: '2b756fe2de4328e598ed511b8828e5c2c6b5cdda1b5e7c1c26f8e0424c81afa9', 937000: '1ae0f15f14a0d4819e34a6c18de9428a9e43e17d75383bffa9ffb18358e93b63', 938000: 'cbd7001825ec87b8c6917d6e9e7dc5c8d7767788b6ffd61a61d0c612dbe5de66', 939000: 'd770d0395aa79076044783fb37a1bb173cb95c93ff1ba82c34a72c4d8e425a03', 940000: '3341d0a0349d091d88d233cd6ea6e0ad553d52039b4d47af51b8a8e7573a7916', 941000: '16123b8758e99344ebe6670cd95826881b274c31d4da2a051052955a32bade3a', 942000: 'ac7430961e77f902918fe79a52cbf6b523e3f2804ec83d0b17908e131ea9ea68', 943000: '2ad08a6877e4687dcb7a623adeddc88403e8082efd6de28328b351282dc141e2', 944000: '81382e8c1f47fa7c03fa1726f9b09ed1cd38140fe50683896eaa1b403d7e5fe3', 945000: '152bfbb166da04dab16030af28ae65b3275819eed1d0bbfc11eba65616ebefd6', 946000: '25b3da0962f87a0d3e4aec8b16483efbcab9514893a42fd31f4cb544ddc45a1f', 947000: '2cb738ba342436628ff292797e3d36c4752d71bdc1af87fe758d469d06e36e0e', 948000: 'b3683e18570fcc8b986720514539181ec43fb5dbc20fe314c56ab6bd31ab766a', 949000: '94ced5bfba55ccffc909bf098d537e047d8d4cbb79f5e2a74146073f39804865', 950000: 'b11543cd2aedae27f6ddc3d2b431c897fdcfe59ed3c926b0777bc1e99de4d12a', 951000: '21508881a7f80fcd0b9b27bbcfba634b39c6525f5313968c4605cd55b4fec446', 952000: 'f9b3ed919c9ca20cd2927d899ee7a86c93c2dd919dafb6fdb792f2d9f1895cb0', 953000: 'cf578d8e80eec4102dc1b5321f10b36020b3b32f4b5d4664c90c412ca2ef6b42', 954000: 'ed17c919ae5c4be835966b47f667d6082c75917b95584b2d2aff0e32f5c8aa98', 955000: '948ea467fa01a20122e2146669214fdd3bb025038554609f7299ece5bca63e39', 956000: 'b50ff4c02957ed8764215d25f206f6f1fe6d0eb712a378b937ff952dd479afd2', 957000: '169922a3e51517ba6104a883d29aac03a9d20b4d448bd2773137b0d790e3db6b', 958000: '92258ac2e8b53167dc30436d93f385d432bd549711ab9790ba4e8263c5c54382', 959000: '7ca824697459eb302bcd7fba9d255fb269555abe7cf9d2dd5e54e196d751e682', 960000: '89f9ec925d23698076d84f9e852ab04fc956ac4465827303de0c3bb0b685eb32', 961000: '41cf75cd71bc12b93674c416e8b01b7410eb9e09eb8727ad93ff0b833c9966c9', 962000: '7db1f1dbff3e389713067879bfedf9513ec74bb1e128b13fc2fe23ad55fd0306', 963000: 'a35e71c611b2227adeac824d151d2f09bdbecd5765a4e62c6e74a3e4290abc66', 964000: 'dc1811130e249d2208d6f85838512b4e5482efb0bd2f619164a68a0c60d7f248', 965000: '92f5e25dd1c03102720dd0c3136b1a0769901bf89fcc0262a5e24405f349ca07', 966000: '08243d780d8ba96a940f409b87d9c6b8a95c92804173b9156ada0dad35b628dc', 967000: 'cb769a8935bb6faeb981da74f4079babbbb89476f825cc897f43e79790295260', 968000: 'ff3fc27d2998f4dc4ac1ff378afe14c7d0f43cc328deb9c978ec0e067d1dfaf9', 969000: 'e41a3452f45d5f025627d08c9c41017679e9c4804371dd1cc02f3ed49f85dbb2', 970000: 'f5eaaf7ba6b47245a4a8096a7785c7b25dc6db342ac2ccbba0c321e97ab58284', 971000: '75414062f1d4ed675dadc8f04ba10147a484aaca1ae316dc0b896a92809b3db6', 972000: '5bcf2ee00133774c7d060a1a1863dfccc20d5127ecb542470f607dec2504fe6f', 973000: '07d15b9656ecde2cd86a9d22c3de8b6505d6bab2aa5a94560b0db9119f1f6f6c', 974000: '2059e7924d7a210a88f5a65abc61152506a82edccd27416e796c81b9b8003f13', 975000: '7fcf5d8b2c0e51cfbdaa2502a9da0bdb323646899dad37dacc39af9f9e16fc5c', 976000: '02acb8cf87a0900436eccfca50371948531041d7b8b410a902205f84dd7fb88e', 977000: '2636dfd5a47016c893265473e78ecbf2000769d886f0d01ee7a91e9397210d15', 978000: 'ce92f52a35096b94bea73a7d4e113bc4564a4a589b66f1ab86f61c822cf9ee76', 979000: '21b8102f5b76be0c8e20d537ebc78ebe46bfcea6b6d2dda950ce5b48e85f72d7', 980000: 'f4df0bd63b36105705de62266d654612d9804bad7069d41344de269657e6f084', 981000: 'f006cd2718d98d774a5cd18394db7744c812fa149c8a63e76bab934aee89f571', 982000: 'da5d6609265d9153022d823b0260aa07e7511ceff7a3fd2ca7ce83cb3900a661', 983000: '3a26f3f02aa145fa8c5268fbe10dd9c3546d7dda57489ca5d4b161beb0d5a6e2', 984000: '968e8cd37a1137797d40f39f106cae62d1e252b46c7473b9434ad5f870ee88fb', 985000: '3129c3bf20deace1a9c92646a9d769da7a07f18dcd5b7a7b1e8cf5fd5390f8e1', 986000: '6ce830ca5da322ddbb97fc572ea03218913d070e5910516b33c6113b02b23c21', 987000: '7fb1a8635623847132ab766a99b792953379f782d1115b9649f5f9c5a742ca04', 988000: '5e8e6c6da7f271129c20c4dd891dcb1df4f9d690ee7cf391c6b7fbd028a0da4c', 989000: '12919e34bb9a9ac1d2a01e221eb8c511117fc4e1b3ae15355d95caf4673bdb08', 990000: '016f8b18227a0c09da55594a98638ad5b0fbb4896e2ab6163ac40b6015b2811e', 991000: 'ddf8cd6e2f4ee07530ae7567cef4fa2c2fd4a655cb20e20422e66fd49bde6489', 992000: 'dca77707c0caa3a9605f3dadf593402339c29448869907fb31f6c624e942dcbd', 993000: 'de9acc4c7c482ecac741fd6acbbc3a333afab52f3fe5eea4130c0770299a56dd', 994000: '54420631f8a801a1b8f391088f599ee22cedc06f24bf67f18272feb8fe70c682', 995000: '4b44b26e3e2495716dfd86fc42594cd4b1e4b70bdab4f0905cce4cb9556e008a', 996000: 'd6e41fd301fc5f519c343ceb39c9ff845656a4482e4e182abdcd3963fd5fde1c', 997000: 'd68b6a509d742b182ffb5a98b0e585a2320a5d3fe6977ad3e6cd06835ef2ea55', 998000: '1efcdcbadbec54ce3a93a1857253614536c34f05a0b1924f24bff194dc3392e1', 999000: '10a7713e46f47527f3819b4a9257a03f3e207d18e4917d6bcb43fdea3ba82b9a', 1000000: '1b4ddb1436df05f07807d6337b93ee1aa8b600fd6a910a8fd5313a39e0440eec', 1001000: 'cde0df1abdae26d2c2bdc111be15fb33231c5e167bb8b8f8eec667d71379fee4', 1002000: 'd7ce7a96a3ca73a4dfd6a1780e23f834f339142519ea7f45d256c113e27e4857', 1003000: 'b1a9b1c562ec62b9dd746d336b4211afc37482d0274ff692a44fa17ac9fe9a28', 1004000: '7afd6d0fb0014fbe16a31c84d3f1731736eaeef35e40bb1a1f232fb00345deae', 1005000: '4af61ce4cda5de58277f7a67cadea5d3f6ce56e54785b188e32306e00b0414df', 1006000: '08e1fb7295efd4a48cb999d899a3d481b682ddbce738fecd88a6d32cbe8234f0', 1007000: '14a367a41603dd690541daee8aa4a2882260059e3f85bd8978b7431e8f7db844', 1008000: 'e673230e62aaefad0678611f94ff35ee8a6e18eb96438bdfb4b614f54f54dba7', 1009000: 'e191af8fb71d0d91419abd19443af3d3f23ee4fe359bb8c390429cc838132bde', 1010000: 'ffdba58f184cf60838b75b7899b6633e7cfd34cf36eded572c0133d07387bc49', 1011000: '40801af3a5546cb9d53e05e21b74be09de9a421b762ca1d52d2266f5c2055ce8', 1012000: '552519acebed0e38102f5270dc60b1da7a123600b6b94169ae74462ae454693f', 1013000: '1eee96f48418929927eaa9642777bc806d326cfffaf077bc8695a7ecd438d631', 1014000: 'a471093e1de2a8db586412d7351c8d88e44ea890f46e9b43251af427a0a4a879', 1015000: '57532f5a522295cc139f008bdcb7a1e6d02e6035d5221b2687c7c216f06297a2', 1016000: 'ec46dba07addcb6e62f58456a53c513d876f1c49ae7d76d230adb8debd26027d', 1017000: '33ea8d25f342a7465ed71e4bab2b91007991e0994c61d321e3625301a1390322', 1018000: '4871c03cc95d4ce0a39bd2cebbb001b2ea1cce1b3561bb841d88f43bb9d12ffd', 1019000: 'f5248257576eb2ff4139d6374cc7ce34121cc942598cf9e04d2bd572e09189bb', 1020000: 'e7785286897c85cfb0276957bff216039eeb11bc1ebca89d0bb586022caa5750', 1021000: 'a30220f17d060634c5f6a1ddc5ea34b01c18fb5eb7e0e8267b66bf5a49525627', 1022000: '6083ea49e64ac0d4507c674237cf87d30b90b285ec63d082e626df0223eb7c9c', 1023000: '1dc5596d716bc33ee0f56fc40c1f073155a58a7692935c9e5854ef3b65b76828', 1024000: '065adfee40dc33abff07fb55339571712b959bc1830dc60b6691e36eab1508ae', 1025000: 'bb6903752d31278570e774b80a80782179c78f099e58c3dc4cba7afea7a471c4', 1026000: 'f3050f3c2f3a76f5084856b0f089383517caa3f51530fbc29335308f5f170625', 1027000: '746ed3701510d07958d11a06f22dbb839d9858373dc5a33249dd69e91bab01fd', 1028000: '43f7a96ea6a45b78c29ad4a2f8680ef184438c2bd3686172b0564e0ae6dd7ba1', 1029000: 'cbb9916099c59e14fe61d284374f4feaa3d43afec59e4698ed92143576f24b34', 1030000: '2e805fc2331e32e586ea692bc3d4e6b11e1ec3f1cab6e331b459f9f1ac9a1f1e', 1031000: '04f324f8f6d4f9901cf65f78dc91d6010ea6cf125f5ac0253b57b5f1f79e81e0', 1032000: '60ca62f52fdfd858b0ee0fdb380648bde85ca14e2a73565205ed4ee0bc861c77', 1033000: 'eb60aac23d599d3099cf98ed8fc3213f1bc06bc1c677429b303e9c81f79f1340', 1034000: 'f0328df2daf119ce673ddfa7a39a84576985f701f7a7dec3f56f58c2019ebd4d', 1035000: 'f9d3cbce3854de168d8835c96917c01be6244c8f82641e8d9398dfffec4e7107', 1036000: '7dca97e6e1d6ed70aa7805f74b768009a270e7ebe1dd951e8727d1d2f2d271f2', 1037000: '5329504126b2845b3044f423b521e77ff58d7d242f24bf87c87f4d8d4e03a947', 1038000: '5bad3ad55e3daa415f3182a1f2a099fe1767e8fae34e9bb95d47e242b8971434', 1039000: 'c29729b8ba49ac0043fe4aa6fc971f8ac3eda68ff92970957ada39a2989b2491', 1040000: 'f303aebfc9267600c081d0c021065743f93790df6f5c924a86b773788e0c45be', 1041000: 'a1cbe5059fa2275707785b77970c36d79b12c1ba93121bc9064ab9b64abacf7b', 1042000: '004b0dd4e438abc54ae832d733df32a6ba35b75e6d3e0c9c1dee5a7950507295', 1043000: '31893a3fe7bb4f6dd546c7a8de4a65990e94046aab442d18c68b6bf6acd54518', 1044000: '2c4dd479948acc42946f94050810000b0539864ad24a67a7251bff1c4971b035', 1045000: '1cea782d60df35a88b30ae205ce37e30abc7cad2b22181722be150bd92c53814', 1046000: 'ee808f0efb0f2ef93e8599d8b7f0e2e7c3cdc42353e4ea5165028b961f43d548', 1047000: '75f057e2a8cb1d46e5c943d63cc56936a6bac8b1cb89300593845a20baf39765', 1048000: '2abcd227f5314baed85e3c5b49d3888a60085c1845c955a8bf96aa3dd6394798', 1049000: '5d0ec24b9acd5ab21b42f68e1f3142b7bf83433b98f2fa9794586c8eff45893e', 1050000: '1d364b13a4c17bd67a6d1e5f77c26d02faa014d7cd152b4da70380f168b8e0ff', 1051000: 'b9a20cec21de84433be9b85817dd4803e875d9275dbc02907b29888431859bae', 1052000: '424cb56b00407d73b309b2081dd0bf89213cf024e3aafb3090506aa0ba10f835', 1053000: '6df3041a32fafd6a4e08778546d077cf591e1a2a16e77fe7a610efc2b542a9ff', 1054000: '78f8dee794f3d4366019339d7ba74ad2b543ecd25dc575620f66e1d535411971', 1055000: '43b8e9dae5addd58a7cccf62ba57ab46ffdaa2dcd113cc8ca537e9101b54c096', 1056000: '86b7f3741343f85d93410b78cc3fbf03d49b60a664e908703016aa56a206ae7e', 1057000: 'b033cf6ec622be6a99dff536a2cf73b36d3c3f8c3835ee17e0dd357403e85c41', 1058000: 'a65a6db692a8358e399a5ac3c818902fdb60595262ae05531084848febead249', 1059000: 'f6d781d2e2fdb4b7b074d1d8123875d899cdbd6be375cb4288e86f1d14a929f6', 1060000: 'cd9019bb1de4926cca16a7bef1a46786f10a3260d467cda0775f73361795abc9', 1061000: 'ed4f5dc6f475f95b40595632fafd9e7e5eef388b6cc15772204c0b0e9ee4e542', 1062000: 'c44d02a890aa66979b10d1cfa597c877f498841b4e12dd9a7bdf8d4a5fccab80', 1063000: '1c093734f5f241b36c1b9971e2759983f88f4033405a2588b4ebfd6998ac7465', 1064000: '9e354a83b71bbb9704053bfeea038a9c3d5daad080c6406c698b047c634706a6', 1065000: '563188accc4a6e311bd5046516a92a233f11f891b2304d37f151c5a6002b6958', 1066000: '333f1b4e996fac87e32dec667533715b31f1736b4342806a81d568b5c5238456', 1067000: 'df59a0b7319d5269bdf55043d91ec62bbb30829bb7054da623717a394b6ed678', 1068000: '06d8b674a205393edaf20c1d837baadc9caf0b0a675645246263cc163302241d', 1069000: 'ac065c48fad1383039d39e23c8367bad7cf9a37e07a5294cd7b04af5827b9961', 1070000: '90cd8b50f94208bc459081356474a961f6b764a1217f8fd291f5e4828081b730', 1071000: '3c0aa207ba9eea45458ab4fa26d6a027862592adb9bcce30915816e777dc6cfc', 1072000: '3d556c08f2300b67b704d3cbf46e22866e3ac164472b5930e2ada23b08475a0f', 1073000: 'a39b5c54c24efe3066aa203358b96baea405cd59aac6b0b48930e77799b4dd7d', 1074000: 'e8c8273d5a50a60e8744716c9f31496fb29eca87b4d68643f4ecd7ec4e400e23', 1075000: 'b8043ae41a1d0d7d4310c85764fcba1424733df347ffc2e8cbda1fe6ccbb5153', 1076000: '58468db1f91805e767d334824d6bffe54e0f900d1fb2a89b105086a493053b3d', 1077000: '04a78749b58465efa3a56d1735cd082c1f0f796e26486c7136950dbaf6effaa4', 1078000: 'e1dd6b58c75b01a67d4a4594dc7b4b2ee9e7d7fa7b25fd6246ce0e86eff33c75', 1079000: 'd239af017a6bb664485b14ad15e0eb703775e43018a045a8612b3697794460da', 1080000: '29ae5503f8c1249fefeb63fd967a71a70588ee0db1c97497e16366163a684341', 1081000: '05103ab27469e0859cbcd3daf42faa2bae798f522534697c7f2b34f7a050ee0f', 1082000: '4553d2cb7e90b6db11d242e287fe96822e6cd60e6388b94bf9006411f202ba03', 1083000: '97995acd178b2a142d571d5ae1c2a3deaf93a909fd91fb9c541d57f73e32dc99', 1084000: '9e3f23376af14d76ab24cd54e321dec019af73ad61067d959ff90043acc5ffcc', 1085000: '81c056b14f13cee0d6d6c8079fdd5a1a84c3a5c76cc9448612e8ef6d3531300e', 1086000: '8a0004f6809bdd075915a804e43991dfe8f22e05679d2fdaf8e373f101bac5c2', 1087000: '27c45a4c9ad24e038f2ebe40835a1c49ac7221d7185082866ee354351ba87c7a', 1088000: 'fd27e21747117b00b4ada1cba161ac49edb57cca540f86ac5ba885050f08f824', 1089000: 'bff867335767103bc3ed15ede5b9fde88016f8ede15dc5bf3e81ea40dcfc61ae', 1090000: '608f75016d1db08888dd59640f63e838c19bdfa833c0cc177ad3d2b818b0db5b', 1091000: '90750b452bd4dedaab6b57fecbfe88f71ce3d5437fad7f9ec0fdd270445c7526', 1092000: '98287b39f9f1233017dc5d932e5c77f0521ca84587eb3f39f0e7b6c297c749af', 1093000: '68a5846ed05c9bb142197849106838765f90f15c10b2cc938eef49b95eaa9d33', 1094000: '5660a1aac2fc763a417fc656c8887fc8186bf613ae1ccbb1a664fb43ce1fa1d6', 1095000: '62bad3db418b3f4cad3596881b645b72479c71deb0d39c7a4c8bd1577dc225fd', 1096000: 'e0e4b2b183591f10dd5614c289412f2fb5e320b7d3278f7c028f42f591872666', 1097000: 'a233a233fc2aa5dab9e75106d91388343ef969458ea974f1409a2ab5fc441911', 1098000: '16dfa5fa6cbd1188e562697b5f00ac206960d0851ed84adf37ae975fd5ffdd6a', 1099000: 'b8a870b7dc6d3263730c00f59d52aa6cce35dc59aa8fba715034cc2d14927260', 1100000: 'a3cd7749743da22a3846dcc2edbf1df21b938e829419389e3bc09284797c5b43', 1101000: '75b14c2a95e2a095949729b7c0b624bd725a2de98404a8e3247b60c977d0198e', 1102000: '4d3af64d37064dd5f57e25d61f248a1e21c1b1cadd7bb1404e35c9fbe06f1fd4', 1103000: 'd73c92bfed358dfcd7659228974ab75ea2fc86f2301ee47133adad8075203872', 1104000: '30cd82354f37bc0b412123867c7e1835206022a7501853bf8c0d3df02f291645', 1105000: '1d2ef984f26693dce77460cd2694e5da46e675077e91a1cea26051733b01a7ef', 1106000: '51c076c304222fe3ca308ba6968c46fef448f85be13a095cecb75b90e7954698', 1107000: '99e2221339e16acc34c9816f2ef7b866c2dd753aa3cbe484ae831959a23ece68', 1108000: '0f1227c250296bfe88eb7eb41703f99f633cfe02870816111e0cadfe778ddb19', 1109000: 'b35447f1ad76f95bc4f5886e4028d33acb3ad7b5000dd15516d3f11ce4baa990', 1110000: 'ac7baff996062bfaaaddd7d496b17e3ec1c8d34b2143095645ff22fb3888ae00', 1111000: '430bbbdcca36b2d69b6a2dd8b07c583a060a467e5f9acbc6de62462e1f7c7036', 1112000: 'e5274dea029dc44baff55c05b0555f91b74d29ffd40e3a8c4e2c5b57f9d40bef', 1113000: 'cf43863249fa42cfe108220dd40169dac702b0dd9cf5cb699cf2fc96feda8371', 1114000: 'fa1c0e551784d21c451564124d2d730e616724f3e535de3c186bcdeb47e80a8f', 1115000: '49fe6ecee35a397b83b5a704e950ad028cfb4b7e7a524021e789f4acc0fd6ffe', 1116000: '74ecded36751aa8b7901b31f0d16d75d111fc3c40b567f649c04f74ed028aa5c', 1117000: 'd9ca760a22190bdf545766b47d963c738a4edcc27f4d15ca801b35751577cfa7', 1118000: 'c28d42f871682800ac4e867608227cfb6bc4c00b618e83a8556f201a1c28813c', 1119000: 'c5fafc4e1785b0b9e84bb052e392154a5ba1aefe612998017e90772bcd554e08', 1120000: 'aa054d428bc9ccee0761da92163817163413065fe1e67ef79a056c5233ea3476', 1121000: '0df295bb944218503bd1bf66d2ece0c50fd22dae3391b80673a7ad1e4e5c3934', 1122000: 'a13abb350a26673b3933b1de307a60a6845ca594d502599548c6253e21a6d8e8', 1123000: 'a4bc6a3abf9ed1f4b14338ff0f03f83456312bc91a93fa89ae6db493050115e1', 1124000: '65869938df99adf0dda76200291ce09a54c9bcc787e4bb62cd72c367db58f4f0', 1125000: 'ea5e918233b14c3c73d488a906e3741c61bdcafe0393bd0404168fe80c950a46', 1126000: 'ce88cd35104fcec51bcee77302e03162dc694802536f5b668786b2245e61bca5', 1127000: 'ea19c0c8d205be4be87d02c5301c9ed331e7d75e25b93d1c2137c248882af515', 1128000: '006f32d63c2a3adcf4fbad0b0629c97f1beab6446a9c27fbde9472f2d066219e', 1129000: '218e5392e1ecf471c3bbc3d79c24dee30ac8db315dbeb61317318efb3f221163', 1130000: '30b9da0bd8364e9cd5551b2529341a01a3b7257a238d15b2560e2c99fdb324e8', 1131000: '8a7f382cfa023d2eba6639443e67206f8883b57d23ce7e1339234b8bb3098a82', 1132000: 'bf9af68a6fe2112d8fe311dfd52334ae2e7b0bac6675c9ebfddb1f386c212668', 1133000: '1a30951e2be633502a47c255a93ddbb9ed231d6bb4c55a807c0e910b437766b3', 1134000: 'a9bcaf3300b7915e701a8e396eb13f0c7287576323420be7aab3c3ba48020f76', 1135000: '337eed9ed072b5ad862af2d3d651f1b49fa852abc590b7e1c2dc381b496f438a', 1136000: '208761dbc29ec58302d722a05e937a3cf9e78bfb6495be395dd7b54f02e169dc', 1137000: '4e5b67ff3324b64e268049fdc3d82982b847ee359d409ade6368864c38a111e5', 1138000: '55d1d0833021a664e85eec8cc90a0985e67cc80d28841aaa8c2231ec28087ebb', 1139000: 'e750ada1ec9fa0f2f2461ed68958c7d116a699a82ec12911da5563139f8df19e', 1140000: '9cf81407b6ccc8046f0233f97484166945758f7392bb54841c912fcb34cf205c', 1141000: 'fccf32b2fae03e3b6b562483776625f9843cd68734c55659e2069cde7e383170', 1142000: 'c3608c215dd6569da6c1871c4d72a09ab1caa9663647f2a9454b5693d5d72a65', 1143000: 'bd39cb8c4e529d15bbea6baeec66afe52ca18afe32bd812f28fbb0676647cdff', 1144000: '6e42d02538565ce7e2d9bf31a304f1fd0ac122d35d17a030160575815901b0b1', 1145000: 'b9722e1de2904ce1219140fffb1f4f9f5a041f885faa634404238d103c738b4c', 1146000: 'd4de4271459966cee774f538a243d7db0689b213b296463d42e45c93194d7861', 1147000: '51fadf109f22bb85574d0fbcbd0b20992983e89aee3d415a7b1c37c44775d9a9', 1148000: '137e1fe8da31680d21a42e7421eb608a883a497314e4404625ce44b0edadde6a', 1149000: 'cb87867eb04203ce15e0763a2f4389376cea75e0a2877f55e2911c575bef07a8', 1150000: '977528ca7953a2c9c19fefaa3aab7ebdec3ac324d74a07d83764ba25d9be0689', 1151000: 'a09c51c832600ded63a19201df008075273ea248fd406886e93a2cbaa3bba46b', 1152000: '0e5367cfa0f00dd932a5bcc00dcc807fa6825161806bed588e16a57947b4b32d', 1153000: '55a9de3dcde2efb56a3c5fea7d22b98c1e180db9a4d4f4f6be7aae1f1cbd7608', 1154000: 'abc58cf71c4691ebfaef920252730cf69abbe9de88b424c03051b9b03e85d45a', 1155000: '4f074ce73c8a096620b8a32498362eb66a072eae95d561f2d53557cd513ae785', 1156000: '540a838a0f0a8834466b17dd456d35b8acae2ec8419f8bd9a704d9ea439062ac', 1157000: 'd5310ac671abdb658ea028db86c23fc729af965f91d67a37218c1412cf32a1f5', 1158000: '162d906a07e6c35e7c3ebf7069a200521605a97920f5b589d31b19bfd7766ee2', 1159000: '600bd8f5e1e62219e220f4dcb650db5812e79956f95ae8a50e83126932685ee0', 1160000: '91319398d1a805fac8582c8485e6d84e7490d6cfa6e44e2c630665b6bce0e6b8', 1161000: 'f7ad3cff6ee76e1e3df4abe70c600e4af66e1df55bf7b03aee12251d4455a1d4', 1162000: '85b9fbba669c2a4d3f85cdb5123f9538c05bd66172b7236d756703f99258454d', 1163000: '966085d767d1e5e2e8baf8eda8c11472ec5351181c418b503585284009aaea79', 1164000: '1c94e1b531215c019b12caf407296d8868481f49524b7180c7161b0363c1f789', 1165000: '803b6bf93735aeae2cf607824e2adf0d754b58da2516c2da1e485c697e472143', 1166000: '872561a82f7991633d0927d25cb659d096bbe556fe6dac7a0b6a679820733069', 1167000: '6bd7cdd605a3179b54c8af88d1638bf8133fab12cbf0a78d37cf21eddf4395a1', 1168000: '79946f5758c1817239cc642d27298bd710983551a8236e49832c6d818b097337', 1169000: 'b0994c60728e74de4aa361f37fa85e5296ce3188ae4e0b66d7b34fe86a239c9c', 1170000: 'a54188a5a64e0cf8da2406d16a0ac3983b087fc7d6231b6f8abf92cf11dc78cd', 1171000: 'ec2924d98e470cc6359821e6468df2c15d60301861d443188730342581230ef2', 1172000: 'b4ac11116aa73ce19428009a80e583e19dc9bcd380f7f7ce272a92921d5868d2', 1173000: '501d3551f762999dd5a799f3c5658fff2a7f3aff0511488272cd7693fefb8f9d', 1174000: '4660074ea48a78ae453cb14b694b2844cc0fb63ed9352ed20d11158bbb5c1f28', 1175000: '0727f6b1d9f8fe5677a9ffa0d475f53f5a419ef90b80896c22c2c95de22175de', 1176000: '150633d6a35496c24a93c9e19817e90f649c56b7e2558f99e97325bfd5df8b17', 1177000: '0849e19f22571b62dba8ff02f6b5a064a7ac36e7ed491321b3663567e8e17294', 1178000: '770dd463e7bad80f689f12934e4ae06e24378d1545dcf211fd143beaef49464e', 1179000: '059d383dcc60a49b658b674d92fc35cab07b06329c58d73818b6387cb0c06534', 1180000: 'e547cb3c636243ca9ae4cfb92c30a0f583eda84e329a5c1e5f64a26fc6fc791e', 1181000: '4521a4396ab02f73d45d7a3393ea1c602d255778d52c12079c88bfbad32aab43', 1182000: '051cfe993e4b0b34233403a9e8c397dd50e8b78a30fb07e9c260604ee9e624a9', 1183000: '44a69c99bb8b85e84ae279f2d8e5400d51cb3d5f0bcd178db49d55548cd66191', 1184000: '2a1d23c9bb3c71a533e0c9d25b03bfa7e9db8e014645f3e7fbede6d99fff0191', 1185000: 'bb90d6c6d77819163a9e909ee621d874707cdb21c91b1d9e861b204cf37d0ffa', 1186000: '4a92051b738ea0e28c64c64f1eb6f0405bc7c3427bef91ff20f4c43cf084d750', 1187000: 'f782ac330ca20fb5d8a094ee0f0f8c086a76e3f03ecc6a2c42f8fd07e52e0f41', 1188000: '94cb7b653dd3d838c186420158cf0e73db73ec28deaf67d9a2ca902caba4141a', 1189000: 'c8128e59b9ec948de890184578a113478ea63f7d57cb75c2c8d5c001a5a724c0', 1190000: '4da643bd35e5b98932ae21515a6bffb9c72f2cd8d514cd2d7eac1922af785c3f', 1191000: '0f922d86658ac3f53c5f9db360c68ab3f3253a925f23e1323820e3384214719a', 1192000: '4c3ab631cf5ba0c236f7c64af6f790fc24448319de6f75dbd28df4e2648d0b7d', 1193000: 'eda118d1fac3470a1f8f01f5c78108c8ecdcd6420be30f6d20f1d1831e7b6975', 1194000: '5723fff88abd9bb5088476fa5f4221a61c6f8a718703a92f13248ad350abeea2', 1195000: '1715846f82d011919e3446c6ce675a65fb80338bd791d4e735702c4767d9adc4', 1196000: 'b497667996aee2db61e88f442e728be15ab0b2b64cfd43198691fcf6cdafacc8', 1197000: '309a6170d837b8cb334fb888a64ed4e47e6592747e93c8e9d1bf7d608cfef87d', 1198000: '3ea918ef64a67dec20051519e6aefaeb7aca2d8583baca9ad5c5bd07073e513a', 1199000: '4ec7b7361b0243e5b2996a16e3b27acd662126b95fe542a487c7030e47ea3667', 1200000: 'b829c742686fcd642d0f9443336d7e2c4eab81667c90ce553df1350ed10b4233', 1201000: '44c022887f1e126fd281b1cae26b2017fa6415a64b105762c87643204ce165a5', 1202000: 'b11cc739eb28a14f4e47be125aa7e62d6d6f90c8f8014ee70044ed506d53d938', 1203000: '997a7c5fd7a98b39c9ca0790519924d73c3567656b605c97a6fdb7b406c3c64d', 1204000: '7d25d872e17195ee277243f7a5a39aa64d8750cec62e4777146acf61a8e76b04', 1205000: 'ce8486ae745a4645bee081ef3291d9505174bed05b0668d963b2998b7643dbb0', 1206000: '46a0bcea3c411c600dffe3e06e3d1dfbf5879a7ec4dcf3848e794cefcbf2bc0b', 1207000: '37e6297bf6e4e2bdd40401d4d7f95e3e3bdafd4a7f76b9c52865cefc6b82b20b', 1208000: 'd09e3982a9827b8cf56a5a2f4031dc6b082926c1fd57b63beaaa6cfd534eb902', 1209000: '54ae9010a9f146c83464e7ee60b30d9dbee36418561abc4e8d61bce9baa2d21d', 1210000: '5dcfd33f8e5ac21c9ba8553758b8cd8afae7961cad428530b5109c2db2ebf39f', 1211000: '91c952348bb2c3dfac0d6531a3dac770ea6dab571af257530e9c55493c96bdd9', 1212000: 'e62cc3fe044a7f5de4c04a8aed5619548f9d5c6fad9f989d3382cb96de1d780d', 1213000: '66b46ffdca8acf1dd04528dadb28b6ac4ce38807c1b84abd685d4ddb3dc59a34', 1214000: '2ce4091756ad23746bab4906f46545953cadaf61deae0d78e8a10d4eb51866b1', 1215000: '83ce3ca087799cdc4b4c5e7cfeb4a127708724a7ca76aa5f7f4ec1ed48b5fca6', 1216000: '7d07b739b7991fbd74926281bf51bba9d5721afab39598720f9ff5f7410a6721', 1217000: '76adf49491670d0e8379058eacf0228f330f3c18955dfea1ebe43bc11ee065f3', 1218000: '77f422e7301a81692dec69e5c6d35fa988a00a4d820ad0ebb1d595add36558cc', 1219000: '8ba9d944f8c468c81799294aeea8dc05ed1bb90bb26552fcd190bd88fedcddf2', 1220000: '00330367c255e0fe51b374597995c53353bc5700ad7d603cbd4197141933fe9c', 1221000: '3ba8b316b7964f31fdf628ed869a6fd023680cca6611257a31efe22e4d17e578', 1222000: '016e58d3fb6a29a3f9281789359460e776e9feb2f0db500482b6e231e1272aef', 1223000: 'fdfe767c29a3de7acd913b627d1e5fa887a1af9974f6a8a6474db822468c785c', 1224000: '92239f6207bff3689c554e92b24fe2e7be4a2203104ad8ef08b2c6bedd9aeccf', 1225000: '9a2f2dd9527b533d3d743efc55236e73e15192171bc8d0cd910918d1ab00aef7', 1226000: 'eb8269c75b8c5f66e6ea88ad70883dddcf8a75a45198ca7a46eb0ec606a791bb', 1227000: '5c82e624390cd57942dc9d64344eaa3d8991e0437e01802473053245b706290c', 1228000: '51e9a7d727f07fc01be7c03e3dd854eb666697f05bf89259baac628520d4402c', 1229000: 'c4bfdb651c9abdeda717fb9c8a4c8a6c9c0f78c13d3e6cae3f24f504d734c643', 1230000: '9f1ce781d16f2334567cbfb22fff42c14d2b9290cc2883746f435a1fb127021d', 1231000: '5c996634b377412ae0a3d8f541f3cc4a354aab72c198aa23a5cfc2678cbabf09', 1232000: '86702316a2d1730fbae01a08f36fffe5bf6d3ebb7d76b35a1617713766698b46', 1233000: 'fb16b63916c0287cb9b01d0c5aad626ced1b73c49a374c9009703aa90fd27a82', 1234000: '7c6f7904602ccd86bfb05cb8d6b5547c989c57cb2e214e93f1220fa4fe29bcb0', 1235000: '898b0f20811f52aa5a6bd0c35eff86fca3fbe3b066e423644fa77b2e269d9513', 1236000: '39128910ef624b6a8bbd390a311b5587c0991cda834eed996d814fe410cac352', 1237000: 'a0709afeedb64af4168ce8cf3dbda667a248df8e91da96acb2333686a2b89325', 1238000: 'e00075e7ba8c18cc277bfc5115ae6ff6b9678e6e99efd6e45f549ef8a3981a3d', 1239000: '3fba891600738f2d37e279209d52bbe6dc7ce005eeed62048247c96f370e7cd5', 1240000: 'def9bf1bec9325db90bb070f532972cfdd74e814c2b5e74a4d5a7c09a963a5f1', 1241000: '6a5d187e32bc189ac786959e1fe846031b97ae1ce202c22e1bdb1d2a963005fd', 1242000: 'a74d7c0b104eaf76c53a3a31ce51b75bbd8e05b5e84c31f593f505a13d83634c', } ================================================ FILE: lbry/wallet/claim_proofs.py ================================================ import struct import binascii from lbry.crypto.hash import double_sha256 class InvalidProofError(Exception): pass def get_hash_for_outpoint(txhash, nout, height_of_last_takeover): return double_sha256( double_sha256(txhash) + double_sha256(str(nout).encode()) + double_sha256(struct.pack('>Q', height_of_last_takeover)) ) # noinspection PyPep8 def verify_proof(proof, root_hash, name): previous_computed_hash = None reverse_computed_name = '' verified_value = False for i, node in enumerate(proof['nodes'][::-1]): found_child_in_chain = False to_hash = b'' previous_child_character = None for child in node['children']: if child['character'] < 0 or child['character'] > 255: raise InvalidProofError("child character not int between 0 and 255") if previous_child_character: if previous_child_character >= child['character']: raise InvalidProofError("children not in increasing order") previous_child_character = child['character'] to_hash += bytes((child['character'],)) if 'nodeHash' in child: if len(child['nodeHash']) != 64: raise InvalidProofError("invalid child nodeHash") to_hash += binascii.unhexlify(child['nodeHash'])[::-1] else: if previous_computed_hash is None: raise InvalidProofError("previous computed hash is None") if found_child_in_chain is True: raise InvalidProofError("already found the next child in the chain") found_child_in_chain = True reverse_computed_name += chr(child['character']) to_hash += previous_computed_hash if not found_child_in_chain: if i != 0: raise InvalidProofError("did not find the alleged child") if i == 0 and 'txhash' in proof and 'nOut' in proof and 'last takeover height' in proof: if len(proof['txhash']) != 64: raise InvalidProofError(f"txhash was invalid: {proof['txhash']}") if not isinstance(proof['nOut'], int): raise InvalidProofError(f"nOut was invalid: {proof['nOut']}") if not isinstance(proof['last takeover height'], int): raise InvalidProofError( f"last takeover height was invalid: {proof['last takeover height']}") to_hash += get_hash_for_outpoint( binascii.unhexlify(proof['txhash'])[::-1], proof['nOut'], proof['last takeover height'] ) verified_value = True elif 'valueHash' in node: if len(node['valueHash']) != 64: raise InvalidProofError("valueHash was invalid") to_hash += binascii.unhexlify(node['valueHash'])[::-1] previous_computed_hash = double_sha256(to_hash) if previous_computed_hash != binascii.unhexlify(root_hash)[::-1]: raise InvalidProofError("computed hash does not match roothash") if 'txhash' in proof and 'nOut' in proof: if not verified_value: raise InvalidProofError("mismatch between proof claim and outcome") target = reverse_computed_name[::-1].encode('ISO-8859-1').decode() if 'txhash' in proof and 'nOut' in proof: if name != target: raise InvalidProofError("name did not match proof") if not name.startswith(target): raise InvalidProofError("name fragment does not match proof") return True ================================================ FILE: lbry/wallet/coinselection.py ================================================ from random import Random from typing import List from lbry.wallet.transaction import OutputEffectiveAmountEstimator MAXIMUM_TRIES = 100000 STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py def strategy(method): STRATEGIES.append(method.__name__) return method class CoinSelector: def __init__(self, target: int, cost_of_change: int, seed: str = None) -> None: self.target = target self.cost_of_change = cost_of_change self.exact_match = False self.tries = 0 self.random = Random(seed) if seed is not None: self.random.seed(seed, version=1) def select( self, txos: List[OutputEffectiveAmountEstimator], strategy_name: str = None) -> List[OutputEffectiveAmountEstimator]: if not txos: return [] available = sum(c.effective_amount for c in txos) if self.target > available: return [] return getattr(self, strategy_name or "standard")(txos, available) @strategy def prefer_confirmed(self, txos: List[OutputEffectiveAmountEstimator], available: int) -> List[OutputEffectiveAmountEstimator]: return ( self.only_confirmed(txos, available) or self.standard(txos, available) ) @strategy def only_confirmed(self, txos: List[OutputEffectiveAmountEstimator], _) -> List[OutputEffectiveAmountEstimator]: confirmed = [t for t in txos if t.txo.tx_ref and t.txo.tx_ref.height > 0] if not confirmed: return [] confirmed_available = sum(c.effective_amount for c in confirmed) if self.target > confirmed_available: return [] return self.standard(confirmed, confirmed_available) @strategy def standard(self, txos: List[OutputEffectiveAmountEstimator], available: int) -> List[OutputEffectiveAmountEstimator]: return ( self.branch_and_bound(txos, available) or self.closest_match(txos, available) or self.random_draw(txos, available) ) @strategy def branch_and_bound(self, txos: List[OutputEffectiveAmountEstimator], available: int) -> List[OutputEffectiveAmountEstimator]: # see bitcoin implementation for more info: # https://github.com/bitcoin/bitcoin/blob/master/src/wallet/coinselection.cpp txos.sort(reverse=True) current_value = 0 current_available_value = available current_selection: List[bool] = [] best_waste = self.cost_of_change best_selection: List[bool] = [] while self.tries < MAXIMUM_TRIES: self.tries += 1 backtrack = False if current_value + current_available_value < self.target or \ current_value > self.target + self.cost_of_change: backtrack = True elif current_value >= self.target: new_waste = current_value - self.target if new_waste <= best_waste: best_waste = new_waste best_selection = current_selection[:] backtrack = True if backtrack: while current_selection and not current_selection[-1]: current_selection.pop() current_available_value += txos[len(current_selection)].effective_amount if not current_selection: break current_selection[-1] = False utxo = txos[len(current_selection) - 1] current_value -= utxo.effective_amount else: utxo = txos[len(current_selection)] current_available_value -= utxo.effective_amount previous_utxo = txos[len(current_selection) - 1] if current_selection else None if current_selection and not current_selection[-1] and previous_utxo and \ utxo.effective_amount == previous_utxo.effective_amount and \ utxo.fee == previous_utxo.fee: current_selection.append(False) else: current_selection.append(True) current_value += utxo.effective_amount if best_selection: self.exact_match = True return [ txos[i] for i, include in enumerate(best_selection) if include ] return [] @strategy def closest_match(self, txos: List[OutputEffectiveAmountEstimator], _) -> List[OutputEffectiveAmountEstimator]: """ Pick one UTXOs that is larger than the target but with the smallest change. """ target = self.target + self.cost_of_change smallest_change = None best_match = None for txo in txos: if txo.effective_amount >= target: change = txo.effective_amount - target if smallest_change is None or change < smallest_change: smallest_change, best_match = change, txo return [best_match] if best_match else [] @strategy def random_draw(self, txos: List[OutputEffectiveAmountEstimator], _) -> List[OutputEffectiveAmountEstimator]: """ Accumulate UTXOs at random until there is enough to cover the target. """ target = self.target + self.cost_of_change self.random.shuffle(txos, random=self.random.random) # pylint: disable=deprecated-argument selection = [] amount = 0 for coin in txos: selection.append(coin) amount += coin.effective_amount if amount >= target: return selection return [] ================================================ FILE: lbry/wallet/constants.py ================================================ NULL_HASH32 = b'\x00'*32 CENT = 1000000 COIN = 100*CENT DUST = 1000 TIMEOUT = 30.0 TXO_TYPES = { "other": 0, "stream": 1, "channel": 2, "support": 3, "purchase": 4, "collection": 5, "repost": 6, } CLAIM_TYPE_NAMES = [ 'stream', 'channel', 'collection', 'repost', ] CLAIM_TYPES = [ TXO_TYPES[name] for name in CLAIM_TYPE_NAMES ] ================================================ FILE: lbry/wallet/database.py ================================================ import os import logging import asyncio import sqlite3 import platform from binascii import hexlify from collections import defaultdict from dataclasses import dataclass from contextvars import ContextVar from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional from datetime import date from prometheus_client import Gauge, Counter, Histogram from lbry.utils import LockWithMetrics from .bip32 import PublicKey from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Input from .constants import TXO_TYPES, CLAIM_TYPES from .util import date_to_julian_day from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order if platform.system() == 'Windows' or ({'ANDROID_ARGUMENT', 'KIVY_BUILD'} & os.environ.keys()): from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported else: from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass log = logging.getLogger(__name__) sqlite3.enable_callback_tracebacks(True) HISTOGRAM_BUCKETS = ( .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') ) @dataclass class ReaderProcessState: cursor: sqlite3.Cursor reader_context: Optional[ContextVar[ReaderProcessState]] = ContextVar('reader_context') def initializer(path): db = sqlite3.connect(path) db.row_factory = dict_row_factory db.executescript("pragma journal_mode=WAL;") reader = ReaderProcessState(db.cursor()) reader_context.set(reader) def run_read_only_fetchall(sql, params): cursor = reader_context.get().cursor try: return cursor.execute(sql, params).fetchall() except (Exception, OSError) as e: log.exception('Error running transaction:', exc_info=e) raise def run_read_only_fetchone(sql, params): cursor = reader_context.get().cursor try: return cursor.execute(sql, params).fetchone() except (Exception, OSError) as e: log.exception('Error running transaction:', exc_info=e) raise class AIOSQLite: reader_executor: ReaderExecutorClass waiting_writes_metric = Gauge( "waiting_writes_count", "Number of waiting db writes", namespace="daemon_database" ) waiting_reads_metric = Gauge( "waiting_reads_count", "Number of waiting db writes", namespace="daemon_database" ) write_count_metric = Counter( "write_count", "Number of database writes", namespace="daemon_database" ) read_count_metric = Counter( "read_count", "Number of database reads", namespace="daemon_database" ) acquire_write_lock_metric = Histogram( 'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS ) held_write_lock_metric = Histogram( 'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS ) def __init__(self): # has to be single threaded as there is no mapping of thread:connection self.writer_executor = ThreadPoolExecutor(max_workers=1) self.writer_connection: Optional[sqlite3.Connection] = None self._closing = False self.query_count = 0 self.write_lock = LockWithMetrics(self.acquire_write_lock_metric, self.held_write_lock_metric) self.writers = 0 self.read_ready = asyncio.Event() self.urgent_read_done = asyncio.Event() @classmethod async def connect(cls, path: Union[bytes, str], *args, **kwargs): sqlite3.enable_callback_tracebacks(True) db = cls() def _connect_writer(): db.writer_connection = sqlite3.connect(path, *args, **kwargs) readers = max(os.cpu_count() - 2, 2) db.reader_executor = ReaderExecutorClass( max_workers=readers, initializer=initializer, initargs=(path, ) ) await asyncio.get_event_loop().run_in_executor(db.writer_executor, _connect_writer) db.read_ready.set() db.urgent_read_done.set() return db async def close(self): if self._closing: return self._closing = True def __checkpoint_and_close(conn: sqlite3.Connection): conn.execute("PRAGMA WAL_CHECKPOINT(FULL);") log.info("DB checkpoint finished.") conn.close() await asyncio.get_event_loop().run_in_executor( self.writer_executor, __checkpoint_and_close, self.writer_connection) self.writer_executor.shutdown(wait=True) self.reader_executor.shutdown(wait=True) self.read_ready.clear() self.writer_connection = None def executemany(self, sql: str, params: Iterable): params = params if params is not None else [] # this fetchall is needed to prevent SQLITE_MISUSE return self.run(lambda conn: conn.executemany(sql, params).fetchall()) def executescript(self, script: str) -> Awaitable: return self.run(lambda conn: conn.executescript(script)) async def _execute_fetch(self, sql: str, parameters: Iterable = None, read_only=False, fetch_all: bool = False) -> List[dict]: read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone parameters = parameters if parameters is not None else [] still_waiting = False urgent_read = False if read_only: self.waiting_reads_metric.inc() self.read_count_metric.inc() try: while self.writers and not self._closing: # more writes can come in while we are waiting for the first if not urgent_read and still_waiting and self.urgent_read_done.is_set(): # throttle the writes if they pile up self.urgent_read_done.clear() urgent_read = True # wait until the running writes have finished await self.read_ready.wait() still_waiting = True if self._closing: raise asyncio.CancelledError() return await asyncio.get_event_loop().run_in_executor( self.reader_executor, read_only_fn, sql, parameters ) finally: if urgent_read: # unthrottle the writers if they had to be throttled self.urgent_read_done.set() self.waiting_reads_metric.dec() if fetch_all: return await self.run(lambda conn: conn.execute(sql, parameters).fetchall()) return await self.run(lambda conn: conn.execute(sql, parameters).fetchone()) async def execute_fetchall(self, sql: str, parameters: Iterable = None, read_only=False) -> List[dict]: return await self._execute_fetch(sql, parameters, read_only, fetch_all=True) async def execute_fetchone(self, sql: str, parameters: Iterable = None, read_only=False) -> List[dict]: return await self._execute_fetch(sql, parameters, read_only, fetch_all=False) def execute(self, sql: str, parameters: Iterable = None) -> Awaitable[sqlite3.Cursor]: parameters = parameters if parameters is not None else [] return self.run(lambda conn: conn.execute(sql, parameters)) async def run(self, fun, *args, **kwargs): self.write_count_metric.inc() self.waiting_writes_metric.inc() # it's possible many writes are coming in one after the other, these can # block reader calls for a long time # if the reader waits for the writers to finish and then has to wait for # yet more, it will clear the urgent_read_done event to block more writers # piling on try: await self.urgent_read_done.wait() except Exception as e: self.waiting_writes_metric.dec() raise e self.writers += 1 # block readers self.read_ready.clear() try: async with self.write_lock: if self._closing: raise asyncio.CancelledError() return await asyncio.get_event_loop().run_in_executor( self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs) ) finally: self.writers -= 1 self.waiting_writes_metric.dec() if not self.writers: # unblock the readers once the last enqueued writer finishes self.read_ready.set() def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs): self.writer_connection.execute('begin') try: self.query_count += 1 result = fun(self.writer_connection, *args, **kwargs) # type: ignore self.writer_connection.commit() return result except (Exception, OSError) as e: log.exception('Error running transaction:', exc_info=e) self.writer_connection.rollback() log.warning("rolled back") raise async def run_with_foreign_keys_disabled(self, fun, *args, **kwargs): self.write_count_metric.inc() self.waiting_writes_metric.inc() try: await self.urgent_read_done.wait() except Exception as e: self.waiting_writes_metric.dec() raise e self.writers += 1 self.read_ready.clear() try: async with self.write_lock: if self._closing: raise asyncio.CancelledError() return await asyncio.get_event_loop().run_in_executor( self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs ) finally: self.writers -= 1 self.waiting_writes_metric.dec() if not self.writers: self.read_ready.set() def __run_transaction_with_foreign_keys_disabled(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], args, kwargs): foreign_keys_enabled, = self.writer_connection.execute("pragma foreign_keys").fetchone() if not foreign_keys_enabled: raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead") try: self.writer_connection.execute('pragma foreign_keys=off').fetchone() return self.__run_transaction(fun, *args, **kwargs) finally: self.writer_connection.execute('pragma foreign_keys=on').fetchone() def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''): sql, values = [], {} for key, constraint in constraints.items(): tag = '0' if '#' in key: key, tag = key[:key.index('#')], key[key.index('#')+1:] col, op, key = key, '=', key.replace('.', '_') if not key: sql.append(constraint) continue if key.startswith('$$'): col, key = col[2:], key[1:] elif key.startswith('$'): values[key] = constraint continue if key.endswith('__not'): col, op = col[:-len('__not')], '!=' elif key.endswith('__is_null'): col = col[:-len('__is_null')] sql.append(f'{col} IS NULL') continue if key.endswith('__is_not_null'): col = col[:-len('__is_not_null')] sql.append(f'{col} IS NOT NULL') continue if key.endswith('__lt'): col, op = col[:-len('__lt')], '<' elif key.endswith('__lte'): col, op = col[:-len('__lte')], '<=' elif key.endswith('__gt'): col, op = col[:-len('__gt')], '>' elif key.endswith('__gte'): col, op = col[:-len('__gte')], '>=' elif key.endswith('__like'): col, op = col[:-len('__like')], 'LIKE' elif key.endswith('__not_like'): col, op = col[:-len('__not_like')], 'NOT LIKE' elif key.endswith('__in') or key.endswith('__not_in'): if key.endswith('__in'): col, op, one_val_op = col[:-len('__in')], 'IN', '=' else: col, op, one_val_op = col[:-len('__not_in')], 'NOT IN', '!=' if constraint: if isinstance(constraint, (list, set, tuple)): if len(constraint) == 1: values[f'{key}{tag}'] = next(iter(constraint)) sql.append(f'{col} {one_val_op} :{key}{tag}') else: keys = [] for i, val in enumerate(constraint): keys.append(f':{key}{tag}_{i}') values[f'{key}{tag}_{i}'] = val sql.append(f'{col} {op} ({", ".join(keys)})') elif isinstance(constraint, str): sql.append(f'{col} {op} ({constraint})') else: raise ValueError(f"{col} requires a list, set or string as constraint value.") continue elif key.endswith('__any') or key.endswith('__or'): where, subvalues = constraints_to_sql(constraint, ' OR ', key+tag+'_') sql.append(f'({where})') values.update(subvalues) continue if key.endswith('__and'): where, subvalues = constraints_to_sql(constraint, ' AND ', key+tag+'_') sql.append(f'({where})') values.update(subvalues) continue sql.append(f'{col} {op} :{prepend_key}{key}{tag}') values[prepend_key+key+tag] = constraint return joiner.join(sql) if sql else '', values def query(select, **constraints) -> Tuple[str, Dict[str, Any]]: sql = [select] limit = constraints.pop('limit', None) offset = constraints.pop('offset', None) order_by = constraints.pop('order_by', None) group_by = constraints.pop('group_by', None) accounts = constraints.pop('accounts', []) if accounts: constraints['account__in'] = [a.public_key.address for a in accounts] where, values = constraints_to_sql(constraints) if where: sql.append('WHERE') sql.append(where) if group_by is not None: sql.append(f'GROUP BY {group_by}') if order_by: sql.append('ORDER BY') if isinstance(order_by, str): sql.append(order_by) elif isinstance(order_by, list): sql.append(', '.join(order_by)) else: raise ValueError("order_by must be string or list") if limit is not None: sql.append(f'LIMIT {limit}') if offset is not None: sql.append(f'OFFSET {offset}') return ' '.join(sql), values def interpolate(sql, values): for k in sorted(values.keys(), reverse=True): value = values[k] if isinstance(value, bytes): value = f"X'{hexlify(value).decode()}'" elif isinstance(value, str): value = f"'{value}'" else: value = str(value) sql = sql.replace(f":{k}", value) return sql def constrain_single_or_list(constraints, column, value, convert=lambda x: x, negate=False): if value is not None: if isinstance(value, list): value = [convert(v) for v in value] if len(value) == 1: if negate: constraints[f"{column}__or"] = { f"{column}__is_null": True, f"{column}__not": value[0] } else: constraints[column] = value[0] elif len(value) > 1: if negate: constraints[f"{column}__or"] = { f"{column}__is_null": True, f"{column}__not_in": value } else: constraints[f"{column}__in"] = value elif negate: constraints[f"{column}__or"] = { f"{column}__is_null": True, f"{column}__not": convert(value) } else: constraints[column] = convert(value) return constraints class SQLiteMixin: SCHEMA_VERSION: Optional[str] = None CREATE_TABLES_QUERY: str MAX_QUERY_VARIABLES = 900 CREATE_VERSION_TABLE = """ create table if not exists version ( version text ); """ def __init__(self, path): self._db_path = path self.db: AIOSQLite = None self.ledger = None async def open(self): log.info("connecting to database: %s", self._db_path) self.db = await AIOSQLite.connect(self._db_path, isolation_level=None) if self.SCHEMA_VERSION: tables = [t[0] for t in await self.db.execute_fetchall( "SELECT name FROM sqlite_master WHERE type='table';" )] if tables: if 'version' in tables: version = await self.db.execute_fetchone("SELECT version FROM version LIMIT 1;") if version == (self.SCHEMA_VERSION,): return if version == ("1.5",) and self.SCHEMA_VERSION == "1.6": await self.db.execute("ALTER TABLE txo ADD COLUMN has_source bool DEFAULT 1;") await self.db.execute("UPDATE version SET version = ?", (self.SCHEMA_VERSION,)) return await self.db.executescript('\n'.join( f"DROP TABLE {table};" for table in tables ) + '\n' + 'PRAGMA WAL_CHECKPOINT(FULL);' + '\n' + 'VACUUM;') await self.db.execute(self.CREATE_VERSION_TABLE) await self.db.execute("INSERT INTO version VALUES (?)", (self.SCHEMA_VERSION,)) await self.db.executescript(self.CREATE_TABLES_QUERY) async def close(self): await self.db.close() @staticmethod def _insert_sql(table: str, data: dict, ignore_duplicate: bool = False, replace: bool = False) -> Tuple[str, List]: columns, values = [], [] for column, value in data.items(): columns.append(column) values.append(value) policy = "" if ignore_duplicate: policy = " OR IGNORE" if replace: policy = " OR REPLACE" sql = "INSERT{} INTO {} ({}) VALUES ({})".format( policy, table, ', '.join(columns), ', '.join(['?'] * len(values)) ) return sql, values @staticmethod def _update_sql(table: str, data: dict, where: str, constraints: Union[list, tuple]) -> Tuple[str, list]: columns, values = [], [] for column, value in data.items(): columns.append(f"{column} = ?") values.append(value) values.extend(constraints) sql = "UPDATE {} SET {} WHERE {}".format( table, ', '.join(columns), where ) return sql, values def dict_row_factory(cursor, row): d = {} for idx, col in enumerate(cursor.description): d[col[0]] = row[idx] return d SQLITE_MAX_INTEGER = 9223372036854775807 def _get_spendable_utxos(transaction: sqlite3.Connection, accounts: List, decoded_transactions: Dict[str, Transaction], result: Dict[Tuple[bytes, int, bool], List[int]], reserved: List[Transaction], amount_to_reserve: int, reserved_amount: int, floor: int, ceiling: int, fee_per_byte: int) -> int: accounts_fmt = ",".join(["?"] * len(accounts)) txo_query = """ SELECT tx.txid, txo.txoid, tx.raw, tx.height, txo.position as nout, tx.is_verified, txo.amount FROM txo INNER JOIN account_address USING (address) LEFT JOIN txi USING (txoid) INNER JOIN tx USING (txid) WHERE txo.txo_type=0 AND txi.txoid IS NULL AND tx.txid IS NOT NULL AND NOT txo.is_reserved AND txo.amount >= ? AND txo.amount < ? """ if accounts: txo_query += f""" AND account_address.account {'= ?' if len(accounts_fmt) == 1 else 'IN (' + accounts_fmt + ')'} """ txo_query += """ ORDER BY txo.amount ASC, tx.height DESC """ # prefer confirmed, but save unconfirmed utxos from this selection in case they are needed unconfirmed = [] for row in transaction.execute(txo_query, (floor, ceiling, *accounts)): (txid, txoid, raw, height, nout, verified, amount) = row.values() # verified or non verified transactions were found- reset the gap count # multiple txos can come from the same tx, only decode it once and cache if txid not in decoded_transactions: # cache the decoded transaction decoded_transactions[txid] = Transaction(raw) decoded_tx = decoded_transactions[txid] # save the unconfirmed txo for possible use later, if still needed if verified: # add the txo to the reservation, minus the fee for including it reserved_amount += amount reserved_amount -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte # mark it as reserved result[(raw, height, verified)].append(nout) reserved.append(txoid) # if we've reserved enough, return if reserved_amount >= amount_to_reserve: return reserved_amount else: unconfirmed.append((txid, txoid, raw, height, nout, verified, amount)) # we're popping the items, so to get them in the order they were seen they are reversed unconfirmed.reverse() # add available unconfirmed txos if any were previously found while unconfirmed and reserved_amount < amount_to_reserve: (txid, txoid, raw, height, nout, verified, amount) = unconfirmed.pop() # it's already decoded decoded_tx = decoded_transactions[txid] # add to the reserved amount reserved_amount += amount reserved_amount -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte result[(raw, height, verified)].append(nout) reserved.append(txoid) return reserved_amount def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: List, amount_to_reserve: int, floor: int, fee_per_byte: int, set_reserved: bool, return_insufficient_funds: bool, base_multiplier: int = 100): txs = defaultdict(list) decoded_transactions = {} reserved = [] reserved_dewies = 0 multiplier = base_multiplier gap_count = 0 while reserved_dewies < amount_to_reserve and gap_count < 5 and floor * multiplier < SQLITE_MAX_INTEGER: previous_reserved_dewies = reserved_dewies reserved_dewies = _get_spendable_utxos( transaction, accounts, decoded_transactions, txs, reserved, amount_to_reserve, reserved_dewies, floor, floor * multiplier, fee_per_byte ) floor *= multiplier if previous_reserved_dewies == reserved_dewies: gap_count += 1 multiplier **= 2 else: gap_count = 0 multiplier = base_multiplier # reserve the accumulated txos if enough were found if reserved_dewies >= amount_to_reserve: if set_reserved: transaction.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?", [(True, txoid) for txoid in reserved]).fetchall() return txs # return_insufficient_funds and set_reserved are used for testing return txs if return_insufficient_funds else {} class Database(SQLiteMixin): SCHEMA_VERSION = "1.6" PRAGMAS = """ pragma journal_mode=WAL; """ CREATE_ACCOUNT_TABLE = """ create table if not exists account_address ( account text not null, address text not null, chain integer not null, pubkey blob not null, chain_code blob not null, n integer not null, depth integer not null, primary key (account, address) ); create index if not exists address_account_idx on account_address (address, account); """ CREATE_PUBKEY_ADDRESS_TABLE = """ create table if not exists pubkey_address ( address text primary key, history text, used_times integer not null default 0 ); """ CREATE_TX_TABLE = """ create table if not exists tx ( txid text primary key, raw blob not null, height integer not null, position integer not null, is_verified boolean not null default 0, purchased_claim_id text, day integer ); create index if not exists tx_purchased_claim_id_idx on tx (purchased_claim_id); """ CREATE_TXO_TABLE = """ create table if not exists txo ( txid text references tx, txoid text primary key, address text references pubkey_address, position integer not null, amount integer not null, script blob not null, is_reserved boolean not null default 0, txo_type integer not null default 0, claim_id text, claim_name text, has_source bool, channel_id text, reposted_claim_id text ); create index if not exists txo_txid_idx on txo (txid); create index if not exists txo_address_idx on txo (address); create index if not exists txo_claim_id_idx on txo (claim_id, txo_type); create index if not exists txo_claim_name_idx on txo (claim_name); create index if not exists txo_txo_type_idx on txo (txo_type); create index if not exists txo_channel_id_idx on txo (channel_id); create index if not exists txo_reposted_claim_idx on txo (reposted_claim_id); """ CREATE_TXI_TABLE = """ create table if not exists txi ( txid text references tx, txoid text references txo primary key, address text references pubkey_address, position integer not null ); create index if not exists txi_address_idx on txi (address); create index if not exists first_input_idx on txi (txid, address) where position=0; """ CREATE_TABLES_QUERY = ( PRAGMAS + CREATE_ACCOUNT_TABLE + CREATE_PUBKEY_ADDRESS_TABLE + CREATE_TX_TABLE + CREATE_TXO_TABLE + CREATE_TXI_TABLE ) async def open(self): await super().open() self.db.writer_connection.row_factory = dict_row_factory def txo_to_row(self, tx, txo): row = { 'txid': tx.id, 'txoid': txo.id, 'address': txo.get_address(self.ledger), 'position': txo.position, 'amount': txo.amount, 'script': sqlite3.Binary(txo.script.source), 'has_source': False, } if txo.is_claim: if txo.can_decode_claim: claim = txo.claim row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream']) if claim.is_repost: row['reposted_claim_id'] = claim.repost.reference.claim_id row['has_source'] = True if claim.is_signed: row['channel_id'] = claim.signing_channel_id if claim.is_stream: row['has_source'] = claim.stream.has_source else: row['txo_type'] = TXO_TYPES['stream'] elif txo.is_support: row['txo_type'] = TXO_TYPES['support'] support = txo.can_decode_support if support and support.is_signed: row['channel_id'] = support.signing_channel_id elif txo.purchase is not None: row['txo_type'] = TXO_TYPES['purchase'] row['claim_id'] = txo.purchased_claim_id if txo.script.is_claim_involved: row['claim_id'] = txo.claim_id row['claim_name'] = txo.claim_name return row def tx_to_row(self, tx): row = { 'txid': tx.id, 'raw': sqlite3.Binary(tx.raw), 'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified, 'day': tx.get_julian_day(self.ledger), } txos = tx.outputs if len(txos) >= 2 and txos[1].can_decode_purchase_data: txos[0].purchase = txos[1] row['purchased_claim_id'] = txos[1].purchase_data.claim_id return row async def insert_transaction(self, tx): await self.db.execute_fetchall(*self._insert_sql('tx', self.tx_to_row(tx))) async def update_transaction(self, tx): await self.db.execute_fetchall(*self._update_sql("tx", { 'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified }, 'txid = ?', (tx.id,))) def _transaction_io(self, conn: sqlite3.Connection, tx: Transaction, address, txhash): conn.execute(*self._insert_sql('tx', self.tx_to_row(tx), replace=True)).fetchall() is_my_input = False for txi in tx.inputs: if txi.txo_ref.txo is not None: txo = txi.txo_ref.txo if txo.has_address and txo.get_address(self.ledger) == address: is_my_input = True conn.execute(*self._insert_sql("txi", { 'txid': tx.id, 'txoid': txo.id, 'address': address, 'position': txi.position }, ignore_duplicate=True)).fetchall() for txo in tx.outputs: if txo.script.is_pay_pubkey_hash and (txo.pubkey_hash == txhash or is_my_input): conn.execute(*self._insert_sql( "txo", self.txo_to_row(tx, txo), ignore_duplicate=True )).fetchall() elif txo.script.is_pay_script_hash and is_my_input: conn.execute(*self._insert_sql( "txo", self.txo_to_row(tx, txo), ignore_duplicate=True )).fetchall() def save_transaction_io(self, tx: Transaction, address, txhash, history): return self.save_transaction_io_batch([tx], address, txhash, history) def save_transaction_io_batch(self, txs: Iterable[Transaction], address, txhash, history): history_count = history.count(':') // 2 def __many(conn): for tx in txs: self._transaction_io(conn, tx, address, txhash) conn.execute( "UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?", (history, history_count, address) ).fetchall() return self.db.run(__many) async def reserve_outputs(self, txos, is_reserved=True): txoids = [(is_reserved, txo.id) for txo in txos] await self.db.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?", txoids) async def release_outputs(self, txos): await self.reserve_outputs(txos, is_reserved=False) async def rewind_blockchain(self, above_height): # pylint: disable=no-self-use # TODO: # 1. delete transactions above_height # 2. update address histories removing deleted TXs return True async def get_spendable_utxos(self, ledger, reserve_amount, accounts: Optional[Iterable], min_amount: int = 1, fee_per_byte: int = 50, set_reserved: bool = True, return_insufficient_funds: bool = False) -> List: to_spend = await self.db.run( get_and_reserve_spendable_utxos, tuple(account.id for account in accounts), reserve_amount, min_amount, fee_per_byte, set_reserved, return_insufficient_funds ) txos = [] for (raw, height, verified), positions in to_spend.items(): tx = Transaction(raw, height=height, is_verified=verified) for nout in positions: txos.append(tx.outputs[nout].get_estimator(ledger)) return txos async def select_transactions(self, cols, accounts=None, read_only=False, **constraints): if not {'txid', 'txid__in'}.intersection(constraints): assert accounts, "'accounts' argument required when no 'txid' constraint is present" where, values = constraints_to_sql({ '$$account_address.account__in': [a.public_key.address for a in accounts] }) constraints['txid__in'] = f""" SELECT txo.txid FROM txo JOIN account_address USING (address) WHERE {where} UNION SELECT txi.txid FROM txi JOIN account_address USING (address) WHERE {where} """ constraints.update(values) return await self.db.execute_fetchall( *query(f"SELECT {cols} FROM tx", **constraints), read_only=read_only ) TXO_NOT_MINE = Output(None, None, is_my_output=False) async def get_transactions(self, wallet=None, **constraints): include_is_spent = constraints.pop('include_is_spent', False) include_is_my_input = constraints.pop('include_is_my_input', False) include_is_my_output = constraints.pop('include_is_my_output', False) tx_rows = await self.select_transactions( 'txid, raw, height, position, is_verified', order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]), **constraints ) if not tx_rows: return [] txids, txs, txi_txoids = [], [], [] for row in tx_rows: txids.append(row['txid']) txs.append(Transaction( raw=row['raw'], height=row['height'], position=row['position'], is_verified=bool(row['is_verified']) )) for txi in txs[-1].inputs: txi_txoids.append(txi.txo_ref.id) step = self.MAX_QUERY_VARIABLES annotated_txos = {} for offset in range(0, len(txids), step): annotated_txos.update({ txo.id: txo for txo in (await self.get_txos( wallet=wallet, txid__in=txids[offset:offset+step], order_by='txo.txid', include_is_spent=include_is_spent, include_is_my_input=include_is_my_input, include_is_my_output=include_is_my_output, )) }) referenced_txos = {} for offset in range(0, len(txi_txoids), step): referenced_txos.update({ txo.id: txo for txo in (await self.get_txos( wallet=wallet, txoid__in=txi_txoids[offset:offset+step], order_by='txo.txoid', include_is_my_output=include_is_my_output, )) }) for tx in txs: for txi in tx.inputs: txo = referenced_txos.get(txi.txo_ref.id) if txo: txi.txo_ref = txo.ref for txo in tx.outputs: _txo = annotated_txos.get(txo.id) if _txo: txo.update_annotations(_txo) else: txo.update_annotations(self.TXO_NOT_MINE) for tx in txs: txos = tx.outputs if len(txos) >= 2 and txos[1].can_decode_purchase_data: txos[0].purchase = txos[1] return txs async def get_transaction_count(self, **constraints): constraints.pop('wallet', None) constraints.pop('offset', None) constraints.pop('limit', None) constraints.pop('order_by', None) count = await self.select_transactions('COUNT(*) as total', **constraints) return count[0]['total'] or 0 async def get_transaction(self, **constraints): txs = await self.get_transactions(limit=1, **constraints) if txs: return txs[0] async def select_txos( self, cols, accounts=None, is_my_input=None, is_my_output=True, is_my_input_or_output=None, exclude_internal_transfers=False, include_is_spent=False, include_is_my_input=False, is_spent=None, read_only=False, **constraints): for rename_col in ('txid', 'txoid'): for rename_constraint in (rename_col, rename_col+'__in', rename_col+'__not_in'): if rename_constraint in constraints: constraints['txo.'+rename_constraint] = constraints.pop(rename_constraint) if accounts: account_in_sql, values = constraints_to_sql({ '$$account__in': [a.public_key.address for a in accounts] }) my_addresses = f"SELECT address FROM account_address WHERE {account_in_sql}" constraints.update(values) if is_my_input_or_output: include_is_my_input = True constraints['received_or_sent__or'] = { 'txo.address__in': my_addresses, 'sent__and': { 'txi.address__is_not_null': True, 'txi.address__in': my_addresses } } else: if is_my_output: constraints['txo.address__in'] = my_addresses elif is_my_output is False: constraints['txo.address__not_in'] = my_addresses if is_my_input: include_is_my_input = True constraints['txi.address__is_not_null'] = True constraints['txi.address__in'] = my_addresses elif is_my_input is False: include_is_my_input = True constraints['is_my_input_false__or'] = { 'txi.address__is_null': True, 'txi.address__not_in': my_addresses } if exclude_internal_transfers: include_is_my_input = True constraints['exclude_internal_payments__or'] = { 'txo.txo_type__not': TXO_TYPES['other'], 'txo.address__not_in': my_addresses, 'txi.address__is_null': True, 'txi.address__not_in': my_addresses, } sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"] if is_spent: constraints['spent.txoid__is_not_null'] = True elif is_spent is False: constraints['is_reserved'] = False constraints['spent.txoid__is_null'] = True if include_is_spent or is_spent is not None: sql.append("LEFT JOIN txi AS spent ON (spent.txoid=txo.txoid)") if include_is_my_input: sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)") return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only) async def get_txos( self, wallet=None, no_tx=False, no_channel_info=False, read_only=False, **constraints ) -> List[Output]: include_is_spent = constraints.get('include_is_spent', False) include_is_my_input = constraints.get('include_is_my_input', False) include_is_my_output = constraints.pop('include_is_my_output', False) include_received_tips = constraints.pop('include_received_tips', False) select_columns = [ "tx.txid, tx.height, tx.position as tx_position, tx.is_verified, " "txo_type, txo.position as txo_position, amount, script" ] if not no_tx: select_columns.append("raw") my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set() my_accounts_sql = "" if include_is_my_output or include_is_my_input: my_accounts_sql, values = constraints_to_sql({'$$account__in#_wallet': my_accounts}) constraints.update(values) if include_is_my_output and my_accounts: if constraints.get('is_my_output', None) in (True, False): select_columns.append(f"{1 if constraints['is_my_output'] else 0} AS is_my_output") else: select_columns.append(f"""( txo.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) ) AS is_my_output""") if include_is_my_input and my_accounts: if constraints.get('is_my_input', None) in (True, False): select_columns.append(f"{1 if constraints['is_my_input'] else 0} AS is_my_input") else: select_columns.append(f"""( txi.address IS NOT NULL AND txi.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) ) AS is_my_input""") if include_is_spent: select_columns.append("spent.txoid IS NOT NULL AS is_spent") if include_received_tips: select_columns.append(f"""( SELECT COALESCE(SUM(support.amount), 0) FROM txo AS support WHERE support.claim_id = txo.claim_id AND support.txo_type = {TXO_TYPES['support']} AND support.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) AND support.txoid NOT IN (SELECT txoid FROM txi) ) AS received_tips""") if 'order_by' not in constraints or constraints['order_by'] == 'height': constraints['order_by'] = [ "tx.height in (0, -1) DESC", "tx.height DESC", "tx.position DESC", "txo.position" ] elif constraints.get('order_by', None) == 'none': del constraints['order_by'] rows = await self.select_txos(', '.join(select_columns), read_only=read_only, **constraints) txos = [] txs = {} for row in rows: if no_tx: txo = Output( amount=row['amount'], script=OutputScript(row['script']), tx_ref=TXRefImmutable.from_id(row['txid'], row['height']), position=row['txo_position'] ) else: if row['txid'] not in txs: txs[row['txid']] = Transaction( row['raw'], height=row['height'], position=row['tx_position'], is_verified=bool(row['is_verified']) ) txo = txs[row['txid']].outputs[row['txo_position']] if include_is_spent: txo.is_spent = bool(row['is_spent']) if include_is_my_input: txo.is_my_input = bool(row['is_my_input']) if include_is_my_output: txo.is_my_output = bool(row['is_my_output']) if include_is_my_input and include_is_my_output: if txo.is_my_input and txo.is_my_output and row['txo_type'] == TXO_TYPES['other']: txo.is_internal_transfer = True else: txo.is_internal_transfer = False if include_received_tips: txo.received_tips = row['received_tips'] txos.append(txo) if not no_channel_info: channel_ids = set() for txo in txos: if txo.is_claim and txo.can_decode_claim: if txo.claim.is_signed: channel_ids.add(txo.claim.signing_channel_id) if txo.claim.is_channel and wallet: for account in wallet.accounts: private_key = await account.get_channel_private_key( txo.claim.channel.public_key_bytes ) if private_key: txo.private_key = private_key break if channel_ids: channels = { txo.claim_id: txo for txo in (await self.get_channels( wallet=wallet, claim_id__in=channel_ids, read_only=read_only )) } for txo in txos: if txo.is_claim and txo.can_decode_claim: txo.channel = channels.get(txo.claim.signing_channel_id, None) return txos @staticmethod def _clean_txo_constraints_for_aggregation(constraints): constraints.pop('include_is_spent', None) constraints.pop('include_is_my_input', None) constraints.pop('include_is_my_output', None) constraints.pop('include_received_tips', None) constraints.pop('wallet', None) constraints.pop('resolve', None) constraints.pop('offset', None) constraints.pop('limit', None) constraints.pop('order_by', None) async def get_txo_count(self, **constraints): self._clean_txo_constraints_for_aggregation(constraints) count = await self.select_txos('COUNT(*) AS total', **constraints) return count[0]['total'] or 0 async def get_txo_sum(self, **constraints): self._clean_txo_constraints_for_aggregation(constraints) result = await self.select_txos('SUM(amount) AS total', **constraints) return result[0]['total'] or 0 async def get_txo_plot(self, start_day=None, days_back=0, end_day=None, days_after=None, **constraints): self._clean_txo_constraints_for_aggregation(constraints) if start_day is None: constraints['day__gte'] = self.ledger.headers.estimated_julian_day( self.ledger.headers.height ) - days_back else: constraints['day__gte'] = date_to_julian_day( date.fromisoformat(start_day) ) if end_day is not None: constraints['day__lte'] = date_to_julian_day( date.fromisoformat(end_day) ) elif days_after is not None: constraints['day__lte'] = constraints['day__gte'] + days_after return await self.select_txos( "DATE(day) AS day, SUM(amount) AS total", group_by='day', order_by='day', **constraints ) def get_utxos(self, read_only=False, **constraints): return self.get_txos(is_spent=False, read_only=read_only, **constraints) def get_utxo_count(self, **constraints): return self.get_txo_count(is_spent=False, **constraints) async def get_balance(self, wallet=None, accounts=None, read_only=False, **constraints): assert wallet or accounts, \ "'wallet' or 'accounts' constraints required to calculate balance" constraints['accounts'] = accounts or wallet.accounts balance = await self.select_txos( 'SUM(amount) as total', is_spent=False, read_only=read_only, **constraints ) return balance[0]['total'] or 0 async def get_detailed_balance(self, accounts, read_only=False, **constraints): constraints['accounts'] = accounts result = (await self.select_txos( f"COALESCE(SUM(amount), 0) AS total," f"COALESCE(SUM(" f" CASE WHEN" f" txo_type NOT IN ({TXO_TYPES['other']}, {TXO_TYPES['purchase']})" f" THEN amount ELSE 0 END), 0) AS reserved," f"COALESCE(SUM(" f" CASE WHEN" f" txo_type IN ({','.join(map(str, CLAIM_TYPES))})" f" THEN amount ELSE 0 END), 0) AS claims," f"COALESCE(SUM(CASE WHEN txo_type = {TXO_TYPES['support']} THEN amount ELSE 0 END), 0) AS supports," f"COALESCE(SUM(" f" CASE WHEN" f" txo_type = {TXO_TYPES['support']} AND" f" TXI.address IS NOT NULL AND" f" TXI.address IN (SELECT address FROM account_address WHERE account = :$account__in0)" f" THEN amount ELSE 0 END), 0) AS my_supports", is_spent=False, include_is_my_input=True, read_only=read_only, **constraints ))[0] return { "total": result["total"], "available": result["total"] - result["reserved"], "reserved": result["reserved"], "reserved_subtotals": { "claims": result["claims"], "supports": result["my_supports"], "tips": result["supports"] - result["my_supports"] } } async def select_addresses(self, cols, read_only=False, **constraints): return await self.db.execute_fetchall(*query( f"SELECT {cols} FROM pubkey_address JOIN account_address USING (address)", **constraints ), read_only=read_only) async def get_addresses(self, cols=None, read_only=False, **constraints): cols = cols or ( 'address', 'account', 'chain', 'history', 'used_times', 'pubkey', 'chain_code', 'n', 'depth' ) addresses = await self.select_addresses(', '.join(cols), read_only=read_only, **constraints) if 'pubkey' in cols: for address in addresses: address['pubkey'] = PublicKey( self.ledger, address.pop('pubkey'), address.pop('chain_code'), address.pop('n'), address.pop('depth') ) return addresses async def get_address_count(self, cols=None, read_only=False, **constraints): self._clean_txo_constraints_for_aggregation(constraints) count = await self.select_addresses('COUNT(*) as total', read_only=read_only, **constraints) return count[0]['total'] or 0 async def get_address(self, read_only=False, **constraints): addresses = await self.get_addresses(read_only=read_only, limit=1, **constraints) if addresses: return addresses[0] async def add_keys(self, account, chain, pubkeys): await self.db.executemany( "insert or ignore into account_address " "(account, address, chain, pubkey, chain_code, n, depth) values " "(?, ?, ?, ?, ?, ?, ?)", (( account.id, k.address, chain, sqlite3.Binary(k.pubkey_bytes), sqlite3.Binary(k.chain_code), k.n, k.depth ) for k in pubkeys) ) await self.db.executemany( "insert or ignore into pubkey_address (address) values (?)", ((pubkey.address,) for pubkey in pubkeys) ) async def _set_address_history(self, address, history): await self.db.execute_fetchall( "UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?", (history, history.count(':')//2, address) ) async def set_address_history(self, address, history): await self._set_address_history(address, history) async def is_channel_key_used(self, account, key: PublicKey): channels = await self.get_txos( accounts=[account], txo_type=TXO_TYPES['channel'], no_tx=True, no_channel_info=True ) other_key_bytes = key.pubkey_bytes for channel in channels: claim = channel.can_decode_claim if claim and claim.channel.public_key_bytes == other_key_bytes: return True return False @staticmethod def constrain_purchases(constraints): accounts = constraints.pop('accounts', None) assert accounts, "'accounts' argument required to find purchases" if not {'purchased_claim_id', 'purchased_claim_id__in'}.intersection(constraints): constraints['purchased_claim_id__is_not_null'] = True constraints.update({ f'$account{i}': a.public_key.address for i, a in enumerate(accounts) }) account_values = ', '.join([f':$account{i}' for i in range(len(accounts))]) constraints['txid__in'] = f""" SELECT txid FROM txi JOIN account_address USING (address) WHERE account_address.account IN ({account_values}) """ async def get_purchases(self, **constraints): self.constrain_purchases(constraints) return [tx.outputs[0] for tx in await self.get_transactions(**constraints)] def get_purchase_count(self, **constraints): self.constrain_purchases(constraints) return self.get_transaction_count(**constraints) @staticmethod def constrain_claims(constraints): if {'txo_type', 'txo_type__in'}.intersection(constraints): return claim_types = constraints.pop('claim_type', None) if claim_types: constrain_single_or_list( constraints, 'txo_type', claim_types, lambda x: TXO_TYPES[x] ) else: constraints['txo_type__in'] = CLAIM_TYPES async def get_claims(self, read_only=False, **constraints) -> List[Output]: self.constrain_claims(constraints) return await self.get_utxos(read_only=read_only, **constraints) def get_claim_count(self, **constraints): self.constrain_claims(constraints) return self.get_utxo_count(**constraints) @staticmethod def constrain_streams(constraints): constraints['txo_type'] = TXO_TYPES['stream'] def get_streams(self, read_only=False, **constraints): self.constrain_streams(constraints) return self.get_claims(read_only=read_only, **constraints) def get_stream_count(self, **constraints): self.constrain_streams(constraints) return self.get_claim_count(**constraints) @staticmethod def constrain_channels(constraints): constraints['txo_type'] = TXO_TYPES['channel'] def get_channels(self, **constraints): self.constrain_channels(constraints) return self.get_claims(**constraints) def get_channel_count(self, **constraints): self.constrain_channels(constraints) return self.get_claim_count(**constraints) @staticmethod def constrain_supports(constraints): constraints['txo_type'] = TXO_TYPES['support'] def get_supports(self, **constraints): self.constrain_supports(constraints) return self.get_utxos(**constraints) def get_support_count(self, **constraints): self.constrain_supports(constraints) return self.get_utxo_count(**constraints) @staticmethod def constrain_collections(constraints): constraints['txo_type'] = TXO_TYPES['collection'] def get_collections(self, **constraints): self.constrain_collections(constraints) return self.get_utxos(**constraints) def get_collection_count(self, **constraints): self.constrain_collections(constraints) return self.get_utxo_count(**constraints) async def release_all_outputs(self, account=None): if account is None: await self.db.execute_fetchall("UPDATE txo SET is_reserved = 0 WHERE is_reserved = 1") else: await self.db.execute_fetchall( "UPDATE txo SET is_reserved = 0 WHERE" " is_reserved = 1 AND txo.address IN (" " SELECT address from account_address WHERE account = ?" " )", (account.public_key.address, ) ) def get_supports_summary(self, read_only=False, **constraints): return self.get_txos( txo_type=TXO_TYPES['support'], is_spent=False, is_my_output=True, include_is_my_input=True, no_tx=True, read_only=read_only, **constraints ) ================================================ FILE: lbry/wallet/dewies.py ================================================ import textwrap from .util import coins_to_satoshis, satoshis_to_coins def lbc_to_dewies(lbc: str) -> int: try: return coins_to_satoshis(lbc) except ValueError: raise ValueError(textwrap.dedent( f""" Decimal inputs require a value in the ones place and in the tenths place separated by a period. The value provided, '{lbc}', is not of the correct format. The following are examples of valid decimal inputs: 1.0 0.001 2.34500 4534.4 2323434.0000 The following are NOT valid: 83 .456 123. """ )) def dewies_to_lbc(dewies) -> str: return satoshis_to_coins(dewies) def dict_values_to_lbc(d): lbc_dict = {} for key, value in d.items(): if isinstance(value, int): lbc_dict[key] = dewies_to_lbc(value) elif isinstance(value, dict): lbc_dict[key] = dict_values_to_lbc(value) else: lbc_dict[key] = value return lbc_dict ================================================ FILE: lbry/wallet/hash.py ================================================ from binascii import hexlify, unhexlify from .constants import NULL_HASH32 class TXRef: __slots__ = '_id', '_hash' def __init__(self): self._id = None self._hash = None @property def id(self): return self._id @property def hash(self): return self._hash @property def height(self): return -1 @property def is_null(self): return self.hash == NULL_HASH32 class TXRefImmutable(TXRef): __slots__ = ('_height',) def __init__(self): super().__init__() self._height = -1 @classmethod def from_hash(cls, tx_hash: bytes, height: int) -> 'TXRefImmutable': ref = cls() ref._hash = tx_hash ref._id = hexlify(tx_hash[::-1]).decode() ref._height = height return ref @classmethod def from_id(cls, tx_id: str, height: int) -> 'TXRefImmutable': ref = cls() ref._id = tx_id ref._hash = unhexlify(tx_id)[::-1] ref._height = height return ref @property def height(self): return self._height ================================================ FILE: lbry/wallet/header.py ================================================ import base64 import os import struct import asyncio import logging import zlib from datetime import date from io import BytesIO from typing import Optional, Iterator, Tuple, Callable from binascii import hexlify, unhexlify from lbry.crypto.hash import sha512, double_sha256, ripemd160 from lbry.wallet.util import ArithUint256, date_to_julian_day from .checkpoints import HASHES log = logging.getLogger(__name__) class InvalidHeader(Exception): def __init__(self, height, message): super().__init__(message) self.message = message self.height = height class Headers: header_size = 112 chunk_size = 10**16 max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' target_timespan = 150 checkpoints = HASHES first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot timestamp_average_offset = 160.6855883050695 # calculated at 733447 validate_difficulty: bool = True def __init__(self, path) -> None: self.io = None self.path = path self._size: Optional[int] = None self.chunk_getter: Optional[Callable] = None self.known_missing_checkpointed_chunks = set() self.check_chunk_lock = asyncio.Lock() async def open(self): self.io = BytesIO() if self.path != ':memory:': def _readit(): if os.path.exists(self.path): with open(self.path, 'r+b') as header_file: self.io.seek(0) self.io.write(header_file.read()) await asyncio.get_event_loop().run_in_executor(None, _readit) bytes_size = self.io.seek(0, os.SEEK_END) self._size = bytes_size // self.header_size max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000 if bytes_size % self.header_size: log.warning("Reader file size doesnt match header size. Repairing, might take a while.") await self.repair() else: # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok) await self.repair(start_height=max_checkpointed_height) await self.ensure_checkpointed_size() await self.get_all_missing_headers() async def close(self): if self.io is not None: def _close(): flags = 'r+b' if os.path.exists(self.path) else 'w+b' with open(self.path, flags) as header_file: header_file.write(self.io.getbuffer()) await asyncio.get_event_loop().run_in_executor(None, _close) self.io.close() self.io = None @staticmethod def serialize(header): return b''.join([ struct.pack('<I', header['version']), unhexlify(header['prev_block_hash'])[::-1], unhexlify(header['merkle_root'])[::-1], unhexlify(header['claim_trie_root'])[::-1], struct.pack('<III', header['timestamp'], header['bits'], header['nonce']) ]) @staticmethod def deserialize(height, header): version, = struct.unpack('<I', header[:4]) timestamp, bits, nonce = struct.unpack('<III', header[100:112]) return { 'version': version, 'prev_block_hash': hexlify(header[4:36][::-1]), 'merkle_root': hexlify(header[36:68][::-1]), 'claim_trie_root': hexlify(header[68:100][::-1]), 'timestamp': timestamp, 'bits': bits, 'nonce': nonce, 'block_height': height, } def get_next_chunk_target(self, chunk: int) -> ArithUint256: return ArithUint256(self.max_target) def get_next_block_target(self, max_target: ArithUint256, previous: Optional[dict], current: Optional[dict]) -> ArithUint256: # https://github.com/lbryio/lbrycrd/blob/master/src/lbry.cpp if previous is None and current is None: return max_target if previous is None: previous = current actual_timespan = current['timestamp'] - previous['timestamp'] modulated_timespan = self.target_timespan + int((actual_timespan - self.target_timespan) / 8) minimum_timespan = self.target_timespan - int(self.target_timespan / 8) # 150 - 18 = 132 maximum_timespan = self.target_timespan + int(self.target_timespan / 2) # 150 + 75 = 225 clamped_timespan = max(minimum_timespan, min(modulated_timespan, maximum_timespan)) target = ArithUint256.from_compact(current['bits']) new_target = min(max_target, (target * clamped_timespan) / self.target_timespan) return new_target def __len__(self) -> int: return self._size def __bool__(self): return True async def get(self, height) -> dict: if isinstance(height, slice): raise NotImplementedError("Slicing of header chain has not been implemented yet.") try: return self.deserialize(height, await self.get_raw_header(height)) except struct.error: raise IndexError(f"failed to get {height}, at {len(self)}") def estimated_timestamp(self, height, try_real_headers=True): if height <= 0: return if try_real_headers and self.has_header(height): offset = height * self.header_size return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0] return int(self.first_block_timestamp + (height * self.timestamp_average_offset)) def estimated_julian_day(self, height): return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False))) async def get_raw_header(self, height) -> bytes: if self.chunk_getter: await self.ensure_chunk_at(height) if not 0 <= height <= self.height: raise IndexError(f"{height} is out of bounds, current height: {self.height}") return self._read(height) def _read(self, height, count=1): offset = height * self.header_size return bytes(self.io.getbuffer()[offset: offset + self.header_size * count]) def chunk_hash(self, start, count): return self.hash_header(self._read(start, count)).decode() async def ensure_checkpointed_size(self): max_checkpointed_height = max(self.checkpoints.keys() or [-1]) if self.height < max_checkpointed_height: self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000)) async def ensure_chunk_at(self, height): async with self.check_chunk_lock: if self.has_header(height): log.debug("has header %s", height) return return await self.fetch_chunk(height) async def fetch_chunk(self, height): log.info("on-demand fetching height %s", height) start = (height // 1000) * 1000 headers = await self.chunk_getter(start) # pylint: disable=not-callable chunk = ( zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000) ) chunk_hash = self.hash_header(chunk).decode() if self.checkpoints.get(start) == chunk_hash: self._write(start, chunk) if start in self.known_missing_checkpointed_chunks: self.known_missing_checkpointed_chunks.remove(start) return elif start not in self.checkpoints: return # todo: fixme raise Exception( f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead." ) def has_header(self, height): normalized_height = (height // 1000) * 1000 if normalized_height in self.checkpoints: return normalized_height not in self.known_missing_checkpointed_chunks empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' return self.chunk_hash(height, 1) not in (empty, all_zeroes) async def get_all_missing_headers(self): # Heavy operation done in one optimized shot for chunk_height, expected_hash in reversed(list(self.checkpoints.items())): if chunk_height in self.known_missing_checkpointed_chunks: continue if self.chunk_hash(chunk_height, 1000) != expected_hash: self.known_missing_checkpointed_chunks.add(chunk_height) return self.known_missing_checkpointed_chunks @property def height(self) -> int: return len(self)-1 @property def bytes_size(self): return len(self) * self.header_size async def hash(self, height=None) -> bytes: return self.hash_header( await self.get_raw_header(height if height is not None else self.height) ) @staticmethod def hash_header(header: bytes) -> bytes: if header is None: return b'0' * 64 return hexlify(double_sha256(header)[::-1]) async def connect(self, start: int, headers: bytes) -> int: added = 0 bail = False for height, chunk in self._iterate_chunks(start, headers): try: # validate_chunk() is CPU bound and reads previous chunks from file system await self.validate_chunk(height, chunk) except InvalidHeader as e: bail = True chunk = chunk[:(height-e.height)*self.header_size] if chunk: added += self._write(height, chunk) if bail: break return added def _write(self, height, verified_chunk): self.io.seek(height * self.header_size, os.SEEK_SET) written = self.io.write(verified_chunk) // self.header_size # self.io.truncate() # .seek()/.write()/.truncate() might also .flush() when needed # the goal here is mainly to ensure we're definitely flush()'ing self.io.flush() self._size = max(self._size or 0, self.io.tell() // self.header_size) return written async def validate_chunk(self, height, chunk): previous_hash, previous_header, previous_previous_header = None, None, None if height > 0: raw = await self.get_raw_header(height-1) previous_header = self.deserialize(height-1, raw) previous_hash = self.hash_header(raw) if height > 1: previous_previous_header = await self.get(height-2) chunk_target = self.get_next_chunk_target(height // 2016 - 1) for current_hash, current_header in self._iterate_headers(height, chunk): block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header) self.validate_header(height, current_hash, current_header, previous_hash, block_target) previous_previous_header = previous_header previous_header = current_header previous_hash = current_hash def validate_header(self, height: int, current_hash: bytes, header: dict, previous_hash: bytes, target: ArithUint256): if previous_hash is None: if self.genesis_hash is not None and self.genesis_hash != current_hash: raise InvalidHeader( height, f"genesis header doesn't match: {current_hash.decode()} " f"vs expected {self.genesis_hash.decode()}") return if header['prev_block_hash'] != previous_hash: raise InvalidHeader( height, "previous hash mismatch: {} vs expected {}".format( header['prev_block_hash'].decode(), previous_hash.decode()) ) if self.validate_difficulty: if header['bits'] != target.compact: raise InvalidHeader( height, "bits mismatch: {} vs expected {}".format( header['bits'], target.compact) ) proof_of_work = self.get_proof_of_work(current_hash) if proof_of_work > target: raise InvalidHeader( height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}" ) async def repair(self, start_height=0): previous_header_hash = fail = None batch_size = 36 for height in range(start_height, self.height, batch_size): headers = self._read(height, batch_size) if len(headers) % self.header_size != 0: headers = headers[:(len(headers) // self.header_size) * self.header_size] for header_hash, header in self._iterate_headers(height, headers): height = header['block_height'] if previous_header_hash: if header['prev_block_hash'] != previous_header_hash: fail = True elif height == 0: if header_hash != self.genesis_hash: fail = True else: # for sanity and clarity, since it is the only way we can end up here assert start_height > 0 and height == start_height if fail: log.warning("Header file corrupted at height %s, truncating it.", height - 1) self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET) self.io.truncate() self.io.flush() self._size = self.io.seek(0, os.SEEK_END) // self.header_size return previous_header_hash = header_hash @classmethod def get_proof_of_work(cls, header_hash: bytes): return ArithUint256(int(b'0x' + cls.header_hash_to_pow_hash(header_hash), 16)) def _iterate_chunks(self, height: int, headers: bytes) -> Iterator[Tuple[int, bytes]]: assert len(headers) % self.header_size == 0, f"{len(headers)} {len(headers)%self.header_size}" start = 0 end = (self.chunk_size - height % self.chunk_size) * self.header_size while start < end: yield height + (start // self.header_size), headers[start:end] start = end end = min(len(headers), end + self.chunk_size * self.header_size) def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]: assert len(headers) % self.header_size == 0, len(headers) for idx in range(len(headers) // self.header_size): start, end = idx * self.header_size, (idx + 1) * self.header_size header = headers[start:end] yield self.hash_header(header), self.deserialize(height+idx, header) @staticmethod def header_hash_to_pow_hash(header_hash: bytes): header_hash_bytes = unhexlify(header_hash)[::-1] h = sha512(header_hash_bytes) pow_hash = double_sha256( ripemd160(h[:len(h) // 2]) + ripemd160(h[len(h) // 2:]) ) return hexlify(pow_hash[::-1]) class UnvalidatedHeaders(Headers): validate_difficulty = False max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556' checkpoints = {} ================================================ FILE: lbry/wallet/ledger.py ================================================ import os import copy import time import asyncio import logging from datetime import datetime from functools import partial from operator import itemgetter from collections import defaultdict from binascii import hexlify, unhexlify from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple from lbry.schema.result import Outputs, INVALID, NOT_FOUND from lbry.schema.url import URL from lbry.crypto.hash import hash160, double_sha256, sha256 from lbry.crypto.base58 import Base58 from lbry.utils import LRUCacheWithMetrics from lbry.wallet.tasks import TaskGroup from lbry.wallet.database import Database from lbry.wallet.stream import StreamController from lbry.wallet.dewies import dewies_to_lbc from lbry.wallet.account import Account, AddressManager, SingleKey from lbry.wallet.network import Network from lbry.wallet.transaction import Transaction, Output from lbry.wallet.header import Headers, UnvalidatedHeaders from lbry.wallet.checkpoints import HASHES from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32 from lbry.wallet.bip32 import PublicKey, PrivateKey from lbry.wallet.coinselection import CoinSelector log = logging.getLogger(__name__) LedgerType = Type['BaseLedger'] class LedgerRegistry(type): ledgers: Dict[str, LedgerType] = {} def __new__(mcs, name, bases, attrs): cls: LedgerType = super().__new__(mcs, name, bases, attrs) if not (name == 'BaseLedger' and not bases): ledger_id = cls.get_id() assert ledger_id not in mcs.ledgers, \ f'Ledger with id "{ledger_id}" already registered.' mcs.ledgers[ledger_id] = cls return cls @classmethod def get_ledger_class(mcs, ledger_id: str) -> LedgerType: return mcs.ledgers[ledger_id] class TransactionEvent(NamedTuple): address: str tx: Transaction class AddressesGeneratedEvent(NamedTuple): address_manager: AddressManager addresses: List[str] class BlockHeightEvent(NamedTuple): height: int change: int class TransactionCacheItem: __slots__ = '_tx', 'lock', 'has_tx', 'pending_verifications' def __init__(self, tx: Optional[Transaction] = None, lock: Optional[asyncio.Lock] = None): self.has_tx = asyncio.Event() self.lock = lock or asyncio.Lock() self._tx = self.tx = tx self.pending_verifications = 0 @property def tx(self) -> Optional[Transaction]: return self._tx @tx.setter def tx(self, tx: Transaction): self._tx = tx if tx is not None: self.has_tx.set() class Ledger(metaclass=LedgerRegistry): name = 'LBRY Credits' symbol = 'LBC' network_name = 'mainnet' headers_class = Headers secret_prefix = bytes((0x1c,)) pubkey_address_prefix = bytes((0x55,)) script_address_prefix = bytes((0x7a,)) extended_public_key_prefix = unhexlify('0488b21e') extended_private_key_prefix = unhexlify('0488ade4') max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' genesis_bits = 0x1f00ffff target_timespan = 150 default_fee_per_byte = 50 default_fee_per_name_char = 0 checkpoints = HASHES def __init__(self, config=None): self.config = config or {} self.db: Database = self.config.get('db') or Database( os.path.join(self.path, "blockchain.db") ) self.db.ledger = self self.headers: Headers = self.config.get('headers') or self.headers_class( os.path.join(self.path, "headers") ) self.headers.checkpoints = self.checkpoints self.network: Network = self.config.get('network') or Network(self) self.network.on_header.listen(self.receive_header) self.network.on_status.listen(self.process_status_update) self.accounts = [] self.fee_per_byte: int = self.config.get('fee_per_byte', self.default_fee_per_byte) self._on_transaction_controller = StreamController() self.on_transaction = self._on_transaction_controller.stream self.on_transaction.listen( lambda e: log.info( '(%s) on_transaction: address=%s, height=%s, is_verified=%s, tx.id=%s', self.get_id(), e.address, e.tx.height, e.tx.is_verified, e.tx.id ) ) self._on_address_controller = StreamController() self.on_address = self._on_address_controller.stream self.on_address.listen( lambda e: log.info('(%s) on_address: %s', self.get_id(), e.addresses) ) self._on_header_controller = StreamController() self.on_header = self._on_header_controller.stream self.on_header.listen( lambda change: log.info( '%s: added %s header blocks, final height %s', self.get_id(), change, self.headers.height ) ) self._download_height = 0 self._on_ready_controller = StreamController() self.on_ready = self._on_ready_controller.stream self._tx_cache = LRUCacheWithMetrics(self.config.get("tx_cache_size", 1024), metric_name='tx') self._update_tasks = TaskGroup() self._other_tasks = TaskGroup() # that we dont need to start self._utxo_reservation_lock = asyncio.Lock() self._header_processing_lock = asyncio.Lock() self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock) self._history_lock = asyncio.Lock() self.coin_selection_strategy = None self._known_addresses_out_of_sync = set() self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char) self._balance_cache = LRUCacheWithMetrics(2 ** 15) @classmethod def get_id(cls): return '{}_{}'.format(cls.symbol.lower(), cls.network_name.lower()) @classmethod def hash160_to_address(cls, h160): raw_address = cls.pubkey_address_prefix + h160 return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4])) @classmethod def hash160_to_script_address(cls, h160): raw_address = cls.script_address_prefix + h160 return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4])) @staticmethod def address_to_hash160(address): return Base58.decode(address)[1:21] @classmethod def is_pubkey_address(cls, address): decoded = Base58.decode_check(address) return decoded[0] == cls.pubkey_address_prefix[0] @classmethod def is_script_address(cls, address): decoded = Base58.decode_check(address) return decoded[0] == cls.script_address_prefix[0] @classmethod def public_key_to_address(cls, public_key): return cls.hash160_to_address(hash160(public_key)) @staticmethod def private_key_to_wif(private_key): return b'\x1c' + private_key + b'\x01' @property def path(self): return os.path.join(self.config['data_path'], self.get_id()) def add_account(self, account: Account): self.accounts.append(account) async def _get_account_and_address_info_for_address(self, wallet, address): match = await self.db.get_address(accounts=wallet.accounts, address=address) if match: for account in wallet.accounts: if match['account'] == account.public_key.address: return account, match async def get_private_key_for_address(self, wallet, address) -> Optional[PrivateKey]: match = await self._get_account_and_address_info_for_address(wallet, address) if match: account, address_info = match return account.get_private_key(address_info['chain'], address_info['pubkey'].n) return None async def get_public_key_for_address(self, wallet, address) -> Optional[PublicKey]: match = await self._get_account_and_address_info_for_address(wallet, address) if match: _, address_info = match return address_info['pubkey'] return None async def get_account_for_address(self, wallet, address): match = await self._get_account_and_address_info_for_address(wallet, address) if match: return match[0] async def get_effective_amount_estimators(self, funding_accounts: Iterable[Account]): estimators = [] for account in funding_accounts: utxos = await account.get_utxos(no_tx=True, no_channel_info=True) for utxo in utxos: estimators.append(utxo.get_estimator(self)) return estimators async def get_addresses(self, **constraints): return await self.db.get_addresses(**constraints) def get_address_count(self, **constraints): return self.db.get_address_count(**constraints) async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']], min_amount=1): min_amount = min(amount // 10, min_amount) fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self) selector = CoinSelector(amount, fee) async with self._utxo_reservation_lock: if self.coin_selection_strategy == 'sqlite': return await self.db.get_spendable_utxos(self, amount + fee, funding_accounts, min_amount=min_amount, fee_per_byte=self.fee_per_byte) txos = await self.get_effective_amount_estimators(funding_accounts) spendables = selector.select(txos, self.coin_selection_strategy) if spendables: await self.reserve_outputs(s.txo for s in spendables) return spendables def reserve_outputs(self, txos): return self.db.reserve_outputs(txos) def release_outputs(self, txos): return self.db.release_outputs(txos) def release_tx(self, tx): return self.release_outputs([txi.txo_ref.txo for txi in tx.inputs]) def get_utxos(self, **constraints): self.constraint_spending_utxos(constraints) return self.db.get_utxos(**constraints) def get_utxo_count(self, **constraints): self.constraint_spending_utxos(constraints) return self.db.get_utxo_count(**constraints) async def get_txos(self, resolve=False, **constraints) -> List[Output]: txos = await self.db.get_txos(**constraints) if resolve: return await self._resolve_for_local_results(constraints.get('accounts', []), txos) return txos def get_txo_count(self, **constraints): return self.db.get_txo_count(**constraints) def get_txo_sum(self, **constraints): return self.db.get_txo_sum(**constraints) def get_txo_plot(self, **constraints): return self.db.get_txo_plot(**constraints) def get_transactions(self, **constraints): return self.db.get_transactions(**constraints) def get_transaction_count(self, **constraints): return self.db.get_transaction_count(**constraints) async def get_local_status_and_history(self, address, history=None): if not history: address_details = await self.db.get_address(address=address) history = (address_details['history'] if address_details else '') or '' parts = history.split(':')[:-1] return ( hexlify(sha256(history.encode())).decode() if history else None, list(zip(parts[0::2], map(int, parts[1::2]))) ) @staticmethod def get_root_of_merkle_tree(branches, branch_positions, working_branch): for i, branch in enumerate(branches): other_branch = unhexlify(branch)[::-1] other_branch_on_left = bool((branch_positions >> i) & 1) if other_branch_on_left: combined = other_branch + working_branch else: combined = working_branch + other_branch working_branch = double_sha256(combined) return hexlify(working_branch[::-1]) async def start(self): if not os.path.exists(self.path): os.mkdir(self.path) await asyncio.wait(map(asyncio.create_task, [ self.db.open(), self.headers.open() ])) fully_synced = self.on_ready.first asyncio.create_task(self.network.start()) await self.network.on_connected.first async with self._header_processing_lock: await self._update_tasks.add(self.initial_headers_sync()) self.network.on_connected.listen(self.join_network) asyncio.ensure_future(self.join_network()) await fully_synced await self.db.release_all_outputs() await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts)) await asyncio.gather(*(a.save_max_gap() for a in self.accounts)) if len(self.accounts) > 10: log.info("Loaded %i accounts", len(self.accounts)) else: await self._report_state() self.on_transaction.listen(self._reset_balance_cache) async def join_network(self, *_): log.info("Subscribing and updating accounts.") await self._update_tasks.add(self.subscribe_accounts()) await self._update_tasks.done.wait() self._on_ready_controller.add(True) async def stop(self): self._update_tasks.cancel() self._other_tasks.cancel() await self._update_tasks.done.wait() await self._other_tasks.done.wait() await self.network.stop() await self.db.close() await self.headers.close() async def tasks_are_done(self): await self._update_tasks.done.wait() await self._other_tasks.done.wait() @property def local_height_including_downloaded_height(self): return max(self.headers.height, self._download_height) async def initial_headers_sync(self): get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=1000, b64=True) self.headers.chunk_getter = get_chunk async def doit(): for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)): async with self._header_processing_lock: await self.headers.ensure_chunk_at(height) self._other_tasks.add(doit()) await self.update_headers() async def update_headers(self, height=None, headers=None, subscription_update=False): rewound = 0 while True: if height is None or height > len(self.headers): # sometimes header subscription updates are for a header in the future # which can't be connected, so we do a normal header sync instead height = len(self.headers) headers = None subscription_update = False if not headers: header_response = await self.network.retriable_call(self.network.get_headers, height, 2001) headers = header_response['hex'] if not headers: # Nothing to do, network thinks we're already at the latest height. return added = await self.headers.connect(height, unhexlify(headers)) if added > 0: height += added self._on_header_controller.add( BlockHeightEvent(self.headers.height, added)) if rewound > 0: # we started rewinding blocks and apparently found # a new chain rewound = 0 await self.db.rewind_blockchain(height) if subscription_update: # subscription updates are for latest header already # so we don't need to check if there are newer / more # on another loop of update_headers(), just return instead return elif added == 0: # we had headers to connect but none got connected, probably a reorganization height -= 1 rewound += 1 log.warning( "Blockchain Reorganization: attempting rewind to height %s from starting height %s", height, height+rewound ) self._tx_cache.clear() else: raise IndexError(f"headers.connect() returned negative number ({added})") if height < 0: raise IndexError( "Blockchain reorganization rewound all the way back to genesis hash. " "Something is very wrong. Maybe you are on the wrong blockchain?" ) if rewound >= 100: raise IndexError( "Blockchain reorganization dropped {} headers. This is highly unusual. " "Will not continue to attempt reorganizing. Please, delete the ledger " "synchronization directory inside your wallet directory (folder: '{}') and " "restart the program to synchronize from scratch." .format(rewound, self.get_id()) ) headers = None # ready to download some more headers # if we made it this far and this was a subscription_update # it means something went wrong and now we're doing a more # robust sync, turn off subscription update shortcut subscription_update = False async def receive_header(self, response): async with self._header_processing_lock: header = response[0] await self.update_headers( height=header['height'], headers=header['hex'], subscription_update=True ) async def subscribe_accounts(self): if self.network.is_connected and self.accounts: log.info("Subscribe to %i accounts", len(self.accounts)) await asyncio.wait(map(asyncio.create_task, [ self.subscribe_account(a) for a in self.accounts ])) async def subscribe_account(self, account: Account): for address_manager in account.address_managers.values(): await self.subscribe_addresses(address_manager, await address_manager.get_addresses()) await account.ensure_address_gap() await account.deterministic_channel_keys.ensure_cache_primed() async def unsubscribe_account(self, account: Account): for address in await account.get_addresses(): await self.network.unsubscribe_address(address) async def announce_addresses(self, address_manager: AddressManager, addresses: List[str]): await self.subscribe_addresses(address_manager, addresses) await self._on_address_controller.add( AddressesGeneratedEvent(address_manager, addresses) ) async def subscribe_addresses(self, address_manager: AddressManager, addresses: List[str], batch_size: int = 1000): if self.network.is_connected and addresses: addresses_remaining = list(addresses) while addresses_remaining: batch = addresses_remaining[:batch_size] results = await self.network.subscribe_address(*batch) for address, remote_status in zip(batch, results): self._update_tasks.add(self.update_history(address, remote_status, address_manager)) addresses_remaining = addresses_remaining[batch_size:] if self.network.client and self.network.client.server_address_and_port: log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining), len(addresses), *self.network.client.server_address_and_port) if self.network.client and self.network.client.server_address_and_port: log.info( "finished subscribing to %i addresses on %s:%i", len(addresses), *self.network.client.server_address_and_port ) def process_status_update(self, update): address, remote_status = update self._update_tasks.add(self.update_history(address, remote_status)) async def update_history(self, address, remote_status, address_manager: AddressManager = None, reattempt_update: bool = True): async with self._address_update_locks[address]: self._known_addresses_out_of_sync.discard(address) local_status, local_history = await self.get_local_status_and_history(address) if local_status == remote_status: return True remote_history = await self.network.retriable_call(self.network.get_history, address) remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history)) we_need = set(remote_history) - set(local_history) if not we_need: remote_missing = set(local_history) - set(remote_history) if remote_missing: log.warning( "%i transactions we have for %s are not in the remote address history", len(remote_missing), address ) return True to_request = {} pending_synced_history = {} already_synced = set() already_synced_offset = 0 for i, (txid, remote_height) in enumerate(remote_history): if i == already_synced_offset and i < len(local_history) and local_history[i] == (txid, remote_height): pending_synced_history[i] = f'{txid}:{remote_height}:' already_synced.add((txid, remote_height)) already_synced_offset += 1 continue tx_indexes = {} for i, (txid, remote_height) in enumerate(remote_history): tx_indexes[txid] = i if (txid, remote_height) in already_synced: continue to_request[i] = (txid, remote_height) log.debug( "request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced), len(remote_history), address ) remote_history_txids = {txid for txid, _ in remote_history} async for tx in self.request_synced_transactions(to_request, remote_history_txids, address): self.maybe_has_channel_key(tx) pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:" if len(pending_synced_history) % 100 == 0: log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request)) log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request)) assert len(pending_synced_history) == len(remote_history), \ f"{len(pending_synced_history)} vs {len(remote_history)} for {address}" synced_history = "" for remote_i, i in zip(range(len(remote_history)), sorted(pending_synced_history.keys())): assert i == remote_i, f"{i} vs {remote_i}" txid, height = remote_history[remote_i] if f"{txid}:{height}:" != pending_synced_history[i]: log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i]) synced_history += pending_synced_history[i] await self.db.set_address_history(address, synced_history) if address_manager is None: address_manager = await self.get_address_manager_for_address(address) if address_manager is not None: await address_manager.ensure_address_gap() local_status, local_history = \ await self.get_local_status_and_history(address, synced_history) if local_status != remote_status: if local_history == remote_history: log.warning( "%s has a synced history but a mismatched status", address ) return True remote_set = set(remote_history) local_set = set(local_history) log.warning( "%s is out of sync after syncing.\n" "Remote: %s with %d items (%i unique), local: %s with %d items (%i unique).\n" "Histories are mismatched on %i items.\n" "Local is missing\n" "%s\n" "Remote is missing\n" "%s\n" "******", address, remote_status, len(remote_history), len(remote_set), local_status, len(local_history), len(local_set), len(remote_set.symmetric_difference(local_set)), "\n".join([f"{txid} - {height}" for txid, height in local_set.difference(remote_set)]), "\n".join([f"{txid} - {height}" for txid, height in remote_set.difference(local_set)]) ) self._known_addresses_out_of_sync.add(address) return False else: log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history)) return True async def maybe_verify_transaction(self, tx, remote_height, merkle=None): tx.height = remote_height if 0 < remote_height < len(self.headers): # can't be tx.pending_verifications == 1 because we have to handle the transaction_show case if not merkle: merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height) if 'merkle' not in merkle: return merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash) header = await self.headers.get(remote_height) tx.position = merkle['pos'] tx.is_verified = merkle_root == header['merkle_root'] return tx def maybe_has_channel_key(self, tx): for txo in tx._outputs: if txo.can_decode_claim and txo.claim.is_channel: for account in self.accounts: account.deterministic_channel_keys.maybe_generate_deterministic_key_for_channel(txo) async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], cached=False): batches = [[]] remote_heights = {} cache_hits = set() for txid, height in sorted(to_request, key=lambda x: x[1]): if cached: cached_tx = self._tx_cache.get(txid) if cached_tx is not None: if cached_tx.tx is not None and cached_tx.tx.is_verified: cache_hits.add(txid) continue else: self._tx_cache[txid] = TransactionCacheItem() remote_heights[txid] = height if len(batches[-1]) == 100: batches.append([]) batches[-1].append(txid) if not batches[-1]: batches.pop() if cached and cache_hits: yield {txid: self._tx_cache[txid].tx for txid in cache_hits} for batch in batches: txs = await self._single_batch(batch, remote_heights) if cached: for txid, tx in txs.items(): self._tx_cache[txid].tx = tx yield txs async def request_synced_transactions(self, to_request, remote_history, address): async for txs in self.request_transactions(((txid, height) for txid, height in to_request.values())): for tx in txs.values(): yield tx await self._sync_and_save_batch(address, remote_history, txs) async def _single_batch(self, batch, remote_heights): heights = {remote_heights[txid] for txid in batch} unrestriced = 0 < min(heights) < max(heights) < max(self.headers.checkpoints or [0]) batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, not unrestriced) txs = {} for txid, (raw, merkle) in batch_result.items(): remote_height = remote_heights[txid] tx = Transaction(unhexlify(raw), height=remote_height) txs[tx.id] = tx await self.maybe_verify_transaction(tx, remote_height, merkle) return txs async def _sync_and_save_batch(self, address, remote_history, pending_txs): await asyncio.gather(*(self._sync(tx, remote_history, pending_txs) for tx in pending_txs.values())) await self.db.save_transaction_io_batch( pending_txs.values(), address, self.address_to_hash160(address), "" ) while pending_txs: self._on_transaction_controller.add(TransactionEvent(address, pending_txs.popitem()[1])) async def _sync(self, tx, remote_history, pending_txs): check_db_for_txos = {} for txi in tx.inputs: if txi.txo_ref.txo is not None: continue wanted_txid = txi.txo_ref.tx_ref.id if wanted_txid not in remote_history: continue if wanted_txid in pending_txs: txi.txo_ref = pending_txs[wanted_txid].outputs[txi.txo_ref.position].ref else: check_db_for_txos[txi] = txi.txo_ref.id referenced_txos = {} if not check_db_for_txos else { txo.id: txo for txo in await self.db.get_txos( txoid__in=list(check_db_for_txos.values()), order_by='txo.txoid', no_tx=True ) } for txi in check_db_for_txos: if txi.txo_ref.id in referenced_txos: txi.txo_ref = referenced_txos[txi.txo_ref.id].ref else: tx_from_db = await self.db.get_transaction(txid=txi.txo_ref.tx_ref.id) if tx_from_db is None: log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id) else: txi.txo_ref = tx_from_db.outputs[txi.txo_ref.position].ref return tx async def get_address_manager_for_address(self, address) -> Optional[AddressManager]: details = await self.db.get_address(address=address) for account in self.accounts: if account.id == details['account']: return account.address_managers[details['chain']] return None async def broadcast_or_release(self, tx, blocking=False): try: await self.broadcast(tx) except: await self.release_tx(tx) raise if blocking: await self.wait(tx, timeout=None) def broadcast(self, tx): # broadcast can't be a retriable call yet return self.network.broadcast(hexlify(tx.raw).decode()) async def wait(self, tx: Transaction, height=-1, timeout=1): timeout = timeout or 600 # after 10 minutes there is almost 0 hope addresses = set() for txi in tx.inputs: if txi.txo_ref.txo is not None: addresses.add( self.hash160_to_address(txi.txo_ref.txo.pubkey_hash) ) for txo in tx.outputs: if txo.is_pubkey_hash: addresses.add(self.hash160_to_address(txo.pubkey_hash)) elif txo.is_script_hash: addresses.add(self.hash160_to_script_address(txo.script_hash)) start = int(time.perf_counter()) while timeout and (int(time.perf_counter()) - start) <= timeout: if await self._wait_round(tx, height, addresses): return raise asyncio.TimeoutError(f'Timed out waiting for transaction. {tx.id}') async def _wait_round(self, tx: Transaction, height: int, addresses: Iterable[str]): records = await self.db.get_addresses(address__in=addresses) _, pending = await asyncio.wait([ self.on_transaction.where(partial( lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id, address_record['address'] )) for address_record in records ], timeout=1) if not pending: return True records = await self.db.get_addresses(address__in=addresses) for record in records: local_history = (await self.get_local_status_and_history( record['address'], history=record['history'] ))[1] if record['history'] else [] for txid, local_height in local_history: if txid == tx.id: if local_height >= height or (local_height == 0 and height > local_height): return True log.warning( "local history has higher height than remote for %s (%i vs %i)", txid, local_height, height ) return False log.warning( "local history does not contain %s, requested height %i", tx.id, height ) return False async def _inflate_outputs( self, query, accounts, include_purchase_receipt=False, include_is_my_output=False, include_sent_supports=False, include_sent_tips=False, include_received_tips=False) -> Tuple[List[Output], dict, int, int]: encoded_outputs = await query outputs = Outputs.from_base64(encoded_outputs or '') # TODO: why is the server returning None? txs: List[Transaction] = [] if len(outputs.txs) > 0: async for tx in self.request_transactions(tuple(outputs.txs), cached=True): txs.extend(tx.values()) _txos, blocked = outputs.inflate(txs) txos = [] for txo in _txos: if isinstance(txo, Output): # transactions and outputs are cached and shared between wallets # we don't want to leak informaion between wallet so we add the # wallet specific metadata on throw away copies of the txos txo = copy.copy(txo) channel = txo.channel txo.purchase_receipt = None txo.update_annotations(None) txo.channel = channel txos.append(txo) includes = ( include_purchase_receipt, include_is_my_output, include_sent_supports, include_sent_tips ) if accounts and any(includes): receipts = {} if include_purchase_receipt: priced_claims = [] for txo in txos: if isinstance(txo, Output) and txo.has_price: priced_claims.append(txo) if priced_claims: receipts = { txo.purchased_claim_id: txo for txo in await self.db.get_purchases( accounts=accounts, purchased_claim_id__in=[c.claim_id for c in priced_claims] ) } for txo in txos: if isinstance(txo, Output) and txo.can_decode_claim: if include_purchase_receipt: txo.purchase_receipt = receipts.get(txo.claim_id) if include_is_my_output: mine = await self.db.get_txo_count( claim_id=txo.claim_id, txo_type__in=CLAIM_TYPES, is_my_output=True, is_spent=False, accounts=accounts ) if mine: txo.is_my_output = True else: txo.is_my_output = False if include_sent_supports: supports = await self.db.get_txo_sum( claim_id=txo.claim_id, txo_type=TXO_TYPES['support'], is_my_input=True, is_my_output=True, is_spent=False, accounts=accounts ) txo.sent_supports = supports if include_sent_tips: tips = await self.db.get_txo_sum( claim_id=txo.claim_id, txo_type=TXO_TYPES['support'], is_my_input=True, is_my_output=False, accounts=accounts ) txo.sent_tips = tips if include_received_tips: tips = await self.db.get_txo_sum( claim_id=txo.claim_id, txo_type=TXO_TYPES['support'], is_my_input=False, is_my_output=True, accounts=accounts ) txo.received_tips = tips return txos, blocked, outputs.offset, outputs.total async def resolve(self, accounts, urls, **kwargs): txos = [] urls_copy = list(urls) resolve = partial(self.network.retriable_call, self.network.resolve) while urls_copy: batch, urls_copy = urls_copy[:100], urls_copy[100:] txos.extend( (await self._inflate_outputs( resolve(batch), accounts, **kwargs ))[0] ) assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received." result = {} for url, txo in zip(urls, txos): if txo: if isinstance(txo, Output) and URL.parse(url).has_stream_in_channel: if not txo.channel or not txo.is_signed_by(txo.channel, self): txo = {'error': {'name': INVALID, 'text': f'{url} has invalid channel signature'}} else: txo = {'error': {'name': NOT_FOUND, 'text': f'{url} did not resolve to a claim'}} result[url] = txo return result async def sum_supports(self, new_sdk_server, **kwargs) -> List[Dict]: return await self.network.sum_supports(new_sdk_server, **kwargs) async def claim_search( self, accounts, include_purchase_receipt=False, include_is_my_output=False, **kwargs) -> Tuple[List[Output], dict, int, int]: return await self._inflate_outputs( self.network.claim_search(**kwargs), accounts, include_purchase_receipt=include_purchase_receipt, include_is_my_output=include_is_my_output ) # async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output: # return await self.network.get_claim_by_id(claim_id) async def get_claim_by_claim_id(self, claim_id, accounts=None, include_purchase_receipt=False, include_is_my_output=False): accounts = accounts or [] # return await self.network.get_claim_by_id(claim_id) inflated = await self._inflate_outputs( self.network.get_claim_by_id(claim_id), accounts, include_purchase_receipt=include_purchase_receipt, include_is_my_output=include_is_my_output, ) txos = inflated[0] if txos: return txos[0] async def _report_state(self): try: for account in self.accounts: balance = dewies_to_lbc(await account.get_balance(include_claims=True)) channel_count = await account.get_channel_count() claim_count = await account.get_claim_count() if isinstance(account.receiving, SingleKey): log.info("Loaded single key account %s with %s LBC. " "%d channels, %d certificates and %d claims", account.id, balance, channel_count, len(account.channel_keys), claim_count) else: total_receiving = len(await account.receiving.get_addresses()) total_change = len(await account.change.get_addresses()) log.info("Loaded account %s with %s LBC, %d receiving addresses (gap: %d), " "%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ", account.id, balance, total_receiving, account.receiving.gap, total_change, account.change.gap, channel_count, len(account.channel_keys), claim_count) except Exception: log.exception( 'Failed to display wallet state, please file issue ' 'for this bug along with the traceback you see below:') async def _reset_balance_cache(self, e: TransactionEvent): account_ids = [ r['account'] for r in await self.db.get_addresses(('account',), address=e.address) ] for account_id in account_ids: if account_id in self._balance_cache: del self._balance_cache[account_id] @staticmethod def constraint_spending_utxos(constraints): constraints['txo_type__in'] = (0, TXO_TYPES['purchase']) async def get_purchases(self, resolve=False, **constraints): purchases = await self.db.get_purchases(**constraints) if resolve: claim_ids = [p.purchased_claim_id for p in purchases] try: resolved, _, _, _ = await self.claim_search([], claim_ids=claim_ids) except Exception: log.exception("Resolve failed while looking up purchased claim ids:") resolved = [] lookup = {claim.claim_id: claim for claim in resolved} for purchase in purchases: purchase.purchased_claim = lookup.get(purchase.purchased_claim_id) return purchases def get_purchase_count(self, resolve=False, **constraints): return self.db.get_purchase_count(**constraints) async def _resolve_for_local_results(self, accounts, txos): txos = await self._resolve_for_local_claim_results(accounts, txos) txos = await self._resolve_for_local_support_results(accounts, txos) return txos async def _resolve_for_local_claim_results(self, accounts, txos): results = [] response = await self.resolve( accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim] ) for txo in txos: resolved = response.get(txo.permanent_url) if txo.can_decode_claim else None if isinstance(resolved, Output): resolved.update_annotations(txo) results.append(resolved) else: if isinstance(resolved, dict) and 'error' in resolved: txo.meta['error'] = resolved['error'] results.append(txo) return results async def _resolve_for_local_support_results(self, accounts, txos): channel_ids = set() signed_support_txos = [] for txo in txos: support = txo.can_decode_support if support and support.signing_channel_id: channel_ids.add(support.signing_channel_id) signed_support_txos.append(txo) if channel_ids: channels = { channel.claim_id: channel for channel in (await self.claim_search(accounts, claim_ids=list(channel_ids)))[0] } for txo in signed_support_txos: txo.channel = channels.get(txo.support.signing_channel_id) return txos async def get_claims(self, resolve=False, **constraints): claims = await self.db.get_claims(**constraints) if resolve: return await self._resolve_for_local_results(constraints.get('accounts', []), claims) return claims def get_claim_count(self, **constraints): return self.db.get_claim_count(**constraints) async def get_streams(self, resolve=False, **constraints): streams = await self.db.get_streams(**constraints) if resolve: return await self._resolve_for_local_results(constraints.get('accounts', []), streams) return streams def get_stream_count(self, **constraints): return self.db.get_stream_count(**constraints) async def get_channels(self, resolve=False, **constraints): channels = await self.db.get_channels(**constraints) if resolve: return await self._resolve_for_local_results(constraints.get('accounts', []), channels) return channels def get_channel_count(self, **constraints): return self.db.get_channel_count(**constraints) async def resolve_collection(self, collection, offset=0, page_size=1): claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset] try: resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids) except Exception: log.exception("Resolve failed while looking up collection claim ids:") return [] claims = [] for claim_id in claim_ids: found = False for txo in resolve_results: if txo.claim_id == claim_id: claims.append(txo) found = True break if not found: claims.append(None) return claims async def get_collections(self, resolve_claims=0, resolve=False, **constraints): collections = await self.db.get_collections(**constraints) if resolve: collections = await self._resolve_for_local_results(constraints.get('accounts', []), collections) if resolve_claims > 0: for collection in collections: collection.claims = await self.resolve_collection(collection, page_size=resolve_claims) return collections def get_collection_count(self, resolve_claims=0, **constraints): return self.db.get_collection_count(**constraints) def get_supports(self, **constraints): return self.db.get_supports(**constraints) def get_support_count(self, **constraints): return self.db.get_support_count(**constraints) async def get_transaction_history(self, read_only=False, **constraints): txs: List[Transaction] = await self.db.get_transactions( include_is_my_output=True, include_is_spent=True, read_only=read_only, **constraints ) headers = self.headers history = [] for tx in txs: # pylint: disable=too-many-nested-blocks ts = headers.estimated_timestamp(tx.height) item = { 'txid': tx.id, 'timestamp': ts, 'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None, 'confirmations': (headers.height + 1) - tx.height if tx.height > 0 else 0, 'claim_info': [], 'update_info': [], 'support_info': [], 'abandon_info': [], 'purchase_info': [] } is_my_inputs = all(txi.is_my_input for txi in tx.inputs) if is_my_inputs: # fees only matter if we are the ones paying them item['value'] = dewies_to_lbc(tx.net_account_balance + tx.fee) item['fee'] = dewies_to_lbc(-tx.fee) else: # someone else paid the fees item['value'] = dewies_to_lbc(tx.net_account_balance) item['fee'] = '0.0' for txo in tx.my_claim_outputs: item['claim_info'].append({ 'address': txo.get_address(self), 'balance_delta': dewies_to_lbc(-txo.amount), 'amount': dewies_to_lbc(txo.amount), 'claim_id': txo.claim_id, 'claim_name': txo.claim_name, 'nout': txo.position, 'is_spent': txo.is_spent, }) for txo in tx.my_update_outputs: if is_my_inputs: # updating my own claim previous = None for txi in tx.inputs: if txi.txo_ref.txo is not None: other_txo = txi.txo_ref.txo if (other_txo.is_claim or other_txo.script.is_support_claim) \ and other_txo.claim_id == txo.claim_id: previous = other_txo break if previous is not None: item['update_info'].append({ 'address': txo.get_address(self), 'balance_delta': dewies_to_lbc(previous.amount - txo.amount), 'amount': dewies_to_lbc(txo.amount), 'claim_id': txo.claim_id, 'claim_name': txo.claim_name, 'nout': txo.position, 'is_spent': txo.is_spent, }) else: # someone sent us their claim item['update_info'].append({ 'address': txo.get_address(self), 'balance_delta': dewies_to_lbc(0), 'amount': dewies_to_lbc(txo.amount), 'claim_id': txo.claim_id, 'claim_name': txo.claim_name, 'nout': txo.position, 'is_spent': txo.is_spent, }) for txo in tx.my_support_outputs: item['support_info'].append({ 'address': txo.get_address(self), 'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount), 'amount': dewies_to_lbc(txo.amount), 'claim_id': txo.claim_id, 'claim_name': txo.claim_name, 'is_tip': not is_my_inputs, 'nout': txo.position, 'is_spent': txo.is_spent, }) if is_my_inputs: for txo in tx.other_support_outputs: item['support_info'].append({ 'address': txo.get_address(self), 'balance_delta': dewies_to_lbc(-txo.amount), 'amount': dewies_to_lbc(txo.amount), 'claim_id': txo.claim_id, 'claim_name': txo.claim_name, 'is_tip': is_my_inputs, 'nout': txo.position, 'is_spent': txo.is_spent, }) for txo in tx.my_abandon_outputs: item['abandon_info'].append({ 'address': txo.get_address(self), 'balance_delta': dewies_to_lbc(txo.amount), 'amount': dewies_to_lbc(txo.amount), 'claim_id': txo.claim_id, 'claim_name': txo.claim_name, 'nout': txo.position }) for txo in tx.any_purchase_outputs: item['purchase_info'].append({ 'address': txo.get_address(self), 'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount), 'amount': dewies_to_lbc(txo.amount), 'claim_id': txo.purchased_claim_id, 'nout': txo.position, 'is_spent': txo.is_spent, }) history.append(item) return history def get_transaction_history_count(self, read_only=False, **constraints): return self.db.get_transaction_count(read_only=read_only, **constraints) async def get_detailed_balance(self, accounts, confirmations=0): result = { 'total': 0, 'available': 0, 'reserved': 0, 'reserved_subtotals': { 'claims': 0, 'supports': 0, 'tips': 0 } } for account in accounts: balance = self._balance_cache.get(account.id) if not balance: balance = self._balance_cache[account.id] = \ await account.get_detailed_balance(confirmations) for key, value in balance.items(): if key == 'reserved_subtotals': for subkey, subvalue in value.items(): result['reserved_subtotals'][subkey] += subvalue else: result[key] += value return result class TestNetLedger(Ledger): network_name = 'testnet' pubkey_address_prefix = bytes((111,)) script_address_prefix = bytes((196,)) extended_public_key_prefix = unhexlify('043587cf') extended_private_key_prefix = unhexlify('04358394') checkpoints = {} class RegTestLedger(Ledger): network_name = 'regtest' headers_class = UnvalidatedHeaders pubkey_address_prefix = bytes((111,)) script_address_prefix = bytes((196,)) extended_public_key_prefix = unhexlify('043587cf') extended_private_key_prefix = unhexlify('04358394') max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556' genesis_bits = 0x207fffff target_timespan = 1 checkpoints = {} ================================================ FILE: lbry/wallet/manager.py ================================================ import os import json import typing import logging import asyncio from binascii import unhexlify from decimal import Decimal from typing import List, Type, MutableSequence, MutableMapping, Optional from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError from lbry.conf import Config, NOT_SET from lbry.wallet.dewies import dewies_to_lbc from lbry.wallet.account import Account from lbry.wallet.ledger import Ledger, LedgerRegistry from lbry.wallet.transaction import Transaction, Output from lbry.wallet.database import Database from lbry.wallet.wallet import Wallet, WalletStorage, ENCRYPT_ON_DISK from lbry.wallet.rpc.jsonrpc import CodeMessageError if typing.TYPE_CHECKING: from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager log = logging.getLogger(__name__) class WalletManager: def __init__(self, wallets: MutableSequence[Wallet] = None, ledgers: MutableMapping[Type[Ledger], Ledger] = None) -> None: self.wallets = wallets or [] self.ledgers = ledgers or {} self.running = False self.config: Optional[Config] = None @classmethod def from_config(cls, config: dict) -> 'WalletManager': manager = cls() for ledger_id, ledger_config in config.get('ledgers', {}).items(): manager.get_or_create_ledger(ledger_id, ledger_config) for wallet_path in config.get('wallets', []): wallet_storage = WalletStorage(wallet_path) wallet = Wallet.from_storage(wallet_storage, manager) manager.wallets.append(wallet) return manager def get_or_create_ledger(self, ledger_id, ledger_config=None): ledger_class = LedgerRegistry.get_ledger_class(ledger_id) ledger = self.ledgers.get(ledger_class) if ledger is None: ledger = ledger_class(ledger_config or {}) self.ledgers[ledger_class] = ledger return ledger def import_wallet(self, path): storage = WalletStorage(path) wallet = Wallet.from_storage(storage, self) self.wallets.append(wallet) return wallet @property def default_wallet(self): for wallet in self.wallets: return wallet @property def default_account(self): for wallet in self.wallets: return wallet.default_account @property def accounts(self): for wallet in self.wallets: yield from wallet.accounts async def start(self): self.running = True await asyncio.gather(*( l.start() for l in self.ledgers.values() )) async def stop(self): await asyncio.gather(*( l.stop() for l in self.ledgers.values() )) self.running = False def get_wallet_or_default(self, wallet_id: Optional[str]) -> Wallet: if wallet_id is None: return self.default_wallet return self.get_wallet_or_error(wallet_id) def get_wallet_or_error(self, wallet_id: str) -> Wallet: for wallet in self.wallets: if wallet.id == wallet_id: return wallet raise WalletNotLoadedError(wallet_id) @staticmethod def get_balance(wallet): accounts = wallet.accounts if not accounts: return 0 return accounts[0].ledger.db.get_balance(wallet=wallet, accounts=accounts) @property def ledger(self) -> Ledger: return self.default_account.ledger @property def db(self) -> Database: return self.ledger.db def check_locked(self): return self.default_wallet.is_locked @staticmethod def migrate_lbryum_to_torba(path): if not os.path.exists(path): return None, None with open(path, 'r') as f: unmigrated_json = f.read() unmigrated = json.loads(unmigrated_json) # TODO: After several public releases of new torba based wallet, we can delete # this lbryum->torba conversion code and require that users who still # have old structured wallets install one of the earlier releases that # still has the below conversion code. if 'master_public_keys' not in unmigrated: return None, None total = unmigrated.get('addr_history') receiving_addresses, change_addresses = set(), set() for _, unmigrated_account in unmigrated.get('accounts', {}).items(): receiving_addresses.update(map(unhexlify, unmigrated_account.get('receiving', []))) change_addresses.update(map(unhexlify, unmigrated_account.get('change', []))) log.info("Wallet migrator found %s receiving addresses and %s change addresses. %s in total on history.", len(receiving_addresses), len(change_addresses), len(total)) migrated_json = json.dumps({ 'version': 1, 'name': 'My Wallet', 'accounts': [{ 'version': 1, 'name': 'Main Account', 'ledger': 'lbc_mainnet', 'encrypted': unmigrated['use_encryption'], 'seed': unmigrated['seed'], 'seed_version': unmigrated['seed_version'], 'private_key': unmigrated['master_private_keys']['x/'], 'public_key': unmigrated['master_public_keys']['x/'], 'certificates': unmigrated.get('claim_certificates', {}), 'address_generator': { 'name': 'deterministic-chain', 'receiving': {'gap': 20, 'maximum_uses_per_address': 1}, 'change': {'gap': 6, 'maximum_uses_per_address': 1} } }] }, indent=4, sort_keys=True) mode = os.stat(path).st_mode i = 1 backup_path_template = os.path.join(os.path.dirname(path), "old_lbryum_wallet") + "_%i" while os.path.isfile(backup_path_template % i): i += 1 os.rename(path, backup_path_template % i) temp_path = f"{path}.tmp.{os.getpid()}" with open(temp_path, "w") as f: f.write(migrated_json) f.flush() os.fsync(f.fileno()) os.rename(temp_path, path) os.chmod(path, mode) return receiving_addresses, change_addresses @classmethod async def from_lbrynet_config(cls, config: Config): ledger_id = { 'lbrycrd_main': 'lbc_mainnet', 'lbrycrd_testnet': 'lbc_testnet', 'lbrycrd_regtest': 'lbc_regtest' }[config.blockchain_name] ledger_config = { 'auto_connect': True, 'explicit_servers': [], 'hub_timeout': config.hub_timeout, 'default_servers': config.lbryum_servers, 'known_hubs': config.known_hubs, 'jurisdiction': config.jurisdiction, 'concurrent_hub_requests': config.concurrent_hub_requests, 'data_path': config.wallet_dir, 'tx_cache_size': config.transaction_cache_size } if 'LBRY_FEE_PER_NAME_CHAR' in os.environ: ledger_config['fee_per_name_char'] = int(os.environ.get('LBRY_FEE_PER_NAME_CHAR')) wallets_directory = os.path.join(config.wallet_dir, 'wallets') if not os.path.exists(wallets_directory): os.mkdir(wallets_directory) receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba( os.path.join(wallets_directory, 'default_wallet') ) if Config.lbryum_servers.is_set_to_default(config): with config.update_config() as c: c.lbryum_servers = NOT_SET manager = cls.from_config({ 'ledgers': {ledger_id: ledger_config}, 'wallets': [ os.path.join(wallets_directory, wallet_file) for wallet_file in config.wallets ] }) manager.config = config ledger = manager.get_or_create_ledger(ledger_id) ledger.coin_selection_strategy = config.coin_selection_strategy default_wallet = manager.default_wallet if default_wallet.default_account is None: log.info('Wallet at %s is empty, generating a default account.', default_wallet.id) default_wallet.generate_account(ledger) default_wallet.save() if default_wallet.is_locked and default_wallet.preferences.get(ENCRYPT_ON_DISK) is None: default_wallet.preferences[ENCRYPT_ON_DISK] = True default_wallet.save() if receiving_addresses or change_addresses: if not os.path.exists(ledger.path): os.mkdir(ledger.path) await ledger.db.open() try: await manager._migrate_addresses(receiving_addresses, change_addresses) finally: await ledger.db.close() return manager async def reset(self): self.ledger.config = { 'auto_connect': True, 'explicit_servers': [], 'default_servers': Config.lbryum_servers.default, 'known_hubs': self.config.known_hubs, 'jurisdiction': self.config.jurisdiction, 'hub_timeout': self.config.hub_timeout, 'concurrent_hub_requests': self.config.concurrent_hub_requests, 'data_path': self.config.wallet_dir, } if Config.lbryum_servers.is_set(self.config): self.ledger.config['explicit_servers'] = self.config.lbryum_servers await self.ledger.stop() await self.ledger.start() async def _migrate_addresses(self, receiving_addresses: set, change_addresses: set): async with self.default_account.receiving.address_generator_lock: migrated_receiving = set(await self.default_account.receiving._generate_keys(0, len(receiving_addresses))) async with self.default_account.change.address_generator_lock: migrated_change = set(await self.default_account.change._generate_keys(0, len(change_addresses))) receiving_addresses = set(map(self.default_account.ledger.public_key_to_address, receiving_addresses)) change_addresses = set(map(self.default_account.ledger.public_key_to_address, change_addresses)) if not any(change_addresses.difference(migrated_change)): log.info("Successfully migrated %s change addresses.", len(change_addresses)) else: log.warning("Failed to migrate %s change addresses!", len(set(change_addresses).difference(set(migrated_change)))) if not any(receiving_addresses.difference(migrated_receiving)): log.info("Successfully migrated %s receiving addresses.", len(receiving_addresses)) else: log.warning("Failed to migrate %s receiving addresses!", len(set(receiving_addresses).difference(set(migrated_receiving)))) async def get_best_blockhash(self): if len(self.ledger.headers) <= 0: return self.ledger.genesis_hash return (await self.ledger.headers.hash(self.ledger.headers.height)).decode() def get_unused_address(self): return self.default_account.receiving.get_or_create_usable_address() async def get_transaction(self, txid: str): tx = await self.db.get_transaction(txid=txid) if tx: return tx try: raw, merkle = await self.ledger.network.get_transaction_and_merkle(txid) except CodeMessageError as e: if 'No such mempool or blockchain transaction.' in e.message: return {'success': False, 'code': 404, 'message': 'transaction not found'} return {'success': False, 'code': e.code, 'message': e.message} height = merkle.get('block_height') tx = Transaction(unhexlify(raw), height=height) if height and height > 0: await self.ledger.maybe_verify_transaction(tx, height, merkle) return tx async def create_purchase_transaction( self, accounts: List[Account], txo: Output, exchange: 'ExchangeRateManager', override_max_key_fee=False): fee = txo.claim.stream.fee fee_amount = exchange.to_dewies(fee.currency, fee.amount) if not override_max_key_fee and self.config.max_key_fee: max_fee = self.config.max_key_fee max_fee_amount = exchange.to_dewies(max_fee['currency'], Decimal(max_fee['amount'])) if max_fee_amount and fee_amount > max_fee_amount: error_fee = f"{dewies_to_lbc(fee_amount)} LBC" if fee.currency != 'LBC': error_fee += f" ({fee.amount} {fee.currency})" error_max_fee = f"{dewies_to_lbc(max_fee_amount)} LBC" if max_fee['currency'] != 'LBC': error_max_fee += f" ({max_fee['amount']} {max_fee['currency']})" raise KeyFeeAboveMaxAllowedError( f"Purchase price of {error_fee} exceeds maximum " f"configured price of {error_max_fee}." ) fee_address = fee.address or txo.get_address(self.ledger) return await Transaction.purchase( txo.claim_id, fee_amount, fee_address, accounts, accounts[0] ) async def broadcast_or_release(self, tx, blocking=False): await self.ledger.broadcast_or_release(tx, blocking=blocking) ================================================ FILE: lbry/wallet/mnemonic.py ================================================ # Copyright (C) 2014 Thomas Voegtlin # Copyright (C) 2018 LBRY Inc. import hmac import math import hashlib import importlib import unicodedata import string from binascii import hexlify from secrets import randbelow import pbkdf2 from lbry.crypto.hash import hmac_sha512 from .words import english # The hash of the mnemonic seed must begin with this SEED_PREFIX = b'01' # Standard wallet SEED_PREFIX_2FA = b'101' # Two-factor authentication SEED_PREFIX_SW = b'100' # Segwit wallet # http://www.asahi-net.or.jp/~ax2s-kmtn/ref/unicode/e_asia.html CJK_INTERVALS = [ (0x4E00, 0x9FFF, 'CJK Unified Ideographs'), (0x3400, 0x4DBF, 'CJK Unified Ideographs Extension A'), (0x20000, 0x2A6DF, 'CJK Unified Ideographs Extension B'), (0x2A700, 0x2B73F, 'CJK Unified Ideographs Extension C'), (0x2B740, 0x2B81F, 'CJK Unified Ideographs Extension D'), (0xF900, 0xFAFF, 'CJK Compatibility Ideographs'), (0x2F800, 0x2FA1D, 'CJK Compatibility Ideographs Supplement'), (0x3190, 0x319F, 'Kanbun'), (0x2E80, 0x2EFF, 'CJK Radicals Supplement'), (0x2F00, 0x2FDF, 'CJK Radicals'), (0x31C0, 0x31EF, 'CJK Strokes'), (0x2FF0, 0x2FFF, 'Ideographic Description Characters'), (0xE0100, 0xE01EF, 'Variation Selectors Supplement'), (0x3100, 0x312F, 'Bopomofo'), (0x31A0, 0x31BF, 'Bopomofo Extended'), (0xFF00, 0xFFEF, 'Halfwidth and Fullwidth Forms'), (0x3040, 0x309F, 'Hiragana'), (0x30A0, 0x30FF, 'Katakana'), (0x31F0, 0x31FF, 'Katakana Phonetic Extensions'), (0x1B000, 0x1B0FF, 'Kana Supplement'), (0xAC00, 0xD7AF, 'Hangul Syllables'), (0x1100, 0x11FF, 'Hangul Jamo'), (0xA960, 0xA97F, 'Hangul Jamo Extended A'), (0xD7B0, 0xD7FF, 'Hangul Jamo Extended B'), (0x3130, 0x318F, 'Hangul Compatibility Jamo'), (0xA4D0, 0xA4FF, 'Lisu'), (0x16F00, 0x16F9F, 'Miao'), (0xA000, 0xA48F, 'Yi Syllables'), (0xA490, 0xA4CF, 'Yi Radicals'), ] def is_cjk(c): n = ord(c) for start, end, _ in CJK_INTERVALS: if start <= n <= end: return True return False def normalize_text(seed): seed = unicodedata.normalize('NFKD', seed) seed = seed.lower() # remove accents seed = ''.join([c for c in seed if not unicodedata.combining(c)]) # normalize whitespaces seed = ' '.join(seed.split()) # remove whitespaces between CJK seed = ''.join([ seed[i] for i in range(len(seed)) if not (seed[i] in string.whitespace and is_cjk(seed[i-1]) and is_cjk(seed[i+1])) ]) return seed def load_words(language_name): if language_name == 'english': return english.words language_module = importlib.import_module('lbry.wallet.client.words.'+language_name) return list(map( lambda s: unicodedata.normalize('NFKD', s), language_module.words )) LANGUAGE_NAMES = { 'en': 'english', 'es': 'spanish', 'ja': 'japanese', 'pt': 'portuguese', 'zh': 'chinese_simplified' } class Mnemonic: # Seed derivation no longer follows BIP39 # Mnemonic phrase uses a hash based checksum, instead of a words-dependent checksum def __init__(self, lang='en'): language_name = LANGUAGE_NAMES.get(lang, 'english') self.words = load_words(language_name) @staticmethod def mnemonic_to_seed(mnemonic, passphrase=''): pbkdf2_rounds = 2048 mnemonic = normalize_text(mnemonic) passphrase = normalize_text(passphrase) return pbkdf2.PBKDF2( mnemonic, passphrase, iterations=pbkdf2_rounds, macmodule=hmac, digestmodule=hashlib.sha512 ).read(64) def mnemonic_encode(self, i): n = len(self.words) words = [] while i: x = i%n i = i//n words.append(self.words[x]) return ' '.join(words) def mnemonic_decode(self, seed): n = len(self.words) words = seed.split() i = 0 while words: word = words.pop() k = self.words.index(word) i = i*n + k return i def make_seed(self, prefix=SEED_PREFIX, num_bits=132): # increase num_bits in order to obtain a uniform distribution for the last word bpw = math.log(len(self.words), 2) # rounding n = int(math.ceil(num_bits/bpw) * bpw) entropy = 1 while 0 < entropy < pow(2, n - bpw): # try again if seed would not contain enough words entropy = randbelow(pow(2, n)) nonce = 0 while True: nonce += 1 i = entropy + nonce seed = self.mnemonic_encode(i) if i != self.mnemonic_decode(seed): raise Exception('Cannot extract same entropy from mnemonic!') if is_new_seed(seed, prefix): break return seed def is_new_seed(seed, prefix): seed = normalize_text(seed) seed_hash = hexlify(hmac_sha512(b"Seed version", seed.encode('utf8'))) return seed_hash.startswith(prefix) ================================================ FILE: lbry/wallet/network.py ================================================ import logging import asyncio import json import socket import random from time import perf_counter from collections import defaultdict from typing import Dict, Optional, Tuple import aiohttp from lbry import __version__ from lbry.utils import resolve_host from lbry.error import IncompatibleWalletServerError from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError from lbry.wallet.stream import StreamController from lbry.wallet.udp import SPVStatusClientProtocol, SPVPong from lbry.conf import KnownHubsList log = logging.getLogger(__name__) class ClientSession(BaseClientSession): def __init__(self, *args, network: 'Network', server, timeout=30, concurrency=32, **kwargs): self.network = network self.server = server super().__init__(*args, **kwargs) self.framer.max_size = self.max_errors = 1 << 32 self.timeout = timeout self.max_seconds_idle = timeout * 2 self.response_time: Optional[float] = None self.connection_latency: Optional[float] = None self._response_samples = 0 self._concurrency = asyncio.Semaphore(concurrency) @property def concurrency(self): return self._concurrency._value @property def available(self): return not self.is_closing() and self.response_time is not None @property def server_address_and_port(self) -> Optional[Tuple[str, int]]: if not self.transport: return None return self.transport.get_extra_info('peername') async def send_timed_server_version_request(self, args=(), timeout=None): timeout = timeout or self.timeout log.debug("send version request to %s:%i", *self.server) start = perf_counter() result = await asyncio.wait_for( super().send_request('server.version', args), timeout=timeout ) current_response_time = perf_counter() - start response_sum = (self.response_time or 0) * self._response_samples + current_response_time self.response_time = response_sum / (self._response_samples + 1) self._response_samples += 1 return result async def send_request(self, method, args=()): log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout) try: await self._concurrency.acquire() if method == 'server.version': return await self.send_timed_server_version_request(args, self.timeout) request = asyncio.ensure_future(super().send_request(method, args)) while not request.done(): done, pending = await asyncio.wait([request], timeout=self.timeout) if pending: log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received) if (perf_counter() - self.last_packet_received) < self.timeout: continue log.warning("timeout sending %s to %s:%i", method, *self.server) raise asyncio.TimeoutError if done: try: return request.result() except ConnectionResetError: log.error( "wallet server (%s) reset connection upon our %s request, json of %i args is %i bytes", self.server[0], method, len(args), len(json.dumps(args)) ) raise except (RPCError, ProtocolError) as e: log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s", *self.server, *e.args) raise e except ConnectionError: log.warning("connection to %s:%i lost", *self.server) self.synchronous_close() raise except asyncio.CancelledError: log.warning("cancelled sending %s to %s:%i", method, *self.server) # self.synchronous_close() raise finally: self._concurrency.release() async def ensure_server_version(self, required=None, timeout=3): required = required or self.network.PROTOCOL_MAX_VERSION response = await asyncio.wait_for( self.send_request('server.version', [self.network.CLIENT_NAME, required]), timeout=timeout ) if tuple(int(piece) for piece in response[1].split(".")) >= self.network.PROTOCOL_MIN_VERSION: return response raise IncompatibleWalletServerError(*self.server) async def keepalive_loop(self, timeout=3, max_idle=60): try: while True: now = perf_counter() if min(self.last_send, self.last_packet_received) + max_idle < now: await asyncio.wait_for( self.send_request('server.ping', []), timeout=timeout ) else: await asyncio.sleep(max(0, max_idle - (now - self.last_send))) except (Exception, asyncio.CancelledError) as err: if isinstance(err, asyncio.CancelledError): log.info("closing connection to %s:%i", *self.server) else: log.exception("lost connection to spv") finally: if not self.is_closing(): self._close() async def create_connection(self, timeout=6): connector = Connector(lambda: self, *self.server) start = perf_counter() await asyncio.wait_for(connector.create_connection(), timeout=timeout) self.connection_latency = perf_counter() - start async def handle_request(self, request): controller = self.network.subscription_controllers[request.method] controller.add(request.args) def connection_lost(self, exc): log.debug("Connection lost: %s:%d", *self.server) super().connection_lost(exc) self.response_time = None self.connection_latency = None self._response_samples = 0 # self._on_disconnect_controller.add(True) if self.network: self.network.disconnect() class Network: CLIENT_VERSION = __version__ CLIENT_NAME = "LBRY SDK " + CLIENT_VERSION PROTOCOL_MIN_VERSION = (0, 65, 0) PROTOCOL_MAX_VERSION = __version__ def __init__(self, ledger): self.ledger = ledger self.client: Optional[ClientSession] = None self.server_features = None # self._switch_task: Optional[asyncio.Task] = None self.running = False self.remote_height: int = 0 self._on_connected_controller = StreamController() self.on_connected = self._on_connected_controller.stream self._on_header_controller = StreamController(merge_repeated_events=True) self.on_header = self._on_header_controller.stream self._on_status_controller = StreamController(merge_repeated_events=True) self.on_status = self._on_status_controller.stream self._on_hub_controller = StreamController(merge_repeated_events=True) self.on_hub = self._on_hub_controller.stream self.subscription_controllers = { 'blockchain.headers.subscribe': self._on_header_controller, 'blockchain.address.subscribe': self._on_status_controller, 'blockchain.peers.subscribe': self._on_hub_controller, } self.aiohttp_session: Optional[aiohttp.ClientSession] = None self._urgent_need_reconnect = asyncio.Event() self._loop_task: Optional[asyncio.Task] = None self._keepalive_task: Optional[asyncio.Task] = None @property def config(self): return self.ledger.config @property def known_hubs(self): if 'known_hubs' not in self.config: return KnownHubsList() return self.config['known_hubs'] @property def jurisdiction(self): return self.config.get("jurisdiction") def disconnect(self): if self._keepalive_task and not self._keepalive_task.done(): self._keepalive_task.cancel() self._keepalive_task = None async def start(self): if not self.running: self.running = True self.aiohttp_session = aiohttp.ClientSession() self.on_header.listen(self._update_remote_height) self.on_hub.listen(self._update_hubs) self._loop_task = asyncio.create_task(self.network_loop()) self._urgent_need_reconnect.set() def loop_task_done_callback(f): try: f.result() except (Exception, asyncio.CancelledError): if self.running: log.exception("wallet server connection loop crashed") self._loop_task.add_done_callback(loop_task_done_callback) async def resolve_spv_dns(self): hostname_to_ip = {} ip_to_hostnames = defaultdict(list) async def resolve_spv(server, port): try: server_addr = await resolve_host(server, port, 'udp') hostname_to_ip[server] = (server_addr, port) ip_to_hostnames[(server_addr, port)].append(server) except socket.error: log.warning("error looking up dns for spv server %s:%i", server, port) except Exception: log.exception("error looking up dns for spv server %s:%i", server, port) # accumulate the dns results if self.config.get('explicit_servers', []): hubs = self.config['explicit_servers'] elif self.known_hubs: hubs = self.known_hubs else: hubs = self.config['default_servers'] await asyncio.gather(*(resolve_spv(server, port) for (server, port) in hubs)) return hostname_to_ip, ip_to_hostnames async def get_n_fastest_spvs(self, timeout=3.0) -> Dict[Tuple[str, int], Optional[SPVPong]]: loop = asyncio.get_event_loop() pong_responses = asyncio.Queue() connection = SPVStatusClientProtocol(pong_responses) sent_ping_timestamps = {} _, ip_to_hostnames = await self.resolve_spv_dns() n = len(ip_to_hostnames) log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config.get('explicit_servers', []))) pongs = {} known_hubs = self.known_hubs try: await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0)) # could raise OSError if it cant bind start = perf_counter() for server in ip_to_hostnames: connection.ping(server) sent_ping_timestamps[server] = perf_counter() while len(pongs) < n: (remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start)) latency = ts - start log.info("%s:%i has latency of %sms (available: %s, height: %i)", '/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2), pong.available, pong.height) known_hubs.hubs.setdefault((ip_to_hostnames[remote][0], remote[1]), {}).update( {"country": pong.country_name} ) if pong.available: pongs[(ip_to_hostnames[remote][0], remote[1])] = pong return pongs except asyncio.TimeoutError: if pongs: log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames)) return pongs else: log.warning("%i spv status probes failed, retrying later. servers tried: %s", len(sent_ping_timestamps), ', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items())) random_server = random.choice(list(ip_to_hostnames.keys())) host, port = random_server log.warning("trying fallback to randomly selected spv: %s:%i", host, port) known_hubs.hubs.setdefault((host, port), {}) return {(host, port): None} finally: connection.close() async def connect_to_fastest(self) -> Optional[ClientSession]: fastest_spvs = await self.get_n_fastest_spvs() for (host, port), pong in fastest_spvs.items(): if (pong is not None and self.jurisdiction is not None) and \ (pong.country_name != self.jurisdiction): continue client = ClientSession(network=self, server=(host, port), timeout=self.config.get('hub_timeout', 30), concurrency=self.config.get('concurrent_hub_requests', 30)) try: await client.create_connection() log.info("Connected to spv server %s:%i", host, port) await client.ensure_server_version() return client except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError): log.warning("Connecting to %s:%d failed", host, port) client._close() return async def network_loop(self): sleep_delay = 30 while self.running: await asyncio.wait( map(asyncio.create_task, [asyncio.sleep(30), self._urgent_need_reconnect.wait()]), return_when=asyncio.FIRST_COMPLETED ) if self._urgent_need_reconnect.is_set(): sleep_delay = 30 self._urgent_need_reconnect.clear() if not self.is_connected: client = await self.connect_to_fastest() if not client: log.warning("failed to connect to any spv servers, retrying later") sleep_delay *= 2 sleep_delay = min(sleep_delay, 300) continue log.debug("get spv server features %s:%i", *client.server) features = await client.send_request('server.features', []) self.client, self.server_features = client, features log.debug("discover other hubs %s:%i", *client.server) await self._update_hubs(await client.send_request('server.peers.subscribe', [])) log.info("subscribe to headers %s:%i", *client.server) self._update_remote_height((await self.subscribe_headers(),)) self._on_connected_controller.add(True) server_str = "%s:%i" % client.server log.info("maintaining connection to spv server %s", server_str) self._keepalive_task = asyncio.create_task(self.client.keepalive_loop()) try: if not self._urgent_need_reconnect.is_set(): await asyncio.wait( [self._keepalive_task, asyncio.create_task(self._urgent_need_reconnect.wait())], return_when=asyncio.FIRST_COMPLETED ) else: await self._keepalive_task if self._urgent_need_reconnect.is_set(): log.warning("urgent reconnect needed") if self._keepalive_task and not self._keepalive_task.done(): self._keepalive_task.cancel() except asyncio.CancelledError: pass finally: self._keepalive_task = None self.client = None self.server_features = None log.info("connection lost to %s", server_str) log.info("network loop finished") async def stop(self): self.running = False self.disconnect() if self._loop_task and not self._loop_task.done(): self._loop_task.cancel() self._loop_task = None if self.aiohttp_session: await self.aiohttp_session.close() self.aiohttp_session = None @property def is_connected(self): return self.client and not self.client.is_closing() def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None): if session or self.is_connected: session = session or self.client return session.send_request(list_or_method, args) else: self._urgent_need_reconnect.set() raise ConnectionError("Attempting to send rpc request when connection is not available.") async def retriable_call(self, function, *args, **kwargs): while self.running: if not self.is_connected: log.warning("Wallet server unavailable, waiting for it to come back and retry.") self._urgent_need_reconnect.set() await self.on_connected.first try: return await function(*args, **kwargs) except asyncio.TimeoutError: log.warning("Wallet server call timed out, retrying.") except ConnectionError: log.warning("connection error") raise asyncio.CancelledError() # if we got here, we are shutting down def _update_remote_height(self, header_args): self.remote_height = header_args[0]["height"] async def _update_hubs(self, hubs): if hubs and hubs != ['']: try: if self.known_hubs.add_hubs(hubs): self.known_hubs.save() except Exception: log.exception("could not add hubs: %s", hubs) def get_transaction(self, tx_hash, known_height=None): # use any server if its old, otherwise restrict to who gave us the history restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10 return self.rpc('blockchain.transaction.get', [tx_hash], restricted) def get_transaction_batch(self, txids, restricted=True): # use any server if its old, otherwise restrict to who gave us the history return self.rpc('blockchain.transaction.get_batch', txids, restricted) def get_transaction_and_merkle(self, tx_hash, known_height=None): # use any server if its old, otherwise restrict to who gave us the history restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10 return self.rpc('blockchain.transaction.info', [tx_hash], restricted) def get_transaction_height(self, tx_hash, known_height=None): restricted = not known_height or 0 > known_height > self.remote_height - 10 return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted) def get_merkle(self, tx_hash, height): restricted = 0 > height > self.remote_height - 10 return self.rpc('blockchain.transaction.get_merkle', [tx_hash, height], restricted) def get_headers(self, height, count=10000, b64=False): restricted = height >= self.remote_height - 100 return self.rpc('blockchain.block.headers', [height, count, 0, b64], restricted) # --- Subscribes, history and broadcasts are always aimed towards the master client directly def get_history(self, address): return self.rpc('blockchain.address.get_history', [address], True) def broadcast(self, raw_transaction): return self.rpc('blockchain.transaction.broadcast', [raw_transaction], True) def subscribe_headers(self): return self.rpc('blockchain.headers.subscribe', [True], True) async def subscribe_address(self, address, *addresses): addresses = list((address, ) + addresses) server_addr_and_port = self.client.server_address_and_port # on disconnect client will be None try: return await self.rpc('blockchain.address.subscribe', addresses, True) except asyncio.TimeoutError: log.warning( "timed out subscribing to addresses from %s:%i", *server_addr_and_port ) # abort and cancel, we can't lose a subscription, it will happen again on reconnect if self.client: self.client.abort() raise asyncio.CancelledError() def unsubscribe_address(self, address): return self.rpc('blockchain.address.unsubscribe', [address], True) def get_server_features(self): return self.rpc('server.features', (), restricted=True) # def get_claims_by_ids(self, claim_ids): # return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids) def get_claim_by_id(self, claim_id): return self.rpc('blockchain.claimtrie.getclaimbyid', [claim_id]) def resolve(self, urls, session_override=None): return self.rpc('blockchain.claimtrie.resolve', urls, False, session_override) def claim_search(self, session_override=None, **kwargs): return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override) async def sum_supports(self, server, **kwargs): message = {"method": "support_sum", "params": kwargs} async with self.aiohttp_session.post(server, json=message) as r: result = await r.json() return result['result'] ================================================ FILE: lbry/wallet/orchstr8/__init__.py ================================================ from lbry.wallet.orchstr8.node import Conductor from lbry.wallet.orchstr8.service import ConductorService ================================================ FILE: lbry/wallet/orchstr8/cli.py ================================================ import logging import argparse import asyncio import aiohttp from lbry import wallet from lbry.wallet.orchstr8.node import ( Conductor, get_lbcd_node_from_ledger, get_lbcwallet_node_from_ledger ) from lbry.wallet.orchstr8.service import ConductorService def get_argument_parser(): parser = argparse.ArgumentParser( prog="orchstr8" ) subparsers = parser.add_subparsers(dest='command', help='sub-command help') subparsers.add_parser("download", help="Download lbcd and lbcwallet node binaries.") start = subparsers.add_parser("start", help="Start orchstr8 service.") start.add_argument("--lbcd", help="Hostname to start lbcd node.") start.add_argument("--lbcwallet", help="Hostname to start lbcwallet node.") start.add_argument("--spv", help="Hostname to start SPV server.") start.add_argument("--wallet", help="Hostname to start wallet daemon.") generate = subparsers.add_parser("generate", help="Call generate method on running orchstr8 instance.") generate.add_argument("blocks", type=int, help="Number of blocks to generate") subparsers.add_parser("transfer", help="Call transfer method on running orchstr8 instance.") return parser async def run_remote_command(command, **kwargs): async with aiohttp.ClientSession() as session: async with session.post('http://localhost:7954/'+command, data=kwargs) as resp: print(resp.status) print(await resp.text()) def main(): parser = get_argument_parser() args = parser.parse_args() command = getattr(args, 'command', 'help') loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) if command == 'download': logging.getLogger('blockchain').setLevel(logging.INFO) get_lbcd_node_from_ledger(wallet).ensure() get_lbcwallet_node_from_ledger(wallet).ensure() elif command == 'generate': loop.run_until_complete(run_remote_command( 'generate', blocks=args.blocks )) elif command == 'start': conductor = Conductor() if getattr(args, 'lbcd', False): conductor.lbcd_node.hostname = args.lbcd loop.run_until_complete(conductor.start_lbcd()) if getattr(args, 'lbcwallet', False): conductor.lbcwallet_node.hostname = args.lbcwallet loop.run_until_complete(conductor.start_lbcwallet()) if getattr(args, 'spv', False): conductor.spv_node.hostname = args.spv loop.run_until_complete(conductor.start_spv()) if getattr(args, 'wallet', False): conductor.wallet_node.hostname = args.wallet loop.run_until_complete(conductor.start_wallet()) service = ConductorService(conductor, loop) loop.run_until_complete(service.start()) try: print('========== Orchstr8 API Service Started ========') loop.run_forever() except KeyboardInterrupt: pass finally: loop.run_until_complete(service.stop()) loop.run_until_complete(conductor.stop()) loop.close() else: parser.print_help() if __name__ == "__main__": main() ================================================ FILE: lbry/wallet/orchstr8/node.py ================================================ # pylint: disable=import-error import os import json import shutil import asyncio import zipfile import tarfile import logging import tempfile import subprocess import platform from binascii import hexlify from typing import Type, Optional import urllib.request from uuid import uuid4 import lbry from lbry.wallet import Wallet, Ledger, RegTestLedger, WalletManager, Account, BlockHeightEvent from lbry.conf import KnownHubsList, Config log = logging.getLogger(__name__) try: from hub.herald.env import ServerEnv from hub.scribe.env import BlockchainEnv from hub.elastic_sync.env import ElasticEnv from hub.herald.service import HubServerService from hub.elastic_sync.service import ElasticSyncService from hub.scribe.service import BlockchainProcessorService except ImportError: pass def get_lbcd_node_from_ledger(ledger_module): return LBCDNode( ledger_module.__lbcd_url__, ledger_module.__lbcd__, ledger_module.__lbcctl__ ) def get_lbcwallet_node_from_ledger(ledger_module): return LBCWalletNode( ledger_module.__lbcwallet_url__, ledger_module.__lbcwallet__, ledger_module.__lbcctl__ ) class Conductor: def __init__(self, seed=None): self.manager_module = WalletManager self.lbcd_node = get_lbcd_node_from_ledger(lbry.wallet) self.lbcwallet_node = get_lbcwallet_node_from_ledger(lbry.wallet) self.spv_node = SPVNode() self.wallet_node = WalletNode( self.manager_module, RegTestLedger, default_seed=seed ) self.lbcd_started = False self.lbcwallet_started = False self.spv_started = False self.wallet_started = False self.log = log.getChild('conductor') async def start_lbcd(self): if not self.lbcd_started: await self.lbcd_node.start() self.lbcd_started = True async def stop_lbcd(self, cleanup=True): if self.lbcd_started: await self.lbcd_node.stop(cleanup) self.lbcd_started = False async def start_spv(self): if not self.spv_started: await self.spv_node.start(self.lbcwallet_node) self.spv_started = True async def stop_spv(self, cleanup=True): if self.spv_started: await self.spv_node.stop(cleanup) self.spv_started = False async def start_wallet(self): if not self.wallet_started: await self.wallet_node.start(self.spv_node) self.wallet_started = True async def stop_wallet(self, cleanup=True): if self.wallet_started: await self.wallet_node.stop(cleanup) self.wallet_started = False async def start_lbcwallet(self, clean=True): if not self.lbcwallet_started: await self.lbcwallet_node.start() if clean: mining_addr = await self.lbcwallet_node.get_new_address() self.lbcwallet_node.mining_addr = mining_addr await self.lbcwallet_node.generate(200) # unlock the wallet for the next 1 hour await self.lbcwallet_node.wallet_passphrase("password", 3600) self.lbcwallet_started = True async def stop_lbcwallet(self, cleanup=True): if self.lbcwallet_started: await self.lbcwallet_node.stop(cleanup) self.lbcwallet_started = False async def start(self): await self.start_lbcd() await self.start_lbcwallet() await self.start_spv() await self.start_wallet() async def stop(self): all_the_stops = [ self.stop_wallet, self.stop_spv, self.stop_lbcwallet, self.stop_lbcd ] for stop in all_the_stops: try: await stop() except Exception as e: log.exception('Exception raised while stopping services:', exc_info=e) async def clear_mempool(self): await self.stop_lbcwallet(cleanup=False) await self.stop_lbcd(cleanup=False) await self.start_lbcd() await self.start_lbcwallet(clean=False) class WalletNode: def __init__(self, manager_class: Type[WalletManager], ledger_class: Type[Ledger], verbose: bool = False, port: int = 5280, default_seed: str = None, data_path: str = None) -> None: self.manager_class = manager_class self.ledger_class = ledger_class self.verbose = verbose self.manager: Optional[WalletManager] = None self.ledger: Optional[Ledger] = None self.wallet: Optional[Wallet] = None self.account: Optional[Account] = None self.data_path: str = data_path or tempfile.mkdtemp() self.port = port self.default_seed = default_seed self.known_hubs = KnownHubsList() async def start(self, spv_node: 'SPVNode', seed=None, connect=True, config=None): wallets_dir = os.path.join(self.data_path, 'wallets') wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json') if not os.path.isdir(wallets_dir): os.mkdir(wallets_dir) with open(wallet_file_name, 'w') as wallet_file: wallet_file.write('{"version": 1, "accounts": []}\n') self.manager = self.manager_class.from_config({ 'ledgers': { self.ledger_class.get_id(): { 'api_port': self.port, 'explicit_servers': [(spv_node.hostname, spv_node.port)], 'default_servers': Config.lbryum_servers.default, 'data_path': self.data_path, 'known_hubs': config.known_hubs if config else KnownHubsList(), 'hub_timeout': 30, 'concurrent_hub_requests': 32, 'fee_per_name_char': 200000 } }, 'wallets': [wallet_file_name] }) self.manager.config = config self.ledger = self.manager.ledgers[self.ledger_class] self.wallet = self.manager.default_wallet if not self.wallet: raise ValueError('Wallet is required.') if seed or self.default_seed: Account.from_dict( self.ledger, self.wallet, {'seed': seed or self.default_seed} ) else: self.wallet.generate_account(self.ledger) self.account = self.wallet.default_account if connect: await self.manager.start() async def stop(self, cleanup=True): try: await self.manager.stop() finally: cleanup and self.cleanup() def cleanup(self): shutil.rmtree(self.data_path, ignore_errors=True) class SPVNode: def __init__(self, node_number=1): self.node_number = node_number self.controller = None self.data_path = None self.server: Optional[HubServerService] = None self.writer: Optional[BlockchainProcessorService] = None self.es_writer: Optional[ElasticSyncService] = None self.hostname = 'localhost' self.port = 50001 + node_number # avoid conflict with default daemon self.udp_port = self.port self.elastic_notifier_port = 19080 + node_number self.elastic_services = f'localhost:9200/localhost:{self.elastic_notifier_port}' self.session_timeout = 600 self.stopped = True self.index_name = uuid4().hex async def start(self, lbcwallet_node: 'LBCWalletNode', extraconf=None): if not self.stopped: log.warning("spv node is already running") return self.stopped = False try: self.data_path = tempfile.mkdtemp() conf = { 'description': '', 'payment_address': '', 'daily_fee': '0', 'db_dir': self.data_path, 'daemon_url': lbcwallet_node.rpc_url, 'reorg_limit': 100, 'host': self.hostname, 'tcp_port': self.port, 'udp_port': self.udp_port, 'elastic_services': self.elastic_services, 'session_timeout': self.session_timeout, 'max_query_workers': 0, 'es_index_prefix': self.index_name, 'chain': 'regtest', 'index_address_status': False } if extraconf: conf.update(extraconf) self.writer = BlockchainProcessorService( BlockchainEnv(db_dir=self.data_path, daemon_url=lbcwallet_node.rpc_url, reorg_limit=100, max_query_workers=0, chain='regtest', index_address_status=False) ) self.server = HubServerService(ServerEnv(**conf)) self.es_writer = ElasticSyncService( ElasticEnv( db_dir=self.data_path, reorg_limit=100, max_query_workers=0, chain='regtest', elastic_notifier_port=self.elastic_notifier_port, es_index_prefix=self.index_name, filtering_channel_ids=(extraconf or {}).get('filtering_channel_ids'), blocking_channel_ids=(extraconf or {}).get('blocking_channel_ids') ) ) await self.writer.start() await self.es_writer.start() await self.server.start() except Exception as e: self.stopped = True log.exception("failed to start spv node") raise e async def stop(self, cleanup=True): if self.stopped: log.warning("spv node is already stopped") return try: await self.server.stop() await self.es_writer.delete_index() await self.es_writer.stop() await self.writer.stop() self.stopped = True except Exception as e: log.exception("failed to stop spv node") raise e finally: cleanup and self.cleanup() def cleanup(self): shutil.rmtree(self.data_path, ignore_errors=True) class LBCDProcess(asyncio.SubprocessProtocol): IGNORE_OUTPUT = [ b'keypool keep', b'keypool reserve', b'keypool return', b'Block submitted', ] def __init__(self): self.ready = asyncio.Event() self.stopped = asyncio.Event() self.log = log.getChild('lbcd') def pipe_data_received(self, fd, data): if self.log and not any(ignore in data for ignore in self.IGNORE_OUTPUT): if b'Error:' in data: self.log.error(data.decode()) else: self.log.info(data.decode()) if b'Error:' in data: self.ready.set() raise SystemError(data.decode()) if b'RPCS: RPC server listening on' in data: self.ready.set() def process_exited(self): self.stopped.set() self.ready.set() class WalletProcess(asyncio.SubprocessProtocol): IGNORE_OUTPUT = [ ] def __init__(self): self.ready = asyncio.Event() self.stopped = asyncio.Event() self.log = log.getChild('lbcwallet') self.transport: Optional[asyncio.transports.SubprocessTransport] = None def pipe_data_received(self, fd, data): if self.log and not any(ignore in data for ignore in self.IGNORE_OUTPUT): if b'Error:' in data: self.log.error(data.decode()) else: self.log.info(data.decode()) if b'Error:' in data: self.ready.set() raise SystemError(data.decode()) if b'WLLT: Finished rescan' in data: self.ready.set() def process_exited(self): self.stopped.set() self.ready.set() class LBCDNode: def __init__(self, url, daemon, cli): self.latest_release_url = url self.project_dir = os.path.dirname(os.path.dirname(__file__)) self.bin_dir = os.path.join(self.project_dir, 'bin') self.daemon_bin = os.path.join(self.bin_dir, daemon) self.cli_bin = os.path.join(self.bin_dir, cli) self.log = log.getChild('lbcd') self.data_path = tempfile.mkdtemp() self.protocol = None self.transport = None self.hostname = 'localhost' self.peerport = 29246 self.rpcport = 29245 self.rpcuser = 'rpcuser' self.rpcpassword = 'rpcpassword' self.stopped = True self.running = asyncio.Event() @property def rpc_url(self): return f'http://{self.rpcuser}:{self.rpcpassword}@{self.hostname}:{self.rpcport}/' @property def exists(self): return ( os.path.exists(self.cli_bin) and os.path.exists(self.daemon_bin) ) def download(self): uname = platform.uname() target_os = str.lower(uname.system) target_arch = str.replace(uname.machine, 'x86_64', 'amd64') target_platform = target_os + '_' + target_arch self.latest_release_url = str.replace(self.latest_release_url, 'TARGET_PLATFORM', target_platform) downloaded_file = os.path.join( self.bin_dir, self.latest_release_url[self.latest_release_url.rfind('/')+1:] ) if not os.path.exists(self.bin_dir): os.mkdir(self.bin_dir) if not os.path.exists(downloaded_file): self.log.info('Downloading: %s', self.latest_release_url) with urllib.request.urlopen(self.latest_release_url) as response: with open(downloaded_file, 'wb') as out_file: shutil.copyfileobj(response, out_file) self.log.info('Extracting: %s', downloaded_file) if downloaded_file.endswith('.zip'): with zipfile.ZipFile(downloaded_file) as dotzip: dotzip.extractall(self.bin_dir) # zipfile bug https://bugs.python.org/issue15795 os.chmod(self.cli_bin, 0o755) os.chmod(self.daemon_bin, 0o755) elif downloaded_file.endswith('.tar.gz'): with tarfile.open(downloaded_file) as tar: tar.extractall(self.bin_dir) return self.exists def ensure(self): return self.exists or self.download() async def start(self): if not self.stopped: return self.stopped = False try: assert self.ensure() loop = asyncio.get_event_loop() asyncio.get_child_watcher().attach_loop(loop) command = [ self.daemon_bin, '--notls', f'--datadir={self.data_path}', '--regtest', f'--listen=127.0.0.1:{self.peerport}', f'--rpclisten=127.0.0.1:{self.rpcport}', '--txindex', f'--rpcuser={self.rpcuser}', f'--rpcpass={self.rpcpassword}' ] self.log.info(' '.join(command)) self.transport, self.protocol = await loop.subprocess_exec( LBCDProcess, *command ) await self.protocol.ready.wait() assert not self.protocol.stopped.is_set() self.running.set() except asyncio.CancelledError: self.running.clear() self.stopped = True raise except Exception as e: self.running.clear() self.stopped = True log.exception('failed to start lbcd', exc_info=e) raise async def stop(self, cleanup=True): if self.stopped: return try: if self.transport: self.transport.terminate() await self.protocol.stopped.wait() self.transport.close() except Exception as e: log.exception('failed to stop lbcd', exc_info=e) raise finally: self.log.info("Done shutting down " + self.daemon_bin) self.stopped = True if cleanup: self.cleanup() self.running.clear() def cleanup(self): assert self.stopped shutil.rmtree(self.data_path, ignore_errors=True) class LBCWalletNode: P2SH_SEGWIT_ADDRESS = "p2sh-segwit" BECH32_ADDRESS = "bech32" def __init__(self, url, lbcwallet, cli): self.latest_release_url = url self.project_dir = os.path.dirname(os.path.dirname(__file__)) self.bin_dir = os.path.join(self.project_dir, 'bin') self.lbcwallet_bin = os.path.join(self.bin_dir, lbcwallet) self.cli_bin = os.path.join(self.bin_dir, cli) self.log = log.getChild('lbcwallet') self.protocol = None self.transport = None self.hostname = 'localhost' self.lbcd_rpcport = 29245 self.lbcwallet_rpcport = 29244 self.rpcuser = 'rpcuser' self.rpcpassword = 'rpcpassword' self.data_path = tempfile.mkdtemp() self.stopped = True self.running = asyncio.Event() self.block_expected = 0 self.mining_addr = '' @property def rpc_url(self): # FIXME: somehow the hub/sdk doesn't learn the blocks through the Walet RPC port, why? # return f'http://{self.rpcuser}:{self.rpcpassword}@{self.hostname}:{self.lbcwallet_rpcport}/' return f'http://{self.rpcuser}:{self.rpcpassword}@{self.hostname}:{self.lbcd_rpcport}/' def is_expected_block(self, e: BlockHeightEvent): return self.block_expected == e.height @property def exists(self): return ( os.path.exists(self.lbcwallet_bin) ) def download(self): uname = platform.uname() target_os = str.lower(uname.system) target_arch = str.replace(uname.machine, 'x86_64', 'amd64') target_platform = target_os + '_' + target_arch self.latest_release_url = str.replace(self.latest_release_url, 'TARGET_PLATFORM', target_platform) downloaded_file = os.path.join( self.bin_dir, self.latest_release_url[self.latest_release_url.rfind('/')+1:] ) if not os.path.exists(self.bin_dir): os.mkdir(self.bin_dir) if not os.path.exists(downloaded_file): self.log.info('Downloading: %s', self.latest_release_url) with urllib.request.urlopen(self.latest_release_url) as response: with open(downloaded_file, 'wb') as out_file: shutil.copyfileobj(response, out_file) self.log.info('Extracting: %s', downloaded_file) if downloaded_file.endswith('.zip'): with zipfile.ZipFile(downloaded_file) as dotzip: dotzip.extractall(self.bin_dir) # zipfile bug https://bugs.python.org/issue15795 os.chmod(self.lbcwallet_bin, 0o755) elif downloaded_file.endswith('.tar.gz'): with tarfile.open(downloaded_file) as tar: tar.extractall(self.bin_dir) return self.exists def ensure(self): return self.exists or self.download() async def start(self): assert self.ensure() loop = asyncio.get_event_loop() asyncio.get_child_watcher().attach_loop(loop) command = [ self.lbcwallet_bin, '--noservertls', '--noclienttls', '--regtest', f'--rpcconnect=127.0.0.1:{self.lbcd_rpcport}', f'--rpclisten=127.0.0.1:{self.lbcwallet_rpcport}', '--createtemp', f'--appdata={self.data_path}', f'--username={self.rpcuser}', f'--password={self.rpcpassword}' ] self.log.info(' '.join(command)) try: self.transport, self.protocol = await loop.subprocess_exec( WalletProcess, *command ) self.protocol.transport = self.transport await self.protocol.ready.wait() assert not self.protocol.stopped.is_set() self.running.set() self.stopped = False except asyncio.CancelledError: self.running.clear() raise except Exception as e: self.running.clear() log.exception('failed to start lbcwallet', exc_info=e) def cleanup(self): assert self.stopped shutil.rmtree(self.data_path, ignore_errors=True) async def stop(self, cleanup=True): if self.stopped: return try: self.transport.terminate() await self.protocol.stopped.wait() self.transport.close() except Exception as e: log.exception('failed to stop lbcwallet', exc_info=e) raise finally: self.log.info("Done shutting down " + self.lbcwallet_bin) self.stopped = True if cleanup: self.cleanup() self.running.clear() async def _cli_cmnd(self, *args): cmnd_args = [ self.cli_bin, f'--rpcuser={self.rpcuser}', f'--rpcpass={self.rpcpassword}', '--notls', '--regtest', '--wallet' ] + list(args) self.log.info(' '.join(cmnd_args)) loop = asyncio.get_event_loop() asyncio.get_child_watcher().attach_loop(loop) process = await asyncio.create_subprocess_exec( *cmnd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) out, err = await process.communicate() result = out.decode().strip() err = err.decode().strip() if len(result) <= 0 and err.startswith('-'): raise Exception(err) if err and 'creating a default config file' not in err: log.warning(err) self.log.info(result) if result.startswith('error code'): raise Exception(result) return result def generate(self, blocks): self.block_expected += blocks return self._cli_cmnd('generatetoaddress', str(blocks), self.mining_addr) def generate_to_address(self, blocks, addr): self.block_expected += blocks return self._cli_cmnd('generatetoaddress', str(blocks), addr) def wallet_passphrase(self, passphrase, timeout): return self._cli_cmnd('walletpassphrase', passphrase, str(timeout)) def invalidate_block(self, blockhash): return self._cli_cmnd('invalidateblock', blockhash) def get_block_hash(self, block): return self._cli_cmnd('getblockhash', str(block)) def sendrawtransaction(self, tx): return self._cli_cmnd('sendrawtransaction', tx) async def get_block(self, block_hash): return json.loads(await self._cli_cmnd('getblock', block_hash, '1')) def get_raw_change_address(self): return self._cli_cmnd('getrawchangeaddress') def get_new_address(self, address_type='legacy'): return self._cli_cmnd('getnewaddress', "", address_type) async def get_balance(self): return await self._cli_cmnd('getbalance') def send_to_address(self, address, amount): return self._cli_cmnd('sendtoaddress', address, str(amount)) def send_raw_transaction(self, tx): return self._cli_cmnd('sendrawtransaction', tx.decode()) def create_raw_transaction(self, inputs, outputs): return self._cli_cmnd('createrawtransaction', json.dumps(inputs), json.dumps(outputs)) async def sign_raw_transaction_with_wallet(self, tx): # the "withwallet" portion should only come into play if we are doing segwit. # and "withwallet" doesn't exist on lbcd yet. result = await self._cli_cmnd('signrawtransaction', tx) return json.loads(result)['hex'].encode() def decode_raw_transaction(self, tx): return self._cli_cmnd('decoderawtransaction', hexlify(tx.raw).decode()) def get_raw_transaction(self, txid): return self._cli_cmnd('getrawtransaction', txid, '1') ================================================ FILE: lbry/wallet/orchstr8/service.py ================================================ import asyncio import logging from aiohttp.web import Application, WebSocketResponse, json_response from aiohttp.http_websocket import WSMsgType, WSCloseCode from lbry.wallet.util import satoshis_to_coins from .node import Conductor PORT = 7954 class WebSocketLogHandler(logging.Handler): def __init__(self, send_message): super().__init__() self.send_message = send_message def emit(self, record): try: self.send_message({ 'type': 'log', 'name': record.name, 'message': self.format(record) }) except Exception: self.handleError(record) class ConductorService: def __init__(self, stack: Conductor, loop: asyncio.AbstractEventLoop) -> None: self.stack = stack self.loop = loop self.app = Application() self.app.router.add_post('/start', self.start_stack) self.app.router.add_post('/generate', self.generate) self.app.router.add_post('/transfer', self.transfer) self.app.router.add_post('/balance', self.balance) self.app.router.add_get('/log', self.log) self.app['websockets'] = set() self.app.on_shutdown.append(self.on_shutdown) self.handler = self.app.make_handler() self.server = None async def start(self): self.server = await self.loop.create_server( self.handler, '0.0.0.0', PORT ) print('serving on', self.server.sockets[0].getsockname()) async def stop(self): await self.stack.stop() self.server.close() await self.server.wait_closed() await self.app.shutdown() await self.handler.shutdown(60.0) await self.app.cleanup() async def start_stack(self, _): #set_logging( # self.stack.ledger_module, logging.DEBUG, WebSocketLogHandler(self.send_message) #) self.stack.lbcd_started or await self.stack.start_lbcd() self.send_message({'type': 'service', 'name': 'lbcd', 'port': self.stack.lbcd_node.port}) self.stack.lbcwallet_started or await self.stack.start_lbcwallet() self.send_message({'type': 'service', 'name': 'lbcwallet', 'port': self.stack.lbcwallet_node.port}) self.stack.spv_started or await self.stack.start_spv() self.send_message({'type': 'service', 'name': 'spv', 'port': self.stack.spv_node.port}) self.stack.wallet_started or await self.stack.start_wallet() self.send_message({'type': 'service', 'name': 'wallet', 'port': self.stack.wallet_node.port}) self.stack.wallet_node.ledger.on_header.listen(self.on_status) self.stack.wallet_node.ledger.on_transaction.listen(self.on_status) return json_response({'started': True}) async def generate(self, request): data = await request.post() blocks = data.get('blocks', 1) await self.stack.lbcwallet_node.generate(int(blocks)) return json_response({'blocks': blocks}) async def transfer(self, request): data = await request.post() address = data.get('address') if not address and self.stack.wallet_started: address = await self.stack.wallet_node.account.receiving.get_or_create_usable_address() if not address: raise ValueError("No address was provided.") amount = data.get('amount', 1) if self.stack.wallet_started: watcher = self.stack.wallet_node.ledger.on_transaction.where( lambda e: e.address == address # and e.tx.id == txid -- might stall; see send_to_address_and_wait ) txid = await self.stack.lbcwallet_node.send_to_address(address, amount) await watcher else: txid = await self.stack.lbcwallet_node.send_to_address(address, amount) return json_response({ 'address': address, 'amount': amount, 'txid': txid }) async def balance(self, _): return json_response({ 'balance': await self.stack.lbcwallet_node.get_balance() }) async def log(self, request): web_socket = WebSocketResponse() await web_socket.prepare(request) self.app['websockets'].add(web_socket) try: async for msg in web_socket: if msg.type == WSMsgType.TEXT: if msg.data == 'close': await web_socket.close() elif msg.type == WSMsgType.ERROR: print('web socket connection closed with exception %s' % web_socket.exception()) finally: self.app['websockets'].remove(web_socket) return web_socket @staticmethod async def on_shutdown(app): for web_socket in app['websockets']: await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown') async def on_status(self, _): if not self.app['websockets']: return self.send_message({ 'type': 'status', 'height': self.stack.wallet_node.ledger.headers.height, 'balance': satoshis_to_coins(await self.stack.wallet_node.account.get_balance()), 'miner': await self.stack.lbcwallet_node.get_balance() }) def send_message(self, msg): for web_socket in self.app['websockets']: self.loop.create_task(web_socket.send_json(msg)) ================================================ FILE: lbry/wallet/rpc/__init__.py ================================================ from .framing import * from .jsonrpc import * from .socks import * from .session import * from .util import * __all__ = (framing.__all__ + jsonrpc.__all__ + socks.__all__ + session.__all__ + util.__all__) ================================================ FILE: lbry/wallet/rpc/framing.py ================================================ # Copyright (c) 2018, Neil Booth # # All rights reserved. # # The MIT License (MIT) # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """RPC message framing in a byte stream.""" __all__ = ('FramerBase', 'NewlineFramer', 'BinaryFramer', 'BitcoinFramer', 'OversizedPayloadError', 'BadChecksumError', 'BadMagicError') from hashlib import sha256 as _sha256 from struct import Struct from asyncio import Queue class FramerBase: """Abstract base class for a framer. A framer breaks an incoming byte stream into protocol messages, buffering if necessary. It also frames outgoing messages into a byte stream. """ def frame(self, message): """Return the framed message.""" raise NotImplementedError def received_bytes(self, data): """Pass incoming network bytes.""" raise NotImplementedError async def receive_message(self): """Wait for a complete unframed message to arrive, and return it.""" raise NotImplementedError class NewlineFramer(FramerBase): """A framer for a protocol where messages are separated by newlines.""" # The default max_size value is motivated by JSONRPC, where a # normal request will be 250 bytes or less, and a reasonable # batch may contain 4000 requests. def __init__(self, max_size=250 * 4000): """max_size - an anti-DoS measure. If, after processing an incoming message, buffered data would exceed max_size bytes, that buffered data is dropped entirely and the framer waits for a newline character to re-synchronize the stream. """ self.max_size = max_size self.queue = Queue() self.received_bytes = self.queue.put_nowait self.synchronizing = False self.residual = b'' def frame(self, message): return message + b'\n' async def receive_message(self): parts = [] buffer_size = 0 while True: part = self.residual self.residual = b'' if not part: part = await self.queue.get() npos = part.find(b'\n') if npos == -1: parts.append(part) buffer_size += len(part) # Ignore over-sized messages; re-synchronize if buffer_size <= self.max_size: continue self.synchronizing = True raise MemoryError(f'dropping message over {self.max_size:,d} ' f'bytes and re-synchronizing') tail, self.residual = part[:npos], part[npos + 1:] if self.synchronizing: self.synchronizing = False return await self.receive_message() else: parts.append(tail) return b''.join(parts) class ByteQueue: """A producer-comsumer queue. Incoming network data is put as it arrives, and the consumer calls an async method waiting for data of a specific length.""" def __init__(self): self.queue = Queue() self.parts = [] self.parts_len = 0 self.put_nowait = self.queue.put_nowait async def receive(self, size): while self.parts_len < size: part = await self.queue.get() self.parts.append(part) self.parts_len += len(part) self.parts_len -= size whole = b''.join(self.parts) self.parts = [whole[size:]] return whole[:size] class BinaryFramer: """A framer for binary messaging protocols.""" def __init__(self): self.byte_queue = ByteQueue() self.message_queue = Queue() self.received_bytes = self.byte_queue.put_nowait def frame(self, message): command, payload = message return b''.join(( self._build_header(command, payload), payload )) async def receive_message(self): command, payload_len, checksum = await self._receive_header() payload = await self.byte_queue.receive(payload_len) payload_checksum = self._checksum(payload) if payload_checksum != checksum: raise BadChecksumError(payload_checksum, checksum) return command, payload def _checksum(self, payload): raise NotImplementedError def _build_header(self, command, payload): raise NotImplementedError async def _receive_header(self): raise NotImplementedError # Helpers struct_le_I = Struct('<I') pack_le_uint32 = struct_le_I.pack def sha256(x): """Simple wrapper of hashlib sha256.""" return _sha256(x).digest() def double_sha256(x): """SHA-256 of SHA-256, as used extensively in bitcoin.""" return sha256(sha256(x)) class BadChecksumError(Exception): pass class BadMagicError(Exception): pass class OversizedPayloadError(Exception): pass class BitcoinFramer(BinaryFramer): """Provides a framer of binary message payloads in the style of the Bitcoin network protocol. Each binary message has the following elements, in order: Magic - to confirm network (currently unused for stream sync) Command - padded command Length - payload length in bytes Checksum - checksum of the payload Payload - binary payload Call frame(command, payload) to get a framed message. Pass incoming network bytes to received_bytes(). Wait on receive_message() to get incoming (command, payload) pairs. """ def __init__(self, magic, max_block_size): def pad_command(command): fill = 12 - len(command) if fill < 0: raise ValueError(f'command {command} too long') return command + bytes(fill) super().__init__() self._magic = magic self._max_block_size = max_block_size self._pad_command = pad_command self._unpack = Struct(f'<4s12sI4s').unpack def _checksum(self, payload): return double_sha256(payload)[:4] def _build_header(self, command, payload): return b''.join(( self._magic, self._pad_command(command), pack_le_uint32(len(payload)), self._checksum(payload) )) async def _receive_header(self): header = await self.byte_queue.receive(24) magic, command, payload_len, checksum = self._unpack(header) if magic != self._magic: raise BadMagicError(magic, self._magic) command = command.rstrip(b'\0') if payload_len > 1024 * 1024: if command != b'block' or payload_len > self._max_block_size: raise OversizedPayloadError(command, payload_len) return command, payload_len, checksum ================================================ FILE: lbry/wallet/rpc/jsonrpc.py ================================================ # Copyright (c) 2018, Neil Booth # # All rights reserved. # # The MIT License (MIT) # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """Classes for JSONRPC versions 1.0 and 2.0, and a loose interpretation.""" __all__ = ('JSONRPC', 'JSONRPCv1', 'JSONRPCv2', 'JSONRPCLoose', 'JSONRPCAutoDetect', 'Request', 'Notification', 'Batch', 'RPCError', 'ProtocolError', 'JSONRPCConnection', 'handler_invocation') import itertools import json import typing import asyncio from functools import partial from numbers import Number import attr from asyncio import Queue, Event, CancelledError from .util import signature_info class SingleRequest: __slots__ = ('method', 'args') def __init__(self, method, args): if not isinstance(method, str): raise ProtocolError(JSONRPC.METHOD_NOT_FOUND, 'method must be a string') if not isinstance(args, (list, tuple, dict)): raise ProtocolError.invalid_args('request arguments must be a ' 'list or a dictionary') self.args = args self.method = method def __repr__(self): return f'{self.__class__.__name__}({self.method!r}, {self.args!r})' def __eq__(self, other): return (isinstance(other, self.__class__) and self.method == other.method and self.args == other.args) class Request(SingleRequest): def send_result(self, response): return None class Notification(SingleRequest): pass class Batch: __slots__ = ('items', ) def __init__(self, items): if not isinstance(items, (list, tuple)): raise ProtocolError.invalid_request('items must be a list') if not items: raise ProtocolError.empty_batch() if not (all(isinstance(item, SingleRequest) for item in items) or all(isinstance(item, Response) for item in items)): raise ProtocolError.invalid_request('batch must be homogeneous') self.items = items def __len__(self): return len(self.items) def __getitem__(self, item): return self.items[item] def __iter__(self): return iter(self.items) def __repr__(self): return f'Batch({len(self.items)} items)' class Response: __slots__ = ('result', ) def __init__(self, result): # Type checking happens when converting to a message self.result = result class CodeMessageError(Exception): @property def code(self): return self.args[0] @property def message(self): return self.args[1] def __eq__(self, other): return (isinstance(other, self.__class__) and self.code == other.code and self.message == other.message) def __hash__(self): # overridden to make the exception hashable # see https://bugs.python.org/issue28603 return hash((self.code, self.message)) @classmethod def invalid_args(cls, message): return cls(JSONRPC.INVALID_ARGS, message) @classmethod def invalid_request(cls, message): return cls(JSONRPC.INVALID_REQUEST, message) @classmethod def empty_batch(cls): return cls.invalid_request('batch is empty') class RPCError(CodeMessageError): pass class ProtocolError(CodeMessageError): def __init__(self, code, message): super().__init__(code, message) # If not None send this unframed message over the network self.error_message = None # If the error was in a JSON response message; its message ID. # Since None can be a response message ID, "id" means the # error was not sent in a JSON response self.response_msg_id = id class JSONRPC: """Abstract base class that interprets and constructs JSON RPC messages.""" # Error codes. See http://www.jsonrpc.org/specification PARSE_ERROR = -32700 INVALID_REQUEST = -32600 METHOD_NOT_FOUND = -32601 INVALID_ARGS = -32602 INTERNAL_ERROR = -32603 QUERY_TIMEOUT = -32000 # Codes specific to this library ERROR_CODE_UNAVAILABLE = -100 # Can be overridden by derived classes allow_batches = True @classmethod def _message_id(cls, message, require_id): """Validate the message is a dictionary and return its ID. Raise an error if the message is invalid or the ID is of an invalid type. If it has no ID, raise an error if require_id is True, otherwise return None. """ raise NotImplementedError @classmethod def _validate_message(cls, message): """Validate other parts of the message other than those done in _message_id.""" pass @classmethod def _request_args(cls, request): """Validate the existence and type of the arguments passed in the request dictionary.""" raise NotImplementedError @classmethod def _process_request(cls, payload): request_id = None try: request_id = cls._message_id(payload, False) cls._validate_message(payload) method = payload.get('method') if request_id is None: item = Notification(method, cls._request_args(payload)) else: item = Request(method, cls._request_args(payload)) return item, request_id except ProtocolError as error: code, message = error.code, error.message raise cls._error(code, message, True, request_id) @classmethod def _process_response(cls, payload): request_id = None try: request_id = cls._message_id(payload, True) cls._validate_message(payload) return Response(cls.response_value(payload)), request_id except ProtocolError as error: code, message = error.code, error.message raise cls._error(code, message, False, request_id) @classmethod def _message_to_payload(cls, message): """Returns a Python object or a ProtocolError.""" try: return json.loads(message.decode()) except UnicodeDecodeError: message = 'messages must be encoded in UTF-8' except json.JSONDecodeError: message = 'invalid JSON' raise cls._error(cls.PARSE_ERROR, message, True, None) @classmethod def _error(cls, code, message, send, msg_id): error = ProtocolError(code, message) if send: error.error_message = cls.response_message(error, msg_id) else: error.response_msg_id = msg_id return error # # External API # @classmethod def message_to_item(cls, message): """Translate an unframed received message and return an (item, request_id) pair. The item can be a Request, Notification, Response or a list. A JSON RPC error response is returned as an RPCError inside a Response object. If a Batch is returned, request_id is an iterable of request ids, one per batch member. If the message violates the protocol in some way a ProtocolError is returned, except if the message was determined to be a response, in which case the ProtocolError is placed inside a Response object. This is so that client code can mark a request as having been responded to even if the response was bad. raises: ProtocolError """ payload = cls._message_to_payload(message) if isinstance(payload, dict): if 'method' in payload: return cls._process_request(payload) else: return cls._process_response(payload) elif isinstance(payload, list) and cls.allow_batches: if not payload: raise cls._error(JSONRPC.INVALID_REQUEST, 'batch is empty', True, None) return payload, None raise cls._error(cls.INVALID_REQUEST, 'request object must be a dictionary', True, None) # Message formation @classmethod def request_message(cls, item, request_id): """Convert an RPCRequest item to a message.""" assert isinstance(item, Request) return cls.encode_payload(cls.request_payload(item, request_id)) @classmethod def notification_message(cls, item): """Convert an RPCRequest item to a message.""" assert isinstance(item, Notification) return cls.encode_payload(cls.request_payload(item, None)) @classmethod def response_message(cls, result, request_id): """Convert a response result (or RPCError) to a message.""" if isinstance(result, CodeMessageError): payload = cls.error_payload(result, request_id) else: payload = cls.response_payload(result, request_id) return cls.encode_payload(payload) @classmethod def batch_message(cls, batch, request_ids): """Convert a request Batch to a message.""" assert isinstance(batch, Batch) if not cls.allow_batches: raise ProtocolError.invalid_request( 'protocol does not permit batches') id_iter = iter(request_ids) rm = cls.request_message nm = cls.notification_message parts = (rm(request, next(id_iter)) if isinstance(request, Request) else nm(request) for request in batch) return cls.batch_message_from_parts(parts) @classmethod def batch_message_from_parts(cls, messages): """Convert messages, one per batch item, into a batch message. At least one message must be passed. """ # Comma-separate the messages and wrap the lot in square brackets middle = b', '.join(messages) if not middle: raise ProtocolError.empty_batch() return b''.join([b'[', middle, b']']) @classmethod def encode_payload(cls, payload): """Encode a Python object as JSON and convert it to bytes.""" try: return json.dumps(payload).encode() except TypeError: msg = f'JSON payload encoding error: {payload}' raise ProtocolError(cls.INTERNAL_ERROR, msg) from None class JSONRPCv1(JSONRPC): """JSON RPC version 1.0.""" allow_batches = False @classmethod def _message_id(cls, message, require_id): # JSONv1 requires an ID always, but without constraint on its type # No need to test for a dictionary here as we don't handle batches. if 'id' not in message: raise ProtocolError.invalid_request('request has no "id"') return message['id'] @classmethod def _request_args(cls, request): args = request.get('params') if not isinstance(args, list): raise ProtocolError.invalid_args( f'invalid request arguments: {args}') return args @classmethod def _best_effort_error(cls, error): # Do our best to interpret the error code = cls.ERROR_CODE_UNAVAILABLE message = 'no error message provided' if isinstance(error, str): message = error elif isinstance(error, int): code = error elif isinstance(error, dict): if isinstance(error.get('message'), str): message = error['message'] if isinstance(error.get('code'), int): code = error['code'] return RPCError(code, message) @classmethod def response_value(cls, payload): if 'result' not in payload or 'error' not in payload: raise ProtocolError.invalid_request( 'response must contain both "result" and "error"') result = payload['result'] error = payload['error'] if error is None: return result # It seems None can be a valid result if result is not None: raise ProtocolError.invalid_request( 'response has a "result" and an "error"') return cls._best_effort_error(error) @classmethod def request_payload(cls, request, request_id): """JSON v1 request (or notification) payload.""" if isinstance(request.args, dict): raise ProtocolError.invalid_args( 'JSONRPCv1 does not support named arguments') return { 'method': request.method, 'params': request.args, 'id': request_id } @classmethod def response_payload(cls, result, request_id): """JSON v1 response payload.""" return { 'result': result, 'error': None, 'id': request_id } @classmethod def error_payload(cls, error, request_id): return { 'result': None, 'error': {'code': error.code, 'message': error.message}, 'id': request_id } class JSONRPCv2(JSONRPC): """JSON RPC version 2.0.""" @classmethod def _message_id(cls, message, require_id): if not isinstance(message, dict): raise ProtocolError.invalid_request( 'request object must be a dictionary') if 'id' in message: request_id = message['id'] if not isinstance(request_id, (Number, str, type(None))): raise ProtocolError.invalid_request( f'invalid "id": {request_id}') return request_id else: if require_id: raise ProtocolError.invalid_request('request has no "id"') return None @classmethod def _validate_message(cls, message): if message.get('jsonrpc') != '2.0': raise ProtocolError.invalid_request('"jsonrpc" is not "2.0"') @classmethod def _request_args(cls, request): args = request.get('params', []) if not isinstance(args, (dict, list)): raise ProtocolError.invalid_args( f'invalid request arguments: {args}') return args @classmethod def response_value(cls, payload): if 'result' in payload: if 'error' in payload: raise ProtocolError.invalid_request( 'response contains both "result" and "error"') return payload['result'] if 'error' not in payload: raise ProtocolError.invalid_request( 'response contains neither "result" nor "error"') # Return an RPCError object error = payload['error'] if isinstance(error, dict): code = error.get('code') message = error.get('message') if isinstance(code, int) and isinstance(message, str): return RPCError(code, message) raise ProtocolError.invalid_request( f'ill-formed response error object: {error}') @classmethod def request_payload(cls, request, request_id): """JSON v2 request (or notification) payload.""" payload = { 'jsonrpc': '2.0', 'method': request.method, } # A notification? if request_id is not None: payload['id'] = request_id # Preserve empty dicts as missing params is read as an array if request.args or request.args == {}: payload['params'] = request.args return payload @classmethod def response_payload(cls, result, request_id): """JSON v2 response payload.""" return { 'jsonrpc': '2.0', 'result': result, 'id': request_id } @classmethod def error_payload(cls, error, request_id): return { 'jsonrpc': '2.0', 'error': {'code': error.code, 'message': error.message}, 'id': request_id } class JSONRPCLoose(JSONRPC): """A relaxed version of JSON RPC.""" # Don't be so loose we accept any old message ID _message_id = JSONRPCv2._message_id _validate_message = JSONRPC._validate_message _request_args = JSONRPCv2._request_args # Outoing messages are JSONRPCv2 so we give the other side the # best chance to assume / detect JSONRPCv2 as default protocol. error_payload = JSONRPCv2.error_payload request_payload = JSONRPCv2.request_payload response_payload = JSONRPCv2.response_payload @classmethod def response_value(cls, payload): # Return result, unless it is None and there is an error if payload.get('error') is not None: if payload.get('result') is not None: raise ProtocolError.invalid_request( 'response contains both "result" and "error"') return JSONRPCv1._best_effort_error(payload['error']) if 'result' not in payload: raise ProtocolError.invalid_request( 'response contains neither "result" nor "error"') # Can be None return payload['result'] class JSONRPCAutoDetect(JSONRPCv2): @classmethod def message_to_item(cls, message): return cls.detect_protocol(message), None @classmethod def detect_protocol(cls, message): """Attempt to detect the protocol from the message.""" main = cls._message_to_payload(message) def protocol_for_payload(payload): if not isinstance(payload, dict): return JSONRPCLoose # Will error # Obey an explicit "jsonrpc" version = payload.get('jsonrpc') if version == '2.0': return JSONRPCv2 if version == '1.0': return JSONRPCv1 # Now to decide between JSONRPCLoose and JSONRPCv1 if possible if 'result' in payload and 'error' in payload: return JSONRPCv1 return JSONRPCLoose if isinstance(main, list): parts = {protocol_for_payload(payload) for payload in main} # If all same protocol, return it if len(parts) == 1: return parts.pop() # If strict protocol detected, return it, preferring JSONRPCv2. # This means a batch of JSONRPCv1 will fail for protocol in (JSONRPCv2, JSONRPCv1): if protocol in parts: return protocol # Will error if no parts return JSONRPCLoose return protocol_for_payload(main) class JSONRPCConnection: """Maintains state of a JSON RPC connection, in particular encapsulating the handling of request IDs. protocol - the JSON RPC protocol to follow max_response_size - responses over this size send an error response instead. """ _id_counter = itertools.count() def __init__(self, protocol): self._protocol = protocol # Sent Requests and Batches that have not received a response. # The key is its request ID; for a batch it is sorted tuple # of request IDs self._requests: typing.Dict[str, typing.Tuple[Request, Event]] = {} # A public attribute intended to be settable dynamically self.max_response_size = 0 def _oversized_response_message(self, request_id): text = f'response too large (over {self.max_response_size:,d} bytes' error = RPCError.invalid_request(text) return self._protocol.response_message(error, request_id) def _receive_response(self, result, request_id): if request_id not in self._requests: if request_id is None and isinstance(result, RPCError): message = f'diagnostic error received: {result}' else: message = f'response to unsent request (ID: {request_id})' raise ProtocolError.invalid_request(message) from None request, event = self._requests.pop(request_id) event.result = result event.set() return [] def _receive_request_batch(self, payloads): def item_send_result(request_id, result): nonlocal size part = protocol.response_message(result, request_id) size += len(part) + 2 if size > self.max_response_size > 0: part = self._oversized_response_message(request_id) parts.append(part) if len(parts) == count: return protocol.batch_message_from_parts(parts) return None parts = [] items = [] size = 0 count = 0 protocol = self._protocol for payload in payloads: try: item, request_id = protocol._process_request(payload) items.append(item) if isinstance(item, Request): count += 1 item.send_result = partial(item_send_result, request_id) except ProtocolError as error: count += 1 parts.append(error.error_message) if not items and parts: protocol_error = ProtocolError(0, "") protocol_error.error_message = protocol.batch_message_from_parts(parts) raise protocol_error return items def _receive_response_batch(self, payloads): request_ids = [] results = [] for payload in payloads: # Let ProtocolError exceptions through item, request_id = self._protocol._process_response(payload) request_ids.append(request_id) results.append(item.result) ordered = sorted(zip(request_ids, results), key=lambda t: t[0]) ordered_ids, ordered_results = zip(*ordered) if ordered_ids not in self._requests: raise ProtocolError.invalid_request('response to unsent batch') request_batch, event = self._requests.pop(ordered_ids) event.result = ordered_results event.set() return [] def _send_result(self, request_id, result): message = self._protocol.response_message(result, request_id) if len(message) > self.max_response_size > 0: message = self._oversized_response_message(request_id) return message def _event(self, request, request_id): event = Event() self._requests[request_id] = (request, event) return event # # External API # def send_request(self, request: Request) -> typing.Tuple[bytes, Event]: """Send a Request. Return a (message, event) pair. The message is an unframed message to send over the network. Wait on the event for the response; which will be in the "result" attribute. Raises: ProtocolError if the request violates the protocol in some way.. """ request_id = next(self._id_counter) message = self._protocol.request_message(request, request_id) return message, self._event(request, request_id) def send_notification(self, notification): return self._protocol.notification_message(notification) def send_batch(self, batch): ids = tuple(next(self._id_counter) for request in batch if isinstance(request, Request)) message = self._protocol.batch_message(batch, ids) event = self._event(batch, ids) if ids else None return message, event def receive_message(self, message): """Call with an unframed message received from the network. Raises: ProtocolError if the message violates the protocol in some way. However, if it happened in a response that can be paired with a request, the ProtocolError is instead set in the result attribute of the send_request() that caused the error. """ try: item, request_id = self._protocol.message_to_item(message) except ProtocolError as e: if e.response_msg_id is not id: return self._receive_response(e, e.response_msg_id) raise if isinstance(item, Request): item.send_result = partial(self._send_result, request_id) return [item] if isinstance(item, Notification): return [item] if isinstance(item, Response): return self._receive_response(item.result, request_id) if isinstance(item, list): if all(isinstance(payload, dict) and ('result' in payload or 'error' in payload) for payload in item): return self._receive_response_batch(item) else: return self._receive_request_batch(item) else: # Protocol auto-detection hack assert issubclass(item, JSONRPC) self._protocol = item return self.receive_message(message) def raise_pending_requests(self, exception): exception = exception or asyncio.TimeoutError() for request, event in self._requests.values(): event.result = exception event.set() self._requests.clear() def pending_requests(self): """All sent requests that have not received a response.""" return [request for request, event in self._requests.values()] def handler_invocation(handler, request): method, args = request.method, request.args if handler is None: raise RPCError(JSONRPC.METHOD_NOT_FOUND, f'unknown method "{method}"') # We must test for too few and too many arguments. How # depends on whether the arguments were passed as a list or as # a dictionary. info = signature_info(handler) if isinstance(args, (tuple, list)): if len(args) < info.min_args: s = '' if len(args) == 1 else 's' raise RPCError.invalid_args( f'{len(args)} argument{s} passed to method ' f'"{method}" but it requires {info.min_args}') if info.max_args is not None and len(args) > info.max_args: s = '' if len(args) == 1 else 's' raise RPCError.invalid_args( f'{len(args)} argument{s} passed to method ' f'{method} taking at most {info.max_args}') return partial(handler, *args) # Arguments passed by name if info.other_names is None: raise RPCError.invalid_args(f'method "{method}" cannot ' f'be called with named arguments') missing = set(info.required_names).difference(args) if missing: s = '' if len(missing) == 1 else 's' missing = ', '.join(sorted(f'"{name}"' for name in missing)) raise RPCError.invalid_args(f'method "{method}" requires ' f'parameter{s} {missing}') if info.other_names is not any: excess = set(args).difference(info.required_names) excess = excess.difference(info.other_names) if excess: s = '' if len(excess) == 1 else 's' excess = ', '.join(sorted(f'"{name}"' for name in excess)) raise RPCError.invalid_args(f'method "{method}" does not ' f'take parameter{s} {excess}') return partial(handler, **args) ================================================ FILE: lbry/wallet/rpc/session.py ================================================ # Copyright (c) 2018, Neil Booth # # All rights reserved. # # The MIT License (MIT) # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. __all__ = ('Connector', 'RPCSession', 'MessageSession', 'Server', 'BatchError') import asyncio from asyncio import Event, CancelledError import logging import time from contextlib import suppress from prometheus_client import Counter, Histogram from lbry.wallet.tasks import TaskGroup from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification from .jsonrpc import RPCError, ProtocolError from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer HISTOGRAM_BUCKETS = ( .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') ) class Connector: def __init__(self, session_factory, host=None, port=None, proxy=None, **kwargs): self.session_factory = session_factory self.host = host self.port = port self.proxy = proxy self.loop = kwargs.get('loop', asyncio.get_event_loop()) self.kwargs = kwargs async def create_connection(self): """Initiate a connection.""" connector = self.proxy or self.loop return await connector.create_connection( self.session_factory, self.host, self.port, **self.kwargs) async def __aenter__(self): transport, self.protocol = await self.create_connection() return self.protocol async def __aexit__(self, exc_type, exc_value, traceback): await self.protocol.close() class SessionBase(asyncio.Protocol): """Base class of networking sessions. There is no client / server distinction other than who initiated the connection. To initiate a connection to a remote server pass host, port and proxy to the constructor, and then call create_connection(). Each successful call should have a corresponding call to close(). Alternatively if used in a with statement, the connection is made on entry to the block, and closed on exit from the block. """ max_errors = 10 def __init__(self, *, framer=None, loop=None): self.framer = framer or self.default_framer() self.loop = loop or asyncio.get_event_loop() self.logger = logging.getLogger(self.__class__.__name__) self.transport = None # Set when a connection is made self._address = None self._proxy_address = None # For logger.debug messages self.verbosity = 0 # Cleared when the send socket is full self._can_send = Event() self._can_send.set() self._pm_task = None self._task_group = TaskGroup(self.loop) # Force-close a connection if a send doesn't succeed in this time self.max_send_delay = 60 # Statistics. The RPC object also keeps its own statistics. self.start_time = time.perf_counter() self.errors = 0 self.send_count = 0 self.send_size = 0 self.last_send = self.start_time self.recv_count = 0 self.recv_size = 0 self.last_recv = self.start_time self.last_packet_received = self.start_time async def _limited_wait(self, secs): try: await asyncio.wait_for(self._can_send.wait(), secs) except asyncio.TimeoutError: self.abort() raise asyncio.TimeoutError(f'task timed out after {secs}s') async def _send_message(self, message): if not self._can_send.is_set(): await self._limited_wait(self.max_send_delay) if not self.is_closing(): framed_message = self.framer.frame(message) self.send_size += len(framed_message) self.send_count += 1 self.last_send = time.perf_counter() if self.verbosity >= 4: self.logger.debug(f'Sending framed message {framed_message}') self.transport.write(framed_message) def _bump_errors(self): self.errors += 1 if self.errors >= self.max_errors: # Don't await self.close() because that is self-cancelling self._close() def _close(self): if self.transport: self.transport.close() # asyncio framework def data_received(self, framed_message): """Called by asyncio when a message comes in.""" self.last_packet_received = time.perf_counter() if self.verbosity >= 4: self.logger.debug(f'Received framed message {framed_message}') self.recv_size += len(framed_message) self.framer.received_bytes(framed_message) def pause_writing(self): """Transport calls when the send buffer is full.""" if not self.is_closing(): self._can_send.clear() self.transport.pause_reading() def resume_writing(self): """Transport calls when the send buffer has room.""" if not self._can_send.is_set(): self._can_send.set() self.transport.resume_reading() def connection_made(self, transport): """Called by asyncio when a connection is established. Derived classes overriding this method must call this first.""" self.transport = transport # This would throw if called on a closed SSL transport. Fixed # in asyncio in Python 3.6.1 and 3.5.4 peer_address = transport.get_extra_info('peername') # If the Socks proxy was used then _address is already set to # the remote address if self._address: self._proxy_address = peer_address else: self._address = peer_address self._pm_task = self.loop.create_task(self._receive_messages()) def connection_lost(self, exc): """Called by asyncio when the connection closes. Tear down things done in connection_made.""" self._address = None self.transport = None self._task_group.cancel() if self._pm_task: self._pm_task.cancel() # Release waiting tasks self._can_send.set() # External API def default_framer(self): """Return a default framer.""" raise NotImplementedError def peer_address(self): """Returns the peer's address (Python networking address), or None if no connection or an error. This is the result of socket.getpeername() when the connection was made. """ return self._address def peer_address_str(self, for_log=True): """Returns the peer's IP address and port as a human-readable string.""" if not self._address: return 'unknown' ip_addr_str, port = self._address[:2] if ':' in ip_addr_str: return f'[{ip_addr_str}]:{port}' else: return f'{ip_addr_str}:{port}' def is_closing(self): """Return True if the connection is closing.""" return not self.transport or self.transport.is_closing() def abort(self): """Forcefully close the connection.""" if self.transport: self.transport.abort() # TODO: replace with synchronous_close async def close(self, *, force_after=30): """Close the connection and return when closed.""" self._close() if self._pm_task: with suppress(CancelledError): await asyncio.wait([self._pm_task], timeout=force_after) self.abort() await self._pm_task def synchronous_close(self): self._close() if self._pm_task and not self._pm_task.done(): self._pm_task.cancel() class MessageSession(SessionBase): """Session class for protocols where messages are not tied to responses, such as the Bitcoin protocol. To use as a client (connection-opening) session, pass host, port and perhaps a proxy. """ async def _receive_messages(self): while not self.is_closing(): try: message = await self.framer.receive_message() except BadMagicError as e: magic, expected = e.args self.logger.error( f'bad network magic: got {magic} expected {expected}, ' f'disconnecting' ) self._close() except OversizedPayloadError as e: command, payload_len = e.args self.logger.error( f'oversized payload of {payload_len:,d} bytes to command ' f'{command}, disconnecting' ) self._close() except BadChecksumError as e: payload_checksum, claimed_checksum = e.args self.logger.warning( f'checksum mismatch: actual {payload_checksum.hex()} ' f'vs claimed {claimed_checksum.hex()}' ) self._bump_errors() else: self.last_recv = time.perf_counter() self.recv_count += 1 await self._task_group.add(self._handle_message(message)) async def _handle_message(self, message): try: await self.handle_message(message) except ProtocolError as e: self.logger.error(f'{e}') self._bump_errors() except CancelledError: raise except Exception: self.logger.exception(f'exception handling {message}') self._bump_errors() # External API def default_framer(self): """Return a bitcoin framer.""" return BitcoinFramer(bytes.fromhex('e3e1f3e8'), 128_000_000) async def handle_message(self, message): """message is a (command, payload) pair.""" pass async def send_message(self, message): """Send a message (command, payload) over the network.""" await self._send_message(message) class BatchError(Exception): def __init__(self, request): self.request = request # BatchRequest object class BatchRequest: """Used to build a batch request to send to the server. Stores the Attributes batch and results are initially None. Adding an invalid request or notification immediately raises a ProtocolError. On exiting the with clause, it will: 1) create a Batch object for the requests in the order they were added. If the batch is empty this raises a ProtocolError. 2) set the "batch" attribute to be that batch 3) send the batch request and wait for a response 4) raise a ProtocolError if the protocol was violated by the server. Currently this only happens if it gave more than one response to any request 5) otherwise there is precisely one response to each Request. Set the "results" attribute to the tuple of results; the responses are ordered to match the Requests in the batch. Notifications do not get a response. 6) if raise_errors is True and any individual response was a JSON RPC error response, or violated the protocol in some way, a BatchError exception is raised. Otherwise the caller can be certain each request returned a standard result. """ def __init__(self, session, raise_errors): self._session = session self._raise_errors = raise_errors self._requests = [] self.batch = None self.results = None def add_request(self, method, args=()): self._requests.append(Request(method, args)) def add_notification(self, method, args=()): self._requests.append(Notification(method, args)) def __len__(self): return len(self._requests) async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_value, traceback): if exc_type is None: self.batch = Batch(self._requests) message, event = self._session.connection.send_batch(self.batch) await self._session._send_message(message) await event.wait() self.results = event.result if self._raise_errors: if any(isinstance(item, Exception) for item in event.result): raise BatchError(self) NAMESPACE = "wallet_server" class RPCSession(SessionBase): """Base class for protocols where a message can lead to a response, for example JSON RPC.""" RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"), buckets=HISTOGRAM_BUCKETS) NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)", namespace=NAMESPACE, labelnames=("method", "version")) REQUEST_ERRORS_COUNT = Counter( "request_error", "Number of requests that returned errors", namespace=NAMESPACE, labelnames=("method", "version") ) RESET_CONNECTIONS = Counter( "reset_clients", "Number of reset connections by client version", namespace=NAMESPACE, labelnames=("version",) ) def __init__(self, *, framer=None, connection=None): super().__init__(framer=framer) self.connection = connection or self.default_connection() self.client_version = 'unknown' async def _receive_messages(self): while not self.is_closing(): try: message = await self.framer.receive_message() except MemoryError: self.logger.warning('received oversized message from %s:%s, dropping connection', self._address[0], self._address[1]) self.RESET_CONNECTIONS.labels(version=self.client_version).inc() self._close() return self.last_recv = time.perf_counter() self.recv_count += 1 try: requests = self.connection.receive_message(message) except ProtocolError as e: self.logger.debug(f'{e}') if e.error_message: await self._send_message(e.error_message) if e.code == JSONRPC.PARSE_ERROR: self.max_errors = 0 self._bump_errors() else: for request in requests: await self._task_group.add(self._handle_request(request)) async def _handle_request(self, request): start = time.perf_counter() try: result = await self.handle_request(request) except (ProtocolError, RPCError) as e: result = e except CancelledError: raise except Exception: reqstr = str(request) self.logger.exception(f'exception handling {reqstr[:16_000]}') result = RPCError(JSONRPC.INTERNAL_ERROR, 'internal server error') if isinstance(request, Request): message = request.send_result(result) self.RESPONSE_TIMES.labels( method=request.method, version=self.client_version ).observe(time.perf_counter() - start) if message: await self._send_message(message) if isinstance(result, Exception): self._bump_errors() self.REQUEST_ERRORS_COUNT.labels( method=request.method, version=self.client_version ).inc() def connection_lost(self, exc): # Cancel pending requests and message processing self.connection.raise_pending_requests(exc) super().connection_lost(exc) # External API def default_connection(self): """Return a default connection if the user provides none.""" return JSONRPCConnection(JSONRPCv2) def default_framer(self): """Return a default framer.""" return NewlineFramer() async def handle_request(self, request): pass async def send_request(self, method, args=()): """Send an RPC request over the network.""" if self.is_closing(): raise asyncio.TimeoutError("Trying to send request on a recently dropped connection.") message, event = self.connection.send_request(Request(method, args)) await self._send_message(message) await event.wait() result = event.result if isinstance(result, Exception): raise result return result async def send_notification(self, method, args=()) -> bool: """Send an RPC notification over the network.""" message = self.connection.send_notification(Notification(method, args)) self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc() try: await self._send_message(message) return True except asyncio.TimeoutError: self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True)) self.abort() return False async def send_notifications(self, notifications) -> bool: """Send an RPC notification over the network.""" message, _ = self.connection.send_batch(notifications) try: await self._send_message(message) return True except asyncio.TimeoutError: self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True)) self.abort() return False def send_batch(self, raise_errors=False): """Return a BatchRequest. Intended to be used like so: async with session.send_batch() as batch: batch.add_request("method1") batch.add_request("sum", (x, y)) batch.add_notification("updated") for result in batch.results: ... Note that in some circumstances exceptions can be raised; see BatchRequest doc string. """ return BatchRequest(self, raise_errors) class Server: """A simple wrapper around an asyncio.Server object.""" def __init__(self, session_factory, host=None, port=None, *, loop=None, **kwargs): self.host = host self.port = port self.loop = loop or asyncio.get_event_loop() self.server = None self._session_factory = session_factory self._kwargs = kwargs async def listen(self): self.server = await self.loop.create_server( self._session_factory, self.host, self.port, **self._kwargs) async def close(self): """Close the listening socket. This does not close any ServerSession objects created to handle incoming connections. """ if self.server: self.server.close() await self.server.wait_closed() self.server = None ================================================ FILE: lbry/wallet/rpc/socks.py ================================================ # Copyright (c) 2018, Neil Booth # # All rights reserved. # # The MIT License (MIT) # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """SOCKS proxying.""" import sys import asyncio import collections import ipaddress import socket import struct from functools import partial __all__ = ('SOCKSUserAuth', 'SOCKS4', 'SOCKS4a', 'SOCKS5', 'SOCKSProxy', 'SOCKSError', 'SOCKSProtocolError', 'SOCKSFailure') SOCKSUserAuth = collections.namedtuple("SOCKSUserAuth", "username password") class SOCKSError(Exception): """Base class for SOCKS exceptions. Each raised exception will be an instance of a derived class.""" class SOCKSProtocolError(SOCKSError): """Raised when the proxy does not follow the SOCKS protocol""" class SOCKSFailure(SOCKSError): """Raised when the proxy refuses or fails to make a connection""" class NeedData(Exception): pass class SOCKSBase: @classmethod def name(cls): return cls.__name__ def __init__(self): self._buffer = bytes() self._state = self._start def _read(self, size): if len(self._buffer) < size: raise NeedData(size - len(self._buffer)) result = self._buffer[:size] self._buffer = self._buffer[size:] return result def receive_data(self, data): self._buffer += data def next_message(self): return self._state() class SOCKS4(SOCKSBase): """SOCKS4 protocol wrapper.""" # See http://ftp.icm.edu.pl/packages/socks/socks4/SOCKS4.protocol REPLY_CODES = { 90: 'request granted', 91: 'request rejected or failed', 92: ('request rejected because SOCKS server cannot connect ' 'to identd on the client'), 93: ('request rejected because the client program and identd ' 'report different user-ids') } def __init__(self, dst_host, dst_port, auth): super().__init__() self._dst_host = self._check_host(dst_host) self._dst_port = dst_port self._auth = auth @classmethod def _check_host(cls, host): if not isinstance(host, ipaddress.IPv4Address): try: host = ipaddress.IPv4Address(host) except ValueError: raise SOCKSProtocolError( f'SOCKS4 requires an IPv4 address: {host}') from None return host def _start(self): self._state = self._first_response if isinstance(self._dst_host, ipaddress.IPv4Address): # SOCKS4 dst_ip_packed = self._dst_host.packed host_bytes = b'' else: # SOCKS4a dst_ip_packed = b'\0\0\0\1' host_bytes = self._dst_host.encode() + b'\0' if isinstance(self._auth, SOCKSUserAuth): user_id = self._auth.username.encode() else: user_id = b'' # Send TCP/IP stream CONNECT request return b''.join([b'\4\1', struct.pack('>H', self._dst_port), dst_ip_packed, user_id, b'\0', host_bytes]) def _first_response(self): # Wait for 8-byte response data = self._read(8) if data[0] != 0: raise SOCKSProtocolError(f'invalid {self.name()} proxy ' f'response: {data}') reply_code = data[1] if reply_code != 90: msg = self.REPLY_CODES.get( reply_code, f'unknown {self.name()} reply code {reply_code}') raise SOCKSFailure(f'{self.name()} proxy request failed: {msg}') # Other fields ignored return None class SOCKS4a(SOCKS4): @classmethod def _check_host(cls, host): if not isinstance(host, (str, ipaddress.IPv4Address)): raise SOCKSProtocolError( f'SOCKS4a requires an IPv4 address or host name: {host}') return host class SOCKS5(SOCKSBase): """SOCKS protocol wrapper.""" # See https://tools.ietf.org/html/rfc1928 ERROR_CODES = { 1: 'general SOCKS server failure', 2: 'connection not allowed by ruleset', 3: 'network unreachable', 4: 'host unreachable', 5: 'connection refused', 6: 'TTL expired', 7: 'command not supported', 8: 'address type not supported', } def __init__(self, dst_host, dst_port, auth): super().__init__() self._dst_bytes = self._destination_bytes(dst_host, dst_port) self._auth_bytes, self._auth_methods = self._authentication(auth) def _destination_bytes(self, host, port): if isinstance(host, ipaddress.IPv4Address): addr_bytes = b'\1' + host.packed elif isinstance(host, ipaddress.IPv6Address): addr_bytes = b'\4' + host.packed elif isinstance(host, str): host = host.encode() if len(host) > 255: raise SOCKSProtocolError(f'hostname too long: ' f'{len(host)} bytes') addr_bytes = b'\3' + bytes([len(host)]) + host else: raise SOCKSProtocolError(f'SOCKS5 requires an IPv4 address, IPv6 ' f'address, or host name: {host}') return addr_bytes + struct.pack('>H', port) def _authentication(self, auth): if isinstance(auth, SOCKSUserAuth): user_bytes = auth.username.encode() if not 0 < len(user_bytes) < 256: raise SOCKSProtocolError(f'username {auth.username} has ' f'invalid length {len(user_bytes)}') pwd_bytes = auth.password.encode() if not 0 < len(pwd_bytes) < 256: raise SOCKSProtocolError(f'password has invalid length ' f'{len(pwd_bytes)}') return b''.join([bytes([1, len(user_bytes)]), user_bytes, bytes([len(pwd_bytes)]), pwd_bytes]), [0, 2] return b'', [0] def _start(self): self._state = self._first_response return (b'\5' + bytes([len(self._auth_methods)]) + bytes(m for m in self._auth_methods)) def _first_response(self): # Wait for 2-byte response data = self._read(2) if data[0] != 5: raise SOCKSProtocolError(f'invalid SOCKS5 proxy response: {data}') if data[1] not in self._auth_methods: raise SOCKSFailure('SOCKS5 proxy rejected authentication methods') # Authenticate if user-password authentication if data[1] == 2: self._state = self._auth_response return self._auth_bytes return self._request_connection() def _auth_response(self): data = self._read(2) if data[0] != 1: raise SOCKSProtocolError(f'invalid SOCKS5 proxy auth ' f'response: {data}') if data[1] != 0: raise SOCKSFailure(f'SOCKS5 proxy auth failure code: ' f'{data[1]}') return self._request_connection() def _request_connection(self): # Send connection request self._state = self._connect_response return b'\5\1\0' + self._dst_bytes def _connect_response(self): data = self._read(5) if data[0] != 5 or data[2] != 0 or data[3] not in (1, 3, 4): raise SOCKSProtocolError(f'invalid SOCKS5 proxy response: {data}') if data[1] != 0: raise SOCKSFailure(self.ERROR_CODES.get( data[1], f'unknown SOCKS5 error code: {data[1]}')) if data[3] == 1: addr_len = 3 # IPv4 elif data[3] == 3: addr_len = data[4] # Hostname else: addr_len = 15 # IPv6 self._state = partial(self._connect_response_rest, addr_len) return self.next_message() def _connect_response_rest(self, addr_len): self._read(addr_len + 2) return None class SOCKSProxy: def __init__(self, address, protocol, auth): """A SOCKS proxy at an address following a SOCKS protocol. auth is an authentication method to use when connecting, or None. address is a (host, port) pair; for IPv6 it can instead be a (host, port, flowinfo, scopeid) 4-tuple. """ self.address = address self.protocol = protocol self.auth = auth # Set on each successful connection via the proxy to the # result of socket.getpeername() self.peername = None def __str__(self): auth = 'username' if self.auth else 'none' return f'{self.protocol.name()} proxy at {self.address}, auth: {auth}' async def _handshake(self, client, sock, loop): while True: count = 0 try: message = client.next_message() except NeedData as e: count = e.args[0] else: if message is None: return await loop.sock_sendall(sock, message) if count: data = await loop.sock_recv(sock, count) if not data: raise SOCKSProtocolError("EOF received") client.receive_data(data) async def _connect_one(self, host, port): """Connect to the proxy and perform a handshake requesting a connection to (host, port). Return the open socket on success, or the exception on failure. """ client = self.protocol(host, port, self.auth) sock = socket.socket() loop = asyncio.get_event_loop() try: # A non-blocking socket is required by loop socket methods sock.setblocking(False) await loop.sock_connect(sock, self.address) await self._handshake(client, sock, loop) self.peername = sock.getpeername() return sock except Exception as e: # Don't close - see https://github.com/kyuupichan/aiorpcX/issues/8 if sys.platform.startswith('linux') or sys.platform == "darwin": sock.close() return e async def _connect(self, addresses): """Connect to the proxy and perform a handshake requesting a connection to each address in addresses. Return an (open_socket, address) pair on success. """ assert len(addresses) > 0 exceptions = [] for address in addresses: host, port = address[:2] sock = await self._connect_one(host, port) if isinstance(sock, socket.socket): return sock, address exceptions.append(sock) strings = {f'{exc!r}' for exc in exceptions} raise (exceptions[0] if len(strings) == 1 else OSError(f'multiple exceptions: {", ".join(strings)}')) async def _detect_proxy(self): """Return True if it appears we can connect to a SOCKS proxy, otherwise False. """ if self.protocol is SOCKS4a: host, port = 'www.apple.com', 80 else: host, port = ipaddress.IPv4Address('8.8.8.8'), 53 sock = await self._connect_one(host, port) if isinstance(sock, socket.socket): sock.close() return True # SOCKSFailure indicates something failed, but that we are # likely talking to a proxy return isinstance(sock, SOCKSFailure) @classmethod async def auto_detect_address(cls, address, auth): """Try to detect a SOCKS proxy at address using the authentication method (or None). SOCKS5, SOCKS4a and SOCKS are tried in order. If a SOCKS proxy is detected a SOCKSProxy object is returned. Returning a SOCKSProxy does not mean it is functioning - for example, it may have no network connectivity. If no proxy is detected return None. """ for protocol in (SOCKS5, SOCKS4a, SOCKS4): proxy = cls(address, protocol, auth) if await proxy._detect_proxy(): return proxy return None @classmethod async def auto_detect_host(cls, host, ports, auth): """Try to detect a SOCKS proxy on a host on one of the ports. Calls auto_detect for the ports in order. Returns SOCKS are tried in order; a SOCKSProxy object for the first detected proxy is returned. Returning a SOCKSProxy does not mean it is functioning - for example, it may have no network connectivity. If no proxy is detected return None. """ for port in ports: address = (host, port) proxy = await cls.auto_detect_address(address, auth) if proxy: return proxy return None async def create_connection(self, protocol_factory, host, port, *, resolve=False, ssl=None, family=0, proto=0, flags=0): """Set up a connection to (host, port) through the proxy. If resolve is True then host is resolved locally with getaddrinfo using family, proto and flags, otherwise the proxy is asked to resolve host. The function signature is similar to loop.create_connection() with the same result. The attribute _address is set on the protocol to the address of the successful remote connection. Additionally raises SOCKSError if something goes wrong with the proxy handshake. """ loop = asyncio.get_event_loop() if resolve: infos = await loop.getaddrinfo(host, port, family=family, type=socket.SOCK_STREAM, proto=proto, flags=flags) addresses = [info[4] for info in infos] else: addresses = [(host, port)] sock, address = await self._connect(addresses) def set_address(): protocol = protocol_factory() protocol._address = address return protocol return await loop.create_connection( set_address, sock=sock, ssl=ssl, server_hostname=host if ssl else None) ================================================ FILE: lbry/wallet/rpc/util.py ================================================ # Copyright (c) 2018, Neil Booth # # All rights reserved. # # The MIT License (MIT) # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. __all__ = () import asyncio from collections import namedtuple import inspect # other_params: None means cannot be called with keyword arguments only # any means any name is good from functools import lru_cache SignatureInfo = namedtuple('SignatureInfo', 'min_args max_args ' 'required_names other_names') @lru_cache(256) def signature_info(func): params = inspect.signature(func).parameters min_args = max_args = 0 required_names = [] other_names = [] no_names = False for p in params.values(): if p.kind == p.POSITIONAL_OR_KEYWORD: max_args += 1 if p.default is p.empty: min_args += 1 required_names.append(p.name) else: other_names.append(p.name) elif p.kind == p.KEYWORD_ONLY: other_names.append(p.name) elif p.kind == p.VAR_POSITIONAL: max_args = None elif p.kind == p.VAR_KEYWORD: other_names = any elif p.kind == p.POSITIONAL_ONLY: max_args += 1 if p.default is p.empty: min_args += 1 no_names = True if no_names: other_names = None return SignatureInfo(min_args, max_args, required_names, other_names) class Concurrency: def __init__(self, max_concurrent): self._require_non_negative(max_concurrent) self._max_concurrent = max_concurrent self.semaphore = asyncio.Semaphore(max_concurrent) def _require_non_negative(self, value): if not isinstance(value, int) or value < 0: raise RuntimeError('concurrency must be a natural number') @property def max_concurrent(self): return self._max_concurrent async def set_max_concurrent(self, value): self._require_non_negative(value) diff = value - self._max_concurrent self._max_concurrent = value if diff >= 0: for _ in range(diff): self.semaphore.release() else: for _ in range(-diff): await self.semaphore.acquire() ================================================ FILE: lbry/wallet/script.py ================================================ from typing import List from itertools import chain from binascii import hexlify from collections import namedtuple from .bcd_data_stream import BCDataStream from .util import subclass_tuple # bitcoin opcodes OP_0 = 0x00 OP_1 = 0x51 OP_16 = 0x60 OP_VERIFY = 0x69 OP_DUP = 0x76 OP_HASH160 = 0xa9 OP_EQUALVERIFY = 0x88 OP_CHECKSIG = 0xac OP_CHECKMULTISIG = 0xae OP_CHECKLOCKTIMEVERIFY = 0xb1 OP_EQUAL = 0x87 OP_PUSHDATA1 = 0x4c OP_PUSHDATA2 = 0x4d OP_PUSHDATA4 = 0x4e OP_RETURN = 0x6a OP_2DROP = 0x6d OP_DROP = 0x75 # lbry custom opcodes # checks OP_PRICECHECK = 0xb0 # checks that the BUY output is >= SELL price # tx types OP_CLAIM_NAME = 0xb5 OP_SUPPORT_CLAIM = 0xb6 OP_UPDATE_CLAIM = 0xb7 OP_SELL_CLAIM = 0xb8 OP_BUY_CLAIM = 0xb9 # template matching opcodes (not real opcodes) # base class for PUSH_DATA related opcodes # pylint: disable=invalid-name PUSH_DATA_OP = namedtuple('PUSH_DATA_OP', 'name') # opcode for variable length strings # pylint: disable=invalid-name PUSH_SINGLE = subclass_tuple('PUSH_SINGLE', PUSH_DATA_OP) # opcode for variable size integers # pylint: disable=invalid-name PUSH_INTEGER = subclass_tuple('PUSH_INTEGER', PUSH_DATA_OP) # opcode for variable number of variable length strings # pylint: disable=invalid-name PUSH_MANY = subclass_tuple('PUSH_MANY', PUSH_DATA_OP) # opcode with embedded subscript parsing # pylint: disable=invalid-name PUSH_SUBSCRIPT = namedtuple('PUSH_SUBSCRIPT', 'name template') def is_push_data_opcode(opcode): return isinstance(opcode, (PUSH_DATA_OP, PUSH_SUBSCRIPT)) def is_push_data_token(token): return 1 <= token <= OP_PUSHDATA4 def push_data(data): size = len(data) if size < OP_PUSHDATA1: yield BCDataStream.uint8.pack(size) elif size <= 0xFF: yield BCDataStream.uint8.pack(OP_PUSHDATA1) yield BCDataStream.uint8.pack(size) elif size <= 0xFFFF: yield BCDataStream.uint8.pack(OP_PUSHDATA2) yield BCDataStream.uint16.pack(size) else: yield BCDataStream.uint8.pack(OP_PUSHDATA4) yield BCDataStream.uint32.pack(size) yield bytes(data) def read_data(token, stream): if token < OP_PUSHDATA1: return stream.read(token) if token == OP_PUSHDATA1: return stream.read(stream.read_uint8()) if token == OP_PUSHDATA2: return stream.read(stream.read_uint16()) return stream.read(stream.read_uint32()) # opcode for OP_1 - OP_16 # pylint: disable=invalid-name SMALL_INTEGER = namedtuple('SMALL_INTEGER', 'name') def is_small_integer(token): return OP_1 <= token <= OP_16 def push_small_integer(num): assert 1 <= num <= 16 yield BCDataStream.uint8.pack(OP_1 + (num - 1)) def read_small_integer(token): return (token - OP_1) + 1 class Token(namedtuple('Token', 'value')): __slots__ = () def __repr__(self): name = None for var_name, var_value in globals().items(): if var_name.startswith('OP_') and var_value == self.value: name = var_name break return name or self.value class DataToken(Token): __slots__ = () def __repr__(self): return f'"{hexlify(self.value)}"' class SmallIntegerToken(Token): __slots__ = () def __repr__(self): return f'SmallIntegerToken({self.value})' def token_producer(source): token = source.read_uint8() while token is not None: if is_push_data_token(token): yield DataToken(read_data(token, source)) elif is_small_integer(token): yield SmallIntegerToken(read_small_integer(token)) else: yield Token(token) token = source.read_uint8() def tokenize(source): return list(token_producer(source)) class ScriptError(Exception): """ General script handling error. """ class ParseError(ScriptError): """ Script parsing error. """ class Parser: def __init__(self, opcodes, tokens): self.opcodes = opcodes self.tokens = tokens self.values = {} self.token_index = 0 self.opcode_index = 0 def parse(self): while self.token_index < len(self.tokens) and self.opcode_index < len(self.opcodes): token = self.tokens[self.token_index] opcode = self.opcodes[self.opcode_index] if token.value == 0 and isinstance(opcode, PUSH_SINGLE): token = DataToken(b'') if isinstance(token, DataToken): if isinstance(opcode, (PUSH_SINGLE, PUSH_INTEGER, PUSH_SUBSCRIPT)): self.push_single(opcode, token.value) elif isinstance(opcode, PUSH_MANY): self.consume_many_non_greedy() else: raise ParseError(f"DataToken found but opcode was '{opcode}'.") elif isinstance(token, SmallIntegerToken): if isinstance(opcode, SMALL_INTEGER): self.values[opcode.name] = token.value else: raise ParseError(f"SmallIntegerToken found but opcode was '{opcode}'.") elif token.value == opcode: pass else: raise ParseError(f"Token is '{token.value}' and opcode is '{opcode}'.") self.token_index += 1 self.opcode_index += 1 if self.token_index < len(self.tokens): raise ParseError("Parse completed without all tokens being consumed.") if self.opcode_index < len(self.opcodes): raise ParseError("Parse completed without all opcodes being consumed.") return self def consume_many_non_greedy(self): """ Allows PUSH_MANY to consume data without being greedy in cases when one or more PUSH_SINGLEs follow a PUSH_MANY. This will prioritize giving all PUSH_SINGLEs some data and only after that subsume the rest into PUSH_MANY. """ token_values = [] while self.token_index < len(self.tokens): token = self.tokens[self.token_index] if not isinstance(token, DataToken): self.token_index -= 1 break token_values.append(token.value) self.token_index += 1 push_opcodes = [] push_many_count = 0 while self.opcode_index < len(self.opcodes): opcode = self.opcodes[self.opcode_index] if not is_push_data_opcode(opcode): self.opcode_index -= 1 break if isinstance(opcode, PUSH_MANY): push_many_count += 1 push_opcodes.append(opcode) self.opcode_index += 1 if push_many_count > 1: raise ParseError( "Cannot have more than one consecutive PUSH_MANY, as there is no way to tell which" " token value should go into which PUSH_MANY." ) if len(push_opcodes) > len(token_values): raise ParseError( "Not enough token values to match all of the PUSH_MANY and PUSH_SINGLE opcodes." ) many_opcode = push_opcodes.pop(0) # consume data into PUSH_SINGLE opcodes, working backwards for opcode in reversed(push_opcodes): self.push_single(opcode, token_values.pop()) # finally PUSH_MANY gets everything that's left self.values[many_opcode.name] = token_values def push_single(self, opcode, value): if isinstance(opcode, PUSH_SINGLE): self.values[opcode.name] = value elif isinstance(opcode, PUSH_INTEGER): self.values[opcode.name] = int.from_bytes(value, 'little') elif isinstance(opcode, PUSH_SUBSCRIPT): self.values[opcode.name] = Script.from_source_with_template(value, opcode.template) else: raise ParseError(f"Not a push single or subscript: {opcode}") class Template: __slots__ = 'name', 'opcodes' def __init__(self, name, opcodes): self.name = name self.opcodes = opcodes def parse(self, tokens): return Parser(self.opcodes, tokens).parse().values if self.opcodes else {} def generate(self, values): source = BCDataStream() for opcode in self.opcodes: if isinstance(opcode, PUSH_SINGLE): data = values[opcode.name] source.write_many(push_data(data)) elif isinstance(opcode, PUSH_INTEGER): data = values[opcode.name] source.write_many(push_data( data.to_bytes((data.bit_length() + 8) // 8, byteorder='little', signed=True) )) elif isinstance(opcode, PUSH_SUBSCRIPT): data = values[opcode.name] source.write_many(push_data(data.source)) elif isinstance(opcode, PUSH_MANY): for data in values[opcode.name]: source.write_many(push_data(data)) elif isinstance(opcode, SMALL_INTEGER): data = values[opcode.name] source.write_many(push_small_integer(data)) else: source.write_uint8(opcode) return source.get_bytes() class Script: __slots__ = 'source', '_template', '_values', '_template_hint' templates: List[Template] = [] NO_SCRIPT = Template('no_script', None) # special case def __init__(self, source=None, template=None, values=None, template_hint=None): self.source = source self._template = template self._values = values self._template_hint = template_hint if source is None and template and values: self.generate() @property def template(self): if self._template is None: self.parse(self._template_hint) return self._template @property def values(self): if self._values is None: self.parse(self._template_hint) return self._values @property def tokens(self): return tokenize(BCDataStream(self.source)) @classmethod def from_source_with_template(cls, source, template): return cls(source, template_hint=template) def parse(self, template_hint=None): tokens = self.tokens if not tokens and not template_hint: template_hint = self.NO_SCRIPT for template in chain((template_hint,), self.templates): if not template: continue try: self._values = template.parse(tokens) self._template = template return except ParseError: continue raise ValueError(f'No matching templates for source: {hexlify(self.source)}') def generate(self): self.source = self.template.generate(self._values) class InputScript(Script): __slots__ = () REDEEM_PUBKEY = Template('pubkey', ( PUSH_SINGLE('signature'), )) REDEEM_PUBKEY_HASH = Template('pubkey_hash', ( PUSH_SINGLE('signature'), PUSH_SINGLE('pubkey') )) MULTI_SIG_SCRIPT = Template('multi_sig', ( SMALL_INTEGER('signatures_count'), PUSH_MANY('pubkeys'), SMALL_INTEGER('pubkeys_count'), OP_CHECKMULTISIG )) REDEEM_SCRIPT_HASH_MULTI_SIG = Template('script_hash+multi_sig', ( OP_0, PUSH_MANY('signatures'), PUSH_SUBSCRIPT('script', MULTI_SIG_SCRIPT) )) TIME_LOCK_SCRIPT = Template('timelock', ( PUSH_INTEGER('height'), OP_CHECKLOCKTIMEVERIFY, OP_DROP, # rest is identical to OutputScript.PAY_PUBKEY_HASH: OP_DUP, OP_HASH160, PUSH_SINGLE('pubkey_hash'), OP_EQUALVERIFY, OP_CHECKSIG )) REDEEM_SCRIPT_HASH_TIME_LOCK = Template('script_hash+timelock', ( PUSH_SINGLE('signature'), PUSH_SINGLE('pubkey'), PUSH_SUBSCRIPT('script', TIME_LOCK_SCRIPT) )) templates = [ REDEEM_PUBKEY, REDEEM_PUBKEY_HASH, REDEEM_SCRIPT_HASH_TIME_LOCK, REDEEM_SCRIPT_HASH_MULTI_SIG, ] @classmethod def redeem_pubkey_hash(cls, signature, pubkey): return cls(template=cls.REDEEM_PUBKEY_HASH, values={ 'signature': signature, 'pubkey': pubkey }) @classmethod def redeem_multi_sig_script_hash(cls, signatures, pubkeys): return cls(template=cls.REDEEM_SCRIPT_HASH_MULTI_SIG, values={ 'signatures': signatures, 'script': cls(template=cls.MULTI_SIG_SCRIPT, values={ 'signatures_count': len(signatures), 'pubkeys': pubkeys, 'pubkeys_count': len(pubkeys) }) }) @classmethod def redeem_time_lock_script_hash(cls, signature, pubkey, height=None, pubkey_hash=None, script_source=None): if height and pubkey_hash: script = cls(template=cls.TIME_LOCK_SCRIPT, values={ 'height': height, 'pubkey_hash': pubkey_hash }) elif script_source: script = cls(source=script_source, template=cls.TIME_LOCK_SCRIPT) script.parse(script.template) else: raise ValueError("script_source or both height and pubkey_hash are required.") return cls(template=cls.REDEEM_SCRIPT_HASH_TIME_LOCK, values={ 'signature': signature, 'pubkey': pubkey, 'script': script }) @property def is_script_hash(self): return self.template.name.startswith('script_hash+') class OutputScript(Script): __slots__ = () # output / payment script templates (aka scriptPubKey) PAY_PUBKEY_FULL = Template('pay_pubkey_full', ( PUSH_SINGLE('pubkey'), OP_CHECKSIG )) PAY_PUBKEY_HASH = Template('pay_pubkey_hash', ( OP_DUP, OP_HASH160, PUSH_SINGLE('pubkey_hash'), OP_EQUALVERIFY, OP_CHECKSIG )) PAY_SCRIPT_HASH = Template('pay_script_hash', ( OP_HASH160, PUSH_SINGLE('script_hash'), OP_EQUAL )) PAY_SEGWIT = Template('pay_script_hash+segwit', ( OP_0, PUSH_SINGLE('script_hash') )) RETURN_DATA = Template('return_data', ( OP_RETURN, PUSH_SINGLE('data') )) CLAIM_NAME_OPCODES = ( OP_CLAIM_NAME, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim'), OP_2DROP, OP_DROP ) CLAIM_NAME_PUBKEY = Template('claim_name+pay_pubkey_hash', ( CLAIM_NAME_OPCODES + PAY_PUBKEY_HASH.opcodes )) CLAIM_NAME_SCRIPT = Template('claim_name+pay_script_hash', ( CLAIM_NAME_OPCODES + PAY_SCRIPT_HASH.opcodes )) SUPPORT_CLAIM_OPCODES = ( OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), OP_2DROP, OP_DROP ) SUPPORT_CLAIM_PUBKEY = Template('support_claim+pay_pubkey_hash', ( SUPPORT_CLAIM_OPCODES + PAY_PUBKEY_HASH.opcodes )) SUPPORT_CLAIM_SCRIPT = Template('support_claim+pay_script_hash', ( SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes )) SUPPORT_CLAIM_DATA_OPCODES = ( OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'), OP_2DROP, OP_2DROP ) SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', ( SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes )) SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', ( SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes )) UPDATE_CLAIM_OPCODES = ( OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'), OP_2DROP, OP_2DROP ) UPDATE_CLAIM_PUBKEY = Template('update_claim+pay_pubkey_hash', ( UPDATE_CLAIM_OPCODES + PAY_PUBKEY_HASH.opcodes )) UPDATE_CLAIM_SCRIPT = Template('update_claim+pay_script_hash', ( UPDATE_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes )) templates = [ PAY_PUBKEY_FULL, PAY_PUBKEY_HASH, PAY_SCRIPT_HASH, PAY_SEGWIT, RETURN_DATA, CLAIM_NAME_PUBKEY, CLAIM_NAME_SCRIPT, SUPPORT_CLAIM_PUBKEY, SUPPORT_CLAIM_SCRIPT, SUPPORT_CLAIM_DATA_PUBKEY, SUPPORT_CLAIM_DATA_SCRIPT, UPDATE_CLAIM_PUBKEY, UPDATE_CLAIM_SCRIPT, ] @classmethod def pay_pubkey_hash(cls, pubkey_hash): return cls(template=cls.PAY_PUBKEY_HASH, values={ 'pubkey_hash': pubkey_hash }) @classmethod def pay_script_hash(cls, script_hash): return cls(template=cls.PAY_SCRIPT_HASH, values={ 'script_hash': script_hash }) @classmethod def return_data(cls, data): return cls(template=cls.RETURN_DATA, values={ 'data': data }) @property def is_pay_pubkey(self): return self.template.name.endswith('pay_pubkey_full') @classmethod def pay_claim_name_pubkey_hash(cls, claim_name, claim, pubkey_hash): return cls(template=cls.CLAIM_NAME_PUBKEY, values={ 'claim_name': claim_name, 'claim': claim, 'pubkey_hash': pubkey_hash }) @classmethod def pay_update_claim_pubkey_hash(cls, claim_name, claim_id, claim, pubkey_hash): return cls(template=cls.UPDATE_CLAIM_PUBKEY, values={ 'claim_name': claim_name, 'claim_id': claim_id, 'claim': claim, 'pubkey_hash': pubkey_hash }) @classmethod def pay_support_pubkey_hash(cls, claim_name: bytes, claim_id: bytes, pubkey_hash: bytes): return cls(template=cls.SUPPORT_CLAIM_PUBKEY, values={ 'claim_name': claim_name, 'claim_id': claim_id, 'pubkey_hash': pubkey_hash }) @classmethod def pay_support_data_pubkey_hash( cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes): return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={ 'claim_name': claim_name, 'claim_id': claim_id, 'support': support, 'pubkey_hash': pubkey_hash }) @property def is_pay_pubkey_hash(self): return self.template.name.endswith('pay_pubkey_hash') @property def is_pay_script_hash(self): return self.template.name.endswith('pay_script_hash') @property def is_return_data(self): return self.template.name.endswith('return_data') @property def is_claim_name(self): return self.template.name.startswith('claim_name+') @property def is_update_claim(self): return self.template.name.startswith('update_claim+') @property def is_support_claim(self): return self.template.name.startswith('support_claim+') @property def is_support_claim_data(self): return self.template.name.startswith('support_claim+data+') @property def is_claim_involved(self): return any((self.is_claim_name, self.is_support_claim, self.is_update_claim)) ================================================ FILE: lbry/wallet/stream.py ================================================ import asyncio class BroadcastSubscription: def __init__(self, controller, on_data, on_error, on_done): self._controller = controller self._previous = self._next = None self._on_data = on_data self._on_error = on_error self._on_done = on_done self.is_paused = False self.is_canceled = False self.is_closed = False def pause(self): self.is_paused = True def resume(self): self.is_paused = False def cancel(self): self._controller._cancel(self) self.is_canceled = True @property def can_fire(self): return not any((self.is_paused, self.is_canceled, self.is_closed)) def _add(self, data): if self.can_fire and self._on_data is not None: return self._on_data(data) def _add_error(self, exception): if self.can_fire and self._on_error is not None: return self._on_error(exception) def _close(self): try: if self.can_fire and self._on_done is not None: return self._on_done() finally: self.is_closed = True class StreamController: def __init__(self, merge_repeated_events=False): self.stream = Stream(self) self._first_subscription = None self._last_subscription = None self._last_event = None self._merge_repeated = merge_repeated_events @property def has_listener(self): return self._first_subscription is not None @property def _iterate_subscriptions(self): next_sub = self._first_subscription while next_sub is not None: subscription = next_sub next_sub = next_sub._next yield subscription def _notify_and_ensure_future(self, notify): tasks = [] for subscription in self._iterate_subscriptions: maybe_coroutine = notify(subscription) if asyncio.iscoroutine(maybe_coroutine): tasks.append(maybe_coroutine) if tasks: return asyncio.ensure_future(asyncio.wait(tasks)) else: f = asyncio.get_event_loop().create_future() f.set_result(None) return f def add(self, event): skip = self._merge_repeated and event == self._last_event self._last_event = event return self._notify_and_ensure_future( lambda subscription: None if skip else subscription._add(event) ) def add_error(self, exception): return self._notify_and_ensure_future( lambda subscription: subscription._add_error(exception) ) def close(self): for subscription in self._iterate_subscriptions: subscription._close() def _cancel(self, subscription): previous = subscription._previous next_sub = subscription._next if previous is None: self._first_subscription = next_sub else: previous._next = next_sub if next_sub is None: self._last_subscription = previous else: next_sub._previous = previous subscription._next = subscription._previous = subscription def _listen(self, on_data, on_error, on_done): subscription = BroadcastSubscription(self, on_data, on_error, on_done) old_last = self._last_subscription self._last_subscription = subscription subscription._previous = old_last subscription._next = None if old_last is None: self._first_subscription = subscription else: old_last._next = subscription return subscription class Stream: def __init__(self, controller): self._controller = controller def listen(self, on_data, on_error=None, on_done=None): return self._controller._listen(on_data, on_error, on_done) def where(self, condition) -> asyncio.Future: future = asyncio.get_event_loop().create_future() def where_test(value): if condition(value): self._cancel_and_callback(subscription, future, value) subscription = self.listen( where_test, lambda exception: self._cancel_and_error(subscription, future, exception) ) return future @property def first(self): future = asyncio.get_event_loop().create_future() subscription = self.listen( lambda value: not future.done() and self._cancel_and_callback(subscription, future, value), lambda exception: not future.done() and self._cancel_and_error(subscription, future, exception) ) return future @staticmethod def _cancel_and_callback(subscription: BroadcastSubscription, future: asyncio.Future, value): subscription.cancel() future.set_result(value) @staticmethod def _cancel_and_error(subscription: BroadcastSubscription, future: asyncio.Future, exception): subscription.cancel() future.set_exception(exception) ================================================ FILE: lbry/wallet/tasks.py ================================================ from asyncio import Event, get_event_loop class TaskGroup: def __init__(self, loop=None): self._loop = loop or get_event_loop() self._tasks = set() self.done = Event() self.started = Event() def __len__(self): return len(self._tasks) def add(self, coro): task = self._loop.create_task(coro) self._tasks.add(task) self.started.set() self.done.clear() task.add_done_callback(self._remove) return task def _remove(self, task): self._tasks.remove(task) if len(self._tasks) < 1: self.done.set() self.started.clear() def cancel(self): for task in self._tasks: task.cancel() self.done.set() self.started.clear() ================================================ FILE: lbry/wallet/transaction.py ================================================ import struct import logging import typing from binascii import hexlify, unhexlify from typing import List, Iterable, Optional, Tuple from lbry.error import InsufficientFundsError from lbry.crypto.hash import hash160, sha256 from lbry.crypto.base58 import Base58 from lbry.schema.url import normalize_name from lbry.schema.claim import Claim from lbry.schema.base import Signable from lbry.schema.purchase import Purchase from lbry.schema.support import Support from .script import InputScript, OutputScript from .constants import COIN, DUST, NULL_HASH32 from .bcd_data_stream import BCDataStream from .hash import TXRef, TXRefImmutable from .util import ReadOnlyList from .bip32 import PrivateKey, PublicKey if typing.TYPE_CHECKING: from lbry.wallet.account import Account from lbry.wallet.ledger import Ledger from lbry.wallet.wallet import Wallet log = logging.getLogger() class TXRefMutable(TXRef): __slots__ = ('tx',) def __init__(self, tx: 'Transaction') -> None: super().__init__() self.tx = tx @property def id(self): if self._id is None: self._id = hexlify(self.hash[::-1]).decode() return self._id @property def hash(self): if self._hash is None: self._hash = sha256(sha256(self.tx.raw_sans_segwit)) return self._hash @property def height(self): return self.tx.height def reset(self): self._id = None self._hash = None class TXORef: __slots__ = 'tx_ref', 'position' def __init__(self, tx_ref: TXRef, position: int) -> None: self.tx_ref = tx_ref self.position = position @property def id(self): return f'{self.tx_ref.id}:{self.position}' @property def hash(self): return self.tx_ref.hash + BCDataStream.uint32.pack(self.position) @property def is_null(self): return self.tx_ref.is_null @property def txo(self) -> Optional['Output']: return None class TXORefResolvable(TXORef): __slots__ = ('_txo',) def __init__(self, txo: 'Output') -> None: assert txo.tx_ref is not None assert txo.position is not None super().__init__(txo.tx_ref, txo.position) self._txo = txo @property def txo(self): return self._txo class InputOutput: __slots__ = 'tx_ref', 'position' def __init__(self, tx_ref: TXRef = None, position: int = None) -> None: self.tx_ref = tx_ref self.position = position @property def size(self) -> int: """ Size of this input / output in bytes. """ stream = BCDataStream() self.serialize_to(stream) return len(stream.get_bytes()) def get_fee(self, ledger): return self.size * ledger.fee_per_byte def serialize_to(self, stream, alternate_script=None): raise NotImplementedError class Input(InputOutput): NULL_SIGNATURE = b'\x00'*72 NULL_PUBLIC_KEY = b'\x00'*33 __slots__ = 'txo_ref', 'sequence', 'coinbase', 'script' def __init__(self, txo_ref: TXORef, script: InputScript, sequence: int = 0xFFFFFFFF, tx_ref: TXRef = None, position: int = None) -> None: super().__init__(tx_ref, position) self.txo_ref = txo_ref self.sequence = sequence self.coinbase = script if txo_ref.is_null else None self.script = script if not txo_ref.is_null else None @property def is_coinbase(self): return self.coinbase is not None @classmethod def spend(cls, txo: 'Output') -> 'Input': """ Create an input to spend the output.""" assert txo.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.' script = InputScript.redeem_pubkey_hash(cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY) return cls(txo.ref, script) @classmethod def spend_time_lock(cls, txo: 'Output', script_source: bytes) -> 'Input': """ Create an input to spend time lock script.""" script = InputScript.redeem_time_lock_script_hash( cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY, script_source=script_source ) return cls(txo.ref, script) @property def amount(self) -> int: """ Amount this input adds to the transaction. """ if self.txo_ref.txo is None: raise ValueError('Cannot resolve output to get amount.') return self.txo_ref.txo.amount @property def is_my_input(self) -> Optional[bool]: """ True if the output this input spends is yours. """ if self.txo_ref.txo is None: return False return self.txo_ref.txo.is_my_output @classmethod def deserialize_from(cls, stream): tx_ref = TXRefImmutable.from_hash(stream.read(32), -1) position = stream.read_uint32() script = stream.read_string() sequence = stream.read_uint32() return cls( TXORef(tx_ref, position), InputScript(script) if not tx_ref.is_null else script, sequence ) def serialize_to(self, stream, alternate_script=None): stream.write(self.txo_ref.tx_ref.hash) stream.write_uint32(self.txo_ref.position) if alternate_script is not None: stream.write_string(alternate_script) else: if self.is_coinbase: stream.write_string(self.coinbase) else: stream.write_string(self.script.source) stream.write_uint32(self.sequence) class OutputEffectiveAmountEstimator: __slots__ = 'txo', 'txi', 'fee', 'effective_amount' def __init__(self, ledger: 'Ledger', txo: 'Output') -> None: self.txo = txo self.txi = Input.spend(txo) self.fee: int = self.txi.get_fee(ledger) self.effective_amount: int = txo.amount - self.fee def __lt__(self, other): return self.effective_amount < other.effective_amount class Output(InputOutput): __slots__ = ( 'amount', 'script', 'is_internal_transfer', 'is_spent', 'is_my_output', 'is_my_input', 'channel', 'private_key', 'meta', 'sent_supports', 'sent_tips', 'received_tips', 'purchase', 'purchased_claim', 'purchase_receipt', 'reposted_claim', 'claims', '_signable' ) def __init__(self, amount: int, script: OutputScript, tx_ref: TXRef = None, position: int = None, is_internal_transfer: Optional[bool] = None, is_spent: Optional[bool] = None, is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None, sent_supports: Optional[int] = None, sent_tips: Optional[int] = None, received_tips: Optional[int] = None, channel: Optional['Output'] = None, private_key: Optional[PrivateKey] = None ) -> None: super().__init__(tx_ref, position) self.amount = amount self.script = script self.is_internal_transfer = is_internal_transfer self.is_spent = is_spent self.is_my_output = is_my_output self.is_my_input = is_my_input self.sent_supports = sent_supports self.sent_tips = sent_tips self.received_tips = received_tips self.channel = channel self.private_key: PrivateKey = private_key self.purchase: 'Output' = None # txo containing purchase metadata self.purchased_claim: 'Output' = None # resolved claim pointed to by purchase self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim self.reposted_claim: 'Output' = None # txo representing claim being reposted self.claims: List['Output'] = None # resolved claims for collection self._signable: Optional[Signable] = None self.meta = {} def update_annotations(self, annotated: 'Output'): if annotated is None: self.is_internal_transfer = None self.is_spent = None self.is_my_output = None self.is_my_input = None self.sent_supports = None self.sent_tips = None self.received_tips = None else: self.is_internal_transfer = annotated.is_internal_transfer self.is_spent = annotated.is_spent self.is_my_output = annotated.is_my_output self.is_my_input = annotated.is_my_input self.sent_supports = annotated.sent_supports self.sent_tips = annotated.sent_tips self.received_tips = annotated.received_tips self.channel = annotated.channel if annotated else None self.private_key = annotated.private_key if annotated else None @property def ref(self): return TXORefResolvable(self) @property def id(self): return self.ref.id @property def is_pubkey_hash(self): return 'pubkey_hash' in self.script.values @property def pubkey_hash(self): return self.script.values['pubkey_hash'] @property def is_script_hash(self): return 'script_hash' in self.script.values @property def script_hash(self): return self.script.values['script_hash'] @property def has_address(self): return self.is_pubkey_hash or self.is_script_hash def get_address(self, ledger): if self.is_pubkey_hash: return ledger.hash160_to_address(self.pubkey_hash) elif self.is_script_hash: return ledger.hash160_to_script_address(self.script_hash) def get_estimator(self, ledger): return OutputEffectiveAmountEstimator(ledger, self) @classmethod def pay_pubkey_hash(cls, amount, pubkey_hash): return cls(amount, OutputScript.pay_pubkey_hash(pubkey_hash)) @classmethod def pay_script_hash(cls, amount, pubkey_hash): return cls(amount, OutputScript.pay_script_hash(pubkey_hash)) @classmethod def deserialize_from(cls, stream): return cls( amount=stream.read_uint64(), script=OutputScript(stream.read_string()) ) def serialize_to(self, stream, alternate_script=None): stream.write_uint64(self.amount) stream.write_string(self.script.source) def get_fee(self, ledger): name_fee = 0 if self.script.is_claim_name: name_fee = len(self.script.values['claim_name']) * ledger.fee_per_name_char return max(name_fee, super().get_fee(ledger)) @property def is_claim(self) -> bool: return self.script.is_claim_name or self.script.is_update_claim @property def is_support(self) -> bool: return self.script.is_support_claim @property def is_support_data(self) -> bool: return self.script.is_support_claim_data @property def claim_hash(self) -> bytes: if self.script.is_claim_name: return hash160(self.tx_ref.hash + struct.pack('>I', self.position)) elif self.script.is_update_claim or self.script.is_support_claim: return self.script.values['claim_id'] else: raise ValueError('No claim_id associated.') @property def claim_id(self) -> str: return hexlify(self.claim_hash[::-1]).decode() @property def claim_name(self) -> str: if self.script.is_claim_involved: return self.script.values['claim_name'].decode() raise ValueError('No claim_name associated.') @property def normalized_name(self) -> str: return normalize_name(self.claim_name) @property def claim(self) -> Claim: if self.is_claim: if not isinstance(self.script.values['claim'], Claim): self.script.values['claim'] = Claim.from_bytes(self.script.values['claim']) return self.script.values['claim'] raise ValueError('Only claim name and claim update have the claim payload.') @property def can_decode_claim(self): try: return self.claim except Exception: return False @property def support(self) -> Support: if self.is_support_data: if not isinstance(self.script.values['support'], Support): self.script.values['support'] = Support.from_bytes(self.script.values['support']) return self.script.values['support'] raise ValueError('Only supports with data can be represented as Supports.') @property def can_decode_support(self): try: return self.support except Exception: return False @property def signable(self) -> Signable: if self._signable is None: if self.is_claim: self._signable = self.claim elif self.is_support_data: self._signable = self.support return self._signable @property def permanent_url(self) -> str: if self.script.is_claim_involved: return f"lbry://{self.claim_name}#{self.claim_id}" raise ValueError('No claim associated.') @property def has_private_key(self): return self.private_key is not None def get_signature_digest(self, ledger): if self.signable.unsigned_payload: pieces = [ Base58.decode(self.get_address(ledger)), self.signable.unsigned_payload, self.signable.signing_channel_hash[::-1] ] else: pieces = [ self.tx_ref.tx.inputs[0].txo_ref.hash, self.signable.signing_channel_hash, self.signable.to_message_bytes() ] return sha256(b''.join(pieces)) @staticmethod def is_signature_valid(signature, digest, public_key_bytes): return PublicKey\ .from_compressed(public_key_bytes)\ .verify(signature, digest) def is_signed_by(self, channel: 'Output', ledger=None): return self.is_signature_valid( self.signable.signature, self.get_signature_digest(ledger), channel.claim.channel.public_key_bytes ) def sign(self, channel: 'Output', first_input_id=None): self.channel = channel self.signable.signing_channel_hash = channel.claim_hash digest = sha256(b''.join([ first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash, self.signable.signing_channel_hash, self.signable.to_message_bytes() ])) self.signable.signature = channel.private_key.sign_compact(digest) self.script.generate() def sign_data(self, data: bytes, timestamp: str) -> str: pieces = [timestamp.encode(), self.claim_hash, data] digest = sha256(b''.join(pieces)) signature = self.private_key.sign_compact(digest) return hexlify(signature).decode() def clear_signature(self): self.channel = None self.signable.clear_signature() def set_channel_private_key(self, private_key: PrivateKey): self.private_key = private_key self.claim.channel.public_key_bytes = private_key.public_key.pubkey_bytes self.script.generate() return self.private_key def is_channel_private_key(self, private_key: PrivateKey): return self.claim.channel.public_key_bytes == private_key.public_key.pubkey_bytes @classmethod def pay_claim_name_pubkey_hash( cls, amount: int, claim_name: str, claim: Claim, pubkey_hash: bytes) -> 'Output': script = OutputScript.pay_claim_name_pubkey_hash( claim_name.encode(), claim, pubkey_hash) return cls(amount, script) @classmethod def pay_update_claim_pubkey_hash( cls, amount: int, claim_name: str, claim_id: str, claim: Claim, pubkey_hash: bytes) -> 'Output': script = OutputScript.pay_update_claim_pubkey_hash( claim_name.encode(), unhexlify(claim_id)[::-1], claim, pubkey_hash ) return cls(amount, script) @classmethod def pay_support_pubkey_hash(cls, amount: int, claim_name: str, claim_id: str, pubkey_hash: bytes) -> 'Output': script = OutputScript.pay_support_pubkey_hash( claim_name.encode(), unhexlify(claim_id)[::-1], pubkey_hash ) return cls(amount, script) @classmethod def pay_support_data_pubkey_hash( cls, amount: int, claim_name: str, claim_id: str, support: Support, pubkey_hash: bytes) -> 'Output': script = OutputScript.pay_support_data_pubkey_hash( claim_name.encode(), unhexlify(claim_id)[::-1], support, pubkey_hash ) return cls(amount, script) @classmethod def add_purchase_data(cls, purchase: Purchase) -> 'Output': script = OutputScript.return_data(purchase) return cls(0, script) @property def is_purchase_data(self) -> bool: return self.script.is_return_data and ( isinstance(self.script.values['data'], Purchase) or Purchase.has_start_byte(self.script.values['data']) ) @property def purchase_data(self) -> Purchase: if self.is_purchase_data: if not isinstance(self.script.values['data'], Purchase): self.script.values['data'] = Purchase.from_bytes(self.script.values['data']) return self.script.values['data'] raise ValueError('Output does not have purchase data.') @property def can_decode_purchase_data(self): try: return self.purchase_data except: # pylint: disable=bare-except return False @property def purchased_claim_id(self): if self.purchase is not None: return self.purchase.purchase_data.claim_id if self.purchased_claim is not None: return self.purchased_claim.claim_id @property def has_price(self): if self.can_decode_claim: claim = self.claim if claim.is_stream: stream = claim.stream return stream.has_fee and stream.fee.amount and stream.fee.amount > 0 return False @property def price(self): return self.claim.stream.fee class Transaction: def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False, height: int = -2, position: int = -1, julian_day: int = None) -> None: self._raw = raw self._raw_sans_segwit = None self._raw_outputs = None self.is_segwit_flag = 0 self.witnesses: List[bytes] = [] self.ref = TXRefMutable(self) self.version = version self.locktime = locktime self._inputs: List[Input] = [] self._outputs: List[Output] = [] self.is_verified = is_verified # Height Progression # -2: not broadcast # -1: in mempool but has unconfirmed inputs # 0: in mempool and all inputs confirmed # +num: confirmed in a specific block (height) self.height = height self.position = position self._day = julian_day if raw is not None: self._deserialize() @property def is_broadcast(self): return self.height > -2 @property def is_mempool(self): return self.height in (-1, 0) @property def is_confirmed(self): return self.height > 0 @property def id(self): return self.ref.id @property def hash(self): return self.ref.hash def get_julian_day(self, ledger): if self._day is None and self.height > 0: self._day = ledger.headers.estimated_julian_day(self.height) return self._day @property def raw(self): if self._raw is None: self._raw = self._serialize() return self._raw @property def raw_sans_segwit(self): if self.is_segwit_flag: if self._raw_sans_segwit is None: self._raw_sans_segwit = self._serialize(sans_segwit=True) return self._raw_sans_segwit return self.raw def _reset(self): self._raw = None self._raw_sans_segwit = None self._raw_outputs = None self.ref.reset() @property def inputs(self) -> ReadOnlyList[Input]: return ReadOnlyList(self._inputs) @property def outputs(self) -> ReadOnlyList[Output]: return ReadOnlyList(self._outputs) def _add(self, existing_ios: List, new_ios: Iterable[InputOutput], reset=False) -> 'Transaction': for txio in new_ios: txio.tx_ref = self.ref txio.position = len(existing_ios) existing_ios.append(txio) if reset: self._reset() return self def add_inputs(self, inputs: Iterable[Input]) -> 'Transaction': return self._add(self._inputs, inputs, True) def add_outputs(self, outputs: Iterable[Output]) -> 'Transaction': return self._add(self._outputs, outputs, True) @property def size(self) -> int: """ Size in bytes of the entire transaction. """ return len(self.raw) @property def base_size(self) -> int: """ Size of transaction without inputs or outputs in bytes. """ return ( self.size - sum(txi.size for txi in self._inputs) - sum(txo.size for txo in self._outputs) ) @property def input_sum(self): return sum(i.amount for i in self.inputs if i.txo_ref.txo is not None) @property def output_sum(self): return sum(o.amount for o in self.outputs) @property def net_account_balance(self) -> int: balance = 0 for txi in self.inputs: if txi.txo_ref.txo is None: continue if txi.is_my_input is True: balance -= txi.amount elif txi.is_my_input is None: raise ValueError( "Cannot access net_account_balance if inputs do not " "have is_my_input set properly." ) for txo in self.outputs: if txo.is_my_output is True: balance += txo.amount elif txo.is_my_output is None: raise ValueError( "Cannot access net_account_balance if outputs do not " "have is_my_output set properly." ) return balance @property def fee(self) -> int: return self.input_sum - self.output_sum def get_base_fee(self, ledger) -> int: """ Fee for base tx excluding inputs and outputs. """ return self.base_size * ledger.fee_per_byte def get_effective_input_sum(self, ledger) -> int: """ Sum of input values *minus* the cost involved to spend them. """ return sum(txi.amount - txi.get_fee(ledger) for txi in self._inputs) def get_total_output_sum(self, ledger) -> int: """ Sum of output values *plus* the cost involved to spend them. """ return sum(txo.amount + txo.get_fee(ledger) for txo in self._outputs) def _serialize(self, with_inputs: bool = True, sans_segwit: bool = False) -> bytes: stream = BCDataStream() stream.write_uint32(self.version) if with_inputs: stream.write_compact_size(len(self._inputs)) for txin in self._inputs: txin.serialize_to(stream) self._serialize_outputs(stream) stream.write_uint32(self.locktime) return stream.get_bytes() def _serialize_for_signature(self, signing_input: int) -> bytes: stream = BCDataStream() stream.write_uint32(self.version) stream.write_compact_size(len(self._inputs)) for i, txin in enumerate(self._inputs): if signing_input == i: if txin.script.is_script_hash: txin.serialize_to(stream, txin.script.values['script'].source) else: assert txin.txo_ref.txo is not None txin.serialize_to(stream, txin.txo_ref.txo.script.source) else: txin.serialize_to(stream, b'') self._serialize_outputs(stream) stream.write_uint32(self.locktime) stream.write_uint32(self.signature_hash_type(1)) # signature hash type: SIGHASH_ALL return stream.get_bytes() def _serialize_outputs(self, stream): if self._raw_outputs is None: self._raw_outputs = BCDataStream() self._raw_outputs.write_compact_size(len(self._outputs)) for txout in self._outputs: txout.serialize_to(self._raw_outputs) stream.write(self._raw_outputs.get_bytes()) def _deserialize(self): if self._raw is not None: stream = BCDataStream(self._raw) self.version = stream.read_uint32() input_count = stream.read_compact_size() if input_count == 0: self.is_segwit_flag = stream.read_uint8() input_count = stream.read_compact_size() self._add(self._inputs, [ Input.deserialize_from(stream) for _ in range(input_count) ]) output_count = stream.read_compact_size() self._add(self._outputs, [ Output.deserialize_from(stream) for _ in range(output_count) ]) if self.is_segwit_flag: # drain witness portion of transaction # too many witnesses for no crime self.witnesses = [] for _ in range(input_count): for _ in range(stream.read_compact_size()): self.witnesses.append(stream.read(stream.read_compact_size())) self.locktime = stream.read_uint32() @classmethod def ensure_all_have_same_ledger_and_wallet( cls, funding_accounts: Iterable['Account'], change_account: 'Account' = None) -> Tuple['Ledger', 'Wallet']: ledger = wallet = None for account in funding_accounts: if ledger is None: ledger = account.ledger wallet = account.wallet if ledger != account.ledger: raise ValueError( 'All funding accounts used to create a transaction must be on the same ledger.' ) if wallet != account.wallet: raise ValueError( 'All funding accounts used to create a transaction must be from the same wallet.' ) if change_account is not None: if change_account.ledger != ledger: raise ValueError('Change account must use same ledger as funding accounts.') if change_account.wallet != wallet: raise ValueError('Change account must use same wallet as funding accounts.') if ledger is None: raise ValueError('No ledger found.') if wallet is None: raise ValueError('No wallet found.') return ledger, wallet @classmethod async def create(cls, inputs: Iterable[Input], outputs: Iterable[Output], funding_accounts: Iterable['Account'], change_account: 'Account', sign: bool = True): """ Find optimal set of inputs when only outputs are provided; add change outputs if only inputs are provided or if inputs are greater than outputs. """ tx = cls() \ .add_inputs(inputs) \ .add_outputs(outputs) ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account) # value of the outputs plus associated fees cost = ( tx.get_base_fee(ledger) + tx.get_total_output_sum(ledger) ) # value of the inputs less the cost to spend those inputs payment = tx.get_effective_input_sum(ledger) try: for _ in range(5): if payment < cost: deficit = cost - payment spendables = await ledger.get_spendable_utxos(deficit, funding_accounts) if not spendables: raise InsufficientFundsError() payment += sum(s.effective_amount for s in spendables) tx.add_inputs(s.txi for s in spendables) cost_of_change = ( tx.get_base_fee(ledger) + Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(ledger) ) if payment > cost: change = payment - cost change_amount = change - cost_of_change if change_amount > DUST: change_address = await change_account.change.get_or_create_usable_address() change_hash160 = change_account.ledger.address_to_hash160(change_address) change_output = Output.pay_pubkey_hash(change_amount, change_hash160) change_output.is_internal_transfer = True tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)]) if tx._outputs: break # this condition and the outer range(5) loop cover an edge case # whereby a single input is just enough to cover the fee and # has some change left over, but the change left over is less # than the cost_of_change: thus the input is completely # consumed and no output is added, which is an invalid tx. # to be able to spend this input we must increase the cost # of the TX and run through the balance algorithm a second time # adding an extra input and change output, making tx valid. # we do this 5 times in case the other UTXOs added are also # less than the fee, after 5 attempts we give up and go home cost += cost_of_change + 1 if sign: await tx.sign(funding_accounts) except Exception as e: log.exception('Failed to create transaction:') await ledger.release_tx(tx) raise e return tx @staticmethod def signature_hash_type(hash_type): return hash_type async def sign(self, funding_accounts: Iterable['Account'], extra_keys: dict = None): self._reset() ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts) for i, txi in enumerate(self._inputs): assert txi.script is not None assert txi.txo_ref.txo is not None txo_script = txi.txo_ref.txo.script if txo_script.is_pay_pubkey_hash or txo_script.is_pay_script_hash: if 'pubkey_hash' in txo_script.values: address = ledger.hash160_to_address(txo_script.values.get('pubkey_hash', '')) private_key = await ledger.get_private_key_for_address(wallet, address) else: private_key = next(iter(extra_keys.values())) assert private_key is not None, 'Cannot find private key for signing output.' tx = self._serialize_for_signature(i) txi.script.values['signature'] = \ private_key.sign(tx) + bytes((self.signature_hash_type(1),)) txi.script.values['pubkey'] = private_key.public_key.pubkey_bytes txi.script.generate() else: raise NotImplementedError("Don't know how to spend this output.") self._reset() @classmethod def pay(cls, amount: int, address: bytes, funding_accounts: List['Account'], change_account: 'Account'): ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account) output = Output.pay_pubkey_hash(amount, ledger.address_to_hash160(address)) return cls.create([], [output], funding_accounts, change_account) @classmethod def claim_create( cls, name: str, claim: Claim, amount: int, holding_address: str, funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None): ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account) claim_output = Output.pay_claim_name_pubkey_hash( amount, name, claim, ledger.address_to_hash160(holding_address) ) if signing_channel is not None: claim_output.sign(signing_channel, b'placeholder txid:nout') return cls.create([], [claim_output], funding_accounts, change_account, sign=False) @classmethod def claim_update( cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str, funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None): ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account) updated_claim = Output.pay_update_claim_pubkey_hash( amount, previous_claim.claim_name, previous_claim.claim_id, claim, ledger.address_to_hash160(holding_address) ) if signing_channel is not None: updated_claim.sign(signing_channel, b'placeholder txid:nout') else: updated_claim.clear_signature() return cls.create( [Input.spend(previous_claim)], [updated_claim], funding_accounts, change_account, sign=False ) @classmethod def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str, funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None, comment: str = None): ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account) if signing_channel is not None or comment is not None: support = Support() if comment is not None: support.comment = comment support_output = Output.pay_support_data_pubkey_hash( amount, claim_name, claim_id, support, ledger.address_to_hash160(holding_address) ) if signing_channel is not None: support_output.sign(signing_channel, b'placeholder txid:nout') else: support_output = Output.pay_support_pubkey_hash( amount, claim_name, claim_id, ledger.address_to_hash160(holding_address) ) return cls.create([], [support_output], funding_accounts, change_account, sign=False) @classmethod def purchase(cls, claim_id: str, amount: int, merchant_address: bytes, funding_accounts: List['Account'], change_account: 'Account'): ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account) payment = Output.pay_pubkey_hash(amount, ledger.address_to_hash160(merchant_address)) data = Output.add_purchase_data(Purchase(claim_id)) return cls.create([], [payment, data], funding_accounts, change_account) @classmethod async def spend_time_lock(cls, time_locked_txo: Output, script: bytes, account: 'Account'): txi = Input.spend_time_lock(time_locked_txo, script) txi.sequence = 0xFFFFFFFE tx = await cls.create([txi], [], [account], account, sign=False) tx.locktime = txi.script.values['script'].values['height'] tx._reset() return tx @property def my_inputs(self): for txi in self.inputs: if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_output: yield txi def _filter_my_outputs(self, f): for txo in self.outputs: if txo.is_my_output and f(txo.script): yield txo def _filter_other_outputs(self, f): for txo in self.outputs: if not txo.is_my_output and f(txo.script): yield txo def _filter_any_outputs(self, f): for txo in self.outputs: if f(txo): yield txo @property def my_claim_outputs(self): return self._filter_my_outputs(lambda s: s.is_claim_name) @property def my_update_outputs(self): return self._filter_my_outputs(lambda s: s.is_update_claim) @property def my_support_outputs(self): return self._filter_my_outputs(lambda s: s.is_support_claim) @property def any_purchase_outputs(self): return self._filter_any_outputs(lambda o: o.purchase is not None) @property def other_support_outputs(self): return self._filter_other_outputs(lambda s: s.is_support_claim) @property def my_abandon_outputs(self): for txi in self.inputs: abandon = txi.txo_ref.txo if abandon is not None and abandon.is_my_output and abandon.script.is_claim_involved: is_update = False if abandon.script.is_claim_name or abandon.script.is_update_claim: for update in self.my_update_outputs: if abandon.claim_id == update.claim_id: is_update = True break if not is_update: yield abandon ================================================ FILE: lbry/wallet/udp.py ================================================ import asyncio import struct from time import perf_counter import logging from typing import Optional, Tuple, NamedTuple from lbry.utils import LRUCache, is_valid_public_ipv4 from lbry.schema.attrs import country_str_to_int, country_int_to_str # from prometheus_client import Counter log = logging.getLogger(__name__) _MAGIC = 1446058291 # genesis blocktime (which is actually wrong) # ping_count_metric = Counter("ping_count", "Number of pings received", namespace='wallet_server_status') _PAD_BYTES = b'\x00' * 64 PROTOCOL_VERSION = 1 class SPVPing(NamedTuple): magic: int protocol_version: int pad_bytes: bytes def encode(self): return struct.pack(b'!lB64s', *self) # pylint: disable=not-an-iterable @staticmethod def make() -> bytes: return SPVPing(_MAGIC, PROTOCOL_VERSION, _PAD_BYTES).encode() @classmethod def decode(cls, packet: bytes): decoded = cls(*struct.unpack(b'!lB64s', packet[:69])) if decoded.magic != _MAGIC: raise ValueError("invalid magic bytes") return decoded PONG_ENCODING = b'!BBL32s4sH' class SPVPong(NamedTuple): protocol_version: int flags: int height: int tip: bytes source_address_raw: bytes country: int def encode(self): return struct.pack(PONG_ENCODING, *self) # pylint: disable=not-an-iterable @staticmethod def encode_address(address: str): return bytes(int(b) for b in address.split(".")) @classmethod def make(cls, flags: int, height: int, tip: bytes, source_address: str, country: str) -> bytes: return SPVPong( PROTOCOL_VERSION, flags, height, tip, cls.encode_address(source_address), country_str_to_int(country) ).encode() @classmethod def make_sans_source_address(cls, flags: int, height: int, tip: bytes, country: str) -> Tuple[bytes, bytes]: pong = cls.make(flags, height, tip, '0.0.0.0', country) return pong[:38], pong[42:] @classmethod def decode(cls, packet: bytes): return cls(*struct.unpack(PONG_ENCODING, packet[:44])) @property def available(self) -> bool: return (self.flags & 0b00000001) > 0 @property def ip_address(self) -> str: return ".".join(map(str, self.source_address_raw)) @property def country_name(self): return country_int_to_str(self.country) def __repr__(self) -> str: return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \ f"available={'True' if self.flags & 1 > 0 else 'False'}," \ f" height={self.height}, tip={self.tip[::-1].hex()}, country={self.country_name})" class SPVServerStatusProtocol(asyncio.DatagramProtocol): def __init__( self, height: int, tip: bytes, country: str, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10, allow_localhost: bool = False, allow_lan: bool = False ): super().__init__() self.transport: Optional[asyncio.transports.DatagramTransport] = None self._height = height self._tip = tip self._flags = 0 self._country = country self._left_cache = self._right_cache = None self.update_cached_response() self._throttle = LRUCache(throttle_cache_size) self._should_log = LRUCache(throttle_cache_size) self._min_delay = 1 / throttle_reqs_per_sec self._allow_localhost = allow_localhost self._allow_lan = allow_lan self.closed = asyncio.Event() def update_cached_response(self): self._left_cache, self._right_cache = SPVPong.make_sans_source_address( self._flags, max(0, self._height), self._tip, self._country ) def set_unavailable(self): self._flags &= 0b11111110 self.update_cached_response() def set_available(self): self._flags |= 0b00000001 self.update_cached_response() def set_height(self, height: int, tip: bytes): self._height, self._tip = height, tip self.update_cached_response() def should_throttle(self, host: str): now = perf_counter() last_requested = self._throttle.get(host, default=0) self._throttle[host] = now if now - last_requested < self._min_delay: log_cnt = self._should_log.get(host, default=0) + 1 if log_cnt % 100 == 0: log.warning("throttle spv status to %s", host) self._should_log[host] = log_cnt return True return False def make_pong(self, host): return self._left_cache + SPVPong.encode_address(host) + self._right_cache def datagram_received(self, data: bytes, addr: Tuple[str, int]): if self.should_throttle(addr[0]): return try: SPVPing.decode(data) except (ValueError, struct.error, AttributeError, TypeError): # log.exception("derp") return if addr[1] >= 1024 and is_valid_public_ipv4( addr[0], allow_localhost=self._allow_localhost, allow_lan=self._allow_lan): self.transport.sendto(self.make_pong(addr[0]), addr) else: log.warning("odd packet from %s:%i", addr[0], addr[1]) # ping_count_metric.inc() def connection_made(self, transport) -> None: self.transport = transport self.closed.clear() def connection_lost(self, exc: Optional[Exception]) -> None: self.transport = None self.closed.set() async def close(self): if self.transport: self.transport.close() await self.closed.wait() class StatusServer: def __init__(self): self._protocol: Optional[SPVServerStatusProtocol] = None async def start(self, height: int, tip: bytes, country: str, interface: str, port: int, allow_lan: bool = False): if self.is_running: return loop = asyncio.get_event_loop() interface = interface if interface.lower() != 'localhost' else '127.0.0.1' self._protocol = SPVServerStatusProtocol( height, tip, country, allow_localhost=interface == '127.0.0.1', allow_lan=allow_lan ) await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port)) log.info("started udp status server on %s:%i", interface, port) async def stop(self): if self.is_running: await self._protocol.close() self._protocol = None @property def is_running(self): return self._protocol is not None def set_unavailable(self): if self.is_running: self._protocol.set_unavailable() def set_available(self): if self.is_running: self._protocol.set_available() def set_height(self, height: int, tip: bytes): if self.is_running: self._protocol.set_height(height, tip) class SPVStatusClientProtocol(asyncio.DatagramProtocol): def __init__(self, responses: asyncio.Queue): super().__init__() self.transport: Optional[asyncio.transports.DatagramTransport] = None self.responses = responses self._ping_packet = SPVPing.make() def datagram_received(self, data: bytes, addr: Tuple[str, int]): try: self.responses.put_nowait(((addr, perf_counter()), SPVPong.decode(data))) except (ValueError, struct.error, AttributeError, TypeError, RuntimeError): return def connection_made(self, transport) -> None: self.transport = transport def connection_lost(self, exc: Optional[Exception]) -> None: self.transport = None log.info("closed udp spv server selection client") def ping(self, server: Tuple[str, int]): self.transport.sendto(self._ping_packet, server) def close(self): # log.info("close udp client") if self.transport: self.transport.close() ================================================ FILE: lbry/wallet/usage_payment.py ================================================ import asyncio import logging from lbry.error import ( InsufficientFundsError, ServerPaymentFeeAboveMaxAllowedError, ServerPaymentInvalidAddressError, ServerPaymentWalletLockedError ) from lbry.wallet.dewies import lbc_to_dewies from lbry.wallet.stream import StreamController from lbry.wallet.transaction import Output, Transaction log = logging.getLogger(__name__) class WalletServerPayer: def __init__(self, payment_period=24 * 60 * 60, max_fee='1.0', analytics_manager=None): self.ledger = None self.wallet = None self.running = False self.task = None self.payment_period = payment_period self.analytics_manager = analytics_manager self.max_fee = max_fee self._on_payment_controller = StreamController() self.on_payment = self._on_payment_controller.stream self.on_payment.listen(None, on_error=lambda e: log.warning(e.args[0])) async def pay(self): while self.running: try: await self._pay() except (asyncio.TimeoutError, ConnectionError): if not self.running: break delay = max(self.payment_period / 24, 10) log.warning("Payement failed. Will retry after %g seconds.", delay) asyncio.sleep(delay) except BaseException as e: if not isinstance(e, asyncio.CancelledError): log.exception("Unexpected exception. Payment task exiting early.") self.running = False raise async def _pay(self): while self.running: await asyncio.sleep(self.payment_period) features = await self.ledger.network.get_server_features() log.debug("pay loop: received server features: %s", str(features)) address = features['payment_address'] amount = str(features['daily_fee']) if not address or not amount: log.debug("pay loop: no address or no amount") continue if not self.ledger.is_pubkey_address(address): log.info("pay loop: address not pubkey") self._on_payment_controller.add_error(ServerPaymentInvalidAddressError(address)) continue if self.wallet.is_locked: log.info("pay loop: wallet is locked") self._on_payment_controller.add_error(ServerPaymentWalletLockedError()) continue amount = lbc_to_dewies(features['daily_fee']) # check that this is in lbc and not dewies limit = lbc_to_dewies(self.max_fee) if amount > limit: log.info("pay loop: amount (%d) > limit (%d)", amount, limit) self._on_payment_controller.add_error( ServerPaymentFeeAboveMaxAllowedError(features['daily_fee'], self.max_fee) ) continue try: tx = await Transaction.create( [], [Output.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))], self.wallet.get_accounts_or_all(None), self.wallet.get_account_or_default(None) ) except InsufficientFundsError: self._on_payment_controller.add_error(InsufficientFundsError()) continue await self.ledger.broadcast_or_release(tx, blocking=True) if self.analytics_manager: await self.analytics_manager.send_credits_sent() self._on_payment_controller.add(tx) async def start(self, ledger=None, wallet=None): if lbc_to_dewies(self.max_fee) < 1: return self.ledger = ledger self.wallet = wallet self.running = True self.task = asyncio.ensure_future(self.pay()) self.task.add_done_callback(self._done_callback) def _done_callback(self, f): if f.cancelled(): reason = "Cancelled" elif f.exception(): reason = f'Exception: {f.exception()}' elif not self.running: reason = "Stopped" else: reason = "" log.info("Stopping wallet server payments. %s", reason) async def stop(self): if self.running: self.running = False self.task.cancel() ================================================ FILE: lbry/wallet/util.py ================================================ import re from typing import TypeVar, Sequence, Optional from .constants import COIN def date_to_julian_day(d): return d.toordinal() + 1721424.5 def coins_to_satoshis(coins): if not isinstance(coins, str): raise ValueError("{coins} must be a string") result = re.search(r'^(\d{1,10})\.(\d{1,8})$', coins) if result is not None: whole, fractional = result.groups() return int(whole+fractional.ljust(8, "0")) raise ValueError("'{lbc}' is not a valid coin decimal") def satoshis_to_coins(satoshis): coins = '{:.8f}'.format(satoshis / COIN).rstrip('0') if coins.endswith('.'): return coins+'0' else: return coins T = TypeVar('T') class ReadOnlyList(Sequence[T]): def __init__(self, lst): self.lst = lst def __getitem__(self, key): return self.lst[key] def __len__(self) -> int: return len(self.lst) def subclass_tuple(name, base): return type(name, (base,), {'__slots__': ()}) class cachedproperty: def __init__(self, f): self.f = f def __get__(self, obj, objtype): obj = obj or objtype value = self.f(obj) setattr(obj, self.f.__name__, value) return value class ArithUint256: # https://github.com/bitcoin/bitcoin/blob/master/src/arith_uint256.cpp __slots__ = '_value', '_compact' def __init__(self, value: int) -> None: self._value = value self._compact: Optional[int] = None @classmethod def from_compact(cls, compact) -> 'ArithUint256': size = compact >> 24 word = compact & 0x007fffff if size <= 3: return cls(word >> 8 * (3 - size)) else: return cls(word << 8 * (size - 3)) @property def value(self) -> int: return self._value @property def compact(self) -> int: if self._compact is None: self._compact = self._calculate_compact() return self._compact @property def negative(self) -> int: return self._calculate_compact(negative=True) @property def bits(self) -> int: """ Returns the position of the highest bit set plus one. """ bits = bin(self._value)[2:] for i, d in enumerate(bits): if d: return (len(bits) - i) + 1 return 0 @property def low64(self) -> int: return self._value & 0xffffffffffffffff def _calculate_compact(self, negative=False) -> int: size = (self.bits + 7) // 8 if size <= 3: compact = self.low64 << 8 * (3 - size) else: compact = ArithUint256(self._value >> 8 * (size - 3)).low64 # The 0x00800000 bit denotes the sign. # Thus, if it is already set, divide the mantissa by 256 and increase the exponent. if compact & 0x00800000: compact >>= 8 size += 1 assert (compact & ~0x007fffff) == 0 assert size < 256 compact |= size << 24 if negative and compact & 0x007fffff: compact |= 0x00800000 return compact def __mul__(self, x): # Take the mod because we are limited to an unsigned 256 bit number return ArithUint256((self._value * x) % 2 ** 256) def __truediv__(self, x): return ArithUint256(int(self._value / x)) def __gt__(self, other): return self._value > other def __lt__(self, other): return self._value < other ================================================ FILE: lbry/wallet/wallet.py ================================================ import os import time import stat import json import zlib import typing import logging from typing import List, Sequence, MutableSequence, Optional from collections import UserDict from hashlib import sha256 from operator import attrgetter from lbry.crypto.crypt import better_aes_encrypt, better_aes_decrypt from lbry.error import InvalidPasswordError from .account import Account if typing.TYPE_CHECKING: from lbry.wallet.manager import WalletManager from lbry.wallet.ledger import Ledger log = logging.getLogger(__name__) ENCRYPT_ON_DISK = 'encrypt-on-disk' class TimestampedPreferences(UserDict): def __init__(self, d: dict = None): super().__init__() if d is not None: self.data = d.copy() def __getitem__(self, key): return self.data[key]['value'] def __setitem__(self, key, value): self.data[key] = { 'value': value, 'ts': int(time.time()) } def __repr__(self): return repr(self.to_dict_without_ts()) def to_dict_without_ts(self): return { key: value['value'] for key, value in self.data.items() } @property def hash(self): return sha256(json.dumps(self.data).encode()).digest() def merge(self, other: dict): for key, value in other.items(): if key in self.data and value['ts'] < self.data[key]['ts']: continue self.data[key] = value class Wallet: """ The primary role of Wallet is to encapsulate a collection of accounts (seed/private keys) and the spending rules / settings for the coins attached to those accounts. Wallets are represented by physical files on the filesystem. """ preferences: TimestampedPreferences encryption_password: Optional[str] def __init__(self, name: str = 'Wallet', accounts: MutableSequence['Account'] = None, storage: 'WalletStorage' = None, preferences: dict = None) -> None: self.name = name self.accounts = accounts or [] self.storage = storage or WalletStorage() self.preferences = TimestampedPreferences(preferences or {}) self.encryption_password = None self.id = self.get_id() def get_id(self): return os.path.basename(self.storage.path) if self.storage.path else self.name def add_account(self, account: 'Account'): self.accounts.append(account) def generate_account(self, ledger: 'Ledger') -> 'Account': return Account.generate(ledger, self) @property def default_account(self) -> Optional['Account']: for account in self.accounts: return account return None def get_account_or_default(self, account_id: str) -> Optional['Account']: if account_id is None: return self.default_account return self.get_account_or_error(account_id) def get_account_or_error(self, account_id: str) -> 'Account': for account in self.accounts: if account.id == account_id: return account raise ValueError(f"Couldn't find account: {account_id}.") def get_accounts_or_all(self, account_ids: List[str]) -> Sequence['Account']: return [ self.get_account_or_error(account_id) for account_id in account_ids ] if account_ids else self.accounts async def get_detailed_accounts(self, **kwargs): accounts = [] for i, account in enumerate(self.accounts): details = await account.get_details(**kwargs) details['is_default'] = i == 0 accounts.append(details) return accounts @classmethod def from_storage(cls, storage: 'WalletStorage', manager: 'WalletManager') -> 'Wallet': json_dict = storage.read() wallet = cls( name=json_dict.get('name', 'Wallet'), preferences=json_dict.get('preferences', {}), storage=storage ) account_dicts: Sequence[dict] = json_dict.get('accounts', []) for account_dict in account_dicts: ledger = manager.get_or_create_ledger(account_dict['ledger']) Account.from_dict(ledger, wallet, account_dict) return wallet def to_dict(self, encrypt_password: str = None): return { 'version': WalletStorage.LATEST_VERSION, 'name': self.name, 'preferences': self.preferences.data, 'accounts': [a.to_dict(encrypt_password) for a in self.accounts] } def to_json(self): assert not self.is_locked, "Cannot serialize a wallet with locked/encrypted accounts." return json.dumps(self.to_dict()) def save(self): if self.preferences.get(ENCRYPT_ON_DISK, False): if self.encryption_password is not None: return self.storage.write(self.to_dict(encrypt_password=self.encryption_password)) elif not self.is_locked: log.warning( "Disk encryption requested but no password available for encryption. " "Resetting encryption preferences and saving wallet in an unencrypted state." ) self.preferences[ENCRYPT_ON_DISK] = False return self.storage.write(self.to_dict()) @property def hash(self) -> bytes: h = sha256() if self.is_encrypted: assert self.encryption_password is not None, \ "Encryption is enabled but no password is available, cannot generate hash." h.update(self.encryption_password.encode()) h.update(self.preferences.hash) for account in sorted(self.accounts, key=attrgetter('id')): h.update(account.hash) return h.digest() def pack(self, password): assert not self.is_locked, "Cannot pack a wallet with locked/encrypted accounts." new_data_compressed = zlib.compress(self.to_json().encode()) return better_aes_encrypt(password, new_data_compressed) @classmethod def unpack(cls, password, encrypted): decrypted = better_aes_decrypt(password, encrypted) try: decompressed = zlib.decompress(decrypted) except zlib.error as e: if "incorrect header check" in e.args[0].lower(): raise InvalidPasswordError() if "unknown compression method" in e.args[0].lower(): raise InvalidPasswordError() if "invalid window size" in e.args[0].lower(): raise InvalidPasswordError() raise return json.loads(decompressed) def merge(self, manager: 'WalletManager', password: str, data: str) -> (List['Account'], List['Account']): assert not self.is_locked, "Cannot sync apply on a locked wallet." added_accounts, merged_accounts = [], [] if password is None: decrypted_data = json.loads(data) else: decrypted_data = self.unpack(password, data) self.preferences.merge(decrypted_data.get('preferences', {})) for account_dict in decrypted_data['accounts']: ledger = manager.get_or_create_ledger(account_dict['ledger']) _, _, pubkey = Account.keys_from_dict(ledger, account_dict) account_id = pubkey.address local_match = None for local_account in self.accounts: if account_id == local_account.id: local_match = local_account break if local_match is not None: local_match.merge(account_dict) merged_accounts.append(local_match) else: new_account = Account.from_dict(ledger, self, account_dict) added_accounts.append(new_account) return added_accounts, merged_accounts @property def is_locked(self) -> bool: for account in self.accounts: if account.encrypted: return True return False async def unlock(self, password): for account in self.accounts: if account.encrypted: if not account.decrypt(password): return False await account.deterministic_channel_keys.ensure_cache_primed() self.encryption_password = password return True def lock(self): assert self.encryption_password is not None, "Cannot lock an unencrypted wallet, encrypt first." for account in self.accounts: if not account.encrypted: account.encrypt(self.encryption_password) return True @property def is_encrypted(self) -> bool: # either its locked or it was unlocked using a password. # if its set to encrypt on preferences but isnt encrypted and no password was given so far, # then its not encrypted return self.is_locked or ( self.preferences.get(ENCRYPT_ON_DISK, False) and self.encryption_password is not None) def decrypt(self): assert not self.is_locked, "Cannot decrypt a locked wallet, unlock first." self.preferences[ENCRYPT_ON_DISK] = False self.save() return True def encrypt(self, password): assert not self.is_locked, "Cannot re-encrypt a locked wallet, unlock first." assert password, "Cannot encrypt with blank password." self.encryption_password = password self.preferences[ENCRYPT_ON_DISK] = True self.save() return True class WalletStorage: LATEST_VERSION = 1 def __init__(self, path=None, default=None): self.path = path self._default = default or { 'version': self.LATEST_VERSION, 'name': 'My Wallet', 'preferences': {}, 'accounts': [] } def read(self): if self.path and os.path.exists(self.path): with open(self.path, 'r') as f: json_data = f.read() json_dict = json.loads(json_data) if json_dict.get('version') == self.LATEST_VERSION and \ set(json_dict) == set(self._default): return json_dict else: return self.upgrade(json_dict) else: return self._default.copy() def upgrade(self, json_dict): json_dict = json_dict.copy() version = json_dict.pop('version', -1) if version == -1: pass upgraded = self._default.copy() upgraded.update(json_dict) return json_dict def write(self, json_dict): json_data = json.dumps(json_dict, indent=4, sort_keys=True) if self.path is None: return json_data temp_path = "{}.tmp.{}".format(self.path, os.getpid()) with open(temp_path, "w") as f: f.write(json_data) f.flush() os.fsync(f.fileno()) if os.path.exists(self.path): mode = os.stat(self.path).st_mode else: mode = stat.S_IREAD | stat.S_IWRITE try: os.rename(temp_path, self.path) except Exception: # pylint: disable=broad-except os.remove(self.path) os.rename(temp_path, self.path) os.chmod(self.path, mode) ================================================ FILE: lbry/wallet/words/__init__.py ================================================ ================================================ FILE: lbry/wallet/words/chinese_simplified.py ================================================ words = [ '的', '一', '是', '在', '不', '了', '有', '和', '人', '这', '中', '大', '为', '上', '个', '国', '我', '以', '要', '他', '时', '来', '用', '们', '生', '到', '作', '地', '于', '出', '就', '分', '对', '成', '会', '可', '主', '发', '年', '动', '同', '工', '也', '能', '下', '过', '子', '说', '产', '种', '面', '而', '方', '后', '多', '定', '行', '学', '法', '所', '民', '得', '经', '十', '三', '之', '进', '着', '等', '部', '度', '家', '电', '力', '里', '如', '水', '化', '高', '自', '二', '理', '起', '小', '物', '现', '实', '加', '量', '都', '两', '体', '制', '机', '当', '使', '点', '从', '业', '本', '去', '把', '性', '好', '应', '开', '它', '合', '还', '因', '由', '其', '些', '然', '前', '外', '天', '政', '四', '日', '那', '社', '义', '事', '平', '形', '相', '全', '表', '间', '样', '与', '关', '各', '重', '新', '线', '内', '数', '正', '心', '反', '你', '明', '看', '原', '又', '么', '利', '比', '或', '但', '质', '气', '第', '向', '道', '命', '此', '变', '条', '只', '没', '结', '解', '问', '意', '建', '月', '公', '无', '系', '军', '很', '情', '者', '最', '立', '代', '想', '已', '通', '并', '提', '直', '题', '党', '程', '展', '五', '果', '料', '象', '员', '革', '位', '入', '常', '文', '总', '次', '品', '式', '活', '设', '及', '管', '特', '件', '长', '求', '老', '头', '基', '资', '边', '流', '路', '级', '少', '图', '山', '统', '接', '知', '较', '将', '组', '见', '计', '别', '她', '手', '角', '期', '根', '论', '运', '农', '指', '几', '九', '区', '强', '放', '决', '西', '被', '干', '做', '必', '战', '先', '回', '则', '任', '取', '据', '处', '队', '南', '给', '色', '光', '门', '即', '保', '治', '北', '造', '百', '规', '热', '领', '七', '海', '口', '东', '导', '器', '压', '志', '世', '金', '增', '争', '济', '阶', '油', '思', '术', '极', '交', '受', '联', '什', '认', '六', '共', '权', '收', '证', '改', '清', '美', '再', '采', '转', '更', '单', '风', '切', '打', '白', '教', '速', '花', '带', '安', '场', '身', '车', '例', '真', '务', '具', '万', '每', '目', '至', '达', '走', '积', '示', '议', '声', '报', '斗', '完', '类', '八', '离', '华', '名', '确', '才', '科', '张', '信', '马', '节', '话', '米', '整', '空', '元', '况', '今', '集', '温', '传', '土', '许', '步', '群', '广', '石', '记', '需', '段', '研', '界', '拉', '林', '律', '叫', '且', '究', '观', '越', '织', '装', '影', '算', '低', '持', '音', '众', '书', '布', '复', '容', '儿', '须', '际', '商', '非', '验', '连', '断', '深', '难', '近', '矿', '千', '周', '委', '素', '技', '备', '半', '办', '青', '省', '列', '习', '响', '约', '支', '般', '史', '感', '劳', '便', '团', '往', '酸', '历', '市', '克', '何', '除', '消', '构', '府', '称', '太', '准', '精', '值', '号', '率', '族', '维', '划', '选', '标', '写', '存', '候', '毛', '亲', '快', '效', '斯', '院', '查', '江', '型', '眼', '王', '按', '格', '养', '易', '置', '派', '层', '片', '始', '却', '专', '状', '育', '厂', '京', '识', '适', '属', '圆', '包', '火', '住', '调', '满', '县', '局', '照', '参', '红', '细', '引', '听', '该', '铁', '价', '严', '首', '底', '液', '官', '德', '随', '病', '苏', '失', '尔', '死', '讲', '配', '女', '黄', '推', '显', '谈', '罪', '神', '艺', '呢', '席', '含', '企', '望', '密', '批', '营', '项', '防', '举', '球', '英', '氧', '势', '告', '李', '台', '落', '木', '帮', '轮', '破', '亚', '师', '围', '注', '远', '字', '材', '排', '供', '河', '态', '封', '另', '施', '减', '树', '溶', '怎', '止', '案', '言', '士', '均', '武', '固', '叶', '鱼', '波', '视', '仅', '费', '紧', '爱', '左', '章', '早', '朝', '害', '续', '轻', '服', '试', '食', '充', '兵', '源', '判', '护', '司', '足', '某', '练', '差', '致', '板', '田', '降', '黑', '犯', '负', '击', '范', '继', '兴', '似', '余', '坚', '曲', '输', '修', '故', '城', '夫', '够', '送', '笔', '船', '占', '右', '财', '吃', '富', '春', '职', '觉', '汉', '画', '功', '巴', '跟', '虽', '杂', '飞', '检', '吸', '助', '升', '阳', '互', '初', '创', '抗', '考', '投', '坏', '策', '古', '径', '换', '未', '跑', '留', '钢', '曾', '端', '责', '站', '简', '述', '钱', '副', '尽', '帝', '射', '草', '冲', '承', '独', '令', '限', '阿', '宣', '环', '双', '请', '超', '微', '让', '控', '州', '良', '轴', '找', '否', '纪', '益', '依', '优', '顶', '础', '载', '倒', '房', '突', '坐', '粉', '敌', '略', '客', '袁', '冷', '胜', '绝', '析', '块', '剂', '测', '丝', '协', '诉', '念', '陈', '仍', '罗', '盐', '友', '洋', '错', '苦', '夜', '刑', '移', '频', '逐', '靠', '混', '母', '短', '皮', '终', '聚', '汽', '村', '云', '哪', '既', '距', '卫', '停', '烈', '央', '察', '烧', '迅', '境', '若', '印', '洲', '刻', '括', '激', '孔', '搞', '甚', '室', '待', '核', '校', '散', '侵', '吧', '甲', '游', '久', '菜', '味', '旧', '模', '湖', '货', '损', '预', '阻', '毫', '普', '稳', '乙', '妈', '植', '息', '扩', '银', '语', '挥', '酒', '守', '拿', '序', '纸', '医', '缺', '雨', '吗', '针', '刘', '啊', '急', '唱', '误', '训', '愿', '审', '附', '获', '茶', '鲜', '粮', '斤', '孩', '脱', '硫', '肥', '善', '龙', '演', '父', '渐', '血', '欢', '械', '掌', '歌', '沙', '刚', '攻', '谓', '盾', '讨', '晚', '粒', '乱', '燃', '矛', '乎', '杀', '药', '宁', '鲁', '贵', '钟', '煤', '读', '班', '伯', '香', '介', '迫', '句', '丰', '培', '握', '兰', '担', '弦', '蛋', '沉', '假', '穿', '执', '答', '乐', '谁', '顺', '烟', '缩', '征', '脸', '喜', '松', '脚', '困', '异', '免', '背', '星', '福', '买', '染', '井', '概', '慢', '怕', '磁', '倍', '祖', '皇', '促', '静', '补', '评', '翻', '肉', '践', '尼', '衣', '宽', '扬', '棉', '希', '伤', '操', '垂', '秋', '宜', '氢', '套', '督', '振', '架', '亮', '末', '宪', '庆', '编', '牛', '触', '映', '雷', '销', '诗', '座', '居', '抓', '裂', '胞', '呼', '娘', '景', '威', '绿', '晶', '厚', '盟', '衡', '鸡', '孙', '延', '危', '胶', '屋', '乡', '临', '陆', '顾', '掉', '呀', '灯', '岁', '措', '束', '耐', '剧', '玉', '赵', '跳', '哥', '季', '课', '凯', '胡', '额', '款', '绍', '卷', '齐', '伟', '蒸', '殖', '永', '宗', '苗', '川', '炉', '岩', '弱', '零', '杨', '奏', '沿', '露', '杆', '探', '滑', '镇', '饭', '浓', '航', '怀', '赶', '库', '夺', '伊', '灵', '税', '途', '灭', '赛', '归', '召', '鼓', '播', '盘', '裁', '险', '康', '唯', '录', '菌', '纯', '借', '糖', '盖', '横', '符', '私', '努', '堂', '域', '枪', '润', '幅', '哈', '竟', '熟', '虫', '泽', '脑', '壤', '碳', '欧', '遍', '侧', '寨', '敢', '彻', '虑', '斜', '薄', '庭', '纳', '弹', '饲', '伸', '折', '麦', '湿', '暗', '荷', '瓦', '塞', '床', '筑', '恶', '户', '访', '塔', '奇', '透', '梁', '刀', '旋', '迹', '卡', '氯', '遇', '份', '毒', '泥', '退', '洗', '摆', '灰', '彩', '卖', '耗', '夏', '择', '忙', '铜', '献', '硬', '予', '繁', '圈', '雪', '函', '亦', '抽', '篇', '阵', '阴', '丁', '尺', '追', '堆', '雄', '迎', '泛', '爸', '楼', '避', '谋', '吨', '野', '猪', '旗', '累', '偏', '典', '馆', '索', '秦', '脂', '潮', '爷', '豆', '忽', '托', '惊', '塑', '遗', '愈', '朱', '替', '纤', '粗', '倾', '尚', '痛', '楚', '谢', '奋', '购', '磨', '君', '池', '旁', '碎', '骨', '监', '捕', '弟', '暴', '割', '贯', '殊', '释', '词', '亡', '壁', '顿', '宝', '午', '尘', '闻', '揭', '炮', '残', '冬', '桥', '妇', '警', '综', '招', '吴', '付', '浮', '遭', '徐', '您', '摇', '谷', '赞', '箱', '隔', '订', '男', '吹', '园', '纷', '唐', '败', '宋', '玻', '巨', '耕', '坦', '荣', '闭', '湾', '键', '凡', '驻', '锅', '救', '恩', '剥', '凝', '碱', '齿', '截', '炼', '麻', '纺', '禁', '废', '盛', '版', '缓', '净', '睛', '昌', '婚', '涉', '筒', '嘴', '插', '岸', '朗', '庄', '街', '藏', '姑', '贸', '腐', '奴', '啦', '惯', '乘', '伙', '恢', '匀', '纱', '扎', '辩', '耳', '彪', '臣', '亿', '璃', '抵', '脉', '秀', '萨', '俄', '网', '舞', '店', '喷', '纵', '寸', '汗', '挂', '洪', '贺', '闪', '柬', '爆', '烯', '津', '稻', '墙', '软', '勇', '像', '滚', '厘', '蒙', '芳', '肯', '坡', '柱', '荡', '腿', '仪', '旅', '尾', '轧', '冰', '贡', '登', '黎', '削', '钻', '勒', '逃', '障', '氨', '郭', '峰', '币', '港', '伏', '轨', '亩', '毕', '擦', '莫', '刺', '浪', '秘', '援', '株', '健', '售', '股', '岛', '甘', '泡', '睡', '童', '铸', '汤', '阀', '休', '汇', '舍', '牧', '绕', '炸', '哲', '磷', '绩', '朋', '淡', '尖', '启', '陷', '柴', '呈', '徒', '颜', '泪', '稍', '忘', '泵', '蓝', '拖', '洞', '授', '镜', '辛', '壮', '锋', '贫', '虚', '弯', '摩', '泰', '幼', '廷', '尊', '窗', '纲', '弄', '隶', '疑', '氏', '宫', '姐', '震', '瑞', '怪', '尤', '琴', '循', '描', '膜', '违', '夹', '腰', '缘', '珠', '穷', '森', '枝', '竹', '沟', '催', '绳', '忆', '邦', '剩', '幸', '浆', '栏', '拥', '牙', '贮', '礼', '滤', '钠', '纹', '罢', '拍', '咱', '喊', '袖', '埃', '勤', '罚', '焦', '潜', '伍', '墨', '欲', '缝', '姓', '刊', '饱', '仿', '奖', '铝', '鬼', '丽', '跨', '默', '挖', '链', '扫', '喝', '袋', '炭', '污', '幕', '诸', '弧', '励', '梅', '奶', '洁', '灾', '舟', '鉴', '苯', '讼', '抱', '毁', '懂', '寒', '智', '埔', '寄', '届', '跃', '渡', '挑', '丹', '艰', '贝', '碰', '拔', '爹', '戴', '码', '梦', '芽', '熔', '赤', '渔', '哭', '敬', '颗', '奔', '铅', '仲', '虎', '稀', '妹', '乏', '珍', '申', '桌', '遵', '允', '隆', '螺', '仓', '魏', '锐', '晓', '氮', '兼', '隐', '碍', '赫', '拨', '忠', '肃', '缸', '牵', '抢', '博', '巧', '壳', '兄', '杜', '讯', '诚', '碧', '祥', '柯', '页', '巡', '矩', '悲', '灌', '龄', '伦', '票', '寻', '桂', '铺', '圣', '恐', '恰', '郑', '趣', '抬', '荒', '腾', '贴', '柔', '滴', '猛', '阔', '辆', '妻', '填', '撤', '储', '签', '闹', '扰', '紫', '砂', '递', '戏', '吊', '陶', '伐', '喂', '疗', '瓶', '婆', '抚', '臂', '摸', '忍', '虾', '蜡', '邻', '胸', '巩', '挤', '偶', '弃', '槽', '劲', '乳', '邓', '吉', '仁', '烂', '砖', '租', '乌', '舰', '伴', '瓜', '浅', '丙', '暂', '燥', '橡', '柳', '迷', '暖', '牌', '秧', '胆', '详', '簧', '踏', '瓷', '谱', '呆', '宾', '糊', '洛', '辉', '愤', '竞', '隙', '怒', '粘', '乃', '绪', '肩', '籍', '敏', '涂', '熙', '皆', '侦', '悬', '掘', '享', '纠', '醒', '狂', '锁', '淀', '恨', '牲', '霸', '爬', '赏', '逆', '玩', '陵', '祝', '秒', '浙', '貌', '役', '彼', '悉', '鸭', '趋', '凤', '晨', '畜', '辈', '秩', '卵', '署', '梯', '炎', '滩', '棋', '驱', '筛', '峡', '冒', '啥', '寿', '译', '浸', '泉', '帽', '迟', '硅', '疆', '贷', '漏', '稿', '冠', '嫩', '胁', '芯', '牢', '叛', '蚀', '奥', '鸣', '岭', '羊', '凭', '串', '塘', '绘', '酵', '融', '盆', '锡', '庙', '筹', '冻', '辅', '摄', '袭', '筋', '拒', '僚', '旱', '钾', '鸟', '漆', '沈', '眉', '疏', '添', '棒', '穗', '硝', '韩', '逼', '扭', '侨', '凉', '挺', '碗', '栽', '炒', '杯', '患', '馏', '劝', '豪', '辽', '勃', '鸿', '旦', '吏', '拜', '狗', '埋', '辊', '掩', '饮', '搬', '骂', '辞', '勾', '扣', '估', '蒋', '绒', '雾', '丈', '朵', '姆', '拟', '宇', '辑', '陕', '雕', '偿', '蓄', '崇', '剪', '倡', '厅', '咬', '驶', '薯', '刷', '斥', '番', '赋', '奉', '佛', '浇', '漫', '曼', '扇', '钙', '桃', '扶', '仔', '返', '俗', '亏', '腔', '鞋', '棱', '覆', '框', '悄', '叔', '撞', '骗', '勘', '旺', '沸', '孤', '吐', '孟', '渠', '屈', '疾', '妙', '惜', '仰', '狠', '胀', '谐', '抛', '霉', '桑', '岗', '嘛', '衰', '盗', '渗', '脏', '赖', '涌', '甜', '曹', '阅', '肌', '哩', '厉', '烃', '纬', '毅', '昨', '伪', '症', '煮', '叹', '钉', '搭', '茎', '笼', '酷', '偷', '弓', '锥', '恒', '杰', '坑', '鼻', '翼', '纶', '叙', '狱', '逮', '罐', '络', '棚', '抑', '膨', '蔬', '寺', '骤', '穆', '冶', '枯', '册', '尸', '凸', '绅', '坯', '牺', '焰', '轰', '欣', '晋', '瘦', '御', '锭', '锦', '丧', '旬', '锻', '垄', '搜', '扑', '邀', '亭', '酯', '迈', '舒', '脆', '酶', '闲', '忧', '酚', '顽', '羽', '涨', '卸', '仗', '陪', '辟', '惩', '杭', '姚', '肚', '捉', '飘', '漂', '昆', '欺', '吾', '郎', '烷', '汁', '呵', '饰', '萧', '雅', '邮', '迁', '燕', '撒', '姻', '赴', '宴', '烦', '债', '帐', '斑', '铃', '旨', '醇', '董', '饼', '雏', '姿', '拌', '傅', '腹', '妥', '揉', '贤', '拆', '歪', '葡', '胺', '丢', '浩', '徽', '昂', '垫', '挡', '览', '贪', '慰', '缴', '汪', '慌', '冯', '诺', '姜', '谊', '凶', '劣', '诬', '耀', '昏', '躺', '盈', '骑', '乔', '溪', '丛', '卢', '抹', '闷', '咨', '刮', '驾', '缆', '悟', '摘', '铒', '掷', '颇', '幻', '柄', '惠', '惨', '佳', '仇', '腊', '窝', '涤', '剑', '瞧', '堡', '泼', '葱', '罩', '霍', '捞', '胎', '苍', '滨', '俩', '捅', '湘', '砍', '霞', '邵', '萄', '疯', '淮', '遂', '熊', '粪', '烘', '宿', '档', '戈', '驳', '嫂', '裕', '徙', '箭', '捐', '肠', '撑', '晒', '辨', '殿', '莲', '摊', '搅', '酱', '屏', '疫', '哀', '蔡', '堵', '沫', '皱', '畅', '叠', '阁', '莱', '敲', '辖', '钩', '痕', '坝', '巷', '饿', '祸', '丘', '玄', '溜', '曰', '逻', '彭', '尝', '卿', '妨', '艇', '吞', '韦', '怨', '矮', '歇' ] ================================================ FILE: lbry/wallet/words/english.py ================================================ words = [ 'abandon', 'ability', 'able', 'about', 'above', 'absent', 'absorb', 'abstract', 'absurd', 'abuse', 'access', 'accident', 'account', 'accuse', 'achieve', 'acid', 'acoustic', 'acquire', 'across', 'act', 'action', 'actor', 'actress', 'actual', 'adapt', 'add', 'addict', 'address', 'adjust', 'admit', 'adult', 'advance', 'advice', 'aerobic', 'affair', 'afford', 'afraid', 'again', 'age', 'agent', 'agree', 'ahead', 'aim', 'air', 'airport', 'aisle', 'alarm', 'album', 'alcohol', 'alert', 'alien', 'all', 'alley', 'allow', 'almost', 'alone', 'alpha', 'already', 'also', 'alter', 'always', 'amateur', 'amazing', 'among', 'amount', 'amused', 'analyst', 'anchor', 'ancient', 'anger', 'angle', 'angry', 'animal', 'ankle', 'announce', 'annual', 'another', 'answer', 'antenna', 'antique', 'anxiety', 'any', 'apart', 'apology', 'appear', 'apple', 'approve', 'april', 'arch', 'arctic', 'area', 'arena', 'argue', 'arm', 'armed', 'armor', 'army', 'around', 'arrange', 'arrest', 'arrive', 'arrow', 'art', 'artefact', 'artist', 'artwork', 'ask', 'aspect', 'assault', 'asset', 'assist', 'assume', 'asthma', 'athlete', 'atom', 'attack', 'attend', 'attitude', 'attract', 'auction', 'audit', 'august', 'aunt', 'author', 'auto', 'autumn', 'average', 'avocado', 'avoid', 'awake', 'aware', 'away', 'awesome', 'awful', 'awkward', 'axis', 'baby', 'bachelor', 'bacon', 'badge', 'bag', 'balance', 'balcony', 'ball', 'bamboo', 'banana', 'banner', 'bar', 'barely', 'bargain', 'barrel', 'base', 'basic', 'basket', 'battle', 'beach', 'bean', 'beauty', 'because', 'become', 'beef', 'before', 'begin', 'behave', 'behind', 'believe', 'below', 'belt', 'bench', 'benefit', 'best', 'betray', 'better', 'between', 'beyond', 'bicycle', 'bid', 'bike', 'bind', 'biology', 'bird', 'birth', 'bitter', 'black', 'blade', 'blame', 'blanket', 'blast', 'bleak', 'bless', 'blind', 'blood', 'blossom', 'blouse', 'blue', 'blur', 'blush', 'board', 'boat', 'body', 'boil', 'bomb', 'bone', 'bonus', 'book', 'boost', 'border', 'boring', 'borrow', 'boss', 'bottom', 'bounce', 'box', 'boy', 'bracket', 'brain', 'brand', 'brass', 'brave', 'bread', 'breeze', 'brick', 'bridge', 'brief', 'bright', 'bring', 'brisk', 'broccoli', 'broken', 'bronze', 'broom', 'brother', 'brown', 'brush', 'bubble', 'buddy', 'budget', 'buffalo', 'build', 'bulb', 'bulk', 'bullet', 'bundle', 'bunker', 'burden', 'burger', 'burst', 'bus', 'business', 'busy', 'butter', 'buyer', 'buzz', 'cabbage', 'cabin', 'cable', 'cactus', 'cage', 'cake', 'call', 'calm', 'camera', 'camp', 'can', 'canal', 'cancel', 'candy', 'cannon', 'canoe', 'canvas', 'canyon', 'capable', 'capital', 'captain', 'car', 'carbon', 'card', 'cargo', 'carpet', 'carry', 'cart', 'case', 'cash', 'casino', 'castle', 'casual', 'cat', 'catalog', 'catch', 'category', 'cattle', 'caught', 'cause', 'caution', 'cave', 'ceiling', 'celery', 'cement', 'census', 'century', 'cereal', 'certain', 'chair', 'chalk', 'champion', 'change', 'chaos', 'chapter', 'charge', 'chase', 'chat', 'cheap', 'check', 'cheese', 'chef', 'cherry', 'chest', 'chicken', 'chief', 'child', 'chimney', 'choice', 'choose', 'chronic', 'chuckle', 'chunk', 'churn', 'cigar', 'cinnamon', 'circle', 'citizen', 'city', 'civil', 'claim', 'clap', 'clarify', 'claw', 'clay', 'clean', 'clerk', 'clever', 'click', 'client', 'cliff', 'climb', 'clinic', 'clip', 'clock', 'clog', 'close', 'cloth', 'cloud', 'clown', 'club', 'clump', 'cluster', 'clutch', 'coach', 'coast', 'coconut', 'code', 'coffee', 'coil', 'coin', 'collect', 'color', 'column', 'combine', 'come', 'comfort', 'comic', 'common', 'company', 'concert', 'conduct', 'confirm', 'congress', 'connect', 'consider', 'control', 'convince', 'cook', 'cool', 'copper', 'copy', 'coral', 'core', 'corn', 'correct', 'cost', 'cotton', 'couch', 'country', 'couple', 'course', 'cousin', 'cover', 'coyote', 'crack', 'cradle', 'craft', 'cram', 'crane', 'crash', 'crater', 'crawl', 'crazy', 'cream', 'credit', 'creek', 'crew', 'cricket', 'crime', 'crisp', 'critic', 'crop', 'cross', 'crouch', 'crowd', 'crucial', 'cruel', 'cruise', 'crumble', 'crunch', 'crush', 'cry', 'crystal', 'cube', 'culture', 'cup', 'cupboard', 'curious', 'current', 'curtain', 'curve', 'cushion', 'custom', 'cute', 'cycle', 'dad', 'damage', 'damp', 'dance', 'danger', 'daring', 'dash', 'daughter', 'dawn', 'day', 'deal', 'debate', 'debris', 'decade', 'december', 'decide', 'decline', 'decorate', 'decrease', 'deer', 'defense', 'define', 'defy', 'degree', 'delay', 'deliver', 'demand', 'demise', 'denial', 'dentist', 'deny', 'depart', 'depend', 'deposit', 'depth', 'deputy', 'derive', 'describe', 'desert', 'design', 'desk', 'despair', 'destroy', 'detail', 'detect', 'develop', 'device', 'devote', 'diagram', 'dial', 'diamond', 'diary', 'dice', 'diesel', 'diet', 'differ', 'digital', 'dignity', 'dilemma', 'dinner', 'dinosaur', 'direct', 'dirt', 'disagree', 'discover', 'disease', 'dish', 'dismiss', 'disorder', 'display', 'distance', 'divert', 'divide', 'divorce', 'dizzy', 'doctor', 'document', 'dog', 'doll', 'dolphin', 'domain', 'donate', 'donkey', 'donor', 'door', 'dose', 'double', 'dove', 'draft', 'dragon', 'drama', 'drastic', 'draw', 'dream', 'dress', 'drift', 'drill', 'drink', 'drip', 'drive', 'drop', 'drum', 'dry', 'duck', 'dumb', 'dune', 'during', 'dust', 'dutch', 'duty', 'dwarf', 'dynamic', 'eager', 'eagle', 'early', 'earn', 'earth', 'easily', 'east', 'easy', 'echo', 'ecology', 'economy', 'edge', 'edit', 'educate', 'effort', 'egg', 'eight', 'either', 'elbow', 'elder', 'electric', 'elegant', 'element', 'elephant', 'elevator', 'elite', 'else', 'embark', 'embody', 'embrace', 'emerge', 'emotion', 'employ', 'empower', 'empty', 'enable', 'enact', 'end', 'endless', 'endorse', 'enemy', 'energy', 'enforce', 'engage', 'engine', 'enhance', 'enjoy', 'enlist', 'enough', 'enrich', 'enroll', 'ensure', 'enter', 'entire', 'entry', 'envelope', 'episode', 'equal', 'equip', 'era', 'erase', 'erode', 'erosion', 'error', 'erupt', 'escape', 'essay', 'essence', 'estate', 'eternal', 'ethics', 'evidence', 'evil', 'evoke', 'evolve', 'exact', 'example', 'excess', 'exchange', 'excite', 'exclude', 'excuse', 'execute', 'exercise', 'exhaust', 'exhibit', 'exile', 'exist', 'exit', 'exotic', 'expand', 'expect', 'expire', 'explain', 'expose', 'express', 'extend', 'extra', 'eye', 'eyebrow', 'fabric', 'face', 'faculty', 'fade', 'faint', 'faith', 'fall', 'false', 'fame', 'family', 'famous', 'fan', 'fancy', 'fantasy', 'farm', 'fashion', 'fat', 'fatal', 'father', 'fatigue', 'fault', 'favorite', 'feature', 'february', 'federal', 'fee', 'feed', 'feel', 'female', 'fence', 'festival', 'fetch', 'fever', 'few', 'fiber', 'fiction', 'field', 'figure', 'file', 'film', 'filter', 'final', 'find', 'fine', 'finger', 'finish', 'fire', 'firm', 'first', 'fiscal', 'fish', 'fit', 'fitness', 'fix', 'flag', 'flame', 'flash', 'flat', 'flavor', 'flee', 'flight', 'flip', 'float', 'flock', 'floor', 'flower', 'fluid', 'flush', 'fly', 'foam', 'focus', 'fog', 'foil', 'fold', 'follow', 'food', 'foot', 'force', 'forest', 'forget', 'fork', 'fortune', 'forum', 'forward', 'fossil', 'foster', 'found', 'fox', 'fragile', 'frame', 'frequent', 'fresh', 'friend', 'fringe', 'frog', 'front', 'frost', 'frown', 'frozen', 'fruit', 'fuel', 'fun', 'funny', 'furnace', 'fury', 'future', 'gadget', 'gain', 'galaxy', 'gallery', 'game', 'gap', 'garage', 'garbage', 'garden', 'garlic', 'garment', 'gas', 'gasp', 'gate', 'gather', 'gauge', 'gaze', 'general', 'genius', 'genre', 'gentle', 'genuine', 'gesture', 'ghost', 'giant', 'gift', 'giggle', 'ginger', 'giraffe', 'girl', 'give', 'glad', 'glance', 'glare', 'glass', 'glide', 'glimpse', 'globe', 'gloom', 'glory', 'glove', 'glow', 'glue', 'goat', 'goddess', 'gold', 'good', 'goose', 'gorilla', 'gospel', 'gossip', 'govern', 'gown', 'grab', 'grace', 'grain', 'grant', 'grape', 'grass', 'gravity', 'great', 'green', 'grid', 'grief', 'grit', 'grocery', 'group', 'grow', 'grunt', 'guard', 'guess', 'guide', 'guilt', 'guitar', 'gun', 'gym', 'habit', 'hair', 'half', 'hammer', 'hamster', 'hand', 'happy', 'harbor', 'hard', 'harsh', 'harvest', 'hat', 'have', 'hawk', 'hazard', 'head', 'health', 'heart', 'heavy', 'hedgehog', 'height', 'hello', 'helmet', 'help', 'hen', 'hero', 'hidden', 'high', 'hill', 'hint', 'hip', 'hire', 'history', 'hobby', 'hockey', 'hold', 'hole', 'holiday', 'hollow', 'home', 'honey', 'hood', 'hope', 'horn', 'horror', 'horse', 'hospital', 'host', 'hotel', 'hour', 'hover', 'hub', 'huge', 'human', 'humble', 'humor', 'hundred', 'hungry', 'hunt', 'hurdle', 'hurry', 'hurt', 'husband', 'hybrid', 'ice', 'icon', 'idea', 'identify', 'idle', 'ignore', 'ill', 'illegal', 'illness', 'image', 'imitate', 'immense', 'immune', 'impact', 'impose', 'improve', 'impulse', 'inch', 'include', 'income', 'increase', 'index', 'indicate', 'indoor', 'industry', 'infant', 'inflict', 'inform', 'inhale', 'inherit', 'initial', 'inject', 'injury', 'inmate', 'inner', 'innocent', 'input', 'inquiry', 'insane', 'insect', 'inside', 'inspire', 'install', 'intact', 'interest', 'into', 'invest', 'invite', 'involve', 'iron', 'island', 'isolate', 'issue', 'item', 'ivory', 'jacket', 'jaguar', 'jar', 'jazz', 'jealous', 'jeans', 'jelly', 'jewel', 'job', 'join', 'joke', 'journey', 'joy', 'judge', 'juice', 'jump', 'jungle', 'junior', 'junk', 'just', 'kangaroo', 'keen', 'keep', 'ketchup', 'key', 'kick', 'kid', 'kidney', 'kind', 'kingdom', 'kiss', 'kit', 'kitchen', 'kite', 'kitten', 'kiwi', 'knee', 'knife', 'knock', 'know', 'lab', 'label', 'labor', 'ladder', 'lady', 'lake', 'lamp', 'language', 'laptop', 'large', 'later', 'latin', 'laugh', 'laundry', 'lava', 'law', 'lawn', 'lawsuit', 'layer', 'lazy', 'leader', 'leaf', 'learn', 'leave', 'lecture', 'left', 'leg', 'legal', 'legend', 'leisure', 'lemon', 'lend', 'length', 'lens', 'leopard', 'lesson', 'letter', 'level', 'liar', 'liberty', 'library', 'license', 'life', 'lift', 'light', 'like', 'limb', 'limit', 'link', 'lion', 'liquid', 'list', 'little', 'live', 'lizard', 'load', 'loan', 'lobster', 'local', 'lock', 'logic', 'lonely', 'long', 'loop', 'lottery', 'loud', 'lounge', 'love', 'loyal', 'lucky', 'luggage', 'lumber', 'lunar', 'lunch', 'luxury', 'lyrics', 'machine', 'mad', 'magic', 'magnet', 'maid', 'mail', 'main', 'major', 'make', 'mammal', 'man', 'manage', 'mandate', 'mango', 'mansion', 'manual', 'maple', 'marble', 'march', 'margin', 'marine', 'market', 'marriage', 'mask', 'mass', 'master', 'match', 'material', 'math', 'matrix', 'matter', 'maximum', 'maze', 'meadow', 'mean', 'measure', 'meat', 'mechanic', 'medal', 'media', 'melody', 'melt', 'member', 'memory', 'mention', 'menu', 'mercy', 'merge', 'merit', 'merry', 'mesh', 'message', 'metal', 'method', 'middle', 'midnight', 'milk', 'million', 'mimic', 'mind', 'minimum', 'minor', 'minute', 'miracle', 'mirror', 'misery', 'miss', 'mistake', 'mix', 'mixed', 'mixture', 'mobile', 'model', 'modify', 'mom', 'moment', 'monitor', 'monkey', 'monster', 'month', 'moon', 'moral', 'more', 'morning', 'mosquito', 'mother', 'motion', 'motor', 'mountain', 'mouse', 'move', 'movie', 'much', 'muffin', 'mule', 'multiply', 'muscle', 'museum', 'mushroom', 'music', 'must', 'mutual', 'myself', 'mystery', 'myth', 'naive', 'name', 'napkin', 'narrow', 'nasty', 'nation', 'nature', 'near', 'neck', 'need', 'negative', 'neglect', 'neither', 'nephew', 'nerve', 'nest', 'net', 'network', 'neutral', 'never', 'news', 'next', 'nice', 'night', 'noble', 'noise', 'nominee', 'noodle', 'normal', 'north', 'nose', 'notable', 'note', 'nothing', 'notice', 'novel', 'now', 'nuclear', 'number', 'nurse', 'nut', 'oak', 'obey', 'object', 'oblige', 'obscure', 'observe', 'obtain', 'obvious', 'occur', 'ocean', 'october', 'odor', 'off', 'offer', 'office', 'often', 'oil', 'okay', 'old', 'olive', 'olympic', 'omit', 'once', 'one', 'onion', 'online', 'only', 'open', 'opera', 'opinion', 'oppose', 'option', 'orange', 'orbit', 'orchard', 'order', 'ordinary', 'organ', 'orient', 'original', 'orphan', 'ostrich', 'other', 'outdoor', 'outer', 'output', 'outside', 'oval', 'oven', 'over', 'own', 'owner', 'oxygen', 'oyster', 'ozone', 'pact', 'paddle', 'page', 'pair', 'palace', 'palm', 'panda', 'panel', 'panic', 'panther', 'paper', 'parade', 'parent', 'park', 'parrot', 'party', 'pass', 'patch', 'path', 'patient', 'patrol', 'pattern', 'pause', 'pave', 'payment', 'peace', 'peanut', 'pear', 'peasant', 'pelican', 'pen', 'penalty', 'pencil', 'people', 'pepper', 'perfect', 'permit', 'person', 'pet', 'phone', 'photo', 'phrase', 'physical', 'piano', 'picnic', 'picture', 'piece', 'pig', 'pigeon', 'pill', 'pilot', 'pink', 'pioneer', 'pipe', 'pistol', 'pitch', 'pizza', 'place', 'planet', 'plastic', 'plate', 'play', 'please', 'pledge', 'pluck', 'plug', 'plunge', 'poem', 'poet', 'point', 'polar', 'pole', 'police', 'pond', 'pony', 'pool', 'popular', 'portion', 'position', 'possible', 'post', 'potato', 'pottery', 'poverty', 'powder', 'power', 'practice', 'praise', 'predict', 'prefer', 'prepare', 'present', 'pretty', 'prevent', 'price', 'pride', 'primary', 'print', 'priority', 'prison', 'private', 'prize', 'problem', 'process', 'produce', 'profit', 'program', 'project', 'promote', 'proof', 'property', 'prosper', 'protect', 'proud', 'provide', 'public', 'pudding', 'pull', 'pulp', 'pulse', 'pumpkin', 'punch', 'pupil', 'puppy', 'purchase', 'purity', 'purpose', 'purse', 'push', 'put', 'puzzle', 'pyramid', 'quality', 'quantum', 'quarter', 'question', 'quick', 'quit', 'quiz', 'quote', 'rabbit', 'raccoon', 'race', 'rack', 'radar', 'radio', 'rail', 'rain', 'raise', 'rally', 'ramp', 'ranch', 'random', 'range', 'rapid', 'rare', 'rate', 'rather', 'raven', 'raw', 'razor', 'ready', 'real', 'reason', 'rebel', 'rebuild', 'recall', 'receive', 'recipe', 'record', 'recycle', 'reduce', 'reflect', 'reform', 'refuse', 'region', 'regret', 'regular', 'reject', 'relax', 'release', 'relief', 'rely', 'remain', 'remember', 'remind', 'remove', 'render', 'renew', 'rent', 'reopen', 'repair', 'repeat', 'replace', 'report', 'require', 'rescue', 'resemble', 'resist', 'resource', 'response', 'result', 'retire', 'retreat', 'return', 'reunion', 'reveal', 'review', 'reward', 'rhythm', 'rib', 'ribbon', 'rice', 'rich', 'ride', 'ridge', 'rifle', 'right', 'rigid', 'ring', 'riot', 'ripple', 'risk', 'ritual', 'rival', 'river', 'road', 'roast', 'robot', 'robust', 'rocket', 'romance', 'roof', 'rookie', 'room', 'rose', 'rotate', 'rough', 'round', 'route', 'royal', 'rubber', 'rude', 'rug', 'rule', 'run', 'runway', 'rural', 'sad', 'saddle', 'sadness', 'safe', 'sail', 'salad', 'salmon', 'salon', 'salt', 'salute', 'same', 'sample', 'sand', 'satisfy', 'satoshi', 'sauce', 'sausage', 'save', 'say', 'scale', 'scan', 'scare', 'scatter', 'scene', 'scheme', 'school', 'science', 'scissors', 'scorpion', 'scout', 'scrap', 'screen', 'script', 'scrub', 'sea', 'search', 'season', 'seat', 'second', 'secret', 'section', 'security', 'seed', 'seek', 'segment', 'select', 'sell', 'seminar', 'senior', 'sense', 'sentence', 'series', 'service', 'session', 'settle', 'setup', 'seven', 'shadow', 'shaft', 'shallow', 'share', 'shed', 'shell', 'sheriff', 'shield', 'shift', 'shine', 'ship', 'shiver', 'shock', 'shoe', 'shoot', 'shop', 'short', 'shoulder', 'shove', 'shrimp', 'shrug', 'shuffle', 'shy', 'sibling', 'sick', 'side', 'siege', 'sight', 'sign', 'silent', 'silk', 'silly', 'silver', 'similar', 'simple', 'since', 'sing', 'siren', 'sister', 'situate', 'six', 'size', 'skate', 'sketch', 'ski', 'skill', 'skin', 'skirt', 'skull', 'slab', 'slam', 'sleep', 'slender', 'slice', 'slide', 'slight', 'slim', 'slogan', 'slot', 'slow', 'slush', 'small', 'smart', 'smile', 'smoke', 'smooth', 'snack', 'snake', 'snap', 'sniff', 'snow', 'soap', 'soccer', 'social', 'sock', 'soda', 'soft', 'solar', 'soldier', 'solid', 'solution', 'solve', 'someone', 'song', 'soon', 'sorry', 'sort', 'soul', 'sound', 'soup', 'source', 'south', 'space', 'spare', 'spatial', 'spawn', 'speak', 'special', 'speed', 'spell', 'spend', 'sphere', 'spice', 'spider', 'spike', 'spin', 'spirit', 'split', 'spoil', 'sponsor', 'spoon', 'sport', 'spot', 'spray', 'spread', 'spring', 'spy', 'square', 'squeeze', 'squirrel', 'stable', 'stadium', 'staff', 'stage', 'stairs', 'stamp', 'stand', 'start', 'state', 'stay', 'steak', 'steel', 'stem', 'step', 'stereo', 'stick', 'still', 'sting', 'stock', 'stomach', 'stone', 'stool', 'story', 'stove', 'strategy', 'street', 'strike', 'strong', 'struggle', 'student', 'stuff', 'stumble', 'style', 'subject', 'submit', 'subway', 'success', 'such', 'sudden', 'suffer', 'sugar', 'suggest', 'suit', 'summer', 'sun', 'sunny', 'sunset', 'super', 'supply', 'supreme', 'sure', 'surface', 'surge', 'surprise', 'surround', 'survey', 'suspect', 'sustain', 'swallow', 'swamp', 'swap', 'swarm', 'swear', 'sweet', 'swift', 'swim', 'swing', 'switch', 'sword', 'symbol', 'symptom', 'syrup', 'system', 'table', 'tackle', 'tag', 'tail', 'talent', 'talk', 'tank', 'tape', 'target', 'task', 'taste', 'tattoo', 'taxi', 'teach', 'team', 'tell', 'ten', 'tenant', 'tennis', 'tent', 'term', 'test', 'text', 'thank', 'that', 'theme', 'then', 'theory', 'there', 'they', 'thing', 'this', 'thought', 'three', 'thrive', 'throw', 'thumb', 'thunder', 'ticket', 'tide', 'tiger', 'tilt', 'timber', 'time', 'tiny', 'tip', 'tired', 'tissue', 'title', 'toast', 'tobacco', 'today', 'toddler', 'toe', 'together', 'toilet', 'token', 'tomato', 'tomorrow', 'tone', 'tongue', 'tonight', 'tool', 'tooth', 'top', 'topic', 'topple', 'torch', 'tornado', 'tortoise', 'toss', 'total', 'tourist', 'toward', 'tower', 'town', 'toy', 'track', 'trade', 'traffic', 'tragic', 'train', 'transfer', 'trap', 'trash', 'travel', 'tray', 'treat', 'tree', 'trend', 'trial', 'tribe', 'trick', 'trigger', 'trim', 'trip', 'trophy', 'trouble', 'truck', 'true', 'truly', 'trumpet', 'trust', 'truth', 'try', 'tube', 'tuition', 'tumble', 'tuna', 'tunnel', 'turkey', 'turn', 'turtle', 'twelve', 'twenty', 'twice', 'twin', 'twist', 'two', 'type', 'typical', 'ugly', 'umbrella', 'unable', 'unaware', 'uncle', 'uncover', 'under', 'undo', 'unfair', 'unfold', 'unhappy', 'uniform', 'unique', 'unit', 'universe', 'unknown', 'unlock', 'until', 'unusual', 'unveil', 'update', 'upgrade', 'uphold', 'upon', 'upper', 'upset', 'urban', 'urge', 'usage', 'use', 'used', 'useful', 'useless', 'usual', 'utility', 'vacant', 'vacuum', 'vague', 'valid', 'valley', 'valve', 'van', 'vanish', 'vapor', 'various', 'vast', 'vault', 'vehicle', 'velvet', 'vendor', 'venture', 'venue', 'verb', 'verify', 'version', 'very', 'vessel', 'veteran', 'viable', 'vibrant', 'vicious', 'victory', 'video', 'view', 'village', 'vintage', 'violin', 'virtual', 'virus', 'visa', 'visit', 'visual', 'vital', 'vivid', 'vocal', 'voice', 'void', 'volcano', 'volume', 'vote', 'voyage', 'wage', 'wagon', 'wait', 'walk', 'wall', 'walnut', 'want', 'warfare', 'warm', 'warrior', 'wash', 'wasp', 'waste', 'water', 'wave', 'way', 'wealth', 'weapon', 'wear', 'weasel', 'weather', 'web', 'wedding', 'weekend', 'weird', 'welcome', 'west', 'wet', 'whale', 'what', 'wheat', 'wheel', 'when', 'where', 'whip', 'whisper', 'wide', 'width', 'wife', 'wild', 'will', 'win', 'window', 'wine', 'wing', 'wink', 'winner', 'winter', 'wire', 'wisdom', 'wise', 'wish', 'witness', 'wolf', 'woman', 'wonder', 'wood', 'wool', 'word', 'work', 'world', 'worry', 'worth', 'wrap', 'wreck', 'wrestle', 'wrist', 'write', 'wrong', 'yard', 'year', 'yellow', 'you', 'young', 'youth', 'zebra', 'zero', 'zone', 'zoo' ] ================================================ FILE: lbry/wallet/words/japanese.py ================================================ words = [ 'あいこくしん', 'あいさつ', 'あいだ', 'あおぞら', 'あかちゃん', 'あきる', 'あけがた', 'あける', 'あこがれる', 'あさい', 'あさひ', 'あしあと', 'あじわう', 'あずかる', 'あずき', 'あそぶ', 'あたえる', 'あたためる', 'あたりまえ', 'あたる', 'あつい', 'あつかう', 'あっしゅく', 'あつまり', 'あつめる', 'あてな', 'あてはまる', 'あひる', 'あぶら', 'あぶる', 'あふれる', 'あまい', 'あまど', 'あまやかす', 'あまり', 'あみもの', 'あめりか', 'あやまる', 'あゆむ', 'あらいぐま', 'あらし', 'あらすじ', 'あらためる', 'あらゆる', 'あらわす', 'ありがとう', 'あわせる', 'あわてる', 'あんい', 'あんがい', 'あんこ', 'あんぜん', 'あんてい', 'あんない', 'あんまり', 'いいだす', 'いおん', 'いがい', 'いがく', 'いきおい', 'いきなり', 'いきもの', 'いきる', 'いくじ', 'いくぶん', 'いけばな', 'いけん', 'いこう', 'いこく', 'いこつ', 'いさましい', 'いさん', 'いしき', 'いじゅう', 'いじょう', 'いじわる', 'いずみ', 'いずれ', 'いせい', 'いせえび', 'いせかい', 'いせき', 'いぜん', 'いそうろう', 'いそがしい', 'いだい', 'いだく', 'いたずら', 'いたみ', 'いたりあ', 'いちおう', 'いちじ', 'いちど', 'いちば', 'いちぶ', 'いちりゅう', 'いつか', 'いっしゅん', 'いっせい', 'いっそう', 'いったん', 'いっち', 'いってい', 'いっぽう', 'いてざ', 'いてん', 'いどう', 'いとこ', 'いない', 'いなか', 'いねむり', 'いのち', 'いのる', 'いはつ', 'いばる', 'いはん', 'いびき', 'いひん', 'いふく', 'いへん', 'いほう', 'いみん', 'いもうと', 'いもたれ', 'いもり', 'いやがる', 'いやす', 'いよかん', 'いよく', 'いらい', 'いらすと', 'いりぐち', 'いりょう', 'いれい', 'いれもの', 'いれる', 'いろえんぴつ', 'いわい', 'いわう', 'いわかん', 'いわば', 'いわゆる', 'いんげんまめ', 'いんさつ', 'いんしょう', 'いんよう', 'うえき', 'うえる', 'うおざ', 'うがい', 'うかぶ', 'うかべる', 'うきわ', 'うくらいな', 'うくれれ', 'うけたまわる', 'うけつけ', 'うけとる', 'うけもつ', 'うける', 'うごかす', 'うごく', 'うこん', 'うさぎ', 'うしなう', 'うしろがみ', 'うすい', 'うすぎ', 'うすぐらい', 'うすめる', 'うせつ', 'うちあわせ', 'うちがわ', 'うちき', 'うちゅう', 'うっかり', 'うつくしい', 'うったえる', 'うつる', 'うどん', 'うなぎ', 'うなじ', 'うなずく', 'うなる', 'うねる', 'うのう', 'うぶげ', 'うぶごえ', 'うまれる', 'うめる', 'うもう', 'うやまう', 'うよく', 'うらがえす', 'うらぐち', 'うらない', 'うりあげ', 'うりきれ', 'うるさい', 'うれしい', 'うれゆき', 'うれる', 'うろこ', 'うわき', 'うわさ', 'うんこう', 'うんちん', 'うんてん', 'うんどう', 'えいえん', 'えいが', 'えいきょう', 'えいご', 'えいせい', 'えいぶん', 'えいよう', 'えいわ', 'えおり', 'えがお', 'えがく', 'えきたい', 'えくせる', 'えしゃく', 'えすて', 'えつらん', 'えのぐ', 'えほうまき', 'えほん', 'えまき', 'えもじ', 'えもの', 'えらい', 'えらぶ', 'えりあ', 'えんえん', 'えんかい', 'えんぎ', 'えんげき', 'えんしゅう', 'えんぜつ', 'えんそく', 'えんちょう', 'えんとつ', 'おいかける', 'おいこす', 'おいしい', 'おいつく', 'おうえん', 'おうさま', 'おうじ', 'おうせつ', 'おうたい', 'おうふく', 'おうべい', 'おうよう', 'おえる', 'おおい', 'おおう', 'おおどおり', 'おおや', 'おおよそ', 'おかえり', 'おかず', 'おがむ', 'おかわり', 'おぎなう', 'おきる', 'おくさま', 'おくじょう', 'おくりがな', 'おくる', 'おくれる', 'おこす', 'おこなう', 'おこる', 'おさえる', 'おさない', 'おさめる', 'おしいれ', 'おしえる', 'おじぎ', 'おじさん', 'おしゃれ', 'おそらく', 'おそわる', 'おたがい', 'おたく', 'おだやか', 'おちつく', 'おっと', 'おつり', 'おでかけ', 'おとしもの', 'おとなしい', 'おどり', 'おどろかす', 'おばさん', 'おまいり', 'おめでとう', 'おもいで', 'おもう', 'おもたい', 'おもちゃ', 'おやつ', 'おやゆび', 'およぼす', 'おらんだ', 'おろす', 'おんがく', 'おんけい', 'おんしゃ', 'おんせん', 'おんだん', 'おんちゅう', 'おんどけい', 'かあつ', 'かいが', 'がいき', 'がいけん', 'がいこう', 'かいさつ', 'かいしゃ', 'かいすいよく', 'かいぜん', 'かいぞうど', 'かいつう', 'かいてん', 'かいとう', 'かいふく', 'がいへき', 'かいほう', 'かいよう', 'がいらい', 'かいわ', 'かえる', 'かおり', 'かかえる', 'かがく', 'かがし', 'かがみ', 'かくご', 'かくとく', 'かざる', 'がぞう', 'かたい', 'かたち', 'がちょう', 'がっきゅう', 'がっこう', 'がっさん', 'がっしょう', 'かなざわし', 'かのう', 'がはく', 'かぶか', 'かほう', 'かほご', 'かまう', 'かまぼこ', 'かめれおん', 'かゆい', 'かようび', 'からい', 'かるい', 'かろう', 'かわく', 'かわら', 'がんか', 'かんけい', 'かんこう', 'かんしゃ', 'かんそう', 'かんたん', 'かんち', 'がんばる', 'きあい', 'きあつ', 'きいろ', 'ぎいん', 'きうい', 'きうん', 'きえる', 'きおう', 'きおく', 'きおち', 'きおん', 'きかい', 'きかく', 'きかんしゃ', 'ききて', 'きくばり', 'きくらげ', 'きけんせい', 'きこう', 'きこえる', 'きこく', 'きさい', 'きさく', 'きさま', 'きさらぎ', 'ぎじかがく', 'ぎしき', 'ぎじたいけん', 'ぎじにってい', 'ぎじゅつしゃ', 'きすう', 'きせい', 'きせき', 'きせつ', 'きそう', 'きぞく', 'きぞん', 'きたえる', 'きちょう', 'きつえん', 'ぎっちり', 'きつつき', 'きつね', 'きてい', 'きどう', 'きどく', 'きない', 'きなが', 'きなこ', 'きぬごし', 'きねん', 'きのう', 'きのした', 'きはく', 'きびしい', 'きひん', 'きふく', 'きぶん', 'きぼう', 'きほん', 'きまる', 'きみつ', 'きむずかしい', 'きめる', 'きもだめし', 'きもち', 'きもの', 'きゃく', 'きやく', 'ぎゅうにく', 'きよう', 'きょうりゅう', 'きらい', 'きらく', 'きりん', 'きれい', 'きれつ', 'きろく', 'ぎろん', 'きわめる', 'ぎんいろ', 'きんかくじ', 'きんじょ', 'きんようび', 'ぐあい', 'くいず', 'くうかん', 'くうき', 'くうぐん', 'くうこう', 'ぐうせい', 'くうそう', 'ぐうたら', 'くうふく', 'くうぼ', 'くかん', 'くきょう', 'くげん', 'ぐこう', 'くさい', 'くさき', 'くさばな', 'くさる', 'くしゃみ', 'くしょう', 'くすのき', 'くすりゆび', 'くせげ', 'くせん', 'ぐたいてき', 'くださる', 'くたびれる', 'くちこみ', 'くちさき', 'くつした', 'ぐっすり', 'くつろぐ', 'くとうてん', 'くどく', 'くなん', 'くねくね', 'くのう', 'くふう', 'くみあわせ', 'くみたてる', 'くめる', 'くやくしょ', 'くらす', 'くらべる', 'くるま', 'くれる', 'くろう', 'くわしい', 'ぐんかん', 'ぐんしょく', 'ぐんたい', 'ぐんて', 'けあな', 'けいかく', 'けいけん', 'けいこ', 'けいさつ', 'げいじゅつ', 'けいたい', 'げいのうじん', 'けいれき', 'けいろ', 'けおとす', 'けおりもの', 'げきか', 'げきげん', 'げきだん', 'げきちん', 'げきとつ', 'げきは', 'げきやく', 'げこう', 'げこくじょう', 'げざい', 'けさき', 'げざん', 'けしき', 'けしごむ', 'けしょう', 'げすと', 'けたば', 'けちゃっぷ', 'けちらす', 'けつあつ', 'けつい', 'けつえき', 'けっこん', 'けつじょ', 'けっせき', 'けってい', 'けつまつ', 'げつようび', 'げつれい', 'けつろん', 'げどく', 'けとばす', 'けとる', 'けなげ', 'けなす', 'けなみ', 'けぬき', 'げねつ', 'けねん', 'けはい', 'げひん', 'けぶかい', 'げぼく', 'けまり', 'けみかる', 'けむし', 'けむり', 'けもの', 'けらい', 'けろけろ', 'けわしい', 'けんい', 'けんえつ', 'けんお', 'けんか', 'げんき', 'けんげん', 'けんこう', 'けんさく', 'けんしゅう', 'けんすう', 'げんそう', 'けんちく', 'けんてい', 'けんとう', 'けんない', 'けんにん', 'げんぶつ', 'けんま', 'けんみん', 'けんめい', 'けんらん', 'けんり', 'こあくま', 'こいぬ', 'こいびと', 'ごうい', 'こうえん', 'こうおん', 'こうかん', 'ごうきゅう', 'ごうけい', 'こうこう', 'こうさい', 'こうじ', 'こうすい', 'ごうせい', 'こうそく', 'こうたい', 'こうちゃ', 'こうつう', 'こうてい', 'こうどう', 'こうない', 'こうはい', 'ごうほう', 'ごうまん', 'こうもく', 'こうりつ', 'こえる', 'こおり', 'ごかい', 'ごがつ', 'ごかん', 'こくご', 'こくさい', 'こくとう', 'こくない', 'こくはく', 'こぐま', 'こけい', 'こける', 'ここのか', 'こころ', 'こさめ', 'こしつ', 'こすう', 'こせい', 'こせき', 'こぜん', 'こそだて', 'こたい', 'こたえる', 'こたつ', 'こちょう', 'こっか', 'こつこつ', 'こつばん', 'こつぶ', 'こてい', 'こてん', 'ことがら', 'ことし', 'ことば', 'ことり', 'こなごな', 'こねこね', 'このまま', 'このみ', 'このよ', 'ごはん', 'こひつじ', 'こふう', 'こふん', 'こぼれる', 'ごまあぶら', 'こまかい', 'ごますり', 'こまつな', 'こまる', 'こむぎこ', 'こもじ', 'こもち', 'こもの', 'こもん', 'こやく', 'こやま', 'こゆう', 'こゆび', 'こよい', 'こよう', 'こりる', 'これくしょん', 'ころっけ', 'こわもて', 'こわれる', 'こんいん', 'こんかい', 'こんき', 'こんしゅう', 'こんすい', 'こんだて', 'こんとん', 'こんなん', 'こんびに', 'こんぽん', 'こんまけ', 'こんや', 'こんれい', 'こんわく', 'ざいえき', 'さいかい', 'さいきん', 'ざいげん', 'ざいこ', 'さいしょ', 'さいせい', 'ざいたく', 'ざいちゅう', 'さいてき', 'ざいりょう', 'さうな', 'さかいし', 'さがす', 'さかな', 'さかみち', 'さがる', 'さぎょう', 'さくし', 'さくひん', 'さくら', 'さこく', 'さこつ', 'さずかる', 'ざせき', 'さたん', 'さつえい', 'ざつおん', 'ざっか', 'ざつがく', 'さっきょく', 'ざっし', 'さつじん', 'ざっそう', 'さつたば', 'さつまいも', 'さてい', 'さといも', 'さとう', 'さとおや', 'さとし', 'さとる', 'さのう', 'さばく', 'さびしい', 'さべつ', 'さほう', 'さほど', 'さます', 'さみしい', 'さみだれ', 'さむけ', 'さめる', 'さやえんどう', 'さゆう', 'さよう', 'さよく', 'さらだ', 'ざるそば', 'さわやか', 'さわる', 'さんいん', 'さんか', 'さんきゃく', 'さんこう', 'さんさい', 'ざんしょ', 'さんすう', 'さんせい', 'さんそ', 'さんち', 'さんま', 'さんみ', 'さんらん', 'しあい', 'しあげ', 'しあさって', 'しあわせ', 'しいく', 'しいん', 'しうち', 'しえい', 'しおけ', 'しかい', 'しかく', 'じかん', 'しごと', 'しすう', 'じだい', 'したうけ', 'したぎ', 'したて', 'したみ', 'しちょう', 'しちりん', 'しっかり', 'しつじ', 'しつもん', 'してい', 'してき', 'してつ', 'じてん', 'じどう', 'しなぎれ', 'しなもの', 'しなん', 'しねま', 'しねん', 'しのぐ', 'しのぶ', 'しはい', 'しばかり', 'しはつ', 'しはらい', 'しはん', 'しひょう', 'しふく', 'じぶん', 'しへい', 'しほう', 'しほん', 'しまう', 'しまる', 'しみん', 'しむける', 'じむしょ', 'しめい', 'しめる', 'しもん', 'しゃいん', 'しゃうん', 'しゃおん', 'じゃがいも', 'しやくしょ', 'しゃくほう', 'しゃけん', 'しゃこ', 'しゃざい', 'しゃしん', 'しゃせん', 'しゃそう', 'しゃたい', 'しゃちょう', 'しゃっきん', 'じゃま', 'しゃりん', 'しゃれい', 'じゆう', 'じゅうしょ', 'しゅくはく', 'じゅしん', 'しゅっせき', 'しゅみ', 'しゅらば', 'じゅんばん', 'しょうかい', 'しょくたく', 'しょっけん', 'しょどう', 'しょもつ', 'しらせる', 'しらべる', 'しんか', 'しんこう', 'じんじゃ', 'しんせいじ', 'しんちく', 'しんりん', 'すあげ', 'すあし', 'すあな', 'ずあん', 'すいえい', 'すいか', 'すいとう', 'ずいぶん', 'すいようび', 'すうがく', 'すうじつ', 'すうせん', 'すおどり', 'すきま', 'すくう', 'すくない', 'すける', 'すごい', 'すこし', 'ずさん', 'すずしい', 'すすむ', 'すすめる', 'すっかり', 'ずっしり', 'ずっと', 'すてき', 'すてる', 'すねる', 'すのこ', 'すはだ', 'すばらしい', 'ずひょう', 'ずぶぬれ', 'すぶり', 'すふれ', 'すべて', 'すべる', 'ずほう', 'すぼん', 'すまい', 'すめし', 'すもう', 'すやき', 'すらすら', 'するめ', 'すれちがう', 'すろっと', 'すわる', 'すんぜん', 'すんぽう', 'せあぶら', 'せいかつ', 'せいげん', 'せいじ', 'せいよう', 'せおう', 'せかいかん', 'せきにん', 'せきむ', 'せきゆ', 'せきらんうん', 'せけん', 'せこう', 'せすじ', 'せたい', 'せたけ', 'せっかく', 'せっきゃく', 'ぜっく', 'せっけん', 'せっこつ', 'せっさたくま', 'せつぞく', 'せつだん', 'せつでん', 'せっぱん', 'せつび', 'せつぶん', 'せつめい', 'せつりつ', 'せなか', 'せのび', 'せはば', 'せびろ', 'せぼね', 'せまい', 'せまる', 'せめる', 'せもたれ', 'せりふ', 'ぜんあく', 'せんい', 'せんえい', 'せんか', 'せんきょ', 'せんく', 'せんげん', 'ぜんご', 'せんさい', 'せんしゅ', 'せんすい', 'せんせい', 'せんぞ', 'せんたく', 'せんちょう', 'せんてい', 'せんとう', 'せんぬき', 'せんねん', 'せんぱい', 'ぜんぶ', 'ぜんぽう', 'せんむ', 'せんめんじょ', 'せんもん', 'せんやく', 'せんゆう', 'せんよう', 'ぜんら', 'ぜんりゃく', 'せんれい', 'せんろ', 'そあく', 'そいとげる', 'そいね', 'そうがんきょう', 'そうき', 'そうご', 'そうしん', 'そうだん', 'そうなん', 'そうび', 'そうめん', 'そうり', 'そえもの', 'そえん', 'そがい', 'そげき', 'そこう', 'そこそこ', 'そざい', 'そしな', 'そせい', 'そせん', 'そそぐ', 'そだてる', 'そつう', 'そつえん', 'そっかん', 'そつぎょう', 'そっけつ', 'そっこう', 'そっせん', 'そっと', 'そとがわ', 'そとづら', 'そなえる', 'そなた', 'そふぼ', 'そぼく', 'そぼろ', 'そまつ', 'そまる', 'そむく', 'そむりえ', 'そめる', 'そもそも', 'そよかぜ', 'そらまめ', 'そろう', 'そんかい', 'そんけい', 'そんざい', 'そんしつ', 'そんぞく', 'そんちょう', 'ぞんび', 'ぞんぶん', 'そんみん', 'たあい', 'たいいん', 'たいうん', 'たいえき', 'たいおう', 'だいがく', 'たいき', 'たいぐう', 'たいけん', 'たいこ', 'たいざい', 'だいじょうぶ', 'だいすき', 'たいせつ', 'たいそう', 'だいたい', 'たいちょう', 'たいてい', 'だいどころ', 'たいない', 'たいねつ', 'たいのう', 'たいはん', 'だいひょう', 'たいふう', 'たいへん', 'たいほ', 'たいまつばな', 'たいみんぐ', 'たいむ', 'たいめん', 'たいやき', 'たいよう', 'たいら', 'たいりょく', 'たいる', 'たいわん', 'たうえ', 'たえる', 'たおす', 'たおる', 'たおれる', 'たかい', 'たかね', 'たきび', 'たくさん', 'たこく', 'たこやき', 'たさい', 'たしざん', 'だじゃれ', 'たすける', 'たずさわる', 'たそがれ', 'たたかう', 'たたく', 'ただしい', 'たたみ', 'たちばな', 'だっかい', 'だっきゃく', 'だっこ', 'だっしゅつ', 'だったい', 'たてる', 'たとえる', 'たなばた', 'たにん', 'たぬき', 'たのしみ', 'たはつ', 'たぶん', 'たべる', 'たぼう', 'たまご', 'たまる', 'だむる', 'ためいき', 'ためす', 'ためる', 'たもつ', 'たやすい', 'たよる', 'たらす', 'たりきほんがん', 'たりょう', 'たりる', 'たると', 'たれる', 'たれんと', 'たろっと', 'たわむれる', 'だんあつ', 'たんい', 'たんおん', 'たんか', 'たんき', 'たんけん', 'たんご', 'たんさん', 'たんじょうび', 'だんせい', 'たんそく', 'たんたい', 'だんち', 'たんてい', 'たんとう', 'だんな', 'たんにん', 'だんねつ', 'たんのう', 'たんぴん', 'だんぼう', 'たんまつ', 'たんめい', 'だんれつ', 'だんろ', 'だんわ', 'ちあい', 'ちあん', 'ちいき', 'ちいさい', 'ちえん', 'ちかい', 'ちから', 'ちきゅう', 'ちきん', 'ちけいず', 'ちけん', 'ちこく', 'ちさい', 'ちしき', 'ちしりょう', 'ちせい', 'ちそう', 'ちたい', 'ちたん', 'ちちおや', 'ちつじょ', 'ちてき', 'ちてん', 'ちぬき', 'ちぬり', 'ちのう', 'ちひょう', 'ちへいせん', 'ちほう', 'ちまた', 'ちみつ', 'ちみどろ', 'ちめいど', 'ちゃんこなべ', 'ちゅうい', 'ちゆりょく', 'ちょうし', 'ちょさくけん', 'ちらし', 'ちらみ', 'ちりがみ', 'ちりょう', 'ちるど', 'ちわわ', 'ちんたい', 'ちんもく', 'ついか', 'ついたち', 'つうか', 'つうじょう', 'つうはん', 'つうわ', 'つかう', 'つかれる', 'つくね', 'つくる', 'つけね', 'つける', 'つごう', 'つたえる', 'つづく', 'つつじ', 'つつむ', 'つとめる', 'つながる', 'つなみ', 'つねづね', 'つのる', 'つぶす', 'つまらない', 'つまる', 'つみき', 'つめたい', 'つもり', 'つもる', 'つよい', 'つるぼ', 'つるみく', 'つわもの', 'つわり', 'てあし', 'てあて', 'てあみ', 'ていおん', 'ていか', 'ていき', 'ていけい', 'ていこく', 'ていさつ', 'ていし', 'ていせい', 'ていたい', 'ていど', 'ていねい', 'ていひょう', 'ていへん', 'ていぼう', 'てうち', 'ておくれ', 'てきとう', 'てくび', 'でこぼこ', 'てさぎょう', 'てさげ', 'てすり', 'てそう', 'てちがい', 'てちょう', 'てつがく', 'てつづき', 'でっぱ', 'てつぼう', 'てつや', 'でぬかえ', 'てぬき', 'てぬぐい', 'てのひら', 'てはい', 'てぶくろ', 'てふだ', 'てほどき', 'てほん', 'てまえ', 'てまきずし', 'てみじか', 'てみやげ', 'てらす', 'てれび', 'てわけ', 'てわたし', 'でんあつ', 'てんいん', 'てんかい', 'てんき', 'てんぐ', 'てんけん', 'てんごく', 'てんさい', 'てんし', 'てんすう', 'でんち', 'てんてき', 'てんとう', 'てんない', 'てんぷら', 'てんぼうだい', 'てんめつ', 'てんらんかい', 'でんりょく', 'でんわ', 'どあい', 'といれ', 'どうかん', 'とうきゅう', 'どうぐ', 'とうし', 'とうむぎ', 'とおい', 'とおか', 'とおく', 'とおす', 'とおる', 'とかい', 'とかす', 'ときおり', 'ときどき', 'とくい', 'とくしゅう', 'とくてん', 'とくに', 'とくべつ', 'とけい', 'とける', 'とこや', 'とさか', 'としょかん', 'とそう', 'とたん', 'とちゅう', 'とっきゅう', 'とっくん', 'とつぜん', 'とつにゅう', 'とどける', 'ととのえる', 'とない', 'となえる', 'となり', 'とのさま', 'とばす', 'どぶがわ', 'とほう', 'とまる', 'とめる', 'ともだち', 'ともる', 'どようび', 'とらえる', 'とんかつ', 'どんぶり', 'ないかく', 'ないこう', 'ないしょ', 'ないす', 'ないせん', 'ないそう', 'なおす', 'ながい', 'なくす', 'なげる', 'なこうど', 'なさけ', 'なたでここ', 'なっとう', 'なつやすみ', 'ななおし', 'なにごと', 'なにもの', 'なにわ', 'なのか', 'なふだ', 'なまいき', 'なまえ', 'なまみ', 'なみだ', 'なめらか', 'なめる', 'なやむ', 'ならう', 'ならび', 'ならぶ', 'なれる', 'なわとび', 'なわばり', 'にあう', 'にいがた', 'にうけ', 'におい', 'にかい', 'にがて', 'にきび', 'にくしみ', 'にくまん', 'にげる', 'にさんかたんそ', 'にしき', 'にせもの', 'にちじょう', 'にちようび', 'にっか', 'にっき', 'にっけい', 'にっこう', 'にっさん', 'にっしょく', 'にっすう', 'にっせき', 'にってい', 'になう', 'にほん', 'にまめ', 'にもつ', 'にやり', 'にゅういん', 'にりんしゃ', 'にわとり', 'にんい', 'にんか', 'にんき', 'にんげん', 'にんしき', 'にんずう', 'にんそう', 'にんたい', 'にんち', 'にんてい', 'にんにく', 'にんぷ', 'にんまり', 'にんむ', 'にんめい', 'にんよう', 'ぬいくぎ', 'ぬかす', 'ぬぐいとる', 'ぬぐう', 'ぬくもり', 'ぬすむ', 'ぬまえび', 'ぬめり', 'ぬらす', 'ぬんちゃく', 'ねあげ', 'ねいき', 'ねいる', 'ねいろ', 'ねぐせ', 'ねくたい', 'ねくら', 'ねこぜ', 'ねこむ', 'ねさげ', 'ねすごす', 'ねそべる', 'ねだん', 'ねつい', 'ねっしん', 'ねつぞう', 'ねったいぎょ', 'ねぶそく', 'ねふだ', 'ねぼう', 'ねほりはほり', 'ねまき', 'ねまわし', 'ねみみ', 'ねむい', 'ねむたい', 'ねもと', 'ねらう', 'ねわざ', 'ねんいり', 'ねんおし', 'ねんかん', 'ねんきん', 'ねんぐ', 'ねんざ', 'ねんし', 'ねんちゃく', 'ねんど', 'ねんぴ', 'ねんぶつ', 'ねんまつ', 'ねんりょう', 'ねんれい', 'のいず', 'のおづま', 'のがす', 'のきなみ', 'のこぎり', 'のこす', 'のこる', 'のせる', 'のぞく', 'のぞむ', 'のたまう', 'のちほど', 'のっく', 'のばす', 'のはら', 'のべる', 'のぼる', 'のみもの', 'のやま', 'のらいぬ', 'のらねこ', 'のりもの', 'のりゆき', 'のれん', 'のんき', 'ばあい', 'はあく', 'ばあさん', 'ばいか', 'ばいく', 'はいけん', 'はいご', 'はいしん', 'はいすい', 'はいせん', 'はいそう', 'はいち', 'ばいばい', 'はいれつ', 'はえる', 'はおる', 'はかい', 'ばかり', 'はかる', 'はくしゅ', 'はけん', 'はこぶ', 'はさみ', 'はさん', 'はしご', 'ばしょ', 'はしる', 'はせる', 'ぱそこん', 'はそん', 'はたん', 'はちみつ', 'はつおん', 'はっかく', 'はづき', 'はっきり', 'はっくつ', 'はっけん', 'はっこう', 'はっさん', 'はっしん', 'はったつ', 'はっちゅう', 'はってん', 'はっぴょう', 'はっぽう', 'はなす', 'はなび', 'はにかむ', 'はぶらし', 'はみがき', 'はむかう', 'はめつ', 'はやい', 'はやし', 'はらう', 'はろうぃん', 'はわい', 'はんい', 'はんえい', 'はんおん', 'はんかく', 'はんきょう', 'ばんぐみ', 'はんこ', 'はんしゃ', 'はんすう', 'はんだん', 'ぱんち', 'ぱんつ', 'はんてい', 'はんとし', 'はんのう', 'はんぱ', 'はんぶん', 'はんぺん', 'はんぼうき', 'はんめい', 'はんらん', 'はんろん', 'ひいき', 'ひうん', 'ひえる', 'ひかく', 'ひかり', 'ひかる', 'ひかん', 'ひくい', 'ひけつ', 'ひこうき', 'ひこく', 'ひさい', 'ひさしぶり', 'ひさん', 'びじゅつかん', 'ひしょ', 'ひそか', 'ひそむ', 'ひたむき', 'ひだり', 'ひたる', 'ひつぎ', 'ひっこし', 'ひっし', 'ひつじゅひん', 'ひっす', 'ひつぜん', 'ぴったり', 'ぴっちり', 'ひつよう', 'ひてい', 'ひとごみ', 'ひなまつり', 'ひなん', 'ひねる', 'ひはん', 'ひびく', 'ひひょう', 'ひほう', 'ひまわり', 'ひまん', 'ひみつ', 'ひめい', 'ひめじし', 'ひやけ', 'ひやす', 'ひよう', 'びょうき', 'ひらがな', 'ひらく', 'ひりつ', 'ひりょう', 'ひるま', 'ひるやすみ', 'ひれい', 'ひろい', 'ひろう', 'ひろき', 'ひろゆき', 'ひんかく', 'ひんけつ', 'ひんこん', 'ひんしゅ', 'ひんそう', 'ぴんち', 'ひんぱん', 'びんぼう', 'ふあん', 'ふいうち', 'ふうけい', 'ふうせん', 'ぷうたろう', 'ふうとう', 'ふうふ', 'ふえる', 'ふおん', 'ふかい', 'ふきん', 'ふくざつ', 'ふくぶくろ', 'ふこう', 'ふさい', 'ふしぎ', 'ふじみ', 'ふすま', 'ふせい', 'ふせぐ', 'ふそく', 'ぶたにく', 'ふたん', 'ふちょう', 'ふつう', 'ふつか', 'ふっかつ', 'ふっき', 'ふっこく', 'ぶどう', 'ふとる', 'ふとん', 'ふのう', 'ふはい', 'ふひょう', 'ふへん', 'ふまん', 'ふみん', 'ふめつ', 'ふめん', 'ふよう', 'ふりこ', 'ふりる', 'ふるい', 'ふんいき', 'ぶんがく', 'ぶんぐ', 'ふんしつ', 'ぶんせき', 'ふんそう', 'ぶんぽう', 'へいあん', 'へいおん', 'へいがい', 'へいき', 'へいげん', 'へいこう', 'へいさ', 'へいしゃ', 'へいせつ', 'へいそ', 'へいたく', 'へいてん', 'へいねつ', 'へいわ', 'へきが', 'へこむ', 'べにいろ', 'べにしょうが', 'へらす', 'へんかん', 'べんきょう', 'べんごし', 'へんさい', 'へんたい', 'べんり', 'ほあん', 'ほいく', 'ぼうぎょ', 'ほうこく', 'ほうそう', 'ほうほう', 'ほうもん', 'ほうりつ', 'ほえる', 'ほおん', 'ほかん', 'ほきょう', 'ぼきん', 'ほくろ', 'ほけつ', 'ほけん', 'ほこう', 'ほこる', 'ほしい', 'ほしつ', 'ほしゅ', 'ほしょう', 'ほせい', 'ほそい', 'ほそく', 'ほたて', 'ほたる', 'ぽちぶくろ', 'ほっきょく', 'ほっさ', 'ほったん', 'ほとんど', 'ほめる', 'ほんい', 'ほんき', 'ほんけ', 'ほんしつ', 'ほんやく', 'まいにち', 'まかい', 'まかせる', 'まがる', 'まける', 'まこと', 'まさつ', 'まじめ', 'ますく', 'まぜる', 'まつり', 'まとめ', 'まなぶ', 'まぬけ', 'まねく', 'まほう', 'まもる', 'まゆげ', 'まよう', 'まろやか', 'まわす', 'まわり', 'まわる', 'まんが', 'まんきつ', 'まんぞく', 'まんなか', 'みいら', 'みうち', 'みえる', 'みがく', 'みかた', 'みかん', 'みけん', 'みこん', 'みじかい', 'みすい', 'みすえる', 'みせる', 'みっか', 'みつかる', 'みつける', 'みてい', 'みとめる', 'みなと', 'みなみかさい', 'みねらる', 'みのう', 'みのがす', 'みほん', 'みもと', 'みやげ', 'みらい', 'みりょく', 'みわく', 'みんか', 'みんぞく', 'むいか', 'むえき', 'むえん', 'むかい', 'むかう', 'むかえ', 'むかし', 'むぎちゃ', 'むける', 'むげん', 'むさぼる', 'むしあつい', 'むしば', 'むじゅん', 'むしろ', 'むすう', 'むすこ', 'むすぶ', 'むすめ', 'むせる', 'むせん', 'むちゅう', 'むなしい', 'むのう', 'むやみ', 'むよう', 'むらさき', 'むりょう', 'むろん', 'めいあん', 'めいうん', 'めいえん', 'めいかく', 'めいきょく', 'めいさい', 'めいし', 'めいそう', 'めいぶつ', 'めいれい', 'めいわく', 'めぐまれる', 'めざす', 'めした', 'めずらしい', 'めだつ', 'めまい', 'めやす', 'めんきょ', 'めんせき', 'めんどう', 'もうしあげる', 'もうどうけん', 'もえる', 'もくし', 'もくてき', 'もくようび', 'もちろん', 'もどる', 'もらう', 'もんく', 'もんだい', 'やおや', 'やける', 'やさい', 'やさしい', 'やすい', 'やすたろう', 'やすみ', 'やせる', 'やそう', 'やたい', 'やちん', 'やっと', 'やっぱり', 'やぶる', 'やめる', 'ややこしい', 'やよい', 'やわらかい', 'ゆうき', 'ゆうびんきょく', 'ゆうべ', 'ゆうめい', 'ゆけつ', 'ゆしゅつ', 'ゆせん', 'ゆそう', 'ゆたか', 'ゆちゃく', 'ゆでる', 'ゆにゅう', 'ゆびわ', 'ゆらい', 'ゆれる', 'ようい', 'ようか', 'ようきゅう', 'ようじ', 'ようす', 'ようちえん', 'よかぜ', 'よかん', 'よきん', 'よくせい', 'よくぼう', 'よけい', 'よごれる', 'よさん', 'よしゅう', 'よそう', 'よそく', 'よっか', 'よてい', 'よどがわく', 'よねつ', 'よやく', 'よゆう', 'よろこぶ', 'よろしい', 'らいう', 'らくがき', 'らくご', 'らくさつ', 'らくだ', 'らしんばん', 'らせん', 'らぞく', 'らたい', 'らっか', 'られつ', 'りえき', 'りかい', 'りきさく', 'りきせつ', 'りくぐん', 'りくつ', 'りけん', 'りこう', 'りせい', 'りそう', 'りそく', 'りてん', 'りねん', 'りゆう', 'りゅうがく', 'りよう', 'りょうり', 'りょかん', 'りょくちゃ', 'りょこう', 'りりく', 'りれき', 'りろん', 'りんご', 'るいけい', 'るいさい', 'るいじ', 'るいせき', 'るすばん', 'るりがわら', 'れいかん', 'れいぎ', 'れいせい', 'れいぞうこ', 'れいとう', 'れいぼう', 'れきし', 'れきだい', 'れんあい', 'れんけい', 'れんこん', 'れんさい', 'れんしゅう', 'れんぞく', 'れんらく', 'ろうか', 'ろうご', 'ろうじん', 'ろうそく', 'ろくが', 'ろこつ', 'ろじうら', 'ろしゅつ', 'ろせん', 'ろてん', 'ろめん', 'ろれつ', 'ろんぎ', 'ろんぱ', 'ろんぶん', 'ろんり', 'わかす', 'わかめ', 'わかやま', 'わかれる', 'わしつ', 'わじまし', 'わすれもの', 'わらう', 'われる' ] ================================================ FILE: lbry/wallet/words/portuguese.py ================================================ words = [ 'abaular', 'abdominal', 'abeto', 'abissinio', 'abjeto', 'ablucao', 'abnegar', 'abotoar', 'abrutalhar', 'absurdo', 'abutre', 'acautelar', 'accessorios', 'acetona', 'achocolatado', 'acirrar', 'acne', 'acovardar', 'acrostico', 'actinomicete', 'acustico', 'adaptavel', 'adeus', 'adivinho', 'adjunto', 'admoestar', 'adnominal', 'adotivo', 'adquirir', 'adriatico', 'adsorcao', 'adutora', 'advogar', 'aerossol', 'afazeres', 'afetuoso', 'afixo', 'afluir', 'afortunar', 'afrouxar', 'aftosa', 'afunilar', 'agentes', 'agito', 'aglutinar', 'aiatola', 'aimore', 'aino', 'aipo', 'airoso', 'ajeitar', 'ajoelhar', 'ajudante', 'ajuste', 'alazao', 'albumina', 'alcunha', 'alegria', 'alexandre', 'alforriar', 'alguns', 'alhures', 'alivio', 'almoxarife', 'alotropico', 'alpiste', 'alquimista', 'alsaciano', 'altura', 'aluviao', 'alvura', 'amazonico', 'ambulatorio', 'ametodico', 'amizades', 'amniotico', 'amovivel', 'amurada', 'anatomico', 'ancorar', 'anexo', 'anfora', 'aniversario', 'anjo', 'anotar', 'ansioso', 'anturio', 'anuviar', 'anverso', 'anzol', 'aonde', 'apaziguar', 'apito', 'aplicavel', 'apoteotico', 'aprimorar', 'aprumo', 'apto', 'apuros', 'aquoso', 'arauto', 'arbusto', 'arduo', 'aresta', 'arfar', 'arguto', 'aritmetico', 'arlequim', 'armisticio', 'aromatizar', 'arpoar', 'arquivo', 'arrumar', 'arsenio', 'arturiano', 'aruaque', 'arvores', 'asbesto', 'ascorbico', 'aspirina', 'asqueroso', 'assustar', 'astuto', 'atazanar', 'ativo', 'atletismo', 'atmosferico', 'atormentar', 'atroz', 'aturdir', 'audivel', 'auferir', 'augusto', 'aula', 'aumento', 'aurora', 'autuar', 'avatar', 'avexar', 'avizinhar', 'avolumar', 'avulso', 'axiomatico', 'azerbaijano', 'azimute', 'azoto', 'azulejo', 'bacteriologista', 'badulaque', 'baforada', 'baixote', 'bajular', 'balzaquiana', 'bambuzal', 'banzo', 'baoba', 'baqueta', 'barulho', 'bastonete', 'batuta', 'bauxita', 'bavaro', 'bazuca', 'bcrepuscular', 'beato', 'beduino', 'begonia', 'behaviorista', 'beisebol', 'belzebu', 'bemol', 'benzido', 'beocio', 'bequer', 'berro', 'besuntar', 'betume', 'bexiga', 'bezerro', 'biatlon', 'biboca', 'bicuspide', 'bidirecional', 'bienio', 'bifurcar', 'bigorna', 'bijuteria', 'bimotor', 'binormal', 'bioxido', 'bipolarizacao', 'biquini', 'birutice', 'bisturi', 'bituca', 'biunivoco', 'bivalve', 'bizarro', 'blasfemo', 'blenorreia', 'blindar', 'bloqueio', 'blusao', 'boazuda', 'bofete', 'bojudo', 'bolso', 'bombordo', 'bonzo', 'botina', 'boquiaberto', 'bostoniano', 'botulismo', 'bourbon', 'bovino', 'boximane', 'bravura', 'brevidade', 'britar', 'broxar', 'bruno', 'bruxuleio', 'bubonico', 'bucolico', 'buda', 'budista', 'bueiro', 'buffer', 'bugre', 'bujao', 'bumerangue', 'burundines', 'busto', 'butique', 'buzios', 'caatinga', 'cabuqui', 'cacunda', 'cafuzo', 'cajueiro', 'camurca', 'canudo', 'caquizeiro', 'carvoeiro', 'casulo', 'catuaba', 'cauterizar', 'cebolinha', 'cedula', 'ceifeiro', 'celulose', 'cerzir', 'cesto', 'cetro', 'ceus', 'cevar', 'chavena', 'cheroqui', 'chita', 'chovido', 'chuvoso', 'ciatico', 'cibernetico', 'cicuta', 'cidreira', 'cientistas', 'cifrar', 'cigarro', 'cilio', 'cimo', 'cinzento', 'cioso', 'cipriota', 'cirurgico', 'cisto', 'citrico', 'ciumento', 'civismo', 'clavicula', 'clero', 'clitoris', 'cluster', 'coaxial', 'cobrir', 'cocota', 'codorniz', 'coexistir', 'cogumelo', 'coito', 'colusao', 'compaixao', 'comutativo', 'contentamento', 'convulsivo', 'coordenativa', 'coquetel', 'correto', 'corvo', 'costureiro', 'cotovia', 'covil', 'cozinheiro', 'cretino', 'cristo', 'crivo', 'crotalo', 'cruzes', 'cubo', 'cucuia', 'cueiro', 'cuidar', 'cujo', 'cultural', 'cunilingua', 'cupula', 'curvo', 'custoso', 'cutucar', 'czarismo', 'dablio', 'dacota', 'dados', 'daguerreotipo', 'daiquiri', 'daltonismo', 'damista', 'dantesco', 'daquilo', 'darwinista', 'dasein', 'dativo', 'deao', 'debutantes', 'decurso', 'deduzir', 'defunto', 'degustar', 'dejeto', 'deltoide', 'demover', 'denunciar', 'deputado', 'deque', 'dervixe', 'desvirtuar', 'deturpar', 'deuteronomio', 'devoto', 'dextrose', 'dezoito', 'diatribe', 'dicotomico', 'didatico', 'dietista', 'difuso', 'digressao', 'diluvio', 'diminuto', 'dinheiro', 'dinossauro', 'dioxido', 'diplomatico', 'dique', 'dirimivel', 'disturbio', 'diurno', 'divulgar', 'dizivel', 'doar', 'dobro', 'docura', 'dodoi', 'doer', 'dogue', 'doloso', 'domo', 'donzela', 'doping', 'dorsal', 'dossie', 'dote', 'doutro', 'doze', 'dravidico', 'dreno', 'driver', 'dropes', 'druso', 'dubnio', 'ducto', 'dueto', 'dulija', 'dundum', 'duodeno', 'duquesa', 'durou', 'duvidoso', 'duzia', 'ebano', 'ebrio', 'eburneo', 'echarpe', 'eclusa', 'ecossistema', 'ectoplasma', 'ecumenismo', 'eczema', 'eden', 'editorial', 'edredom', 'edulcorar', 'efetuar', 'efigie', 'efluvio', 'egiptologo', 'egresso', 'egua', 'einsteiniano', 'eira', 'eivar', 'eixos', 'ejetar', 'elastomero', 'eldorado', 'elixir', 'elmo', 'eloquente', 'elucidativo', 'emaranhar', 'embutir', 'emerito', 'emfa', 'emitir', 'emotivo', 'empuxo', 'emulsao', 'enamorar', 'encurvar', 'enduro', 'enevoar', 'enfurnar', 'enguico', 'enho', 'enigmista', 'enlutar', 'enormidade', 'enpreendimento', 'enquanto', 'enriquecer', 'enrugar', 'entusiastico', 'enunciar', 'envolvimento', 'enxuto', 'enzimatico', 'eolico', 'epiteto', 'epoxi', 'epura', 'equivoco', 'erario', 'erbio', 'ereto', 'erguido', 'erisipela', 'ermo', 'erotizar', 'erros', 'erupcao', 'ervilha', 'esburacar', 'escutar', 'esfuziante', 'esguio', 'esloveno', 'esmurrar', 'esoterismo', 'esperanca', 'espirito', 'espurio', 'essencialmente', 'esturricar', 'esvoacar', 'etario', 'eterno', 'etiquetar', 'etnologo', 'etos', 'etrusco', 'euclidiano', 'euforico', 'eugenico', 'eunuco', 'europio', 'eustaquio', 'eutanasia', 'evasivo', 'eventualidade', 'evitavel', 'evoluir', 'exaustor', 'excursionista', 'exercito', 'exfoliado', 'exito', 'exotico', 'expurgo', 'exsudar', 'extrusora', 'exumar', 'fabuloso', 'facultativo', 'fado', 'fagulha', 'faixas', 'fajuto', 'faltoso', 'famoso', 'fanzine', 'fapesp', 'faquir', 'fartura', 'fastio', 'faturista', 'fausto', 'favorito', 'faxineira', 'fazer', 'fealdade', 'febril', 'fecundo', 'fedorento', 'feerico', 'feixe', 'felicidade', 'felipe', 'feltro', 'femur', 'fenotipo', 'fervura', 'festivo', 'feto', 'feudo', 'fevereiro', 'fezinha', 'fiasco', 'fibra', 'ficticio', 'fiduciario', 'fiesp', 'fifa', 'figurino', 'fijiano', 'filtro', 'finura', 'fiorde', 'fiquei', 'firula', 'fissurar', 'fitoteca', 'fivela', 'fixo', 'flavio', 'flexor', 'flibusteiro', 'flotilha', 'fluxograma', 'fobos', 'foco', 'fofura', 'foguista', 'foie', 'foliculo', 'fominha', 'fonte', 'forum', 'fosso', 'fotossintese', 'foxtrote', 'fraudulento', 'frevo', 'frivolo', 'frouxo', 'frutose', 'fuba', 'fucsia', 'fugitivo', 'fuinha', 'fujao', 'fulustreco', 'fumo', 'funileiro', 'furunculo', 'fustigar', 'futurologo', 'fuxico', 'fuzue', 'gabriel', 'gado', 'gaelico', 'gafieira', 'gaguejo', 'gaivota', 'gajo', 'galvanoplastico', 'gamo', 'ganso', 'garrucha', 'gastronomo', 'gatuno', 'gaussiano', 'gaviao', 'gaxeta', 'gazeteiro', 'gear', 'geiser', 'geminiano', 'generoso', 'genuino', 'geossinclinal', 'gerundio', 'gestual', 'getulista', 'gibi', 'gigolo', 'gilete', 'ginseng', 'giroscopio', 'glaucio', 'glacial', 'gleba', 'glifo', 'glote', 'glutonia', 'gnostico', 'goela', 'gogo', 'goitaca', 'golpista', 'gomo', 'gonzo', 'gorro', 'gostou', 'goticula', 'gourmet', 'governo', 'gozo', 'graxo', 'grevista', 'grito', 'grotesco', 'gruta', 'guaxinim', 'gude', 'gueto', 'guizo', 'guloso', 'gume', 'guru', 'gustativo', 'gustavo', 'gutural', 'habitue', 'haitiano', 'halterofilista', 'hamburguer', 'hanseniase', 'happening', 'harpista', 'hastear', 'haveres', 'hebreu', 'hectometro', 'hedonista', 'hegira', 'helena', 'helminto', 'hemorroidas', 'henrique', 'heptassilabo', 'hertziano', 'hesitar', 'heterossexual', 'heuristico', 'hexagono', 'hiato', 'hibrido', 'hidrostatico', 'hieroglifo', 'hifenizar', 'higienizar', 'hilario', 'himen', 'hino', 'hippie', 'hirsuto', 'historiografia', 'hitlerista', 'hodometro', 'hoje', 'holograma', 'homus', 'honroso', 'hoquei', 'horto', 'hostilizar', 'hotentote', 'huguenote', 'humilde', 'huno', 'hurra', 'hutu', 'iaia', 'ialorixa', 'iambico', 'iansa', 'iaque', 'iara', 'iatista', 'iberico', 'ibis', 'icar', 'iceberg', 'icosagono', 'idade', 'ideologo', 'idiotice', 'idoso', 'iemenita', 'iene', 'igarape', 'iglu', 'ignorar', 'igreja', 'iguaria', 'iidiche', 'ilativo', 'iletrado', 'ilharga', 'ilimitado', 'ilogismo', 'ilustrissimo', 'imaturo', 'imbuzeiro', 'imerso', 'imitavel', 'imovel', 'imputar', 'imutavel', 'inaveriguavel', 'incutir', 'induzir', 'inextricavel', 'infusao', 'ingua', 'inhame', 'iniquo', 'injusto', 'inning', 'inoxidavel', 'inquisitorial', 'insustentavel', 'intumescimento', 'inutilizavel', 'invulneravel', 'inzoneiro', 'iodo', 'iogurte', 'ioio', 'ionosfera', 'ioruba', 'iota', 'ipsilon', 'irascivel', 'iris', 'irlandes', 'irmaos', 'iroques', 'irrupcao', 'isca', 'isento', 'islandes', 'isotopo', 'isqueiro', 'israelita', 'isso', 'isto', 'iterbio', 'itinerario', 'itrio', 'iuane', 'iugoslavo', 'jabuticabeira', 'jacutinga', 'jade', 'jagunco', 'jainista', 'jaleco', 'jambo', 'jantarada', 'japones', 'jaqueta', 'jarro', 'jasmim', 'jato', 'jaula', 'javel', 'jazz', 'jegue', 'jeitoso', 'jejum', 'jenipapo', 'jeova', 'jequitiba', 'jersei', 'jesus', 'jetom', 'jiboia', 'jihad', 'jilo', 'jingle', 'jipe', 'jocoso', 'joelho', 'joguete', 'joio', 'jojoba', 'jorro', 'jota', 'joule', 'joviano', 'jubiloso', 'judoca', 'jugular', 'juizo', 'jujuba', 'juliano', 'jumento', 'junto', 'jururu', 'justo', 'juta', 'juventude', 'labutar', 'laguna', 'laico', 'lajota', 'lanterninha', 'lapso', 'laquear', 'lastro', 'lauto', 'lavrar', 'laxativo', 'lazer', 'leasing', 'lebre', 'lecionar', 'ledo', 'leguminoso', 'leitura', 'lele', 'lemure', 'lento', 'leonardo', 'leopardo', 'lepton', 'leque', 'leste', 'letreiro', 'leucocito', 'levitico', 'lexicologo', 'lhama', 'lhufas', 'liame', 'licoroso', 'lidocaina', 'liliputiano', 'limusine', 'linotipo', 'lipoproteina', 'liquidos', 'lirismo', 'lisura', 'liturgico', 'livros', 'lixo', 'lobulo', 'locutor', 'lodo', 'logro', 'lojista', 'lombriga', 'lontra', 'loop', 'loquaz', 'lorota', 'losango', 'lotus', 'louvor', 'luar', 'lubrificavel', 'lucros', 'lugubre', 'luis', 'luminoso', 'luneta', 'lustroso', 'luto', 'luvas', 'luxuriante', 'luzeiro', 'maduro', 'maestro', 'mafioso', 'magro', 'maiuscula', 'majoritario', 'malvisto', 'mamute', 'manutencao', 'mapoteca', 'maquinista', 'marzipa', 'masturbar', 'matuto', 'mausoleu', 'mavioso', 'maxixe', 'mazurca', 'meandro', 'mecha', 'medusa', 'mefistofelico', 'megera', 'meirinho', 'melro', 'memorizar', 'menu', 'mequetrefe', 'mertiolate', 'mestria', 'metroviario', 'mexilhao', 'mezanino', 'miau', 'microssegundo', 'midia', 'migratorio', 'mimosa', 'minuto', 'miosotis', 'mirtilo', 'misturar', 'mitzvah', 'miudos', 'mixuruca', 'mnemonico', 'moagem', 'mobilizar', 'modulo', 'moer', 'mofo', 'mogno', 'moita', 'molusco', 'monumento', 'moqueca', 'morubixaba', 'mostruario', 'motriz', 'mouse', 'movivel', 'mozarela', 'muarra', 'muculmano', 'mudo', 'mugir', 'muitos', 'mumunha', 'munir', 'muon', 'muquira', 'murros', 'musselina', 'nacoes', 'nado', 'naftalina', 'nago', 'naipe', 'naja', 'nalgum', 'namoro', 'nanquim', 'napolitano', 'naquilo', 'nascimento', 'nautilo', 'navios', 'nazista', 'nebuloso', 'nectarina', 'nefrologo', 'negus', 'nelore', 'nenufar', 'nepotismo', 'nervura', 'neste', 'netuno', 'neutron', 'nevoeiro', 'newtoniano', 'nexo', 'nhenhenhem', 'nhoque', 'nigeriano', 'niilista', 'ninho', 'niobio', 'niponico', 'niquelar', 'nirvana', 'nisto', 'nitroglicerina', 'nivoso', 'nobreza', 'nocivo', 'noel', 'nogueira', 'noivo', 'nojo', 'nominativo', 'nonuplo', 'noruegues', 'nostalgico', 'noturno', 'nouveau', 'nuanca', 'nublar', 'nucleotideo', 'nudista', 'nulo', 'numismatico', 'nunquinha', 'nupcias', 'nutritivo', 'nuvens', 'oasis', 'obcecar', 'obeso', 'obituario', 'objetos', 'oblongo', 'obnoxio', 'obrigatorio', 'obstruir', 'obtuso', 'obus', 'obvio', 'ocaso', 'occipital', 'oceanografo', 'ocioso', 'oclusivo', 'ocorrer', 'ocre', 'octogono', 'odalisca', 'odisseia', 'odorifico', 'oersted', 'oeste', 'ofertar', 'ofidio', 'oftalmologo', 'ogiva', 'ogum', 'oigale', 'oitavo', 'oitocentos', 'ojeriza', 'olaria', 'oleoso', 'olfato', 'olhos', 'oliveira', 'olmo', 'olor', 'olvidavel', 'ombudsman', 'omeleteira', 'omitir', 'omoplata', 'onanismo', 'ondular', 'oneroso', 'onomatopeico', 'ontologico', 'onus', 'onze', 'opalescente', 'opcional', 'operistico', 'opio', 'oposto', 'oprobrio', 'optometrista', 'opusculo', 'oratorio', 'orbital', 'orcar', 'orfao', 'orixa', 'orla', 'ornitologo', 'orquidea', 'ortorrombico', 'orvalho', 'osculo', 'osmotico', 'ossudo', 'ostrogodo', 'otario', 'otite', 'ouro', 'ousar', 'outubro', 'ouvir', 'ovario', 'overnight', 'oviparo', 'ovni', 'ovoviviparo', 'ovulo', 'oxala', 'oxente', 'oxiuro', 'oxossi', 'ozonizar', 'paciente', 'pactuar', 'padronizar', 'paete', 'pagodeiro', 'paixao', 'pajem', 'paludismo', 'pampas', 'panturrilha', 'papudo', 'paquistanes', 'pastoso', 'patua', 'paulo', 'pauzinhos', 'pavoroso', 'paxa', 'pazes', 'peao', 'pecuniario', 'pedunculo', 'pegaso', 'peixinho', 'pejorativo', 'pelvis', 'penuria', 'pequno', 'petunia', 'pezada', 'piauiense', 'pictorico', 'pierro', 'pigmeu', 'pijama', 'pilulas', 'pimpolho', 'pintura', 'piorar', 'pipocar', 'piqueteiro', 'pirulito', 'pistoleiro', 'pituitaria', 'pivotar', 'pixote', 'pizzaria', 'plistoceno', 'plotar', 'pluviometrico', 'pneumonico', 'poco', 'podridao', 'poetisa', 'pogrom', 'pois', 'polvorosa', 'pomposo', 'ponderado', 'pontudo', 'populoso', 'poquer', 'porvir', 'posudo', 'potro', 'pouso', 'povoar', 'prazo', 'prezar', 'privilegios', 'proximo', 'prussiano', 'pseudopode', 'psoriase', 'pterossauros', 'ptialina', 'ptolemaico', 'pudor', 'pueril', 'pufe', 'pugilista', 'puir', 'pujante', 'pulverizar', 'pumba', 'punk', 'purulento', 'pustula', 'putsch', 'puxe', 'quatrocentos', 'quetzal', 'quixotesco', 'quotizavel', 'rabujice', 'racista', 'radonio', 'rafia', 'ragu', 'rajado', 'ralo', 'rampeiro', 'ranzinza', 'raptor', 'raquitismo', 'raro', 'rasurar', 'ratoeira', 'ravioli', 'razoavel', 'reavivar', 'rebuscar', 'recusavel', 'reduzivel', 'reexposicao', 'refutavel', 'regurgitar', 'reivindicavel', 'rejuvenescimento', 'relva', 'remuneravel', 'renunciar', 'reorientar', 'repuxo', 'requisito', 'resumo', 'returno', 'reutilizar', 'revolvido', 'rezonear', 'riacho', 'ribossomo', 'ricota', 'ridiculo', 'rifle', 'rigoroso', 'rijo', 'rimel', 'rins', 'rios', 'riqueza', 'riquixa', 'rissole', 'ritualistico', 'rivalizar', 'rixa', 'robusto', 'rococo', 'rodoviario', 'roer', 'rogo', 'rojao', 'rolo', 'rompimento', 'ronronar', 'roqueiro', 'rorqual', 'rosto', 'rotundo', 'rouxinol', 'roxo', 'royal', 'ruas', 'rucula', 'rudimentos', 'ruela', 'rufo', 'rugoso', 'ruivo', 'rule', 'rumoroso', 'runico', 'ruptura', 'rural', 'rustico', 'rutilar', 'saariano', 'sabujo', 'sacudir', 'sadomasoquista', 'safra', 'sagui', 'sais', 'samurai', 'santuario', 'sapo', 'saquear', 'sartriano', 'saturno', 'saude', 'sauva', 'saveiro', 'saxofonista', 'sazonal', 'scherzo', 'script', 'seara', 'seborreia', 'secura', 'seduzir', 'sefardim', 'seguro', 'seja', 'selvas', 'sempre', 'senzala', 'sepultura', 'sequoia', 'sestercio', 'setuplo', 'seus', 'seviciar', 'sezonismo', 'shalom', 'siames', 'sibilante', 'sicrano', 'sidra', 'sifilitico', 'signos', 'silvo', 'simultaneo', 'sinusite', 'sionista', 'sirio', 'sisudo', 'situar', 'sivan', 'slide', 'slogan', 'soar', 'sobrio', 'socratico', 'sodomizar', 'soerguer', 'software', 'sogro', 'soja', 'solver', 'somente', 'sonso', 'sopro', 'soquete', 'sorveteiro', 'sossego', 'soturno', 'sousafone', 'sovinice', 'sozinho', 'suavizar', 'subverter', 'sucursal', 'sudoriparo', 'sufragio', 'sugestoes', 'suite', 'sujo', 'sultao', 'sumula', 'suntuoso', 'suor', 'supurar', 'suruba', 'susto', 'suturar', 'suvenir', 'tabuleta', 'taco', 'tadjique', 'tafeta', 'tagarelice', 'taitiano', 'talvez', 'tampouco', 'tanzaniano', 'taoista', 'tapume', 'taquion', 'tarugo', 'tascar', 'tatuar', 'tautologico', 'tavola', 'taxionomista', 'tchecoslovaco', 'teatrologo', 'tectonismo', 'tedioso', 'teflon', 'tegumento', 'teixo', 'telurio', 'temporas', 'tenue', 'teosofico', 'tepido', 'tequila', 'terrorista', 'testosterona', 'tetrico', 'teutonico', 'teve', 'texugo', 'tiara', 'tibia', 'tiete', 'tifoide', 'tigresa', 'tijolo', 'tilintar', 'timpano', 'tintureiro', 'tiquete', 'tiroteio', 'tisico', 'titulos', 'tive', 'toar', 'toboga', 'tofu', 'togoles', 'toicinho', 'tolueno', 'tomografo', 'tontura', 'toponimo', 'toquio', 'torvelinho', 'tostar', 'toto', 'touro', 'toxina', 'trazer', 'trezentos', 'trivialidade', 'trovoar', 'truta', 'tuaregue', 'tubular', 'tucano', 'tudo', 'tufo', 'tuiste', 'tulipa', 'tumultuoso', 'tunisino', 'tupiniquim', 'turvo', 'tutu', 'ucraniano', 'udenista', 'ufanista', 'ufologo', 'ugaritico', 'uiste', 'uivo', 'ulceroso', 'ulema', 'ultravioleta', 'umbilical', 'umero', 'umido', 'umlaut', 'unanimidade', 'unesco', 'ungulado', 'unheiro', 'univoco', 'untuoso', 'urano', 'urbano', 'urdir', 'uretra', 'urgente', 'urinol', 'urna', 'urologo', 'urro', 'ursulina', 'urtiga', 'urupe', 'usavel', 'usbeque', 'usei', 'usineiro', 'usurpar', 'utero', 'utilizar', 'utopico', 'uvular', 'uxoricidio', 'vacuo', 'vadio', 'vaguear', 'vaivem', 'valvula', 'vampiro', 'vantajoso', 'vaporoso', 'vaquinha', 'varziano', 'vasto', 'vaticinio', 'vaudeville', 'vazio', 'veado', 'vedico', 'veemente', 'vegetativo', 'veio', 'veja', 'veludo', 'venusiano', 'verdade', 'verve', 'vestuario', 'vetusto', 'vexatorio', 'vezes', 'viavel', 'vibratorio', 'victor', 'vicunha', 'vidros', 'vietnamita', 'vigoroso', 'vilipendiar', 'vime', 'vintem', 'violoncelo', 'viquingue', 'virus', 'visualizar', 'vituperio', 'viuvo', 'vivo', 'vizir', 'voar', 'vociferar', 'vodu', 'vogar', 'voile', 'volver', 'vomito', 'vontade', 'vortice', 'vosso', 'voto', 'vovozinha', 'voyeuse', 'vozes', 'vulva', 'vupt', 'western', 'xadrez', 'xale', 'xampu', 'xango', 'xarope', 'xaual', 'xavante', 'xaxim', 'xenonio', 'xepa', 'xerox', 'xicara', 'xifopago', 'xiita', 'xilogravura', 'xinxim', 'xistoso', 'xixi', 'xodo', 'xogum', 'xucro', 'zabumba', 'zagueiro', 'zambiano', 'zanzar', 'zarpar', 'zebu', 'zefiro', 'zeloso', 'zenite', 'zumbi' ] ================================================ FILE: lbry/wallet/words/spanish.py ================================================ words = [ 'ábaco', 'abdomen', 'abeja', 'abierto', 'abogado', 'abono', 'aborto', 'abrazo', 'abrir', 'abuelo', 'abuso', 'acabar', 'academia', 'acceso', 'acción', 'aceite', 'acelga', 'acento', 'aceptar', 'ácido', 'aclarar', 'acné', 'acoger', 'acoso', 'activo', 'acto', 'actriz', 'actuar', 'acudir', 'acuerdo', 'acusar', 'adicto', 'admitir', 'adoptar', 'adorno', 'aduana', 'adulto', 'aéreo', 'afectar', 'afición', 'afinar', 'afirmar', 'ágil', 'agitar', 'agonía', 'agosto', 'agotar', 'agregar', 'agrio', 'agua', 'agudo', 'águila', 'aguja', 'ahogo', 'ahorro', 'aire', 'aislar', 'ajedrez', 'ajeno', 'ajuste', 'alacrán', 'alambre', 'alarma', 'alba', 'álbum', 'alcalde', 'aldea', 'alegre', 'alejar', 'alerta', 'aleta', 'alfiler', 'alga', 'algodón', 'aliado', 'aliento', 'alivio', 'alma', 'almeja', 'almíbar', 'altar', 'alteza', 'altivo', 'alto', 'altura', 'alumno', 'alzar', 'amable', 'amante', 'amapola', 'amargo', 'amasar', 'ámbar', 'ámbito', 'ameno', 'amigo', 'amistad', 'amor', 'amparo', 'amplio', 'ancho', 'anciano', 'ancla', 'andar', 'andén', 'anemia', 'ángulo', 'anillo', 'ánimo', 'anís', 'anotar', 'antena', 'antiguo', 'antojo', 'anual', 'anular', 'anuncio', 'añadir', 'añejo', 'año', 'apagar', 'aparato', 'apetito', 'apio', 'aplicar', 'apodo', 'aporte', 'apoyo', 'aprender', 'aprobar', 'apuesta', 'apuro', 'arado', 'araña', 'arar', 'árbitro', 'árbol', 'arbusto', 'archivo', 'arco', 'arder', 'ardilla', 'arduo', 'área', 'árido', 'aries', 'armonía', 'arnés', 'aroma', 'arpa', 'arpón', 'arreglo', 'arroz', 'arruga', 'arte', 'artista', 'asa', 'asado', 'asalto', 'ascenso', 'asegurar', 'aseo', 'asesor', 'asiento', 'asilo', 'asistir', 'asno', 'asombro', 'áspero', 'astilla', 'astro', 'astuto', 'asumir', 'asunto', 'atajo', 'ataque', 'atar', 'atento', 'ateo', 'ático', 'atleta', 'átomo', 'atraer', 'atroz', 'atún', 'audaz', 'audio', 'auge', 'aula', 'aumento', 'ausente', 'autor', 'aval', 'avance', 'avaro', 'ave', 'avellana', 'avena', 'avestruz', 'avión', 'aviso', 'ayer', 'ayuda', 'ayuno', 'azafrán', 'azar', 'azote', 'azúcar', 'azufre', 'azul', 'baba', 'babor', 'bache', 'bahía', 'baile', 'bajar', 'balanza', 'balcón', 'balde', 'bambú', 'banco', 'banda', 'baño', 'barba', 'barco', 'barniz', 'barro', 'báscula', 'bastón', 'basura', 'batalla', 'batería', 'batir', 'batuta', 'baúl', 'bazar', 'bebé', 'bebida', 'bello', 'besar', 'beso', 'bestia', 'bicho', 'bien', 'bingo', 'blanco', 'bloque', 'blusa', 'boa', 'bobina', 'bobo', 'boca', 'bocina', 'boda', 'bodega', 'boina', 'bola', 'bolero', 'bolsa', 'bomba', 'bondad', 'bonito', 'bono', 'bonsái', 'borde', 'borrar', 'bosque', 'bote', 'botín', 'bóveda', 'bozal', 'bravo', 'brazo', 'brecha', 'breve', 'brillo', 'brinco', 'brisa', 'broca', 'broma', 'bronce', 'brote', 'bruja', 'brusco', 'bruto', 'buceo', 'bucle', 'bueno', 'buey', 'bufanda', 'bufón', 'búho', 'buitre', 'bulto', 'burbuja', 'burla', 'burro', 'buscar', 'butaca', 'buzón', 'caballo', 'cabeza', 'cabina', 'cabra', 'cacao', 'cadáver', 'cadena', 'caer', 'café', 'caída', 'caimán', 'caja', 'cajón', 'cal', 'calamar', 'calcio', 'caldo', 'calidad', 'calle', 'calma', 'calor', 'calvo', 'cama', 'cambio', 'camello', 'camino', 'campo', 'cáncer', 'candil', 'canela', 'canguro', 'canica', 'canto', 'caña', 'cañón', 'caoba', 'caos', 'capaz', 'capitán', 'capote', 'captar', 'capucha', 'cara', 'carbón', 'cárcel', 'careta', 'carga', 'cariño', 'carne', 'carpeta', 'carro', 'carta', 'casa', 'casco', 'casero', 'caspa', 'castor', 'catorce', 'catre', 'caudal', 'causa', 'cazo', 'cebolla', 'ceder', 'cedro', 'celda', 'célebre', 'celoso', 'célula', 'cemento', 'ceniza', 'centro', 'cerca', 'cerdo', 'cereza', 'cero', 'cerrar', 'certeza', 'césped', 'cetro', 'chacal', 'chaleco', 'champú', 'chancla', 'chapa', 'charla', 'chico', 'chiste', 'chivo', 'choque', 'choza', 'chuleta', 'chupar', 'ciclón', 'ciego', 'cielo', 'cien', 'cierto', 'cifra', 'cigarro', 'cima', 'cinco', 'cine', 'cinta', 'ciprés', 'circo', 'ciruela', 'cisne', 'cita', 'ciudad', 'clamor', 'clan', 'claro', 'clase', 'clave', 'cliente', 'clima', 'clínica', 'cobre', 'cocción', 'cochino', 'cocina', 'coco', 'código', 'codo', 'cofre', 'coger', 'cohete', 'cojín', 'cojo', 'cola', 'colcha', 'colegio', 'colgar', 'colina', 'collar', 'colmo', 'columna', 'combate', 'comer', 'comida', 'cómodo', 'compra', 'conde', 'conejo', 'conga', 'conocer', 'consejo', 'contar', 'copa', 'copia', 'corazón', 'corbata', 'corcho', 'cordón', 'corona', 'correr', 'coser', 'cosmos', 'costa', 'cráneo', 'cráter', 'crear', 'crecer', 'creído', 'crema', 'cría', 'crimen', 'cripta', 'crisis', 'cromo', 'crónica', 'croqueta', 'crudo', 'cruz', 'cuadro', 'cuarto', 'cuatro', 'cubo', 'cubrir', 'cuchara', 'cuello', 'cuento', 'cuerda', 'cuesta', 'cueva', 'cuidar', 'culebra', 'culpa', 'culto', 'cumbre', 'cumplir', 'cuna', 'cuneta', 'cuota', 'cupón', 'cúpula', 'curar', 'curioso', 'curso', 'curva', 'cutis', 'dama', 'danza', 'dar', 'dardo', 'dátil', 'deber', 'débil', 'década', 'decir', 'dedo', 'defensa', 'definir', 'dejar', 'delfín', 'delgado', 'delito', 'demora', 'denso', 'dental', 'deporte', 'derecho', 'derrota', 'desayuno', 'deseo', 'desfile', 'desnudo', 'destino', 'desvío', 'detalle', 'detener', 'deuda', 'día', 'diablo', 'diadema', 'diamante', 'diana', 'diario', 'dibujo', 'dictar', 'diente', 'dieta', 'diez', 'difícil', 'digno', 'dilema', 'diluir', 'dinero', 'directo', 'dirigir', 'disco', 'diseño', 'disfraz', 'diva', 'divino', 'doble', 'doce', 'dolor', 'domingo', 'don', 'donar', 'dorado', 'dormir', 'dorso', 'dos', 'dosis', 'dragón', 'droga', 'ducha', 'duda', 'duelo', 'dueño', 'dulce', 'dúo', 'duque', 'durar', 'dureza', 'duro', 'ébano', 'ebrio', 'echar', 'eco', 'ecuador', 'edad', 'edición', 'edificio', 'editor', 'educar', 'efecto', 'eficaz', 'eje', 'ejemplo', 'elefante', 'elegir', 'elemento', 'elevar', 'elipse', 'élite', 'elixir', 'elogio', 'eludir', 'embudo', 'emitir', 'emoción', 'empate', 'empeño', 'empleo', 'empresa', 'enano', 'encargo', 'enchufe', 'encía', 'enemigo', 'enero', 'enfado', 'enfermo', 'engaño', 'enigma', 'enlace', 'enorme', 'enredo', 'ensayo', 'enseñar', 'entero', 'entrar', 'envase', 'envío', 'época', 'equipo', 'erizo', 'escala', 'escena', 'escolar', 'escribir', 'escudo', 'esencia', 'esfera', 'esfuerzo', 'espada', 'espejo', 'espía', 'esposa', 'espuma', 'esquí', 'estar', 'este', 'estilo', 'estufa', 'etapa', 'eterno', 'ética', 'etnia', 'evadir', 'evaluar', 'evento', 'evitar', 'exacto', 'examen', 'exceso', 'excusa', 'exento', 'exigir', 'exilio', 'existir', 'éxito', 'experto', 'explicar', 'exponer', 'extremo', 'fábrica', 'fábula', 'fachada', 'fácil', 'factor', 'faena', 'faja', 'falda', 'fallo', 'falso', 'faltar', 'fama', 'familia', 'famoso', 'faraón', 'farmacia', 'farol', 'farsa', 'fase', 'fatiga', 'fauna', 'favor', 'fax', 'febrero', 'fecha', 'feliz', 'feo', 'feria', 'feroz', 'fértil', 'fervor', 'festín', 'fiable', 'fianza', 'fiar', 'fibra', 'ficción', 'ficha', 'fideo', 'fiebre', 'fiel', 'fiera', 'fiesta', 'figura', 'fijar', 'fijo', 'fila', 'filete', 'filial', 'filtro', 'fin', 'finca', 'fingir', 'finito', 'firma', 'flaco', 'flauta', 'flecha', 'flor', 'flota', 'fluir', 'flujo', 'flúor', 'fobia', 'foca', 'fogata', 'fogón', 'folio', 'folleto', 'fondo', 'forma', 'forro', 'fortuna', 'forzar', 'fosa', 'foto', 'fracaso', 'frágil', 'franja', 'frase', 'fraude', 'freír', 'freno', 'fresa', 'frío', 'frito', 'fruta', 'fuego', 'fuente', 'fuerza', 'fuga', 'fumar', 'función', 'funda', 'furgón', 'furia', 'fusil', 'fútbol', 'futuro', 'gacela', 'gafas', 'gaita', 'gajo', 'gala', 'galería', 'gallo', 'gamba', 'ganar', 'gancho', 'ganga', 'ganso', 'garaje', 'garza', 'gasolina', 'gastar', 'gato', 'gavilán', 'gemelo', 'gemir', 'gen', 'género', 'genio', 'gente', 'geranio', 'gerente', 'germen', 'gesto', 'gigante', 'gimnasio', 'girar', 'giro', 'glaciar', 'globo', 'gloria', 'gol', 'golfo', 'goloso', 'golpe', 'goma', 'gordo', 'gorila', 'gorra', 'gota', 'goteo', 'gozar', 'grada', 'gráfico', 'grano', 'grasa', 'gratis', 'grave', 'grieta', 'grillo', 'gripe', 'gris', 'grito', 'grosor', 'grúa', 'grueso', 'grumo', 'grupo', 'guante', 'guapo', 'guardia', 'guerra', 'guía', 'guiño', 'guion', 'guiso', 'guitarra', 'gusano', 'gustar', 'haber', 'hábil', 'hablar', 'hacer', 'hacha', 'hada', 'hallar', 'hamaca', 'harina', 'haz', 'hazaña', 'hebilla', 'hebra', 'hecho', 'helado', 'helio', 'hembra', 'herir', 'hermano', 'héroe', 'hervir', 'hielo', 'hierro', 'hígado', 'higiene', 'hijo', 'himno', 'historia', 'hocico', 'hogar', 'hoguera', 'hoja', 'hombre', 'hongo', 'honor', 'honra', 'hora', 'hormiga', 'horno', 'hostil', 'hoyo', 'hueco', 'huelga', 'huerta', 'hueso', 'huevo', 'huida', 'huir', 'humano', 'húmedo', 'humilde', 'humo', 'hundir', 'huracán', 'hurto', 'icono', 'ideal', 'idioma', 'ídolo', 'iglesia', 'iglú', 'igual', 'ilegal', 'ilusión', 'imagen', 'imán', 'imitar', 'impar', 'imperio', 'imponer', 'impulso', 'incapaz', 'índice', 'inerte', 'infiel', 'informe', 'ingenio', 'inicio', 'inmenso', 'inmune', 'innato', 'insecto', 'instante', 'interés', 'íntimo', 'intuir', 'inútil', 'invierno', 'ira', 'iris', 'ironía', 'isla', 'islote', 'jabalí', 'jabón', 'jamón', 'jarabe', 'jardín', 'jarra', 'jaula', 'jazmín', 'jefe', 'jeringa', 'jinete', 'jornada', 'joroba', 'joven', 'joya', 'juerga', 'jueves', 'juez', 'jugador', 'jugo', 'juguete', 'juicio', 'junco', 'jungla', 'junio', 'juntar', 'júpiter', 'jurar', 'justo', 'juvenil', 'juzgar', 'kilo', 'koala', 'labio', 'lacio', 'lacra', 'lado', 'ladrón', 'lagarto', 'lágrima', 'laguna', 'laico', 'lamer', 'lámina', 'lámpara', 'lana', 'lancha', 'langosta', 'lanza', 'lápiz', 'largo', 'larva', 'lástima', 'lata', 'látex', 'latir', 'laurel', 'lavar', 'lazo', 'leal', 'lección', 'leche', 'lector', 'leer', 'legión', 'legumbre', 'lejano', 'lengua', 'lento', 'leña', 'león', 'leopardo', 'lesión', 'letal', 'letra', 'leve', 'leyenda', 'libertad', 'libro', 'licor', 'líder', 'lidiar', 'lienzo', 'liga', 'ligero', 'lima', 'límite', 'limón', 'limpio', 'lince', 'lindo', 'línea', 'lingote', 'lino', 'linterna', 'líquido', 'liso', 'lista', 'litera', 'litio', 'litro', 'llaga', 'llama', 'llanto', 'llave', 'llegar', 'llenar', 'llevar', 'llorar', 'llover', 'lluvia', 'lobo', 'loción', 'loco', 'locura', 'lógica', 'logro', 'lombriz', 'lomo', 'lonja', 'lote', 'lucha', 'lucir', 'lugar', 'lujo', 'luna', 'lunes', 'lupa', 'lustro', 'luto', 'luz', 'maceta', 'macho', 'madera', 'madre', 'maduro', 'maestro', 'mafia', 'magia', 'mago', 'maíz', 'maldad', 'maleta', 'malla', 'malo', 'mamá', 'mambo', 'mamut', 'manco', 'mando', 'manejar', 'manga', 'maniquí', 'manjar', 'mano', 'manso', 'manta', 'mañana', 'mapa', 'máquina', 'mar', 'marco', 'marea', 'marfil', 'margen', 'marido', 'mármol', 'marrón', 'martes', 'marzo', 'masa', 'máscara', 'masivo', 'matar', 'materia', 'matiz', 'matriz', 'máximo', 'mayor', 'mazorca', 'mecha', 'medalla', 'medio', 'médula', 'mejilla', 'mejor', 'melena', 'melón', 'memoria', 'menor', 'mensaje', 'mente', 'menú', 'mercado', 'merengue', 'mérito', 'mes', 'mesón', 'meta', 'meter', 'método', 'metro', 'mezcla', 'miedo', 'miel', 'miembro', 'miga', 'mil', 'milagro', 'militar', 'millón', 'mimo', 'mina', 'minero', 'mínimo', 'minuto', 'miope', 'mirar', 'misa', 'miseria', 'misil', 'mismo', 'mitad', 'mito', 'mochila', 'moción', 'moda', 'modelo', 'moho', 'mojar', 'molde', 'moler', 'molino', 'momento', 'momia', 'monarca', 'moneda', 'monja', 'monto', 'moño', 'morada', 'morder', 'moreno', 'morir', 'morro', 'morsa', 'mortal', 'mosca', 'mostrar', 'motivo', 'mover', 'móvil', 'mozo', 'mucho', 'mudar', 'mueble', 'muela', 'muerte', 'muestra', 'mugre', 'mujer', 'mula', 'muleta', 'multa', 'mundo', 'muñeca', 'mural', 'muro', 'músculo', 'museo', 'musgo', 'música', 'muslo', 'nácar', 'nación', 'nadar', 'naipe', 'naranja', 'nariz', 'narrar', 'nasal', 'natal', 'nativo', 'natural', 'náusea', 'naval', 'nave', 'navidad', 'necio', 'néctar', 'negar', 'negocio', 'negro', 'neón', 'nervio', 'neto', 'neutro', 'nevar', 'nevera', 'nicho', 'nido', 'niebla', 'nieto', 'niñez', 'niño', 'nítido', 'nivel', 'nobleza', 'noche', 'nómina', 'noria', 'norma', 'norte', 'nota', 'noticia', 'novato', 'novela', 'novio', 'nube', 'nuca', 'núcleo', 'nudillo', 'nudo', 'nuera', 'nueve', 'nuez', 'nulo', 'número', 'nutria', 'oasis', 'obeso', 'obispo', 'objeto', 'obra', 'obrero', 'observar', 'obtener', 'obvio', 'oca', 'ocaso', 'océano', 'ochenta', 'ocho', 'ocio', 'ocre', 'octavo', 'octubre', 'oculto', 'ocupar', 'ocurrir', 'odiar', 'odio', 'odisea', 'oeste', 'ofensa', 'oferta', 'oficio', 'ofrecer', 'ogro', 'oído', 'oír', 'ojo', 'ola', 'oleada', 'olfato', 'olivo', 'olla', 'olmo', 'olor', 'olvido', 'ombligo', 'onda', 'onza', 'opaco', 'opción', 'ópera', 'opinar', 'oponer', 'optar', 'óptica', 'opuesto', 'oración', 'orador', 'oral', 'órbita', 'orca', 'orden', 'oreja', 'órgano', 'orgía', 'orgullo', 'oriente', 'origen', 'orilla', 'oro', 'orquesta', 'oruga', 'osadía', 'oscuro', 'osezno', 'oso', 'ostra', 'otoño', 'otro', 'oveja', 'óvulo', 'óxido', 'oxígeno', 'oyente', 'ozono', 'pacto', 'padre', 'paella', 'página', 'pago', 'país', 'pájaro', 'palabra', 'palco', 'paleta', 'pálido', 'palma', 'paloma', 'palpar', 'pan', 'panal', 'pánico', 'pantera', 'pañuelo', 'papá', 'papel', 'papilla', 'paquete', 'parar', 'parcela', 'pared', 'parir', 'paro', 'párpado', 'parque', 'párrafo', 'parte', 'pasar', 'paseo', 'pasión', 'paso', 'pasta', 'pata', 'patio', 'patria', 'pausa', 'pauta', 'pavo', 'payaso', 'peatón', 'pecado', 'pecera', 'pecho', 'pedal', 'pedir', 'pegar', 'peine', 'pelar', 'peldaño', 'pelea', 'peligro', 'pellejo', 'pelo', 'peluca', 'pena', 'pensar', 'peñón', 'peón', 'peor', 'pepino', 'pequeño', 'pera', 'percha', 'perder', 'pereza', 'perfil', 'perico', 'perla', 'permiso', 'perro', 'persona', 'pesa', 'pesca', 'pésimo', 'pestaña', 'pétalo', 'petróleo', 'pez', 'pezuña', 'picar', 'pichón', 'pie', 'piedra', 'pierna', 'pieza', 'pijama', 'pilar', 'piloto', 'pimienta', 'pino', 'pintor', 'pinza', 'piña', 'piojo', 'pipa', 'pirata', 'pisar', 'piscina', 'piso', 'pista', 'pitón', 'pizca', 'placa', 'plan', 'plata', 'playa', 'plaza', 'pleito', 'pleno', 'plomo', 'pluma', 'plural', 'pobre', 'poco', 'poder', 'podio', 'poema', 'poesía', 'poeta', 'polen', 'policía', 'pollo', 'polvo', 'pomada', 'pomelo', 'pomo', 'pompa', 'poner', 'porción', 'portal', 'posada', 'poseer', 'posible', 'poste', 'potencia', 'potro', 'pozo', 'prado', 'precoz', 'pregunta', 'premio', 'prensa', 'preso', 'previo', 'primo', 'príncipe', 'prisión', 'privar', 'proa', 'probar', 'proceso', 'producto', 'proeza', 'profesor', 'programa', 'prole', 'promesa', 'pronto', 'propio', 'próximo', 'prueba', 'público', 'puchero', 'pudor', 'pueblo', 'puerta', 'puesto', 'pulga', 'pulir', 'pulmón', 'pulpo', 'pulso', 'puma', 'punto', 'puñal', 'puño', 'pupa', 'pupila', 'puré', 'quedar', 'queja', 'quemar', 'querer', 'queso', 'quieto', 'química', 'quince', 'quitar', 'rábano', 'rabia', 'rabo', 'ración', 'radical', 'raíz', 'rama', 'rampa', 'rancho', 'rango', 'rapaz', 'rápido', 'rapto', 'rasgo', 'raspa', 'rato', 'rayo', 'raza', 'razón', 'reacción', 'realidad', 'rebaño', 'rebote', 'recaer', 'receta', 'rechazo', 'recoger', 'recreo', 'recto', 'recurso', 'red', 'redondo', 'reducir', 'reflejo', 'reforma', 'refrán', 'refugio', 'regalo', 'regir', 'regla', 'regreso', 'rehén', 'reino', 'reír', 'reja', 'relato', 'relevo', 'relieve', 'relleno', 'reloj', 'remar', 'remedio', 'remo', 'rencor', 'rendir', 'renta', 'reparto', 'repetir', 'reposo', 'reptil', 'res', 'rescate', 'resina', 'respeto', 'resto', 'resumen', 'retiro', 'retorno', 'retrato', 'reunir', 'revés', 'revista', 'rey', 'rezar', 'rico', 'riego', 'rienda', 'riesgo', 'rifa', 'rígido', 'rigor', 'rincón', 'riñón', 'río', 'riqueza', 'risa', 'ritmo', 'rito', 'rizo', 'roble', 'roce', 'rociar', 'rodar', 'rodeo', 'rodilla', 'roer', 'rojizo', 'rojo', 'romero', 'romper', 'ron', 'ronco', 'ronda', 'ropa', 'ropero', 'rosa', 'rosca', 'rostro', 'rotar', 'rubí', 'rubor', 'rudo', 'rueda', 'rugir', 'ruido', 'ruina', 'ruleta', 'rulo', 'rumbo', 'rumor', 'ruptura', 'ruta', 'rutina', 'sábado', 'saber', 'sabio', 'sable', 'sacar', 'sagaz', 'sagrado', 'sala', 'saldo', 'salero', 'salir', 'salmón', 'salón', 'salsa', 'salto', 'salud', 'salvar', 'samba', 'sanción', 'sandía', 'sanear', 'sangre', 'sanidad', 'sano', 'santo', 'sapo', 'saque', 'sardina', 'sartén', 'sastre', 'satán', 'sauna', 'saxofón', 'sección', 'seco', 'secreto', 'secta', 'sed', 'seguir', 'seis', 'sello', 'selva', 'semana', 'semilla', 'senda', 'sensor', 'señal', 'señor', 'separar', 'sepia', 'sequía', 'ser', 'serie', 'sermón', 'servir', 'sesenta', 'sesión', 'seta', 'setenta', 'severo', 'sexo', 'sexto', 'sidra', 'siesta', 'siete', 'siglo', 'signo', 'sílaba', 'silbar', 'silencio', 'silla', 'símbolo', 'simio', 'sirena', 'sistema', 'sitio', 'situar', 'sobre', 'socio', 'sodio', 'sol', 'solapa', 'soldado', 'soledad', 'sólido', 'soltar', 'solución', 'sombra', 'sondeo', 'sonido', 'sonoro', 'sonrisa', 'sopa', 'soplar', 'soporte', 'sordo', 'sorpresa', 'sorteo', 'sostén', 'sótano', 'suave', 'subir', 'suceso', 'sudor', 'suegra', 'suelo', 'sueño', 'suerte', 'sufrir', 'sujeto', 'sultán', 'sumar', 'superar', 'suplir', 'suponer', 'supremo', 'sur', 'surco', 'sureño', 'surgir', 'susto', 'sutil', 'tabaco', 'tabique', 'tabla', 'tabú', 'taco', 'tacto', 'tajo', 'talar', 'talco', 'talento', 'talla', 'talón', 'tamaño', 'tambor', 'tango', 'tanque', 'tapa', 'tapete', 'tapia', 'tapón', 'taquilla', 'tarde', 'tarea', 'tarifa', 'tarjeta', 'tarot', 'tarro', 'tarta', 'tatuaje', 'tauro', 'taza', 'tazón', 'teatro', 'techo', 'tecla', 'técnica', 'tejado', 'tejer', 'tejido', 'tela', 'teléfono', 'tema', 'temor', 'templo', 'tenaz', 'tender', 'tener', 'tenis', 'tenso', 'teoría', 'terapia', 'terco', 'término', 'ternura', 'terror', 'tesis', 'tesoro', 'testigo', 'tetera', 'texto', 'tez', 'tibio', 'tiburón', 'tiempo', 'tienda', 'tierra', 'tieso', 'tigre', 'tijera', 'tilde', 'timbre', 'tímido', 'timo', 'tinta', 'tío', 'típico', 'tipo', 'tira', 'tirón', 'titán', 'títere', 'título', 'tiza', 'toalla', 'tobillo', 'tocar', 'tocino', 'todo', 'toga', 'toldo', 'tomar', 'tono', 'tonto', 'topar', 'tope', 'toque', 'tórax', 'torero', 'tormenta', 'torneo', 'toro', 'torpedo', 'torre', 'torso', 'tortuga', 'tos', 'tosco', 'toser', 'tóxico', 'trabajo', 'tractor', 'traer', 'tráfico', 'trago', 'traje', 'tramo', 'trance', 'trato', 'trauma', 'trazar', 'trébol', 'tregua', 'treinta', 'tren', 'trepar', 'tres', 'tribu', 'trigo', 'tripa', 'triste', 'triunfo', 'trofeo', 'trompa', 'tronco', 'tropa', 'trote', 'trozo', 'truco', 'trueno', 'trufa', 'tubería', 'tubo', 'tuerto', 'tumba', 'tumor', 'túnel', 'túnica', 'turbina', 'turismo', 'turno', 'tutor', 'ubicar', 'úlcera', 'umbral', 'unidad', 'unir', 'universo', 'uno', 'untar', 'uña', 'urbano', 'urbe', 'urgente', 'urna', 'usar', 'usuario', 'útil', 'utopía', 'uva', 'vaca', 'vacío', 'vacuna', 'vagar', 'vago', 'vaina', 'vajilla', 'vale', 'válido', 'valle', 'valor', 'válvula', 'vampiro', 'vara', 'variar', 'varón', 'vaso', 'vecino', 'vector', 'vehículo', 'veinte', 'vejez', 'vela', 'velero', 'veloz', 'vena', 'vencer', 'venda', 'veneno', 'vengar', 'venir', 'venta', 'venus', 'ver', 'verano', 'verbo', 'verde', 'vereda', 'verja', 'verso', 'verter', 'vía', 'viaje', 'vibrar', 'vicio', 'víctima', 'vida', 'vídeo', 'vidrio', 'viejo', 'viernes', 'vigor', 'vil', 'villa', 'vinagre', 'vino', 'viñedo', 'violín', 'viral', 'virgo', 'virtud', 'visor', 'víspera', 'vista', 'vitamina', 'viudo', 'vivaz', 'vivero', 'vivir', 'vivo', 'volcán', 'volumen', 'volver', 'voraz', 'votar', 'voto', 'voz', 'vuelo', 'vulgar', 'yacer', 'yate', 'yegua', 'yema', 'yerno', 'yeso', 'yodo', 'yoga', 'yogur', 'zafiro', 'zanja', 'zapato', 'zarza', 'zona', 'zorro', 'zumo', 'zurdo' ] ================================================ FILE: lbry/winpaths.py ================================================ # Copyright (c) 2014 Michael Kropat import sys import ctypes from ctypes import windll, wintypes from uuid import UUID # http://msdn.microsoft.com/en-us/library/windows/desktop/aa373931.aspx class GUID(ctypes.Structure): _fields_ = [ ("Data1", wintypes.DWORD), ("Data2", wintypes.WORD), ("Data3", wintypes.WORD), ("Data4", wintypes.BYTE * 8) ] def __init__(self, uuid_): super().__init__() self.Data1, self.Data2, self.Data3, self.Data4[0], self.Data4[1], rest = uuid_.fields for i in range(2, 8): self.Data4[i] = rest>>(8 - i - 1)*8 & 0xff # http://msdn.microsoft.com/en-us/library/windows/desktop/dd378457.aspx class FOLDERID: # pylint: disable=bad-whitespace AccountPictures = UUID('{008ca0b1-55b4-4c56-b8a8-4de4b299d3be}') AdminTools = UUID('{724EF170-A42D-4FEF-9F26-B60E846FBA4F}') ApplicationShortcuts = UUID('{A3918781-E5F2-4890-B3D9-A7E54332328C}') CameraRoll = UUID('{AB5FB87B-7CE2-4F83-915D-550846C9537B}') CDBurning = UUID('{9E52AB10-F80D-49DF-ACB8-4330F5687855}') CommonAdminTools = UUID('{D0384E7D-BAC3-4797-8F14-CBA229B392B5}') CommonOEMLinks = UUID('{C1BAE2D0-10DF-4334-BEDD-7AA20B227A9D}') CommonPrograms = UUID('{0139D44E-6AFE-49F2-8690-3DAFCAE6FFB8}') CommonStartMenu = UUID('{A4115719-D62E-491D-AA7C-E74B8BE3B067}') CommonStartup = UUID('{82A5EA35-D9CD-47C5-9629-E15D2F714E6E}') CommonTemplates = UUID('{B94237E7-57AC-4347-9151-B08C6C32D1F7}') Contacts = UUID('{56784854-C6CB-462b-8169-88E350ACB882}') Cookies = UUID('{2B0F765D-C0E9-4171-908E-08A611B84FF6}') Desktop = UUID('{B4BFCC3A-DB2C-424C-B029-7FE99A87C641}') DeviceMetadataStore = UUID('{5CE4A5E9-E4EB-479D-B89F-130C02886155}') Documents = UUID('{FDD39AD0-238F-46AF-ADB4-6C85480369C7}') DocumentsLibrary = UUID('{7B0DB17D-9CD2-4A93-9733-46CC89022E7C}') Downloads = UUID('{374DE290-123F-4565-9164-39C4925E467B}') Favorites = UUID('{1777F761-68AD-4D8A-87BD-30B759FA33DD}') Fonts = UUID('{FD228CB7-AE11-4AE3-864C-16F3910AB8FE}') GameTasks = UUID('{054FAE61-4DD8-4787-80B6-090220C4B700}') History = UUID('{D9DC8A3B-B784-432E-A781-5A1130A75963}') ImplicitAppShortcuts = UUID('{BCB5256F-79F6-4CEE-B725-DC34E402FD46}') InternetCache = UUID('{352481E8-33BE-4251-BA85-6007CAEDCF9D}') Libraries = UUID('{1B3EA5DC-B587-4786-B4EF-BD1DC332AEAE}') Links = UUID('{bfb9d5e0-c6a9-404c-b2b2-ae6db6af4968}') LocalAppData = UUID('{F1B32785-6FBA-4FCF-9D55-7B8E7F157091}') LocalAppDataLow = UUID('{A520A1A4-1780-4FF6-BD18-167343C5AF16}') LocalizedResourcesDir = UUID('{2A00375E-224C-49DE-B8D1-440DF7EF3DDC}') Music = UUID('{4BD8D571-6D19-48D3-BE97-422220080E43}') MusicLibrary = UUID('{2112AB0A-C86A-4FFE-A368-0DE96E47012E}') NetHood = UUID('{C5ABBF53-E17F-4121-8900-86626FC2C973}') OriginalImages = UUID('{2C36C0AA-5812-4b87-BFD0-4CD0DFB19B39}') PhotoAlbums = UUID('{69D2CF90-FC33-4FB7-9A0C-EBB0F0FCB43C}') PicturesLibrary = UUID('{A990AE9F-A03B-4E80-94BC-9912D7504104}') Pictures = UUID('{33E28130-4E1E-4676-835A-98395C3BC3BB}') Playlists = UUID('{DE92C1C7-837F-4F69-A3BB-86E631204A23}') PrintHood = UUID('{9274BD8D-CFD1-41C3-B35E-B13F55A758F4}') Profile = UUID('{5E6C858F-0E22-4760-9AFE-EA3317B67173}') ProgramData = UUID('{62AB5D82-FDC1-4DC3-A9DD-070D1D495D97}') ProgramFiles = UUID('{905e63b6-c1bf-494e-b29c-65b732d3d21a}') ProgramFilesX64 = UUID('{6D809377-6AF0-444b-8957-A3773F02200E}') ProgramFilesX86 = UUID('{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}') ProgramFilesCommon = UUID('{F7F1ED05-9F6D-47A2-AAAE-29D317C6F066}') ProgramFilesCommonX64 = UUID('{6365D5A7-0F0D-45E5-87F6-0DA56B6A4F7D}') ProgramFilesCommonX86 = UUID('{DE974D24-D9C6-4D3E-BF91-F4455120B917}') Programs = UUID('{A77F5D77-2E2B-44C3-A6A2-ABA601054A51}') Public = UUID('{DFDF76A2-C82A-4D63-906A-5644AC457385}') PublicDesktop = UUID('{C4AA340D-F20F-4863-AFEF-F87EF2E6BA25}') PublicDocuments = UUID('{ED4824AF-DCE4-45A8-81E2-FC7965083634}') PublicDownloads = UUID('{3D644C9B-1FB8-4f30-9B45-F670235F79C0}') PublicGameTasks = UUID('{DEBF2536-E1A8-4c59-B6A2-414586476AEA}') PublicLibraries = UUID('{48DAF80B-E6CF-4F4E-B800-0E69D84EE384}') PublicMusic = UUID('{3214FAB5-9757-4298-BB61-92A9DEAA44FF}') PublicPictures = UUID('{B6EBFB86-6907-413C-9AF7-4FC2ABF07CC5}') PublicRingtones = UUID('{E555AB60-153B-4D17-9F04-A5FE99FC15EC}') PublicUserTiles = UUID('{0482af6c-08f1-4c34-8c90-e17ec98b1e17}') PublicVideos = UUID('{2400183A-6185-49FB-A2D8-4A392A602BA3}') QuickLaunch = UUID('{52a4f021-7b75-48a9-9f6b-4b87a210bc8f}') Recent = UUID('{AE50C081-EBD2-438A-8655-8A092E34987A}') RecordedTVLibrary = UUID('{1A6FDBA2-F42D-4358-A798-B74D745926C5}') ResourceDir = UUID('{8AD10C31-2ADB-4296-A8F7-E4701232C972}') Ringtones = UUID('{C870044B-F49E-4126-A9C3-B52A1FF411E8}') RoamingAppData = UUID('{3EB685DB-65F9-4CF6-A03A-E3EF65729F3D}') RoamedTileImages = UUID('{AAA8D5A5-F1D6-4259-BAA8-78E7EF60835E}') RoamingTiles = UUID('{00BCFC5A-ED94-4e48-96A1-3F6217F21990}') SampleMusic = UUID('{B250C668-F57D-4EE1-A63C-290EE7D1AA1F}') SamplePictures = UUID('{C4900540-2379-4C75-844B-64E6FAF8716B}') SamplePlaylists = UUID('{15CA69B3-30EE-49C1-ACE1-6B5EC372AFB5}') SampleVideos = UUID('{859EAD94-2E85-48AD-A71A-0969CB56A6CD}') SavedGames = UUID('{4C5C32FF-BB9D-43b0-B5B4-2D72E54EAAA4}') SavedSearches = UUID('{7d1d3a04-debb-4115-95cf-2f29da2920da}') Screenshots = UUID('{b7bede81-df94-4682-a7d8-57a52620b86f}') SearchHistory = UUID('{0D4C3DB6-03A3-462F-A0E6-08924C41B5D4}') SearchTemplates = UUID('{7E636BFE-DFA9-4D5E-B456-D7B39851D8A9}') SendTo = UUID('{8983036C-27C0-404B-8F08-102D10DCFD74}') SidebarDefaultParts = UUID('{7B396E54-9EC5-4300-BE0A-2482EBAE1A26}') SidebarParts = UUID('{A75D362E-50FC-4fb7-AC2C-A8BEAA314493}') SkyDrive = UUID('{A52BBA46-E9E1-435f-B3D9-28DAA648C0F6}') SkyDriveCameraRoll = UUID('{767E6811-49CB-4273-87C2-20F355E1085B}') SkyDriveDocuments = UUID('{24D89E24-2F19-4534-9DDE-6A6671FBB8FE}') SkyDrivePictures = UUID('{339719B5-8C47-4894-94C2-D8F77ADD44A6}') StartMenu = UUID('{625B53C3-AB48-4EC1-BA1F-A1EF4146FC19}') Startup = UUID('{B97D20BB-F46A-4C97-BA10-5E3608430854}') System = UUID('{1AC14E77-02E7-4E5D-B744-2EB1AE5198B7}') SystemX86 = UUID('{D65231B0-B2F1-4857-A4CE-A8E7C6EA7D27}') Templates = UUID('{A63293E8-664E-48DB-A079-DF759E0509F7}') UserPinned = UUID('{9E3995AB-1F9C-4F13-B827-48B24B6C7174}') UserProfiles = UUID('{0762D272-C50A-4BB0-A382-697DCD729B80}') UserProgramFiles = UUID('{5CD7AEE2-2219-4A67-B85D-6C9CE15660CB}') UserProgramFilesCommon = UUID('{BCBD3057-CA5C-4622-B42D-BC56DB0AE516}') Videos = UUID('{18989B1D-99B5-455B-841C-AB7C74E4DDFC}') VideosLibrary = UUID('{491E922F-5643-4AF4-A7EB-4E7A138D8174}') Windows = UUID('{F38BF404-1D43-42F2-9305-67DE0B28FC23}') # http://msdn.microsoft.com/en-us/library/windows/desktop/bb762188.aspx class UserHandle: current = wintypes.HANDLE(0) common = wintypes.HANDLE(-1) # http://msdn.microsoft.com/en-us/library/windows/desktop/ms680722.aspx _CoTaskMemFree = windll.ole32.CoTaskMemFree _CoTaskMemFree.restype = None _CoTaskMemFree.argtypes = [ctypes.c_void_p] # http://msdn.microsoft.com/en-us/library/windows/desktop/bb762188.aspx # http://www.themacaque.com/?p=954 _SHGetKnownFolderPath = windll.shell32.SHGetKnownFolderPath _SHGetKnownFolderPath.argtypes = [ ctypes.POINTER(GUID), wintypes.DWORD, wintypes.HANDLE, ctypes.POINTER(ctypes.c_wchar_p) ] class PathNotFoundException(Exception): pass def get_path(folderid, user_handle=UserHandle.common): fid = GUID(folderid) pPath = ctypes.c_wchar_p() S_OK = 0 if _SHGetKnownFolderPath(ctypes.byref(fid), 0, user_handle, ctypes.byref(pPath)) != S_OK: raise PathNotFoundException() path = pPath.value _CoTaskMemFree(pPath) return path if __name__ == '__main__': if len(sys.argv) < 2 or sys.argv[1] in ['-?', '/?']: print('python winpaths.py FOLDERID {current|common}') sys.exit(0) try: folderid = getattr(FOLDERID, sys.argv[1]) except AttributeError: print('Unknown folder id "%s"' % sys.argv[1], file=sys.stderr) sys.exit(1) try: if len(sys.argv) == 2: print(get_path(folderid)) else: print(get_path(folderid, getattr(UserHandle, sys.argv[2]))) except PathNotFoundException: print('Folder not found "%s"' % ' '.join(sys.argv[1:]), file=sys.stderr) sys.exit(1) ================================================ FILE: scripts/Dockerfile.lbry_orchstr8 ================================================ FROM debian:buster-slim ARG TORBA_VERSION=master RUN apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ build-essential \ git \ python3.7 \ python3.7-dev \ python3-pip && \ rm -rf /var/lib/apt/lists/* RUN python3.7 -m pip install --upgrade pip setuptools wheel COPY . /app WORKDIR /app RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba RUN python3.7 -m pip install -e . # Orchstr8 API EXPOSE 7954 # Wallet Server EXPOSE 5280 # SPV Server EXPOSE 50002 # blockchain EXPOSE 9246 ENV TORBA_LEDGER lbry.wallet RUN /usr/local/bin/orchstr8 download ENTRYPOINT ["/usr/local/bin/orchstr8"] ================================================ FILE: scripts/check_signature.py ================================================ import argparse import sqlite3 from binascii import hexlify from lbry.wallet.transaction import Output def check(db_path, claim_id): db = sqlite3.connect(db_path) db.row_factory = sqlite3.Row claim = db.execute('select * from claim where claim_id=?', (claim_id,)).fetchone() if not claim: print('Could not find claim.') return channel = db.execute('select * from claim where claim_hash=?', (claim['channel_hash'],)).fetchone() if not channel: print('Could not find channel for this claim.') print(f"Claim: {claim['claim_name']}") print(f"Channel: {channel['claim_name']}") print(f"Signature: {hexlify(claim['signature']).decode()}") print(f"Digest: {hexlify(claim['signature_digest']).decode()}") print(f"Pubkey: {hexlify(channel['public_key_bytes']).decode()}") print("Valid: {}".format(Output.is_signature_valid( claim['signature'], claim['signature_digest'], channel['public_key_bytes'] ))) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('db_path') parser.add_argument('claim_id') args = parser.parse_args() check(args.db_path, args.claim_id) ================================================ FILE: scripts/check_video.py ================================================ #!/usr/bin/env python3 import asyncio import logging import platform import sys # noinspection PyUnresolvedReferences import lbry.wallet # needed to make the following line work (it's a bug): from lbry.conf import TranscodeConfig from lbry.file_analysis import VideoFileAnalyzer def enable_logging(): root = logging.getLogger() root.setLevel(logging.DEBUG) handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) root.addHandler(handler) async def process_video(analyzer, video_file): try: await analyzer.verify_or_repair(True, False, video_file) print("No concerns. Ship it!") except (FileNotFoundError, ValueError) as e: print("Analysis failed.", str(e)) except Exception as e: print(str(e)) transcode = input("Would you like to make a repaired clone now? [y/N] ") if transcode == "y": try: new_video_file, _ = await analyzer.verify_or_repair(True, True, video_file) print("Successfully created ", new_video_file) except Exception as e: print("Unable to complete the transcode. Message: ", str(e)) def main(): if len(sys.argv) < 2: print("Usage: check_video.py <path to video file>", file=sys.stderr) sys.exit(1) enable_logging() video_file = sys.argv[1] conf = TranscodeConfig() analyzer = VideoFileAnalyzer(conf) try: asyncio.run(process_video(analyzer, video_file)) except KeyboardInterrupt: pass if __name__ == '__main__': main() ================================================ FILE: scripts/checkpoints.py ================================================ import asyncio import os from lbry.extras.cli import ensure_directory_exists from lbry.conf import Config from lbry.wallet.header import Headers import lbry.wallet.checkpoints async def main(): outpath = lbry.wallet.checkpoints.__file__ ledger_path = os.path.join(Config().wallet_dir, 'lbc_mainnet') ensure_directory_exists(ledger_path) headers_path = os.path.join(ledger_path, 'headers') headers = Headers(headers_path) await headers.open() print(f"Working on headers at {outpath}") print("Verifying integrity, might take a while.") await headers.repair() target = ((headers.height - 100) // 1000) * 1000 current_checkpoint_tip = max(lbry.wallet.checkpoints.HASHES.keys()) if target <= current_checkpoint_tip: print(f"We have nothing to add: Local: {target}, checkpoint: {current_checkpoint_tip}") return print(f"Headers file at {headers.height}, checkpointing up to {target}." f"Current checkpoint at {current_checkpoint_tip}.") with open(outpath, 'w') as outfile: print('HASHES = {', file=outfile) for height in range(0, target, 1000): print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile) print('}', file=outfile) if __name__ == "__main__": asyncio.get_event_loop().run_until_complete(main()) ================================================ FILE: scripts/checktrie.py ================================================ import sys import asyncio from binascii import hexlify from lbry.wallet.server.db.writer import SQLDB from lbry.wallet.server.coin import LBC from lbry.wallet.server.daemon import Daemon def hex_reverted(value: bytes) -> str: return hexlify(value[::-1]).decode() def match(name, what, value, expected): if value != expected: print(f'{name}: {what} mismatch, {value} is not {expected}') return value == expected def checkrecord(record, expected_winner, expected_claim): assert record['is_controlling'] == record['claim_hash'], dict(record) name = record['normalized'] claim_id = hex_reverted(record['claim_hash']) takover = record['activation_height'] if not expected_winner: print(f"{name} not on lbrycrd. We have {claim_id} at {takover} takeover height.") return if not match(name, 'claim id', claim_id, expected_winner['claimId']): print(f"-- {name} has the wrong winner") if not expected_claim: print(f'{name}: {claim_id} not found, we possibly have an abandoned claim as winner') return match(name, 'height', record['height'], expected_claim['height']) match(name, 'activation height', takover, expected_claim['valid at height']) match(name, 'name', record['normalized'], expected_claim['normalized_name']) match(name, 'amount', record['amount'], expected_claim['amount']) match(name, 'effective amount', record['effective_amount'], expected_claim['effective amount']) match(name, 'txid', hex_reverted(record['txo_hash'][:-4]), expected_claim['txid']) match(name, 'nout', int.from_bytes(record['txo_hash'][-4:], 'little', signed=False), expected_claim['n']) async def checkcontrolling(daemon: Daemon, db: SQLDB): records, names, futs = [], [], [] for record in db.get_claims('claimtrie.claim_hash as is_controlling, claim.*', is_controlling=True): records.append(record) claim_id = hex_reverted(record['claim_hash']) names.append((record['normalized'], (claim_id,), "", True)) # last parameter is IncludeValues if len(names) > 50000: futs.append(daemon._send_vector('getclaimsfornamebyid', names)) names.clear() if names: futs.append(daemon._send_vector('getclaimsfornamebyid', names)) names.clear() while futs: winners, claims = futs.pop(0), futs.pop(0) for winner, claim in zip(await winners, await claims): checkrecord(records.pop(0), winner, claim) if __name__ == '__main__': if len(sys.argv) != 3: print("usage: <db_file_path> <lbrycrd_url>") sys.exit(1) db_path, lbrycrd_url = sys.argv[1:] # pylint: disable=W0632 daemon = Daemon(LBC(), url=lbrycrd_url) db = SQLDB(None, db_path) db.open() asyncio.get_event_loop().run_until_complete(checkcontrolling(daemon, db)) ================================================ FILE: scripts/deploy_dev_wallet_server.sh ================================================ #!/usr/bin/env bash # usage: update_dev_wallet_server.sh <host to update> TARGET_HOST=$1 SCRIPTS_DIR=`dirname $0` LBRY_DIR=`dirname $SCRIPTS_DIR` # build the image docker build -f $LBRY_DIR/docker/Dockerfile.wallet_server -t lbry/wallet-server:development $LBRY_DIR IMAGE=`docker image inspect lbry/wallet-server:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"` # push the image to the server ssh $TARGET_HOST docker image prune --force docker save $IMAGE | ssh $TARGET_HOST docker load ssh $TARGET_HOST docker tag $IMAGE lbry/wallet-server:development # restart the wallet server ssh $TARGET_HOST docker-compose down ssh $TARGET_HOST WALLET_SERVER_TAG="development" docker-compose up -d ================================================ FILE: scripts/dht_crawler.py ================================================ import sys import datetime import logging import asyncio import os.path import random import time import typing from dataclasses import dataclass, astuple, replace from aiohttp import web from prometheus_client import Gauge, generate_latest as prom_generate_latest, Counter, Histogram import lbry.dht.error from lbry.dht.constants import generate_id from lbry.dht.node import Node from lbry.dht.peer import make_kademlia_peer, PeerManager, decode_tcp_peer_from_compact_address from lbry.dht.protocol.distance import Distance from lbry.dht.protocol.iterative_find import FindValueResponse, FindNodeResponse, FindResponse from lbry.extras.daemon.storage import SQLiteMixin from lbry.conf import Config from lbry.utils import resolve_host logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s") log = logging.getLogger(__name__) class SDHashSamples: def __init__(self, samples_file_path): with open(samples_file_path, "rb") as sample_file: self._samples = sample_file.read() assert len(self._samples) % 48 == 0 self.size = len(self._samples) // 48 def read_samples(self, count=1): for _ in range(count): offset = 48 * random.randrange(0, self.size) yield self._samples[offset:offset + 48] class PeerStorage(SQLiteMixin): CREATE_TABLES_QUERY = """ PRAGMA JOURNAL_MODE=WAL; CREATE TABLE IF NOT EXISTS peer ( peer_id INTEGER NOT NULL, node_id VARCHAR(96), address VARCHAR, udp_port INTEGER, tcp_port INTEGER, first_online DATETIME, errors INTEGER, last_churn INTEGER, added_on DATETIME NOT NULL, last_check DATETIME, last_seen DATETIME, latency INTEGER, PRIMARY KEY (peer_id) ); CREATE TABLE IF NOT EXISTS connection ( from_peer_id INTEGER NOT NULL, to_peer_id INTEGER NOT NULL, PRIMARY KEY (from_peer_id, to_peer_id), FOREIGN KEY(from_peer_id) REFERENCES peer (peer_id), FOREIGN KEY(to_peer_id) REFERENCES peer (peer_id) ); """ async def open(self): await super().open() self.db.writer_connection.row_factory = dict_row_factory async def all_peers(self): return [ DHTPeer(**peer) for peer in await self.db.execute_fetchall( "select * from peer where latency > 0 or last_seen > datetime('now', '-1 hour')") ] async def save_peers(self, *peers): log.info("Saving graph nodes (peers) to DB") await self.db.executemany( "INSERT OR REPLACE INTO peer(" "node_id, address, udp_port, tcp_port, first_online, errors, last_churn," "added_on, last_check, last_seen, latency, peer_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", [astuple(peer) for peer in peers] ) log.info("Finished saving graph nodes (peers) to DB") async def save_connections(self, connections_map): log.info("Saving graph edges (connections) to DB") await self.db.executemany( "DELETE FROM connection WHERE from_peer_id = ?", [(key,) for key in connections_map]) for from_peer_id in connections_map: await self.db.executemany( "INSERT INTO connection(from_peer_id, to_peer_id) VALUES(?,?)", [(from_peer_id, to_peer_id) for to_peer_id in connections_map[from_peer_id]]) log.info("Finished saving graph edges (connections) to DB") @dataclass(frozen=True) class DHTPeer: node_id: str address: str udp_port: int tcp_port: int = None first_online: datetime.datetime = None errors: int = None last_churn: int = None added_on: datetime.datetime = None last_check: datetime.datetime = None last_seen: datetime.datetime = None latency: int = None peer_id: int = None @classmethod def from_kad_peer(cls, peer, peer_id): node_id = peer.node_id.hex() if peer.node_id else None return DHTPeer( node_id=node_id, address=peer.address, udp_port=peer.udp_port, tcp_port=peer.tcp_port, peer_id=peer_id, added_on=datetime.datetime.utcnow()) def to_kad_peer(self): node_id = bytes.fromhex(self.node_id) if self.node_id else None return make_kademlia_peer(node_id, self.address, self.udp_port, self.tcp_port) def new_node(address="0.0.0.0", udp_port=0, node_id=None): node_id = node_id or generate_id() loop = asyncio.get_event_loop() return Node(loop, PeerManager(loop), node_id, udp_port, udp_port, 3333, address) class Crawler: unique_total_hosts_metric = Gauge( "unique_total_hosts", "Number of unique hosts seen in the last interval", namespace="dht_crawler_node", ) reachable_hosts_metric = Gauge( "reachable_hosts", "Number of hosts that replied in the last interval", namespace="dht_crawler_node", ) total_historic_hosts_metric = Gauge( "history_total_hosts", "Number of hosts seen since first run.", namespace="dht_crawler_node", ) pending_check_hosts_metric = Gauge( "pending_hosts", "Number of hosts on queue to be checked.", namespace="dht_crawler_node", ) hosts_with_errors_metric = Gauge( "error_hosts", "Number of hosts that raised errors during contact.", namespace="dht_crawler_node", ) ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS = tuple(map(float, range(100))) + ( 500., 1000., 2000., float('inf') ) connections_found_metric = Histogram( "connections_found", "Number of hosts returned by the last successful contact.", namespace="dht_crawler_node", buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS ) known_connections_found_metric = Histogram( "known_connections_found", "Number of already known hosts returned by last contact.", namespace="dht_crawler_node", buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS ) reachable_connections_found_metric = Histogram( "reachable_connections_found", "Number of reachable known hosts returned by last contact.", namespace="dht_crawler_node", buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS ) LATENCY_HISTOGRAM_BUCKETS = ( 0., 5., 10., 15., 30., 60., 120., 180., 240., 300., 600., 1200., 1800., 4000., 6000., float('inf') ) host_latency_metric = Histogram( "host_latency", "Time spent on the last request, in milliseconds.", namespace="dht_crawler_node", buckets=LATENCY_HISTOGRAM_BUCKETS ) probed_streams_metric = Counter( "probed_streams", "Amount of streams probed.", namespace="dht_crawler_node", ) announced_streams_metric = Counter( "announced_streams", "Amount of streams where announcements were found.", namespace="dht_crawler_node", ) working_streams_metric = Counter( "working_streams", "Amount of streams with reachable hosts.", namespace="dht_crawler_node", ) def __init__(self, db_path: str, sd_hash_samples: SDHashSamples): self.node = new_node() self.db = PeerStorage(db_path) self.sd_hashes = sd_hash_samples self._memory_peers = {} self._reachable_by_node_id = {} self._connections = {} async def open(self): await self.db.open() self._memory_peers = { (peer.address, peer.udp_port): peer for peer in await self.db.all_peers() } self.refresh_reachable_set() def refresh_reachable_set(self): self._reachable_by_node_id = { bytes.fromhex(peer.node_id): peer for peer in self._memory_peers.values() if (peer.latency or 0) > 0 } async def probe_files(self): if not self.sd_hashes: return while True: for sd_hash in self.sd_hashes.read_samples(10_000): self.refresh_reachable_set() distance = Distance(sd_hash) node_ids = list(self._reachable_by_node_id.keys()) node_ids.sort(key=lambda node_id: distance(node_id)) k_closest = [self._reachable_by_node_id[node_id] for node_id in node_ids[:8]] found = False working = False for response in asyncio.as_completed( [self.request_peers(peer.address, peer.udp_port, peer.node_id, sd_hash) for peer in k_closest]): response = await response if response and response.found: found = True blob_peers = [] for compact_addr in response.found_compact_addresses: try: blob_peers.append(decode_tcp_peer_from_compact_address(compact_addr)) except ValueError as e: log.error("Error decoding compact peers: %s", e) for blob_peer in blob_peers: response = await self.request_peers(blob_peer.address, blob_peer.tcp_port, blob_peer.node_id, sd_hash) if response: working = True log.info("Found responsive peer for %s: %s:%d(%d)", sd_hash.hex()[:8], blob_peer.address, blob_peer.udp_port or -1, blob_peer.tcp_port or -1) else: log.info("Found dead peer for %s: %s:%d(%d)", sd_hash.hex()[:8], blob_peer.address, blob_peer.udp_port or -1, blob_peer.tcp_port or -1) self.probed_streams_metric.inc() if found: self.announced_streams_metric.inc() if working: self.working_streams_metric.inc() log.info("Done querying stream %s for peers. Found: %s, working: %s", sd_hash.hex()[:8], found, working) await asyncio.sleep(.5) @property def refresh_limit(self): return datetime.datetime.utcnow() - datetime.timedelta(hours=1) @property def all_peers(self): return [ peer for peer in self._memory_peers.values() if (peer.last_seen and peer.last_seen > self.refresh_limit) or (peer.latency or 0) > 0 ] @property def active_peers_count(self): return len(self.all_peers) @property def checked_peers_count(self): return len([peer for peer in self.all_peers if peer.last_check and peer.last_check > self.refresh_limit]) @property def unreachable_peers_count(self): return len([peer for peer in self.all_peers if peer.last_check and peer.last_check > self.refresh_limit and not peer.latency]) @property def peers_with_errors_count(self): return len([peer for peer in self.all_peers if (peer.errors or 0) > 0]) def get_peers_needing_check(self): to_check = [peer for peer in self.all_peers if peer.last_check is None or peer.last_check < self.refresh_limit] return to_check def remove_expired_peers(self): for key, peer in list(self._memory_peers.items()): if (peer.latency or 0) < 1 and peer.last_seen < self.refresh_limit: del self._memory_peers[key] def add_peers(self, *peers): for peer in peers: db_peer = self.get_from_peer(peer) if db_peer and db_peer.node_id is None and peer.node_id is not None: db_peer = replace(db_peer, node_id=peer.node_id.hex()) elif not db_peer: db_peer = DHTPeer.from_kad_peer(peer, len(self._memory_peers) + 1) db_peer = replace(db_peer, last_seen=datetime.datetime.utcnow()) self._memory_peers[(peer.address, peer.udp_port)] = db_peer async def flush_to_db(self): await self.db.save_peers(*self._memory_peers.values()) connections_to_save = self._connections self._connections = {} # await self.db.save_connections(connections_to_save) heavy call self.remove_expired_peers() def get_from_peer(self, peer): return self._memory_peers.get((peer.address, peer.udp_port), None) def set_latency(self, peer, latency=None): if latency: self.host_latency_metric.observe(latency / 1_000_000.0) db_peer = self.get_from_peer(peer) if not db_peer: return db_peer = replace(db_peer, latency=latency) if not db_peer.node_id and peer.node_id: db_peer = replace(db_peer, node_id=peer.node_id.hex()) if db_peer.first_online and latency is None: db_peer = replace(db_peer, last_churn=(datetime.datetime.utcnow() - db_peer.first_online).seconds) elif latency is not None and db_peer.first_online is None: db_peer = replace(db_peer, first_online=datetime.datetime.utcnow()) db_peer = replace(db_peer, last_check=datetime.datetime.utcnow()) self._memory_peers[(db_peer.address, db_peer.udp_port)] = db_peer def inc_errors(self, peer): db_peer = self.get_from_peer(peer) self._memory_peers[(peer.address, peer.node_id)] = replace(db_peer, errors=(db_peer.errors or 0) + 1) def associate_peers(self, peer, other_peers): self._connections[self.get_from_peer(peer).peer_id] = [ self.get_from_peer(other_peer).peer_id for other_peer in other_peers] async def request_peers(self, host, port, node_id, key=None) -> typing.Optional[FindResponse]: key = key or node_id peer = make_kademlia_peer(key, await resolve_host(host, port, 'udp'), port) for attempt in range(3): try: req_start = time.perf_counter_ns() if key == node_id: response = await self.node.protocol.get_rpc_peer(peer).find_node(key) response = FindNodeResponse(key, response) latency = time.perf_counter_ns() - req_start self.set_latency(peer, latency) else: response = await self.node.protocol.get_rpc_peer(peer).find_value(key) response = FindValueResponse(key, response) await asyncio.sleep(0.05) return response except asyncio.TimeoutError: if key == node_id: self.set_latency(peer, None) continue except lbry.dht.error.TransportNotConnected: log.info("Transport unavailable, waiting 1s to retry") await asyncio.sleep(1) except lbry.dht.error.RemoteException as e: log.info('Peer errored: %s:%d attempt #%d - %s', host, port, (attempt + 1), str(e)) if key == node_id: self.inc_errors(peer) self.set_latency(peer, None) continue async def crawl_routing_table(self, host, port, node_id=None): start = time.time() log.debug("querying %s:%d", host, port) address = await resolve_host(host, port, 'udp') key = node_id or self.node.protocol.peer_manager.get_node_id_for_endpoint(address, port) peer = make_kademlia_peer(key, address, port) self.add_peers(peer) if not key: latency = None for _ in range(3): try: ping_start = time.perf_counter_ns() await self.node.protocol.get_rpc_peer(peer).ping() await asyncio.sleep(0.05) key = key or self.node.protocol.peer_manager.get_node_id_for_endpoint(address, port) peer = make_kademlia_peer(key, address, port) latency = time.perf_counter_ns() - ping_start break except asyncio.TimeoutError: pass except lbry.dht.error.RemoteException: self.inc_errors(peer) pass self.set_latency(peer, latency if peer.node_id else None) if not latency or not peer.node_id: if latency and not peer.node_id: log.warning("No node id from %s:%d", host, port) return set() distance = Distance(key) max_distance = int.from_bytes(bytes([0xff] * 48), 'big') peers = set() factor = 2048 for i in range(1000): response = await self.request_peers(address, port, key) new_peers = list(response.get_close_kademlia_peers(peer)) if response else None if not new_peers: break new_peers.sort(key=lambda peer: distance(peer.node_id)) peers.update(new_peers) far_key = new_peers[-1].node_id if distance(far_key) <= distance(key): current_distance = distance(key) next_jump = current_distance + int(max_distance // factor) # jump closer factor /= 2 if factor > 8 and next_jump < max_distance: key = int.from_bytes(peer.node_id, 'big') ^ next_jump if key.bit_length() > 384: break key = key.to_bytes(48, 'big') else: break else: key = far_key factor = 2048 if peers: log.info("Done querying %s:%d in %.2f seconds: %d peers found over %d requests.", host, port, (time.time() - start), len(peers), i) if peers: self.connections_found_metric.observe(len(peers)) known_peers = 0 reachable_connections = 0 for peer in peers: known_peer = self.get_from_peer(peer) known_peers += 1 if known_peer else 0 reachable_connections += 1 if known_peer and (known_peer.latency or 0) > 0 else 0 self.known_connections_found_metric.observe(known_peers) self.reachable_connections_found_metric.observe(reachable_connections) self.add_peers(*peers) self.associate_peers(peer, peers) return peers async def process(self): to_process = {} def submit(_peer): f = asyncio.ensure_future( self.crawl_routing_table(_peer.address, _peer.udp_port, bytes.fromhex(_peer.node_id))) to_process[_peer.peer_id] = f f.add_done_callback(lambda _: to_process.pop(_peer.peer_id)) to_check = self.get_peers_needing_check() last_flush = datetime.datetime.utcnow() while True: for peer in to_check[:200]: if peer.peer_id not in to_process: submit(peer) await asyncio.sleep(.05) await asyncio.sleep(0) self.unique_total_hosts_metric.set(self.checked_peers_count) self.reachable_hosts_metric.set(self.checked_peers_count - self.unreachable_peers_count) self.total_historic_hosts_metric.set(len(self._memory_peers)) self.pending_check_hosts_metric.set(len(to_check)) self.hosts_with_errors_metric.set(self.peers_with_errors_count) log.info("%d known, %d contacted recently, %d unreachable, %d error, %d processing, %d on queue", self.active_peers_count, self.checked_peers_count, self.unreachable_peers_count, self.peers_with_errors_count, len(to_process), len(to_check)) if to_process: await asyncio.wait(to_process.values(), return_when=asyncio.FIRST_COMPLETED) to_check = self.get_peers_needing_check() if (datetime.datetime.utcnow() - last_flush).seconds > 60: log.info("flushing to db") await self.flush_to_db() last_flush = datetime.datetime.utcnow() while not to_check and not to_process: port = self.node.listening_port.get_extra_info('socket').getsockname()[1] self.node.stop() await self.node.start_listening() log.info("Idle, sleeping a minute. Port changed to %d", port) await asyncio.sleep(60.0) to_check = self.get_peers_needing_check() class SimpleMetrics: def __init__(self, port): self.prometheus_port = port async def handle_metrics_get_request(self, _): try: return web.Response( text=prom_generate_latest().decode(), content_type='text/plain; version=0.0.4' ) except Exception: log.exception('could not generate prometheus data') raise async def start(self): prom_app = web.Application() prom_app.router.add_get('/metrics', self.handle_metrics_get_request) metrics_runner = web.AppRunner(prom_app) await metrics_runner.setup() prom_site = web.TCPSite(metrics_runner, "0.0.0.0", self.prometheus_port) await prom_site.start() def dict_row_factory(cursor, row): d = {} for idx, col in enumerate(cursor.description): if col[0] in ('added_on', 'first_online', 'last_seen', 'last_check'): d[col[0]] = datetime.datetime.fromisoformat(row[idx]) if row[idx] else None else: d[col[0]] = row[idx] return d async def test(): db_path = "/tmp/peers.db" if len(sys.argv) == 1 else sys.argv[-1] asyncio.get_event_loop().set_debug(True) metrics = SimpleMetrics('8080') await metrics.start() conf = Config() hosting_samples = SDHashSamples("test.sample") if os.path.isfile("test.sample") else None crawler = Crawler(db_path, hosting_samples) await crawler.open() await crawler.flush_to_db() await crawler.node.start_listening() if crawler.active_peers_count < 100: probes = [] for (host, port) in conf.known_dht_nodes: probes.append(asyncio.create_task(crawler.crawl_routing_table(host, port))) await asyncio.gather(*probes) await crawler.flush_to_db() await asyncio.gather(crawler.process(), crawler.probe_files()) if __name__ == '__main__': asyncio.run(test()) ================================================ FILE: scripts/dht_monitor.py ================================================ import curses import time import asyncio import lbry.wallet from lbry.conf import Config from lbry.extras.daemon.client import daemon_rpc stdscr = curses.initscr() def init_curses(): curses.noecho() curses.cbreak() stdscr.nodelay(1) stdscr.keypad(1) def teardown_curses(): curses.nocbreak() stdscr.keypad(0) curses.echo() curses.endwin() def refresh(routing_table_info): height, width = stdscr.getmaxyx() node_id = routing_table_info['node_id'] for y in range(height): stdscr.addstr(y, 0, " " * (width - 1)) buckets = routing_table_info['buckets'] stdscr.addstr(0, 0, f"node id: {node_id}") stdscr.addstr(1, 0, f"{len(buckets)} buckets") y = 3 for i in range(len(buckets)): stdscr.addstr(y, 0, "bucket %s" % i) y += 1 for peer in buckets[str(i)]: stdscr.addstr(y, 0, f"{peer['node_id'][:8]} ({peer['address']}:{peer['udp_port']})") y += 1 y += 1 stdscr.addstr(y + 1, 0, str(time.time())) stdscr.refresh() async def main(): conf = Config() try: init_curses() c = None while c not in [ord('q'), ord('Q')]: routing_info = await daemon_rpc(conf, 'routing_table_get') refresh(routing_info) c = stdscr.getch() time.sleep(0.1) finally: teardown_curses() if __name__ == "__main__": asyncio.run(main()) ================================================ FILE: scripts/dht_node.py ================================================ import asyncio import argparse import logging import csv import os.path from io import StringIO from typing import Optional from aiohttp import web from prometheus_client import generate_latest as prom_generate_latest from lbry.dht.constants import generate_id from lbry.dht.node import Node from lbry.dht.peer import PeerManager from lbry.extras.daemon.storage import SQLiteStorage from lbry.conf import Config logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s") log = logging.getLogger(__name__) class SimpleMetrics: def __init__(self, port, node): self.prometheus_port = port self.dht_node: Node = node async def handle_metrics_get_request(self, _): try: return web.Response( text=prom_generate_latest().decode(), content_type='text/plain; version=0.0.4' ) except Exception: log.exception('could not generate prometheus data') raise async def handle_peers_csv(self, _): out = StringIO() writer = csv.DictWriter(out, fieldnames=["ip", "port", "dht_id"]) writer.writeheader() for peer in self.dht_node.protocol.routing_table.get_peers(): writer.writerow({"ip": peer.address, "port": peer.udp_port, "dht_id": peer.node_id.hex()}) return web.Response(text=out.getvalue(), content_type='text/csv') async def handle_blobs_csv(self, _): out = StringIO() writer = csv.DictWriter(out, fieldnames=["blob_hash"]) writer.writeheader() for blob in self.dht_node.protocol.data_store.keys(): writer.writerow({"blob_hash": blob.hex()}) return web.Response(text=out.getvalue(), content_type='text/csv') async def start(self): prom_app = web.Application() prom_app.router.add_get('/metrics', self.handle_metrics_get_request) if self.dht_node: prom_app.router.add_get('/peers.csv', self.handle_peers_csv) prom_app.router.add_get('/blobs.csv', self.handle_blobs_csv) metrics_runner = web.AppRunner(prom_app) await metrics_runner.setup() prom_site = web.TCPSite(metrics_runner, "0.0.0.0", self.prometheus_port) await prom_site.start() async def main(host: str, port: int, db_file_path: str, bootstrap_node: Optional[str], prometheus_port: int, export: bool): loop = asyncio.get_event_loop() conf = Config() if not db_file_path.startswith(':memory:'): node_id_file_path = db_file_path + 'node_id' if os.path.exists(node_id_file_path): with open(node_id_file_path, 'rb') as node_id_file: node_id = node_id_file.read() else: with open(node_id_file_path, 'wb') as node_id_file: node_id = generate_id() node_id_file.write(node_id) storage = SQLiteStorage(conf, db_file_path, loop, loop.time) if bootstrap_node: nodes = bootstrap_node.split(':') nodes = [(nodes[0], int(nodes[1]))] else: nodes = conf.known_dht_nodes await storage.open() node = Node( loop, PeerManager(loop), node_id, port, port, 3333, None, storage=storage, is_bootstrap_node=True ) if prometheus_port > 0: metrics = SimpleMetrics(prometheus_port, node if export else None) await metrics.start() node.start(host, nodes) log.info("Peer with id %s started", node_id.hex()) while True: await asyncio.sleep(10) log.info("Known peers: %d. Storing contact information for %d blobs from %d peers.", len(node.protocol.routing_table.get_peers()), len(node.protocol.data_store), len(node.protocol.data_store.get_storing_contacts())) if __name__ == '__main__': parser = argparse.ArgumentParser( description="Starts a single DHT node, which then can be used as a seed node or just a contributing node.") parser.add_argument("--host", default='0.0.0.0', type=str, help="Host to listen for requests. Default: 0.0.0.0") parser.add_argument("--port", default=4444, type=int, help="Port to listen for requests. Default: 4444") parser.add_argument("--db_file", default='/tmp/dht.db', type=str, help="DB file to save peers. Default: /tmp/dht.db") parser.add_argument("--bootstrap_node", default=None, type=str, help="Node to connect for bootstraping this node. Leave unset to use the default ones. " "Format: host:port Example: lbrynet1.lbry.com:4444") parser.add_argument("--metrics_port", default=0, type=int, help="Port for Prometheus metrics. 0 to disable. Default: 0") parser.add_argument("--enable_csv_export", action='store_true', help="Enable CSV endpoints on metrics server.") args = parser.parse_args() asyncio.run(main(args.host, args.port, args.db_file, args.bootstrap_node, args.metrics_port, args.enable_csv_export)) ================================================ FILE: scripts/download_blob_from_peer.py ================================================ """A simple script that attempts to directly download a single blob. To Do: ------ Currently `lbrynet blob get <hash>` does not work to download single blobs which are not already present in the system. The function locks up and never returns. It only works for blobs that are in the `blobfiles` directory already. This bug is reported in lbryio/lbry-sdk, issue #2070. Maybe this script can be investigated, and certain parts can be added to `lbry.extras.daemon.daemon.jsonrpc_blob_get` in order to solve the previous issue, and finally download single blobs from the network (peers or reflector servers). """ import sys import os import asyncio import socket import ipaddress import lbry.wallet from lbry.conf import Config from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_manager import BlobManager from lbry.blob_exchange.client import BlobExchangeClientProtocol, request_blob import logging log = logging.getLogger("lbry") log.addHandler(logging.StreamHandler()) log.setLevel(logging.DEBUG) async def main(blob_hash: str, url: str): conf = Config() loop = asyncio.get_running_loop() host_url, port = url.split(":") try: host = None if ipaddress.ip_address(host_url): host = host_url except ValueError: host = None if not host: host_info = await loop.getaddrinfo( host_url, 'https', proto=socket.IPPROTO_TCP, ) host = host_info[0][4][0] storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite")) blob_manager = BlobManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage, conf) await storage.open() await blob_manager.setup() blob = blob_manager.get_blob(blob_hash) success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout, conf.blob_download_timeout) print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n" f"keep connection: {keep}") if blob.get_is_verified(): await blob_manager.delete_blobs([blob.blob_hash]) print(f"deleted {blob_hash}") if __name__ == "__main__": if len(sys.argv) < 2: print("usage: download_blob_from_peer.py <blob_hash> [host_url:port]") sys.exit(1) url = 'reflector.lbry.com:5567' if len(sys.argv) > 2: url = sys.argv[2] asyncio.run(main(sys.argv[1], url)) ================================================ FILE: scripts/find_max_server_load.py ================================================ import time import asyncio import random from argparse import ArgumentParser from lbry.wallet.network import ClientSession class AgentSmith(ClientSession): async def do_nefarious_things(self): await self.send_request('blockchain.claimtrie.search', { 'no_totals': True, 'offset': random.choice(range(0, 300, 20)), 'limit': 20, 'any_tags': ( random.choice([[ random.choice(['gaming', 'games', 'game']) + random.choice(['entertainment', 'playthrough', 'funny']) + random.choice(['xbox', 'xbox one', 'xbox news']) ], [ random.choice(['aliens', 'alien', 'ufo', 'ufos']) + random.choice(['news', 'sighting', 'sightings']) ], [ random.choice(['art', 'automotive']), random.choice(['blockchain', 'economics', 'food']), random.choice(['funny', 'learnings', 'nature']), random.choice(['news', 'science', 'technology']) ] ]) ), 'not_tags': random.choice([[], [ 'porn', 'mature', 'xxx', 'nsfw' ]]), 'order_by': random.choice([ ['release_time'], ['trending_global', 'trending_mixed'], ['effective_amount'] ]) }) class AgentSmithProgram: def __init__(self, host, port): self.host, self.port = host, port self.agent_smiths = [] async def make_one_more_of_them(self): smith = AgentSmith(network=None, server=(self.host, self.port)) await smith.create_connection() self.agent_smiths.append(smith) async def coordinate_nefarious_activity(self): start = time.perf_counter() await asyncio.gather( *(s.do_nefarious_things() for s in self.agent_smiths), return_exceptions=True ) return time.perf_counter() - start def __len__(self): return len(self.agent_smiths) async def delete_one_smith(self): if self.agent_smiths: await self.agent_smiths.pop().close() async def delete_program(self): await asyncio.gather(*( s.close() for s in self.agent_smiths )) async def main(host, port): smiths = AgentSmithProgram(host, port) await smiths.make_one_more_of_them() activity = asyncio.create_task(smiths.coordinate_nefarious_activity()) ease_off = 0 for i in range(1000): await asyncio.sleep(1) if activity.done() and activity.result() < .9: print('more, more, more...') await asyncio.gather(*( asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20) )) else: print('!!!!!!!!!!!!!!') print('IS NEO LOSING?') print('!!!!!!!!!!!!!!') await asyncio.gather(*( asyncio.create_task(smiths.delete_one_smith()) for _ in range(21) )) print(f'coordinate all {len(smiths)} smiths to action') activity = asyncio.create_task(smiths.coordinate_nefarious_activity()) print('finishing up any remaining actions') await activity print('neo has won, deleting agents...') await smiths.delete_program() print('done.') if __name__ == "__main__": parser = ArgumentParser() parser.add_argument('--host', dest='host', default='localhost', type=str) parser.add_argument('--port', dest='port', default=50001, type=int) args = parser.parse_args() asyncio.run(main(args.host, args.port)) ================================================ FILE: scripts/generate_json_api.py ================================================ import os import re import json import inspect import tempfile import asyncio import time from docopt import docopt from binascii import unhexlify from textwrap import indent from lbry.testcase import CommandTestCase from lbry.extras.cli import set_kwargs, get_argument_parser from lbry.extras.daemon.daemon import ( Daemon, jsonrpc_dumps_pretty, encode_pagination_doc ) from lbry.extras.daemon.json_response_encoder import ( encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc, encode_wallet_doc ) RETURN_DOCS = { 'Account': encode_account_doc(), 'Wallet': encode_wallet_doc(), 'File': encode_file_doc(), 'Transaction': encode_tx_doc(), 'Output': encode_txo_doc(), 'Address': 'an address in base58', 'Dict': 'glorious data in dictionary', } class ExampleRecorder: def __init__(self, test): self.test = test self.examples = {} async def __call__(self, title, *command): parser = get_argument_parser() args, command_args = parser.parse_known_args(command) api_method_name = args.api_method_name parsed = docopt(args.doc, command_args) kwargs = set_kwargs(parsed) for k, v in kwargs.items(): if v and isinstance(v, str) and (v[0], v[-1]) == ('"', '"'): kwargs[k] = v[1:-1] params = json.dumps({"method": api_method_name, "params": kwargs}) method = getattr(self.test.daemon, f'jsonrpc_{api_method_name}') result = method(**kwargs) if asyncio.iscoroutine(result): result = await result output = jsonrpc_dumps_pretty(result, ledger=self.test.daemon.ledger) self.examples.setdefault(api_method_name, []).append({ 'title': title, 'curl': f"curl -d'{params}' http://localhost:5279/", 'lbrynet': 'lbrynet ' + ' '.join(command), 'python': f'requests.post("http://localhost:5279", json={params}).json()', 'output': output.strip() }) return json.loads(output)['result'] class Examples(CommandTestCase): async def asyncSetUp(self): await super().asyncSetUp() self.recorder = ExampleRecorder(self) async def play(self): r = self.recorder # general sdk await r( 'Get status', 'status' ) await r( 'Get version', 'version' ) # settings await r( 'Get settings', 'settings', 'get' ) await r( 'Set settings', 'settings', 'set', '"tcp_port"', '99' ) # preferences await r( 'Set preference', 'preference', 'set', '"theme"', '"dark"' ) await r( 'Get preferences', 'preference', 'get' ) # wallets await r( 'List your wallets', 'wallet', 'list' ) # accounts await r( 'List your accounts', 'account', 'list' ) account = await r( 'Create an account', 'account', 'create', '"generated account"' ) await r( 'Remove an account', 'account', 'remove', account['id'] ) await r( 'Add an account from seed', 'account', 'add', '"new account"', f"--seed=\"{account['seed']}\"" ) await r( 'Modify maximum number of times a change address can be reused', 'account', 'set', account['id'], '--change_max_uses=10' ) # addresses await r( 'List addresses in default account', 'address', 'list' ) an_address = await r( 'Get an unused address', 'address', 'unused' ) address_list_by_id = await r( 'List addresses in specified account', 'address', 'list', f"--account_id=\"{account['id']}\"" ) await r( 'Check if address is mine', 'address', 'is_mine', an_address ) # sends/funds transfer = await r( 'Transfer 2 LBC from default account to specific account', 'account', 'fund', f"--to_account=\"{account['id']}\"", "--amount=2.0", "--broadcast" ) await self.on_transaction_dict(transfer) await self.generate(1) await self.on_transaction_dict(transfer) await r( 'Get default account balance', 'account', 'balance' ) txlist = await r( 'List your transactions', 'transaction', 'list' ) await r( 'Get balance for specific account by id', 'account', 'balance', f"\"{account['id']}\"" ) spread_transaction = await r( 'Spread LBC between multiple addresses', 'account', 'fund', f"--to_account=\"{account['id']}\"", f"--from_account=\"{account['id']}\"", '--amount=1.5', '--outputs=2', '--broadcast' ) await self.on_transaction_dict(spread_transaction) await self.generate(1) await self.on_transaction_dict(spread_transaction) await r( 'Transfer all LBC to a specified account', 'account', 'fund', f"--from_account=\"{account['id']}\"", "--everything", "--broadcast" ) # channels channel = await r( 'Create a channel claim without metadata', 'channel', 'create', '@channel', '1.0' ) channel_id = self.get_claim_id(channel) await self.on_transaction_dict(channel) await self.generate(1) await self.on_transaction_dict(channel) await r( 'List your channel claims', 'channel', 'list' ) await r( 'Paginate your channel claims', 'channel', 'list', '--page=1', '--page_size=20' ) channel = await r( 'Update a channel claim', 'channel', 'update', self.get_claim_id(channel), '--title="New Channel"' ) await self.on_transaction_dict(channel) await self.generate(1) await self.on_transaction_dict(channel) big_channel = await r( 'Create a channel claim with all metadata', 'channel', 'create', '@bigchannel', '1.0', '--title="Big Channel"', '--description="A channel with lots of videos."', '--email="creator@smallmedia.com"', '--tags=music', '--tags=art', '--languages=pt-BR', '--languages=uk', '--locations=BR', '--locations=UA::Kiyv', '--website_url="http://smallmedia.com"', '--thumbnail_url="http://smallmedia.com/logo.jpg"', '--cover_url="http://smallmedia.com/logo.jpg"' ) await self.on_transaction_dict(big_channel) await self.generate(1) await self.on_transaction_dict(big_channel) await self.daemon.jsonrpc_channel_abandon(self.get_claim_id(big_channel)) await self.generate(1) # stream claims with tempfile.NamedTemporaryFile() as file: file.write(b'hello world') file.flush() stream = await r( 'Create a stream claim without metadata', 'stream', 'create', 'astream', '1.0', file.name ) await self.on_transaction_dict(stream) await self.generate(1) await self.on_transaction_dict(stream) stream_id = self.get_claim_id(stream) stream_name = stream['outputs'][0]['name'] stream = await r( 'Update a stream claim to add channel', 'stream', 'update', stream_id, f'--channel_id="{channel_id}"' ) await self.on_transaction_dict(stream) await self.generate(1) await self.on_transaction_dict(stream) await r( 'List all your claims', 'claim', 'list' ) await r( 'Paginate your claims', 'claim', 'list', '--page=1', '--page_size=20' ) await r( 'List all your stream claims', 'stream', 'list' ) await r( 'Paginate your stream claims', 'stream', 'list', '--page=1', '--page_size=20' ) await r( 'Search for all claims in channel', 'claim', 'search', '--channel=@channel' ) await r( 'Search for claims matching a name', 'claim', 'search', f'--name="{stream_name}"' ) with tempfile.NamedTemporaryFile(suffix='.png') as file: file.write(unhexlify( b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc' b'510b9000000097048597300000b1300000b1301009a9c1800000015494441' b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004' b'9454e44ae426082' )) file.flush() big_stream = await r( 'Create an image stream claim with all metadata and fee', 'stream', 'create', 'blank-image', '1.0', file.name, '--tags=blank', '--tags=art', '--languages=en', '--locations=US:NH:Manchester', '--fee_currency=LBC', '--fee_amount=0.3', '--title="Blank Image"', '--description="A blank PNG that is 5x7."', '--author=Picaso', '--license="Public Domain"', '--license_url=http://public-domain.org', '--thumbnail_url="http://smallmedia.com/thumbnail.jpg"', f'--release_time={int(time.time())}', f'--channel_id="{channel_id}"' ) await self.on_transaction_dict(big_stream) await self.generate(1) await self.on_transaction_dict(big_stream) await self.daemon.jsonrpc_channel_abandon(self.get_claim_id(big_stream)) await self.generate(1) # collections collection = await r( 'Create a collection of one stream', 'collection', 'create', '--name=tom', '--bid=1.0', f'--channel_id={channel_id}', f'--claims={stream_id}' ) await self.on_transaction_dict(collection) await self.generate(1) await self.on_transaction_dict(collection) await r( 'List collections', 'collection', 'list', '--resolve', '--resolve_claims=1', ) # files file_list_result = (await r( 'List local files', 'file', 'list' ))['items'] file_uri = f"{file_list_result[0]['claim_name']}#{file_list_result[0]['claim_id']}" await r( 'Resolve a claim', 'resolve', file_uri ) await r( 'List files matching a parameter', 'file', 'list', f"--claim_id=\"{file_list_result[0]['claim_id']}\"" ) await r( 'Delete a file', 'file', 'delete', f"--claim_id=\"{file_list_result[0]['claim_id']}\"" ) await r( 'Get a file', 'get', file_uri ) await r( 'Save a file to the downloads directory', 'file', 'save', f"--sd_hash=\"{file_list_result[0]['sd_hash']}\"" ) # blobs bloblist = await r( 'List your local blobs', 'blob', 'list' ) await r( 'Delete a blob', 'blob', 'delete', f"{bloblist['items'][0]}" ) # abandon all the things abandon_stream = await r( 'Abandon a stream claim', 'stream', 'abandon', stream_id ) await self.on_transaction_dict(abandon_stream) await self.generate(1) await self.on_transaction_dict(abandon_stream) abandon_channel = await r( 'Abandon a channel claim', 'channel', 'abandon', channel_id ) await self.on_transaction_dict(abandon_channel) await self.generate(1) await self.on_transaction_dict(abandon_channel) with tempfile.NamedTemporaryFile() as file: file.write(b'hello world') file.flush() stream = await r( 'Publish a file', 'publish', 'a-new-stream', '--bid=1.0', f'--file_path={file.name}' ) await self.on_transaction_dict(stream) await self.generate(1) await self.on_transaction_dict(stream) def get_examples(): player = Examples('play') result = player.run() if result.errors: for error in result.errors: print(error[1]) raise Exception('See above for errors while running the examples.') return player.recorder.examples SECTIONS = re.compile("(.*?)Usage:(.*?)Options:(.*?)Returns:(.*)", re.DOTALL) REQUIRED_OPTIONS = re.compile(r"\(<(.*?)>.*?\)") ARGUMENT_NAME = re.compile("--([^=]+)") ARGUMENT_TYPE = re.compile(r"\s*\((.*?)\)(.*)") def get_return_def(returns): result = returns.strip() if (result[0], result[-1]) == ('{', '}'): obj_type = result[1:-1] if '[' in obj_type: sub_type = obj_type[obj_type.index('[')+1:-1] obj_type = obj_type[:obj_type.index('[')] if obj_type == 'Paginated': obj_def = encode_pagination_doc(RETURN_DOCS[sub_type]) elif obj_type == 'List': obj_def = [RETURN_DOCS[sub_type]] else: raise NameError(f'Unknown return type: {obj_type}') else: obj_def = RETURN_DOCS[obj_type] return indent(json.dumps(obj_def, indent=4), ' '*12) return result def get_api(name, examples): obj = Daemon.callable_methods[name] docstr = inspect.getdoc(obj).strip() try: description, usage, options, returns = SECTIONS.search(docstr).groups() except: raise ValueError(f"Doc string format error for {obj.__name__}.") required = re.findall(REQUIRED_OPTIONS, usage) arguments = [] for line in options.splitlines(): line = line.strip() if not line: continue if line.startswith('--'): arg, desc = line.split(':', 1) arg_name = ARGUMENT_NAME.search(arg).group(1) arg_type, arg_desc = ARGUMENT_TYPE.search(desc).groups() arguments.append({ 'name': arg_name.strip(), 'type': arg_type.strip(), 'description': [arg_desc.strip()], 'is_required': arg_name in required }) elif line == 'None': continue else: arguments[-1]['description'].append(line.strip()) for arg in arguments: arg['description'] = ' '.join(arg['description']) return { 'name': name, 'description': description.strip(), 'arguments': arguments, 'returns': get_return_def(returns), 'examples': examples } def write_api(f): examples = get_examples() api_definitions = Daemon.get_api_definitions() apis = { 'main': { 'doc': 'Ungrouped commands.', 'commands': [] } } for group_name, group_doc in api_definitions['groups'].items(): apis[group_name] = { 'doc': group_doc, 'commands': [] } for method_name, command in api_definitions['commands'].items(): if 'replaced_by' in command: continue apis[command['group'] or 'main']['commands'].append(get_api( method_name, examples.get(method_name, []) )) json.dump(apis, f, indent=4) if __name__ == '__main__': parent = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_file = os.path.join(parent, 'docs', 'api.json') with open(html_file, 'w+') as f: write_api(f) ================================================ FILE: scripts/hook-coincurve.py ================================================ """ Hook for coincurve. """ import os.path from PyInstaller.utils.hooks import get_module_file_attribute coincurve_dir = os.path.dirname(get_module_file_attribute('coincurve')) binaries = [(os.path.join(coincurve_dir, 'libsecp256k1.dll'), 'coincurve')] ================================================ FILE: scripts/hook-libtorrent.py ================================================ """ Hook for libtorrent. """ import os import glob import os.path from PyInstaller.utils.hooks import get_module_file_attribute from PyInstaller import compat def get_binaries(): if compat.is_win: files = ('c:/Windows/System32/libssl-1_1-x64.dll', 'c:/Windows/System32/libcrypto-1_1-x64.dll') for file in files: if not os.path.isfile(file): print(f"MISSING {file}") return [(file, '.') for file in files] return [] binaries = get_binaries() for file in glob.glob(os.path.join(get_module_file_attribute('libtorrent'), 'libtorrent*pyd*')): binaries.append((file, 'libtorrent')) ================================================ FILE: scripts/idea/lbry-sdk.iml ================================================ <?xml version="1.0" encoding="UTF-8"?> <module type="PYTHON_MODULE" version="4"> <component name="NewModuleRootManager"> <content url="file://$MODULE_DIR$"> <sourceFolder url="file://$MODULE_DIR$/" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/scripts" isTestSource="false" /> </content> <orderEntry type="jdk" jdkName="Python 3.7 (lbry)" jdkType="Python SDK" /> <orderEntry type="sourceFolder" forTests="false" /> </component> <component name="TestRunnerService"> <option name="PROJECT_TEST_RUNNER" value="Unittests" /> </component> </module> ================================================ FILE: scripts/idea/modules.xml ================================================ <?xml version="1.0" encoding="UTF-8"?> <project version="4"> <component name="ProjectModuleManager"> <modules> <module fileurl="file://$PROJECT_DIR$/.idea/lbry-sdk.iml" filepath="$PROJECT_DIR$/.idea/lbry-sdk.iml" /> </modules> </component> </project> ================================================ FILE: scripts/idea/vcs.xml ================================================ <?xml version="1.0" encoding="UTF-8"?> <project version="4"> <component name="VcsDirectoryMappings"> <mapping directory="$PROJECT_DIR$" vcs="Git" /> </component> </project> ================================================ FILE: scripts/initialize_hub_from_snapshot.sh ================================================ #!/bin/bash SNAPSHOT_HEIGHT="1072108" HUB_VOLUME_PATH="/var/lib/docker/volumes/${USER}_wallet_server" ES_VOLUME_PATH="/var/lib/docker/volumes/${USER}_es01" SNAPSHOT_TAR_NAME="wallet_server_snapshot_${SNAPSHOT_HEIGHT}.tar.gz" ES_SNAPSHOT_TAR_NAME="es_snapshot_${SNAPSHOT_HEIGHT}.tar.gz" SNAPSHOT_URL="https://snapshots.lbry.com/hub/${SNAPSHOT_TAR_NAME}" ES_SNAPSHOT_URL="https://snapshots.lbry.com/hub/${ES_SNAPSHOT_TAR_NAME}" echo "fetching wallet server snapshot" wget $SNAPSHOT_URL echo "decompressing wallet server snapshot" tar -xf $SNAPSHOT_TAR_NAME sudo mkdir -p $HUB_VOLUME_PATH sudo rm -rf "${HUB_VOLUME_PATH}/_data" sudo chown -R 999:999 "snapshot_${SNAPSHOT_HEIGHT}" sudo mv "snapshot_${SNAPSHOT_HEIGHT}" "${HUB_VOLUME_PATH}/_data" echo "finished setting up wallet server snapshot" echo "fetching elasticsearch snapshot" wget $ES_SNAPSHOT_URL echo "decompressing elasticsearch snapshot" tar -xf $ES_SNAPSHOT_TAR_NAME sudo chown -R $USER:root "snapshot_es_${SNAPSHOT_HEIGHT}" sudo chmod -R 775 "snapshot_es_${SNAPSHOT_HEIGHT}" sudo mkdir -p $ES_VOLUME_PATH sudo rm -rf "${ES_VOLUME_PATH}/_data" sudo mv "snapshot_es_${SNAPSHOT_HEIGHT}" "${ES_VOLUME_PATH}/_data" echo "finished setting up elasticsearch snapshot" ================================================ FILE: scripts/monitor_slow_queries.py ================================================ import os, asyncio, aiohttp, json, slack, sqlparse async def listen(slack_client, url): async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session: print(f"connecting to {url}") try: ws = await session.ws_connect(url) except (aiohttp.ClientConnectorError, asyncio.TimeoutError): print(f"failed to connect to {url}") return print(f"connected to {url}") async for msg in ws: r = json.loads(msg.data) try: queries = r["api"]["search"]["interrupted_queries"] except KeyError: continue for q in queries: # clean = re.sub(r"\s+", " ", q) clean = sqlparse.format(q, reindent=True, keyword_case='upper') print(f'{url}: {clean}') response = await slack_client.chat_postMessage( username=url, icon_emoji=":hourglass_flowing_sand:", channel='#clubhouse-de-obscure', text="*Query timed out:* " + clean ) if not response["ok"]: print("SLACK ERROR:\n", response) print() async def main(): try: slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True) except KeyError: print("Error: SLACK_TOKEN env var required") return num_servers = 5 tasks = [] for i in range(1, num_servers+1): tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005'))) await asyncio.gather(*tasks) asyncio.run(main()) ================================================ FILE: scripts/publish_performance.py ================================================ import os import time from random import Random from pyqtgraph.Qt import QtCore, QtGui app = QtGui.QApplication([]) from qtreactor import pyqt4reactor pyqt4reactor.install() from twisted.internet import defer, task, threads from orchstr8.services import LbryServiceStack import pyqtgraph as pg class Profiler: pens = [ (230, 25, 75), # red (60, 180, 75), # green (255, 225, 25), # yellow (0, 130, 200), # blue (245, 130, 48), # orange (145, 30, 180), # purple (70, 240, 240), # cyan (240, 50, 230), # magenta (210, 245, 60), # lime (250, 190, 190), # pink (0, 128, 128), # teal ] def __init__(self, graph=None): self.times = {} self.graph = graph def start(self, name): if name in self.times: self.times[name]['start'] = time.time() else: self.times[name] = { 'start': time.time(), 'data': [], 'plot': self.graph.plot( pen=self.pens[len(self.times)], symbolBrush=self.pens[len(self.times)], name=name ) } def stop(self, name): elapsed = time.time() - self.times[name]['start'] self.times[name]['start'] = None self.times[name]['data'].append(elapsed) def draw(self): for plot in self.times.values(): plot['plot'].setData(plot['data']) class ThePublisherOfThings: def __init__(self, blocks=100, txns_per_block=100, seed=2015, start_blocks=110): self.blocks = blocks self.txns_per_block = txns_per_block self.start_blocks = start_blocks self.random = Random(seed) self.profiler = Profiler() self.service = LbryServiceStack(verbose=True, profiler=self.profiler) self.publish_file = None @defer.inlineCallbacks def start(self): yield self.service.startup( after_lbrycrd_start=lambda: self.service.lbrycrd.generate(1010) ) wallet = self.service.lbry.wallet address = yield wallet.get_least_used_address() sendtxid = yield self.service.lbrycrd.sendtoaddress(address, 100) yield self.service.lbrycrd.generate(1) yield wallet.wait_for_tx_in_wallet(sendtxid) yield wallet.update_balance() self.publish_file = os.path.join(self.service.lbry.download_directory, 'the_file') with open(self.publish_file, 'w') as _publish_file: _publish_file.write('message that will be heard around the world\n') yield threads.deferToThread(time.sleep, 0.5) @defer.inlineCallbacks def generate_publishes(self): win = pg.GraphicsLayoutWidget(show=True) win.setWindowTitle('orchstr8: performance monitor') win.resize(1800, 600) p4 = win.addPlot() p4.addLegend() p4.setDownsampling(mode='peak') p4.setClipToView(True) self.profiler.graph = p4 for block in range(self.blocks): for txn in range(self.txns_per_block): name = f'block{block}txn{txn}' self.profiler.start('total') yield self.service.lbry.daemon.jsonrpc_publish( name=name, bid=self.random.randrange(1, 5)/1000.0, file_path=self.publish_file, metadata={ "description": "Some interesting content", "title": "My interesting content", "author": "Video shot by me@example.com", "language": "en", "license": "LBRY Inc", "nsfw": False } ) self.profiler.stop('total') self.profiler.draw() yield self.service.lbrycrd.generate(1) def stop(self): return self.service.shutdown(cleanup=False) @defer.inlineCallbacks def generate_publishes(_): pub = ThePublisherOfThings(50, 10) yield pub.start() yield pub.generate_publishes() yield pub.stop() print(f'lbrycrd: {pub.service.lbrycrd.data_path}') print(f'lbrynet: {pub.service.lbry.data_path}') print(f'lbryumserver: {pub.service.lbryumserver.data_path}') if __name__ == "__main__": task.react(generate_publishes) ================================================ FILE: scripts/release.py ================================================ import os import re import io import sys import yaml import argparse import unittest from datetime import date from getpass import getpass try: import github3 except ImportError: print('To run release tool you need to install github3.py:') print('') print(' $ pip install github3.py') print('') sys.exit(1) AREA_RENAME = { 'api': 'API', 'dht': 'DHT' } def get_github(): config_path = os.path.expanduser('~/.config/gh/hosts.yml') if os.path.exists(config_path): with open(config_path, 'r') as config_file: config = yaml.load(config_file, Loader=yaml.FullLoader) return github3.login(token=config['github.com']['oauth_token']) print('To run release tool you need to first login using the github cli:') print('') print(' $ gh auth login') print('') sys.exit(1) def get_labels(pr, prefix): for label in pr.labels: label_name = label['name'] if label_name.startswith(f'{prefix}: '): yield label_name[len(f'{prefix}: '):] def get_label(pr, prefix): for label in get_labels(pr, prefix): return label BACKWARDS_INCOMPATIBLE = 'backwards-incompatible:' RELEASE_TEXT = 'release-text:' RELEASE_TEXT_LINES = 'release-text-lines:' def get_backwards_incompatible(desc: str): for line in desc.splitlines(): if line.startswith(BACKWARDS_INCOMPATIBLE): yield line[len(BACKWARDS_INCOMPATIBLE):] def get_release_text(desc: str): in_release_lines = False for line in desc.splitlines(): if in_release_lines: yield line.rstrip() elif line.startswith(RELEASE_TEXT_LINES): in_release_lines = True elif line.startswith(RELEASE_TEXT): yield line[len(RELEASE_TEXT):].strip() yield '' class Version: def __init__(self, major=0, minor=0, micro=0): self.major = int(major) self.minor = int(minor) self.micro = int(micro) @classmethod def from_string(cls, version_string): (major, minor, micro), rc = version_string.split('.'), None if 'rc' in micro: micro, rc = micro.split('rc') return cls(major, minor, micro) @classmethod def from_content(cls, content): src = content.decoded.decode('utf-8') version = re.search('__version__ = "(.*?)"', src).group(1) return cls.from_string(version) def increment(self, action): cls = self.__class__ if action == 'major': return cls(self.major+1) elif action == 'minor': return cls(self.major, self.minor+1) elif action == 'micro': return cls(self.major, self.minor, self.micro+1) raise ValueError(f'unknown action: {action}') @property def tag(self): return f'v{self}' def __str__(self): return '.'.join(str(p) for p in [self.major, self.minor, self.micro]) def release(args): gh = get_github() repo = gh.repository('lbryio', 'lbry-sdk') version_file = repo.file_contents('lbry/__init__.py') if not args.confirm: print("\nDRY RUN ONLY. RUN WITH --confirm TO DO A REAL RELEASE.\n") current_version = Version.from_content(version_file) print(f'Current Version: {current_version}') if args.action == 'current': new_version = current_version else: new_version = current_version.increment(args.action) print(f' New Version: {new_version}') previous_release = repo.release_from_tag(args.start_tag or current_version.tag) print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})') print() incompats = [] release_texts = [] unlabeled = [] fixups = [] areas = {} for pr in gh.search_issues(f"merged:>={previous_release._json_data['created_at']} repo:lbryio/lbry-sdk"): area_labels = list(get_labels(pr, 'area')) type_label = get_label(pr, 'type') pr_url = f'[#{pr.number}]({pr.html_url})' user_url = f'[{pr.user["login"]}]({pr.user["html_url"]})' if area_labels and type_label: for area_name in area_labels: for incompat in get_backwards_incompatible(pr.body or ""): incompats.append(f' * [{area_name}] {incompat.strip()} ({pr_url})') for release_text in get_release_text(pr.body or ""): release_texts.append(release_text) if type_label == 'fixup': fixups.append(f' * {pr.title} ({pr_url}) by {user_url}') else: area = areas.setdefault(area_name, []) area.append(f' * [{type_label}] {pr.title} ({pr_url}) by {user_url}') else: unlabeled.append(f' * {pr.title} ({pr_url}) by {user_url}') area_names = list(areas.keys()) area_names.sort() body = io.StringIO() w = lambda s: body.write(s+'\n') w(f'## [{new_version}] - {date.today().isoformat()}') if release_texts: w('') for release_text in release_texts: w(release_text) if incompats: w('') w(f'### Backwards Incompatible Changes') for incompat in incompats: w(incompat) for area in area_names: prs = areas[area] area = AREA_RENAME.get(area.lower(), area.capitalize()) w('') w(f'### {area}') for pr in prs: w(pr) print(body.getvalue()) if unlabeled: print('The following PRs were skipped and not included in changelog:') for skipped in unlabeled: print(skipped) if fixups: print('The following PRs were marked as fixups and not included in changelog:') for skipped in fixups: print(skipped) if args.confirm: commit = version_file.update( new_version.tag, version_file.decoded.decode('utf-8').replace(str(current_version), str(new_version)).encode() )['commit'] repo.create_tag( tag=new_version.tag, message=new_version.tag, sha=commit.sha, obj_type='commit', tagger=commit.committer ) repo.create_release( new_version.tag, name=new_version.tag, body=body.getvalue(), draft=True, ) return 0 class TestReleaseTool(unittest.TestCase): def test_version_parsing(self): self.assertTrue(str(Version.from_string('1.2.3')), '1.2.3') self.assertTrue(str(Version.from_string('1.2.3rc4')), '1.2.3rc4') def test_version_increment(self): v = Version.from_string('1.2.3') self.assertTrue(str(v.increment('major')), '2.0.0') self.assertTrue(str(v.increment('minor')), '1.3.0') self.assertTrue(str(v.increment('micro')), '1.2.4') def test(): runner = unittest.TextTestRunner(verbosity=2) loader = unittest.TestLoader() suite = loader.loadTestsFromTestCase(TestReleaseTool) return 0 if runner.run(suite).wasSuccessful() else 1 def main(): parser = argparse.ArgumentParser() parser.add_argument("--confirm", default=False, action="store_true", help="without this flag, it will only print what it will do but will not actually do it") parser.add_argument("--start-tag", help="custom starting tag for changelog generation") parser.add_argument("action", choices=['test', 'current', 'major', 'minor', 'micro']) args = parser.parse_args() if args.action == "test": code = test() else: code = release(args) print() return code if __name__ == "__main__": sys.exit(main()) ================================================ FILE: scripts/repair_0_31_1_db.py ================================================ import os import binascii import sqlite3 from lbry.conf import Config def main(): conf = Config() db = sqlite3.connect(os.path.join(conf.data_dir, 'lbrynet.sqlite')) cur = db.cursor() files = cur.execute("select stream_hash, file_name, download_directory from file").fetchall() update = {} for stream_hash, file_name, download_directory in files: try: binascii.unhexlify(file_name) except binascii.Error: try: binascii.unhexlify(download_directory) except binascii.Error: update[stream_hash] = ( binascii.hexlify(file_name.encode()).decode(), binascii.hexlify(download_directory.encode()).decode() ) if update: print(f"repair {len(update)} streams") for stream_hash, (file_name, download_directory) in update.items(): cur.execute('update file set file_name=?, download_directory=? where stream_hash=?', (file_name, download_directory, stream_hash)) db.commit() db.close() if __name__ == "__main__": main() ================================================ FILE: scripts/sd_hash_sampler.py ================================================ import asyncio from typing import Iterable from lbry.extras.daemon.client import daemon_rpc from lbry.conf import Config conf = Config() async def sample_prefix(prefix: bytes): result = await daemon_rpc(conf, "claim_search", sd_hash=prefix.hex(), page_size=50) total_pages = result['total_pages'] print(total_pages) sd_hashes = set() for page in range(1, total_pages + 1): if page > 1: result = await daemon_rpc(conf, "claim_search", sd_hash=prefix.hex(), page=page, page_size=50) for item in result['items']: sd_hash = item.get('value', {}).get('source', {}).get('sd_hash') if not sd_hash: print('err', item) continue sd_hashes.add(sd_hash) print('page', page, len(sd_hashes)) return sd_hashes def save_sample(name: str, samples: Iterable[str]): with open(name, 'wb') as outfile: for sample in samples: outfile.write(bytes.fromhex(sample)) outfile.flush() print(outfile.tell()) async def main(): samples = set() futs = [asyncio.ensure_future(sample_prefix(bytes([i]))) for i in range(256)] for i, completed in enumerate(asyncio.as_completed(futs)): samples.update(await completed) print(i, len(samples)) print(save_sample("test.sample", samples)) if __name__ == "__main__": asyncio.run(main()) ================================================ FILE: scripts/standalone_blob_server.py ================================================ import sys import os import asyncio from lbry.blob.blob_manager import BlobManager from lbry.blob_exchange.server import BlobServer from lbry.schema.address import decode_address from lbry.extras.daemon.storage import SQLiteStorage async def main(address: str): try: decode_address(address) except: print(f"'{address}' is not a valid lbrycrd address") return 1 loop = asyncio.get_running_loop() storage = SQLiteStorage(os.path.expanduser("~/.lbrynet/lbrynet.sqlite")) await storage.open() blob_manager = BlobManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage) await blob_manager.setup() server = await loop.create_server( lambda: BlobServer(loop, blob_manager, address), '0.0.0.0', 4444) try: async with server: await server.serve_forever() finally: await storage.close() if __name__ == "__main__": asyncio.run(main(sys.argv[1])) ================================================ FILE: scripts/test_claim_search.py ================================================ import asyncio from lbry.wallet.network import ClientSession from lbry.wallet.rpc.jsonrpc import RPCError import logging import json import sys logging.getLogger('lbry.wallet').setLevel(logging.CRITICAL) async def main(): try: hostname = sys.argv[1] except IndexError: hostname = 'spv11.lbry.com' loop = asyncio.get_event_loop() client = ClientSession(network=None, server=(hostname, 50001)) error = None args = { 'any_tags': ['art'], 'not_tags': ['xxx', 'porn', 'mature', 'nsfw', 'titan'], 'order_by': ["name"], 'offset': 3000, 'limit': 200, 'no_totals': False, } start = loop.time() try: await client.create_connection() try: await client.send_request('blockchain.claimtrie.search', args) except RPCError as err: error = err finally: await client.close() finally: print(json.dumps({ "time": loop.time() - start, "error": error.__str__() if error else None, "args": args, }, indent=4)) if __name__ == "__main__": asyncio.run(main()) ================================================ FILE: scripts/time_to_first_byte.py ================================================ import os import sys import json import argparse import asyncio import time import aiohttp from aiohttp import ClientConnectorError from lbry import __version__ from lbry.blob.blob_file import MAX_BLOB_SIZE from lbry.conf import Config from lbry.extras.daemon.client import daemon_rpc from lbry.extras import system_info async def report_to_slack(output, webhook): payload = { "text": f"lbrynet {__version__} ({system_info.get_platform()['platform']}) time to first byte:\n{output}" } async with aiohttp.request('post', webhook, data=json.dumps(payload)): pass def confidence(times, z, plus_err=True): mean = sum(times) / len(times) standard_dev = (sum((t - sum(times) / len(times)) ** 2.0 for t in times) / len(times)) ** 0.5 err = (z * standard_dev) / (len(times) ** 0.5) return f"{round((mean + err) if plus_err else (mean - err), 3)}" def variance(times): mean = sum(times) / len(times) return round(sum((i - mean) ** 2.0 for i in times) / (len(times) - 1), 3) async def wait_for_done(conf, claim_name, timeout): blobs_completed, last_completed = 0, time.perf_counter() while True: file = (await daemon_rpc(conf, "file_list", claim_name=claim_name))['items'][0] if file['status'] in ['finished', 'stopped']: return True, file['blobs_completed'], file['blobs_in_stream'] elif blobs_completed < int(file['blobs_completed']): blobs_completed, last_completed = int(file['blobs_completed']), time.perf_counter() elif (time.perf_counter() - last_completed) > timeout: return False, file['blobs_completed'], file['blobs_in_stream'] await asyncio.sleep(1.0) async def main(cmd_args=None): print('Time to first byte started using parameters:') for key, value in vars(cmd_args).items(): print(f"{key}: {value}") conf = Config() url_to_claim = {} try: for page in range(1, cmd_args.download_pages + 1): start = time.perf_counter() kwargs = { 'page': page, 'claim_type': 'stream', 'any_tags': [ 'art', 'automotive', 'blockchain', 'comedy', 'economics', 'education', 'gaming', 'music', 'news', 'science', 'sports', 'technology', ], 'order_by': ['trending_global', 'trending_mixed'], 'no_totals': True } if not cmd_args.allow_fees: kwargs['fee_amount'] = 0 response = await daemon_rpc( conf, 'claim_search', **kwargs ) if 'error' in response or not response.get('items'): print(f'Error getting claim list page {page}:') print(response) return 1 else: url_to_claim.update({ claim['permanent_url']: claim for claim in response['items'] if claim['value_type'] == 'stream' }) print(f'Claim search page {page} took: {time.perf_counter() - start}') except (ClientConnectorError, ConnectionError): print("Could not connect to daemon") return 1 print("**********************************************") print(f"Attempting to download {len(url_to_claim)} claim_search streams") first_byte_times = [] download_speeds = [] download_successes = [] failed_to = {} await asyncio.gather(*( daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name']) for claim in url_to_claim.values() if not cmd_args.keep_files )) for i, (url, claim) in enumerate(url_to_claim.items()): start = time.perf_counter() response = await daemon_rpc(conf, 'get', uri=url, save_file=not cmd_args.head_blob_only) if 'error' in response: print(f"{i + 1}/{len(url_to_claim)} - failed to start {url}: {response['error']}") failed_to[url] = 'start' if cmd_args.exit_on_error: return continue first_byte = time.perf_counter() first_byte_times.append(first_byte - start) print(f"{i + 1}/{len(url_to_claim)} - {first_byte - start} {url}") if not cmd_args.head_blob_only: downloaded, amount_downloaded, blobs_in_stream = await wait_for_done( conf, claim['name'], cmd_args.stall_download_timeout ) if downloaded: download_successes.append(url) else: failed_to[url] = 'finish' mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.perf_counter() - start) / 1000000, 2) download_speeds.append(mbs) print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {url} at " f"{mbs}mb/s") if not cmd_args.keep_files: await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name']) await asyncio.sleep(0.1) print("**********************************************") result = f"Started {len(first_byte_times)} of {len(url_to_claim)} attempted front page streams\n" if first_byte_times: result += f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \ f"Best first byte time: {round(min(first_byte_times), 2)}\n" \ f"*95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s*\n" \ f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \ f"Variance: {variance(first_byte_times)}\n" if download_successes: result += f"Downloaded {len(download_successes)}/{len(url_to_claim)}\n" \ f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \ f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \ f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \ f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n" for reason in ('start', 'finish'): failures = [url for url, why in failed_to.items() if reason == why] if failures: result += f"\nFailed to {reason}:\n" + "\n•".join(failures) print(result) webhook = os.environ.get('TTFB_SLACK_TOKEN', None) if webhook: await report_to_slack(result, webhook) return 0 if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--allow_fees", action='store_true') parser.add_argument("--exit_on_error", action='store_true') parser.add_argument("--stall_download_timeout", default=5, type=int) parser.add_argument("--keep_files", action='store_true') parser.add_argument("--head_blob_only", action='store_true') parser.add_argument("--download_pages", type=int, default=10) sys.exit(asyncio.run(main(cmd_args=parser.parse_args())) or 0) ================================================ FILE: scripts/troubleshoot_p2p_and_dht_webservice.py ================================================ import asyncio from aiohttp import web from lbry.blob_exchange.serialization import BlobRequest, BlobResponse from lbry.dht.constants import generate_id from lbry.dht.node import Node from lbry.dht.peer import make_kademlia_peer, PeerManager from lbry.extras.daemon.storage import SQLiteStorage loop = asyncio.get_event_loop() NODE = Node( loop, PeerManager(loop), generate_id(), 60600, 60600, 3333, None, storage=SQLiteStorage(None, ":memory:", loop, loop.time) ) async def check_p2p(ip, port): writer = None try: reader, writer = await asyncio.open_connection(ip, port) writer.write(BlobRequest.make_request_for_blob_hash('0'*96).serialize()) return BlobResponse.deserialize(await reader.readuntil(b'}')).get_address_response().lbrycrd_address except OSError: return None finally: if writer: writer.close() await writer.wait_closed() async def check_dht(ip, port): peer = make_kademlia_peer(None, ip, udp_port=int(port)) return await NODE.protocol.get_rpc_peer(peer).ping() async def endpoint_p2p(request): p2p_port = request.match_info.get('p2p_port', "3333") try: address = await asyncio.wait_for(check_p2p(request.remote, p2p_port), 3) except asyncio.TimeoutError: address = None return {"status": address is not None, "port": p2p_port, "payment_address": address} async def endpoint_dht(request): dht_port = request.match_info.get('dht_port', "3333") try: response = await check_dht(request.remote, dht_port) except asyncio.TimeoutError: response = None return {"status": response == b'pong', "port": dht_port} async def endpoint_default(request): return {"dht_status": await endpoint_dht(request), "p2p_status": await endpoint_p2p(request)} def as_json_response_wrapper(endpoint): async def json_endpoint(*args, **kwargs): return web.json_response(await endpoint(*args, **kwargs)) return json_endpoint app = web.Application() app.add_routes([web.get('/', as_json_response_wrapper(endpoint_default)), web.get('/dht/{dht_port}', as_json_response_wrapper(endpoint_dht)), web.get('/p2p/{p2p_port}', as_json_response_wrapper(endpoint_p2p))]) if __name__ == '__main__': loop.create_task(NODE.start_listening("0.0.0.0")) web.run_app(app, port=60666) ================================================ FILE: scripts/wallet_server_monitor.py ================================================ import sys import json import random import asyncio import argparse import traceback import signal from time import time from datetime import datetime try: import aiohttp import psycopg2 import slack except ImportError: print(f"To run {sys.argv[0]} you need to install aiohttp, psycopg2 and slackclient:") print(f"") print(f" $ pip install aiohttp psycopg2 slackclient") print("") sys.exit(1) if not sys.version_info >= (3, 7): print("Please use Python 3.7 or higher, this script expects that dictionary keys preserve order.") sys.exit(1) async def handle_slow_query(cursor, server, command, queries): for query in queries: cursor.execute(""" INSERT INTO wallet_server_slow_queries (server, command, query, event_time) VALUES (%s,%s,%s,%s); """, (server, command, query, datetime.now())) async def handle_analytics_event(cursor, event, server): cursor.execute(""" INSERT INTO wallet_server_stats (server, sessions, event_time) VALUES (%s,%s,%s); """, (server, event['status']['sessions'], datetime.now())) for command, stats in event["api"].items(): data = { 'server': server, 'command': command, 'event_time': datetime.now() } for key, value in stats.items(): if key.endswith("_queries"): if key == "interrupted_queries": await handle_slow_query(cursor, server, command, value) continue if isinstance(value, list): data.update({ key + '_avg': value[0], key + '_min': value[1], key + '_five': value[2], key + '_twenty_five': value[3], key + '_fifty': value[4], key + '_seventy_five': value[5], key + '_ninety_five': value[6], key + '_max': value[7], }) else: data[key] = value cursor.execute(f""" INSERT INTO wallet_server_command_stats ({','.join(data)}) VALUES ({','.join('%s' for _ in data)}); """, list(data.values())) SLACKCLIENT = None async def boris_says(what_boris_says): if SLACKCLIENT: await SLACKCLIENT.chat_postMessage( username="boris the wallet monitor", icon_emoji=":boris:", channel='#tech-sdk', text=what_boris_says ) else: print(what_boris_says) async def monitor(db, server): c = db.cursor() delay = 30 height_changed = None, time() height_change_reported = False first_attempt = True while True: try: async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: try: ws = await session.ws_connect(server) except (aiohttp.ClientConnectionError, asyncio.TimeoutError): if first_attempt: print(f"failed connecting to {server}") await boris_says(random.choice([ f"{server} is not responding, probably dead, will not connect again.", ])) return raise if first_attempt: await boris_says(f"{server} is online") else: await boris_says(f"{server} is back online") delay = 30 first_attempt = False print(f"connected to {server}") async for msg in ws: event = json.loads(msg.data) height = event['status'].get('height') height_change_time = int(time()-height_changed[1]) if height is None: pass elif height_changed[0] != height: height_changed = (height, time()) if height_change_reported: await boris_says( f"Server {server} received new block after {height_change_time / 60:.1f} minutes.", ) height_change_reported = False elif height_change_time > 30*60: if not height_change_reported or height_change_time % (2*60) == 0: await boris_says( f"It's been {height_change_time/60:.1f} minutes since {server} received a new block.", ) height_change_reported = True await handle_analytics_event(c, event, server) db.commit() except (aiohttp.ClientConnectionError, asyncio.TimeoutError): await boris_says(random.choice([ f"<!channel> Guys, we have a problem! Nobody home at {server}. Will check on it again in {delay} seconds.", f"<!channel> Something wrong with {server}. I think dead. Will poke it again in {delay} seconds.", f"<!channel> Don't hear anything from {server}, maybe dead. Will try it again in {delay} seconds.", ])) await asyncio.sleep(delay) delay += 30 async def main(dsn, servers): db = ensure_database(dsn) await boris_says(random.choice([ "No fear, Boris is here! I will monitor the servers now and will try not to fall asleep again.", "Comrad the Cat and Boris are here now, monitoring wallet servers.", ])) await asyncio.gather(*( asyncio.create_task(monitor(db, server)) for server in servers )) def ensure_database(dsn): db = psycopg2.connect(**dsn) c = db.cursor() c.execute("SELECT to_regclass('wallet_server_stats');") if c.fetchone()[0] is None: print("creating table 'wallet_server_stats'...") c.execute(""" CREATE TABLE wallet_server_stats ( server text, sessions integer, event_time timestamp ); """) c.execute("SELECT to_regclass('wallet_server_slow_queries');") if c.fetchone()[0] is None: print("creating table 'wallet_server_slow_queries'...") c.execute(""" CREATE TABLE wallet_server_slow_queries ( server text, command text, query text, event_time timestamp ); """) c.execute("SELECT to_regclass('wallet_server_command_stats');") if c.fetchone()[0] is None: print("creating table 'wallet_server_command_stats'...") c.execute(""" CREATE TABLE wallet_server_command_stats ( server text, command text, event_time timestamp, -- total requests received during event window receive_count integer, -- sum of these is total responses made cache_response_count integer, query_response_count integer, intrp_response_count integer, error_response_count integer, -- millisecond timings for non-cache responses (response_*, interrupt_*, error_*) response_avg float, response_min float, response_five float, response_twenty_five float, response_fifty float, response_seventy_five float, response_ninety_five float, response_max float, interrupt_avg float, interrupt_min float, interrupt_five float, interrupt_twenty_five float, interrupt_fifty float, interrupt_seventy_five float, interrupt_ninety_five float, interrupt_max float, error_avg float, error_min float, error_five float, error_twenty_five float, error_fifty float, error_seventy_five float, error_ninety_five float, error_max float, -- response, interrupt and error each also report the python, wait and sql stats python_avg float, python_min float, python_five float, python_twenty_five float, python_fifty float, python_seventy_five float, python_ninety_five float, python_max float, wait_avg float, wait_min float, wait_five float, wait_twenty_five float, wait_fifty float, wait_seventy_five float, wait_ninety_five float, wait_max float, sql_avg float, sql_min float, sql_five float, sql_twenty_five float, sql_fifty float, sql_seventy_five float, sql_ninety_five float, sql_max float, -- extended timings for individual sql executions individual_sql_avg float, individual_sql_min float, individual_sql_five float, individual_sql_twenty_five float, individual_sql_fifty float, individual_sql_seventy_five float, individual_sql_ninety_five float, individual_sql_max float, individual_sql_count integer ); """) db.commit() return db def get_dsn(args): dsn = {} for attr in ('dbname', 'user', 'password', 'host', 'port'): value = getattr(args, f'pg_{attr}') if value: dsn[attr] = value return dsn def get_servers(args): servers = [] for s in args.server_range.split(","): if '..' in s: start, end = s.split('..') servers.extend(range(int(start), int(end)+1)) else: servers.append(int(s)) return [args.server_url.format(i) for i in servers] def get_slack_client(args): if args.slack_token: return slack.WebClient(token=args.slack_token, run_async=True) def get_args(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--pg-dbname", default="analytics", help="PostgreSQL database name") parser.add_argument("--pg-user", help="PostgreSQL username") parser.add_argument("--pg-password", help="PostgreSQL password") parser.add_argument("--pg-host", default="localhost", help="PostgreSQL host") parser.add_argument("--pg-port", default="5432", help="PostgreSQL port") parser.add_argument("--server-url", default="http://spv{}.lbry.com:50005", help="URL with '{}' placeholder") parser.add_argument("--server-range", default="1..5", help="Range of numbers or single number to use in URL placeholder") parser.add_argument("--slack-token") return parser.parse_args() async def shutdown(signal, loop): await boris_says(f"I got signal {signal.name}. Shutting down.") tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] [task.cancel() for task in tasks] await asyncio.gather(*tasks, return_exceptions=True) # loop.stop() if __name__ == "__main__": loop = asyncio.get_event_loop() for sig in (signal.SIGHUP, signal.SIGTERM, signal.SIGINT): loop.add_signal_handler(sig, lambda s=sig: asyncio.create_task(shutdown(s, loop))) args = get_args() SLACKCLIENT = get_slack_client(args) try: loop.run_until_complete(main(get_dsn(args), get_servers(args))) except asyncio.CancelledError as e: pass except Exception as e: loop.run_until_complete(boris_says("<!channel> I crashed with the following exception:")) loop.run_until_complete(boris_says(traceback.format_exc())) finally: loop.run_until_complete( boris_says(random.choice([ "Wallet servers will have to watch themselves, I'm leaving now.", "I'm going to go take a nap, hopefully nothing blows up while I'm gone.", "Babushka is calling, I'll be back later, someone else watch the servers while I'm gone.", ])) ) ================================================ FILE: setup.cfg ================================================ [coverage:run] branch = True [coverage:paths] source = lbry .tox/*/lib/python*/site-packages/lbry omit = lbry/wallet/orchstr8/ .tox/*/lib/python*/site-packages/lbry/wallet/orchstr8/node.py [cryptography.*,coincurve.*,pbkdf2,libtorrent] ignore_missing_imports = True [pylint] jobs=8 ignore=words,server,rpc,schema,winpaths.py,migrator,undecorated.py max-parents=10 max-args=10 max-line-length=120 good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id,r,iv,ts,l,pk valid-metaclass-classmethod-first-arg=mcs disable= c-extension-no-member, fixme, broad-except, raise-missing-from, no-else-return, cyclic-import, missing-docstring, consider-using-f-string, duplicate-code, expression-not-assigned, inconsistent-return-statements, too-few-public-methods, too-many-lines, too-many-locals, too-many-branches, too-many-arguments, too-many-statements, too-many-nested-blocks, too-many-public-methods, too-many-return-statements, too-many-instance-attributes, unspecified-encoding, protected-access, unused-argument ================================================ FILE: setup.py ================================================ import os import sys from lbry import __name__, __version__ from setuptools import setup, find_packages BASE = os.path.dirname(__file__) with open(os.path.join(BASE, 'README.md'), encoding='utf-8') as fh: long_description = fh.read() setup( name=__name__, version=__version__, author="LBRY Inc.", author_email="hello@lbry.com", url="https://lbry.com", description="A decentralized media library and marketplace", long_description=long_description, long_description_content_type="text/markdown", keywords="lbry protocol media", license='MIT', python_requires='>=3.8', packages=find_packages(exclude=('tests',)), zip_safe=False, entry_points={ 'console_scripts': [ 'lbrynet=lbry.extras.cli:main', 'orchstr8=lbry.wallet.orchstr8.cli:main' ], }, install_requires=[ 'aiohttp==3.7.4', 'aioupnp==0.0.18', 'appdirs==1.4.3', 'certifi>=2021.10.08', 'colorama==0.3.7', 'distro==1.4.0', 'base58==1.0.0', 'cffi==1.13.2', 'cryptography==3.4.7', 'protobuf==3.17.2', 'prometheus_client==0.7.1', 'ecdsa==0.13.3', 'pyyaml==5.3.1', 'docopt==0.6.2', 'hachoir==3.1.2', 'coincurve==15.0.0', 'pbkdf2==1.3', 'filetype==1.0.9', 'libtorrent==2.0.6', ], extras_require={ 'lint': [ 'pylint==2.13.9' ], 'test': [ 'coverage', 'jsonschema==4.4.0', ], 'hub': [ 'hub@git+https://github.com/lbryio/hub.git@929448d64bcbe6c5e476757ec78456beaa85e56a' ] }, classifiers=[ 'Framework :: AsyncIO', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3', 'Operating System :: OS Independent', 'Topic :: Internet', 'Topic :: Software Development :: Testing', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Distributed Computing', 'Topic :: Utilities', ], ) ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/dht_mocks.py ================================================ import typing import contextlib import socket from unittest import mock import functools import asyncio if typing.TYPE_CHECKING: from lbry.dht.protocol.protocol import KademliaProtocol def get_time_accelerator(loop: asyncio.AbstractEventLoop, instant_step: bool = False) -> typing.Callable[[float], typing.Awaitable[None]]: """ Returns an async advance() function This provides a way to advance() the BaseEventLoop.time for the scheduled TimerHandles made by call_later, call_at, and call_soon. """ original = loop.time _drift = 0 loop.time = functools.wraps(loop.time)(lambda: original() + _drift) async def accelerate_time(seconds: float) -> None: nonlocal _drift if seconds < 0: raise ValueError(f'Cannot go back in time ({seconds} seconds)') _drift += seconds await asyncio.sleep(0) async def accelerator(seconds: float): steps = seconds * 10.0 if not instant_step else 1 for _ in range(max(int(steps), 1)): await accelerate_time(seconds/steps) return accelerator @contextlib.contextmanager def mock_network_loop(loop: asyncio.AbstractEventLoop, dht_network: typing.Optional[typing.Dict[typing.Tuple[str, int], 'KademliaProtocol']] = None): dht_network: typing.Dict[typing.Tuple[str, int], 'KademliaProtocol'] = dht_network if dht_network is not None else {} async def create_datagram_endpoint(proto_lam: typing.Callable[[], 'KademliaProtocol'], from_addr: typing.Tuple[str, int]): def sendto(data, to_addr): rx = dht_network.get(to_addr) if rx and rx.external_ip: # print(f"{from_addr[0]}:{from_addr[1]} -{len(data)} bytes-> {rx.external_ip}:{rx.udp_port}") return rx.datagram_received(data, from_addr) protocol = proto_lam() transport = mock.Mock(spec=asyncio.DatagramTransport) transport.get_extra_info = lambda k: {'socket': mock_sock}[k] transport.is_closing = lambda: False transport.close = lambda: mock_sock.close() mock_sock.sendto = sendto transport.sendto = mock_sock.sendto protocol.connection_made(transport) dht_network[from_addr] = protocol return transport, protocol mock_sock = mock.Mock(spec=socket.socket) mock_sock.setsockopt = lambda *_: None mock_sock.bind = lambda *_: None mock_sock.setblocking = lambda *_: None mock_sock.getsockname = lambda: "0.0.0.0" mock_sock.getpeername = lambda: "" mock_sock.close = lambda: None mock_sock.type = socket.SOCK_DGRAM mock_sock.fileno = lambda: 7 loop.create_datagram_endpoint = create_datagram_endpoint yield ================================================ FILE: tests/integration/__init__.py ================================================ ================================================ FILE: tests/integration/blockchain/__init__.py ================================================ ================================================ FILE: tests/integration/blockchain/test_account_commands.py ================================================ from binascii import hexlify, unhexlify from lbry.testcase import CommandTestCase from lbry.wallet.script import InputScript from lbry.wallet.dewies import dewies_to_lbc from lbry.wallet.account import DeterministicChannelKeyManager from lbry.crypto.hash import hash160 from lbry.crypto.base58 import Base58 def extract(d, keys): return {k: d[k] for k in keys} class AccountManagement(CommandTestCase): async def test_account_list_set_create_remove_add(self): # check initial account accounts = await self.daemon.jsonrpc_account_list() self.assertItemCount(accounts, 1) # change account name and gap account_id = accounts['items'][0]['id'] self.daemon.jsonrpc_account_set( account_id=account_id, new_name='test account', receiving_gap=95, receiving_max_uses=96, change_gap=97, change_max_uses=98 ) accounts = (await self.daemon.jsonrpc_account_list())['items'][0] self.assertEqual(accounts['name'], 'test account') self.assertEqual( accounts['address_generator']['receiving'], {'gap': 95, 'maximum_uses_per_address': 96} ) self.assertEqual( accounts['address_generator']['change'], {'gap': 97, 'maximum_uses_per_address': 98} ) # create another account await self.daemon.jsonrpc_account_create('second account') accounts = await self.daemon.jsonrpc_account_list() self.assertItemCount(accounts, 2) self.assertEqual(accounts['items'][1]['name'], 'second account') account_id2 = accounts['items'][1]['id'] # make new account the default self.daemon.jsonrpc_account_set(account_id=account_id2, default=True) accounts = await self.daemon.jsonrpc_account_list(show_seed=True) self.assertEqual(accounts['items'][0]['name'], 'second account') account_seed = accounts['items'][1]['seed'] # remove account self.daemon.jsonrpc_account_remove(accounts['items'][1]['id']) accounts = await self.daemon.jsonrpc_account_list() self.assertItemCount(accounts, 1) # add account await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed) accounts = await self.daemon.jsonrpc_account_list() self.assertItemCount(accounts, 2) self.assertEqual(accounts['items'][1]['name'], 'recreated account') # list specific account accounts = await self.daemon.jsonrpc_account_list(account_id, include_claims=True) self.assertEqual(accounts['items'][0]['name'], 'recreated account') async def test_wallet_migration(self): old_id, new_id, valid_key = ( 'mi9E8KqFfW5ngktU22pN2jpgsdf81ZbsGY', 'mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8', '-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95' '466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh' '0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS' 'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n' ) # null certificates should get deleted self.account.channel_keys = { new_id: 'not valid key', 'foo': 'bar', } await self.account.maybe_migrate_certificates() self.assertEqual(self.account.channel_keys, {}) self.account.channel_keys = { new_id: 'not valid key', 'foo': 'bar', 'invalid address': valid_key, } await self.account.maybe_migrate_certificates() self.assertEqual(self.account.channel_keys, { new_id: valid_key }) async def assertFindsClaims(self, claim_names, awaitable): self.assertEqual(claim_names, [txo.claim_name for txo in (await awaitable)['items']]) async def assertOutputAmount(self, amounts, awaitable): self.assertEqual(amounts, [dewies_to_lbc(txo.amount) for txo in (await awaitable)['items']]) async def test_commands_across_accounts(self): channel_list = self.daemon.jsonrpc_channel_list stream_list = self.daemon.jsonrpc_stream_list support_list = self.daemon.jsonrpc_support_list utxo_list = self.daemon.jsonrpc_utxo_list default_account = self.wallet.default_account second_account = await self.daemon.jsonrpc_account_create('second account') tx = await self.daemon.jsonrpc_account_send( '0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id), blocking=True ) await self.confirm_tx(tx.id) await self.assertOutputAmount(['0.05', '9.949876'], utxo_list()) await self.assertOutputAmount(['0.05'], utxo_list(account_id=second_account.id)) await self.assertOutputAmount(['9.949876'], utxo_list(account_id=default_account.id)) channel1 = await self.channel_create('@channel-in-account1', '0.01') channel2 = await self.channel_create( '@channel-in-account2', '0.01', account_id=second_account.id, funding_account_ids=[default_account.id] ) await self.assertFindsClaims(['@channel-in-account2', '@channel-in-account1'], channel_list()) await self.assertFindsClaims(['@channel-in-account1'], channel_list(account_id=default_account.id)) await self.assertFindsClaims(['@channel-in-account2'], channel_list(account_id=second_account.id)) stream1 = await self.stream_create('stream-in-account1', '0.01', channel_id=self.get_claim_id(channel1)) stream2 = await self.stream_create( 'stream-in-account2', '0.01', channel_id=self.get_claim_id(channel2), account_id=second_account.id, funding_account_ids=[default_account.id] ) await self.assertFindsClaims(['stream-in-account2', 'stream-in-account1'], stream_list()) await self.assertFindsClaims(['stream-in-account1'], stream_list(account_id=default_account.id)) await self.assertFindsClaims(['stream-in-account2'], stream_list(account_id=second_account.id)) await self.assertFindsClaims( ['stream-in-account2', 'stream-in-account1', '@channel-in-account2', '@channel-in-account1'], self.daemon.jsonrpc_claim_list() ) await self.assertFindsClaims( ['stream-in-account1', '@channel-in-account1'], self.daemon.jsonrpc_claim_list(account_id=default_account.id) ) await self.assertFindsClaims( ['stream-in-account2', '@channel-in-account2'], self.daemon.jsonrpc_claim_list(account_id=second_account.id) ) support1 = await self.support_create(self.get_claim_id(stream1), '0.01') support2 = await self.support_create( self.get_claim_id(stream2), '0.01', account_id=second_account.id, funding_account_ids=[default_account.id] ) self.assertEqual([support2['txid'], support1['txid']], [txo.tx_ref.id for txo in (await support_list())['items']]) self.assertEqual([support1['txid']], [txo.tx_ref.id for txo in (await support_list(account_id=default_account.id))['items']]) self.assertEqual([support2['txid']], [txo.tx_ref.id for txo in (await support_list(account_id=second_account.id))['items']]) history = await self.daemon.jsonrpc_transaction_list() self.assertItemCount(history, 8) history = history['items'] self.assertEqual(extract(history[0]['support_info'][0], ['claim_name', 'is_tip', 'amount', 'balance_delta']), { 'claim_name': 'stream-in-account2', 'is_tip': False, 'amount': '0.01', 'balance_delta': '-0.01' }) self.assertEqual(extract(history[1]['support_info'][0], ['claim_name', 'is_tip', 'amount', 'balance_delta']), { 'claim_name': 'stream-in-account1', 'is_tip': False, 'amount': '0.01', 'balance_delta': '-0.01' }) self.assertEqual(extract(history[2]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), { 'claim_name': 'stream-in-account2', 'amount': '0.01', 'balance_delta': '-0.01' }) self.assertEqual(extract(history[3]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), { 'claim_name': 'stream-in-account1', 'amount': '0.01', 'balance_delta': '-0.01' }) self.assertEqual(extract(history[4]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), { 'claim_name': '@channel-in-account2', 'amount': '0.01', 'balance_delta': '-0.01' }) self.assertEqual(extract(history[5]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), { 'claim_name': '@channel-in-account1', 'amount': '0.01', 'balance_delta': '-0.01' }) self.assertEqual(history[6]['value'], '0.0') self.assertEqual(history[7]['value'], '10.0') async def test_address_validation(self): address = await self.daemon.jsonrpc_address_unused() bad_address = address[0:20] + '9999999' + address[27:] with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"): await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address]) async def test_hybrid_channel_keys(self): # non-deterministic channel self.account.channel_keys = { 'mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8': '-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95' '466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh' '0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS' 'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n' } channel1 = await self.create_nondeterministic_channel('@foo1', '1.0', unhexlify( '3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1' '66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16' 'a97a6d6a4a8effd29d748901bb9789352519cd00b13d' )) await self.confirm_tx(channel1['txid']) # deterministic channel channel2 = await self.channel_create('@foo2') await self.stream_create('stream-in-channel1', '0.01', channel_id=self.get_claim_id(channel1)) await self.stream_create('stream-in-channel2', '0.01', channel_id=self.get_claim_id(channel2)) resolved_stream1 = await self.resolve('@foo1/stream-in-channel1') self.assertEqual('stream-in-channel1', resolved_stream1['name']) self.assertTrue(resolved_stream1['is_channel_signature_valid']) resolved_stream2 = await self.resolve('@foo2/stream-in-channel2') self.assertEqual('stream-in-channel2', resolved_stream2['name']) self.assertTrue(resolved_stream2['is_channel_signature_valid']) async def test_deterministic_channel_keys(self): seed = self.account.seed keys = self.account.deterministic_channel_keys # create two channels and make sure they have different keys channel1a = await self.channel_create('@foo1') channel2a = await self.channel_create('@foo2') self.assertNotEqual( channel1a['outputs'][0]['value']['public_key'], channel2a['outputs'][0]['value']['public_key'], ) # start another daemon from the same seed self.daemon2 = await self.add_daemon(seed=seed) channel2b, channel1b = (await self.daemon2.jsonrpc_channel_list())['items'] # both daemons end up with the same channel signing keys automagically self.assertTrue(channel1b.has_private_key) self.assertEqual( channel1a['outputs'][0]['value']['public_key_id'], channel1b.private_key.address ) self.assertTrue(channel2b.has_private_key) self.assertEqual( channel2a['outputs'][0]['value']['public_key_id'], channel2b.private_key.address ) # repeatedly calling next channel key returns the same key when not used current_known = keys.last_known next_key = await keys.generate_next_key() self.assertEqual(current_known, keys.last_known) self.assertEqual(next_key.address, (await keys.generate_next_key()).address) # again, should be idempotent next_key = await keys.generate_next_key() self.assertEqual(current_known, keys.last_known) self.assertEqual(next_key.address, (await keys.generate_next_key()).address) # create third channel while both daemons running, second daemon should pick it up channel3a = await self.channel_create('@foo3') self.assertEqual(current_known+1, keys.last_known) self.assertNotEqual(next_key.address, (await keys.generate_next_key()).address) channel3b, = (await self.daemon2.jsonrpc_channel_list(name='@foo3'))['items'] self.assertTrue(channel3b.has_private_key) self.assertEqual( channel3a['outputs'][0]['value']['public_key_id'], channel3b.private_key.address ) # channel key cache re-populated after simulated restart # reset cache self.account.deterministic_channel_keys = DeterministicChannelKeyManager(self.account) channel3c, channel2c, channel1c = (await self.daemon.jsonrpc_channel_list())['items'] self.assertFalse(channel1c.has_private_key) self.assertFalse(channel2c.has_private_key) self.assertFalse(channel3c.has_private_key) # repopulate cache await self.account.deterministic_channel_keys.ensure_cache_primed() self.assertEqual(self.account.deterministic_channel_keys.last_known, keys.last_known) channel3c, channel2c, channel1c = (await self.daemon.jsonrpc_channel_list())['items'] self.assertTrue(channel1c.has_private_key) self.assertTrue(channel2c.has_private_key) self.assertTrue(channel3c.has_private_key) async def test_time_locked_transactions(self): address = await self.account.receiving.get_or_create_usable_address() private_key = await self.ledger.get_private_key_for_address(self.wallet, address) script = InputScript( template=InputScript.TIME_LOCK_SCRIPT, values={'height': 210, 'pubkey_hash': self.ledger.address_to_hash160(address)} ) script_address = self.ledger.hash160_to_script_address(hash160(script.source)) script_source = hexlify(script.source).decode() await self.assertBalance(self.account, '10.0') tx = await self.daemon.jsonrpc_account_send('4.0', script_address) await self.confirm_tx(tx.id) await self.generate(510) await self.assertBalance(self.account, '5.999877') tx = await self.daemon.jsonrpc_account_deposit( tx.id, 0, script_source, Base58.encode_check(self.ledger.private_key_to_wif(private_key.private_key_bytes)) ) await self.confirm_tx(tx.id) await self.assertBalance(self.account, '9.9997545') ================================================ FILE: tests/integration/blockchain/test_blockchain_reorganization.py ================================================ import logging import asyncio from binascii import hexlify from lbry.testcase import CommandTestCase class BlockchainReorganizationTests(CommandTestCase): VERBOSITY = logging.WARN async def assertBlockHash(self, height): bp = self.conductor.spv_node.writer reader = self.conductor.spv_node.server def get_txids(): return [ reader.db.fs_tx_hash(tx_num)[0][::-1].hex() for tx_num in range(bp.db.tx_counts[height - 1], bp.db.tx_counts[height]) ] block_hash = await self.blockchain.get_block_hash(height) self.assertEqual(block_hash, (await self.ledger.headers.hash(height)).decode()) self.assertEqual(block_hash, (await reader.db.fs_block_hashes(height, 1))[0][::-1].hex()) txids = await asyncio.get_event_loop().run_in_executor(None, get_txids) txs = await reader.db.get_transactions_and_merkles(txids) block_txs = (await bp.daemon.deserialised_block(block_hash))['tx'] self.assertSetEqual(set(block_txs), set(txs.keys()), msg='leveldb/lbrycrd is missing transactions') self.assertListEqual(block_txs, list(txs.keys()), msg='leveldb/lbrycrd transactions are of order') async def test_reorg(self): bp = self.conductor.spv_node.writer bp.reorg_count_metric.set(0) # invalidate current block, move forward 2 height = 206 self.assertEqual(self.ledger.headers.height, height) await self.assertBlockHash(height) block_hash = (await self.ledger.headers.hash(206)).decode() await self.blockchain.invalidate_block(block_hash) await self.blockchain.generate(2) await asyncio.wait_for(self.on_header(207), 3.0) self.assertEqual(self.ledger.headers.height, 207) await self.assertBlockHash(206) await self.assertBlockHash(207) self.assertEqual(1, bp.reorg_count_metric._samples()[0][2]) # invalidate current block, move forward 3 await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode()) await self.blockchain.generate(3) await asyncio.wait_for(self.on_header(208), 3.0) self.assertEqual(self.ledger.headers.height, 208) await self.assertBlockHash(206) await self.assertBlockHash(207) await self.assertBlockHash(208) self.assertEqual(2, bp.reorg_count_metric._samples()[0][2]) await self.blockchain.generate(3) await asyncio.wait_for(self.on_header(211), 3.0) await self.assertBlockHash(209) await self.assertBlockHash(210) await self.assertBlockHash(211) still_valid = await self.daemon.jsonrpc_stream_create( 'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(still_valid) await self.blockchain.generate(1) await asyncio.wait_for(self.on_header(212), 1.0) claim_id = still_valid.outputs[0].claim_id c1 = (await self.resolve(f'still-valid#{claim_id}'))['claim_id'] c2 = (await self.resolve(f'still-valid#{claim_id[:2]}'))['claim_id'] c3 = (await self.resolve(f'still-valid'))['claim_id'] self.assertTrue(c1 == c2 == c3) abandon_tx = await self.daemon.jsonrpc_stream_abandon(claim_id=claim_id) await self.blockchain.generate(1) await asyncio.wait_for(self.on_header(213), 1.0) c1 = await self.resolve(f'still-valid#{still_valid.outputs[0].claim_id}') c2 = await self.daemon.jsonrpc_resolve([f'still-valid#{claim_id[:2]}']) c3 = await self.daemon.jsonrpc_resolve([f'still-valid']) async def test_reorg_change_claim_height(self): # sanity check result = await self.resolve('hovercraft') # TODO: do these for claim_search and resolve both self.assertIn('error', result) still_valid = await self.daemon.jsonrpc_stream_create( 'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(still_valid) await self.generate(1) # create a claim and verify it's returned by claim_search self.assertEqual(self.ledger.headers.height, 207) await self.assertBlockHash(207) broadcast_tx = await self.daemon.jsonrpc_stream_create( 'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(broadcast_tx) await self.generate(1) await self.ledger.wait(broadcast_tx, self.blockchain.block_expected) self.assertEqual(self.ledger.headers.height, 208) await self.assertBlockHash(208) claim = await self.resolve('hovercraft') self.assertEqual(claim['txid'], broadcast_tx.id) self.assertEqual(claim['height'], 208) # check that our tx is in block 208 as returned by lbrycrdd invalidated_block_hash = (await self.ledger.headers.hash(208)).decode() block_207 = await self.blockchain.get_block(invalidated_block_hash) self.assertIn(claim['txid'], block_207['tx']) self.assertEqual(208, claim['height']) # reorg the last block dropping our claim tx await self.blockchain.invalidate_block(invalidated_block_hash) await self.conductor.clear_mempool() await self.blockchain.generate(2) await asyncio.wait_for(self.on_header(209), 3.0) await self.assertBlockHash(207) await self.assertBlockHash(208) await self.assertBlockHash(209) # verify the claim was dropped from block 208 as returned by lbrycrdd reorg_block_hash = await self.blockchain.get_block_hash(208) self.assertNotEqual(invalidated_block_hash, reorg_block_hash) block_207 = await self.blockchain.get_block(reorg_block_hash) self.assertNotIn(claim['txid'], block_207['tx']) client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode() self.assertEqual(client_reorg_block_hash, reorg_block_hash) # verify the dropped claim is no longer returned by claim search self.assertDictEqual( {'error': {'name': 'NOT_FOUND', 'text': 'Could not find claim at "hovercraft".'}}, await self.resolve('hovercraft') ) # verify the claim published a block earlier wasn't also reverted self.assertEqual(207, (await self.resolve('still-valid'))['height']) # broadcast the claim in a different block new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode()) self.assertEqual(broadcast_tx.id, new_txid) await self.blockchain.generate(1) await asyncio.wait_for(self.on_header(210), 1.0) # verify the claim is in the new block and that it is returned by claim_search republished = await self.resolve('hovercraft') self.assertEqual(210, republished['height']) self.assertEqual(claim['claim_id'], republished['claim_id']) # this should still be unchanged self.assertEqual(207, (await self.resolve('still-valid'))['height']) async def test_reorg_drop_claim(self): # sanity check result = await self.resolve('hovercraft') # TODO: do these for claim_search and resolve both self.assertIn('error', result) still_valid = await self.daemon.jsonrpc_stream_create( 'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(still_valid) await self.generate(1) # create a claim and verify it's returned by claim_search self.assertEqual(self.ledger.headers.height, 207) await self.assertBlockHash(207) broadcast_tx = await self.daemon.jsonrpc_stream_create( 'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(broadcast_tx) await self.generate(1) await self.ledger.wait(broadcast_tx, self.blockchain.block_expected) self.assertEqual(self.ledger.headers.height, 208) await self.assertBlockHash(208) claim = await self.resolve('hovercraft') self.assertEqual(claim['txid'], broadcast_tx.id) self.assertEqual(claim['height'], 208) # check that our tx is in block 208 as returned by lbrycrdd invalidated_block_hash = (await self.ledger.headers.hash(208)).decode() block_207 = await self.blockchain.get_block(invalidated_block_hash) self.assertIn(claim['txid'], block_207['tx']) self.assertEqual(208, claim['height']) # reorg the last block dropping our claim tx await self.blockchain.invalidate_block(invalidated_block_hash) await self.conductor.clear_mempool() await self.blockchain.generate(2) # wait for the client to catch up and verify the reorg await asyncio.wait_for(self.on_header(209), 3.0) await self.assertBlockHash(207) await self.assertBlockHash(208) await self.assertBlockHash(209) # verify the claim was dropped from block 208 as returned by lbrycrdd reorg_block_hash = await self.blockchain.get_block_hash(208) self.assertNotEqual(invalidated_block_hash, reorg_block_hash) block_207 = await self.blockchain.get_block(reorg_block_hash) self.assertNotIn(claim['txid'], block_207['tx']) client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode() self.assertEqual(client_reorg_block_hash, reorg_block_hash) # verify the dropped claim is no longer returned by claim search self.assertDictEqual( {'error': {'name': 'NOT_FOUND', 'text': 'Could not find claim at "hovercraft".'}}, await self.resolve('hovercraft') ) # verify the claim published a block earlier wasn't also reverted self.assertEqual(207, (await self.resolve('still-valid'))['height']) # broadcast the claim in a different block new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode()) self.assertEqual(broadcast_tx.id, new_txid) await self.blockchain.generate(1) await asyncio.wait_for(self.on_header(210), 1.0) # verify the claim is in the new block and that it is returned by claim_search republished = await self.resolve('hovercraft') self.assertEqual(210, republished['height']) self.assertEqual(claim['claim_id'], republished['claim_id']) # this should still be unchanged self.assertEqual(207, (await self.resolve('still-valid'))['height']) ================================================ FILE: tests/integration/blockchain/test_network.py ================================================ import asyncio import hub from unittest.mock import Mock from hub.herald import HUB_PROTOCOL_VERSION from hub.herald.udp import StatusServer from hub.herald.session import LBRYElectrumX from hub.scribe.network import LBCRegTest from lbry.wallet.network import Network from lbry.wallet.orchstr8 import Conductor from lbry.wallet.orchstr8.node import SPVNode from lbry.wallet.rpc import RPCSession from lbry.testcase import IntegrationTestCase, AsyncioTestCase from lbry.conf import Config class NetworkTests(IntegrationTestCase): async def test_remote_height_updated_automagically(self): initial_height = self.ledger.network.remote_height await self.blockchain.generate(1) await self.ledger.network.on_header.first self.assertEqual(self.ledger.network.remote_height, initial_height + 1) async def test_server_features(self): self.assertDictEqual({ 'genesis_hash': LBCRegTest.GENESIS_HASH, 'hash_function': 'sha256', 'hosts': {}, 'protocol_max': '0.199.0', 'protocol_min': '0.54.0', 'pruning': None, 'description': '', 'payment_address': '', 'donation_address': '', 'daily_fee': '0', 'server_version': HUB_PROTOCOL_VERSION, 'trending_algorithm': 'fast_ar', }, await self.ledger.network.get_server_features()) # await self.conductor.spv_node.stop() payment_address, donation_address = await self.account.get_addresses(limit=2) original_address = self.conductor.spv_node.server.env.payment_address original_donation_address = self.conductor.spv_node.server.env.donation_address original_description = self.conductor.spv_node.server.env.description original_daily_fee = self.conductor.spv_node.server.env.daily_fee self.conductor.spv_node.server.env.payment_address = payment_address self.conductor.spv_node.server.env.donation_address = donation_address self.conductor.spv_node.server.env.description = 'Fastest server in the west.' self.conductor.spv_node.server.env.daily_fee = '42' LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env) # await self.ledger.network.on_connected.first self.assertDictEqual({ 'genesis_hash': LBCRegTest.GENESIS_HASH, 'hash_function': 'sha256', 'hosts': {}, 'protocol_max': '0.199.0', 'protocol_min': '0.54.0', 'pruning': None, 'description': 'Fastest server in the west.', 'payment_address': payment_address, 'donation_address': donation_address, 'daily_fee': '42', 'server_version': HUB_PROTOCOL_VERSION, 'trending_algorithm': 'fast_ar', }, await self.ledger.network.get_server_features()) # cleanup the changes since the attributes are set on the class self.conductor.spv_node.server.env.payment_address = original_address self.conductor.spv_node.server.env.donation_address = original_donation_address self.conductor.spv_node.server.env.description = original_description self.conductor.spv_node.server.env.daily_fee = original_daily_fee LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env) class ReconnectTests(IntegrationTestCase): async def test_multiple_servers(self): # we have a secondary node that connects later, so node2 = SPVNode(node_number=2) await node2.start(self.blockchain) self.ledger.network.config['explicit_servers'].append((node2.hostname, node2.port)) self.ledger.network.config['explicit_servers'].reverse() self.assertEqual(50002, self.ledger.network.client.server[1]) await self.ledger.stop() await self.ledger.start() self.assertTrue(self.ledger.network.is_connected) self.assertEqual(50003, self.ledger.network.client.server[1]) await node2.stop(True) self.assertFalse(self.ledger.network.is_connected) await self.ledger.resolve([], ['derp']) self.assertEqual(50002, self.ledger.network.client.server[1]) async def test_direct_sync(self): await self.ledger.stop() initial_height = self.ledger.local_height_including_downloaded_height await self.blockchain.generate(100) while self.conductor.spv_node.server.session_manager.notified_height < initial_height + 100: await asyncio.sleep(0.1) self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height) await self.ledger.headers.open() await self.ledger.network.start() await self.ledger.network.on_connected.first await self.ledger.initial_headers_sync() self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height) async def test_connection_drop_still_receives_events_after_reconnected(self): address1 = await self.account.receiving.get_or_create_usable_address() # disconnect and send a new tx, should reconnect and get it self.ledger.network.client.transport.close() self.assertFalse(self.ledger.network.is_connected) await self.ledger.resolve([], ['derp']) sendtxid = await self.send_to_address_and_wait(address1, 1.1337, 1) self.assertLess(self.ledger.network.client.response_time, 1) # response time properly set lower, we are fine await self.assertBalance(self.account, '1.1337') # is it real? are we rich!? let me see this tx... d = self.ledger.network.get_transaction(sendtxid) # what's that smoke on my ethernet cable? oh no! master_client = self.ledger.network.client self.ledger.network.client.connection_lost(Exception()) with self.assertRaises(asyncio.TimeoutError): await d self.assertIsNone(master_client.response_time) # response time unknown as it failed # rich but offline? no way, no water, let's retry with self.assertRaisesRegex(ConnectionError, 'connection is not available'): await self.ledger.network.get_transaction(sendtxid) # * goes to pick some water outside... * time passes by and another donation comes in sendtxid = await self.blockchain.send_to_address(address1, 42) await self.blockchain.generate(1) # (this is just so the test doesn't hang forever if it doesn't reconnect) if not self.ledger.network.is_connected: await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=10.0) # omg, the burned cable still works! torba is fire proof! await self.ledger.on_header.where(self.blockchain.is_expected_block) await self.ledger.network.get_transaction(sendtxid) async def test_timeout_then_reconnect(self): # tests that it connects back after some failed attempts await self.conductor.spv_node.stop() self.assertFalse(self.ledger.network.is_connected) await asyncio.sleep(0.2) # let it retry and fail once await self.conductor.spv_node.start(self.conductor.lbcwallet_node) await self.ledger.network.on_connected.first self.assertTrue(self.ledger.network.is_connected) async def test_timeout_and_concurrency_propagated_from_config(self): conf = Config() self.assertEqual(self.ledger.network.client.timeout, 30) self.assertEqual(self.ledger.network.client.concurrency, 32) conf.hub_timeout = 123.0 conf.concurrent_hub_requests = 42 conf.known_hubs = self.ledger.config['known_hubs'] conf.wallet_dir = self.ledger.config['data_path'] self.manager.config = conf await self.manager.reset() self.assertEqual(self.ledger.network.client.timeout, 123) self.assertEqual(self.ledger.network.client.concurrency, 42) # async def test_online_but_still_unavailable(self): # # Edge case. See issue #2445 for context # self.assertIsNotNone(self.ledger.network.session_pool.fastest_session) # for session in self.ledger.network.session_pool.sessions: # session.response_time = None # self.assertIsNone(self.ledger.network.session_pool.fastest_session) class UDPServerFailDiscoveryTest(AsyncioTestCase): async def test_wallet_connects_despite_lack_of_udp(self): conductor = Conductor() conductor.spv_node.udp_port = '0' await conductor.start_lbcd() self.addCleanup(conductor.stop_lbcd) await conductor.start_lbcwallet() self.addCleanup(conductor.stop_lbcwallet) await conductor.start_spv() self.addCleanup(conductor.stop_spv) self.assertFalse(conductor.spv_node.server.status_server.is_running) await asyncio.wait_for(conductor.start_wallet(), timeout=5) self.addCleanup(conductor.stop_wallet) self.assertTrue(conductor.wallet_node.ledger.network.is_connected) class ServerPickingTestCase(AsyncioTestCase): async def _make_udp_server(self, port, latency) -> StatusServer: s = StatusServer() await s.start(0, b'\x00' * 32, 'US', '127.0.0.1', port, True) s.set_available() sendto = s._protocol.transport.sendto def mock_sendto(data, addr): self.loop.call_later(latency, sendto, data, addr) s._protocol.transport.sendto = mock_sendto self.addCleanup(s.stop) return s async def _make_fake_server(self, latency=1.0, port=1): # local fake server with artificial latency class FakeSession(RPCSession): async def handle_request(self, request): await asyncio.sleep(latency) if request.method == 'server.version': return tuple(request.args) return {'height': 1} server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port) self.addCleanup(server.close) await self._make_udp_server(port, latency) return '127.0.0.1', port async def _make_bad_server(self, port=42420): async def echo(reader, writer): while True: writer.write(await reader.read()) server = await asyncio.start_server(echo, host='127.0.0.1', port=port) self.addCleanup(server.close) await self._make_udp_server(port, 0) return '127.0.0.1', port async def test_pick_fastest(self): ledger = Mock(config={ 'default_servers': [ # fast but unhealthy, should be discarded # await self._make_bad_server(), ('localhost', 1), ('example.that.doesnt.resolve', 9000), await self._make_fake_server(latency=1.0, port=1340), await self._make_fake_server(latency=0.1, port=1337), await self._make_fake_server(latency=0.4, port=1339), ], 'connect_timeout': 3 }) network = Network(ledger) self.addCleanup(network.stop) await network.start() await asyncio.wait_for(network.on_connected.first, timeout=10) self.assertTrue(network.is_connected) self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337)) # self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions])) # ensure we are connected to all of them after a while # await asyncio.sleep(1) # self.assertEqual(len(list(network.session_pool.available_sessions)), 3) ================================================ FILE: tests/integration/blockchain/test_purchase_command.py ================================================ from typing import Optional from lbry.testcase import CommandTestCase from lbry.schema.purchase import Purchase from lbry.wallet.transaction import Transaction from lbry.wallet.dewies import lbc_to_dewies, dewies_to_lbc class PurchaseCommandTests(CommandTestCase): async def asyncSetUp(self): await super().asyncSetUp() self.merchant_address = await self.blockchain.get_raw_change_address() async def priced_stream( self, name='stream', price: Optional[str] = '0.2', currency='LBC', mine=False ) -> Transaction: kwargs = {} if price and currency: kwargs = { 'fee_amount': price, 'fee_currency': currency, 'fee_address': self.merchant_address } if not mine: kwargs['claim_address'] = self.merchant_address file_path = self.create_upload_file(data=b'high value content') tx = await self.daemon.jsonrpc_stream_create( name, '0.01', file_path=file_path, **kwargs ) await self.ledger.wait(tx) await self.generate(1) await self.ledger.wait(tx) await self.daemon.jsonrpc_file_delete(claim_name=name) return tx async def create_purchase(self, name, price): stream = await self.priced_stream(name, price) claim_id = stream.outputs[0].claim_id purchase = await self.daemon.jsonrpc_purchase_create(claim_id) await self.ledger.wait(purchase) return claim_id async def assertStreamPurchased(self, stream: Transaction, operation): await self.account.release_all_outputs() buyer_balance = await self.account.get_balance() merchant_balance = lbc_to_dewies(await self.blockchain.get_balance()) pre_purchase_count = (await self.daemon.jsonrpc_purchase_list())['total_items'] purchase = await operation() stream_txo, purchase_txo = stream.outputs[0], purchase.outputs[0] stream_fee = stream_txo.claim.stream.fee self.assertEqual(stream_fee.dewies, purchase_txo.amount) self.assertEqual(stream_fee.address, purchase_txo.get_address(self.ledger)) await self.ledger.wait(purchase) await self.generate(1) merchant_balance += lbc_to_dewies('1.0') # block reward await self.ledger.wait(purchase) self.assertEqual( await self.account.get_balance(), buyer_balance - (purchase.input_sum-purchase.outputs[2].amount)) self.assertEqual( str(float(await self.blockchain.get_balance())), dewies_to_lbc(merchant_balance + purchase_txo.amount) ) purchases = await self.daemon.jsonrpc_purchase_list() self.assertEqual(purchases['total_items'], pre_purchase_count+1) tx = purchases['items'][0].tx_ref.tx self.assertEqual(len(tx.outputs), 3) # purchase txo, purchase data, change txo0 = tx.outputs[0] txo1 = tx.outputs[1] self.assertEqual(txo0.purchase, txo1) # purchase txo has reference to purchase data self.assertTrue(txo1.is_purchase_data) self.assertTrue(txo1.can_decode_purchase_data) self.assertIsInstance(txo1.purchase_data, Purchase) self.assertEqual(txo1.purchase_data.claim_id, stream_txo.claim_id) async def test_purchasing(self): stream = await self.priced_stream() claim_id = stream.outputs[0].claim_id # explicit purchase of claim await self.assertStreamPurchased(stream, lambda: self.daemon.jsonrpc_purchase_create(claim_id)) # check that `get` doesn't purchase it again balance = await self.account.get_balance() response = await self.daemon.jsonrpc_get('lbry://stream') self.assertIsNone(response.content_fee) self.assertEqual(await self.account.get_balance(), balance) self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 1) # `get` does purchase a stream we don't have yet another_stream = await self.priced_stream('another') async def imagine_its_a_lambda(): response = await self.daemon.jsonrpc_get('lbry://another') return response.content_fee await self.assertStreamPurchased(another_stream, imagine_its_a_lambda) # purchase non-existent claim fails with self.assertRaisesRegex(Exception, "Could not find claim with claim_id"): await self.daemon.jsonrpc_purchase_create('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') # purchase stream with no price fails no_price_stream = await self.priced_stream('no_price_stream', price=None) with self.assertRaisesRegex(Exception, "does not have a purchase price"): await self.daemon.jsonrpc_purchase_create(no_price_stream.outputs[0].claim_id) # purchase claim you already own fails with self.assertRaisesRegex(Exception, "You already have a purchase for claim_id"): await self.daemon.jsonrpc_purchase_create(claim_id) # force purchasing claim you already own await self.assertStreamPurchased( stream, lambda: self.daemon.jsonrpc_purchase_create(claim_id, allow_duplicate_purchase=True) ) # purchase by uri abc_stream = await self.priced_stream('abc') await self.assertStreamPurchased(abc_stream, lambda: self.daemon.jsonrpc_purchase_create(url='lbry://abc')) # purchase without valid exchange rate fails erm = self.daemon.component_manager.get_component('exchange_rate_manager') for feed in erm.market_feeds: feed.last_check -= 10_000 with self.assertRaisesRegex(Exception, "Unable to convert 50 from USD to LBC"): await self.daemon.jsonrpc_purchase_create(claim_id, allow_duplicate_purchase=True) async def test_purchase_and_transaction_list(self): self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0) self.assertItemCount(await self.daemon.jsonrpc_transaction_list(), 1) claim_id1 = await self.create_purchase('a', '1.0') claim_id2 = await self.create_purchase('b', '1.0') result = await self.out(self.daemon.jsonrpc_purchase_list()) self.assertItemCount(await self.daemon.jsonrpc_transaction_list(), 5) self.assertItemCount(result, 2) self.assertEqual(result['items'][0]['type'], 'purchase') self.assertEqual(result['items'][0]['claim_id'], claim_id2) self.assertNotIn('claim', result['items'][0]) self.assertEqual(result['items'][1]['type'], 'purchase') self.assertEqual(result['items'][1]['claim_id'], claim_id1) self.assertNotIn('claim', result['items'][1]) result = await self.out(self.daemon.jsonrpc_purchase_list(resolve=True)) self.assertEqual(result['items'][0]['claim']['name'], 'b') self.assertEqual(result['items'][1]['claim']['name'], 'a') result = await self.daemon.jsonrpc_transaction_list() self.assertEqual(result['items'][0]['purchase_info'][0]['claim_id'], claim_id2) self.assertEqual(result['items'][2]['purchase_info'][0]['claim_id'], claim_id1) result = await self.claim_search(include_purchase_receipt=True) self.assertEqual(result[0]['claim_id'], result[0]['purchase_receipt']['claim_id']) self.assertEqual(result[1]['claim_id'], result[1]['purchase_receipt']['claim_id']) url = result[0]['canonical_url'] resolve = await self.resolve(url, include_purchase_receipt=True) self.assertEqual(result[0]['claim_id'], resolve['purchase_receipt']['claim_id']) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) await self.daemon.jsonrpc_get('lbry://a') await self.daemon.jsonrpc_get('lbry://b') files = await self.file_list() self.assertEqual(files[0]['claim_id'], files[0]['purchase_receipt']['claim_id']) self.assertEqual(files[1]['claim_id'], files[1]['purchase_receipt']['claim_id']) async def test_seller_can_spend_received_purchase_funds(self): self.merchant_address = await self.account.receiving.get_or_create_usable_address() daemon2 = await self.add_daemon() address2 = await daemon2.wallet_manager.default_account.receiving.get_or_create_usable_address() await self.send_to_address_and_wait(address2, 2, 1, ledger=daemon2.ledger) stream = await self.priced_stream('a', '1.0') await self.assertBalance(self.account, '9.987893') self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 1) purchase = await daemon2.jsonrpc_purchase_create(stream.outputs[0].claim_id) await self.ledger.wait(purchase) await self.generate(1) await self.ledger.wait(purchase) # confirm that available and reserved take into account purchase received self.assertEqual(await self.account.get_detailed_balance(), { 'total': 1099789300, 'available': 1098789300, 'reserved': 1000000, 'reserved_subtotals': {'claims': 1000000, 'supports': 0, 'tips': 0} }) self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 2) spend = await self.daemon.jsonrpc_wallet_send('10.5', address2) await self.ledger.wait(spend) await self.generate(1) await self.ledger.wait(spend) await self.assertBalance(self.account, '0.487695') self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 1) async def test_owner_not_required_purchase_own_content(self): await self.priced_stream(mine=True) # check that `get` doesn't purchase own claim balance = await self.account.get_balance() response = await self.daemon.jsonrpc_get('lbry://stream') self.assertIsNone(response.content_fee) self.assertEqual(await self.account.get_balance(), balance) self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0) ================================================ FILE: tests/integration/blockchain/test_sync.py ================================================ import asyncio import logging from lbry.testcase import IntegrationTestCase, WalletNode from lbry.constants import CENT from lbry.wallet import WalletManager, RegTestLedger, Transaction, Output class SyncTests(IntegrationTestCase): VERBOSITY = logging.WARN def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.api_port = 5280 self.started_nodes = [] async def asyncTearDown(self): for node in self.started_nodes: try: await node.stop(cleanup=True) except Exception as e: print(e) await super().asyncTearDown() async def make_wallet_node(self, seed=None): self.api_port += 1 wallet_node = WalletNode(WalletManager, RegTestLedger, port=self.api_port) await wallet_node.start(self.conductor.spv_node, seed) self.started_nodes.append(wallet_node) return wallet_node async def test_nodes_with_same_account_stay_in_sync(self): # destination node/account for receiving TXs node0 = await self.make_wallet_node() account0 = node0.account # main node/account creating TXs node1 = self.wallet_node account1 = self.wallet_node.account # mirror node/account, expected to reflect everything in main node as it happens node2 = await self.make_wallet_node(account1.seed) account2 = node2.account self.assertNotEqual(account0.id, account1.id) self.assertEqual(account1.id, account2.id) await self.assertBalance(account0, '0.0') await self.assertBalance(account1, '0.0') await self.assertBalance(account2, '0.0') self.assertEqual(await account0.get_address_count(chain=0), 20) self.assertEqual(await account1.get_address_count(chain=0), 20) self.assertEqual(await account2.get_address_count(chain=0), 20) self.assertEqual(await account1.get_address_count(chain=1), 6) self.assertEqual(await account2.get_address_count(chain=1), 6) # check that main node and mirror node generate 5 address to fill gap fifth_address = (await account1.receiving.get_addresses())[4] await self.blockchain.send_to_address(fifth_address, 1.00) await asyncio.wait([ account1.ledger.on_address.first, account2.ledger.on_address.first ]) self.assertEqual(await account1.get_address_count(chain=0), 25) self.assertEqual(await account2.get_address_count(chain=0), 25) await self.assertBalance(account1, '1.0') await self.assertBalance(account2, '1.0') await self.generate(1) # pay 0.01 from main node to receiving node, would have increased change addresses address0 = (await account0.receiving.get_addresses())[0] hash0 = self.ledger.address_to_hash160(address0) tx = await Transaction.create( [], [Output.pay_pubkey_hash(CENT, hash0)], [account1], account1 ) await self.broadcast(tx) await asyncio.wait([ account0.ledger.wait(tx), account1.ledger.wait(tx), account2.ledger.wait(tx), ]) await self.generate(1) await asyncio.wait([ account0.ledger.wait(tx), account1.ledger.wait(tx), account2.ledger.wait(tx), ]) self.assertEqual(await account0.get_address_count(chain=0), 21) self.assertGreater(await account1.get_address_count(chain=1), 6) self.assertGreater(await account2.get_address_count(chain=1), 6) await self.assertBalance(account0, '0.01') await self.assertBalance(account1, '0.989876') await self.assertBalance(account2, '0.989876') await self.generate(1) # create a new mirror node and see if it syncs to same balance from scratch node3 = await self.make_wallet_node(account1.seed) account3 = node3.account await self.assertBalance(account3, '0.989876') ================================================ FILE: tests/integration/blockchain/test_wallet_commands.py ================================================ import asyncio import json import string from binascii import unhexlify from random import Random from lbry.wallet import ENCRYPT_ON_DISK from lbry.error import InvalidPasswordError from lbry.testcase import CommandTestCase from lbry.wallet.dewies import dict_values_to_lbc class WalletCommands(CommandTestCase): async def test_wallet_create_and_add_subscribe(self): session = next(iter(self.conductor.spv_node.server.session_manager.sessions.values())) self.assertEqual(len(session.hashX_subs), 27) wallet = await self.daemon.jsonrpc_wallet_create('foo', create_account=True, single_key=True) self.assertEqual(len(session.hashX_subs), 28) await self.daemon.jsonrpc_wallet_remove(wallet.id) self.assertEqual(len(session.hashX_subs), 27) await self.daemon.jsonrpc_wallet_add(wallet.id) self.assertEqual(len(session.hashX_subs), 28) async def test_wallet_syncing_status(self): address = await self.daemon.jsonrpc_address_unused() await self.ledger._update_tasks.done.wait() self.assertFalse(self.daemon.jsonrpc_wallet_status()['is_syncing']) await self.send_to_address_and_wait(address, 1) await self.ledger._update_tasks.started.wait() self.assertTrue(self.daemon.jsonrpc_wallet_status()['is_syncing']) await self.ledger._update_tasks.done.wait() self.assertFalse(self.daemon.jsonrpc_wallet_status()['is_syncing']) wallet = self.daemon.component_manager.get_actual_component('wallet') wallet_manager = wallet.wallet_manager # when component manager hasn't started yet wallet.wallet_manager = None self.assertEqual( {'is_encrypted': None, 'is_syncing': None, 'is_locked': None}, self.daemon.jsonrpc_wallet_status() ) wallet.wallet_manager = wallet_manager self.assertEqual( {'is_encrypted': False, 'is_syncing': False, 'is_locked': False}, self.daemon.jsonrpc_wallet_status() ) async def test_wallet_reconnect(self): status = await self.daemon.jsonrpc_status() self.assertEqual(len(status['wallet']['servers']), 1) self.assertEqual(status['wallet']['servers'][0]['port'], 50002) await self.conductor.spv_node.stop() self.conductor.spv_node.port = 54320 await self.conductor.spv_node.start(self.conductor.lbcwallet_node) status = await self.daemon.jsonrpc_status() self.assertEqual(len(status['wallet']['servers']), 0) self.daemon.jsonrpc_settings_set('lbryum_servers', ['localhost:54320']) await self.daemon.jsonrpc_wallet_reconnect() status = await self.daemon.jsonrpc_status() self.assertEqual(len(status['wallet']['servers']), 1) self.assertEqual(status['wallet']['servers'][0]['port'], 54320) async def test_sending_to_scripthash_address(self): bal = await self.blockchain.get_balance() await self.assertBalance(self.account, '10.0') p2sh_address1 = await self.blockchain.get_new_address(self.blockchain.P2SH_SEGWIT_ADDRESS) tx = await self.account_send('2.0', p2sh_address1) self.assertEqual(tx['outputs'][0]['address'], p2sh_address1) self.assertEqual(await self.blockchain.get_balance(), str(float(bal)+3)) # +1 lbc for confirm block await self.assertBalance(self.account, '7.999877') await self.wallet_send('3.0', p2sh_address1) self.assertEqual(await self.blockchain.get_balance(), str(float(bal)+7)) # +1 lbc for confirm block await self.assertBalance(self.account, '4.999754') async def test_balance_caching(self): account2 = await self.daemon.jsonrpc_account_create("Tip-er") address2 = await self.daemon.jsonrpc_address_unused(account2.id) await self.send_to_address_and_wait(address2, 10, 2) await self.ledger.tasks_are_done() # don't mess with the query count while we need it wallet_balance = self.daemon.jsonrpc_wallet_balance ledger = self.ledger query_count = self.ledger.db.db.query_count expected = { 'total': '20.0', 'available': '20.0', 'reserved': '0.0', 'reserved_subtotals': {'claims': '0.0', 'supports': '0.0', 'tips': '0.0'} } self.assertIsNone(ledger._balance_cache.get(self.account.id)) query_count += 2 balance = await wallet_balance() self.assertEqual(self.ledger.db.db.query_count, query_count) self.assertEqual(balance, expected) self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0') self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0') # calling again uses cache balance = await wallet_balance() self.assertEqual(self.ledger.db.db.query_count, query_count) self.assertEqual(balance, expected) self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0') self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0') await self.stream_create() await self.generate(1) expected = { 'total': '19.979893', 'available': '18.979893', 'reserved': '1.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '0.0', 'tips': '0.0'} } # on_transaction event reset balance cache query_count = self.ledger.db.db.query_count self.assertEqual(await wallet_balance(), expected) query_count += 1 # only one of the accounts changed self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '9.979893') self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0') self.assertEqual(self.ledger.db.db.query_count, query_count) async def test_granular_balances(self): account2 = await self.daemon.jsonrpc_account_create("Tip-er") wallet2 = await self.daemon.jsonrpc_wallet_create('foo', create_account=True) account3 = wallet2.default_account address3 = await self.daemon.jsonrpc_address_unused(account3.id, wallet2.id) await self.send_to_address_and_wait(address3, 1, 1) account_balance = self.daemon.jsonrpc_account_balance wallet_balance = self.daemon.jsonrpc_wallet_balance expected = { 'total': '10.0', 'available': '10.0', 'reserved': '0.0', 'reserved_subtotals': {'claims': '0.0', 'supports': '0.0', 'tips': '0.0'} } self.assertEqual(await account_balance(), expected) self.assertEqual(await wallet_balance(), expected) # claim with update + supporting our own claim stream1 = await self.stream_create('granularity', '3.0') await self.stream_update(self.get_claim_id(stream1), data=b'news', bid='1.0') await self.support_create(self.get_claim_id(stream1), '2.0') expected = { 'total': '9.977534', 'available': '6.977534', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} } self.assertEqual(await account_balance(), expected) self.assertEqual(await wallet_balance(), expected) address2 = await self.daemon.jsonrpc_address_unused(account2.id) # send lbc to someone else tx = await self.daemon.jsonrpc_account_send('1.0', address2, blocking=True) await self.confirm_tx(tx.id) self.assertEqual(await account_balance(), { 'total': '8.97741', 'available': '5.97741', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) self.assertEqual(await wallet_balance(), { 'total': '9.97741', 'available': '6.97741', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) # tip received support1 = await self.support_create( self.get_claim_id(stream1), '0.3', tip=True, wallet_id=wallet2.id ) self.assertEqual(await account_balance(), { 'total': '9.27741', 'available': '5.97741', 'reserved': '3.3', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'} }) self.assertEqual(await wallet_balance(), { 'total': '10.27741', 'available': '6.97741', 'reserved': '3.3', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'} }) # tip claimed tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0, blocking=True) await self.confirm_tx(tx.id) self.assertEqual(await account_balance(), { 'total': '9.277303', 'available': '6.277303', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) self.assertEqual(await wallet_balance(), { 'total': '10.277303', 'available': '7.277303', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) stream2 = await self.stream_create( 'granularity-is-cool', '0.1', account_id=account2.id, funding_account_ids=[account2.id] ) # tip another claim await self.support_create( self.get_claim_id(stream2), '0.2', tip=True, wallet_id=wallet2.id ) self.assertEqual(await account_balance(), { 'total': '9.277303', 'available': '6.277303', 'reserved': '3.0', 'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'} }) self.assertEqual(await wallet_balance(), { 'total': '10.439196', 'available': '7.139196', 'reserved': '3.3', 'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'} }) class WalletEncryptionAndSynchronization(CommandTestCase): SEED = ( "carbon smart garage balance margin twelve chest " "sword toast envelope bottom stomach absent" ) async def asyncSetUp(self): await super().asyncSetUp() self.daemon2 = await self.add_daemon( seed="chest sword toast envelope bottom stomach absent " "carbon smart garage balance margin twelve" ) address = (await self.daemon2.wallet_manager.default_account.receiving.get_addresses(limit=1, only_usable=True))[0] await self.send_to_address_and_wait(address, 1, 1, ledger=self.daemon2.ledger) def assertWalletEncrypted(self, wallet_path, encrypted): with open(wallet_path) as opened: wallet = json.load(opened) self.assertEqual(wallet['accounts'][0]['private_key'][1:4] != 'prv', encrypted) async def test_sync(self): daemon, daemon2 = self.daemon, self.daemon2 # Preferences self.assertFalse(daemon.jsonrpc_preference_get()) self.assertFalse(daemon2.jsonrpc_preference_get()) daemon.jsonrpc_preference_set("fruit", '["peach", "apricot"]') daemon.jsonrpc_preference_set("one", "1") daemon.jsonrpc_preference_set("conflict", "1") daemon2.jsonrpc_preference_set("another", "A") await asyncio.sleep(1) # these preferences will win after merge since they are "newer" daemon2.jsonrpc_preference_set("two", "2") daemon2.jsonrpc_preference_set("conflict", "2") daemon.jsonrpc_preference_set("another", "B") self.assertDictEqual(daemon.jsonrpc_preference_get(), { "one": "1", "conflict": "1", "another": "B", "fruit": ["peach", "apricot"] }) self.assertDictEqual(daemon2.jsonrpc_preference_get(), { "two": "2", "conflict": "2", "another": "A" }) self.assertItemCount(await daemon.jsonrpc_account_list(), 1) data = await daemon2.jsonrpc_sync_apply('password') await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True) self.assertItemCount(await daemon.jsonrpc_account_list(), 2) self.assertDictEqual( # "two" key added and "conflict" value changed to "2" daemon.jsonrpc_preference_get(), {"one": "1", "two": "2", "conflict": "2", "another": "B", "fruit": ["peach", "apricot"]} ) # Channel Certificate # non-deterministic channel self.daemon2.wallet_manager.default_account.channel_keys['mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8'] = ( '-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95' '466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh' '0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS' 'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n' ) channel = await self.create_nondeterministic_channel('@foo', '0.1', unhexlify( '3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1' '66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16' 'a97a6d6a4a8effd29d748901bb9789352519cd00b13d' ), self.daemon2, blocking=True) await self.confirm_tx(channel['txid'], self.daemon2.ledger) # both daemons will have the channel but only one has the cert so far self.assertItemCount(await daemon.jsonrpc_channel_list(), 1) self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0) self.assertItemCount(await daemon2.jsonrpc_channel_list(), 1) self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1) data = await daemon2.jsonrpc_sync_apply('password') await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True) # both daemons have the cert after sync'ing self.assertEqual( daemon2.wallet_manager.default_account.channel_keys, daemon.wallet_manager.default_wallet.accounts[1].channel_keys ) async def test_encryption_and_locking(self): daemon = self.daemon wallet = daemon.wallet_manager.default_wallet wallet.save() self.assertEqual(daemon.jsonrpc_wallet_status(), { 'is_locked': False, 'is_encrypted': False, 'is_syncing': False }) self.assertIsNone(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK)) self.assertWalletEncrypted(wallet.storage.path, False) # can't lock an unencrypted account with self.assertRaisesRegex(AssertionError, "Cannot lock an unencrypted wallet, encrypt first."): daemon.jsonrpc_wallet_lock() # safe to call unlock and decrypt, they are no-ops at this point await daemon.jsonrpc_wallet_unlock('password') # already unlocked daemon.jsonrpc_wallet_decrypt() # already not encrypted daemon.jsonrpc_wallet_encrypt('password') self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True, 'is_syncing': False}) self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True}) self.assertWalletEncrypted(wallet.storage.path, True) daemon.jsonrpc_wallet_lock() self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': True, 'is_encrypted': True, 'is_syncing': False}) # can't sign transactions with locked wallet with self.assertRaises(AssertionError): await daemon.jsonrpc_channel_create('@foo', '1.0') await daemon.jsonrpc_wallet_unlock('password') self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True, 'is_syncing': False}) await daemon.jsonrpc_channel_create('@foo', '1.0') daemon.jsonrpc_wallet_decrypt() self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False, 'is_syncing': False}) self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': False}) self.assertWalletEncrypted(wallet.storage.path, False) async def test_encryption_with_imported_channel(self): daemon, daemon2 = self.daemon, self.daemon2 channel = await self.channel_create() exported = await daemon.jsonrpc_channel_export(self.get_claim_id(channel)) await daemon2.jsonrpc_channel_import(exported) self.assertTrue(daemon2.jsonrpc_wallet_encrypt('password')) self.assertTrue(daemon2.jsonrpc_wallet_lock()) self.assertTrue(await daemon2.jsonrpc_wallet_unlock("password")) self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True, 'is_syncing': False}) async def test_locking_unlocking_does_not_break_deterministic_channels(self): self.assertTrue(self.daemon.jsonrpc_wallet_encrypt("password")) self.assertTrue(self.daemon.jsonrpc_wallet_lock()) self.account.deterministic_channel_keys._private_key = None self.assertTrue(await self.daemon.jsonrpc_wallet_unlock("password")) await self.channel_create() async def test_sync_with_encryption_and_password_change(self): daemon, daemon2 = self.daemon, self.daemon2 wallet, wallet2 = daemon.wallet_manager.default_wallet, daemon2.wallet_manager.default_wallet self.assertEqual(wallet2.encryption_password, None) self.assertEqual(wallet2.encryption_password, None) daemon.jsonrpc_wallet_encrypt('password') self.assertEqual(wallet.encryption_password, 'password') data = await daemon2.jsonrpc_sync_apply('password2') # sync_apply doesn't save password if encrypt-on-disk is False self.assertEqual(wallet2.encryption_password, None) # Need to use new password2 in sync_apply. Attempts with other passwords # should fail consistently with InvalidPasswordError. random = Random('password') for i in range(200): bad_guess = ''.join(random.choices(string.digits + string.ascii_letters + string.punctuation, k=40)) self.assertNotEqual(bad_guess, 'password2') with self.assertRaises(InvalidPasswordError): await daemon.jsonrpc_sync_apply(bad_guess, data=data['data'], blocking=True) await daemon.jsonrpc_sync_apply('password2', data=data['data'], blocking=True) # sync_apply with new password2 also sets it as new local password self.assertEqual(wallet.encryption_password, 'password2') self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True, 'is_syncing': True}) self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True}) self.assertWalletEncrypted(wallet.storage.path, True) # check new password is active daemon.jsonrpc_wallet_lock() self.assertFalse(await daemon.jsonrpc_wallet_unlock('password')) self.assertTrue(await daemon.jsonrpc_wallet_unlock('password2')) # propagate disk encryption to daemon2 data = await daemon.jsonrpc_sync_apply('password3') # sync_apply (even with no data) on wallet with encrypt-on-disk updates local password self.assertEqual(wallet.encryption_password, 'password3') self.assertEqual(wallet2.encryption_password, None) await daemon2.jsonrpc_sync_apply('password3', data=data['data'], blocking=True) # the other device got new password and on disk encryption self.assertEqual(wallet2.encryption_password, 'password3') self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True, 'is_syncing': True}) self.assertEqual(daemon2.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True}) self.assertWalletEncrypted(wallet2.storage.path, True) daemon2.jsonrpc_wallet_lock() self.assertTrue(await daemon2.jsonrpc_wallet_unlock('password3')) async def test_wallet_import_and_export(self): daemon, daemon2 = self.daemon, self.daemon2 # Preferences self.assertFalse(daemon.jsonrpc_preference_get()) self.assertFalse(daemon2.jsonrpc_preference_get()) daemon.jsonrpc_preference_set("fruit", '["peach", "apricot"]') daemon.jsonrpc_preference_set("one", "1") daemon.jsonrpc_preference_set("conflict", "1") daemon2.jsonrpc_preference_set("another", "A") await asyncio.sleep(1) # these preferences will win after merge since they are "newer" daemon2.jsonrpc_preference_set("two", "2") daemon2.jsonrpc_preference_set("conflict", "2") daemon.jsonrpc_preference_set("another", "B") self.assertDictEqual(daemon.jsonrpc_preference_get(), { "one": "1", "conflict": "1", "another": "B", "fruit": ["peach", "apricot"] }) self.assertDictEqual(daemon2.jsonrpc_preference_get(), { "two": "2", "conflict": "2", "another": "A" }) self.assertItemCount(await daemon.jsonrpc_account_list(), 1) data = await daemon2.jsonrpc_wallet_export(password='password') await daemon.jsonrpc_wallet_import(data=data, password='password', blocking=True) self.assertItemCount(await daemon.jsonrpc_account_list(), 2) self.assertDictEqual( # "two" key added and "conflict" value changed to "2" daemon.jsonrpc_preference_get(), {"one": "1", "two": "2", "conflict": "2", "another": "B", "fruit": ["peach", "apricot"]} ) # Channel Certificate # non-deterministic channel self.daemon2.wallet_manager.default_account.channel_keys['mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8'] = ( '-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95' '466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh' '0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS' 'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n' ) channel = await self.create_nondeterministic_channel('@foo', '0.1', unhexlify( '3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1' '66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16' 'a97a6d6a4a8effd29d748901bb9789352519cd00b13d' ), self.daemon2, blocking=True) await self.confirm_tx(channel['txid'], self.daemon2.ledger) # both daemons will have the channel but only one has the cert so far self.assertItemCount(await daemon.jsonrpc_channel_list(), 1) self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0) self.assertItemCount(await daemon2.jsonrpc_channel_list(), 1) self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1) data = await daemon2.jsonrpc_wallet_export(password='password') await daemon.jsonrpc_wallet_import(data=data, password='password', blocking=True) # both daemons have the cert after sync'ing self.assertEqual( daemon2.wallet_manager.default_account.channel_keys, daemon.wallet_manager.default_wallet.accounts[1].channel_keys ) # test without passwords data = await daemon2.jsonrpc_wallet_export() json_data = json.loads(data) self.assertEqual(json_data["name"], "Wallet") self.assertNotIn("four", json_data["preferences"]) json_data["preferences"]["four"] = {"value": 4, "ts": 0} await daemon.jsonrpc_wallet_import(data=json.dumps(json_data), blocking=True) self.assertEqual(daemon.jsonrpc_preference_get("four"), {"four": 4}) # if password is empty string, export is encrypted data = await daemon2.jsonrpc_wallet_export(password="") self.assertNotEqual(data[0], "{") # if password is empty string, import is decrypted await daemon.jsonrpc_wallet_import(data, password="") ================================================ FILE: tests/integration/blockchain/test_wallet_server_sessions.py ================================================ import asyncio from hub.herald import HUB_PROTOCOL_VERSION from hub.herald.session import LBRYElectrumX from lbry.error import InsufficientFundsError, ServerPaymentFeeAboveMaxAllowedError from lbry.wallet.network import ClientSession from lbry.wallet.rpc import RPCError from lbry.testcase import IntegrationTestCase, CommandTestCase from lbry.wallet.orchstr8.node import SPVNode class TestSessions(IntegrationTestCase): """ Tests that server cleans up stale connections after session timeout and client times out too. """ async def test_session_bloat_from_socket_timeout(self): await self.conductor.stop_spv() await self.ledger.stop() self.conductor.spv_node.session_timeout = 1 await self.conductor.start_spv() session = ClientSession( network=None, server=(self.conductor.spv_node.hostname, self.conductor.spv_node.port), timeout=0.2 ) await session.create_connection() await session.send_request('server.banner', ()) self.assertEqual(len(self.conductor.spv_node.server.session_manager.sessions), 1) self.assertFalse(session.is_closing()) await asyncio.sleep(1.1) with self.assertRaises(asyncio.TimeoutError): await session.send_request('server.banner', ()) self.assertTrue(session.is_closing()) self.assertEqual(len(self.conductor.spv_node.server.session_manager.sessions), 0) async def test_proper_version(self): info = await self.ledger.network.get_server_features() self.assertEqual(HUB_PROTOCOL_VERSION, info['server_version']) async def test_client_errors(self): # Goal is ensuring thsoe are raised and not trapped accidentally with self.assertRaisesRegex(Exception, 'not a valid address'): await self.ledger.network.get_history('of the world') with self.assertRaisesRegex(Exception, 'rejected by network rules.*TX decode failed'): await self.ledger.network.broadcast('13370042004200') class TestUsagePayment(CommandTestCase): async def test_single_server_payment(self): wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments') self.assertFalse(wallet_pay_service.running) wallet_pay_service.payment_period = 0.5 # only starts with a positive max key fee wallet_pay_service.max_fee = "0.0" await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet) self.assertFalse(wallet_pay_service.running) wallet_pay_service.max_fee = "1.0" await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet) self.assertTrue(wallet_pay_service.running) await wallet_pay_service.stop() await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet) address = await self.blockchain.get_raw_change_address() _, history = await self.ledger.get_local_status_and_history(address) self.assertEqual(history, []) node = SPVNode(node_number=2) await node.start(self.blockchain, extraconf={"payment_address": address, "daily_fee": "1.1"}) self.addCleanup(node.stop) self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"]) await self.daemon.jsonrpc_wallet_reconnect() LBRYElectrumX.set_server_features(node.server.env) features = await self.ledger.network.get_server_features() self.assertEqual(features["payment_address"], address) self.assertEqual(features["daily_fee"], "1.1") with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError): await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30) node.server.env.daily_fee = "1.0" node.server.env.payment_address = address LBRYElectrumX.set_server_features(node.server.env) # self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"]) await self.daemon.jsonrpc_wallet_reconnect() features = await self.ledger.network.get_server_features() self.assertEqual(features["payment_address"], address) self.assertEqual(features["daily_fee"], "1.0") tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30) self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted self.assertEqual(tx.outputs[0].amount, 100000000) self.assertEqual(tx.outputs[0].get_address(self.ledger), address) # continue paying until account is out of funds with self.assertRaises(InsufficientFundsError): for i in range(10): await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30) self.assertTrue(wallet_pay_service.running) class TestESSync(CommandTestCase): async def test_es_sync_utility(self): es_writer = self.conductor.spv_node.es_writer server_search_client = self.conductor.spv_node.server.session_manager.search_index for i in range(10): await self.stream_create(f"stream{i}", bid='0.001') await self.generate(1) self.assertEqual(10, len(await self.claim_search(order_by=['height']))) # delete the index and verify nothing is returned by claim search await es_writer.delete_index() server_search_client.clear_caches() self.assertEqual(0, len(await self.claim_search(order_by=['height']))) # reindex, 10 claims should be returned await es_writer.reindex(force=True) self.assertEqual(10, len(await self.claim_search(order_by=['height']))) server_search_client.clear_caches() self.assertEqual(10, len(await self.claim_search(order_by=['height']))) # reindex again, this should not appear to do anything but will delete and reinsert the same 10 claims await es_writer.reindex(force=True) self.assertEqual(10, len(await self.claim_search(order_by=['height']))) server_search_client.clear_caches() self.assertEqual(10, len(await self.claim_search(order_by=['height']))) # delete the index again and stop the writer, upon starting it the writer should reindex automatically await es_writer.delete_index() await es_writer.stop() server_search_client.clear_caches() self.assertEqual(0, len(await self.claim_search(order_by=['height']))) await es_writer.start(reindex=True) self.assertEqual(10, len(await self.claim_search(order_by=['height']))) # stop the es writer and advance the chain by 1, adding a new claim. upon resuming the es writer, it should # add the new claim await es_writer.stop() stream11 = self.get_claim_id(await self.stream_create(f"stream11", bid='0.001', confirm=False)) current_height = self.conductor.spv_node.writer.height generate_block_task = asyncio.create_task(self.generate(1)) await self.conductor.spv_node.writer.wait_until_block(current_height + 1) await es_writer.start() await generate_block_task self.assertEqual(11, len(await self.claim_search(order_by=['height']))) # stop/delete es and advance the chain by 1, removing stream11 await es_writer.delete_index() await es_writer.stop() server_search_client.clear_caches() await self.stream_abandon(stream11, confirm=False) current_height = self.conductor.spv_node.writer.height generate_block_task = asyncio.create_task(self.generate(1)) await self.conductor.spv_node.writer.wait_until_block(current_height + 1) await es_writer.start(reindex=True) await generate_block_task self.assertEqual(10, len(await self.claim_search(order_by=['height']))) # # this time we will test a migration from unversioned to v1 # await db.search_index.sync_client.indices.delete_template(db.search_index.index) # await db.search_index.stop() # # await make_es_index_and_run_sync(env, db=db, index_name=db.search_index.index, force=True) # await db.search_index.start() # # await es_writer.reindex() # self.assertEqual(10, len(await self.claim_search(order_by=['height']))) class TestHubDiscovery(CommandTestCase): async def test_hub_discovery(self): us_final_node = SPVNode(node_number=2) await us_final_node.start(self.blockchain, extraconf={"country": "US"}) self.addCleanup(us_final_node.stop) final_node_host = f"{us_final_node.hostname}:{us_final_node.port}" kp_final_node = SPVNode(node_number=3) await kp_final_node.start(self.blockchain, extraconf={"country": "KP"}) self.addCleanup(kp_final_node.stop) kp_final_node_host = f"{kp_final_node.hostname}:{kp_final_node.port}" relay_node = SPVNode(node_number=4) await relay_node.start(self.blockchain, extraconf={ "country": "FR", "peer_hubs": ",".join([kp_final_node_host, final_node_host]) }) relay_node_host = f"{relay_node.hostname}:{relay_node.port}" self.addCleanup(relay_node.stop) self.assertEqual(list(self.daemon.conf.known_hubs), []) self.assertEqual( self.daemon.ledger.network.client.server_address_and_port, ('127.0.0.1', 50002) ) # connect to relay hub which will tell us about the final hubs self.daemon.jsonrpc_settings_set('lbryum_servers', [relay_node_host]) await self.daemon.jsonrpc_wallet_reconnect() self.assertEqual( self.daemon.conf.known_hubs.filter(), { (relay_node.hostname, relay_node.port): {"country": "FR"}, (us_final_node.hostname, us_final_node.port): {}, # discovered from relay but not contacted yet (kp_final_node.hostname, kp_final_node.port): {}, # discovered from relay but not contacted yet } ) self.assertEqual( self.daemon.ledger.network.client.server_address_and_port, ('127.0.0.1', relay_node.port) ) # use known_hubs to connect to final US hub self.daemon.jsonrpc_settings_clear('lbryum_servers') self.daemon.conf.jurisdiction = "US" await self.daemon.jsonrpc_wallet_reconnect() self.assertEqual( self.daemon.conf.known_hubs.filter(), { (relay_node.hostname, relay_node.port): {"country": "FR"}, (us_final_node.hostname, us_final_node.port): {"country": "US"}, (kp_final_node.hostname, kp_final_node.port): {"country": "KP"}, } ) self.assertEqual( self.daemon.ledger.network.client.server_address_and_port, ('127.0.0.1', us_final_node.port) ) # connection to KP jurisdiction self.daemon.conf.jurisdiction = "KP" await self.daemon.jsonrpc_wallet_reconnect() self.assertEqual( self.daemon.ledger.network.client.server_address_and_port, ('127.0.0.1', kp_final_node.port) ) kp_final_node.server.session_manager._notify_peer('127.0.0.1:9988') await self.daemon.ledger.network.on_hub.first await asyncio.sleep(0.5) # wait for above event to be processed by other listeners self.assertEqual( self.daemon.conf.known_hubs.filter(), { (relay_node.hostname, relay_node.port): {"country": "FR"}, (us_final_node.hostname, us_final_node.port): {"country": "US"}, (kp_final_node.hostname, kp_final_node.port): {"country": "KP"}, ('127.0.0.1', 9988): {} } ) class TestStressFlush(CommandTestCase): # async def test_flush_over_66_thousand(self): # history = self.conductor.spv_node.server.db.history # history.flush_count = 66_000 # history.flush() # self.assertEqual(history.flush_count, 66_001) # await self.generate(1) # self.assertEqual(history.flush_count, 66_002) async def test_thousands_claim_ids_on_search(self): await self.stream_create() with self.assertRaises(RPCError) as err: await self.claim_search(not_channel_ids=[("%040x" % i) for i in range(8196)]) # in the go hub this doesnt have a `.` at the end, in python it does self.assertTrue(err.exception.message.startswith('not_channel_ids cant have more than 2048 items')) ================================================ FILE: tests/integration/claims/__init__.py ================================================ ================================================ FILE: tests/integration/claims/test_claim_commands.py ================================================ import os.path import tempfile import logging import asyncio from binascii import unhexlify from unittest import skip from urllib.request import urlopen import ecdsa from lbry.error import InsufficientFundsError from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE from lbry.testcase import CommandTestCase from lbry.wallet.orchstr8.node import SPVNode from lbry.wallet.transaction import Transaction, Output from lbry.wallet.util import satoshis_to_coins as lbc from lbry.crypto.hash import sha256 log = logging.getLogger(__name__) STREAM_TYPES = { 'video': 1, 'audio': 2, 'image': 3, 'document': 4, 'binary': 5, 'model': 6, } def verify(channel, data, signature, channel_hash=None): pieces = [ signature['salt'].encode(), channel_hash or channel.claim_hash, data ] return Output.is_signature_valid( unhexlify(signature['signature']), sha256(b''.join(pieces)), channel.claim.channel.public_key_bytes ) class ClaimTestCase(CommandTestCase): files_directory = os.path.join(os.path.dirname(__file__), 'files') video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4' video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4') image_data = unhexlify( b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc' b'510b9000000097048597300000b1300000b1301009a9c1800000015494441' b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004' b'9454e44ae426082' ) def setUp(self): if not os.path.exists(self.video_file_name): if not os.path.exists(self.files_directory): os.mkdir(self.files_directory) log.info(f'downloading test video from {self.video_file_name}') with urlopen(self.video_file_url) as response, \ open(self.video_file_name, 'wb') as video_file: video_file.write(response.read()) class ClaimSearchCommand(ClaimTestCase): async def create_channel(self): self.channel = await self.channel_create('@abc', '1.0') self.channel_id = self.get_claim_id(self.channel) async def create_lots_of_streams(self): tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True) await self.confirm_tx(tx.id) # 4 claims per block, 3 blocks. Sorted by height (descending) then claim name (ascending). self.streams = [] for j in range(4): same_height_claims = [] for k in range(5): claim_tx = await self.stream_create( f'c{j}-{k}', '0.000001', channel_id=self.channel_id, confirm=False) same_height_claims.append(claim_tx['outputs'][0]['name']) await self.on_transaction_dict(claim_tx) claim_tx = await self.stream_create( f'c{j}-6', '0.000001', channel_id=self.channel_id, confirm=True) same_height_claims.append(claim_tx['outputs'][0]['name']) self.streams = same_height_claims + self.streams async def assertFindsClaim(self, claim, **kwargs): await self.assertFindsClaims([claim], **kwargs) async def assertFindsClaims(self, claims, **kwargs): kwargs.setdefault('order_by', ['height', '^name']) results = await self.claim_search(**kwargs) self.assertEqual( len(claims), len(results), f"{[claim['outputs'][0]['name'] for claim in claims]} != {[result['name'] for result in results]}") for claim, result in zip(claims, results): self.assertEqual( (claim['txid'], self.get_claim_id(claim)), (result['txid'], result['claim_id']), f"(expected {claim['outputs'][0]['name']}) != (got {result['name']})" ) async def assertListsClaims(self, claims, **kwargs): kwargs.setdefault('order_by', 'height') results = await self.claim_list(**kwargs) self.assertEqual(len(claims), len(results)) for claim, result in zip(claims, results): self.assertEqual( (claim['txid'], self.get_claim_id(claim)), (result['txid'], result['claim_id']), f"(expected {claim['outputs'][0]['name']}) != (got {result['name']})" ) @skip("doesnt happen on ES...?") async def test_disconnect_on_memory_error(self): claim_ids = [ '0000000000000000000000000000000000000000', ] * 23828 self.assertListEqual([], await self.claim_search(claim_ids=claim_ids)) # this should do nothing... if the resolve (which is retried) results in the server disconnecting, # it kerplodes await asyncio.wait_for(self.daemon.jsonrpc_resolve([ f'0000000000000000000000000000000000000000{i}' for i in range(30000) ]), 30) # 23829 claim ids makes the request just large enough claim_ids = [ '0000000000000000000000000000000000000000', ] * 33829 with self.assertRaises(ConnectionResetError): await self.claim_search(claim_ids=claim_ids) async def test_basic_claim_search(self): await self.create_channel() channel_txo = self.channel['outputs'][0] channel2 = await self.channel_create('@abc', '0.1', allow_duplicate_name=True) channel_txo2 = channel2['outputs'][0] channel_id2 = channel_txo2['claim_id'] # finding a channel await self.assertFindsClaims([channel2, self.channel], name='@abc') await self.assertFindsClaim(self.channel, name='@abc', is_controlling=True) await self.assertFindsClaim(self.channel, claim_id=self.channel_id) await self.assertFindsClaim(self.channel, txid=self.channel['txid'], nout=0) await self.assertFindsClaim(channel2, claim_id=channel_id2) await self.assertFindsClaim(channel2, txid=channel2['txid'], nout=0) await self.assertFindsClaim( channel2, public_key_id=channel_txo2['value']['public_key_id']) await self.assertFindsClaim( self.channel, public_key_id=channel_txo['value']['public_key_id']) signed = await self.stream_create('on-channel-claim', '0.001', channel_id=self.channel_id) signed2 = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel_id2, allow_duplicate_name=True) unsigned = await self.stream_create('unsigned', '0.0001') # finding claims with and without a channel await self.assertFindsClaims([signed2, signed], name='on-channel-claim') await self.assertFindsClaims([signed2, signed], channel_ids=[self.channel_id, channel_id2]) await self.assertFindsClaim(signed, name='on-channel-claim', channel_ids=[self.channel_id]) await self.assertFindsClaim(signed2, name='on-channel-claim', channel_ids=[channel_id2]) await self.assertFindsClaim(unsigned, name='unsigned') await self.assertFindsClaim(unsigned, txid=unsigned['txid'], nout=0) await self.assertFindsClaim(unsigned, claim_id=self.get_claim_id(unsigned)) two = await self.stream_create('on-channel-claim-2', '0.0001', channel_id=self.channel_id) three = await self.stream_create('on-channel-claim-3', '0.0001', channel_id=self.channel_id) # three streams in channel, zero streams in abandoned channel claims = [three, two, signed] await self.assertFindsClaims(claims, channel_ids=[self.channel_id]) await self.assertFindsClaims(claims, channel=f"@abc#{self.channel_id}") await self.assertFindsClaims(claims, channel=f"@abc#{self.channel_id}", valid_channel_signature=True) await self.assertFindsClaims(claims, channel=f"@abc#{self.channel_id}", has_channel_signature=True, valid_channel_signature=True) await self.assertFindsClaims([], channel=f"@abc#{self.channel_id}", has_channel_signature=True, invalid_channel_signature=True) # fixme await self.assertFindsClaims([], channel=f"@inexistent") await self.assertFindsClaims([three, two, signed2, signed], channel_ids=[channel_id2, self.channel_id]) await self.channel_abandon(claim_id=self.channel_id) await self.assertFindsClaims([], channel=f"@abc#{self.channel_id}", valid_channel_signature=True) await self.assertFindsClaims([], channel_ids=[self.channel_id], valid_channel_signature=True) await self.assertFindsClaims([signed2], channel_ids=[channel_id2], valid_channel_signature=True) # pass `invalid_channel_signature=False` to catch a bug in argument processing await self.assertFindsClaims([signed2], channel_ids=[channel_id2, self.channel_id], valid_channel_signature=True, invalid_channel_signature=False) # invalid signature still returns channel_id self.ledger._tx_cache.clear() invalid_claims = await self.claim_search(invalid_channel_signature=True, has_channel_signature=True) self.assertEqual(3, len(invalid_claims)) self.assertTrue(all([not c['is_channel_signature_valid'] for c in invalid_claims])) self.assertEqual({'channel_id': self.channel_id}, invalid_claims[0]['signing_channel']) valid_claims = await self.claim_search(valid_channel_signature=True, has_channel_signature=True) self.assertEqual(1, len(valid_claims)) self.assertTrue(all([c['is_channel_signature_valid'] for c in valid_claims])) self.assertEqual('@abc', valid_claims[0]['signing_channel']['name']) # abandoned stream won't show up for streams in channel search await self.stream_abandon(txid=signed2['txid'], nout=0) await self.assertFindsClaims([], channel_ids=[channel_id2]) # resolve by claim ids await self.assertFindsClaims([three, two], claim_ids=[self.get_claim_id(three), self.get_claim_id(two)]) await self.assertFindsClaims([three], claim_id=self.get_claim_id(three)) await self.assertFindsClaims([three], claim_id=self.get_claim_id(three), text='*') # resolve by sd hash two_sd_hash = two['outputs'][0]['value']['source']['sd_hash'] await self.assertFindsClaims([two], sd_hash=two_sd_hash) await self.assertFindsClaims([two], sd_hash=two_sd_hash[:4]) async def test_source_filter(self): channel = await self.channel_create('@abc') no_source = await self.stream_create('no-source', data=None) normal = await self.stream_create('normal', data=b'normal') normal_repost = await self.stream_repost(self.get_claim_id(normal), 'normal-repost') no_source_repost = await self.stream_repost(self.get_claim_id(no_source), 'no-source-repost') channel_repost = await self.stream_repost(self.get_claim_id(channel), 'channel-repost') await self.assertFindsClaims([channel_repost, no_source_repost, no_source, channel], has_no_source=True) await self.assertListsClaims([no_source, channel], has_no_source=True) await self.assertFindsClaims([channel_repost, normal_repost, normal, channel], has_source=True) await self.assertListsClaims([channel_repost, no_source_repost, normal_repost, normal], has_source=True) await self.assertFindsClaims([channel_repost, no_source_repost, normal_repost, normal, no_source, channel]) await self.assertListsClaims([channel_repost, no_source_repost, normal_repost, normal, no_source, channel]) await self.assertFindsClaims([normal_repost, normal], stream_types=list(STREAM_TYPES.keys())) async def test_pagination(self): await self.create_channel() await self.create_lots_of_streams() # with and without totals results = await self.daemon.jsonrpc_claim_search() self.assertEqual(results['total_pages'], 2) self.assertEqual(results['total_items'], 25) results = await self.daemon.jsonrpc_claim_search(no_totals=True) self.assertNotIn('total_pages', results) self.assertNotIn('total_items', results) # defaults page = await self.claim_search(channel='@abc', order_by=['height', '^name']) page_claim_ids = [item['name'] for item in page] self.assertEqual(page_claim_ids, self.streams[:DEFAULT_PAGE_SIZE]) # page with default page_size page = await self.claim_search(page=2, channel='@abc', order_by=['height', '^name']) page_claim_ids = [item['name'] for item in page] self.assertEqual(page_claim_ids, self.streams[DEFAULT_PAGE_SIZE:(DEFAULT_PAGE_SIZE*2)]) # page_size larger than dataset page = await self.claim_search(page_size=50, channel='@abc', order_by=['height', '^name']) page_claim_ids = [item['name'] for item in page] self.assertEqual(page_claim_ids, self.streams) # page_size less than dataset page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name']) page_claim_ids = [item['name'] for item in page] self.assertEqual(page_claim_ids, self.streams[:6]) # page and page_size page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name']) page_claim_ids = [item['name'] for item in page] self.assertEqual(page_claim_ids, self.streams[6:12]) out_of_bounds = await self.claim_search(page=4, page_size=20, channel='@abc') self.assertEqual(out_of_bounds, []) async def test_tag_search(self): claim1 = await self.stream_create('claim1', tags=['aBc']) claim2 = await self.stream_create('claim2', tags=['#abc', 'def']) claim3 = await self.stream_create('claim3', tags=['abc', 'ghi', 'jkl']) claim4 = await self.stream_create('claim4', tags=['abc\t', 'ghi', 'mno']) claim5 = await self.stream_create('claim5', tags=['pqr']) # any_tags await self.assertFindsClaims([claim5, claim4, claim3, claim2, claim1], any_tags=['\tabc', 'pqr']) await self.assertFindsClaims([claim4, claim3, claim2, claim1], any_tags=['abc']) await self.assertFindsClaims([claim4, claim3, claim2, claim1], any_tags=['abc', 'ghi']) await self.assertFindsClaims([claim4, claim3], any_tags=['ghi']) await self.assertFindsClaims([claim4, claim3], any_tags=['ghi', 'xyz']) await self.assertFindsClaims([], any_tags=['xyz']) # all_tags await self.assertFindsClaims([], all_tags=['abc', 'pqr']) await self.assertFindsClaims([claim4, claim3, claim2, claim1], all_tags=['ABC']) await self.assertFindsClaims([claim4, claim3], all_tags=['abc', 'ghi']) await self.assertFindsClaims([claim4, claim3], all_tags=['ghi']) await self.assertFindsClaims([], all_tags=['ghi', 'xyz']) await self.assertFindsClaims([], all_tags=['xyz']) # not_tags await self.assertFindsClaims([], not_tags=['abc', 'pqr']) await self.assertFindsClaims([claim5], not_tags=['abC']) await self.assertFindsClaims([claim5], not_tags=['abc', 'ghi']) await self.assertFindsClaims([claim5, claim2, claim1], not_tags=['ghi']) await self.assertFindsClaims([claim5, claim2, claim1], not_tags=['ghi', 'xyz']) await self.assertFindsClaims([claim5, claim4, claim3, claim2, claim1], not_tags=['xyz']) # combinations await self.assertFindsClaims([claim3], all_tags=['abc', 'ghi'], not_tags=['mno']) await self.assertFindsClaims([claim3], all_tags=['abc', 'ghi'], any_tags=['jkl'], not_tags=['mno']) await self.assertFindsClaims([claim4, claim3, claim2], all_tags=['abc'], any_tags=['def', 'ghi']) async def test_order_by(self): height = self.ledger.network.remote_height claims = [await self.stream_create(f'claim{i}') for i in range(5)] await self.assertFindsClaims(claims, order_by=["^height"]) await self.assertFindsClaims(list(reversed(claims)), order_by=["height"]) await self.assertFindsClaims([claims[0]], height=height+1) await self.assertFindsClaims([claims[4]], height=height+5) await self.assertFindsClaims(claims[:1], height=f'<{height+2}', order_by=["^height"]) await self.assertFindsClaims(claims[:2], height=f'<={height+2}', order_by=["^height"]) await self.assertFindsClaims(claims[2:], height=f'>{height+2}', order_by=["^height"]) await self.assertFindsClaims(claims[1:], height=f'>={height+2}', order_by=["^height"]) await self.assertFindsClaims(claims, order_by=["^name"]) async def test_search_by_fee(self): claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc') claim2 = await self.stream_create('claim2', fee_amount='0.9', fee_currency='lbc') claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc') claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc') claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd') repost1 = await self.stream_repost(self.get_claim_id(claim1), 'repost1') repost5 = await self.stream_repost(self.get_claim_id(claim5), 'repost5') await self.assertFindsClaims([repost5, repost1, claim5, claim4, claim3, claim2, claim1], fee_amount='>0') await self.assertFindsClaims([repost1, claim4, claim3, claim2, claim1], fee_currency='lbc') await self.assertFindsClaims([repost1, claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc') await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc') await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc') await self.assertFindsClaims([repost5, claim5], fee_currency='usd') async def test_search_by_language(self): claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc') claim2 = await self.stream_create('claim2', fee_amount='0.9', fee_currency='lbc') claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc', languages='en') claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc', languages='en') claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd', languages='es') await self.assertFindsClaims([claim4, claim3], any_languages=['en']) await self.assertFindsClaims([claim2, claim1], any_languages=['none']) await self.assertFindsClaims([claim4, claim3, claim2, claim1], any_languages=['none', 'en']) await self.assertFindsClaims([claim5], any_languages=['es']) await self.assertFindsClaims([claim5, claim4, claim3], any_languages=['en', 'es']) await self.assertFindsClaims([], fee_currency='foo') async def test_search_by_channel(self): match = self.assertFindsClaims chan1_id = self.get_claim_id(await self.channel_create('@chan1')) chan2_id = self.get_claim_id(await self.channel_create('@chan2')) chan3_id = self.get_claim_id(await self.channel_create('@chan3')) chan4 = await self.channel_create('@chan4', '0.1') claim1 = await self.stream_create('claim1') claim2 = await self.stream_create('claim2', channel_id=chan1_id) claim3 = await self.stream_create('claim3', channel_id=chan1_id) claim4 = await self.stream_create('claim4', channel_id=chan2_id) claim5 = await self.stream_create('claim5', channel_id=chan2_id) claim6 = await self.stream_create('claim6', channel_id=chan3_id) await self.channel_abandon(chan3_id) # {has/valid/invalid}_channel_signature await match([claim6, claim5, claim4, claim3, claim2], has_channel_signature=True) await match([claim5, claim4, claim3, claim2, claim1], valid_channel_signature=True, claim_type='stream') await match([claim6, claim1], invalid_channel_signature=True, claim_type='stream') await match([claim5, claim4, claim3, claim2], has_channel_signature=True, valid_channel_signature=True) await match([claim6], has_channel_signature=True, invalid_channel_signature=True) # not_channel_ids await match([claim6, claim5, claim4, claim3, claim2, claim1], not_channel_ids=['abc123'], claim_type='stream') await match([claim5, claim4, claim3, claim2, claim1], not_channel_ids=[chan3_id], claim_type='stream') await match([claim6, claim5, claim4, claim1], not_channel_ids=[chan1_id], claim_type='stream') await match([claim6, claim3, claim2, claim1], not_channel_ids=[chan2_id], claim_type='stream') await match([claim6, claim1], not_channel_ids=[chan1_id, chan2_id], claim_type='stream') await match([claim6, claim1, chan4], not_channel_ids=[chan1_id, chan2_id]) # not_channel_ids + valid_channel_signature await match([claim5, claim4, claim3, claim2, claim1], not_channel_ids=['abc123'], valid_channel_signature=True, claim_type='stream') await match([claim5, claim4, claim1], not_channel_ids=[chan1_id], valid_channel_signature=True, claim_type='stream') await match([claim3, claim2, claim1], not_channel_ids=[chan2_id], valid_channel_signature=True, claim_type='stream') await match([claim1], not_channel_ids=[chan1_id, chan2_id], valid_channel_signature=True, claim_type='stream') # not_channel_ids + has_channel_signature await match([claim6, claim5, claim4, claim3, claim2], not_channel_ids=['abc123'], has_channel_signature=True) await match([claim6, claim5, claim4], not_channel_ids=[chan1_id], has_channel_signature=True) await match([claim6, claim3, claim2], not_channel_ids=[chan2_id], has_channel_signature=True) await match([claim6], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True) # not_channel_ids + has_channel_signature + valid_channel_signature await match([claim5, claim4, claim3, claim2], not_channel_ids=['abc123'], has_channel_signature=True, valid_channel_signature=True) await match([claim5, claim4], not_channel_ids=[chan1_id], has_channel_signature=True, valid_channel_signature=True) await match([claim3, claim2], not_channel_ids=[chan2_id], has_channel_signature=True, valid_channel_signature=True) await match([], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True, valid_channel_signature=True) @skip async def test_no_source_and_valid_channel_signature_and_media_type(self): await self.channel_create('@spam2', '1.0') await self.stream_create('barrrrrr', '1.0', channel_name='@spam2', file_path=self.video_file_name) paradox_no_source_claims = await self.claim_search(has_no_source=True, valid_channel_signature=True, media_type="video/mp4") mp4_claims = await self.claim_search(media_type="video/mp4") no_source_claims = await self.claim_search(has_no_source=True, valid_channel_signature=True) self.assertEqual(0, len(paradox_no_source_claims)) self.assertEqual(1, len(no_source_claims)) self.assertEqual(1, len(mp4_claims)) async def test_limit_claims_per_channel(self): match = self.assertFindsClaims chan1_id = self.get_claim_id(await self.channel_create('@chan1')) chan2_id = self.get_claim_id(await self.channel_create('@chan2')) claim1 = await self.stream_create('claim1') claim2 = await self.stream_create('claim2', channel_id=chan1_id) claim3 = await self.stream_create('claim3', channel_id=chan1_id) claim4 = await self.stream_create('claim4', channel_id=chan1_id) claim5 = await self.stream_create('claim5', channel_id=chan2_id) claim6 = await self.stream_create('claim6', channel_id=chan2_id) await match( [claim6, claim5, claim4, claim3, claim1], limit_claims_per_channel=2, claim_type='stream' ) await match( [claim6, claim5, claim4, claim3, claim2, claim1], limit_claims_per_channel=3, claim_type='stream' ) async def test_no_duplicates(self): await self.generate(10) match = self.assertFindsClaims claims = [] channels = [] first = await self.stream_create('original_claim0') second = await self.stream_create('original_claim1') for i in range(10): repost_id = self.get_claim_id(second if i % 2 == 0 else first) channel = await self.channel_create(f'@chan{i}', bid='0.001') channels.append(channel) claims.append( await self.stream_repost(repost_id, f'claim{i}', bid='0.001', channel_id=self.get_claim_id(channel))) await match([first, second] + channels, remove_duplicates=True, order_by=['^height']) await match(list(reversed(channels)) + [second, first], remove_duplicates=True, order_by=['height']) # the original claims doesn't show up, so we pick the oldest reposts await match([channels[0], claims[0], channels[1], claims[1]] + channels[2:], height='>218', remove_duplicates=True, order_by=['^height']) # limit claims per channel, invert order, oldest ones are still chosen await match(channels[2:][::-1] + [claims[1], channels[1], claims[0], channels[0]], height='>218', limit_claims_per_channel=1, remove_duplicates=True, order_by=['height']) async def test_limit_claims_per_channel_across_sorted_pages(self): await self.generate(10) match = self.assertFindsClaims channel_id = self.get_claim_id(await self.channel_create('@chan0')) claims = [] first = await self.stream_create('claim0', channel_id=channel_id) second = await self.stream_create('claim1', channel_id=channel_id) for i in range(2, 10): some_chan = self.get_claim_id(await self.channel_create(f'@chan{i}', bid='0.001')) claims.append(await self.stream_create(f'claim{i}', bid='0.001', channel_id=some_chan)) last = await self.stream_create('claim10', channel_id=channel_id) await match( [first, second, claims[0], claims[1]], page_size=4, limit_claims_per_channel=3, claim_type='stream', order_by=['^height'] ) # second goes out await match( [first, claims[0], claims[1], claims[2]], page_size=4, limit_claims_per_channel=1, claim_type='stream', order_by=['^height'] ) # second appears, from replacement queue await match( [second, claims[3], claims[4], claims[5]], page_size=4, page=2, limit_claims_per_channel=1, claim_type='stream', order_by=['^height'] ) # last is unaffected, as the limit applies per page await match( [claims[6], claims[7], last], page_size=4, page=3, limit_claims_per_channel=1, claim_type='stream', order_by=['^height'] ) # feature disabled on 0 or negative values for limit in [None, 0, -1]: await match( [first, second] + claims + [last], limit_claims_per_channel=limit, claim_type='stream', order_by=['^height'] ) async def test_claim_type_and_media_type_search(self): # create an invalid/unknown claim address = await self.account.receiving.get_or_create_usable_address() tx = await Transaction.claim_create( 'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account) await tx.sign([self.account]) await self.broadcast_and_confirm(tx) octet = await self.stream_create() video = await self.stream_create('chrome', file_path=self.video_file_name) image = await self.stream_create('blank-image', data=self.image_data, suffix='.png') image_repost = await self.stream_repost(self.get_claim_id(image), 'image-repost') video_repost = await self.stream_repost(self.get_claim_id(video), 'video-repost') collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)]) channel = await self.channel_create() unknown = self.sout(tx) # claim_type await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream') await self.assertFindsClaims([channel], claim_type='channel') await self.assertFindsClaims([video_repost, image_repost], claim_type='repost') await self.assertFindsClaims([collection], claim_type='collection') # stream_type await self.assertFindsClaims([octet, unknown], stream_types=['binary']) await self.assertFindsClaims([video_repost, video], stream_types=['video']) await self.assertFindsClaims([image_repost, image], stream_types=['image']) await self.assertFindsClaims([video_repost, image_repost, image, video], stream_types=['video', 'image']) # media_type await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream']) await self.assertFindsClaims([video_repost, video], media_types=['video/mp4']) await self.assertFindsClaims([image_repost, image], media_types=['image/png']) await self.assertFindsClaims([video_repost, image_repost, image, video], media_types=['video/mp4', 'image/png']) # duration await self.assertFindsClaims([video_repost, video], duration='>14') await self.assertFindsClaims([video_repost, video], duration='<16') await self.assertFindsClaims([video_repost, video], duration=15) await self.assertFindsClaims([], duration='>100') await self.assertFindsClaims([], duration='<14') async def test_search_by_text(self): chan1_id = self.get_claim_id(await self.channel_create('@SatoshiNakamoto')) chan2_id = self.get_claim_id(await self.channel_create('@Bitcoin')) chan3_id = self.get_claim_id(await self.channel_create('@IAmSatoshi')) claim1 = await self.stream_create( "the-real-satoshi", title="The Real Satoshi Nakamoto", description="Documentary about the real Satoshi Nakamoto, creator of bitcoin.", tags=['satoshi nakamoto', 'bitcoin', 'documentary'] ) claim2 = await self.stream_create( "about-me", channel_id=chan1_id, title="Satoshi Nakamoto Autobiography", description="I am Satoshi Nakamoto and this is my autobiography.", tags=['satoshi nakamoto', 'bitcoin', 'documentary', 'autobiography'] ) claim3 = await self.stream_create( "history-of-bitcoin", channel_id=chan2_id, title="History of Bitcoin", description="History of bitcoin and its creator Satoshi Nakamoto.", tags=['satoshi nakamoto', 'bitcoin', 'documentary', 'history'] ) claim4 = await self.stream_create( "satoshi-conspiracies", channel_id=chan3_id, title="Satoshi Nakamoto Conspiracies", description="Documentary detailing various conspiracies surrounding Satoshi Nakamoto.", tags=['conspiracies', 'bitcoin', 'satoshi nakamoto'] ) await self.assertFindsClaims([], text='cheese') await self.assertFindsClaims([claim2], text='autobiography') await self.assertFindsClaims([claim3], text='history') await self.assertFindsClaims([claim4], text='conspiracy') await self.assertFindsClaims([], text='conspiracy+history') await self.assertFindsClaims([claim4, claim3], text='conspiracy|history') await self.assertFindsClaims([claim1, claim4, claim2, claim3], text='documentary', order_by=[]) # todo: check why claim1 and claim2 order changed. used to be ...claim1, claim2... await self.assertFindsClaims([claim4, claim2, claim1, claim3], text='satoshi', order_by=[]) claim2 = await self.stream_update( self.get_claim_id(claim2), clear_tags=True, tags=['cloud'], title="Satoshi Nakamoto Nography", description="I am Satoshi Nakamoto and this is my nography.", ) await self.assertFindsClaims([], text='autobiography') await self.assertFindsClaims([claim2], text='cloud') await self.stream_abandon(self.get_claim_id(claim2)) await self.assertFindsClaims([], text='cloud') class TransactionCommands(ClaimTestCase): async def test_transaction_list(self): channel_id = self.get_claim_id(await self.channel_create()) await self.channel_update(channel_id, bid='0.5') await self.channel_abandon(claim_id=channel_id) stream_id = self.get_claim_id(await self.stream_create()) await self.stream_update(stream_id, bid='0.5') await self.stream_abandon(claim_id=stream_id) r = await self.transaction_list() self.assertEqual(7, len(r)) self.assertEqual(stream_id, r[0]['abandon_info'][0]['claim_id']) self.assertEqual(stream_id, r[1]['update_info'][0]['claim_id']) self.assertEqual(stream_id, r[2]['claim_info'][0]['claim_id']) self.assertEqual(channel_id, r[3]['abandon_info'][0]['claim_id']) self.assertEqual(channel_id, r[4]['update_info'][0]['claim_id']) self.assertEqual(channel_id, r[5]['claim_info'][0]['claim_id']) class TransactionOutputCommands(ClaimTestCase): async def test_support_with_comment(self): channel = self.get_claim_id(await self.channel_create('@identity')) stream = self.get_claim_id(await self.stream_create()) support = await self.support_create(stream, channel_id=channel, comment="nice!") self.assertEqual(support['outputs'][0]['value']['comment'], "nice!") r, = await self.txo_list(type='support') self.assertEqual(r['txid'], support['txid']) self.assertEqual(r['value']['comment'], "nice!") await self.support_abandon(txid=support['txid'], nout=0, blocking=True) support = await self.support_create(stream, comment="anonymously great!") self.assertEqual(support['outputs'][0]['value']['comment'], "anonymously great!") r, = await self.txo_list(type='support', is_not_spent=True) self.assertEqual(r['txid'], support['txid']) self.assertEqual(r['value']['comment'], "anonymously great!") async def test_txo_list_resolve_supports(self): channel = self.get_claim_id(await self.channel_create('@identity')) stream = self.get_claim_id(await self.stream_create()) support = await self.support_create(stream, channel_id=channel) r, = await self.txo_list(type='support') self.assertEqual(r['txid'], support['txid']) self.assertNotIn('name', r['signing_channel']) r, = await self.txo_list(type='support', resolve=True) self.assertIn('name', r['signing_channel']) self.assertEqual(r['signing_channel']['name'], '@identity') async def test_txo_list_by_channel_filtering(self): channel_foo = self.get_claim_id(await self.channel_create('@foo')) channel_bar = self.get_claim_id(await self.channel_create('@bar')) stream_a = self.get_claim_id(await self.stream_create('a', channel_id=channel_foo)) stream_b = self.get_claim_id(await self.stream_create('b', channel_id=channel_foo)) stream_c = self.get_claim_id(await self.stream_create('c', channel_id=channel_bar)) stream_d = self.get_claim_id(await self.stream_create('d')) support_c = await self.support_create(stream_c, '0.3', channel_id=channel_foo) support_d = await self.support_create(stream_d, '0.3', channel_id=channel_bar) r = await self.txo_list(type='stream') self.assertEqual({stream_a, stream_b, stream_c, stream_d}, {c['claim_id'] for c in r}) r = await self.txo_list(type='stream', channel_id=channel_foo) self.assertEqual({stream_a, stream_b}, {c['claim_id'] for c in r}) r = await self.txo_list(type='support', channel_id=channel_foo) self.assertEqual({support_c['txid']}, {s['txid'] for s in r}) r = await self.txo_list(type='support', channel_id=channel_bar) self.assertEqual({support_d['txid']}, {s['txid'] for s in r}) r = await self.txo_list(type='stream', channel_id=[channel_foo, channel_bar]) self.assertEqual({stream_a, stream_b, stream_c}, {c['claim_id'] for c in r}) r = await self.txo_list(type='stream', not_channel_id=channel_foo) self.assertEqual({stream_c, stream_d}, {c['claim_id'] for c in r}) r = await self.txo_list(type='stream', not_channel_id=[channel_foo, channel_bar]) self.assertEqual({stream_d}, {c['claim_id'] for c in r}) async def test_txo_list_and_sum_filtering(self): channel_id = self.get_claim_id(await self.channel_create()) self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True))) await self.channel_update(channel_id, bid='0.5') self.assertEqual('0.5', lbc(await self.txo_sum(type='channel', is_not_spent=True))) self.assertEqual('1.5', lbc(await self.txo_sum(type='channel'))) stream_id = self.get_claim_id(await self.stream_create(bid='1.3')) self.assertEqual('1.3', lbc(await self.txo_sum(type='stream', is_not_spent=True))) await self.stream_update(stream_id, bid='0.7') self.assertEqual('0.7', lbc(await self.txo_sum(type='stream', is_not_spent=True))) self.assertEqual('2.0', lbc(await self.txo_sum(type='stream'))) self.assertEqual('1.2', lbc(await self.txo_sum(type=['stream', 'channel'], is_not_spent=True))) self.assertEqual('3.5', lbc(await self.txo_sum(type=['stream', 'channel']))) # type filtering r = await self.txo_list(type='channel') self.assertEqual(2, len(r)) self.assertEqual('channel', r[0]['value_type']) self.assertFalse(r[0]['is_spent']) self.assertEqual('channel', r[1]['value_type']) self.assertTrue(r[1]['is_spent']) r = await self.txo_list(type='stream') self.assertEqual(2, len(r)) self.assertEqual('stream', r[0]['value_type']) self.assertFalse(r[0]['is_spent']) self.assertEqual('stream', r[1]['value_type']) self.assertTrue(r[1]['is_spent']) r = await self.txo_list(type=['stream', 'channel']) self.assertEqual(4, len(r)) self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r}) # claim_id filtering r = await self.txo_list(claim_id=stream_id) self.assertEqual(2, len(r)) self.assertEqual({stream_id}, {c['claim_id'] for c in r}) r = await self.txo_list(claim_id=[stream_id, channel_id]) self.assertEqual(4, len(r)) self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) stream_name, _, channel_name, _ = (c['name'] for c in r) r = await self.txo_list(claim_id=['beef']) self.assertEqual(0, len(r)) # claim_name filtering r = await self.txo_list(name=stream_name) self.assertEqual(2, len(r)) self.assertEqual({stream_id}, {c['claim_id'] for c in r}) r = await self.txo_list(name=[stream_name, channel_name]) self.assertEqual(4, len(r)) self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) r = await self.txo_list(name=['beef']) self.assertEqual(0, len(r)) r = await self.txo_list() self.assertEqual(9, len(r)) await self.stream_abandon(claim_id=stream_id) r = await self.txo_list() self.assertEqual(10, len(r)) r = await self.txo_list(claim_id=stream_id) self.assertEqual(2, len(r)) self.assertTrue(r[0]['is_spent']) self.assertTrue(r[1]['is_spent']) async def test_txo_list_my_input_output_filtering(self): wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True) address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id) await self.channel_create('@kept-channel') await self.channel_create('@sent-channel', claim_address=address2) await self.wallet_send('2.9', address2) # all txos on second wallet received_payment, received_channel = await self.txo_list( wallet_id=wallet2.id, is_my_input_or_output=True) self.assertEqual('1.0', received_channel['amount']) self.assertFalse(received_channel['is_my_input']) self.assertTrue(received_channel['is_my_output']) self.assertFalse(received_channel['is_internal_transfer']) self.assertEqual('2.9', received_payment['amount']) self.assertFalse(received_payment['is_my_input']) self.assertTrue(received_payment['is_my_output']) self.assertFalse(received_payment['is_internal_transfer']) # all txos on default wallet r = await self.txo_list(is_my_input_or_output=True) self.assertEqual( ['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'], [t['amount'] for t in r] ) sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r self.assertTrue(sent_payment['is_my_input']) self.assertFalse(sent_payment['is_my_output']) self.assertFalse(sent_payment['is_internal_transfer']) self.assertTrue(change3['is_my_input']) self.assertTrue(change3['is_my_output']) self.assertTrue(change3['is_internal_transfer']) self.assertTrue(sent_channel['is_my_input']) self.assertFalse(sent_channel['is_my_output']) self.assertFalse(sent_channel['is_internal_transfer']) self.assertTrue(change2['is_my_input']) self.assertTrue(change2['is_my_output']) self.assertTrue(change2['is_internal_transfer']) self.assertTrue(kept_channel['is_my_input']) self.assertTrue(kept_channel['is_my_output']) self.assertFalse(kept_channel['is_internal_transfer']) self.assertTrue(change1['is_my_input']) self.assertTrue(change1['is_my_output']) self.assertTrue(change1['is_internal_transfer']) self.assertFalse(initial_funds['is_my_input']) self.assertTrue(initial_funds['is_my_output']) self.assertFalse(initial_funds['is_internal_transfer']) # my stuff and stuff i sent excluding "change" r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True) self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r) # my unspent stuff and stuff i sent excluding "change" r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True) self.assertEqual([sent_payment, sent_channel, kept_channel], r) # only "change" r = await self.txo_list(is_my_input=True, is_my_output=True, type="other") self.assertEqual([change3, change2, change1], r) # only unspent "change" r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True) self.assertEqual([change3], r) # only spent "change" r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True) self.assertEqual([change2, change1], r) # all my unspent stuff r = await self.txo_list(is_my_output=True, is_not_spent=True) self.assertEqual([change3, kept_channel], r) # stuff i sent r = await self.txo_list(is_not_my_output=True) self.assertEqual([sent_payment, sent_channel], r) async def test_txo_plot(self): day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset) stream_id = self.get_claim_id(await self.stream_create()) await self.support_create(stream_id, '0.3') await self.support_create(stream_id, '0.2') await self.generate(day_blocks // 2) await self.stream_update(stream_id) await self.generate(day_blocks // 2) await self.support_create(stream_id, '0.4') await self.support_create(stream_id, '0.5') await self.stream_update(stream_id) await self.generate(day_blocks // 2) await self.stream_update(stream_id) await self.generate(day_blocks // 2) await self.support_create(stream_id, '0.6') plot = await self.txo_plot(type='support') self.assertEqual([ {'day': '2016-06-25', 'total': '0.6'}, ], plot) plot = await self.txo_plot(type='support', days_back=1) self.assertEqual([ {'day': '2016-06-24', 'total': '0.9'}, {'day': '2016-06-25', 'total': '0.6'}, ], plot) plot = await self.txo_plot(type='support', days_back=2) self.assertEqual([ {'day': '2016-06-23', 'total': '0.5'}, {'day': '2016-06-24', 'total': '0.9'}, {'day': '2016-06-25', 'total': '0.6'}, ], plot) plot = await self.txo_plot(type='support', start_day='2016-06-23') self.assertEqual([ {'day': '2016-06-23', 'total': '0.5'}, {'day': '2016-06-24', 'total': '0.9'}, {'day': '2016-06-25', 'total': '0.6'}, ], plot) plot = await self.txo_plot(type='support', start_day='2016-06-24') self.assertEqual([ {'day': '2016-06-24', 'total': '0.9'}, {'day': '2016-06-25', 'total': '0.6'}, ], plot) plot = await self.txo_plot(type='support', start_day='2016-06-23', end_day='2016-06-24') self.assertEqual([ {'day': '2016-06-23', 'total': '0.5'}, {'day': '2016-06-24', 'total': '0.9'}, ], plot) plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=1) self.assertEqual([ {'day': '2016-06-23', 'total': '0.5'}, {'day': '2016-06-24', 'total': '0.9'}, ], plot) plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=2) self.assertEqual([ {'day': '2016-06-23', 'total': '0.5'}, {'day': '2016-06-24', 'total': '0.9'}, {'day': '2016-06-25', 'total': '0.6'}, ], plot) async def test_txo_spend(self): stream_id = self.get_claim_id(await self.stream_create()) for _ in range(10): await self.support_create(stream_id, '0.1') await self.assertBalance(self.account, '7.978478') self.assertEqual('1.0', lbc(await self.txo_sum(type='support', is_not_spent=True))) txs = await self.txo_spend(type='support', batch_size=3, include_full_tx=True) self.assertEqual(4, len(txs)) self.assertEqual(3, len(txs[0]['inputs'])) self.assertEqual(3, len(txs[1]['inputs'])) self.assertEqual(3, len(txs[2]['inputs'])) self.assertEqual(1, len(txs[3]['inputs'])) self.assertEqual('0.0', lbc(await self.txo_sum(type='support', is_not_spent=True))) await self.assertBalance(self.account, '8.977606') await self.support_create(stream_id, '0.1') txs = await self.daemon.jsonrpc_txo_spend(type='support', batch_size=3) self.assertEqual(1, len(txs)) self.assertEqual({'txid'}, set(txs[0])) class ClaimCommands(ClaimTestCase): async def test_claim_list_filtering(self): channel_id = self.get_claim_id(await self.channel_create()) stream_id = self.get_claim_id(await self.stream_create()) await self.stream_update(stream_id, title='foo') # type filtering r = await self.claim_list(claim_type='channel') self.assertEqual(1, len(r)) self.assertEqual('channel', r[0]['value_type']) # catch a bug where cli sends is_spent=False by default r = await self.claim_list(claim_type='stream', is_spent=False) self.assertEqual(1, len(r)) self.assertEqual('stream', r[0]['value_type']) r = await self.claim_list(claim_type=['stream', 'channel']) self.assertEqual(2, len(r)) self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r}) # claim_id filtering r = await self.claim_list(claim_id=stream_id) self.assertEqual(1, len(r)) self.assertEqual({stream_id}, {c['claim_id'] for c in r}) r = await self.claim_list(claim_id=[stream_id, channel_id]) self.assertEqual(2, len(r)) self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) stream_name, channel_name = (c['name'] for c in r) r = await self.claim_list(claim_id=['beef']) self.assertEqual(0, len(r)) # claim_name filtering r = await self.claim_list(name=stream_name) self.assertEqual(1, len(r)) self.assertEqual({stream_id}, {c['claim_id'] for c in r}) r = await self.claim_list(name=[stream_name, channel_name]) self.assertEqual(2, len(r)) self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r}) r = await self.claim_list(name=['beef']) self.assertEqual(0, len(r)) async def test_claim_stream_channel_list_with_resolve(self): self.assertListEqual([], await self.claim_list(resolve=True)) await self.channel_create() await self.stream_create() r = await self.claim_list() self.assertNotIn('short_url', r[0]) self.assertNotIn('short_url', r[1]) self.assertNotIn('short_url', (await self.stream_list())[0]) self.assertNotIn('short_url', (await self.channel_list())[0]) r = await self.claim_list(resolve=True) self.assertIn('short_url', r[0]) self.assertIn('short_url', r[1]) self.assertIn('short_url', (await self.stream_list(resolve=True))[0]) self.assertIn('short_url', (await self.channel_list(resolve=True))[0]) # unconfirmed channel won't resolve channel_tx = await self.daemon.jsonrpc_channel_create('@foo', '1.0') await self.ledger.wait(channel_tx) r = await self.claim_list(resolve=True) self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name']) self.assertTrue(r[1]['meta']['is_controlling']) r = await self.channel_list(resolve=True) self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name']) self.assertTrue(r[1]['meta']['is_controlling']) # confirm it await self.generate(1) await self.ledger.wait(channel_tx, self.blockchain.block_expected) # all channel claims resolve r = await self.claim_list(resolve=True) self.assertTrue(r[0]['meta']['is_controlling']) self.assertTrue(r[1]['meta']['is_controlling']) r = await self.channel_list(resolve=True) self.assertTrue(r[0]['meta']['is_controlling']) self.assertTrue(r[1]['meta']['is_controlling']) # unconfirmed stream won't resolve stream_tx = await self.daemon.jsonrpc_stream_create( 'foo', '1.0', file_path=self.create_upload_file(data=b'hi') ) await self.ledger.wait(stream_tx) r = await self.claim_list(resolve=True) self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name']) self.assertTrue(r[1]['meta']['is_controlling']) r = await self.stream_list(resolve=True) self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name']) self.assertTrue(r[1]['meta']['is_controlling']) # confirm it await self.generate(1) await self.ledger.wait(stream_tx, self.blockchain.block_expected) # all claims resolve r = await self.claim_list(resolve=True) self.assertTrue(r[0]['meta']['is_controlling']) self.assertTrue(r[1]['meta']['is_controlling']) self.assertTrue(r[2]['meta']['is_controlling']) self.assertTrue(r[3]['meta']['is_controlling']) r = await self.stream_list(resolve=True) self.assertTrue(r[0]['meta']['is_controlling']) self.assertTrue(r[1]['meta']['is_controlling']) r = await self.channel_list(resolve=True) self.assertTrue(r[0]['meta']['is_controlling']) self.assertTrue(r[1]['meta']['is_controlling']) # check that metadata is transfered self.assertTrue(r[0]['is_my_output']) async def assertClaimList(self, claim_ids, **kwargs): self.assertEqual(claim_ids, [c['claim_id'] for c in await self.claim_list(**kwargs)]) async def test_list_streams_in_channel_and_order_by(self): channel1_id = self.get_claim_id(await self.channel_create('@chan-one')) channel2_id = self.get_claim_id(await self.channel_create('@chan-two')) stream1_id = self.get_claim_id(await self.stream_create('stream-a', bid='0.3', channel_id=channel1_id)) stream2_id = self.get_claim_id(await self.stream_create('stream-b', bid='0.9', channel_id=channel1_id)) stream3_id = self.get_claim_id(await self.stream_create('stream-c', bid='0.6', channel_id=channel2_id)) await self.assertClaimList([stream2_id, stream1_id], channel_id=channel1_id) await self.assertClaimList([stream3_id], channel_id=channel2_id) await self.assertClaimList([stream3_id, stream2_id, stream1_id], channel_id=[channel1_id, channel2_id]) await self.assertClaimList([stream1_id, stream2_id, stream3_id], claim_type='stream', order_by='name') await self.assertClaimList([stream1_id, stream3_id, stream2_id], claim_type='stream', order_by='amount') await self.assertClaimList([stream3_id, stream2_id, stream1_id], claim_type='stream', order_by='height') async def test_claim_list_with_tips(self): wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True) address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id) await self.wallet_send('5.0', address2) stream1_id = self.get_claim_id(await self.stream_create('one')) stream2_id = self.get_claim_id(await self.stream_create('two')) claims = await self.claim_list() self.assertNotIn('received_tips', claims[0]) self.assertNotIn('received_tips', claims[1]) claims = await self.claim_list(include_received_tips=True) self.assertEqual('0.0', claims[0]['received_tips']) self.assertEqual('0.0', claims[1]['received_tips']) await self.support_create(stream1_id, '0.7', tip=True) await self.support_create(stream1_id, '0.3', tip=True, wallet_id=wallet2.id) await self.support_create(stream1_id, '0.2', tip=True, wallet_id=wallet2.id) await self.support_create(stream2_id, '0.4', tip=True, wallet_id=wallet2.id) await self.support_create(stream2_id, '0.5', tip=True, wallet_id=wallet2.id) await self.support_create(stream2_id, '0.1', tip=True, wallet_id=wallet2.id) claims = await self.claim_list(include_received_tips=True) self.assertEqual('1.0', claims[0]['received_tips']) self.assertEqual('1.2', claims[1]['received_tips']) await self.support_abandon(stream1_id) claims = await self.claim_list(include_received_tips=True) self.assertEqual('1.0', claims[0]['received_tips']) self.assertEqual('0.0', claims[1]['received_tips']) await self.support_abandon(stream2_id) claims = await self.claim_list(include_received_tips=True) self.assertEqual('0.0', claims[0]['received_tips']) self.assertEqual('0.0', claims[1]['received_tips']) async def stream_update_and_wait(self, claim_id, **kwargs): tx = await self.daemon.jsonrpc_stream_update(claim_id, **kwargs) await self.ledger.wait(tx) async def test_claim_list_pending_edits_ordering(self): stream5_id = self.get_claim_id(await self.stream_create('five')) stream4_id = self.get_claim_id(await self.stream_create('four')) stream3_id = self.get_claim_id(await self.stream_create('three')) stream2_id = self.get_claim_id(await self.stream_create('two')) stream1_id = self.get_claim_id(await self.stream_create('one')) await self.assertClaimList([stream1_id, stream2_id, stream3_id, stream4_id, stream5_id]) await self.stream_update_and_wait(stream4_id, title='foo') await self.assertClaimList([stream4_id, stream1_id, stream2_id, stream3_id, stream5_id]) await self.stream_update_and_wait(stream3_id, title='foo') await self.assertClaimList([stream4_id, stream3_id, stream1_id, stream2_id, stream5_id]) class ChannelCommands(CommandTestCase): async def test_create_channel_names(self): # claim new name await self.channel_create('@foo') self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '8.991893') # fail to claim duplicate with self.assertRaisesRegex(Exception, "You already have a channel under the name '@foo'."): await self.channel_create('@foo') # fail to claim invalid name with self.assertRaisesRegex(Exception, "Channel names must start with '@' symbol."): await self.channel_create('foo') # nothing's changed after failed attempts self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '8.991893') # succeed overriding duplicate restriction await self.channel_create('@foo', allow_duplicate_name=True) self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 2) await self.assertBalance(self.account, '7.983786') async def test_channel_bids(self): # enough funds tx = await self.channel_create('@foo', '5.0') claim_id = self.get_claim_id(tx) self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '4.991893') # bid preserved on update tx = await self.channel_update(claim_id) self.assertEqual(tx['outputs'][0]['amount'], '5.0') # bid changed on update tx = await self.channel_update(claim_id, bid='4.0') self.assertEqual(tx['outputs'][0]['amount'], '4.0') await self.assertBalance(self.account, '5.991503') # not enough funds with self.assertRaisesRegex( InsufficientFundsError, "Not enough funds to cover this transaction."): await self.channel_create('@foo2', '9.0') self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1) await self.assertBalance(self.account, '5.991503') # spend exactly amount available, no change tx = await self.channel_create('@foo3', '5.981322') await self.assertBalance(self.account, '0.0') self.assertEqual(len(tx['outputs']), 1) # no change self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 2) async def test_setting_channel_fields(self): values = { 'title': "Cool Channel", 'description': "Best channel on LBRY.", 'thumbnail_url': "https://co.ol/thumbnail.png", 'tags': ["cool", "awesome"], 'languages': ["en-US"], 'locations': ['US::Manchester'], 'email': "human@email.com", 'website_url': "https://co.ol", 'cover_url': "https://co.ol/cover.png", 'featured': ['cafe'] } fixed_values = values.copy() fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')} fixed_values['locations'] = [{'country': 'US', 'city': 'Manchester'}] fixed_values['cover'] = {'url': fixed_values.pop('cover_url')} # create new channel with all fields set tx = await self.out(self.channel_create('@bigchannel', **values)) channel = tx['outputs'][0]['value'] self.assertEqual(channel, { 'public_key': channel['public_key'], 'public_key_id': channel['public_key_id'], **fixed_values }) # create channel with nothing set tx = await self.out(self.channel_create('@lightchannel')) channel = tx['outputs'][0]['value'] self.assertEqual( channel, {'public_key': channel['public_key'], 'public_key_id': channel['public_key_id']}) # create channel with just a featured claim tx = await self.out(self.channel_create('@featurechannel', featured='beef')) txo = tx['outputs'][0] claim_id, channel = txo['claim_id'], txo['value'] fixed_values['public_key'] = channel['public_key'] fixed_values['public_key_id'] = channel['public_key_id'] self.assertEqual(channel, { 'public_key': fixed_values['public_key'], 'public_key_id': fixed_values['public_key_id'], 'featured': ['beef'] }) # update channel "@featurechannel" setting all fields tx = await self.out(self.channel_update(claim_id, **values)) channel = tx['outputs'][0]['value'] fixed_values['featured'].insert(0, 'beef') # existing featured claim self.assertEqual(channel, fixed_values) # clearing and settings featured content tx = await self.out(self.channel_update(claim_id, featured='beefcafe', clear_featured=True)) channel = tx['outputs'][0]['value'] fixed_values['featured'] = ['beefcafe'] self.assertEqual(channel, fixed_values) # reset signing key tx = await self.out(self.channel_update(claim_id, new_signing_key=True)) channel = tx['outputs'][0]['value'] self.assertNotEqual(channel['public_key'], fixed_values['public_key']) # replace mode (clears everything except public_key) tx = await self.out(self.channel_update(claim_id, replace=True, title='foo', email='new@email.com')) self.assertEqual(tx['outputs'][0]['value'], { 'public_key': channel['public_key'], 'public_key_id': channel['public_key_id'], 'title': 'foo', 'email': 'new@email.com'} ) # move channel to another account new_account = await self.out(self.daemon.jsonrpc_account_create('second account')) account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id']) # before moving self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 3) self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 0) other_address = await account2.receiving.get_or_create_usable_address() tx = await self.out(self.channel_update(claim_id, claim_address=other_address)) # after moving self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 3) self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=self.account.id), 2) self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 1) async def test_sign_hex_encoded_data(self): data_to_sign = "CAFEBABE" # claim new name await self.channel_create('@someotherchan') channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1', blocking=True) await self.confirm_tx(channel_tx.id) channel = channel_tx.outputs[0] signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign)) signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign)) signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef')) signature4 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef')) self.assertNotEqual(signature2, signature3) self.assertEqual(signature3, signature4) self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1)) self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2)) self.assertTrue(verify(channel, unhexlify(data_to_sign), signature3)) signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=99)) self.assertTrue(verify(channel, unhexlify('99'), signature3)) async def test_channel_export_import_before_sending_channel(self): # export tx = await self.channel_create('@foo', '1.0') claim_id = self.get_claim_id(tx) channel_private_key = (await self.account.get_channels())[0].private_key exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id)) # import daemon2 = await self.add_daemon() self.assertItemCount(await daemon2.jsonrpc_channel_list(), 0) await daemon2.jsonrpc_channel_import(exported_data) channels = (await daemon2.jsonrpc_channel_list())['items'] self.assertEqual(1, len(channels)) self.assertEqual(channel_private_key.private_key_bytes, channels[0].private_key.private_key_bytes) # second wallet can't update until channel is sent to it with self.assertRaisesRegex(AssertionError, 'Cannot find private key for signing output.'): await daemon2.jsonrpc_channel_update(claim_id, bid='0.5') # now send the channel as well await self.channel_update(claim_id, claim_address=await daemon2.jsonrpc_address_unused()) # second wallet should be able to update now await daemon2.jsonrpc_channel_update(claim_id, bid='0.5') async def test_channel_update_across_accounts(self): account2 = await self.daemon.jsonrpc_account_create('second account') channel = await self.out(self.channel_create('@spam', '1.0', account_id=account2.id)) # channel not in account1 with self.assertRaisesRegex(Exception, "Can't find the channel"): await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=self.account.id) # channel is in account2 await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=account2.id) result = (await self.out(self.daemon.jsonrpc_channel_list()))['items'] self.assertEqual(result[0]['amount'], '2.0') # check all accounts for channel await self.channel_update(self.get_claim_id(channel), bid='3.0') result = (await self.out(self.daemon.jsonrpc_channel_list()))['items'] self.assertEqual(result[0]['amount'], '3.0') await self.channel_abandon(self.get_claim_id(channel)) async def test_tag_normalization(self): tx1 = await self.channel_create('@abc', '1.0', tags=['aBc', ' ABC ', 'xYZ ', 'xyz']) claim_id = self.get_claim_id(tx1) self.assertCountEqual(tx1['outputs'][0]['value']['tags'], ['abc', 'xyz']) tx2 = await self.channel_update(claim_id, tags=[' pqr', 'PQr ']) self.assertCountEqual(tx2['outputs'][0]['value']['tags'], ['abc', 'xyz', 'pqr']) tx3 = await self.channel_update(claim_id, tags=' pqr') self.assertCountEqual(tx3['outputs'][0]['value']['tags'], ['abc', 'xyz', 'pqr']) tx4 = await self.channel_update(claim_id, tags=[' pqr', 'PQr '], clear_tags=True) self.assertEqual(tx4['outputs'][0]['value']['tags'], ['pqr']) class StreamCommands(ClaimTestCase): async def test_create_stream_names(self): # claim new name await self.stream_create('foo') self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '8.993893') # fail to claim duplicate with self.assertRaisesRegex( Exception, "You already have a stream claim published under the name 'foo'."): await self.stream_create('foo') # fail claim starting with @ with self.assertRaisesRegex( Exception, "Stream names cannot start with '@' symbol."): await self.stream_create('@foo') self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '8.993893') # succeed overriding duplicate restriction await self.stream_create('foo', allow_duplicate_name=True) self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 2) await self.assertBalance(self.account, '7.987786') async def test_stream_bids(self): # enough funds tx = await self.stream_create('foo', '2.0') claim_id = self.get_claim_id(tx) self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '7.993893') # bid preserved on update tx = await self.stream_update(claim_id) self.assertEqual(tx['outputs'][0]['amount'], '2.0') # bid changed on update tx = await self.stream_update(claim_id, bid='3.0') self.assertEqual(tx['outputs'][0]['amount'], '3.0') await self.assertBalance(self.account, '6.993319') # not enough funds with self.assertRaisesRegex( InsufficientFundsError, "Not enough funds to cover this transaction."): await self.stream_create('foo2', '9.0') self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 1) await self.assertBalance(self.account, '6.993319') # spend exactly amount available, no change tx = await self.stream_create('foo3', '6.98523') await self.assertBalance(self.account, '0.0') self.assertEqual(len(tx['outputs']), 1) # no change self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 2) async def test_stream_update_and_abandon_across_accounts(self): account2 = await self.daemon.jsonrpc_account_create('second account') stream = await self.out(self.stream_create('spam', '1.0', account_id=account2.id)) # stream not in account1 with self.assertRaisesRegex(Exception, "Can't find the stream"): await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=self.account.id) # stream is in account2 await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=account2.id) result = (await self.out(self.daemon.jsonrpc_stream_list()))['items'] self.assertEqual(result[0]['amount'], '2.0') # check all accounts for stream await self.stream_update(self.get_claim_id(stream), bid='3.0') result = (await self.out(self.daemon.jsonrpc_stream_list()))['items'] self.assertEqual(result[0]['amount'], '3.0') await self.stream_abandon(self.get_claim_id(stream)) async def test_publishing_checks_all_accounts_for_channel(self): account1_id, account1 = self.account.id, self.account new_account = await self.out(self.daemon.jsonrpc_account_create('second account')) account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id']) await self.out(self.channel_create('@spam', '1.0')) self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available']) result = await self.out(self.daemon.jsonrpc_account_send( '5.0', await self.daemon.jsonrpc_address_unused(account2_id), blocking=True )) await self.confirm_tx(result['txid']) self.assertEqual('3.989769', (await self.daemon.jsonrpc_account_balance())['available']) self.assertEqual('5.0', (await self.daemon.jsonrpc_account_balance(account2_id))['available']) baz_tx = await self.out(self.channel_create('@baz', '1.0', account_id=account2_id)) baz_id = self.get_claim_id(baz_tx) channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id)) self.assertItemCount(channels, 1) self.assertEqual(channels['items'][0]['name'], '@spam') self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list(account1_id))) channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id)) self.assertItemCount(channels, 1) self.assertEqual(channels['items'][0]['name'], '@baz') channels = await self.out(self.daemon.jsonrpc_channel_list()) self.assertItemCount(channels, 2) self.assertEqual(channels['items'][0]['name'], '@baz') self.assertEqual(channels['items'][1]['name'], '@spam') # defaults to using all accounts to lookup channel await self.stream_create('hovercraft1', '0.1', channel_id=baz_id) self.assertEqual((await self.claim_search(name='hovercraft1'))[0]['signing_channel']['name'], '@baz') # lookup by channel_name in all accounts await self.stream_create('hovercraft2', '0.1', channel_name='@baz') self.assertEqual((await self.claim_search(name='hovercraft2'))[0]['signing_channel']['name'], '@baz') # uses only the specific accounts which contains the channel await self.stream_create('hovercraft3', '0.1', channel_id=baz_id, channel_account_id=[account2_id]) self.assertEqual((await self.claim_search(name='hovercraft3'))[0]['signing_channel']['name'], '@baz') # lookup by channel_name in specific account await self.stream_create('hovercraft4', '0.1', channel_name='@baz', channel_account_id=[account2_id]) self.assertEqual((await self.claim_search(name='hovercraft4'))[0]['signing_channel']['name'], '@baz') # fails when specifying account which does not contain channel with self.assertRaisesRegex(ValueError, "Couldn't find channel with channel_id"): await self.stream_create( 'hovercraft5', '0.1', channel_id=baz_id, channel_account_id=[account1_id] ) # fail with channel_name with self.assertRaisesRegex(ValueError, "Couldn't find channel with channel_name '@baz'"): await self.stream_create( 'hovercraft5', '0.1', channel_name='@baz', channel_account_id=[account1_id] ) # signing with channel works even if channel and certificate are in different accounts await self.channel_update( baz_id, account_id=account2_id, claim_address=await self.daemon.jsonrpc_address_unused(account1_id) ) await self.stream_create( 'hovercraft5', '0.1', channel_id=baz_id ) async def test_preview_works_with_signed_streams(self): await self.channel_create('@spam', '1.0') signed = await self.stream_create('bar', '1.0', channel_name='@spam', preview=True, confirm=False) self.assertTrue(signed['outputs'][0]['is_channel_signature_valid']) async def test_repost(self): tx = await self.channel_create('@goodies', '1.0') goodies_claim_id = self.get_claim_id(tx) tx = await self.channel_create('@spam', '1.0') spam_claim_id = self.get_claim_id(tx) tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies', tags=['foo', 'gaming']) claim_id = self.get_claim_id(tx) self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0) self.assertItemCount(await self.daemon.jsonrpc_txo_list(reposted_claim_id=claim_id), 0) self.assertItemCount(await self.daemon.jsonrpc_txo_list(type='repost'), 0) tx = await self.stream_repost( claim_id, 'newstuff-again', '1.1', channel_name='@spam', title="repost title", description="repost desc", tags=["tag1", "tag2"] ) repost_id = self.get_claim_id(tx) # test inflating reposted channels works repost_url = f'newstuff-again:{repost_id}' self.ledger._tx_cache.clear() repost_resolve = await self.out(self.daemon.jsonrpc_resolve(repost_url)) repost = repost_resolve[repost_url] self.assertEqual(goodies_claim_id, repost['reposted_claim']['signing_channel']['claim_id']) self.assertEqual("repost title", repost["value"]["title"]) self.assertEqual("repost desc", repost["value"]["description"]) self.assertEqual(["tag1", "tag2"], repost["value"]["tags"]) await self.stream_update( repost_id, title="title 2", description="desc 2", tags=["tag3"] ) repost_resolve = await self.out(self.daemon.jsonrpc_resolve(repost_url)) repost = repost_resolve[repost_url] self.assertEqual(goodies_claim_id, repost['reposted_claim']['signing_channel']['claim_id']) self.assertEqual("title 2", repost["value"]["title"]) self.assertEqual("desc 2", repost["value"]["description"]) self.assertEqual(["tag1", "tag2", "tag3"], repost["value"]["tags"]) self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 1) self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1) self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id) self.assertEqual((await self.txo_list(reposted_claim_id=claim_id))[0]['claim_id'], repost_id) self.assertEqual((await self.txo_list(type='repost'))[0]['claim_id'], repost_id) # tags are inherited (non-common / indexed tags) self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2) self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['foo'], claim_type=['stream', 'repost']), 2) self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['foo'], claim_type=['stream', 'repost']), 0) # "common" / indexed tags work too self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['gaming'], claim_type=['stream', 'repost']), 2) self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['gaming'], claim_type=['stream', 'repost']), 2) self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['gaming'], claim_type=['stream', 'repost']), 0) await self.channel_create('@reposting-goodies', '1.0') await self.stream_repost(claim_id, 'repost-on-channel', '1.1', channel_name='@reposting-goodies') self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 2) self.assertItemCount(await self.daemon.jsonrpc_claim_search(reposted_claim_id=claim_id), 2) self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 2) search_results = await self.claim_search(reposted='>=2') self.assertEqual(len(search_results), 1) self.assertEqual(search_results[0]['name'], 'newstuff') search_results = await self.claim_search(name='repost-on-channel') self.assertEqual(len(search_results), 1) search = search_results[0] self.assertEqual(search['name'], 'repost-on-channel') self.assertEqual(search['signing_channel']['name'], '@reposting-goodies') self.assertEqual(search['reposted_claim']['name'], 'newstuff') self.assertEqual(search['reposted_claim']['meta']['reposted'], 2) self.assertEqual(search['reposted_claim']['signing_channel']['name'], '@goodies') resolved = await self.out( self.daemon.jsonrpc_resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again']) ) self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search) self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff') await self.stream_update(repost_id, bid='0.42') searched_repost = (await self.claim_search(claim_id=repost_id))[0] self.assertEqual(searched_repost['amount'], '0.42') self.assertEqual(searched_repost['signing_channel']['claim_id'], spam_claim_id) async def test_filtering_channels_for_removing_content(self): some_channel_id = self.get_claim_id(await self.channel_create('@some_channel', '0.1')) await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good']) bad_content_id = self.get_claim_id( await self.stream_create('bad_content', '0.1', channel_name='@some_channel', tags=['bad']) ) filtering_channel_id = self.get_claim_id( await self.channel_create('@filtering', '0.1') ) self.conductor.spv_node.server.db.filtering_channel_hashes.add(bytes.fromhex(filtering_channel_id)) self.conductor.spv_node.es_writer.db.filtering_channel_hashes.add(bytes.fromhex(filtering_channel_id)) self.assertEqual(0, len(self.conductor.spv_node.es_writer.db.filtered_streams)) await self.stream_repost(bad_content_id, 'filter1', '0.1', channel_name='@filtering') self.assertEqual(1, len(self.conductor.spv_node.es_writer.db.filtered_streams)) self.assertEqual('0.1', (await self.out(self.daemon.jsonrpc_resolve('bad_content')))['bad_content']['amount']) # search for filtered content directly result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content')) blocked = result['blocked'] self.assertEqual([], result['items']) self.assertEqual(1, blocked['total']) self.assertEqual(1, len(blocked['channels'])) self.assertEqual(1, blocked['channels'][0]['blocked']) self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#')) # same search, but details omitted by 'no_totals' last_result = result result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content', no_totals=True)) self.assertEqual(result['items'], last_result['items']) # search inside channel containing filtered content result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel')) filtered = result['blocked'] self.assertEqual(1, len(result['items'])) self.assertEqual(1, filtered['total']) self.assertEqual(1, len(filtered['channels'])) self.assertEqual(1, filtered['channels'][0]['blocked']) self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#')) # same search, but details omitted by 'no_totals' last_result = result result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', no_totals=True)) self.assertEqual(result['items'], last_result['items']) # content was filtered by not_tag before censoring result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', not_tags=["good", "bad"])) self.assertEqual(0, len(result['items'])) self.assertEqual({"channels": [], "total": 0}, result['blocked']) # filtered content can still be resolved result = await self.resolve('lbry://@some_channel/bad_content') self.assertEqual(bad_content_id, result['claim_id']) blocking_channel_id = self.get_claim_id( await self.channel_create('@blocking', '0.1') ) # test setting from env vars and starting from scratch await self.conductor.spv_node.stop(False) await self.conductor.spv_node.start(self.conductor.lbcwallet_node, extraconf={'blocking_channel_ids': [blocking_channel_id], 'filtering_channel_ids': [filtering_channel_id]}) await self.daemon.wallet_manager.reset() self.assertEqual(0, len(self.conductor.spv_node.es_writer.db.blocked_streams)) await self.stream_repost(bad_content_id, 'block1', '0.1', channel_name='@blocking') self.assertEqual(1, len(self.conductor.spv_node.es_writer.db.blocked_streams)) # blocked content is not resolveable error = (await self.resolve('lbry://@some_channel/bad_content'))['error'] self.assertEqual(error['name'], 'BLOCKED') self.assertTrue(error['text'].startswith(f"Resolve of 'lbry://@some_channel#{some_channel_id[:1]}/bad_content#{bad_content_id[:1]}' was blocked")) self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#')) # local claim list still finds local reposted content that's blocked claims = await self.claim_list(reposted_claim_id=bad_content_id) self.assertEqual(claims[0]['name'], 'block1') self.assertEqual(claims[0]['value']['claim_id'], bad_content_id) self.assertEqual(claims[1]['name'], 'filter1') self.assertEqual(claims[1]['value']['claim_id'], bad_content_id) # a filtered/blocked channel impacts all content inside it bad_channel_id = self.get_claim_id( await self.channel_create('@bad_channel', '0.1', tags=['bad-stuff']) ) worse_content_id = self.get_claim_id( await self.stream_create('worse_content', '0.1', channel_name='@bad_channel', tags=['bad-stuff']) ) # check search before filtering channel result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height'])) self.assertEqual(2, result['total_items']) self.assertEqual('worse_content', result['items'][0]['name']) self.assertEqual('@bad_channel', result['items'][1]['name']) # filter channel out self.assertEqual(0, len(self.conductor.spv_node.server.db.filtered_channels)) await self.stream_repost(bad_channel_id, 'filter2', '0.1', channel_name='@filtering') self.assertEqual(1, len(self.conductor.spv_node.server.db.filtered_channels)) # same claim search as previous now returns 0 results result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height'])) filtered = result['blocked'] self.assertEqual(0, len(result['items'])) self.assertEqual(3, filtered['total']) self.assertEqual(1, len(filtered['channels'])) self.assertEqual(3, filtered['channels'][0]['blocked']) self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#')) # same search, but details omitted by 'no_totals' last_result = result result = await self.out( self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height'], no_totals=True) ) self.assertEqual(result['items'], last_result['items']) # filtered channel should still resolve result = await self.resolve('lbry://@bad_channel') self.assertEqual(bad_channel_id, result['claim_id']) result = await self.resolve('lbry://@bad_channel/worse_content') self.assertEqual(worse_content_id, result['claim_id']) # block channel self.assertEqual(0, len(self.conductor.spv_node.server.db.blocked_channels)) await self.stream_repost(bad_channel_id, 'block2', '0.1', channel_name='@blocking') self.assertEqual(1, len(self.conductor.spv_node.server.db.blocked_channels)) # channel, claim in channel or claim individually no longer resolve self.assertEqual((await self.resolve('lbry://@bad_channel'))['error']['name'], 'BLOCKED') self.assertEqual((await self.resolve('lbry://worse_content'))['error']['name'], 'BLOCKED') self.assertEqual((await self.resolve('lbry://@bad_channel/worse_content'))['error']['name'], 'BLOCKED') await self.stream_update(worse_content_id, channel_name='@bad_channel', tags=['bad-stuff']) self.assertEqual((await self.resolve('lbry://@bad_channel'))['error']['name'], 'BLOCKED') self.assertEqual((await self.resolve('lbry://worse_content'))['error']['name'], 'BLOCKED') self.assertEqual((await self.resolve('lbry://@bad_channel/worse_content'))['error']['name'], 'BLOCKED') async def test_publish_updates_file_list(self): tx = await self.stream_create(title='created') txo = tx['outputs'][0] claim_id, expected = txo['claim_id'], txo['value'] files = await self.file_list() self.assertEqual(1, len(files)) self.assertEqual(tx['txid'], files[0]['txid']) self.assertEqual(expected, files[0]['metadata']) # update with metadata-only changes tx = await self.stream_update(claim_id, title='update 1') files = await self.file_list() expected['title'] = 'update 1' self.assertEqual(1, len(files)) self.assertEqual(tx['txid'], files[0]['txid']) self.assertEqual(expected, files[0]['metadata']) # update with new data tx = await self.stream_update(claim_id, title='update 2', data=b'updated data') expected = tx['outputs'][0]['value'] files = await self.file_list() self.assertEqual(1, len(files)) self.assertEqual(tx['txid'], files[0]['txid']) self.assertEqual(expected, files[0]['metadata']) async def test_setting_stream_fields(self): values = { 'title': "Cool Content", 'description': "Best content on LBRY.", 'thumbnail_url': "https://co.ol/thumbnail.png", 'tags': ["cool", "awesome"], 'languages': ["en"], 'locations': ['US:NH:Manchester:03101:42.990605:-71.460989'], 'author': "Jules Verne", 'license': 'Public Domain', 'license_url': "https://co.ol/license", 'release_time': 123456, 'fee_currency': 'usd', 'fee_amount': '2.99', 'fee_address': 'mmCsWAiXMUVecFQ3fVzUwvpT9XFMXno2Ca', } fixed_values = values.copy() fixed_values['locations'] = [{ 'country': 'US', 'state': 'NH', 'city': 'Manchester', 'code': '03101', 'latitude': '42.990605', 'longitude': '-71.460989' }] fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')} fixed_values['release_time'] = str(values['release_time']) fixed_values['stream_type'] = 'binary' fixed_values['source'] = { 'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e', 'media_type': 'application/octet-stream', 'size': '3' } fixed_values['fee'] = { 'address': fixed_values.pop('fee_address'), 'amount': fixed_values.pop('fee_amount'), 'currency': fixed_values.pop('fee_currency').upper() } # create new stream with all fields set tx = await self.out(self.stream_create('big', **values)) stream = tx['outputs'][0]['value'] fixed_values['source']['name'] = stream['source']['name'] fixed_values['source']['sd_hash'] = stream['source']['sd_hash'] self.assertEqual(stream, fixed_values) # create stream with nothing set tx = await self.out(self.stream_create('light')) stream = tx['outputs'][0]['value'] self.assertEqual( stream, { 'stream_type': 'binary', 'source': { 'size': '3', 'media_type': 'application/octet-stream', 'name': stream['source']['name'], 'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e', 'sd_hash': stream['source']['sd_hash'] }, } ) # create stream with just some tags, langs and locations tx = await self.out(self.stream_create('updated', tags='blah', languages='uk', locations='UA::Kyiv')) txo = tx['outputs'][0] claim_id, stream = txo['claim_id'], txo['value'] fixed_values['source']['name'] = stream['source']['name'] fixed_values['source']['sd_hash'] = stream['source']['sd_hash'] self.assertEqual( stream, { 'stream_type': 'binary', 'source': { 'size': '3', 'media_type': 'application/octet-stream', 'name': fixed_values['source']['name'], 'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e', 'sd_hash': fixed_values['source']['sd_hash'], }, 'tags': ['blah'], 'languages': ['uk'], 'locations': [{'country': 'UA', 'city': 'Kyiv'}] } ) # update stream setting all fields, 'source' doesn't change tx = await self.out(self.stream_update(claim_id, **values)) stream = tx['outputs'][0]['value'] fixed_values['tags'].insert(0, 'blah') # existing tag fixed_values['languages'].insert(0, 'uk') # existing language fixed_values['locations'].insert(0, {'country': 'UA', 'city': 'Kyiv'}) # existing location self.assertEqual(stream, fixed_values) # clearing and settings tags, languages and locations tx = await self.out(self.stream_update( claim_id, tags='single', clear_tags=True, languages='pt', clear_languages=True, locations='BR', clear_locations=True, )) txo = tx['outputs'][0] fixed_values['tags'] = ['single'] fixed_values['languages'] = ['pt'] fixed_values['locations'] = [{'country': 'BR'}] self.assertEqual(txo['value'], fixed_values) # modifying hash/size/name fixed_values['source']['name'] = 'changed_name' fixed_values['source']['hash'] = 'cafebeef' fixed_values['source']['size'] = '42' tx = await self.out(self.stream_update( claim_id, file_name='changed_name', file_hash='cafebeef', file_size=42 )) self.assertEqual(tx['outputs'][0]['value'], fixed_values) # stream_update re-signs with the same channel channel_id = self.get_claim_id(await self.channel_create('@chan')) tx = await self.stream_update(claim_id, channel_id=channel_id) self.assertEqual(tx['outputs'][0]['signing_channel']['name'], '@chan') tx = await self.stream_update(claim_id, title='channel re-signs') self.assertEqual(tx['outputs'][0]['value']['title'], 'channel re-signs') self.assertEqual(tx['outputs'][0]['signing_channel']['name'], '@chan') # send claim to someone else new_account = await self.out(self.daemon.jsonrpc_account_create('second account')) account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id']) # before sending self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 4) self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=self.account.id), 4) self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=account2_id), 0) other_address = await account2.receiving.get_or_create_usable_address() tx = await self.out(self.stream_update(claim_id, claim_address=other_address)) # after sending self.assertItemCount(await self.daemon.jsonrpc_claim_list(), 4) self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=self.account.id), 3) self.assertItemCount(await self.daemon.jsonrpc_claim_list(account_id=account2_id), 1) self.assertEqual(4, len(await self.claim_search(release_time='>0', order_by=['release_time']))) self.assertEqual(3, len(await self.claim_search(release_time='>0', order_by=['release_time'], claim_type='stream'))) self.assertEqual(4, len(await self.claim_search(release_time='>=0', order_by=['release_time']))) self.assertEqual(4, len(await self.claim_search(order_by=['release_time']))) self.assertEqual(3, len(await self.claim_search(claim_type='stream', order_by=['release_time']))) self.assertEqual(1, len(await self.claim_search(claim_type='channel', order_by=['release_time']))) self.assertEqual(2, len(await self.claim_search(release_time='>=123456', order_by=['release_time']))) self.assertEqual(1, len(await self.claim_search(release_time='>=123456', order_by=['release_time'], claim_type='stream'))) self.assertEqual(1, len(await self.claim_search(release_time='>123456', order_by=['release_time'], claim_type='stream'))) self.assertEqual(2, len(await self.claim_search(release_time='>123456', order_by=['release_time']))) self.assertEqual(3, len(await self.claim_search(release_time='<123457', order_by=['release_time']))) self.assertEqual(2, len(await self.claim_search(release_time='<123457', order_by=['release_time'], claim_type='stream'))) self.assertEqual(2, len(await self.claim_search(release_time=['<123457'], order_by=['release_time'], claim_type='stream'))) self.assertEqual(3, len(await self.claim_search(release_time=['<123457'], order_by=['release_time']))) self.assertEqual(3, len(await self.claim_search(release_time=['>0', '<123457'], order_by=['release_time']))) self.assertEqual(2, len(await self.claim_search(release_time=['>0', '<123457'], order_by=['release_time'], claim_type='stream'))) self.assertEqual(3, len(await self.claim_search(release_time=['<123457'], order_by=['release_time'], height=['>0']))) self.assertEqual(4, len(await self.claim_search(order_by=['release_time'], height=['>0']))) self.assertEqual(4, len(await self.claim_search(order_by=['release_time'], height=['>0'], claim_type=['stream', 'channel']))) self.assertEqual( 3, len(await self.claim_search(release_time=['>=123097', '<123457'], order_by=['release_time'])) ) self.assertEqual( 3, len(await self.claim_search(release_time=['<123457', '>0'], order_by=['release_time'])) ) async def test_setting_fee_fields(self): tx = await self.out(self.stream_create('paid-stream')) txo = tx['outputs'][0] claim_id, stream = txo['claim_id'], txo['value'] fee_address = 'mmCsWAiXMUVecFQ3fVzUwvpT9XFMXno2Ca' self.assertNotIn('fee', stream) # --replace=false # validation with self.assertRaisesRegex(Exception, 'please specify a fee currency'): await self.stream_update(claim_id, fee_amount='0.1') with self.assertRaisesRegex(Exception, 'unknown currency provided: foo'): await self.stream_update(claim_id, fee_amount='0.1', fee_currency="foo") with self.assertRaisesRegex(Exception, 'please specify a fee amount'): await self.stream_update(claim_id, fee_currency='usd') with self.assertRaisesRegex(Exception, 'please specify a fee amount'): await self.stream_update(claim_id, fee_address=fee_address) # set just amount and currency with default address tx = await self.stream_update( claim_id, fee_amount='0.99', fee_currency='lbc' ) self.assertEqual( tx['outputs'][0]['value']['fee'], {'amount': '0.99', 'currency': 'LBC', 'address': txo['address']} ) # set all fee fields tx = await self.stream_update( claim_id, fee_amount='0.1', fee_currency='usd', fee_address=fee_address ) self.assertEqual( tx['outputs'][0]['value']['fee'], {'amount': '0.1', 'currency': 'USD', 'address': fee_address} ) # change just address tx = await self.stream_update(claim_id, fee_address=txo['address']) self.assertEqual( tx['outputs'][0]['value']['fee'], {'amount': '0.1', 'currency': 'USD', 'address': txo['address']} ) # change just amount (does not reset fee_address) tx = await self.stream_update(claim_id, fee_amount='0.2') self.assertEqual( tx['outputs'][0]['value']['fee'], {'amount': '0.2', 'currency': 'USD', 'address': txo['address']} ) # changing currency without an amount is never allowed, even if previous amount exists with self.assertRaises(Exception, msg='In order to set a fee currency, please specify a fee amount'): await self.stream_update(claim_id, fee_currency='usd') # clearing fee tx = await self.out(self.stream_update(claim_id, clear_fee=True)) self.assertNotIn('fee', tx['outputs'][0]['value']) # --replace=true # set just amount and currency with default address tx = await self.stream_update( claim_id, fee_amount='0.99', fee_currency='lbc', replace=True ) self.assertEqual( tx['outputs'][0]['value']['fee'], {'amount': '0.99', 'currency': 'LBC', 'address': txo['address']} ) # set all fee fields tx = await self.stream_update( claim_id, fee_amount='0.1', fee_currency='usd', fee_address=fee_address, replace=True ) self.assertEqual( tx['outputs'][0]['value']['fee'], {'amount': '0.1', 'currency': 'USD', 'address': fee_address} ) # validation with self.assertRaisesRegex(Exception, 'please specify a fee currency'): await self.stream_update(claim_id, fee_amount='0.1', replace=True) with self.assertRaisesRegex(Exception, 'unknown currency provided: foo'): await self.stream_update(claim_id, fee_amount='0.1', fee_currency="foo", replace=True) with self.assertRaisesRegex(Exception, 'please specify a fee amount'): await self.stream_update(claim_id, fee_currency='usd', replace=True) with self.assertRaisesRegex(Exception, 'please specify a fee amount'): await self.stream_update(claim_id, fee_address=fee_address, replace=True) async def test_automatic_type_and_metadata_detection_for_image(self): txo = (await self.stream_create('blank-image', data=self.image_data, suffix='.png'))['outputs'][0] self.assertEqual( txo['value'], { 'source': { 'size': '99', 'name': txo['value']['source']['name'], 'media_type': 'image/png', 'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545', 'sd_hash': txo['value']['source']['sd_hash'], }, 'stream_type': 'image', 'image': { 'width': 5, 'height': 7 } } ) async def test_automatic_type_and_metadata_detection_for_video(self): txo = (await self.stream_create('chrome', file_path=self.video_file_name))['outputs'][0] self.assertEqual( txo['value'], { 'source': { 'size': '2299653', 'name': 'ForBiggerEscapes.mp4', 'media_type': 'video/mp4', 'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315', 'sd_hash': txo['value']['source']['sd_hash'], }, 'stream_type': 'video', 'video': { 'width': 1280, 'height': 720, 'duration': 15 } } ) async def test_overriding_automatic_metadata_detection(self): tx = await self.out( self.daemon.jsonrpc_stream_create( 'chrome', '1.0', file_path=self.video_file_name, width=99, height=88, duration=9 ) ) txo = tx['outputs'][0] self.assertEqual( txo['value'], { 'source': { 'size': '2299653', 'name': 'ForBiggerEscapes.mp4', 'media_type': 'video/mp4', 'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315', 'sd_hash': txo['value']['source']['sd_hash'], }, 'stream_type': 'video', 'video': { 'width': 99, 'height': 88, 'duration': 9 } } ) async def test_update_file_type(self): video_txo = (await self.stream_create('chrome', file_path=self.video_file_name))['outputs'][0] self.assertSetEqual(set(video_txo['value']), {'source', 'stream_type', 'video'}) self.assertEqual(video_txo['value']['stream_type'], 'video') self.assertEqual(video_txo['value']['source']['media_type'], 'video/mp4') self.assertEqual( video_txo['value']['video'], { 'duration': 15, 'height': 720, 'width': 1280 } ) claim_id = video_txo['claim_id'] binary_txo = (await self.stream_update(claim_id, data=b'hi!'))['outputs'][0] self.assertEqual(binary_txo['value']['stream_type'], 'binary') self.assertEqual(binary_txo['value']['source']['media_type'], 'application/octet-stream') self.assertSetEqual(set(binary_txo['value']), {'source', 'stream_type'}) image_txo = (await self.stream_update(claim_id, data=self.image_data, suffix='.png'))['outputs'][0] self.assertSetEqual(set(image_txo['value']), {'source', 'stream_type', 'image'}) self.assertEqual(image_txo['value']['stream_type'], 'image') self.assertEqual(image_txo['value']['source']['media_type'], 'image/png') self.assertEqual(image_txo['value']['image'], {'height': 7, 'width': 5}) async def test_replace_mode_preserves_source_and_type(self): expected = { 'tags': ['blah'], 'languages': ['uk'], 'locations': [{'country': 'UA', 'city': 'Kyiv'}], 'source': { 'size': '2299653', 'name': 'ForBiggerEscapes.mp4', 'media_type': 'video/mp4', 'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315', }, 'stream_type': 'video', 'video': { 'width': 1280, 'height': 720, 'duration': 15 } } channel = await self.channel_create('@chan') tx = await self.out(self.daemon.jsonrpc_stream_create( 'chrome', '1.0', file_path=self.video_file_name, tags='blah', languages='uk', locations='UA::Kyiv', channel_id=self.get_claim_id(channel) )) await self.on_transaction_dict(tx) txo = tx['outputs'][0] expected['source']['sd_hash'] = txo['value']['source']['sd_hash'] self.assertEqual(txo['value'], expected) self.assertEqual(txo['signing_channel']['name'], '@chan') tx = await self.out(self.daemon.jsonrpc_stream_update( txo['claim_id'], title='new title', replace=True )) txo = tx['outputs'][0] expected['title'] = 'new title' del expected['tags'] del expected['languages'] del expected['locations'] self.assertEqual(txo['value'], expected) self.assertNotIn('signing_channel', txo) async def test_create_update_and_abandon_stream(self): await self.assertBalance(self.account, '10.0') tx = await self.stream_create(bid='2.5') # creates new claim claim_id = self.get_claim_id(tx) txs = await self.transaction_list() self.assertEqual(len(txs[0]['claim_info']), 1) self.assertEqual(txs[0]['confirmations'], 1) self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5') self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim_id) self.assertFalse(txs[0]['claim_info'][0]['is_spent']) self.assertEqual(txs[0]['value'], '0.0') self.assertEqual(txs[0]['fee'], '-0.020107') await self.assertBalance(self.account, '7.479893') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) await self.daemon.jsonrpc_file_delete(delete_all=True) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) await self.stream_update(claim_id, bid='1.0') # updates previous claim txs = await self.transaction_list() self.assertEqual(len(txs[0]['update_info']), 1) self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5') self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id) self.assertFalse(txs[0]['update_info'][0]['is_spent']) self.assertTrue(txs[1]['claim_info'][0]['is_spent']) self.assertEqual(txs[0]['value'], '0.0') self.assertEqual(txs[0]['fee'], '-0.0002165') await self.assertBalance(self.account, '8.9796765') await self.stream_abandon(claim_id) txs = await self.transaction_list() self.assertEqual(len(txs[0]['abandon_info']), 1) self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0') self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id) self.assertTrue(txs[1]['update_info'][0]['is_spent']) self.assertTrue(txs[2]['claim_info'][0]['is_spent']) self.assertEqual(txs[0]['value'], '0.0') self.assertEqual(txs[0]['fee'], '-0.000107') await self.assertBalance(self.account, '9.9795695') async def test_abandoning_stream_at_loss(self): await self.assertBalance(self.account, '10.0') tx = await self.stream_create(bid='0.0001') await self.assertBalance(self.account, '9.979793') await self.stream_abandon(self.get_claim_id(tx)) await self.assertBalance(self.account, '9.97968399') async def test_publish(self): # errors on missing arguments to create a stream with self.assertRaisesRegex(Exception, "'bid' is a required argument for new publishes."): await self.daemon.jsonrpc_publish('foo') # successfully create stream with tempfile.NamedTemporaryFile() as file: file.write(b'hi') file.flush() tx1 = await self.publish('foo', bid='1.0', file_path=file.name) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) # doesn't error on missing arguments when doing an update stream tx2 = await self.publish('foo', tags='updated') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.assertEqual(self.get_claim_id(tx1), self.get_claim_id(tx2)) # update conflict with two claims of the same name tx3 = await self.stream_create('foo', allow_duplicate_name=True) with self.assertRaisesRegex(Exception, "There are 2 claims for 'foo'"): await self.daemon.jsonrpc_publish('foo') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2) # abandon duplicate stream await self.stream_abandon(self.get_claim_id(tx3)) # publish to a channel await self.channel_create('@abc') tx3 = await self.publish('foo', channel_name='@abc') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2) r = await self.resolve('lbry://@abc/foo') self.assertEqual( r['claim_id'], self.get_claim_id(tx3) ) # publishing again clears channel tx4 = await self.publish('foo', languages='uk-UA', tags=['Anime', 'anime ']) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2) claim = await self.resolve('lbry://foo') self.assertEqual(claim['txid'], tx4['outputs'][0]['txid']) self.assertNotIn('signing_channel', claim) self.assertEqual(claim['value']['languages'], ['uk-UA']) self.assertEqual(claim['value']['tags'], ['anime']) # publish a stream with no source tx5 = await self.publish( 'future-release', bid='0.1', languages='uk-UA', tags=['Anime', 'anime '] ) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2) claim = await self.resolve('lbry://future-release') self.assertEqual(claim['txid'], tx5['outputs'][0]['txid']) self.assertNotIn('signing_channel', claim) self.assertEqual(claim['value']['languages'], ['uk-UA']) self.assertEqual(claim['value']['tags'], ['anime']) self.assertNotIn('source', claim['value']) # change metadata before the release await self.publish( 'future-release', bid='0.1', tags=['Anime', 'anime ', 'psy-trance'], title='Psy will be over 9000!!!' ) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2) claim = await self.resolve('lbry://future-release') self.assertEqual(claim['value']['tags'], ['anime', 'psy-trance']) self.assertEqual(claim['value']['title'], 'Psy will be over 9000!!!') self.assertNotIn('source', claim['value']) # update the stream to have a source tx6 = await self.publish( 'future-release', sd_hash='beef', file_hash='beef', file_name='blah.mp3', tags=['something-else'] ) claim = await self.resolve('lbry://future-release') self.assertEqual(claim['txid'], tx6['outputs'][0]['txid']) self.assertEqual(claim['value']['tags'], ['something-else']) self.assertEqual(claim['value']['source']['sd_hash'], 'beef') self.assertEqual(claim['value']['source']['hash'], 'beef') self.assertEqual(claim['value']['source']['name'], 'blah.mp3') self.assertEqual(claim['value']['source']['media_type'], 'audio/mpeg') class SupportCommands(CommandTestCase): async def test_regular_supports_and_tip_supports(self): wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True) account2 = wallet2.accounts[0] # send account2 5 LBC out of the 10 LBC in account1 result = await self.out(self.daemon.jsonrpc_account_send( '5.0', await self.daemon.jsonrpc_address_unused(wallet_id='wallet2') )) await self.on_transaction_dict(result) # account1 and account2 balances: await self.assertBalance(self.account, '4.999876') await self.assertBalance(account2, '5.0') # create the claim we'll be tipping and supporting claim_id = self.get_claim_id(await self.stream_create()) # account1 and account2 balances: await self.assertBalance(self.account, '3.979769') await self.assertBalance(account2, '5.0') # send a tip to the claim using account2 tip = await self.out( self.daemon.jsonrpc_support_create( claim_id, '1.0', True, account_id=account2.id, wallet_id='wallet2', funding_account_ids=[account2.id], blocking=True) ) await self.confirm_tx(tip['txid']) # tips don't affect balance so account1 balance is same but account2 balance went down await self.assertBalance(self.account, '3.979769') await self.assertBalance(account2, '3.9998585') # verify that the incoming tip is marked correctly as is_tip=True in account1 txs = await self.transaction_list(account_id=self.account.id) self.assertEqual(len(txs[0]['support_info']), 1) self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0') self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id) self.assertTrue(txs[0]['support_info'][0]['is_tip']) self.assertFalse(txs[0]['support_info'][0]['is_spent']) self.assertEqual(txs[0]['value'], '1.0') self.assertEqual(txs[0]['fee'], '0.0') # verify that the outgoing tip is marked correctly as is_tip=True in account2 txs2 = await self.transaction_list(wallet_id='wallet2', account_id=account2.id) self.assertEqual(len(txs2[0]['support_info']), 1) self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0') self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id) self.assertTrue(txs2[0]['support_info'][0]['is_tip']) self.assertFalse(txs2[0]['support_info'][0]['is_spent']) self.assertEqual(txs2[0]['value'], '-1.0') self.assertEqual(txs2[0]['fee'], '-0.0001415') # send a support to the claim using account2 support = await self.out( self.daemon.jsonrpc_support_create( claim_id, '2.0', False, account_id=account2.id, wallet_id='wallet2', funding_account_ids=[account2.id], blocking=True) ) await self.confirm_tx(support['txid']) # account2 balance went down ~2 await self.assertBalance(self.account, '3.979769') await self.assertBalance(account2, '1.999717') # verify that the outgoing support is marked correctly as is_tip=False in account2 txs2 = await self.transaction_list(wallet_id='wallet2') self.assertEqual(len(txs2[0]['support_info']), 1) self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0') self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id) self.assertFalse(txs2[0]['support_info'][0]['is_tip']) self.assertFalse(txs2[0]['support_info'][0]['is_spent']) self.assertEqual(txs2[0]['value'], '0.0') self.assertEqual(txs2[0]['fee'], '-0.0001415') # abandoning the tip increases balance and shows tip as spent await self.support_abandon(claim_id) await self.assertBalance(self.account, '4.979662') txs = await self.transaction_list(account_id=self.account.id) self.assertEqual(len(txs[0]['abandon_info']), 1) self.assertEqual(len(txs[1]['support_info']), 1) self.assertTrue(txs[1]['support_info'][0]['is_tip']) self.assertTrue(txs[1]['support_info'][0]['is_spent']) async def test_signed_supports_with_no_change_txo_regression(self): # reproduces a bug where transactions did not get properly signed # if there was no change and just a single output # lbrycrd returned 'the transaction was rejected by network rules.' channel_id = self.get_claim_id(await self.channel_create()) stream_id = self.get_claim_id(await self.stream_create()) tx = await self.support_create(stream_id, '7.967601', channel_id=channel_id) self.assertEqual(len(tx['outputs']), 1) # must be one to reproduce bug self.assertTrue(tx['outputs'][0]['is_channel_signature_valid']) class CollectionCommands(CommandTestCase): async def test_collections(self): claim_ids = [ self.get_claim_id(tx) for tx in [ await self.stream_create('stream-one'), await self.stream_create('stream-two') ] ] claim_ids.append(claim_ids[0]) claim_ids.append('beef') tx = await self.collection_create('radjingles', claims=claim_ids, title="boring title") claim_id = self.get_claim_id(tx) collections = await self.out(self.daemon.jsonrpc_collection_list()) self.assertEqual(collections['items'][0]['value']['title'], 'boring title') self.assertEqual(collections['items'][0]['value']['claims'], claim_ids) self.assertEqual(collections['items'][0]['value_type'], 'collection') self.assertItemCount(collections, 1) await self.assertBalance(self.account, '6.939679') with self.assertRaisesRegex(Exception, "You already have a collection under the name 'radjingles'."): await self.collection_create('radjingles', claims=claim_ids) self.assertItemCount(await self.daemon.jsonrpc_collection_list(), 1) await self.assertBalance(self.account, '6.939679') collections = await self.out(self.daemon.jsonrpc_collection_list()) self.assertEqual(collections['items'][0]['value']['title'], 'boring title') await self.collection_update(claim_id, title='fancy title') collections = await self.out(self.daemon.jsonrpc_collection_list()) self.assertEqual(collections['items'][0]['value']['title'], 'fancy title') self.assertEqual(collections['items'][0]['value']['claims'], claim_ids) self.assertNotIn('claims', collections['items'][0]) tx = await self.collection_create('radjingles', claims=claim_ids, allow_duplicate_name=True) claim_id2 = self.get_claim_id(tx) self.assertItemCount(await self.daemon.jsonrpc_collection_list(), 2) # with clear_claims await self.collection_update(claim_id, clear_claims=True, claims=claim_ids[:2]) collections = await self.out(self.daemon.jsonrpc_collection_list()) self.assertEqual(len(collections['items']), 2) self.assertNotIn('canonical_url', collections['items'][0]) resolved_collections = await self.out(self.daemon.jsonrpc_collection_list(resolve=True)) self.assertIn('canonical_url', resolved_collections['items'][0]) # with replace await self.collection_update(claim_id, replace=True, claims=claim_ids[::-1][:2], tags=['cool']) updated = await self.claim_search(claim_id=claim_id) self.assertEqual(updated[0]['value']['tags'], ['cool']) self.assertEqual(updated[0]['value']['claims'], claim_ids[::-1][:2]) await self.collection_update(claim_id, replace=True, claims=claim_ids[:4], languages=['en', 'pt-BR']) updated = await self.resolve(f'radjingles:{claim_id}') self.assertEqual(updated['value']['claims'], claim_ids[:4]) self.assertNotIn('tags', updated['value']) self.assertEqual(updated['value']['languages'], ['en', 'pt-BR']) await self.collection_abandon(claim_id) self.assertItemCount(await self.daemon.jsonrpc_collection_list(), 1) collections = await self.out(self.daemon.jsonrpc_collection_list(resolve_claims=2)) self.assertEqual(len(collections['items'][0]['claims']), 2) collections = await self.out(self.daemon.jsonrpc_collection_list(resolve_claims=10)) self.assertEqual(len(collections['items'][0]['claims']), 4) self.assertEqual(collections['items'][0]['claims'][0]['name'], 'stream-one') self.assertEqual(collections['items'][0]['claims'][1]['name'], 'stream-two') self.assertEqual(collections['items'][0]['claims'][2]['name'], 'stream-one') self.assertIsNone(collections['items'][0]['claims'][3]) claims = await self.out(self.daemon.jsonrpc_claim_list()) self.assertEqual(claims['items'][0]['name'], 'radjingles') self.assertEqual(claims['items'][1]['name'], 'stream-two') self.assertEqual(claims['items'][2]['name'], 'stream-one') claims = await self.out(self.daemon.jsonrpc_collection_resolve(claim_id2)) self.assertEqual(claims['items'][0]['name'], 'stream-one') self.assertEqual(claims['items'][1]['name'], 'stream-two') self.assertEqual(claims['items'][2]['name'], 'stream-one') claims = await self.out(self.daemon.jsonrpc_collection_resolve(claim_id2, page=10)) self.assertEqual(claims['items'], []) ================================================ FILE: tests/integration/datanetwork/__init__.py ================================================ ================================================ FILE: tests/integration/datanetwork/test_dht.py ================================================ import asyncio from binascii import hexlify from lbry.extras.daemon.storage import SQLiteStorage from lbry.conf import Config from lbry.dht import constants from lbry.dht.node import Node from lbry.dht import peer as dht_peer from lbry.dht.peer import PeerManager, make_kademlia_peer from lbry.testcase import AsyncioTestCase class DHTIntegrationTest(AsyncioTestCase): async def asyncSetUp(self): dht_peer.ALLOW_LOCALHOST = True self.addCleanup(setattr, dht_peer, 'ALLOW_LOCALHOST', False) import logging logging.getLogger('asyncio').setLevel(logging.ERROR) logging.getLogger('lbry.dht').setLevel(logging.WARN) self.nodes = [] self.known_node_addresses = [] async def create_node(self, node_id, port, external_ip='127.0.0.1'): storage = SQLiteStorage(Config(), ":memory:", self.loop, self.loop.time) await storage.open() node = Node(self.loop, PeerManager(self.loop), node_id=node_id, udp_port=port, internal_udp_port=port, peer_port=3333, external_ip=external_ip, storage=storage) self.addCleanup(node.stop) node.protocol.rpc_timeout = .5 node.protocol.ping_queue._default_delay = .5 return node async def setup_network(self, size: int, start_port=40000, seed_nodes=1, external_ip='127.0.0.1'): for i in range(size): node_port = start_port + i node_id = constants.generate_id(i) node = await self.create_node(node_id, node_port) self.nodes.append(node) self.known_node_addresses.append((external_ip, node_port)) for node in self.nodes: node.start(external_ip, self.known_node_addresses[:seed_nodes]) async def test_replace_bad_nodes(self): await self.setup_network(20) await asyncio.gather(*[node.joined.wait() for node in self.nodes]) self.assertEqual(len(self.nodes), 20) node = self.nodes[0] bad_peers = [] for candidate in self.nodes[1:10]: address, port, node_id = candidate.protocol.external_ip, candidate.protocol.udp_port, candidate.protocol.node_id peer = make_kademlia_peer(node_id, address, udp_port=port) bad_peers.append(peer) node.protocol.add_peer(peer) candidate.stop() await asyncio.sleep(.3) # let pending events settle for bad_peer in bad_peers: self.assertIn(bad_peer, node.protocol.routing_table.get_peers()) await node.refresh_node(True) await asyncio.sleep(.3) # let pending events settle good_nodes = {good_node.protocol.node_id for good_node in self.nodes[10:]} for peer in node.protocol.routing_table.get_peers(): self.assertIn(peer.node_id, good_nodes) async def test_re_join(self): await self.setup_network(20, seed_nodes=10) await asyncio.gather(*[node.joined.wait() for node in self.nodes]) node = self.nodes[-1] self.assertTrue(node.joined.is_set()) self.assertTrue(node.protocol.routing_table.get_peers()) for network_node in self.nodes[:-1]: network_node.stop() await node.refresh_node(True) await asyncio.sleep(.3) # let pending events settle self.assertFalse(node.protocol.routing_table.get_peers()) for network_node in self.nodes[:-1]: network_node.start('127.0.0.1', self.known_node_addresses) self.assertFalse(node.protocol.routing_table.get_peers()) timeout = 20 while not node.protocol.routing_table.get_peers(): await asyncio.sleep(.1) timeout -= 1 if not timeout: self.fail("node didn't join back after 2 seconds") async def test_announce_no_peers(self): await self.setup_network(1) node = self.nodes[0] blob_hash = hexlify(constants.generate_id(1337)).decode() peers = await node.announce_blob(blob_hash) self.assertEqual(len(peers), 0) async def test_get_token_on_announce(self): await self.setup_network(2, seed_nodes=2) await asyncio.gather(*[node.joined.wait() for node in self.nodes]) node1, node2 = self.nodes node1.protocol.peer_manager.clear_token(node2.protocol.node_id) blob_hash = hexlify(constants.generate_id(1337)).decode() node_ids = await node1.announce_blob(blob_hash) self.assertIn(node2.protocol.node_id, node_ids) node2.protocol.node_rpc.refresh_token() node_ids = await node1.announce_blob(blob_hash) self.assertIn(node2.protocol.node_id, node_ids) node2.protocol.node_rpc.refresh_token() node_ids = await node1.announce_blob(blob_hash) self.assertIn(node2.protocol.node_id, node_ids) async def test_peer_search_removes_bad_peers(self): # that's an edge case discovered by Tom, but an important one # imagine that you only got bad peers and refresh will happen in one hour # instead of failing for one hour we should be able to recover by scheduling pings to bad peers we find await self.setup_network(2, seed_nodes=2) await asyncio.gather(*[node.joined.wait() for node in self.nodes]) node1, node2 = self.nodes node2.stop() # forcefully make it a bad peer but don't remove it from routing table address, port, node_id = node2.protocol.external_ip, node2.protocol.udp_port, node2.protocol.node_id peer = make_kademlia_peer(node_id, address, udp_port=port) self.assertTrue(node1.protocol.peer_manager.peer_is_good(peer)) node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port) node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port) self.assertFalse(node1.protocol.peer_manager.peer_is_good(peer)) # now a search happens, which removes bad peers while contacting them self.assertTrue(node1.protocol.routing_table.get_peers()) await node1.peer_search(node2.protocol.node_id) await asyncio.sleep(.3) # let pending events settle self.assertFalse(node1.protocol.routing_table.get_peers()) async def test_peer_persistance(self): num_nodes = 6 start_port = 40000 num_seeds = 2 external_ip = '127.0.0.1' # Start a node await self.setup_network(num_nodes, start_port=start_port, seed_nodes=num_seeds) await asyncio.gather(*[node.joined.wait() for node in self.nodes]) node1 = self.nodes[-1] peer_args = [(n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port, n.protocol.peer_port) for n in self.nodes[:num_seeds]] peers = [make_kademlia_peer(*args) for args in peer_args] # node1 is bootstrapped from the fixed seeds self.assertCountEqual(peers, node1.protocol.routing_table.get_peers()) # Refresh and assert that the peers were persisted await node1.refresh_node(True) self.assertEqual(len(peer_args), len(await node1._storage.get_persisted_kademlia_peers())) node1.stop() # Start a fresh node with the same node_id and storage, but no known peers node2 = await self.create_node(constants.generate_id(num_nodes-1), start_port+num_nodes-1) node2._storage = node1._storage node2.start(external_ip, []) await node2.joined.wait() # The peers are restored self.assertEqual(num_seeds, len(node2.protocol.routing_table.get_peers())) for bucket1, bucket2 in zip(node1.protocol.routing_table.buckets, node2.protocol.routing_table.buckets): self.assertEqual((bucket1.range_min, bucket1.range_max), (bucket2.range_min, bucket2.range_max)) ================================================ FILE: tests/integration/datanetwork/test_file_commands.py ================================================ import unittest from unittest import skipIf import asyncio import os from binascii import hexlify from lbry.schema import Claim from lbry.stream.background_downloader import BackgroundDownloader from lbry.stream.descriptor import StreamDescriptor from lbry.testcase import CommandTestCase from lbry.extras.daemon.components import TorrentSession, BACKGROUND_DOWNLOADER_COMPONENT from lbry.wallet import Transaction from lbry.torrent.tracker import UDPTrackerServerProtocol class FileCommands(CommandTestCase): def __init__(self, *a, **kw): super().__init__(*a, **kw) self.skip_libtorrent = False async def add_forever(self): while True: for handle in self.client_session._handles.values(): handle._handle.connect_peer(('127.0.0.1', 4040)) await asyncio.sleep(.1) async def initialize_torrent(self, tx_to_update=None): if not hasattr(self, 'seeder_session'): self.seeder_session = TorrentSession(self.loop, None) self.addCleanup(self.seeder_session.stop) await self.seeder_session.bind('127.0.0.1', port=4040) btih = await self.seeder_session.add_fake_torrent() address = await self.account.receiving.get_or_create_usable_address() if not tx_to_update: claim = Claim() claim.stream.update(bt_infohash=btih) tx = await Transaction.claim_create( 'torrent', claim, 1, address, [self.account], self.account ) else: claim = tx_to_update.outputs[0].claim claim.stream.update(bt_infohash=btih) tx = await Transaction.claim_update( tx_to_update.outputs[0], claim, 1, address, [self.account], self.account ) await tx.sign([self.account]) await self.broadcast_and_confirm(tx) self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session self.client_session.wait_start = False # fixme: this is super slow on tests task = asyncio.create_task(self.add_forever()) self.addCleanup(task.cancel) return tx, btih @skipIf(TorrentSession is None, "libtorrent not installed") async def test_download_torrent(self): tx, btih = await self.initialize_torrent() self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) # second call, see its there and move on self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, btih) self.assertIn(btih, self.client_session._handles) tx, new_btih = await self.initialize_torrent(tx) self.assertNotEqual(btih, new_btih) # claim now points to another torrent, update to it self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, new_btih) self.assertIn(new_btih, self.client_session._handles) self.assertNotIn(btih, self.client_session._handles) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) await self.daemon.jsonrpc_file_delete(delete_all=True) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) self.assertNotIn(new_btih, self.client_session._handles) async def create_streams_in_range(self, *args, **kwargs): self.stream_claim_ids = [] for i in range(*args, **kwargs): t = await self.stream_create(f'Stream_{i}', '0.00001') self.stream_claim_ids.append(t['outputs'][0]['claim_id']) async def test_file_reflect(self): tx = await self.stream_create('mirror', '0.01') sd_hash = tx['outputs'][0]['value']['source']['sd_hash'] self.assertEqual([], await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash)) all_except_sd = [ blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash ] await self.reflector.blob_manager.delete_blobs(all_except_sd) self.assertEqual(all_except_sd, await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash)) async def test_sd_blob_fields_fallback(self): claim_id = self.get_claim_id(await self.stream_create('foo', '0.01', suffix='.txt')) stream = (await self.daemon.jsonrpc_file_list())["items"][0] stream.descriptor.suggested_file_name = ' ' stream.descriptor.stream_name = ' ' stream.descriptor.stream_hash = stream.descriptor.get_stream_hash() sd_hash = stream.descriptor.sd_hash = stream.descriptor.calculate_sd_hash() await stream.descriptor.make_sd_blob() await self.daemon.jsonrpc_file_delete(claim_name='foo') await self.stream_update(claim_id=claim_id, sd_hash=sd_hash) file_dict = await self.out(self.daemon.jsonrpc_get('lbry://foo', save_file=True)) self.assertEqual(file_dict['suggested_file_name'], stream.file_name) self.assertEqual(file_dict['stream_name'], stream.file_name) self.assertEqual(file_dict['mime_type'], 'text/plain') async def test_file_management(self): await self.stream_create('foo', '0.01') await self.stream_create('foo2', '0.01') file1, file2 = await self.file_list('claim_name') self.assertEqual(file1['claim_name'], 'foo') self.assertEqual(file2['claim_name'], 'foo2') self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=[file1['claim_id'], file2['claim_id']]), 2) self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=file1['claim_id']), 1) self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=[file1['outpoint'], file2['outpoint']]), 2) self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=file1['outpoint']), 1) await self.daemon.jsonrpc_file_delete(claim_name='foo') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) await self.daemon.jsonrpc_file_delete(claim_name='foo2') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) await self.daemon.jsonrpc_get('lbry://foo') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) async def test_tracker_discovery(self): port = 50990 server = UDPTrackerServerProtocol() transport, _ = await self.loop.create_datagram_endpoint(lambda: server, local_addr=("127.0.0.1", port)) self.addCleanup(transport.close) self.daemon.conf.fixed_peers = [] self.daemon.conf.tracker_servers = [("127.0.0.1", port)] tx = await self.stream_create('foo', '0.01') sd_hash = tx['outputs'][0]['value']['source']['sd_hash'] self.assertNotIn(bytes.fromhex(sd_hash)[:20], server.peers) server.add_peer(bytes.fromhex(sd_hash)[:20], "127.0.0.1", 5567) self.assertEqual(1, len(server.peers[bytes.fromhex(sd_hash)[:20]])) self.assertTrue(await self.daemon.jsonrpc_file_delete(delete_all=True)) stream = await self.daemon.jsonrpc_get('foo', save_file=True) await self.wait_files_to_complete() self.assertEqual(0, stream.blobs_remaining) self.assertEqual(2, len(server.peers[bytes.fromhex(sd_hash)[:20]])) self.assertEqual([{'address': '127.0.0.1', 'node_id': None, 'tcp_port': 5567, 'udp_port': None}, {'address': '127.0.0.1', 'node_id': None, 'tcp_port': 4444, 'udp_port': None}], (await self.daemon.jsonrpc_peer_list(sd_hash))['items']) async def test_announces(self): # announces on publish self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), []) await self.stream_create('foo', '0.01') stream = (await self.daemon.jsonrpc_file_list())["items"][0] self.assertSetEqual(set(await self.daemon.storage.get_blobs_to_announce()), {stream.sd_hash}) self.assertTrue(await self.daemon.jsonrpc_file_delete(delete_all=True)) # announces on download self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), []) stream = await self.daemon.jsonrpc_get('foo') self.assertSetEqual(set(await self.daemon.storage.get_blobs_to_announce()), {stream.sd_hash}) async def _purge_file(self, claim_name, full_path): self.assertTrue( await self.daemon.jsonrpc_file_delete(claim_name=claim_name, delete_from_download_dir=True) ) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) self.assertFalse(os.path.isfile(full_path)) async def test_publish_with_illegal_chars(self): def check_prefix_suffix(name, prefix, suffix): self.assertTrue(name.startswith(prefix)) self.assertTrue(name.endswith(suffix)) # Stream a file with file name containing invalid chars claim_name = 'lolwindows' prefix, suffix = 'derp?', '.ext.' san_prefix, san_suffix = 'derp', '.ext' tx = await self.stream_create(claim_name, '0.01', prefix=prefix, suffix=suffix) stream = (await self.daemon.jsonrpc_file_list())["items"][0] claim_id = self.get_claim_id(tx) # Assert that file list and source contains the local unsanitized name, but suggested name is sanitized full_path = (await self.daemon.jsonrpc_get('lbry://' + claim_name)).full_path stream_file_name = os.path.basename(full_path) source_file_name = tx['outputs'][0]['value']['source']['name'] file_list_name = stream.file_name suggested_file_name = stream.descriptor.suggested_file_name self.assertTrue(os.path.isfile(full_path)) check_prefix_suffix(stream_file_name, prefix, suffix) self.assertEqual(stream_file_name, source_file_name) self.assertEqual(stream_file_name, file_list_name) check_prefix_suffix(suggested_file_name, san_prefix, san_suffix) await self._purge_file(claim_name, full_path) # Re-download deleted file and assert that the file name is sanitized full_path = (await self.daemon.jsonrpc_get('lbry://' + claim_name, save_file=True)).full_path stream_file_name = os.path.basename(full_path) stream = (await self.daemon.jsonrpc_file_list())["items"][0] file_list_name = stream.file_name suggested_file_name = stream.descriptor.suggested_file_name self.assertTrue(os.path.isfile(full_path)) check_prefix_suffix(stream_file_name, san_prefix, san_suffix) self.assertEqual(stream_file_name, file_list_name) self.assertEqual(stream_file_name, suggested_file_name) await self._purge_file(claim_name, full_path) # Assert that the downloaded file name is not sanitized when user provides custom file name custom_name = 'cust*m_name' full_path = (await self.daemon.jsonrpc_get( 'lbry://' + claim_name, file_name=custom_name, save_file=True)).full_path file_name_on_disk = os.path.basename(full_path) self.assertTrue(os.path.isfile(full_path)) self.assertEqual(custom_name, file_name_on_disk) # Update the stream and assert the file name is not sanitized, but the suggested file name is prefix, suffix = 'derpyderp?', '.ext.' san_prefix, san_suffix = 'derpyderp', '.ext' tx = await self.stream_update(claim_id, data=b'amazing content', prefix=prefix, suffix=suffix) full_path = (await self.daemon.jsonrpc_get('lbry://' + claim_name, save_file=True)).full_path updated_stream = (await self.daemon.jsonrpc_file_list())["items"][0] stream_file_name = os.path.basename(full_path) source_file_name = tx['outputs'][0]['value']['source']['name'] file_list_name = updated_stream.file_name suggested_file_name = updated_stream.descriptor.suggested_file_name self.assertTrue(os.path.isfile(full_path)) check_prefix_suffix(stream_file_name, prefix, suffix) self.assertEqual(stream_file_name, source_file_name) self.assertEqual(stream_file_name, file_list_name) check_prefix_suffix(suggested_file_name, san_prefix, san_suffix) async def test_file_list_fields(self): await self.stream_create('foo', '0.01') file_list = await self.file_list() self.assertEqual( file_list[0]['timestamp'], self.ledger.headers.estimated_timestamp(file_list[0]['height']) ) self.assertEqual(file_list[0]['confirmations'], -1) await self.daemon.jsonrpc_resolve('foo') file_list = await self.file_list() self.assertEqual( file_list[0]['timestamp'], self.ledger.headers.estimated_timestamp(file_list[0]['height']) ) self.assertEqual(file_list[0]['confirmations'], 1) async def test_get_doesnt_touch_user_written_files_between_calls(self): await self.stream_create('foo', '0.01', data=bytes([0] * (2 << 23))) self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo')) first_path = (await self.daemon.jsonrpc_get('lbry://foo', save_file=True)).full_path await self.wait_files_to_complete() self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo')) with open(first_path, 'wb') as f: f.write(b' ') f.flush() second_path = await self.daemon.jsonrpc_get('lbry://foo', save_file=True) await self.wait_files_to_complete() self.assertNotEqual(first_path, second_path) @unittest.SkipTest # FIXME: claimname/updateclaim is gone. #3480 wip, unblock #3479" async def test_file_list_updated_metadata_on_resolve(self): await self.stream_create('foo', '0.01') txo = (await self.daemon.resolve(self.wallet.accounts, ['lbry://foo']))['lbry://foo'] claim = txo.claim await self.daemon.jsonrpc_file_delete(claim_name='foo') txid = await self.blockchain_claim_name('bar', hexlify(claim.to_bytes()).decode(), '0.01') await self.daemon.jsonrpc_get('lbry://bar') claim.stream.description = "fix typos, fix the world" await self.blockchain_update_name(txid, hexlify(claim.to_bytes()).decode(), '0.01') await self.daemon.jsonrpc_resolve('lbry://bar') file_list = (await self.daemon.jsonrpc_file_list())['items'] self.assertEqual(file_list[0].stream_claim_info.claim.stream.description, claim.stream.description) async def test_sourceless_content(self): # claim has no source, then it has one tx = await self.stream_create('foo', '0.01', data=None) claim_id = self.get_claim_id(tx) await self.daemon.jsonrpc_file_delete(claim_name='foo') response = await self.out(self.daemon.jsonrpc_get('lbry://foo')) self.assertIn('error', response) self.assertIn('nothing to download', response['error']) # source is set (there isn't a way to clear the source field, so we stop here for now) await self.stream_update(claim_id, data=b'surpriiiiiiiise') response = await self.out(self.daemon.jsonrpc_get('lbry://foo')) self.assertNotIn('error', response) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) async def test_file_list_paginated_output(self): await self.create_streams_in_range(0, 20) page = await self.file_list(page_size=20) page_claim_ids = [item['claim_id'] for item in page] self.assertListEqual(page_claim_ids, self.stream_claim_ids) page = await self.file_list(page_size=6) page_claim_ids = [item['claim_id'] for item in page] self.assertListEqual(page_claim_ids, self.stream_claim_ids[:6]) page = await self.file_list(page_size=6, page=2) page_claim_ids = [item['claim_id'] for item in page] self.assertListEqual(page_claim_ids, self.stream_claim_ids[6:12]) out_of_bounds = await self.file_list(page=5, page_size=6) self.assertEqual(out_of_bounds, []) complete = await self.daemon.jsonrpc_file_list() self.assertEqual(complete['total_pages'], 1) self.assertEqual(complete['total_items'], 20) page = await self.daemon.jsonrpc_file_list(page_size=10, page=1) self.assertEqual(page['total_pages'], 2) self.assertEqual(page['total_items'], 20) self.assertEqual(page['page'], 1) full = await self.out(self.daemon.jsonrpc_file_list(page_size=20, page=1)) page1 = await self.file_list(page=1, page_size=10) page2 = await self.file_list(page=2, page_size=10) self.assertEqual(page1 + page2, full['items']) async def test_download_different_timeouts(self): tx = await self.stream_create('foo', '0.01') sd_hash = tx['outputs'][0]['value']['source']['sd_hash'] await self.daemon.jsonrpc_file_delete(claim_name='foo') all_except_sd = [ blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash ] await self.server.blob_manager.delete_blobs(all_except_sd) resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True) self.assertIn('error', resp) self.assertEqual('Failed to download data blobs for sd hash %s within timeout.' % sd_hash, resp['error']) self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didn't create a file") await self.server.blob_manager.delete_blobs([sd_hash]) resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True) self.assertIn('error', resp) self.assertEqual('Failed to download sd blob %s within timeout.' % sd_hash, resp['error']) async def wait_files_to_complete(self): while await self.file_list(status='running'): await asyncio.sleep(0.01) async def test_filename_conflicts_management_on_resume_download(self): await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23))) file_info = (await self.file_list())[0] original_path = os.path.join(self.daemon.conf.download_dir, file_info['file_name']) await self.daemon.jsonrpc_file_delete(claim_name='foo') await self.daemon.jsonrpc_get('lbry://foo') with open(original_path, 'wb') as handle: handle.write(b'some other stuff was there instead') await self.daemon.file_manager.stop() await self.daemon.file_manager.start() await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed # check that internal state got through up to the file list API stream = self.daemon.file_manager.get_filtered(stream_hash=file_info['stream_hash'])[0] file_info = (await self.file_list())[0] self.assertEqual(stream.file_name, file_info['file_name']) # checks if what the API shows is what he have at the very internal level. self.assertEqual(stream.full_path, file_info['download_path']) async def test_incomplete_downloads_erases_output_file_on_stop(self): tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000) sd_hash = tx['outputs'][0]['value']['source']['sd_hash'] file_info = (await self.file_list())[0] blobs = await self.daemon.storage.get_blobs_for_stream( await self.daemon.storage.get_stream_hash_for_sd_hash(sd_hash) ) await self.daemon.jsonrpc_file_delete(claim_name='foo') self.assertEqual(5, len(blobs)) all_except_sd_and_head = [ blob.blob_hash for blob in blobs[1:-1] ] await self.server.blob_manager.delete_blobs(all_except_sd_and_head) path = os.path.join(self.daemon.conf.download_dir, file_info['file_name']) self.assertFalse(os.path.isfile(path)) resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2)) self.assertNotIn('error', resp) self.assertTrue(os.path.isfile(path)) await self.daemon.file_manager.stop() self.assertFalse(os.path.isfile(path)) async def test_incomplete_downloads_retry(self): tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000) sd_hash = tx['outputs'][0]['value']['source']['sd_hash'] blobs = await self.daemon.storage.get_blobs_for_stream( await self.daemon.storage.get_stream_hash_for_sd_hash(sd_hash) ) self.assertEqual(5, len(blobs)) await self.daemon.jsonrpc_file_delete(claim_name='foo') all_except_sd_and_head = [ blob.blob_hash for blob in blobs[1:-1] ] # backup server blobs for blob_hash in all_except_sd_and_head: blob = self.server_blob_manager.get_blob(blob_hash) os.rename(blob.file_path, blob.file_path + '__') # erase all except sd blob await self.server.blob_manager.delete_blobs(all_except_sd_and_head) # start the download resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2)) self.assertNotIn('error', resp) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.assertEqual('running', (await self.file_list())[0]['status']) # recover blobs for blob_hash in all_except_sd_and_head: blob = self.server_blob_manager.get_blob(blob_hash) os.rename(blob.file_path + '__', blob.file_path) self.server_blob_manager.blobs.clear() await self.server_blob_manager.blob_completed(self.server_blob_manager.get_blob(blob_hash)) await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) file_info = (await self.file_list())[0] self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream']) self.assertEqual('finished', file_info['status']) async def test_paid_download(self): target_address = await self.blockchain.get_raw_change_address() # FAIL: beyond available balance await self.stream_create( 'expensive', '0.01', data=b'pay me if you can', fee_currency='LBC', fee_amount='11.0', fee_address=target_address, claim_address=target_address ) await self.daemon.jsonrpc_file_delete(claim_name='expensive') response = await self.out(self.daemon.jsonrpc_get('lbry://expensive')) self.assertEqual(response['error'], 'Not enough funds to cover this transaction.') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) # FAIL: beyond maximum key fee await self.stream_create( 'maxkey', '0.01', data=b'no pay me, no', fee_currency='LBC', fee_amount='111.0', fee_address=target_address, claim_address=target_address ) await self.daemon.jsonrpc_file_delete(claim_name='maxkey') response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey')) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) self.assertEqual( response['error'], 'Purchase price of 111.0 LBC exceeds maximum configured price of 100.0 LBC (50.0 USD).' ) # PASS: purchase is successful await self.stream_create( 'icanpay', '0.01', data=b'I got the power!', fee_currency='LBC', fee_amount='1.0', fee_address=target_address, claim_address=target_address ) await self.daemon.jsonrpc_file_delete(claim_name='icanpay') await self.assertBalance(self.account, '9.925679') response = await self.daemon.jsonrpc_get('lbry://icanpay') raw_content_fee = response.content_fee.raw await self.ledger.wait(response.content_fee) await self.assertBalance(self.account, '8.925538') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) await asyncio.wait_for(self.wait_files_to_complete(), timeout=1) # check that the fee was received starting_balance = float(await self.blockchain.get_balance()) await self.generate(1) block_reward_and_claim_fee = 2.0 self.assertEqual( float(await self.blockchain.get_balance()), starting_balance + block_reward_and_claim_fee ) # restart the daemon and make sure the fee is still there await self.daemon.file_manager.stop() await self.daemon.file_manager.start() self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].content_fee.raw, raw_content_fee) await self.daemon.jsonrpc_file_delete(claim_name='icanpay') # PASS: no fee address --> use the claim address to pay tx = await self.stream_create( 'nofeeaddress', '0.01', data=b'free stuff?', ) await self.__raw_value_update_no_fee_address( tx, fee_amount='2.0', fee_currency='LBC', claim_address=target_address ) await self.daemon.jsonrpc_file_delete(claim_name='nofeeaddress') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) response = await self.out(self.daemon.jsonrpc_get('lbry://nofeeaddress')) self.assertIsNone((await self.daemon.jsonrpc_file_list())['items'][0].stream_claim_info.claim.stream.fee.address) self.assertIsNotNone(response['content_fee']) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.assertEqual(response['content_fee']['outputs'][0]['amount'], '2.0') self.assertEqual(response['content_fee']['outputs'][0]['address'], target_address) async def test_null_max_key_fee(self): target_address = await self.blockchain.get_raw_change_address() self.daemon.conf.max_key_fee = None await self.stream_create( 'somename', '0.5', data=b'Yes, please', fee_currency='LBC', fee_amount='1.0', fee_address=target_address, claim_address=target_address ) self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename')) # Assert the fee and bid are subtracted await self.assertBalance(self.account, '9.483893') response = await self.daemon.jsonrpc_get('lbry://somename') await self.ledger.wait(response.content_fee) await self.assertBalance(self.account, '8.483752') # Assert the file downloads await asyncio.wait_for(self.wait_files_to_complete(), timeout=1) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) # Assert the transaction is recorded to the blockchain starting_balance = float(await self.blockchain.get_balance()) await self.generate(1) block_reward_and_claim_fee = 2.0 self.assertEqual( float(await self.blockchain.get_balance()), starting_balance + block_reward_and_claim_fee ) async def test_null_fee(self): target_address = await self.blockchain.get_raw_change_address() tx = await self.stream_create( 'nullfee', '0.01', data=b'no pay me, no', fee_currency='LBC', fee_address=target_address, fee_amount='1.0' ) await self.__raw_value_update_no_fee_amount(tx, target_address) await self.daemon.jsonrpc_file_delete(claim_name='nullfee') response = await self.daemon.jsonrpc_get('lbry://nullfee') self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.assertIsNone(response.content_fee) self.assertTrue(response.stream_claim_info.claim.stream.has_fee) self.assertDictEqual( response.stream_claim_info.claim.stream.to_dict()['fee'], {'currency': 'LBC', 'address': target_address} ) await self.daemon.jsonrpc_file_delete(claim_name='nullfee') async def __raw_value_update_no_fee_address(self, tx, claim_address, **kwargs): tx = await self.daemon.jsonrpc_stream_update( self.get_claim_id(tx), preview=True, claim_address=claim_address, **kwargs ) tx.outputs[0].claim.stream.fee.address_bytes = b'' tx.outputs[0].script.generate() await tx.sign([self.account]) await self.broadcast_and_confirm(tx) async def __raw_value_update_no_fee_amount(self, tx, claim_address): tx = await self.daemon.jsonrpc_stream_update( self.get_claim_id(tx), preview=True, fee_currency='LBC', fee_amount='1.0', fee_address=claim_address, claim_address=claim_address ) tx.outputs[0].claim.stream.fee.message.ClearField('amount') tx.outputs[0].script.generate() await tx.sign([self.account]) await self.broadcast_and_confirm(tx) class DiskSpaceManagement(CommandTestCase): async def get_referenced_blobs(self, tx): sd_hash = tx['outputs'][0]['value']['source']['sd_hash'] stream_hash = await self.daemon.storage.get_stream_hash_for_sd_hash(sd_hash) return tx['outputs'][0]['value']['source']['sd_hash'], set(await self.blob_list( stream_hash=stream_hash )) async def test_file_management(self): status = await self.status() self.assertIn('disk_space', status) self.assertEqual(0, status['disk_space']['total_used_mb']) self.assertEqual(True, status['disk_space']['running']) sd_hash1, blobs1 = await self.get_referenced_blobs( await self.stream_create('foo1', '0.01', data=('0' * 2 * 1024 * 1024).encode()) ) sd_hash2, blobs2 = await self.get_referenced_blobs( await self.stream_create('foo2', '0.01', data=('0' * 3 * 1024 * 1024).encode()) ) sd_hash3, blobs3 = await self.get_referenced_blobs( await self.stream_create('foo3', '0.01', data=('0' * 3 * 1024 * 1024).encode()) ) sd_hash4, blobs4 = await self.get_referenced_blobs( await self.stream_create('foo4', '0.01', data=('0' * 2 * 1024 * 1024).encode()) ) await self.daemon.storage.update_blob_ownership(sd_hash1, False) await self.daemon.storage.update_blob_ownership(sd_hash3, False) await self.daemon.storage.update_blob_ownership(sd_hash4, False) await self.blob_clean() # just to refresh caches, has no effect self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(10, (await self.status())['disk_space']['total_used_mb']) self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list())) await self.blob_clean() self.assertEqual(10, (await self.status())['disk_space']['total_used_mb']) self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb']) self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list())) self.daemon.conf.blob_storage_limit = 6 await self.blob_clean() self.assertEqual(5, (await self.status())['disk_space']['total_used_mb']) self.assertEqual(2, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb']) blobs = set(await self.blob_list()) self.assertFalse(blobs1.issubset(blobs)) self.assertTrue(blobs2.issubset(blobs)) self.assertFalse(blobs3.issubset(blobs)) self.assertTrue(blobs4.issubset(blobs)) # check that pending blobs are not accounted (#3617) await self.daemon.storage.db.execute_fetchall("update blob set status='pending'") await self.blob_clean() # just to refresh caches, has no effect self.assertEqual(0, (await self.status())['disk_space']['total_used_mb']) self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(0, (await self.status())['disk_space']['published_blobs_storage_used_mb']) # check that added_on gets set on downloads (was a bug) self.assertLess(0, await self.daemon.storage.run_and_return_one_or_none("select min(added_on) from blob")) await self.daemon.jsonrpc_file_delete(delete_all=True) await self.daemon.jsonrpc_get("foo4", save_file=False) self.assertLess(0, await self.daemon.storage.run_and_return_one_or_none("select min(added_on) from blob")) class TestBackgroundDownloaderComponent(CommandTestCase): async def get_blobs_from_sd_blob(self, sd_blob): descriptor = await StreamDescriptor.from_stream_descriptor_blob( asyncio.get_running_loop(), self.daemon.blob_manager.blob_dir, sd_blob ) return descriptor.blobs async def assertBlobs(self, *sd_hashes, no_files=True): # checks that we have ony the finished blobs needed for the the referenced streams seen = set(sd_hashes) for sd_hash in sd_hashes: sd_blob = self.daemon.blob_manager.get_blob(sd_hash) self.assertTrue(sd_blob.get_is_verified()) blobs = await self.get_blobs_from_sd_blob(sd_blob) for blob in blobs[:-1]: self.assertTrue(self.daemon.blob_manager.get_blob(blob.blob_hash).get_is_verified()) seen.update(blob.blob_hash for blob in blobs if blob.blob_hash) if no_files: self.assertEqual(seen, self.daemon.blob_manager.completed_blob_hashes) self.assertEqual(0, len(await self.file_list())) async def clear(self): await self.daemon.jsonrpc_file_delete(delete_all=True) self.assertEqual(0, len(await self.file_list())) await self.daemon.blob_manager.delete_blobs(list(self.daemon.blob_manager.completed_blob_hashes), True) self.assertEqual(0, len((await self.daemon.jsonrpc_blob_list())['items'])) async def test_download(self): content1 = await self.stream_create('content1', '0.01', data=bytes([0] * 32 * 1024 * 1024)) content1 = content1['outputs'][0]['value']['source']['sd_hash'] content2 = await self.stream_create('content2', '0.01', data=bytes([0] * 16 * 1024 * 1024)) content2 = content2['outputs'][0]['value']['source']['sd_hash'] self.assertEqual(48, (await self.status())['disk_space']['published_blobs_storage_used_mb']) self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) background_downloader = BackgroundDownloader(self.daemon.conf, self.daemon.storage, self.daemon.blob_manager) self.daemon.conf.network_storage_limit = 32 await self.clear() await self.blob_clean() self.assertEqual(0, (await self.status())['disk_space']['total_used_mb']) await background_downloader.download_blobs(content1) await self.assertBlobs(content1) await self.blob_clean() self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(32, (await self.status())['disk_space']['seed_blobs_storage_used_mb']) self.daemon.conf.network_storage_limit = 48 await background_downloader.download_blobs(content2) await self.assertBlobs(content1, content2) await self.blob_clean() self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(48, (await self.status())['disk_space']['seed_blobs_storage_used_mb']) await self.clear() await background_downloader.download_blobs(content2) await self.assertBlobs(content2) await self.blob_clean() self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(16, (await self.status())['disk_space']['seed_blobs_storage_used_mb']) # tests that an attempt to download something that isn't a sd blob will download the single blob and stop blobs = await self.get_blobs_from_sd_blob(self.reflector.blob_manager.get_blob(content1)) await self.clear() await background_downloader.download_blobs(blobs[0].blob_hash) self.assertEqual({blobs[0].blob_hash}, self.daemon.blob_manager.completed_blob_hashes) # test that disk space manager doesn't delete orphan network blobs await background_downloader.download_blobs(content1) await self.daemon.storage.db.execute_fetchall("update blob set added_on=0") # so it is preferred for cleaning await self.daemon.jsonrpc_get("content2", save_file=False) while (await self.file_list())[0]['status'] != 'stopped': await asyncio.sleep(0.5) await self.assertBlobs(content1, no_files=False) self.daemon.conf.blob_storage_limit = 1 await self.blob_clean() await self.assertBlobs(content1, no_files=False) self.daemon.conf.network_storage_limit = 0 await self.blob_clean() self.assertEqual(0, (await self.status())['disk_space']['seed_blobs_storage_used_mb']) ================================================ FILE: tests/integration/datanetwork/test_streaming.py ================================================ import os import hashlib import aiohttp import aiohttp.web import asyncio import contextlib from lbry.file.source import ManagedDownloadSource from lbry.utils import aiohttp_request from lbry.blob.blob_file import MAX_BLOB_SIZE from lbry.testcase import CommandTestCase def get_random_bytes(n: int) -> bytes: result = b''.join(hashlib.sha256(os.urandom(4)).digest() for _ in range(n // 16)) if len(result) < n: result += os.urandom(n - len(result)) elif len(result) > n: result = result[:-(len(result) - n)] assert len(result) == n, (n, len(result)) return result class RangeRequests(CommandTestCase): async def _restart_stream_manager(self): await self.daemon.file_manager.stop() await self.daemon.file_manager.start() return async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0): self.daemon.conf.save_blobs = save_blobs self.daemon.conf.save_files = save_files self.data = data await self.stream_create('foo', '0.01', data=self.data, file_size=file_size) if save_blobs: self.assertGreater(len(os.listdir(self.daemon.blob_manager.blob_dir)), 1) await (await self.daemon.jsonrpc_file_list())['items'][0].fully_reflected.wait() await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo') self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir))) # await self._restart_stream_manager() await self.daemon.streaming_runner.setup() site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host, self.daemon.conf.streaming_port) await site.start() self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) async def _test_range_requests(self): name = 'foo' url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/{name}' async with aiohttp_request('get', url) as req: self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream') content_range = req.headers.get('Content-Range') content_length = int(req.headers.get('Content-Length')) streamed_bytes = await req.content.read() self.assertEqual(content_length, len(streamed_bytes)) return streamed_bytes, content_range, content_length async def test_range_requests_2_byte(self): self.data = b'hi' await self._setup_stream(self.data) streamed, content_range, content_length = await self._test_range_requests() self.assertEqual(15, content_length) self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed) self.assertEqual('bytes 0-14/15', content_range) async def test_range_requests_15_byte(self): self.data = b'123456789abcdef' await self._setup_stream(self.data) streamed, content_range, content_length = await self._test_range_requests() self.assertEqual(15, content_length) self.assertEqual(15, len(streamed)) self.assertEqual(self.data, streamed) self.assertEqual('bytes 0-14/15', content_range) async def test_range_requests_0_padded_bytes(self, size: int = (MAX_BLOB_SIZE - 1) * 4, expected_range: str = 'bytes 0-8388603/8388604', padding=b'', file_size=0): self.data = get_random_bytes(size) await self._setup_stream(self.data, file_size=file_size) streamed, content_range, content_length = await self._test_range_requests() self.assertEqual(len(self.data + padding), content_length) self.assertEqual(streamed, self.data + padding) self.assertEqual(expected_range, content_range) async def test_range_requests_1_padded_bytes(self): await self.test_range_requests_0_padded_bytes( ((MAX_BLOB_SIZE - 1) * 4) - 1, padding=b'\x00' ) async def test_range_requests_2_padded_bytes(self): await self.test_range_requests_0_padded_bytes( ((MAX_BLOB_SIZE - 1) * 4) - 2, padding=b'\x00' * 2 ) async def test_range_requests_14_padded_bytes(self): await self.test_range_requests_0_padded_bytes( ((MAX_BLOB_SIZE - 1) * 4) - 14, padding=b'\x00' * 14 ) async def test_range_requests_no_padding_size_from_claim(self): size = ((MAX_BLOB_SIZE - 1) * 4) - 14 await self.test_range_requests_0_padded_bytes(size, padding=b'', file_size=size, expected_range=f"bytes 0-{size-1}/{size}") async def test_range_requests_15_padded_bytes(self): await self.test_range_requests_0_padded_bytes( ((MAX_BLOB_SIZE - 1) * 4) - 15, padding=b'\x00' * 15 ) async def test_forbidden(self): self.data = get_random_bytes(1000) await self._setup_stream(self.data, file_size=1000) url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/foo' self.daemon.conf.streaming_get = False async with aiohttp_request('get', url) as req: self.assertEqual(403, req.status) async def test_range_requests_last_block_of_last_blob_padding(self): self.data = get_random_bytes(((MAX_BLOB_SIZE - 1) * 4) - 16) await self._setup_stream(self.data) streamed, content_range, content_length = await self._test_range_requests() self.assertEqual(len(self.data), content_length) self.assertEqual(streamed, self.data) self.assertEqual('bytes 0-8388587/8388588', content_range) async def test_streaming_only_with_blobs(self): self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4) await self._setup_stream(self.data) await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) # test that repeated range requests do not create duplicate files for _ in range(3): await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) # test that a range request after restart does not create a duplicate file await self._restart_stream_manager() current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) async def test_streaming_only_without_blobs(self): self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4) await self._setup_stream(self.data, save_blobs=False) await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) # test that repeated range requests do not create duplicate files for _ in range(3): await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) # test that a range request after restart does not create a duplicate file await self._restart_stream_manager() current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNone(stream.download_directory) self.assertIsNone(stream.full_path) current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir)) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) async def test_stream_and_save_file_with_blobs(self): self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4) await self._setup_stream(self.data, save_files=True) await self._test_range_requests() streams = (await self.daemon.jsonrpc_file_list())['items'] self.assertEqual(1, len(streams)) stream = streams[0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) full_path = stream.full_path files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) for _ in range(3): await self._test_range_requests() streams = (await self.daemon.jsonrpc_file_list())['items'] self.assertEqual(1, len(streams)) stream = streams[0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) await self._restart_stream_manager() current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) streams = (await self.daemon.jsonrpc_file_list())['items'] self.assertEqual(1, len(streams)) stream = streams[0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) await self._test_range_requests() streams = (await self.daemon.jsonrpc_file_list())['items'] self.assertEqual(1, len(streams)) stream = streams[0] self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path)) self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) with open(stream.full_path, 'rb') as f: self.assertEqual(self.data, f.read()) async def test_stream_and_save_file_without_blobs(self): self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4) await self._setup_stream(self.data, save_files=True) self.daemon.conf.save_blobs = False await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) full_path = stream.full_path files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) for _ in range(3): await self._test_range_requests() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) await self._restart_stream_manager() current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) streams = (await self.daemon.jsonrpc_file_list())['items'] self.assertEqual(1, len(streams)) stream = streams[0] self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) await self._test_range_requests() streams = (await self.daemon.jsonrpc_file_list())['items'] self.assertEqual(1, len(streams)) stream = streams[0] self.assertTrue(os.path.isdir(stream.download_directory)) self.assertTrue(os.path.isfile(stream.full_path)) current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path))) self.assertEqual( len(files_in_download_dir), len(current_files_in_download_dir) ) with open(stream.full_path, 'rb') as f: self.assertEqual(self.data, f.read()) async def test_switch_save_blobs_while_running(self): await self.test_streaming_only_without_blobs() self.daemon.conf.save_blobs = True blobs_in_stream = (await self.daemon.jsonrpc_file_list())['items'][0].blobs_in_stream sd_hash = (await self.daemon.jsonrpc_file_list())['items'][0].sd_hash start_file_count = len(os.listdir(self.daemon.blob_manager.blob_dir)) await self._test_range_requests() self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir))) self.assertEqual(0, (await self.daemon.jsonrpc_file_list())['items'][0].blobs_remaining) # switch back self.daemon.conf.save_blobs = False await self._test_range_requests() self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir))) self.assertEqual(0, (await self.daemon.jsonrpc_file_list())['items'][0].blobs_remaining) await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=sd_hash) self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir))) await self._test_range_requests() self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir))) self.assertEqual(blobs_in_stream, (await self.daemon.jsonrpc_file_list())['items'][0].blobs_remaining) async def test_file_save_streaming_only_save_blobs(self): await self.test_streaming_only_with_blobs() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNone(stream.full_path) self.server.stop_server() await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir) stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNotNone(stream.full_path) await stream.finished_writing.wait() with open(stream.full_path, 'rb') as f: self.assertEqual(self.data, f.read()) await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=stream.sd_hash) async def test_file_save_stop_before_finished_streaming_only(self, wait_for_start_writing=False): await self.test_streaming_only_with_blobs() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNone(stream.full_path) self.server.stop_server() await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir) stream = (await self.daemon.jsonrpc_file_list())['items'][0] path = stream.full_path self.assertIsNotNone(path) if wait_for_start_writing: with contextlib.suppress(asyncio.CancelledError): await stream.started_writing.wait() self.assertTrue(os.path.isfile(path)) await self.daemon.file_manager.stop() # while stopped, we get no response to query and no file is present self.assertEqual((await self.daemon.jsonrpc_file_list())['items'], []) self.assertEqual(os.path.isfile(path), stream.status == ManagedDownloadSource.STATUS_FINISHED) await self.daemon.file_manager.start() # after restart, we get a response to query and same file path stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNotNone(stream.full_path) self.assertEqual(stream.full_path, path) if wait_for_start_writing: with contextlib.suppress(asyncio.CancelledError): await stream.started_writing.wait() self.assertTrue(os.path.isfile(path)) async def test_file_save_stop_before_finished_streaming_only_wait_for_start(self): return await self.test_file_save_stop_before_finished_streaming_only(wait_for_start_writing=True) async def test_file_save_streaming_only_dont_save_blobs(self): await self.test_streaming_only_without_blobs() stream = (await self.daemon.jsonrpc_file_list())['items'][0] self.assertIsNone(stream.full_path) await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir) stream = (await self.daemon.jsonrpc_file_list())['items'][0] await stream.finished_writing.wait() with open(stream.full_path, 'rb') as f: self.assertEqual(self.data, f.read()) class RangeRequestsLRUCache(CommandTestCase): blob_lru_cache_size = 32 async def _request_stream(self): name = 'foo' url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/{name}' async with aiohttp_request('get', url) as req: self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream') content_range = req.headers.get('Content-Range') content_length = int(req.headers.get('Content-Length')) streamed_bytes = await req.content.read() self.assertEqual(content_length, len(streamed_bytes)) self.assertEqual(15, content_length) self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed_bytes) self.assertEqual('bytes 0-14/15', content_range) async def test_range_requests_with_blob_lru_cache(self): self.data = b'hi' self.daemon.conf.save_blobs = False self.daemon.conf.save_files = False await self.stream_create('foo', '0.01', data=self.data, file_size=0) await (await self.daemon.jsonrpc_file_list())['items'][0].fully_reflected.wait() await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo') self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir))) await self.daemon.streaming_runner.setup() site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host, self.daemon.conf.streaming_port) await site.start() self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) await self._request_stream() self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.server.stop_server() # running with cache size 0 gets through without errors without # this since the server doesn't stop immediately await asyncio.sleep(1) await self._request_stream() ================================================ FILE: tests/integration/other/__init__.py ================================================ ================================================ FILE: tests/integration/other/test_chris45.py ================================================ from lbry.testcase import CommandTestCase class EpicAdventuresOfChris45(CommandTestCase): async def test_no_this_is_not_a_test_its_an_adventure(self): # Chris45 is an avid user of LBRY and this is his story. It's fact and fiction # and everything in between; it's also the setting of some record setting # integration tests. # Chris45 starts everyday by checking his balance. result = await self.daemon.jsonrpc_account_balance() self.assertEqual(result['available'], '10.0') # "10 LBC, yippy! I can do a lot with that.", he thinks to himself, # enthusiastically. But he is hungry so he goes into the kitchen # to make himself a spamdwich. # While making the spamdwich he wonders... has anyone on LBRY # registered the @spam channel yet? "I should do that!" he # exclaims and goes back to his computer to do just that! tx = await self.channel_create('@spam', '1.0') channel_id = self.get_claim_id(tx) # Do we have it locally? channels = await self.out(self.daemon.jsonrpc_channel_list()) self.assertItemCount(channels, 1) self.assertEqual(channels['items'][0]['name'], '@spam') # As the new channel claim travels through the intertubes and makes its # way into the mempool and then a block and then into the claimtrie, # Chris doesn't sit idly by: he checks his balance! result = await self.daemon.jsonrpc_account_balance() self.assertEqual(result['available'], '8.989893') # He waits for 6 more blocks (confirmations) to make sure the balance has been settled. await self.generate(6) result = await self.daemon.jsonrpc_account_balance(confirmations=6) self.assertEqual(result['available'], '8.989893') # And is the channel resolvable and empty? response = await self.resolve('lbry://@spam') self.assertEqual(response['value_type'], 'channel') # "What goes well with spam?" ponders Chris... # "A hovercraft with eels!" he exclaims. # "That's what goes great with spam!" he further confirms. # And so, many hours later, Chris is finished writing his epic story # about eels driving a hovercraft across the wetlands while eating spam # and decides it's time to publish it to the @spam channel. tx = await self.stream_create( 'hovercraft', '1.0', data=b'[insert long story about eels driving hovercraft]', channel_id=channel_id ) claim_id = self.get_claim_id(tx) # He quickly checks the unconfirmed balance to make sure everything looks # correct. result = await self.daemon.jsonrpc_account_balance() self.assertEqual(result['available'], '7.969786') # Also checks that his new story can be found on the blockchain before # giving the link to all his friends. response = await self.resolve('lbry://@spam/hovercraft') self.assertEqual(response['value_type'], 'stream') # He goes to tell everyone about it and in the meantime 5 blocks are confirmed. await self.generate(5) # When he comes back he verifies the confirmed balance. result = await self.daemon.jsonrpc_account_balance() self.assertEqual(result['available'], '7.969786') # As people start reading his story they discover some typos and notify # Chris who explains in despair "Oh! Noooooos!" but then remembers # "No big deal! I can update my claim." And so he updates his claim. await self.stream_update(claim_id, data=b'[typo fixing sounds being made]') # After some soul searching Chris decides that his story needs more # heart and a better ending. He takes down the story and begins the rewrite. abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=True)) self.assertEqual(abandon['inputs'][0]['claim_id'], claim_id) await self.confirm_tx(abandon['txid']) # And now checks that the claim doesn't resolve anymore. self.assertEqual( {'error': { 'name': 'NOT_FOUND', 'text': 'Could not find claim at "lbry://@spam/hovercraft".' }}, await self.resolve('lbry://@spam/hovercraft') ) # After abandoning he just waits for his LBCs to be returned to his account await self.generate(5) result = await self.daemon.jsonrpc_account_balance() self.assertEqual(result['available'], '8.9693455') # Amidst all this Chris receives a call from his friend Ramsey # who says that it is of utmost urgency that Chris transfer him # 1 LBC to which Chris readily obliges ramsey_account_id = (await self.out(self.daemon.jsonrpc_account_create("Ramsey")))['id'] ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id) result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address, blocking=True)) self.assertIn("txid", result) await self.confirm_tx(result['txid']) # Chris then eagerly waits for 6 confirmations to check his balance and then calls Ramsey to verify whether # he received it or not await self.generate(5) result = await self.daemon.jsonrpc_account_balance() # Chris' balance was correct self.assertEqual(result['available'], '7.9692215') # Ramsey too assured him that he had received the 1 LBC and thanks him result = await self.daemon.jsonrpc_account_balance(ramsey_account_id) self.assertEqual(result['available'], '1.0') # After Chris is done with all the "helping other people" stuff he decides that it's time to # write a new story and publish it to lbry. All he needed was a fresh start and he came up with: tx = await self.stream_create( 'fresh-start', '1.0', data=b'Amazingly Original First Line', channel_id=channel_id ) claim_id2 = self.get_claim_id(tx) await self.generate(5) # He gives the link of his story to all his friends and hopes that this is the much needed break for him uri = 'lbry://@spam/fresh-start' # And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit # Now to keep this claim winning on the lbry blockchain he immediately supports the claim tx = await self.out(self.daemon.jsonrpc_support_create( claim_id2, '0.2', account_id=ramsey_account_id, blocking=True )) await self.confirm_tx(tx['txid']) # And check if his support showed up resolve_result = await self.resolve(uri) # It obviously did! Because, blockchain baby \O/ self.assertEqual(resolve_result['amount'], '1.0') self.assertEqual(resolve_result['meta']['effective_amount'], '1.2') await self.generate(5) # Now he also wanted to support the original creator of the Award Winning Novel # So he quickly decides to send a tip to him tx = await self.out( self.daemon.jsonrpc_support_create(claim_id2, '0.3', tip=True, account_id=ramsey_account_id, blocking=True) ) await self.confirm_tx(tx['txid']) # And again checks if it went to the just right place resolve_result = await self.resolve(uri) # Which it obviously did. Because....????? self.assertEqual(resolve_result['meta']['effective_amount'], '1.5') await self.generate(5) # Seeing the ravishing success of his novel Chris adds support to his claim too tx = await self.out(self.daemon.jsonrpc_support_create(claim_id2, '0.4', blocking=True)) await self.confirm_tx(tx['txid']) # And check if his support showed up resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri)) # It did! self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.9') await self.generate(5) # Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that # the instruments, the theatre, the ads, everything, EXCEPT lyrics!! He panicked.. But then he remembered # something, so he un-panicked. He quickly calls up his best bud Chris and requests him to write hit lyrics for # his song, seeing as his novel had smashed all the records, he was the perfect candidate! # ....... # Chris agrees.. 17 hours 43 minutes and 14 seconds later, he makes his publish tx = await self.stream_create( 'hit-song', '1.0', data=b'The Whale and The Bookmark', channel_id=channel_id ) await self.generate(5) # He sends the link to Ramsey, all happy and proud uri = 'lbry://@spam/hit-song' # But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris... # Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=True)) self.assertTrue(abandon['inputs'][0]['txid'], tx['txid']) await self.confirm_tx(abandon['txid']) # He them checks that the claim doesn't resolve anymore. self.assertEqual( {'error': { 'name': 'NOT_FOUND', 'text': f'Could not find claim at "{uri}".' }}, await self.resolve(uri) ) # He closes and opens the wallet server databases to see how horribly they break db = self.conductor.spv_node.server.db db.close() db.open_db() await db.initialize_caches() # They didn't! (error would be AssertionError: 276 vs 266 (264 counts) on startup) ================================================ FILE: tests/integration/other/test_cli.py ================================================ import contextlib import os import tempfile from io import StringIO from lbry.testcase import AsyncioTestCase from lbry.conf import Config from lbry.extras import cli from lbry.extras.daemon.components import ( DATABASE_COMPONENT, DISK_SPACE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT, LIBTORRENT_COMPONENT, BACKGROUND_DOWNLOADER_COMPONENT, TRACKER_ANNOUNCER_COMPONENT ) from lbry.extras.daemon.daemon import Daemon class CLIIntegrationTest(AsyncioTestCase): async def asyncSetUp(self): conf = Config() conf.data_dir = '/tmp' conf.share_usage_data = False conf.api = 'localhost:5299' conf.components_to_skip = ( DATABASE_COMPONENT, DISK_SPACE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT, LIBTORRENT_COMPONENT, BACKGROUND_DOWNLOADER_COMPONENT, TRACKER_ANNOUNCER_COMPONENT ) Daemon.component_attributes = {} self.daemon = Daemon(conf) await self.daemon.start() self.addCleanup(self.daemon.stop) def test_cli_status_command_with_auth(self): actual_output = StringIO() with contextlib.redirect_stdout(actual_output): cli.main(["--api", "localhost:5299", "status"]) actual_output = actual_output.getvalue() self.assertIn("is_running", actual_output) def test_when_download_dir_non_writable_on_start_then_daemon_dies_with_helpful_msg(self): with tempfile.TemporaryDirectory() as download_dir: os.chmod(download_dir, mode=0o555) # makes download dir non-writable, readable and executable with self.assertRaisesRegex(PermissionError, f"The following directory is not writable: {download_dir}"): cli.main(["start", "--download-dir", download_dir]) ================================================ FILE: tests/integration/other/test_exchange_rate_manager.py ================================================ import asyncio from decimal import Decimal from lbry.testcase import AsyncioTestCase from lbry.extras.daemon.exchange_rate_manager import ( ExchangeRate, ExchangeRateManager, FEEDS, MarketFeed ) class TestExchangeRateManager(AsyncioTestCase): # async def test_exchange_rate_manager(self): # manager = ExchangeRateManager(FEEDS) # manager.start() # self.addCleanup(manager.stop) # for feed in manager.market_feeds: # self.assertFalse(feed.is_online) # self.assertIsNone(feed.rate) # await manager.wait() # failures = set() # for feed in manager.market_feeds: # if feed.is_online: # self.assertIsInstance(feed.rate, ExchangeRate) # else: # failures.add(feed.name) # self.assertFalse(feed.has_rate) # self.assertLessEqual(len(failures), 1, f"feed failures: {failures}. Please check exchange rate feeds!") # lbc = manager.convert_currency('USD', 'LBC', Decimal('1.0')) # self.assertGreaterEqual(lbc, 2.0) # self.assertLessEqual(lbc, 120.0) # lbc = manager.convert_currency('BTC', 'LBC', Decimal('0.01')) # self.assertGreaterEqual(lbc, 1_000) # self.assertLessEqual(lbc, 30_000) async def test_it_handles_feed_being_offline(self): class FakeFeed(MarketFeed): name = "fake" url = "http://impossi.bru" manager = ExchangeRateManager((FakeFeed,)) manager.start() self.addCleanup(manager.stop) for feed in manager.market_feeds: self.assertFalse(feed.is_online) self.assertIsNone(feed.rate) await asyncio.wait_for(manager.wait(), 2) for feed in manager.market_feeds: self.assertFalse(feed.is_online) self.assertFalse(feed.has_rate) ================================================ FILE: tests/integration/other/test_other_commands.py ================================================ from lbry.testcase import CommandTestCase class AddressManagement(CommandTestCase): async def test_address_list(self): addresses = await self.out(self.daemon.jsonrpc_address_list()) self.assertItemCount(addresses, 27) single = await self.out(self.daemon.jsonrpc_address_list(addresses['items'][11]['address'])) self.assertItemCount(single, 1) self.assertEqual(single['items'][0], addresses['items'][11]) class SettingsManagement(CommandTestCase): async def test_settings(self): self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('localhost', 50002)) setting = self.daemon.jsonrpc_settings_set('lbryum_servers', ['server:50001']) self.assertEqual(setting['lbryum_servers'][0], ('server', 50001)) self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('server', 50001)) setting = self.daemon.jsonrpc_settings_clear('lbryum_servers') self.assertEqual(setting['lbryum_servers'][0], ('spv11.lbry.com', 50001)) self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv11.lbry.com', 50001)) # test_privacy_settings (merged for reducing test time, unmerge when its fast) # tests that changing share_usage_data propagates to the relevant properties self.assertFalse(self.daemon.jsonrpc_settings_get()['share_usage_data']) self.daemon.jsonrpc_settings_set('share_usage_data', True) self.assertTrue(self.daemon.jsonrpc_settings_get()['share_usage_data']) self.assertTrue(self.daemon.analytics_manager.enabled) self.daemon.jsonrpc_settings_set('share_usage_data', False) class TroubleshootingCommands(CommandTestCase): async def test_tracemalloc_commands(self): self.addCleanup(self.daemon.jsonrpc_tracemalloc_disable) self.assertFalse(self.daemon.jsonrpc_tracemalloc_disable()) self.assertTrue(self.daemon.jsonrpc_tracemalloc_enable()) class WeirdObject(): pass hold_em = [WeirdObject() for _ in range(500)] top = self.daemon.jsonrpc_tracemalloc_top(1) self.assertEqual(1, len(top)) self.assertEqual('hold_em = [WeirdObject() for _ in range(500)]', top[0]['code']) self.assertTrue(top[0]['line'].startswith('other/test_other_commands.py:')) self.assertGreaterEqual(top[0]['count'], 500) self.assertGreater(top[0]['size'], 0) # just matters that its a positive integer ================================================ FILE: tests/integration/other/test_transcoding.py ================================================ import logging import pathlib import time from ..claims.test_claim_commands import ClaimTestCase from lbry.conf import TranscodeConfig from lbry.file_analysis import VideoFileAnalyzer log = logging.getLogger(__name__) class MeasureTime: def __init__(self, text): print(text, end="...", flush=True) def __enter__(self): self.start = time.perf_counter() def __exit__(self, exc_type, exc_val, exc_tb): end = time.perf_counter() print(f" done in {end - self.start:.6f}s", flush=True) class TranscodeValidation(ClaimTestCase): def make_name(self, name, extension=""): path = pathlib.Path(self.video_file_name) return path.parent / f"{path.stem}_{name}{extension or path.suffix}" async def asyncSetUp(self): await super().asyncSetUp() self.conf = TranscodeConfig() self.conf.volume_analysis_time = 0 # disable it as the test file isn't very good here self.analyzer = VideoFileAnalyzer(self.conf) self.assertTrue((await self.analyzer.status())["available"]) # ensure ffmpeg path detected file_ogg = self.make_name("ogg", ".ogg") self.video_file_ogg = str(file_ogg) if not file_ogg.exists(): command = f'-i "{self.video_file_name}" -c:v libtheora -q:v 4 -c:a libvorbis -q:a 4 ' \ f'-c:s copy -c:d copy "{file_ogg}"' with MeasureTime(f"Creating {file_ogg.name}"): output, code = await self.analyzer._execute_ffmpeg(command) self.assertEqual(code, 0, output) file_webm = self.make_name("webm", ".webm") self.video_file_webm = str(file_webm) if not file_webm.exists(): command = f'-i "{self.video_file_name}" -c:v libvpx-vp9 -crf 36 -b:v 0 -cpu-used 2 ' \ f'-c:a libopus -b:a 128k -c:s copy -c:d copy "{file_webm}"' with MeasureTime(f"Creating {file_webm.name}"): output, code = await self.analyzer._execute_ffmpeg(command) self.assertEqual(code, 0, output) async def test_should_work(self): new_file_name, _ = await self.analyzer.verify_or_repair(True, False, self.video_file_name) self.assertEqual(self.video_file_name, new_file_name) new_file_name, _ = await self.analyzer.verify_or_repair(True, False, self.video_file_ogg) self.assertEqual(self.video_file_ogg, new_file_name) new_file_name, spec = await self.analyzer.verify_or_repair(True, False, self.video_file_webm) self.assertEqual(self.video_file_webm, new_file_name) self.assertEqual(spec["width"], 1280) self.assertEqual(spec["height"], 720) self.assertEqual(spec["duration"], 16) async def test_volume(self): self.conf.volume_analysis_time = 200 with self.assertRaisesRegex(Exception, "lower than prime"): await self.analyzer.verify_or_repair(True, False, self.video_file_name) async def test_container(self): file_name = self.make_name("bad_container", ".avi") if not file_name.exists(): command = f'-i "{self.video_file_name}" -c copy -map 0 "{file_name}"' with MeasureTime(f"Creating {file_name.name}"): output, code = await self.analyzer._execute_ffmpeg(command) self.assertEqual(code, 0, output) with self.assertRaisesRegex(Exception, "Container format is not in the approved list"): await self.analyzer.verify_or_repair(True, False, file_name) fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name) pathlib.Path(fixed_file).unlink() async def test_video_codec(self): file_name = self.make_name("bad_video_codec_1") if not file_name.exists(): command = f'-i "{self.video_file_name}" -c copy -map 0 -c:v libx265 -preset superfast "{file_name}"' with MeasureTime(f"Creating {file_name.name}"): output, code = await self.analyzer._execute_ffmpeg(command) self.assertEqual(code, 0, output) with self.assertRaisesRegex(Exception, "Video codec is not in the approved list"): await self.analyzer.verify_or_repair(True, False, file_name) with self.assertRaisesRegex(Exception, "faststart flag was not used"): await self.analyzer.verify_or_repair(True, False, file_name) fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name) pathlib.Path(fixed_file).unlink() async def test_max_bit_rate(self): self.conf.video_bitrate_maximum = 100 with self.assertRaisesRegex(Exception, "The bit rate is above the configured maximum"): await self.analyzer.verify_or_repair(True, False, self.video_file_name) async def test_video_format(self): file_name = self.make_name("bad_video_format_1") if not file_name.exists(): command = f'-i "{self.video_file_name}" -c copy -map 0 -c:v libx264 ' \ f'-vf format=yuv444p "{file_name}"' with MeasureTime(f"Creating {file_name.name}"): output, code = await self.analyzer._execute_ffmpeg(command) self.assertEqual(code, 0, output) with self.assertRaisesRegex(Exception, "pixel format does not match the approved"): await self.analyzer.verify_or_repair(True, False, file_name) fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name) pathlib.Path(fixed_file).unlink() async def test_audio_codec(self): file_name = self.make_name("bad_audio_codec_1", ".mkv") if not file_name.exists(): command = f'-i "{self.video_file_name}" -c copy -map 0 -c:a pcm_s16le "{file_name}"' with MeasureTime(f"Creating {file_name.name}"): output, code = await self.analyzer._execute_ffmpeg(command) self.assertEqual(code, 0, output) with self.assertRaisesRegex(Exception, "Audio codec is not in the approved list"): await self.analyzer.verify_or_repair(True, False, file_name) fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name) pathlib.Path(fixed_file).unlink() async def test_extension_choice(self): scan_data = await self.analyzer._get_scan_data(True, self.video_file_name) extension = self.analyzer._get_best_container_extension(scan_data, "") self.assertEqual(extension, pathlib.Path(self.video_file_name).suffix[1:]) scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg) extension = self.analyzer._get_best_container_extension(scan_data, "") self.assertEqual(extension, "ogv") scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm) extension = self.analyzer._get_best_container_extension(scan_data, "") self.assertEqual(extension, "webm") extension = self.analyzer._get_best_container_extension("", "libx264 -crf 23") self.assertEqual("mp4", extension) extension = self.analyzer._get_best_container_extension("", "libvpx-vp9 -crf 23") self.assertEqual("webm", extension) extension = self.analyzer._get_best_container_extension("", "libtheora") self.assertEqual("ogv", extension) async def test_no_ffmpeg(self): self.conf.ffmpeg_path = "I don't really exist/" self.analyzer._env_copy.pop("PATH", None) await self.analyzer.status(reset=True) with self.assertRaisesRegex(Exception, "Unable to locate"): await self.analyzer.verify_or_repair(True, False, self.video_file_name) async def test_dont_recheck_ffmpeg_installation(self): call_count = 0 original = self.daemon._video_file_analyzer._verify_ffmpeg_installed def _verify_ffmpeg_installed(): nonlocal call_count call_count += 1 return original() self.daemon._video_file_analyzer._verify_ffmpeg_installed = _verify_ffmpeg_installed self.assertEqual(0, call_count) await self.daemon.jsonrpc_status() self.assertEqual(1, call_count) # counter should not go up again await self.daemon.jsonrpc_status() self.assertEqual(1, call_count) # this should force rechecking the installation await self.daemon.jsonrpc_ffmpeg_find() self.assertEqual(2, call_count) ================================================ FILE: tests/integration/takeovers/__init__.py ================================================ ================================================ FILE: tests/integration/takeovers/test_resolve_command.py ================================================ import asyncio import json import hashlib import sys from bisect import bisect_right from binascii import hexlify, unhexlify from collections import defaultdict from typing import NamedTuple, List from lbry.testcase import CommandTestCase from lbry.wallet.transaction import Transaction, Output from lbry.schema.compat import OldClaimMessage from lbry.crypto.hash import sha256 from lbry.crypto.base58 import Base58 class ClaimStateValue(NamedTuple): claim_id: str activation_height: int active_in_lbrycrd: bool class BaseResolveTestCase(CommandTestCase): def assertMatchESClaim(self, claim_from_es, claim_from_db): self.assertEqual(claim_from_es['claim_hash'][::-1].hex(), claim_from_db.claim_hash.hex()) self.assertEqual(claim_from_es['claim_id'], claim_from_db.claim_hash.hex()) self.assertEqual(claim_from_es['activation_height'], claim_from_db.activation_height, f"es height: {claim_from_es['activation_height']}, rocksdb height: {claim_from_db.activation_height}") self.assertEqual(claim_from_es['last_take_over_height'], claim_from_db.last_takeover_height) self.assertEqual(claim_from_es['tx_id'], claim_from_db.tx_hash[::-1].hex()) self.assertEqual(claim_from_es['tx_nout'], claim_from_db.position) self.assertEqual(claim_from_es['amount'], claim_from_db.amount) self.assertEqual(claim_from_es['effective_amount'], claim_from_db.effective_amount) def assertMatchDBClaim(self, expected, claim): self.assertEqual(expected['claimid'], claim.claim_hash.hex()) self.assertEqual(expected['validatheight'], claim.activation_height) self.assertEqual(expected['lasttakeoverheight'], claim.last_takeover_height) self.assertEqual(expected['txid'], claim.tx_hash[::-1].hex()) self.assertEqual(expected['n'], claim.position) self.assertEqual(expected['amount'], claim.amount) self.assertEqual(expected['effectiveamount'], claim.effective_amount) async def assertResolvesToClaimId(self, name, claim_id): other = await self.resolve(name) if claim_id is None: self.assertIn('error', other) self.assertEqual(other['error']['name'], 'NOT_FOUND') claims_from_es = (await self.conductor.spv_node.server.session_manager.search_index.search(name=name))[0] claims_from_es = [c['claim_hash'][::-1].hex() for c in claims_from_es] self.assertNotIn(claim_id, claims_from_es) else: claim_from_es = await self.conductor.spv_node.server.session_manager.search_index.search(claim_id=claim_id) self.assertEqual(claim_id, other['claim_id']) self.assertEqual(claim_id, claim_from_es[0][0]['claim_hash'][::-1].hex()) async def assertNoClaimForName(self, name: str): lbrycrd_winning = json.loads(await self.blockchain._cli_cmnd('getclaimsforname', name)) stream, channel, _, _ = await self.conductor.spv_node.server.db.resolve(name) if 'claims' in lbrycrd_winning and lbrycrd_winning['claims'] is not None: self.assertEqual(len(lbrycrd_winning['claims']), 0) if stream is not None: self.assertIsInstance(stream, LookupError) else: self.assertIsInstance(channel, LookupError) claim_from_es = await self.conductor.spv_node.server.session_manager.search_index.search(name=name) self.assertListEqual([], claim_from_es[0]) async def assertNoClaim(self, name: str, claim_id: str): expected = json.loads(await self.blockchain._cli_cmnd('getclaimsfornamebyid', name, '["' + claim_id + '"]')) if 'claims' in expected and expected['claims'] is not None: # ensure that if we do have the matching claim that it is not active self.assertEqual(expected['claims'][0]['effectiveamount'], 0) claim_from_es = await self.conductor.spv_node.server.session_manager.search_index.search(claim_id=claim_id) self.assertListEqual([], claim_from_es[0]) claim = await self.conductor.spv_node.server.db.fs_getclaimbyid(claim_id) self.assertIsNone(claim) async def assertMatchWinningClaim(self, name): expected = json.loads(await self.blockchain._cli_cmnd('getclaimsfornamebybid', name, "[0]")) stream, channel, _, _ = await self.conductor.spv_node.server.db.resolve(name) claim = stream if stream else channel expected['claims'][0]['lasttakeoverheight'] = expected['lasttakeoverheight'] await self._assertMatchClaim(expected['claims'][0], claim) return claim async def _assertMatchClaim(self, expected, claim): self.assertMatchDBClaim(expected, claim) claim_from_es = await self.conductor.spv_node.server.session_manager.search_index.search( claim_id=claim.claim_hash.hex() ) self.assertEqual(len(claim_from_es[0]), 1) self.assertMatchESClaim(claim_from_es[0][0], claim) self._check_supports(claim.claim_hash.hex(), expected.get('supports', []), claim_from_es[0][0]['support_amount']) async def assertMatchClaim(self, name, claim_id, is_active_in_lbrycrd=True): claim = await self.conductor.spv_node.server.db.fs_getclaimbyid(claim_id) claim_from_es = await self.conductor.spv_node.server.session_manager.search_index.search( claim_id=claim.claim_hash.hex() ) self.assertEqual(len(claim_from_es[0]), 1) self.assertEqual(claim_from_es[0][0]['claim_hash'][::-1].hex(), claim.claim_hash.hex()) self.assertMatchESClaim(claim_from_es[0][0], claim) expected = json.loads(await self.blockchain._cli_cmnd('getclaimsfornamebyid', name, '["' + claim_id + '"]')) if is_active_in_lbrycrd: if not expected: self.assertIsNone(claim) return expected['claims'][0]['lasttakeoverheight'] = expected['lasttakeoverheight'] self.assertMatchDBClaim(expected['claims'][0], claim) self._check_supports(claim.claim_hash.hex(), expected['claims'][0].get('supports', []), claim_from_es[0][0]['support_amount']) else: if 'claims' in expected and expected['claims'] is not None: # ensure that if we do have the matching claim that it is not active self.assertEqual(expected['claims'][0]['effectiveamount'], 0) return claim async def assertMatchClaimIsWinning(self, name, claim_id): self.assertEqual(claim_id, (await self.assertMatchWinningClaim(name)).claim_hash.hex()) await self.assertMatchClaimsForName(name) def _check_supports(self, claim_id, lbrycrd_supports, es_support_amount): total_lbrycrd_amount = 0.0 total_es_amount = 0.0 active_es_amount = 0.0 db = self.conductor.spv_node.server.db es_supports = db.get_supports(bytes.fromhex(claim_id)) # we're only concerned about active supports here, and they should match self.assertTrue(len(es_supports) >= len(lbrycrd_supports)) for i, (tx_num, position, amount) in enumerate(es_supports): total_es_amount += amount valid_height = db.get_activation(tx_num, position, is_support=True) if valid_height > db.db_height: continue active_es_amount += amount txid = db.prefix_db.tx_hash.get(tx_num, deserialize_value=False)[::-1].hex() support = next(filter(lambda s: s['txid'] == txid and s['n'] == position, lbrycrd_supports)) total_lbrycrd_amount += support['amount'] self.assertEqual(support['height'], bisect_right(db.tx_counts, tx_num)) self.assertEqual(support['validatheight'], valid_height) self.assertEqual(total_es_amount, es_support_amount) self.assertEqual(active_es_amount, total_lbrycrd_amount) async def assertMatchClaimsForName(self, name): expected = json.loads(await self.blockchain._cli_cmnd('getclaimsforname', name, "", "true")) db = self.conductor.spv_node.server.db for c in expected['claims']: c['lasttakeoverheight'] = expected['lasttakeoverheight'] claim_id = c['claimid'] claim_hash = bytes.fromhex(claim_id) claim = db._fs_get_claim_by_hash(claim_hash) self.assertMatchDBClaim(c, claim) claim_from_es = await self.conductor.spv_node.server.session_manager.search_index.search( claim_id=claim_id ) self.assertEqual(len(claim_from_es[0]), 1) self.assertEqual(claim_from_es[0][0]['claim_hash'][::-1].hex(), claim_id) self.assertMatchESClaim(claim_from_es[0][0], claim) self._check_supports(claim_id, c.get('supports', []), claim_from_es[0][0]['support_amount']) async def assertNameState(self, height: int, name: str, winning_claim_id: str, last_takeover_height: int, non_winning_claims: List[ClaimStateValue]): self.assertEqual(height, self.conductor.spv_node.server.db.db_height) await self.assertMatchClaimIsWinning(name, winning_claim_id) for non_winning in non_winning_claims: claim = await self.assertMatchClaim( name, non_winning.claim_id, is_active_in_lbrycrd=non_winning.active_in_lbrycrd ) self.assertEqual(non_winning.activation_height, claim.activation_height) self.assertEqual(last_takeover_height, claim.last_takeover_height) class ResolveCommand(BaseResolveTestCase): async def test_colliding_short_id(self): prefixes = defaultdict(list) colliding_claim_ids = [] first_claims_one_char_shortid = {} while True: chan = self.get_claim_id( await self.channel_create('@abc', '0.01', allow_duplicate_name=True) ) if chan[:1] not in first_claims_one_char_shortid: first_claims_one_char_shortid[chan[:1]] = chan prefixes[chan[:2]].append(chan) if len(prefixes[chan[:2]]) > 1: colliding_claim_ids.extend(prefixes[chan[:2]]) break first_claim = first_claims_one_char_shortid[colliding_claim_ids[0][:1]] await self.assertResolvesToClaimId( f'@abc#{colliding_claim_ids[0][:1]}', first_claim ) collision_depth = 0 for c1, c2 in zip(colliding_claim_ids[0], colliding_claim_ids[1]): if c1 == c2: collision_depth += 1 else: break await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[0][:2]}', colliding_claim_ids[0]) await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[0][:7]}', colliding_claim_ids[0]) await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[0][:17]}', colliding_claim_ids[0]) await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[0]}', colliding_claim_ids[0]) await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[1][:collision_depth + 1]}', colliding_claim_ids[1]) await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[1][:7]}', colliding_claim_ids[1]) await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[1][:17]}', colliding_claim_ids[1]) await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[1]}', colliding_claim_ids[1]) # test resolving different streams for a channel using short urls self.get_claim_id( await self.stream_create('foo1', '0.01', channel_id=colliding_claim_ids[0]) ) self.get_claim_id( await self.stream_create('foo2', '0.01', channel_id=colliding_claim_ids[0]) ) duplicated_resolved = list(( await self.ledger.resolve([], [ f'@abc#{colliding_claim_ids[0][:2]}/foo1', f'@abc#{colliding_claim_ids[0][:2]}/foo2' ]) ).values()) self.assertEqual('foo1', duplicated_resolved[0].normalized_name) self.assertEqual('foo2', duplicated_resolved[1].normalized_name) async def test_abandon_channel_and_claims_in_same_tx(self): channel_id = self.get_claim_id( await self.channel_create('@abc', '0.01') ) await self.stream_create('foo', '0.01', channel_id=channel_id) await self.channel_update(channel_id, bid='0.001') foo2_id = self.get_claim_id(await self.stream_create('foo2', '0.01', channel_id=channel_id)) await self.stream_update(foo2_id, bid='0.0001', channel_id=channel_id, confirm=False) tx = await self.stream_create('foo3', '0.01', channel_id=channel_id, confirm=False, return_tx=True) await self.ledger.wait(tx) # db = self.conductor.spv_node.server.bp.db # claims = list(db.all_claims_producer()) # print("claims", claims) await self.daemon.jsonrpc_txo_spend(blocking=True) await self.generate(1) await self.assertNoClaimForName('@abc') await self.assertNoClaimForName('foo') await self.assertNoClaimForName('foo2') await self.assertNoClaimForName('foo3') async def test_resolve_response(self): channel_id = self.get_claim_id( await self.channel_create('@abc', '0.01') ) # resolving a channel @abc response = await self.resolve('lbry://@abc') self.assertEqual(response['name'], '@abc') self.assertEqual(response['value_type'], 'channel') self.assertEqual(response['meta']['claims_in_channel'], 0) await self.stream_create('foo', '0.01', channel_id=channel_id) await self.stream_create('foo2', '0.01', channel_id=channel_id) # resolving a channel @abc with some claims in it response['confirmations'] += 2 response['meta']['claims_in_channel'] = 2 self.assertEqual(response, await self.resolve('lbry://@abc')) # resolving claim foo within channel @abc claim = await self.resolve('lbry://@abc/foo') self.assertEqual(claim['name'], 'foo') self.assertEqual(claim['value_type'], 'stream') self.assertEqual(claim['signing_channel']['name'], '@abc') self.assertTrue(claim['is_channel_signature_valid']) self.assertEqual( claim['timestamp'], self.ledger.headers.estimated_timestamp(claim['height']) ) self.assertEqual( claim['signing_channel']['timestamp'], self.ledger.headers.estimated_timestamp(claim['signing_channel']['height']) ) # resolving claim foo by itself self.assertEqual(claim, await self.resolve('lbry://foo')) # resolving from the given permanent url self.assertEqual(claim, await self.resolve(claim['permanent_url'])) # resolving multiple at once response = await self.out(self.daemon.jsonrpc_resolve(['lbry://foo', 'lbry://foo2'])) self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response)) claim = response['lbry://foo2'] self.assertEqual(claim['name'], 'foo2') self.assertEqual(claim['value_type'], 'stream') self.assertEqual(claim['signing_channel']['name'], '@abc') self.assertTrue(claim['is_channel_signature_valid']) # resolve has correct confirmations tx_details = await self.blockchain.get_raw_transaction(claim['txid']) self.assertEqual(claim['confirmations'], json.loads(tx_details)['confirmations']) # FIXME : claimname/updateclaim is gone. #3480 wip, unblock #3479" # resolve handles invalid data # await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1") # await self.generate(1) # response = await self.out(self.daemon.jsonrpc_resolve("lbry://gibberish")) # self.assertSetEqual({'lbry://gibberish'}, set(response)) # claim = response['lbry://gibberish'] # self.assertEqual(claim['name'], 'gibberish') # self.assertNotIn('value', claim) # resolve retries await self.conductor.spv_node.stop() resolve_task = asyncio.create_task(self.resolve('foo')) await self.conductor.spv_node.start(self.conductor.lbcwallet_node) self.assertIsNotNone((await resolve_task)['claim_id']) async def test_winning_by_effective_amount(self): # first one remains winner unless something else changes claim_id1 = self.get_claim_id( await self.channel_create('@foo', allow_duplicate_name=True)) await self.assertResolvesToClaimId('@foo', claim_id1) claim_id2 = self.get_claim_id( await self.channel_create('@foo', allow_duplicate_name=True)) await self.assertResolvesToClaimId('@foo', claim_id1) claim_id3 = self.get_claim_id( await self.channel_create('@foo', allow_duplicate_name=True)) await self.assertResolvesToClaimId('@foo', claim_id1) # supports change the winner await self.support_create(claim_id3, '0.09') await self.assertResolvesToClaimId('@foo', claim_id3) await self.support_create(claim_id2, '0.19') await self.assertResolvesToClaimId('@foo', claim_id2) await self.support_create(claim_id1, '0.29') await self.assertResolvesToClaimId('@foo', claim_id1) await self.support_abandon(claim_id1) await self.assertResolvesToClaimId('@foo', claim_id2) async def test_resolve_duplicate_name_in_channel(self): db_resolve = self.conductor.spv_node.server.db.resolve # first one remains winner unless something else changes channel_id = self.get_claim_id(await self.channel_create('@foo')) file_path = self.create_upload_file(data=b'hi!') tx = await self.daemon.jsonrpc_stream_create('duplicate', '0.1', file_path=file_path, allow_duplicate_name=True, channel_id=channel_id) await self.ledger.wait(tx) first_claim = tx.outputs[0].claim_id file_path = self.create_upload_file(data=b'hi!') tx = await self.daemon.jsonrpc_stream_create('duplicate', '0.1', file_path=file_path, allow_duplicate_name=True, channel_id=channel_id) await self.ledger.wait(tx) duplicate_claim = tx.outputs[0].claim_id await self.generate(1) stream, channel, _, _ = await db_resolve(f"@foo:{channel_id}/duplicate:{first_claim}") self.assertEqual(stream.claim_hash.hex(), first_claim) self.assertEqual(channel.claim_hash.hex(), channel_id) stream, channel, _, _ = await db_resolve(f"@foo:{channel_id}/duplicate:{duplicate_claim}") self.assertEqual(stream.claim_hash.hex(), duplicate_claim) self.assertEqual(channel.claim_hash.hex(), channel_id) async def test_advanced_resolve(self): claim_id1 = self.get_claim_id( await self.stream_create('foo', '0.7', allow_duplicate_name=True)) await self.assertResolvesToClaimId('foo$1', claim_id1) claim_id2 = self.get_claim_id( await self.stream_create('foo', '0.8', allow_duplicate_name=True)) await self.assertResolvesToClaimId('foo$1', claim_id2) await self.assertResolvesToClaimId('foo$2', claim_id1) claim_id3 = self.get_claim_id( await self.stream_create('foo', '0.9', allow_duplicate_name=True)) # plain winning claim await self.assertResolvesToClaimId('foo', claim_id3) # amount order resolution await self.assertResolvesToClaimId('foo$1', claim_id3) await self.assertResolvesToClaimId('foo$2', claim_id2) await self.assertResolvesToClaimId('foo$3', claim_id1) await self.assertResolvesToClaimId('foo$4', None) # async def test_partial_claim_id_resolve(self): # # add some noise # await self.channel_create('@abc', '0.1', allow_duplicate_name=True) # await self.channel_create('@abc', '0.2', allow_duplicate_name=True) # await self.channel_create('@abc', '1.0', allow_duplicate_name=True) # # channel_id = self.get_claim_id(await self.channel_create('@abc', '1.1', allow_duplicate_name=True)) # await self.assertResolvesToClaimId(f'@abc', channel_id) # await self.assertResolvesToClaimId(f'@abc#{channel_id[:10]}', channel_id) # await self.assertResolvesToClaimId(f'@abc#{channel_id}', channel_id) # # channel = await self.claim_get(channel_id) # await self.assertResolvesToClaimId(channel['short_url'], channel_id) # await self.assertResolvesToClaimId(channel['canonical_url'], channel_id) # await self.assertResolvesToClaimId(channel['permanent_url'], channel_id) # # # add some noise # await self.stream_create('foo', '0.1', allow_duplicate_name=True, channel_id=channel['claim_id']) # await self.stream_create('foo', '0.2', allow_duplicate_name=True, channel_id=channel['claim_id']) # await self.stream_create('foo', '0.3', allow_duplicate_name=True, channel_id=channel['claim_id']) # # claim_id1 = self.get_claim_id( # await self.stream_create('foo', '0.7', allow_duplicate_name=True, channel_id=channel['claim_id'])) # claim1 = await self.claim_get(claim_id=claim_id1) # # await self.assertResolvesToClaimId('foo', claim_id1) # await self.assertResolvesToClaimId('@abc/foo', claim_id1) # await self.assertResolvesToClaimId(claim1['short_url'], claim_id1) # await self.assertResolvesToClaimId(claim1['canonical_url'], claim_id1) # await self.assertResolvesToClaimId(claim1['permanent_url'], claim_id1) # # claim_id2 = self.get_claim_id( # await self.stream_create('foo', '0.8', allow_duplicate_name=True, channel_id=channel['claim_id'])) # claim2 = await self.claim_get(claim_id=claim_id2) # await self.assertResolvesToClaimId('foo', claim_id2) # await self.assertResolvesToClaimId('@abc/foo', claim_id2) # await self.assertResolvesToClaimId(claim2['short_url'], claim_id2) # await self.assertResolvesToClaimId(claim2['canonical_url'], claim_id2) # await self.assertResolvesToClaimId(claim2['permanent_url'], claim_id2) async def test_abandoned_channel_with_signed_claims(self): channel = (await self.channel_create('@abc', '1.0'))['outputs'][0] orphan_claim = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel['claim_id']) abandoned_channel_id = channel['claim_id'] await self.channel_abandon(txid=channel['txid'], nout=0) channel = (await self.channel_create('@abc', '1.0'))['outputs'][0] orphan_claim_id = self.get_claim_id(orphan_claim) # Original channel doesn't exists anymore, so the signature is invalid. For invalid signatures, resolution is # only possible outside a channel self.assertEqual( {'error': { 'name': 'NOT_FOUND', 'text': 'Could not find claim at "lbry://@abc/on-channel-claim".', }}, await self.resolve('lbry://@abc/on-channel-claim') ) response = await self.resolve('lbry://on-channel-claim') self.assertFalse(response['is_channel_signature_valid']) self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel']) direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id response = await self.resolve(direct_uri) self.assertFalse(response['is_channel_signature_valid']) self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel']) await self.stream_abandon(claim_id=orphan_claim_id) uri = 'lbry://@abc/on-channel-claim' # now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore) valid_claim = await self.stream_create('on-channel-claim', '0.00000001', channel_id=channel['claim_id']) # resolves normally response = await self.resolve(uri) self.assertTrue(response['is_channel_signature_valid']) # ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition) await self.stream_create( 'on-channel-claim', '0.00000001', channel_id=channel['claim_id'], allow_duplicate_name=True ) # it still resolves! but to the older claim response = await self.resolve(uri) self.assertTrue(response['is_channel_signature_valid']) self.assertEqual(response['txid'], valid_claim['txid']) claims = [await self.resolve('on-channel-claim'), await self.resolve('on-channel-claim$2')] self.assertEqual(2, len(claims)) self.assertEqual( {channel['claim_id']}, {claim['signing_channel']['claim_id'] for claim in claims} ) async def test_normalization_resolution(self): one = 'ΣίσυφοςfiÆ' two = 'ΣΊΣΥΦΟσFIæ' c1 = await self.stream_create(one, '0.1') c2 = await self.stream_create(two, '0.2') loser_id = self.get_claim_id(c1) winner_id = self.get_claim_id(c2) # winning_one = await self.check_lbrycrd_winning(one) await self.assertMatchClaimIsWinning(two, winner_id) claim1 = await self.resolve(f'lbry://{one}') claim2 = await self.resolve(f'lbry://{two}') claim3 = await self.resolve(f'lbry://{one}:{winner_id[:5]}') claim4 = await self.resolve(f'lbry://{two}:{winner_id[:5]}') claim5 = await self.resolve(f'lbry://{one}:{loser_id[:5]}') claim6 = await self.resolve(f'lbry://{two}:{loser_id[:5]}') self.assertEqual(winner_id, claim1['claim_id']) self.assertEqual(winner_id, claim2['claim_id']) self.assertEqual(winner_id, claim3['claim_id']) self.assertEqual(winner_id, claim4['claim_id']) self.assertEqual(two, claim1['name']) self.assertEqual(two, claim2['name']) self.assertEqual(two, claim3['name']) self.assertEqual(two, claim4['name']) self.assertEqual(loser_id, claim5['claim_id']) self.assertEqual(loser_id, claim6['claim_id']) self.assertEqual(one, claim5['name']) self.assertEqual(one, claim6['name']) async def test_resolve_old_claim(self): channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0', blocking=True) await self.confirm_tx(channel.id) address = channel.outputs[0].get_address(self.account.ledger) claim = generate_signed_legacy(address, channel.outputs[0]) tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account) await tx.sign([self.account]) await self.broadcast_and_confirm(tx) response = await self.resolve('@olds/example') self.assertTrue('is_channel_signature_valid' in response, str(response)) self.assertTrue(response['is_channel_signature_valid']) claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature)) tx = await Transaction.claim_create( 'bad_example', claim.SerializeToString(), 1, address, [self.account], self.account ) await tx.sign([self.account]) await self.broadcast_and_confirm(tx) response = await self.resolve('bad_example') self.assertFalse(response['is_channel_signature_valid']) self.assertEqual( {'error': { 'name': 'NOT_FOUND', 'text': 'Could not find claim at "@olds/bad_example".', }}, await self.resolve('@olds/bad_example') ) async def test_resolve_with_includes(self): wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True) address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id) await self.wallet_send('1.0', address2) stream = await self.stream_create( 'priced', '0.1', wallet_id=wallet2.id, fee_amount='0.5', fee_currency='LBC', fee_address=address2 ) stream_id = self.get_claim_id(stream) resolve = await self.resolve('priced') self.assertNotIn('is_my_output', resolve) self.assertNotIn('purchase_receipt', resolve) self.assertNotIn('sent_supports', resolve) self.assertNotIn('sent_tips', resolve) self.assertNotIn('received_tips', resolve) # is_my_output resolve = await self.resolve('priced', include_is_my_output=True) self.assertFalse(resolve['is_my_output']) resolve = await self.resolve('priced', wallet_id=wallet2.id, include_is_my_output=True) self.assertTrue(resolve['is_my_output']) # purchase receipt resolve = await self.resolve('priced', include_purchase_receipt=True) self.assertNotIn('purchase_receipt', resolve) await self.purchase_create(stream_id) resolve = await self.resolve('priced', include_purchase_receipt=True) self.assertEqual('0.5', resolve['purchase_receipt']['amount']) # my supports and my tips resolve = await self.resolve( 'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True ) self.assertEqual('0.0', resolve['sent_supports']) self.assertEqual('0.0', resolve['sent_tips']) self.assertEqual('0.0', resolve['received_tips']) await self.support_create(stream_id, '0.3') await self.support_create(stream_id, '0.2') await self.support_create(stream_id, '0.4', tip=True) await self.support_create(stream_id, '0.5', tip=True) resolve = await self.resolve( 'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True ) self.assertEqual('0.5', resolve['sent_supports']) self.assertEqual('0.9', resolve['sent_tips']) self.assertEqual('0.0', resolve['received_tips']) resolve = await self.resolve( 'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True, wallet_id=wallet2.id ) self.assertEqual('0.0', resolve['sent_supports']) self.assertEqual('0.0', resolve['sent_tips']) self.assertEqual('0.9', resolve['received_tips']) self.assertEqual('1.4', resolve['meta']['support_amount']) # make sure nothing is leaked between wallets through cached tx/txos resolve = await self.resolve('priced') self.assertNotIn('is_my_output', resolve) self.assertNotIn('purchase_receipt', resolve) self.assertNotIn('sent_supports', resolve) self.assertNotIn('sent_tips', resolve) self.assertNotIn('received_tips', resolve) class ResolveClaimTakeovers(BaseResolveTestCase): async def test_channel_invalidation(self): channel_id = (await self.channel_create('@test', '0.1'))['outputs'][0]['claim_id'] channel_id2 = (await self.channel_create('@other', '0.1'))['outputs'][0]['claim_id'] async def make_claim(name, amount, channel_id=None): return ( await self.stream_create(name, amount, channel_id=channel_id) )['outputs'][0]['claim_id'] unsigned_then_signed = await make_claim('unsigned_then_signed', '0.1') unsigned_then_updated_then_signed = await make_claim('unsigned_then_updated_then_signed', '0.1') signed_then_unsigned = await make_claim( 'signed_then_unsigned', '0.01', channel_id=channel_id ) signed_then_signed_different_chan = await make_claim( 'signed_then_signed_different_chan', '0.01', channel_id=channel_id ) self.assertIn("error", await self.resolve('@test/unsigned_then_signed')) await self.assertMatchClaimIsWinning('unsigned_then_signed', unsigned_then_signed) self.assertIn("error", await self.resolve('@test/unsigned_then_updated_then_signed')) await self.assertMatchClaimIsWinning('unsigned_then_updated_then_signed', unsigned_then_updated_then_signed) self.assertDictEqual( await self.resolve('@test/signed_then_unsigned'), await self.resolve('signed_then_unsigned') ) await self.assertMatchClaimIsWinning('signed_then_unsigned', signed_then_unsigned) # sign 'unsigned_then_signed' and update it await self.ledger.wait(await self.daemon.jsonrpc_stream_update( unsigned_then_signed, '0.09', channel_id=channel_id)) await self.ledger.wait(await self.daemon.jsonrpc_stream_update(unsigned_then_updated_then_signed, '0.09')) await self.ledger.wait(await self.daemon.jsonrpc_stream_update( unsigned_then_updated_then_signed, '0.09', channel_id=channel_id)) await self.ledger.wait(await self.daemon.jsonrpc_stream_update( signed_then_unsigned, '0.09', clear_channel=True)) await self.ledger.wait(await self.daemon.jsonrpc_stream_update( signed_then_signed_different_chan, '0.09', channel_id=channel_id2)) await self.daemon.jsonrpc_txo_spend(type='channel', claim_id=channel_id) signed3 = await make_claim('signed3', '0.01', channel_id=channel_id) signed4 = await make_claim('signed4', '0.01', channel_id=channel_id2) self.assertIn("error", await self.resolve('@test')) self.assertIn("error", await self.resolve('@test/signed1')) self.assertIn("error", await self.resolve('@test/unsigned_then_updated_then_signed')) self.assertIn("error", await self.resolve('@test/unsigned_then_signed')) self.assertIn("error", await self.resolve('@test/signed3')) self.assertIn("error", await self.resolve('@test/signed4')) await self.assertMatchClaimIsWinning('signed_then_unsigned', signed_then_unsigned) await self.assertMatchClaimIsWinning('unsigned_then_signed', unsigned_then_signed) await self.assertMatchClaimIsWinning('unsigned_then_updated_then_signed', unsigned_then_updated_then_signed) await self.assertMatchClaimIsWinning('signed_then_signed_different_chan', signed_then_signed_different_chan) await self.assertMatchClaimIsWinning('signed3', signed3) await self.assertMatchClaimIsWinning('signed4', signed4) self.assertDictEqual(await self.resolve('@other/signed_then_signed_different_chan'), await self.resolve('signed_then_signed_different_chan')) self.assertDictEqual(await self.resolve('@other/signed4'), await self.resolve('signed4')) self.assertEqual(2, len(await self.claim_search(channel_ids=[channel_id2]))) await self.channel_update(channel_id2) await make_claim('third_signed', '0.01', channel_id=channel_id2) self.assertEqual(3, len(await self.claim_search(channel_ids=[channel_id2]))) async def _test_activation_delay(self): name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) # a claim of higher amount made now will have a takeover delay of 10 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] # sanity check self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(9) # not yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) # the new claim should have activated await self.assertMatchClaimIsWinning(name, second_claim_id) return first_claim_id, second_claim_id async def test_activation_delay(self): await self._test_activation_delay() async def test_activation_delay_then_abandon_then_reclaim(self): name = 'derp' first_claim_id, second_claim_id = await self._test_activation_delay() await self.daemon.jsonrpc_txo_spend(type='stream', claim_id=first_claim_id) await self.daemon.jsonrpc_txo_spend(type='stream', claim_id=second_claim_id) await self.generate(1) await self.assertNoClaimForName(name) await self._test_activation_delay() async def create_stream_claim(self, amount: str, name='derp') -> str: return (await self.stream_create(name, amount, allow_duplicate_name=True))['outputs'][0]['claim_id'] async def assertNameState(self, height: int, name: str, winning_claim_id: str, last_takeover_height: int, non_winning_claims: List[ClaimStateValue]): self.assertEqual(height, self.conductor.spv_node.server.db.db_height) await self.assertMatchClaimIsWinning(name, winning_claim_id) for non_winning in non_winning_claims: claim = await self.assertMatchClaim(name, non_winning.claim_id, is_active_in_lbrycrd=non_winning.active_in_lbrycrd ) self.assertEqual(non_winning.activation_height, claim.activation_height) self.assertEqual(last_takeover_height, claim.last_takeover_height) async def test_delay_takeover_with_update(self): name = 'derp' first_claim_id = await self.create_stream_claim('0.2', name) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) second_claim_id = await self.create_stream_claim('0.1', name) third_claim_id = await self.create_stream_claim('0.1', name) await self.generate(8) await self.assertNameState( height=537, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=False), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=538, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=539, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=True) ] ) await self.daemon.jsonrpc_stream_update(third_claim_id, '0.21') await self.generate(1) await self.assertNameState( height=540, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(9) await self.assertNameState( height=549, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=550, name=name, winning_claim_id=third_claim_id, last_takeover_height=550, non_winning_claims=[ ClaimStateValue(first_claim_id, activation_height=207, active_in_lbrycrd=True), ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True) ] ) async def test_delay_takeover_with_update_then_update_to_lower_before_takeover(self): name = 'derp' first_claim_id = await self.create_stream_claim('0.2', name) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) second_claim_id = await self.create_stream_claim('0.1', name) third_claim_id = await self.create_stream_claim('0.1', name) await self.generate(8) await self.assertNameState( height=537, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=False), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=538, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=539, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=True) ] ) await self.daemon.jsonrpc_stream_update(third_claim_id, '0.21') await self.generate(1) await self.assertNameState( height=540, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(8) await self.assertNameState( height=548, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.daemon.jsonrpc_stream_update(third_claim_id, '0.09') await self.generate(1) await self.assertNameState( height=549, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=559, active_in_lbrycrd=False) ] ) await self.generate(10) await self.assertNameState( height=559, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=559, active_in_lbrycrd=True) ] ) async def test_delay_takeover_with_update_then_update_to_lower_on_takeover(self): name = 'derp' first_claim_id = await self.create_stream_claim('0.2', name) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) second_claim_id = await self.create_stream_claim('0.1', name) third_claim_id = await self.create_stream_claim('0.1', name) await self.generate(8) await self.assertNameState( height=537, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=False), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=538, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=539, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=True) ] ) await self.daemon.jsonrpc_stream_update(third_claim_id, '0.21') await self.generate(1) await self.assertNameState( height=540, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(8) await self.assertNameState( height=548, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=549, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.daemon.jsonrpc_stream_update(third_claim_id, '0.09') await self.generate(1) await self.assertNameState( height=550, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=560, active_in_lbrycrd=False) ] ) await self.generate(10) await self.assertNameState( height=560, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=560, active_in_lbrycrd=True) ] ) async def test_delay_takeover_with_update_then_update_to_lower_after_takeover(self): name = 'derp' first_claim_id = await self.create_stream_claim('0.2', name) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) second_claim_id = await self.create_stream_claim('0.1', name) third_claim_id = await self.create_stream_claim('0.1', name) await self.generate(8) await self.assertNameState( height=537, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=False), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=538, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=539, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=539, active_in_lbrycrd=True) ] ) await self.daemon.jsonrpc_stream_update(third_claim_id, '0.21') await self.generate(1) await self.assertNameState( height=540, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(8) await self.assertNameState( height=548, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=549, name=name, winning_claim_id=first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=550, active_in_lbrycrd=False) ] ) await self.generate(1) await self.assertNameState( height=550, name=name, winning_claim_id=third_claim_id, last_takeover_height=550, non_winning_claims=[ ClaimStateValue(first_claim_id, activation_height=207, active_in_lbrycrd=True), ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True) ] ) await self.daemon.jsonrpc_stream_update(third_claim_id, '0.09') await self.generate(1) await self.assertNameState( height=551, name=name, winning_claim_id=first_claim_id, last_takeover_height=551, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True), ClaimStateValue(third_claim_id, activation_height=551, active_in_lbrycrd=True) ] ) async def test_resolve_signed_claims_with_fees(self): channel_name = '@abc' channel_id = self.get_claim_id( await self.channel_create(channel_name, '0.01') ) self.assertEqual(channel_id, (await self.assertMatchWinningClaim(channel_name)).claim_hash.hex()) stream_name = 'foo' stream_with_no_fee = self.get_claim_id( await self.stream_create(stream_name, '0.01', channel_id=channel_id) ) stream_with_fee = self.get_claim_id( await self.stream_create('with_a_fee', '0.01', channel_id=channel_id, fee_amount='1', fee_currency='LBC') ) greater_than_or_equal_to_zero = [ claim['claim_id'] for claim in ( await self.conductor.spv_node.server.session_manager.search_index.search( channel_id=channel_id, fee_amount=">=0" ))[0] ] self.assertEqual(2, len(greater_than_or_equal_to_zero)) self.assertSetEqual(set(greater_than_or_equal_to_zero), {stream_with_no_fee, stream_with_fee}) greater_than_zero = [ claim['claim_id'] for claim in ( await self.conductor.spv_node.server.session_manager.search_index.search( channel_id=channel_id, fee_amount=">0" ))[0] ] self.assertEqual(1, len(greater_than_zero)) self.assertSetEqual(set(greater_than_zero), {stream_with_fee}) equal_to_zero = [ claim['claim_id'] for claim in ( await self.conductor.spv_node.server.session_manager.search_index.search( channel_id=channel_id, fee_amount="<=0" ))[0] ] self.assertEqual(1, len(equal_to_zero)) self.assertSetEqual(set(equal_to_zero), {stream_with_no_fee}) async def test_spec_example(self): # https://spec.lbry.com/#claim-activation-example # this test has adjusted block heights from the example because it uses the regtest chain instead of mainnet # on regtest, claims expire much faster, so we can't do the ~1000 block delay in the spec example exactly name = 'test' await self.generate(494) address = (await self.account.receiving.get_addresses(True))[0] await self.send_to_address_and_wait(address, 400.0) await self.account.ledger.on_address.first await self.generate(100) self.assertEqual(800, self.conductor.spv_node.server.db.db_height) # Block 801: Claim A for 10 LBC is accepted. # It is the first claim, so it immediately becomes active and controlling. # State: A(10) is controlling claim_id_A = (await self.stream_create(name, '10.0', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, claim_id_A) # Block 1121: Claim B for 20 LBC is accepted. # Its activation height is 1121 + min(4032, floor((1121-801) / 32)) = 1121 + 10 = 1131. # State: A(10) is controlling, B(20) is accepted. await self.generate(32 * 10 - 1) self.assertEqual(1120, self.conductor.spv_node.server.db.db_height) claim_id_B = (await self.stream_create(name, '20.0', allow_duplicate_name=True))['outputs'][0]['claim_id'] claim_B, _, _, _ = await self.conductor.spv_node.server.db.resolve(f"{name}:{claim_id_B}") self.assertEqual(1121, self.conductor.spv_node.server.db.db_height) self.assertEqual(1131, claim_B.activation_height) await self.assertMatchClaimIsWinning(name, claim_id_A) # Block 1122: Support X for 14 LBC for claim A is accepted. # Since it is a support for the controlling claim, it activates immediately. # State: A(10+14) is controlling, B(20) is accepted. await self.support_create(claim_id_A, bid='14.0') self.assertEqual(1122, self.conductor.spv_node.server.db.db_height) await self.assertMatchClaimIsWinning(name, claim_id_A) # Block 1123: Claim C for 50 LBC is accepted. # The activation height is 1123 + min(4032, floor((1123-801) / 32)) = 1123 + 10 = 1133. # State: A(10+14) is controlling, B(20) is accepted, C(50) is accepted. claim_id_C = (await self.stream_create(name, '50.0', allow_duplicate_name=True))['outputs'][0]['claim_id'] self.assertEqual(1123, self.conductor.spv_node.server.db.db_height) claim_C, _, _, _ = await self.conductor.spv_node.server.db.resolve(f"{name}:{claim_id_C}") self.assertEqual(1133, claim_C.activation_height) await self.assertMatchClaimIsWinning(name, claim_id_A) await self.generate(7) self.assertEqual(1130, self.conductor.spv_node.server.db.db_height) await self.assertMatchClaimIsWinning(name, claim_id_A) await self.generate(1) # Block 1131: Claim B activates. It has 20 LBC, while claim A has 24 LBC (10 original + 14 from support X). There is no takeover, and claim A remains controlling. # State: A(10+14) is controlling, B(20) is active, C(50) is accepted. self.assertEqual(1131, self.conductor.spv_node.server.db.db_height) await self.assertMatchClaimIsWinning(name, claim_id_A) # Block 1132: Claim D for 300 LBC is accepted. The activation height is 1132 + min(4032, floor((1132-801) / 32)) = 1132 + 10 = 1142. # State: A(10+14) is controlling, B(20) is active, C(50) is accepted, D(300) is accepted. claim_id_D = (await self.stream_create(name, '300.0', allow_duplicate_name=True))['outputs'][0]['claim_id'] self.assertEqual(1132, self.conductor.spv_node.server.db.db_height) claim_D, _, _, _ = await self.conductor.spv_node.server.db.resolve(f"{name}:{claim_id_D}") self.assertEqual(False, claim_D.is_controlling) self.assertEqual(801, claim_D.last_takeover_height) self.assertEqual(1142, claim_D.activation_height) await self.assertMatchClaimIsWinning(name, claim_id_A) # Block 1133: Claim C activates. It has 50 LBC, while claim A has 24 LBC, so a takeover is initiated. The takeover height for this name is set to 1133, and therefore the activation delay for all the claims becomes min(4032, floor((1133-1133) / 32)) = 0. All the claims become active. The totals for each claim are recalculated, and claim D becomes controlling because it has the highest total. # State: A(10+14) is active, B(20) is active, C(50) is active, D(300) is controlling await self.generate(1) self.assertEqual(1133, self.conductor.spv_node.server.db.db_height) claim_D, _, _, _ = await self.conductor.spv_node.server.db.resolve(f"{name}:{claim_id_D}") self.assertEqual(True, claim_D.is_controlling) self.assertEqual(1133, claim_D.last_takeover_height) self.assertEqual(1133, claim_D.activation_height) await self.assertMatchClaimIsWinning(name, claim_id_D) async def test_early_takeover(self): name = 'derp' # block 207 first_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(96) # block 304, activates at 307 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] # block 305, activates at 308 (but gets triggered early by the takeover by the second claim) third_claim_id = (await self.stream_create(name, '0.3', allow_duplicate_name=True))['outputs'][0]['claim_id'] self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) async def test_early_takeover_zero_delay(self): name = 'derp' # block 207 first_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(96) # block 304, activates at 307 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) # on block 307 make a third claim with a yet higher amount, it takes over with no delay because the # second claim activates and begins the takeover on this block third_claim_id = (await self.stream_create(name, '0.3', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, third_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) async def test_early_takeover_from_support_zero_delay(self): name = 'derp' # block 207 first_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(96) # block 304, activates at 307 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) third_claim_id = (await self.stream_create(name, '0.19', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) tx = await self.daemon.jsonrpc_support_create(third_claim_id, '0.1') await self.ledger.wait(tx) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) async def test_early_takeover_from_support_and_claim_zero_delay(self): name = 'derp' # block 207 first_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(96) # block 304, activates at 307 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) file_path = self.create_upload_file(data=b'hi!') tx = await self.daemon.jsonrpc_stream_create(name, '0.19', file_path=file_path, allow_duplicate_name=True) await self.ledger.wait(tx) third_claim_id = tx.outputs[0].claim_id wallet = self.daemon.wallet_manager.get_wallet_or_default(None) funding_accounts = wallet.get_accounts_or_all(None) amount = self.daemon.get_dewies_or_error("amount", '0.1') account = wallet.get_account_or_default(None) claim_address = await account.receiving.get_or_create_usable_address() tx = await Transaction.support( 'derp', third_claim_id, amount, claim_address, funding_accounts, funding_accounts[0], None ) await tx.sign(funding_accounts) await self.daemon.broadcast_or_release(tx, True) await self.ledger.wait(tx) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) async def test_early_takeover_abandoned_controlling_support(self): name = 'derp' # block 207 first_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0][ 'claim_id'] tx = await self.daemon.jsonrpc_support_create(first_claim_id, '0.2') await self.ledger.wait(tx) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(96) # block 304, activates at 307 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0][ 'claim_id'] # block 305, activates at 308 (but gets triggered early by the takeover by the second claim) third_claim_id = (await self.stream_create(name, '0.3', allow_duplicate_name=True))['outputs'][0][ 'claim_id'] self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.daemon.jsonrpc_txo_spend(type='support', txid=tx.id) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, third_claim_id) async def test_block_takeover_with_delay_1_support(self): name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] self.assertEqual(first_claim_id, (await self.assertMatchWinningClaim(name)).claim_hash.hex()) await self.generate(320) # a claim of higher amount made now will have a takeover delay of 10 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] # sanity check self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) for _ in range(8): await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) # prevent the takeover by adding a support one block before the takeover happens await self.support_create(first_claim_id, bid='1.0') await self.assertMatchClaimIsWinning(name, first_claim_id) # one more block until activation await self.generate(1) await self.assertMatchClaimIsWinning(name, first_claim_id) async def test_block_takeover_with_delay_0_support(self): name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) # a claim of higher amount made now will have a takeover delay of 10 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] # sanity check await self.assertMatchClaimIsWinning(name, first_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(9) await self.assertMatchClaimIsWinning(name, first_claim_id) # prevent the takeover by adding a support on the same block the takeover would happen await self.support_create(first_claim_id, bid='1.0') await self.assertMatchClaimIsWinning(name, first_claim_id) async def _test_almost_prevent_takeover(self, name: str, blocks: int = 9): # initially claim the name first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) # a claim of higher amount made now will have a takeover delay of 10 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] # sanity check self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(blocks) await self.assertMatchClaimIsWinning(name, first_claim_id) # prevent the takeover by adding a support on the same block the takeover would happen tx = await self.daemon.jsonrpc_support_create(first_claim_id, '1.0') await self.ledger.wait(tx) return first_claim_id, second_claim_id, tx async def test_almost_prevent_takeover_remove_support_same_block_supported(self): name = 'derp' first_claim_id, second_claim_id, tx = await self._test_almost_prevent_takeover(name, 9) await self.daemon.jsonrpc_txo_spend(type='support', txid=tx.id) await self.generate(1) await self.assertMatchClaimIsWinning(name, second_claim_id) async def test_almost_prevent_takeover_remove_support_one_block_after_supported(self): name = 'derp' first_claim_id, second_claim_id, tx = await self._test_almost_prevent_takeover(name, 8) await self.generate(1) await self.daemon.jsonrpc_txo_spend(type='support', txid=tx.id) await self.generate(1) await self.assertMatchClaimIsWinning(name, second_claim_id) async def test_abandon_before_takeover(self): name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) # a claim of higher amount made now will have a takeover delay of 10 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] # sanity check self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(8) await self.assertMatchClaimIsWinning(name, first_claim_id) # abandon the winning claim await self.daemon.jsonrpc_txo_spend(type='stream', claim_id=first_claim_id) await self.generate(1) # the takeover and activation should happen a block earlier than they would have absent the abandon await self.assertMatchClaimIsWinning(name, second_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, second_claim_id) async def test_abandon_before_takeover_no_delay_update(self): # TODO: fix race condition line 506 name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) # block 527 # a claim of higher amount made now will have a takeover delay of 10 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] # block 528 # sanity check self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.assertMatchClaimsForName(name) await self.generate(8) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.assertMatchClaimsForName(name) # abandon the winning claim await self.daemon.jsonrpc_txo_spend(type='stream', claim_id=first_claim_id) await self.daemon.jsonrpc_stream_update(second_claim_id, '0.1') await self.generate(1) # the takeover and activation should happen a block earlier than they would have absent the abandon await self.assertMatchClaimIsWinning(name, second_claim_id) await self.assertMatchClaimsForName(name) await self.generate(1) # await self.ledger.on_header.where(lambda e: e.height == 537) await self.assertMatchClaimIsWinning(name, second_claim_id) await self.assertMatchClaimsForName(name) async def test_abandon_controlling_support_before_pending_takeover(self): name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] controlling_support_tx = await self.daemon.jsonrpc_support_create(first_claim_id, '0.9') await self.ledger.wait(controlling_support_tx) self.assertEqual(first_claim_id, (await self.assertMatchWinningClaim(name)).claim_hash.hex()) await self.generate(321) second_claim_id = (await self.stream_create(name, '0.9', allow_duplicate_name=True))['outputs'][0]['claim_id'] self.assertNotEqual(first_claim_id, second_claim_id) # takeover should not have happened yet await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(8) await self.assertMatchClaimIsWinning(name, first_claim_id) # abandon the support that causes the winning claim to have the highest staked tx = await self.daemon.jsonrpc_txo_spend(type='support', txid=controlling_support_tx.id, blocking=True) await self.generate(1) await self.assertNameState(538, name, first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=539, active_in_lbrycrd=False) ]) await self.generate(1) await self.assertNameState(539, name, second_claim_id, last_takeover_height=539, non_winning_claims=[ ClaimStateValue(first_claim_id, activation_height=207, active_in_lbrycrd=True) ]) async def test_remove_controlling_support(self): name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.2'))['outputs'][0]['claim_id'] first_support_tx = await self.daemon.jsonrpc_support_create(first_claim_id, '0.9') await self.ledger.wait(first_support_tx) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(320) # give the first claim long enough for a 10 block takeover delay await self.assertNameState(527, name, first_claim_id, last_takeover_height=207, non_winning_claims=[]) # make a second claim which will take over the name second_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertNameState(528, name, first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=False) ]) second_claim_support_tx = await self.daemon.jsonrpc_support_create(second_claim_id, '1.5') await self.ledger.wait(second_claim_support_tx) await self.generate(1) # neither the second claim or its support have activated yet await self.assertNameState(529, name, first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=False) ]) await self.generate(9) # claim activates, but is not yet winning await self.assertNameState(538, name, first_claim_id, last_takeover_height=207, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True) ]) await self.generate(1) # support activates, takeover happens await self.assertNameState(539, name, second_claim_id, last_takeover_height=539, non_winning_claims=[ ClaimStateValue(first_claim_id, activation_height=207, active_in_lbrycrd=True) ]) await self.daemon.jsonrpc_txo_spend(type='support', claim_id=second_claim_id, blocking=True) await self.generate(1) # support activates, takeover happens await self.assertNameState(540, name, first_claim_id, last_takeover_height=540, non_winning_claims=[ ClaimStateValue(second_claim_id, activation_height=538, active_in_lbrycrd=True) ]) async def test_claim_expiration(self): name = 'derp' # starts at height 206 vanishing_claim = (await self.stream_create('vanish', '0.1'))['outputs'][0]['claim_id'] await self.generate(493) # in block 701 and 702 first_claim_id = (await self.stream_create(name, '0.3'))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning('vanish', vanishing_claim) await self.generate(100) # block 801, expiration fork happened await self.assertNoClaimForName('vanish') # second claim is in block 802 second_claim_id = (await self.stream_create(name, '0.2', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(498) await self.assertMatchClaimIsWinning(name, first_claim_id) await self.generate(1) await self.assertMatchClaimIsWinning(name, second_claim_id) await self.generate(100) await self.assertMatchClaimIsWinning(name, second_claim_id) await self.generate(1) await self.assertNoClaimForName(name) async def _test_add_non_winning_already_claimed(self): name = 'derp' # initially claim the name first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] self.assertEqual(first_claim_id, (await self.assertMatchWinningClaim(name)).claim_hash.hex()) await self.generate(32) second_claim_id = (await self.stream_create(name, '0.01', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertNoClaim(name, second_claim_id) self.assertEqual( len((await self.conductor.spv_node.server.session_manager.search_index.search(claim_name=name))[0]), 1 ) await self.generate(1) await self.assertMatchClaim(name, second_claim_id) self.assertEqual( len((await self.conductor.spv_node.server.session_manager.search_index.search(claim_name=name))[0]), 2 ) async def test_abandon_controlling_same_block_as_new_claim(self): name = 'derp' first_claim_id = (await self.stream_create(name, '0.1'))['outputs'][0]['claim_id'] await self.generate(64) await self.assertNameState(271, name, first_claim_id, last_takeover_height=207, non_winning_claims=[]) await self.daemon.jsonrpc_txo_spend(type='stream', claim_id=first_claim_id) second_claim_id = (await self.stream_create(name, '0.1', allow_duplicate_name=True))['outputs'][0]['claim_id'] await self.assertNameState(272, name, second_claim_id, last_takeover_height=272, non_winning_claims=[]) async def test_trending(self): async def get_trending_score(claim_id): return (await self.conductor.spv_node.server.session_manager.search_index.search( claim_id=claim_id ))[0][0]['trending_score'] claim_id1 = (await self.stream_create('derp', '1.0'))['outputs'][0]['claim_id'] COIN = int(1E8) self.assertEqual(self.conductor.spv_node.writer.height, 207) self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put( (208, bytes.fromhex(claim_id1)), (0, 10 * COIN) ) await self.generate(1) self.assertEqual(self.conductor.spv_node.writer.height, 208) self.assertEqual(1.7090807854206793, await get_trending_score(claim_id1)) self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put( (209, bytes.fromhex(claim_id1)), (10 * COIN, 100 * COIN) ) await self.generate(1) self.assertEqual(self.conductor.spv_node.writer.height, 209) self.assertEqual(2.2437974397778886, await get_trending_score(claim_id1)) self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put( (309, bytes.fromhex(claim_id1)), (100 * COIN, 1000000 * COIN) ) await self.generate(100) self.assertEqual(self.conductor.spv_node.writer.height, 309) self.assertEqual(5.157053472135866, await get_trending_score(claim_id1)) self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put( (409, bytes.fromhex(claim_id1)), (1000000 * COIN, 1 * COIN) ) await self.generate(99) self.assertEqual(self.conductor.spv_node.writer.height, 408) self.assertEqual(5.157053472135866, await get_trending_score(claim_id1)) await self.generate(1) self.assertEqual(self.conductor.spv_node.writer.height, 409) self.assertEqual(-3.4256156592205627, await get_trending_score(claim_id1)) search_results = (await self.conductor.spv_node.server.session_manager.search_index.search(claim_name="derp"))[0] self.assertEqual(1, len(search_results)) self.assertListEqual([claim_id1], [c['claim_id'] for c in search_results]) class ResolveAfterReorg(BaseResolveTestCase): async def reorg(self, start): blocks = self.ledger.headers.height - start self.blockchain.block_expected = start - 1 # go back to start await self.blockchain.invalidate_block((await self.ledger.headers.hash(start)).decode()) # go to previous + 1 await self.generate(blocks + 2) async def assertBlockHash(self, height): reader_db = self.conductor.spv_node.server.db block_hash = await self.blockchain.get_block_hash(height) self.assertEqual(block_hash, (await self.ledger.headers.hash(height)).decode()) self.assertEqual(block_hash, (await reader_db.fs_block_hashes(height, 1))[0][::-1].hex()) txids = [ tx_hash[::-1].hex() for tx_hash in reader_db.get_block_txs(height) ] txs = await reader_db.get_transactions_and_merkles(txids) block_txs = (await self.conductor.spv_node.server.daemon.deserialised_block(block_hash))['tx'] self.assertSetEqual(set(block_txs), set(txs.keys()), msg='leveldb/lbrycrd is missing transactions') self.assertListEqual(block_txs, list(txs.keys()), msg='leveldb/lbrycrd transactions are of order') async def test_reorg(self): self.assertEqual(self.ledger.headers.height, 206) channel_name = '@abc' channel_id = self.get_claim_id( await self.channel_create(channel_name, '0.01') ) await self.assertNameState( height=207, name='@abc', winning_claim_id=channel_id, last_takeover_height=207, non_winning_claims=[] ) await self.reorg(206) await self.assertNameState( height=208, name='@abc', winning_claim_id=channel_id, last_takeover_height=207, non_winning_claims=[] ) # await self.assertNoClaimForName(channel_name) # self.assertNotIn('error', await self.resolve(channel_name)) stream_name = 'foo' stream_id = self.get_claim_id( await self.stream_create(stream_name, '0.01', channel_id=channel_id) ) await self.assertNameState( height=209, name=stream_name, winning_claim_id=stream_id, last_takeover_height=209, non_winning_claims=[] ) await self.reorg(206) await self.assertNameState( height=210, name=stream_name, winning_claim_id=stream_id, last_takeover_height=209, non_winning_claims=[] ) await self.support_create(stream_id, '0.01') await self.assertNameState( height=211, name=stream_name, winning_claim_id=stream_id, last_takeover_height=209, non_winning_claims=[] ) await self.reorg(206) # self.assertNotIn('error', await self.resolve(stream_name)) await self.assertNameState( height=212, name=stream_name, winning_claim_id=stream_id, last_takeover_height=209, non_winning_claims=[] ) await self.stream_abandon(stream_id) self.assertNotIn('error', await self.resolve(channel_name)) self.assertIn('error', await self.resolve(stream_name)) self.assertEqual(channel_id, (await self.assertMatchWinningClaim(channel_name)).claim_hash.hex()) await self.assertNoClaimForName(stream_name) # TODO: check @abc/foo too await self.reorg(206) self.assertNotIn('error', await self.resolve(channel_name)) self.assertIn('error', await self.resolve(stream_name)) self.assertEqual(channel_id, (await self.assertMatchWinningClaim(channel_name)).claim_hash.hex()) await self.assertNoClaimForName(stream_name) await self.channel_abandon(channel_id) self.assertIn('error', await self.resolve(channel_name)) self.assertIn('error', await self.resolve(stream_name)) await self.reorg(206) self.assertIn('error', await self.resolve(channel_name)) self.assertIn('error', await self.resolve(stream_name)) async def test_reorg_change_claim_height(self): # sanity check result = await self.resolve('hovercraft') # TODO: do these for claim_search and resolve both self.assertIn('error', result) still_valid = await self.daemon.jsonrpc_stream_create( 'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(still_valid) await self.generate(1) # create a claim and verify it's returned by claim_search self.assertEqual(self.ledger.headers.height, 207) await self.assertBlockHash(207) broadcast_tx = await self.daemon.jsonrpc_stream_create( 'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(broadcast_tx) await self.support_create(still_valid.outputs[0].claim_id, '0.01') await self.ledger.wait(broadcast_tx, self.blockchain.block_expected) self.assertEqual(self.ledger.headers.height, 208) await self.assertBlockHash(208) claim = await self.resolve('hovercraft') self.assertEqual(claim['txid'], broadcast_tx.id) self.assertEqual(claim['height'], 208) # check that our tx is in block 208 as returned by lbrycrdd invalidated_block_hash = (await self.ledger.headers.hash(208)).decode() block_207 = await self.blockchain.get_block(invalidated_block_hash) self.assertIn(claim['txid'], block_207['tx']) self.assertEqual(208, claim['height']) # reorg the last block dropping our claim tx await self.blockchain.invalidate_block(invalidated_block_hash) await self.conductor.clear_mempool() await self.blockchain.generate(2) # wait for the client to catch up and verify the reorg await asyncio.wait_for(self.on_header(209), 3.0) await self.assertBlockHash(207) await self.assertBlockHash(208) await self.assertBlockHash(209) # verify the claim was dropped from block 208 as returned by lbrycrdd reorg_block_hash = await self.blockchain.get_block_hash(208) self.assertNotEqual(invalidated_block_hash, reorg_block_hash) block_207 = await self.blockchain.get_block(reorg_block_hash) self.assertNotIn(claim['txid'], block_207['tx']) client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode() self.assertEqual(client_reorg_block_hash, reorg_block_hash) # verify the dropped claim is no longer returned by claim search self.assertDictEqual( {'error': {'name': 'NOT_FOUND', 'text': 'Could not find claim at "hovercraft".'}}, await self.resolve('hovercraft') ) # verify the claim published a block earlier wasn't also reverted self.assertEqual(207, (await self.resolve('still-valid'))['height']) # broadcast the claim in a different block new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode()) self.assertEqual(broadcast_tx.id, new_txid) await self.blockchain.generate(1) # wait for the client to catch up await asyncio.wait_for(self.on_header(210), 3.0) # verify the claim is in the new block and that it is returned by claim_search republished = await self.resolve('hovercraft') self.assertEqual(210, republished['height']) self.assertEqual(claim['claim_id'], republished['claim_id']) # this should still be unchanged self.assertEqual(207, (await self.resolve('still-valid'))['height']) async def test_reorg_drop_claim(self): # sanity check result = await self.resolve('hovercraft') # TODO: do these for claim_search and resolve both self.assertIn('error', result) still_valid = await self.daemon.jsonrpc_stream_create( 'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(still_valid) await self.generate(1) # create a claim and verify it's returned by claim_search self.assertEqual(self.ledger.headers.height, 207) await self.assertBlockHash(207) broadcast_tx = await self.daemon.jsonrpc_stream_create( 'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!') ) await self.ledger.wait(broadcast_tx) await self.generate(1) await self.ledger.wait(broadcast_tx, self.blockchain.block_expected) self.assertEqual(self.ledger.headers.height, 208) await self.assertBlockHash(208) claim = await self.resolve('hovercraft') self.assertEqual(claim['txid'], broadcast_tx.id) self.assertEqual(claim['height'], 208) # check that our tx is in block 208 as returned by lbrycrdd invalidated_block_hash = (await self.ledger.headers.hash(208)).decode() block_207 = await self.blockchain.get_block(invalidated_block_hash) self.assertIn(claim['txid'], block_207['tx']) self.assertEqual(208, claim['height']) # reorg the last block dropping our claim tx await self.blockchain.invalidate_block(invalidated_block_hash) await self.conductor.clear_mempool() await self.blockchain.generate(2) # wait for the client to catch up and verify the reorg await asyncio.wait_for(self.on_header(209), 30.0) await self.assertBlockHash(207) await self.assertBlockHash(208) await self.assertBlockHash(209) # verify the claim was dropped from block 208 as returned by lbrycrdd reorg_block_hash = await self.blockchain.get_block_hash(208) self.assertNotEqual(invalidated_block_hash, reorg_block_hash) block_207 = await self.blockchain.get_block(reorg_block_hash) self.assertNotIn(claim['txid'], block_207['tx']) client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode() self.assertEqual(client_reorg_block_hash, reorg_block_hash) # verify the dropped claim is no longer returned by claim search self.assertDictEqual( {'error': {'name': 'NOT_FOUND', 'text': 'Could not find claim at "hovercraft".'}}, await self.resolve('hovercraft') ) # verify the claim published a block earlier wasn't also reverted self.assertEqual(207, (await self.resolve('still-valid'))['height']) # broadcast the claim in a different block new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode()) self.assertEqual(broadcast_tx.id, new_txid) await self.blockchain.generate(1) # wait for the client to catch up await asyncio.wait_for(self.on_header(210), 1.0) # verify the claim is in the new block and that it is returned by claim_search republished = await self.resolve('hovercraft') self.assertEqual(210, republished['height']) self.assertEqual(claim['claim_id'], republished['claim_id']) # this should still be unchanged self.assertEqual(207, (await self.resolve('still-valid'))['height']) def generate_signed_legacy(address: bytes, output: Output): decoded_address = Base58.decode(address) claim = OldClaimMessage() claim.ParseFromString(unhexlify( '080110011aee04080112a604080410011a2b4865726520617265203520526561736f6e73204920e29da4e' 'fb88f204e657874636c6f7564207c20544c4722920346696e64206f7574206d6f72652061626f7574204e' '657874636c6f75643a2068747470733a2f2f6e657874636c6f75642e636f6d2f0a0a596f752063616e206' '6696e64206d65206f6e20746865736520736f6369616c733a0a202a20466f72756d733a2068747470733a' '2f2f666f72756d2e6865617679656c656d656e742e696f2f0a202a20506f64636173743a2068747470733' 'a2f2f6f6666746f706963616c2e6e65740a202a2050617472656f6e3a2068747470733a2f2f7061747265' '6f6e2e636f6d2f7468656c696e757867616d65720a202a204d657263683a2068747470733a2f2f7465657' '37072696e672e636f6d2f73746f7265732f6f6666696369616c2d6c696e75782d67616d65720a202a2054' '77697463683a2068747470733a2f2f7477697463682e74762f786f6e64616b0a202a20547769747465723' 'a2068747470733a2f2f747769747465722e636f6d2f7468656c696e757867616d65720a0a2e2e2e0a6874' '7470733a2f2f7777772e796f75747562652e636f6d2f77617463683f763d4672546442434f535f66632a0' 'f546865204c696e75782047616d6572321c436f7079726967687465642028636f6e746163742061757468' '6f722938004a2968747470733a2f2f6265726b2e6e696e6a612f7468756d626e61696c732f46725464424' '34f535f666352005a001a41080110011a30040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc22' 'f0bff70c4fe0b91fd36da9a375e3e1c171db825bf5d1f32209766964656f2f6d70342a5c080110031a406' '2b2dd4c45e364030fbfad1a6fefff695ebf20ea33a5381b947753e2a0ca359989a5cc7d15e5392a0d354c' '0b68498382b2701b22c03beb8dcb91089031b871e72214feb61536c007cdf4faeeaab4876cb397feaf6b51' )) claim.ClearField("publisherSignature") digest = sha256(b''.join([ decoded_address, claim.SerializeToString(), output.claim_hash[::-1] ])) signature = output.private_key.sign_compact(digest) claim.publisherSignature.version = 1 claim.publisherSignature.signatureType = 1 claim.publisherSignature.signature = signature claim.publisherSignature.certificateId = output.claim_hash[::-1] return claim ================================================ FILE: tests/integration/transactions/__init__.py ================================================ ================================================ FILE: tests/integration/transactions/test_internal_transaction_api.py ================================================ import asyncio from lbry.testcase import IntegrationTestCase import lbry.wallet from lbry.schema.claim import Claim from lbry.wallet.transaction import Transaction, Output, Input from lbry.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d class BasicTransactionTest(IntegrationTestCase): LEDGER = lbry.wallet async def test_creating_updating_and_abandoning_claim_with_channel(self): await self.account.ensure_address_gap() address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True) notifications = asyncio.create_task(asyncio.wait( [asyncio.ensure_future(self.on_address_update(address1)), asyncio.ensure_future(self.on_address_update(address2))] )) await self.send_to_address_and_wait(address1, 5) await self.send_to_address_and_wait(address2, 5, 1) await notifications self.assertEqual(d2l(await self.account.get_balance()), '10.0') channel = Claim() channel_txo = Output.pay_claim_name_pubkey_hash( l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1) ) channel_txo.set_channel_private_key( await self.account.generate_channel_private_key() ) channel_txo.script.generate() channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account) stream = Claim() stream.stream.source.media_type = "video/mp4" stream_txo = Output.pay_claim_name_pubkey_hash( l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1) ) stream_tx = await Transaction.create([], [stream_txo], [self.account], self.account) stream_txo.sign(channel_txo) await stream_tx.sign([self.account]) notifications = asyncio.create_task(asyncio.wait( [asyncio.ensure_future(self.ledger.wait(channel_tx)), asyncio.ensure_future(self.ledger.wait(stream_tx))] )) await self.broadcast(channel_tx) await self.broadcast(stream_tx) await notifications notifications = asyncio.create_task(asyncio.wait( [asyncio.ensure_future(self.ledger.wait(channel_tx)), asyncio.ensure_future(self.ledger.wait(stream_tx))] )) await self.generate(1) await notifications self.assertEqual(d2l(await self.account.get_balance()), '7.985786') self.assertEqual(d2l(await self.account.get_balance(include_claims=True)), '9.985786') response = await self.ledger.resolve([], ['lbry://@bar/foo']) self.assertEqual(response['lbry://@bar/foo'].claim.claim_type, 'stream') abandon_tx = await Transaction.create([Input.spend(stream_tx.outputs[0])], [], [self.account], self.account) notify = asyncio.create_task(self.ledger.wait(abandon_tx)) await self.broadcast(abandon_tx) await notify notify = asyncio.create_task(self.ledger.wait(abandon_tx)) await self.generate(1) await notify response = await self.ledger.resolve([], ['lbry://@bar/foo']) self.assertIn('error', response['lbry://@bar/foo']) # checks for expected format in inexistent URIs response = await self.ledger.resolve([], ['lbry://404', 'lbry://@404', 'lbry://@404/404']) self.assertEqual('Could not find claim at "lbry://404".', response['lbry://404']['error']['text']) self.assertEqual('Could not find channel in "lbry://@404".', response['lbry://@404']['error']['text']) self.assertEqual('Could not find channel in "lbry://@404/404".', response['lbry://@404/404']['error']['text']) ================================================ FILE: tests/integration/transactions/test_transaction_commands.py ================================================ import asyncio import unittest from lbry.testcase import CommandTestCase from lbry.wallet import Transaction class TransactionCommandsTestCase(CommandTestCase): async def test_txo_dust_prevention(self): address = await self.daemon.jsonrpc_address_unused(self.account.id) tx = await self.account_send('9.9997758', address) # dust prevention threshold not reached, small txo created self.assertEqual(2, len(tx['outputs'])) self.assertEqual(tx['outputs'][1]['amount'], '0.0001002') tx = await self.account_send('9.999706', address) # dust prevention prevented dust self.assertEqual(1, len(tx['outputs'])) self.assertEqual(tx['outputs'][0]['amount'], '9.999706') async def test_transaction_show(self): # local tx result = await self.out(self.daemon.jsonrpc_account_send( '5.0', await self.daemon.jsonrpc_address_unused(self.account.id), blocking=True )) await self.confirm_tx(result['txid']) tx = await self.daemon.jsonrpc_transaction_show(result['txid']) self.assertEqual(tx.id, result['txid']) # someone's tx change_address = await self.blockchain.get_raw_change_address() sendtxid = await self.blockchain.send_to_address(change_address, 10) # After a few tries, Hub should have the transaction (in mempool). for i in range(5): tx = await self.daemon.jsonrpc_transaction_show(sendtxid) # Retry if Hub is not aware of the transaction. if isinstance(tx, dict): # Fields: 'success', 'code', 'message' self.assertFalse(tx['success'], tx) self.assertEqual(tx['code'], 404, tx) self.assertEqual(tx['message'], "transaction not found", tx) await asyncio.sleep(0.1) continue break # verify transaction show (in mempool) self.assertTrue(isinstance(tx, Transaction), str(tx)) # Fields: 'txid', 'raw', 'height', 'position', 'is_verified', and more. self.assertEqual(tx.id, sendtxid, vars(tx)) self.assertEqual(tx.height, -1, vars(tx)) self.assertEqual(tx.is_verified, False, vars(tx)) # transaction is confirmed and leaves mempool await self.generate(1) # verify transaction show tx = await self.daemon.jsonrpc_transaction_show(sendtxid) self.assertTrue(isinstance(tx, Transaction), str(tx)) self.assertEqual(tx.id, sendtxid, vars(tx)) self.assertEqual(tx.height, self.ledger.headers.height, vars(tx)) self.assertEqual(tx.is_verified, True, vars(tx)) # inexistent result = await self.daemon.jsonrpc_transaction_show('0'*64) self.assertTrue(isinstance(result, dict), result) # Fields: 'success', 'code', 'message' self.assertFalse(result['success'], result) self.assertEqual(result['code'], 404, result) self.assertEqual(result['message'], "transaction not found", result) async def test_utxo_release(self): await self.send_to_address_and_wait( await self.account.receiving.get_or_create_usable_address(), 1, 1 ) await self.assertBalance(self.account, '11.0') await self.ledger.reserve_outputs(await self.account.get_utxos()) await self.assertBalance(self.account, '0.0') await self.daemon.jsonrpc_utxo_release() await self.assertBalance(self.account, '11.0') class TestSegwit(CommandTestCase): @unittest.SkipTest async def test_segwit(self): p2sh_address1 = await self.blockchain.get_new_address(self.blockchain.P2SH_SEGWIT_ADDRESS) p2sh_address2 = await self.blockchain.get_new_address(self.blockchain.P2SH_SEGWIT_ADDRESS) p2sh_address3 = await self.blockchain.get_new_address(self.blockchain.P2SH_SEGWIT_ADDRESS) bech32_address1 = await self.blockchain.get_new_address(self.blockchain.BECH32_ADDRESS) bech32_address2 = await self.blockchain.get_new_address(self.blockchain.BECH32_ADDRESS) bech32_address3 = await self.blockchain.get_new_address(self.blockchain.BECH32_ADDRESS) # fund specific addresses for later use p2sh_txid1 = await self.blockchain.send_to_address(p2sh_address1, '1.0') p2sh_txid2 = await self.blockchain.send_to_address(p2sh_address2, '1.0') bech32_txid1 = await self.blockchain.send_to_address(bech32_address1, '1.0') bech32_txid2 = await self.blockchain.send_to_address(bech32_address2, '1.0') await self.generate(1) # P2SH & BECH32 can pay to P2SH address tx = await self.blockchain.create_raw_transaction([ {"txid": p2sh_txid1, "vout": 0}, {"txid": bech32_txid1, "vout": 0}, ], {p2sh_address3: 1.9} ) tx = await self.blockchain.sign_raw_transaction_with_wallet(tx) p2sh_txid3 = await self.blockchain.send_raw_transaction(tx) await self.generate(1) # P2SH & BECH32 can pay to BECH32 address tx = await self.blockchain.create_raw_transaction([ {"txid": p2sh_txid2, "vout": 0}, {"txid": bech32_txid2, "vout": 0}, ], {bech32_address3: 1.9} ) tx = await self.blockchain.sign_raw_transaction_with_wallet(tx) bech32_txid3 = await self.blockchain.send_raw_transaction(tx) await self.generate(1) # P2SH & BECH32 can pay lbry wallet P2PKH address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0] tx = await self.blockchain.create_raw_transaction([ {"txid": p2sh_txid3, "vout": 0}, {"txid": bech32_txid3, "vout": 0}, ], {address: 3.5} ) tx = await self.blockchain.sign_raw_transaction_with_wallet(tx) txid = await self.blockchain.send_raw_transaction(tx) await self.generate_and_wait(1, [txid]) await self.assertBalance(self.account, '13.5') ================================================ FILE: tests/integration/transactions/test_transactions.py ================================================ import asyncio import random import lbry.wallet.rpc.jsonrpc from lbry.wallet.transaction import Transaction, Output, Input from lbry.testcase import IntegrationTestCase from lbry.wallet.util import satoshis_to_coins, coins_to_satoshis from lbry.wallet.manager import WalletManager class BasicTransactionTests(IntegrationTestCase): async def test_variety_of_transactions_and_longish_history(self): await self.generate(300) await self.assertBalance(self.account, '0.0') addresses = await self.account.receiving.get_addresses() # send 10 coins to first 10 receiving addresses and then 10 transactions worth 10 coins each # to the 10th receiving address for a total of 30 UTXOs on the entire account for i in range(10): notification = asyncio.ensure_future(self.on_address_update(addresses[i])) _ = await self.send_to_address_and_wait(addresses[i], 10) await notification notification = asyncio.ensure_future(self.on_address_update(addresses[9])) _ = await self.send_to_address_and_wait(addresses[9], 10) await notification # use batching to reduce issues with send_to_address on cli await self.assertBalance(self.account, '200.0') self.assertEqual(20, await self.account.get_utxo_count()) # address gap should have increase by 10 to cover the first 10 addresses we've used up addresses = await self.account.receiving.get_addresses() self.assertEqual(30, len(addresses)) # there used to be a sync bug which failed to save TXIs between # daemon restarts, clearing cache replicates that behavior self.ledger._tx_cache.clear() # spend from each of the first 10 addresses to the subsequent 10 addresses txs = [] for address in addresses[10:20]: txs.append(await Transaction.create( [], [Output.pay_pubkey_hash( coins_to_satoshis('1.0'), self.ledger.address_to_hash160(address) )], [self.account], self.account )) await asyncio.wait([self.broadcast(tx) for tx in txs]) await asyncio.wait([self.ledger.wait(tx) for tx in txs]) # verify that a previous bug which failed to save TXIs doesn't come back # this check must happen before generating a new block self.assertTrue(all([ tx.inputs[0].txo_ref.txo is not None for tx in await self.ledger.db.get_transactions(txid__in=[tx.id for tx in txs]) ])) await self.generate(1) await asyncio.wait([self.ledger.wait(tx) for tx in txs]) await self.assertBalance(self.account, '199.99876') # 10 of the UTXOs have been split into a 1 coin UTXO and a 9 UTXO change self.assertEqual(30, await self.account.get_utxo_count()) # spend all 30 UTXOs into a a 199 coin UTXO and change tx = await Transaction.create( [], [Output.pay_pubkey_hash( coins_to_satoshis('199.0'), self.ledger.address_to_hash160(addresses[-1]) )], [self.account], self.account ) await self.broadcast(tx) await self.ledger.wait(tx) await self.generate(1) await self.ledger.wait(tx) self.assertEqual(2, await self.account.get_utxo_count()) # 199 + change await self.assertBalance(self.account, '199.99649') async def test_sending_and_receiving(self): account1, account2 = self.account, self.wallet.generate_account(self.ledger) await self.ledger.subscribe_account(account2) await self.assertBalance(account1, '0.0') await self.assertBalance(account2, '0.0') addresses = await account1.receiving.get_addresses() txids = [] for address in addresses[:5]: txids.append(await self.send_to_address_and_wait(address, 1.1)) await self.generate_and_wait(1, txids) await self.assertBalance(account1, '5.5') await self.assertBalance(account2, '0.0') address2 = await account2.receiving.get_or_create_usable_address() tx = await Transaction.create( [], [Output.pay_pubkey_hash( coins_to_satoshis('2.0'), self.ledger.address_to_hash160(address2) )], [account1], account1 ) await self.broadcast(tx) await self.ledger.wait(tx) # mempool await self.generate(1) await self.ledger.wait(tx) # confirmed await self.assertBalance(account1, '3.499802') await self.assertBalance(account2, '2.0') utxos = await self.account.get_utxos() tx = await Transaction.create( [Input.spend(utxos[0])], [], [account1], account1 ) await self.broadcast(tx) await self.ledger.wait(tx) # mempool await self.generate(1) await self.ledger.wait(tx) # confirmed tx = (await account1.get_transactions(include_is_my_input=True, include_is_my_output=True))[1] self.assertEqual(satoshis_to_coins(tx.inputs[0].amount), '1.1') self.assertEqual(satoshis_to_coins(tx.inputs[1].amount), '1.1') self.assertEqual(satoshis_to_coins(tx.outputs[0].amount), '2.0') self.assertEqual(tx.outputs[0].get_address(self.ledger), address2) self.assertTrue(tx.outputs[0].is_internal_transfer) self.assertTrue(tx.outputs[1].is_internal_transfer) async def test_history_edge_cases(self): await self.generate(300) await self.assertBalance(self.account, '0.0') address = await self.account.receiving.get_or_create_usable_address() # evil trick: mempool is unsorted on real life, but same order between python instances. reproduce it original_summary = self.conductor.spv_node.server.mempool.transaction_summaries def random_summary(*args, **kwargs): summary = original_summary(*args, **kwargs) if summary and len(summary) > 2: ordered = summary.copy() while summary == ordered: random.shuffle(summary) return summary self.conductor.spv_node.server.mempool.transaction_summaries = random_summary # 10 unconfirmed txs, all from blockchain wallet for i in range(10): await self.send_to_address_and_wait(address, 10) remote_status = await self.ledger.network.subscribe_address(address) self.assertTrue(await self.ledger.update_history(address, remote_status)) # 20 unconfirmed txs, 10 from blockchain, 10 from local to local utxos = await self.account.get_utxos() txs = [] for utxo in utxos: tx = await Transaction.create( [Input.spend(utxo)], [], [self.account], self.account ) await self.broadcast(tx) txs.append(tx) await asyncio.wait([self.on_transaction_address(tx, address) for tx in txs], timeout=1) remote_status = await self.ledger.network.subscribe_address(address) self.assertTrue(await self.ledger.update_history(address, remote_status)) # server history grows unordered await self.send_to_address_and_wait(address, 1) self.assertTrue(await self.ledger.update_history(address, remote_status)) self.assertEqual(21, len((await self.ledger.get_local_status_and_history(address))[1])) self.assertEqual(0, len(self.ledger._known_addresses_out_of_sync)) async def _test_transaction(self, send_amount, address, inputs, change): tx = await Transaction.create( [], [Output.pay_pubkey_hash(send_amount, self.ledger.address_to_hash160(address))], [self.account], self.account ) await self.ledger.broadcast(tx) input_amounts = [txi.amount for txi in tx.inputs] self.assertListEqual(inputs, input_amounts) self.assertEqual(len(inputs), len(tx.inputs)) self.assertEqual(2, len(tx.outputs)) self.assertEqual(send_amount, tx.outputs[0].amount) self.assertEqual(change, tx.outputs[1].amount) return tx async def assertSpendable(self, amounts): spendable = await self.ledger.db.get_spendable_utxos( self.ledger, 2000000000000, [self.account], set_reserved=False, return_insufficient_funds=True ) got_amounts = [estimator.effective_amount for estimator in spendable] self.assertListEqual(sorted(amounts), sorted(got_amounts)) async def test_sqlite_coin_chooser(self): wallet_manager = WalletManager([self.wallet], {self.ledger.get_id(): self.ledger}) await self.generate(300) await self.assertBalance(self.account, '0.0') address = await self.account.receiving.get_or_create_usable_address() other_account = self.wallet.generate_account(self.ledger) other_address = await other_account.receiving.get_or_create_usable_address() self.ledger.coin_selection_strategy = 'sqlite' await self.ledger.subscribe_account(other_account) accepted = asyncio.ensure_future(self.on_address_update(address)) _ = await self.send_to_address_and_wait(address, 1.0) await accepted accepted = asyncio.ensure_future(self.on_address_update(address)) _ = await self.send_to_address_and_wait(address, 1.0) await accepted accepted = asyncio.ensure_future(self.on_address_update(address)) _ = await self.send_to_address_and_wait(address, 3.0) await accepted accepted = asyncio.ensure_future(self.on_address_update(address)) _ = await self.send_to_address_and_wait(address, 5.0) await accepted accepted = asyncio.ensure_future(self.on_address_update(address)) _ = await self.send_to_address_and_wait(address, 10.0) await accepted await self.assertBalance(self.account, '20.0') await self.assertSpendable([99992600, 99992600, 299992600, 499992600, 999992600]) # send 1.5 lbc first_tx = await Transaction.create( [], [Output.pay_pubkey_hash(150000000, self.ledger.address_to_hash160(other_address))], [self.account], self.account ) self.assertEqual(2, len(first_tx.inputs)) self.assertEqual(2, len(first_tx.outputs)) self.assertEqual(100000000, first_tx.inputs[0].amount) self.assertEqual(100000000, first_tx.inputs[1].amount) self.assertEqual(150000000, first_tx.outputs[0].amount) self.assertEqual(49980200, first_tx.outputs[1].amount) await self.assertBalance(self.account, '18.0') await self.assertSpendable([299992600, 499992600, 999992600]) await wallet_manager.broadcast_or_release(first_tx, blocking=True) await self.assertSpendable([49972800, 299992600, 499992600, 999992600]) # 0.499, 3.0, 5.0, 10.0 await self.assertBalance(self.account, '18.499802') # send 1.5lbc again second_tx = await self._test_transaction(150000000, other_address, [49980200, 300000000], 199960400) await self.assertSpendable([499992600, 999992600]) # replicate cancelling the api call after the tx broadcast while ledger.wait'ing it e = asyncio.Event() real_broadcast = self.ledger.broadcast async def broadcast(tx): try: return await real_broadcast(tx) except lbry.wallet.rpc.jsonrpc.RPCError as err: # this is expected in tests where we try to double spend. if 'the transaction was rejected by network rules.' in str(err): pass else: raise err finally: e.set() self.ledger.broadcast = broadcast broadcast_task = asyncio.create_task(wallet_manager.broadcast_or_release(second_tx, blocking=True)) # wait for the broadcast to finish await e.wait() # cancel the api call broadcast_task.cancel() with self.assertRaises(asyncio.CancelledError): await broadcast_task # test if sending another 1.5 lbc will try to double spend the inputs from the cancelled tx tx1 = await self._test_transaction(150000000, other_address, [500000000], 349987600) await self.ledger.wait(tx1, timeout=1) # wait for the cancelled transaction too, so that it's in the database # needed to keep everything deterministic await self.ledger.wait(second_tx, timeout=1) await self.assertSpendable([199953000, 349980200, 999992600]) # spend deep into the mempool and see what else breaks tx2 = await self._test_transaction(150000000, other_address, [199960400], 49948000) await self.assertSpendable([349980200, 999992600]) await self.ledger.wait(tx2, timeout=1) await self.assertSpendable([49940600, 349980200, 999992600]) tx3 = await self._test_transaction(150000000, other_address, [49948000, 349987600], 249915800) await self.assertSpendable([999992600]) await self.ledger.wait(tx3, timeout=1) await self.assertSpendable([249908400, 999992600]) tx4 = await self._test_transaction(150000000, other_address, [249915800], 99903400) await self.assertSpendable([999992600]) await self.ledger.wait(tx4, timeout=1) await self.assertBalance(self.account, '10.999034') await self.assertSpendable([99896000, 999992600]) # spend more tx5 = await self._test_transaction(100000000, other_address, [99903400, 1000000000], 999883600) await self.assertSpendable([]) await self.ledger.wait(tx5, timeout=1) await self.assertSpendable([999876200]) await self.assertBalance(self.account, '9.998836') ================================================ FILE: tests/test_utils.py ================================================ import datetime import time import os import tempfile import shutil from unittest import mock from binascii import hexlify DEFAULT_TIMESTAMP = datetime.datetime(2016, 1, 1) DEFAULT_ISO_TIME = time.mktime(DEFAULT_TIMESTAMP.timetuple()) def mk_db_and_blob_dir(): db_dir = tempfile.mkdtemp() blob_dir = tempfile.mkdtemp() return db_dir, blob_dir def rm_db_and_blob_dir(db_dir, blob_dir): shutil.rmtree(db_dir, ignore_errors=True) shutil.rmtree(blob_dir, ignore_errors=True) def random_lbry_hash(): return hexlify(os.urandom(48)).decode() def reset_time(test_case, timestamp=DEFAULT_TIMESTAMP): iso_time = time.mktime(timestamp.timetuple()) patcher = mock.patch('time.time') patcher.start().return_value = iso_time test_case.addCleanup(patcher.stop) patcher = mock.patch('lbry.utils.now') patcher.start().return_value = timestamp test_case.addCleanup(patcher.stop) patcher = mock.patch('lbry.utils.utcnow') patcher.start().return_value = timestamp test_case.addCleanup(patcher.stop) def is_android(): return 'ANDROID_ARGUMENT' in os.environ # detect Android using the Kivy way ================================================ FILE: tests/unit/__init__.py ================================================ ================================================ FILE: tests/unit/analytics/__init__.py ================================================ ================================================ FILE: tests/unit/analytics/test_track.py ================================================ import lbry.wallet from lbry.extras.daemon import analytics import unittest @unittest.SkipTest class TrackTest(unittest.TestCase): def test_empty_summarize_is_none(self): track = analytics.Manager(None, 'x', 'y', 'z') _, result = track.summarize_and_reset('a') self.assertIsNone(result) def test_can_get_sum_of_metric(self): track = analytics.Manager(None, 'x', 'y', 'z') track.add_observation('b', 1) track.add_observation('b', 2) _, result = track.summarize_and_reset('b') self.assertEqual(3, result) def test_summarize_resets_metric(self): track = analytics.Manager(None, 'x', 'y', 'z') track.add_observation('metric', 1) track.add_observation('metric', 2) track.summarize_and_reset('metric') _, result = track.summarize_and_reset('metric') self.assertIsNone(result) ================================================ FILE: tests/unit/blob/__init__.py ================================================ ================================================ FILE: tests/unit/blob/test_blob_file.py ================================================ import asyncio import tempfile import shutil import os from lbry.testcase import AsyncioTestCase from lbry.error import InvalidDataError, InvalidBlobHashError from lbry.conf import Config from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_file import BlobFile, BlobBuffer, AbstractBlob class TestBlob(AsyncioTestCase): blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" blob_bytes = b'1' * ((2 * 2 ** 20) - 1) async def asyncSetUp(self): self.tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(self.tmp_dir)) self.loop = asyncio.get_running_loop() self.config = Config() self.storage = SQLiteStorage(self.config, ":memory:", self.loop) self.blob_manager = BlobManager(self.loop, self.tmp_dir, self.storage, self.config) await self.storage.open() def _get_blob(self, blob_class=AbstractBlob, blob_directory=None): blob = blob_class(self.loop, self.blob_hash, len(self.blob_bytes), self.blob_manager.blob_completed, blob_directory=blob_directory) self.assertFalse(blob.get_is_verified()) self.addCleanup(blob.close) return blob async def _test_create_blob(self, blob_class=AbstractBlob, blob_directory=None): blob = self._get_blob(blob_class, blob_directory) writer = blob.get_blob_writer() writer.write(self.blob_bytes) await blob.verified.wait() self.assertTrue(blob.get_is_verified()) await asyncio.sleep(0) # wait for the db save task return blob async def _test_close_writers_on_finished(self, blob_class=AbstractBlob, blob_directory=None): blob = self._get_blob(blob_class, blob_directory=blob_directory) writers = [blob.get_blob_writer('1.2.3.4', port) for port in range(5)] self.assertEqual(5, len(blob.writers)) # test that writing too much causes the writer to fail with InvalidDataError and to be removed with self.assertRaises(InvalidDataError): writers[1].write(self.blob_bytes * 2) await writers[1].finished await asyncio.sleep(0) self.assertEqual(4, len(blob.writers)) # write the blob other = writers[2] writers[3].write(self.blob_bytes) await blob.verified.wait() self.assertTrue(blob.get_is_verified()) self.assertEqual(0, len(blob.writers)) with self.assertRaises(IOError): other.write(self.blob_bytes) def _test_ioerror_if_length_not_set(self, blob_class=AbstractBlob, blob_directory=None): blob = blob_class( self.loop, self.blob_hash, blob_completed_callback=self.blob_manager.blob_completed, blob_directory=blob_directory ) self.addCleanup(blob.close) writer = blob.get_blob_writer() with self.assertRaises(IOError): writer.write(b'') async def _test_invalid_blob_bytes(self, blob_class=AbstractBlob, blob_directory=None): blob = blob_class( self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed, blob_directory=blob_directory ) self.addCleanup(blob.close) writer = blob.get_blob_writer() writer.write(self.blob_bytes[:-4] + b'fake') with self.assertRaises(InvalidBlobHashError): await writer.finished async def test_add_blob_buffer_to_db(self): blob = await self._test_create_blob(BlobBuffer) db_status = await self.storage.get_blob_status(blob.blob_hash) self.assertEqual(db_status, 'pending') async def test_add_blob_file_to_db(self): blob = await self._test_create_blob(BlobFile, self.tmp_dir) db_status = await self.storage.get_blob_status(blob.blob_hash) self.assertEqual(db_status, 'finished') async def test_invalid_blob_bytes(self): await self._test_invalid_blob_bytes(BlobBuffer) await self._test_invalid_blob_bytes(BlobFile, self.tmp_dir) def test_ioerror_if_length_not_set(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) self._test_ioerror_if_length_not_set(BlobBuffer) self._test_ioerror_if_length_not_set(BlobFile, tmp_dir) async def test_create_blob_file(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) blob = await self._test_create_blob(BlobFile, tmp_dir) self.assertIsInstance(blob, BlobFile) self.assertTrue(os.path.isfile(blob.file_path)) for _ in range(2): with blob.reader_context() as reader: self.assertEqual(self.blob_bytes, reader.read()) async def test_create_blob_buffer(self): blob = await self._test_create_blob(BlobBuffer) self.assertIsInstance(blob, BlobBuffer) self.assertIsNotNone(blob._verified_bytes) # check we can only read the bytes once, and that the buffer is torn down with blob.reader_context() as reader: self.assertEqual(self.blob_bytes, reader.read()) self.assertIsNone(blob._verified_bytes) with self.assertRaises(OSError): with blob.reader_context() as reader: self.assertEqual(self.blob_bytes, reader.read()) self.assertIsNone(blob._verified_bytes) async def test_close_writers_on_finished(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) await self._test_close_writers_on_finished(BlobBuffer) await self._test_close_writers_on_finished(BlobFile, tmp_dir) async def test_concurrency_and_premature_closes(self): blob_directory = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(blob_directory)) blob = self._get_blob(BlobBuffer, blob_directory=blob_directory) writer = blob.get_blob_writer('1.1.1.1', 1337) self.assertEqual(1, len(blob.writers)) with self.assertRaises(OSError): blob.get_blob_writer('1.1.1.1', 1337) writer.close_handle() self.assertTrue(blob.writers[('1.1.1.1', 1337)].closed()) writer = blob.get_blob_writer('1.1.1.1', 1337) self.assertEqual(blob.writers[('1.1.1.1', 1337)], writer) writer.close_handle() await asyncio.sleep(0.000000001) # flush callbacks self.assertEqual(0, len(blob.writers)) async def test_delete(self): blob_buffer = await self._test_create_blob(BlobBuffer) self.assertIsInstance(blob_buffer, BlobBuffer) self.assertIsNotNone(blob_buffer._verified_bytes) self.assertTrue(blob_buffer.get_is_verified()) blob_buffer.delete() self.assertIsNone(blob_buffer._verified_bytes) self.assertFalse(blob_buffer.get_is_verified()) tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) blob_file = await self._test_create_blob(BlobFile, tmp_dir) self.assertIsInstance(blob_file, BlobFile) self.assertTrue(os.path.isfile(blob_file.file_path)) self.assertTrue(blob_file.get_is_verified()) blob_file.delete() self.assertFalse(os.path.isfile(blob_file.file_path)) self.assertFalse(blob_file.get_is_verified()) async def test_delete_corrupt(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) blob = BlobFile( self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed, blob_directory=tmp_dir ) writer = blob.get_blob_writer() writer.write(self.blob_bytes) await blob.verified.wait() blob.close() blob = BlobFile( self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed, blob_directory=tmp_dir ) self.assertTrue(blob.get_is_verified()) with open(blob.file_path, 'wb+') as f: f.write(b'\x00') blob = BlobFile( self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed, blob_directory=tmp_dir ) self.assertFalse(blob.get_is_verified()) self.assertFalse(os.path.isfile(blob.file_path)) def test_invalid_blob_hash(self): self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, '', len(self.blob_bytes)) self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'x' * 96, len(self.blob_bytes)) self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'a' * 97, len(self.blob_bytes)) async def _test_close_reader(self, blob_class=AbstractBlob, blob_directory=None): blob = await self._test_create_blob(blob_class, blob_directory) reader = blob.reader_context() self.assertEqual(0, len(blob.readers)) async def read_blob_buffer(): with reader as read_handle: self.assertEqual(1, len(blob.readers)) await asyncio.sleep(2) self.assertEqual(0, len(blob.readers)) return read_handle.read() self.loop.call_later(1, blob.close) with self.assertRaises(ValueError) as err: read_task = self.loop.create_task(read_blob_buffer()) await read_task self.assertEqual(err.exception, ValueError("I/O operation on closed file")) async def test_close_reader(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) await self._test_close_reader(BlobBuffer) await self._test_close_reader(BlobFile, tmp_dir) ================================================ FILE: tests/unit/blob/test_blob_manager.py ================================================ import tempfile import shutil import os from lbry.testcase import AsyncioTestCase from lbry.conf import Config from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_manager import BlobManager class TestBlobManager(AsyncioTestCase): async def setup_blob_manager(self, save_blobs=True): tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) self.config = Config(save_blobs=save_blobs) self.storage = SQLiteStorage(self.config, os.path.join(tmp_dir, "lbrynet.sqlite")) self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config) await self.storage.open() async def test_memory_blobs_arent_verified_but_real_ones_are(self): for save_blobs in (False, True): await self.setup_blob_manager(save_blobs=save_blobs) # add a blob file blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" blob_bytes = b'1' * ((2 * 2 ** 20) - 1) blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes)) blob.save_verified_blob(blob_bytes) await blob.verified.wait() self.assertTrue(blob.get_is_verified()) self.blob_manager.blob_completed(blob) self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs) async def test_sync_blob_file_manager_on_startup(self): await self.setup_blob_manager(save_blobs=True) # add a blob file blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" blob_bytes = b'1' * ((2 * 2 ** 20) - 1) with open(os.path.join(self.blob_manager.blob_dir, blob_hash), 'wb') as f: f.write(blob_bytes) # it should not have been added automatically on startup await self.blob_manager.setup() self.assertSetEqual(self.blob_manager.completed_blob_hashes, set()) # make sure we can add the blob await self.blob_manager.blob_completed(self.blob_manager.get_blob(blob_hash, len(blob_bytes))) self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash}) # stop the blob manager and restart it, make sure the blob is there self.blob_manager.stop() self.assertSetEqual(self.blob_manager.completed_blob_hashes, set()) await self.blob_manager.setup() self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash}) # test that the blob is removed upon the next startup after the file being manually deleted self.blob_manager.stop() # manually delete the blob file and restart the blob manager os.remove(os.path.join(self.blob_manager.blob_dir, blob_hash)) await self.blob_manager.setup() self.assertSetEqual(self.blob_manager.completed_blob_hashes, set()) # check that the deleted blob was updated in the database self.assertEqual( 'pending', ( await self.storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash) ) ) ================================================ FILE: tests/unit/blob_exchange/__init__.py ================================================ ================================================ FILE: tests/unit/blob_exchange/test_transfer_blob.py ================================================ import asyncio import tempfile from io import BytesIO from unittest import mock import shutil import os import copy from lbry.blob_exchange.serialization import BlobRequest from lbry.testcase import AsyncioTestCase from lbry.conf import Config from lbry.extras.daemon.storage import SQLiteStorage from lbry.extras.daemon.daemon import Daemon from lbry.blob.blob_manager import BlobManager from lbry.blob_exchange.server import BlobServer, BlobServerProtocol from lbry.blob_exchange.client import request_blob from lbry.dht.peer import PeerManager, make_kademlia_peer from lbry.dht.node import Node # import logging # logging.getLogger("lbry").setLevel(logging.DEBUG) def mock_config(): config = Config(save_files=True) config.fixed_peer_delay = 10000 return config class BlobExchangeTestBase(AsyncioTestCase): async def asyncSetUp(self): self.loop = asyncio.get_event_loop() self.client_wallet_dir = tempfile.mkdtemp() self.client_dir = tempfile.mkdtemp() self.server_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.client_wallet_dir) self.addCleanup(shutil.rmtree, self.client_dir) self.addCleanup(shutil.rmtree, self.server_dir) self.server_config = Config( data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir, save_files=True, fixed_peers=[] ) self.server_config.transaction_cache_size = 10000 self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite")) self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config) self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP') self.client_config = Config( data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_wallet_dir, save_files=True, fixed_peers=[], tracker_servers=[] ) self.client_config.transaction_cache_size = 10000 self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite")) self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config) self.client_peer_manager = PeerManager(self.loop) self.server_from_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333, allow_localhost=True) await self.client_storage.open() await self.server_storage.open() await self.client_blob_manager.setup() await self.server_blob_manager.setup() self.server.start_server(33333, '127.0.0.1') self.addCleanup(self.server.stop_server) await self.server.started_listening.wait() class TestBlobExchange(BlobExchangeTestBase): async def _add_blob_to_server(self, blob_hash: str, blob_bytes: bytes): # add the blob on the server server_blob = self.server_blob_manager.get_blob(blob_hash, len(blob_bytes)) writer = server_blob.get_blob_writer() writer.write(blob_bytes) await server_blob.verified.wait() self.assertTrue(os.path.isfile(server_blob.file_path)) self.assertTrue(server_blob.get_is_verified()) self.assertTrue(writer.closed()) async def _test_transfer_blob(self, blob_hash: str): client_blob = self.client_blob_manager.get_blob(blob_hash) # download the blob downloaded, transport = await request_blob(self.loop, client_blob, self.server_from_client.address, self.server_from_client.tcp_port, 2, 3) self.assertIsNotNone(transport) self.addCleanup(transport.close) await client_blob.verified.wait() self.assertTrue(client_blob.get_is_verified()) self.assertTrue(downloaded) client_blob.close() async def test_transfer_sd_blob(self): sd_hash = "3e2706157a59aaa47ef52bc264fce488078b4026c0b9bab649a8f2fe1ecc5e5cad7182a2bb7722460f856831a1ac0f02" mock_sd_blob_bytes = b"""{"blobs": [{"blob_hash": "6f53c72de100f6f007aa1b9720632e2d049cc6049e609ad790b556dba262159f739d5a14648d5701afc84b991254206a", "blob_num": 0, "iv": "3b6110c2d8e742bff66e4314863dee7e", "length": 2097152}, {"blob_hash": "18493bc7c5164b00596153859a0faffa45765e47a6c3f12198a4f7be4658111505b7f8a15ed0162306a0672c4a9b505d", "blob_num": 1, "iv": "df973fa64e73b4ff2677d682cdc32d3e", "length": 2097152}, {"blob_num": 2, "iv": "660d2dc2645da7c7d4540a466fcb0c60", "length": 0}], "key": "6465616462656566646561646265656664656164626565666465616462656566", "stream_hash": "22423c6786584974bd6b462af47ecb03e471da0ef372fe85a4e71a78bef7560c4afb0835c689f03916105404653b7bdf", "stream_name": "746573745f66696c65", "stream_type": "lbryfile", "suggested_file_name": "746573745f66696c65"}""" await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes) return await self._test_transfer_blob(sd_hash) async def test_transfer_blob(self): blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1) await self._add_blob_to_server(blob_hash, mock_blob_bytes) return await self._test_transfer_blob(blob_hash) async def test_host_same_blob_to_multiple_peers_at_once(self): blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1) second_client_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, second_client_dir) second_client_conf = Config(save_files=True) second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite")) second_client_blob_manager = BlobManager( self.loop, second_client_dir, second_client_storage, second_client_conf ) server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333, allow_localhost=True) await second_client_storage.open() await second_client_blob_manager.setup() await self._add_blob_to_server(blob_hash, mock_blob_bytes) second_client_blob = second_client_blob_manager.get_blob(blob_hash) # download the blob await asyncio.gather( request_blob( self.loop, second_client_blob, server_from_second_client.address, server_from_second_client.tcp_port, 2, 3 ), self._test_transfer_blob(blob_hash) ) await second_client_blob.verified.wait() self.assertTrue(second_client_blob.get_is_verified()) async def test_blob_writers_concurrency(self): blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1) blob = self.server_blob_manager.get_blob(blob_hash) write_blob = blob._write_blob write_called_count = 0 async def _wrap_write_blob(blob_bytes): nonlocal write_called_count write_called_count += 1 await write_blob(blob_bytes) def wrap_write_blob(blob_bytes): return asyncio.create_task(_wrap_write_blob(blob_bytes)) blob._write_blob = wrap_write_blob writer1 = blob.get_blob_writer(peer_port=1) writer2 = blob.get_blob_writer(peer_port=2) reader1_ctx_before_write = blob.reader_context() with self.assertRaises(OSError): blob.get_blob_writer(peer_port=2) with self.assertRaises(OSError): with blob.reader_context(): pass blob.set_length(len(mock_blob_bytes)) results = {} def check_finished_callback(writer, num): def inner(writer_future: asyncio.Future): results[num] = writer_future.result() writer.finished.add_done_callback(inner) check_finished_callback(writer1, 1) check_finished_callback(writer2, 2) def write_task(writer): async def _inner(): writer.write(mock_blob_bytes) return self.loop.create_task(_inner()) await asyncio.gather(write_task(writer1), write_task(writer2)) self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results) self.assertEqual(1, write_called_count) await blob.verified.wait() self.assertTrue(blob.get_is_verified()) self.assertDictEqual({}, blob.writers) with reader1_ctx_before_write as f: self.assertEqual(mock_blob_bytes, f.read()) with blob.reader_context() as f: self.assertEqual(mock_blob_bytes, f.read()) with blob.reader_context() as f: blob.close() with self.assertRaises(ValueError): f.read() self.assertListEqual([], blob.readers) async def test_host_different_blobs_to_multiple_peers_at_once(self): blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1) sd_hash = "3e2706157a59aaa47ef52bc264fce488078b4026c0b9bab649a8f2fe1ecc5e5cad7182a2bb7722460f856831a1ac0f02" mock_sd_blob_bytes = b"""{"blobs": [{"blob_hash": "6f53c72de100f6f007aa1b9720632e2d049cc6049e609ad790b556dba262159f739d5a14648d5701afc84b991254206a", "blob_num": 0, "iv": "3b6110c2d8e742bff66e4314863dee7e", "length": 2097152}, {"blob_hash": "18493bc7c5164b00596153859a0faffa45765e47a6c3f12198a4f7be4658111505b7f8a15ed0162306a0672c4a9b505d", "blob_num": 1, "iv": "df973fa64e73b4ff2677d682cdc32d3e", "length": 2097152}, {"blob_num": 2, "iv": "660d2dc2645da7c7d4540a466fcb0c60", "length": 0}], "key": "6465616462656566646561646265656664656164626565666465616462656566", "stream_hash": "22423c6786584974bd6b462af47ecb03e471da0ef372fe85a4e71a78bef7560c4afb0835c689f03916105404653b7bdf", "stream_name": "746573745f66696c65", "stream_type": "lbryfile", "suggested_file_name": "746573745f66696c65"}""" second_client_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, second_client_dir) second_client_conf = Config(save_files=True) second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite")) second_client_blob_manager = BlobManager( self.loop, second_client_dir, second_client_storage, second_client_conf ) server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333, allow_localhost=True) await second_client_storage.open() await second_client_blob_manager.setup() await self._add_blob_to_server(blob_hash, mock_blob_bytes) await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes) second_client_blob = self.client_blob_manager.get_blob(blob_hash) await asyncio.gather( request_blob( self.loop, second_client_blob, server_from_second_client.address, server_from_second_client.tcp_port, 2, 3 ), self._test_transfer_blob(sd_hash), second_client_blob.verified.wait() ) self.assertTrue(second_client_blob.get_is_verified()) async def test_server_chunked_request(self): blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" server_protocol = BlobServerProtocol(self.loop, self.server_blob_manager, self.server.lbrycrd_address) transport = mock.Mock(spec=asyncio.Transport) transport.get_extra_info = lambda k: {'peername': ('ip', 90)}[k] received_data = BytesIO() transport.is_closing = lambda: received_data.closed transport.write = received_data.write server_protocol.connection_made(transport) blob_request = BlobRequest.make_request_for_blob_hash(blob_hash).serialize() for byte in blob_request: server_protocol.data_received(bytes([byte])) await asyncio.sleep(0.1) # yield execution self.assertGreater(len(received_data.getvalue()), 0) async def test_idle_timeout(self): self.server.idle_timeout = 1 blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1) await self._add_blob_to_server(blob_hash, mock_blob_bytes) client_blob = self.client_blob_manager.get_blob(blob_hash) # download the blob downloaded, protocol = await request_blob(self.loop, client_blob, self.server_from_client.address, self.server_from_client.tcp_port, 2, 3) self.assertIsNotNone(protocol) self.assertFalse(protocol.transport.is_closing()) await client_blob.verified.wait() self.assertTrue(client_blob.get_is_verified()) self.assertTrue(downloaded) client_blob.delete() # wait for less than the idle timeout await asyncio.sleep(0.5) # download the blob again downloaded, protocol2 = await request_blob(self.loop, client_blob, self.server_from_client.address, self.server_from_client.tcp_port, 2, 3, connected_protocol=protocol) self.assertIs(protocol, protocol2) self.assertFalse(protocol.transport.is_closing()) await client_blob.verified.wait() self.assertTrue(client_blob.get_is_verified()) self.assertTrue(downloaded) client_blob.delete() # check that the connection times out from the server side await asyncio.sleep(0.9) self.assertFalse(protocol.transport.is_closing()) self.assertIsNotNone(protocol.transport._sock) await asyncio.sleep(0.1) self.assertIsNone(protocol.transport) def test_max_request_size(self): protocol = BlobServerProtocol(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP') called = asyncio.Event() protocol.close = called.set protocol.data_received(b'0' * 1199) self.assertFalse(called.is_set()) protocol.data_received(b'0') self.assertTrue(called.is_set()) def test_bad_json(self): protocol = BlobServerProtocol(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP') called = asyncio.Event() protocol.close = called.set protocol.data_received(b'{{0}') self.assertTrue(called.is_set()) def test_no_request(self): protocol = BlobServerProtocol(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP') called = asyncio.Event() protocol.close = called.set protocol.data_received(b'{}') self.assertTrue(called.is_set()) async def test_transfer_timeout(self): self.server.transfer_timeout = 1 blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1) await self._add_blob_to_server(blob_hash, mock_blob_bytes) client_blob = self.client_blob_manager.get_blob(blob_hash) server_blob = self.server_blob_manager.get_blob(blob_hash) async def sendfile(writer): await asyncio.sleep(2) return 0 server_blob.sendfile = sendfile with self.assertRaises(asyncio.CancelledError): await request_blob(self.loop, client_blob, self.server_from_client.address, self.server_from_client.tcp_port, 2, 3) async def test_download_blob_using_jsonrpc_blob_get(self): blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1) await self._add_blob_to_server(blob_hash, mock_blob_bytes) # setup RPC Daemon daemon_config = copy.deepcopy(self.client_config) daemon_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)] daemon = Daemon(daemon_config) mock_node = mock.Mock(spec=Node) def _mock_accumulate_peers(q1, q2=None): async def _task(): pass q2 = q2 or asyncio.Queue() return q2, self.loop.create_task(_task()) mock_node.accumulate_peers = _mock_accumulate_peers with mock.patch('lbry.extras.daemon.componentmanager.ComponentManager.all_components_running', return_value=True): with mock.patch('lbry.extras.daemon.daemon.Daemon.dht_node', new_callable=mock.PropertyMock) \ as daemon_mock_dht: with mock.patch('lbry.extras.daemon.daemon.Daemon.blob_manager', new_callable=mock.PropertyMock) \ as daemon_mock_blob_manager: daemon_mock_dht.return_value = mock_node daemon_mock_blob_manager.return_value = self.client_blob_manager result = await daemon.jsonrpc_blob_get(blob_hash, read=True) self.assertIsNotNone(result) self.assertEqual(mock_blob_bytes.decode(), result, "Downloaded blob is different than server blob") ================================================ FILE: tests/unit/components/__init__.py ================================================ ================================================ FILE: tests/unit/components/test_component_manager.py ================================================ import asyncio from lbry.testcase import AsyncioTestCase, AdvanceTimeTestCase from lbry.conf import Config from lbry.extras.daemon.componentmanager import ComponentManager from lbry.extras.daemon.components import DATABASE_COMPONENT, DISK_SPACE_COMPONENT, DHT_COMPONENT, \ BACKGROUND_DOWNLOADER_COMPONENT from lbry.extras.daemon.components import HASH_ANNOUNCER_COMPONENT, UPNP_COMPONENT from lbry.extras.daemon.components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT from lbry.extras.daemon import components class TestComponentManager(AsyncioTestCase): def setUp(self): self.default_components_sort = [ [ components.DatabaseComponent, components.ExchangeRateManagerComponent, components.TorrentComponent, components.UPnPComponent ], [ components.BlobComponent, components.DHTComponent, components.WalletComponent ], [ components.DiskSpaceComponent, components.FileManagerComponent, components.HashAnnouncerComponent, components.PeerProtocolServerComponent, components.WalletServerPaymentsComponent ], [ components.BackgroundDownloaderComponent, components.TrackerAnnouncerComponent ] ] self.component_manager = ComponentManager(Config()) def test_sort_components(self): stages = self.component_manager.sort_components() for stage_list, sorted_stage_list in zip(stages, self.default_components_sort): self.assertEqual([type(stage) for stage in stage_list], sorted_stage_list) def test_sort_components_reverse(self): rev_stages = self.component_manager.sort_components(reverse=True) reverse_default_components_sort = reversed(self.default_components_sort) for stage_list, sorted_stage_list in zip(rev_stages, reverse_default_components_sort): self.assertEqual([type(stage) for stage in stage_list], sorted_stage_list) def test_get_component_not_exists(self): with self.assertRaises(NameError): self.component_manager.get_component("random_component") class TestComponentManagerOverrides(AsyncioTestCase): def test_init_with_overrides(self): class FakeWallet: component_name = "wallet" depends_on = [] def __init__(self, component_manager): self.component_manager = component_manager @property def component(self): return self new_component_manager = ComponentManager(Config(), wallet=FakeWallet) fake_wallet = new_component_manager.get_component("wallet") # wallet should be an instance of FakeWallet and not WalletComponent from components.py self.assertIsInstance(fake_wallet, FakeWallet) self.assertNotIsInstance(fake_wallet, components.WalletComponent) def test_init_with_wrong_overrides(self): class FakeRandomComponent: component_name = "someComponent" depends_on = [] with self.assertRaises(SyntaxError): ComponentManager(Config(), randomComponent=FakeRandomComponent) class FakeComponent: depends_on = [] component_name = None def __init__(self, component_manager): self.component_manager = component_manager self._running = False @property def running(self): return self._running async def start(self): pass async def stop(self): pass @property def component(self): return self async def _setup(self): result = await self.start() self._running = True return result async def _stop(self): result = await self.stop() self._running = False return result async def get_status(self): return {} def __lt__(self, other): return self.component_name < other.component_name class FakeDelayedWallet(FakeComponent): component_name = "wallet" depends_on = [] ledger = None default_wallet = None async def stop(self): await asyncio.sleep(1) class FakeDelayedBlobManager(FakeComponent): component_name = "blob_manager" depends_on = [FakeDelayedWallet.component_name] async def start(self): await asyncio.sleep(1) async def stop(self): await asyncio.sleep(1) class FakeDelayedFileManager(FakeComponent): component_name = "file_manager" depends_on = [FakeDelayedBlobManager.component_name] async def start(self): await asyncio.sleep(1) def get_filtered(self): return [] class TestComponentManagerProperStart(AdvanceTimeTestCase): def setUp(self): self.component_manager = ComponentManager( Config(), skip_components=[ DATABASE_COMPONENT, DISK_SPACE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, BACKGROUND_DOWNLOADER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT], wallet=FakeDelayedWallet, file_manager=FakeDelayedFileManager, blob_manager=FakeDelayedBlobManager ) async def test_proper_starting_of_components(self): asyncio.create_task(self.component_manager.start()) await self.advance(0) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertFalse(self.component_manager.get_component('blob_manager').running) self.assertFalse(self.component_manager.get_component('file_manager').running) await self.advance(1) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) self.assertFalse(self.component_manager.get_component('file_manager').running) await self.advance(1) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) self.assertTrue(self.component_manager.get_component('file_manager').running) async def test_proper_stopping_of_components(self): asyncio.create_task(self.component_manager.start()) await self.advance(0) await self.advance(1) await self.advance(1) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) self.assertTrue(self.component_manager.get_component('file_manager').running) asyncio.create_task(self.component_manager.stop()) await self.advance(0) self.assertFalse(self.component_manager.get_component('file_manager').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) self.assertTrue(self.component_manager.get_component('wallet').running) await self.advance(1) self.assertFalse(self.component_manager.get_component('file_manager').running) self.assertFalse(self.component_manager.get_component('blob_manager').running) self.assertTrue(self.component_manager.get_component('wallet').running) await self.advance(1) self.assertFalse(self.component_manager.get_component('file_manager').running) self.assertFalse(self.component_manager.get_component('blob_manager').running) self.assertFalse(self.component_manager.get_component('wallet').running) ================================================ FILE: tests/unit/core/__init__.py ================================================ ================================================ FILE: tests/unit/core/test_utils.py ================================================ import unittest import asyncio from lbry import utils from lbry.testcase import AsyncioTestCase class CompareVersionTest(unittest.TestCase): def test_compare_versions_isnot_lexographic(self): self.assertTrue(utils.version_is_greater_than('0.3.10', '0.3.6')) def test_same_versions_return_false(self): self.assertFalse(utils.version_is_greater_than('1.3.9', '1.3.9')) def test_same_release_is_greater_then_beta(self): self.assertTrue(utils.version_is_greater_than('1.3.9', '1.3.9b1')) def test_version_can_have_four_parts(self): self.assertTrue(utils.version_is_greater_than('1.3.9.1', '1.3.9')) def test_release_is_greater_than_rc(self): self.assertTrue(utils.version_is_greater_than('1.3.9', '1.3.9rc0')) class ObfuscationTest(unittest.TestCase): def test_deobfuscation_reverses_obfuscation(self): plain = "my_test_string" obf = utils.obfuscate(plain.encode()) self.assertEqual(plain, utils.deobfuscate(obf)) def test_can_use_unicode(self): plain = '☃' obf = utils.obfuscate(plain.encode()) self.assertEqual(plain, utils.deobfuscate(obf)) class SdHashTests(unittest.TestCase): def test_none_in_none_out(self): self.assertIsNone(utils.get_sd_hash(None)) def test_ordinary_dict(self): claim = { "claim": { "value": { "stream": { "source": { "source": "0123456789ABCDEF" } } } } } self.assertEqual("0123456789ABCDEF", utils.get_sd_hash(claim)) def test_old_shape_fails(self): claim = { "stream": { "source": { "source": "0123456789ABCDEF" } } } self.assertIsNone(utils.get_sd_hash(claim)) class CacheConcurrentDecoratorTests(AsyncioTestCase): def setUp(self): self.called = [] self.finished = [] self.counter = 0 @utils.cache_concurrent async def foo(self, arg1, arg2=None, delay=1): self.called.append((arg1, arg2, delay)) await asyncio.sleep(delay) self.counter += 1 self.finished.append((arg1, arg2, delay)) return object() async def test_gather_duplicates(self): result = await asyncio.gather( self.loop.create_task(self.foo(1)), self.loop.create_task(self.foo(1)) ) self.assertEqual(1, len(self.called)) self.assertEqual(1, len(self.finished)) self.assertEqual(1, self.counter) self.assertIs(result[0], result[1]) self.assertEqual(2, len(result)) async def test_one_cancelled_all_cancel(self): t1 = self.loop.create_task(self.foo(1)) self.loop.call_later(0.1, t1.cancel) with self.assertRaises(asyncio.CancelledError): await asyncio.gather( t1, self.loop.create_task(self.foo(1)) ) self.assertEqual(1, len(self.called)) self.assertEqual(0, len(self.finished)) self.assertEqual(0, self.counter) async def test_error_after_success(self): def cause_type_error(): self.counter = "" self.loop.call_later(0.1, cause_type_error) t1 = self.loop.create_task(self.foo(1)) t2 = self.loop.create_task(self.foo(1)) with self.assertRaises(TypeError): await t2 self.assertEqual(1, len(self.called)) self.assertEqual(0, len(self.finished)) self.assertTrue(t1.done()) self.assertEqual("", self.counter) # test that the task is run fresh, it should not error self.counter = 0 t3 = self.loop.create_task(self.foo(1)) self.assertTrue(await t3) self.assertEqual(1, self.counter) # the previously failed call should still raise if awaited with self.assertRaises(TypeError): await t1 self.assertEqual(1, self.counter) async def test_break_it(self): t1 = self.loop.create_task(self.foo(1)) t2 = self.loop.create_task(self.foo(1)) t3 = self.loop.create_task(self.foo(2, delay=0)) t3.add_done_callback(lambda _: t2.cancel()) with self.assertRaises(asyncio.CancelledError): await asyncio.gather(t1, t2, t3) ================================================ FILE: tests/unit/database/__init__.py ================================================ ================================================ FILE: tests/unit/database/test_SQLiteStorage.py ================================================ import shutil import tempfile import unittest import asyncio import logging import hashlib from lbry.testcase import AsyncioTestCase from lbry.conf import Config from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_info import BlobInfo from lbry.blob.blob_manager import BlobManager from lbry.stream.descriptor import StreamDescriptor from tests.test_utils import random_lbry_hash from lbry.dht.peer import make_kademlia_peer log = logging.getLogger() def blob_info_dict(blob_info): info = { "length": blob_info.length, "blob_num": blob_info.blob_num, "iv": blob_info.iv } if blob_info.length: info['blob_hash'] = blob_info.blob_hash return info fake_claim_info = { 'name': "test", 'claim_id': 'deadbeef' * 5, 'address': "bT6wc54qiUUYt34HQF9wnW8b2o2yQTXf2S", 'claim_sequence': 1, 'value': { "version": "_0_0_1", "claimType": "streamType", "stream": { "source": { "source": 'deadbeef' * 12, "version": "_0_0_1", "contentType": "video/mp4", "sourceType": "lbry_sd_hash" }, "version": "_0_0_1", "metadata": { "license": "LBRY inc", "description": "What is LBRY? An introduction with Alex Tabarrok", "language": "en", "title": "What is LBRY?", "author": "Samuel Bryan", "version": "_0_1_0", "nsfw": False, "licenseUrl": "", "preview": "", "thumbnail": "https://s3.amazonaws.com/files.lbry.io/logo.png" } } }, 'height': 10000, 'amount': '1.0', 'effective_amount': '1.0', 'nout': 0, 'txid': "deadbeef" * 8, 'supports': [], 'channel_claim_id': None, 'channel_name': None } class StorageTest(AsyncioTestCase): async def asyncSetUp(self): self.conf = Config() self.storage = SQLiteStorage(self.conf, ':memory:') self.blob_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.blob_dir) self.blob_manager = BlobManager(asyncio.get_event_loop(), self.blob_dir, self.storage, self.conf) await self.storage.open() async def asyncTearDown(self): await self.storage.close() async def store_fake_blob(self, blob_hash, length=100): await self.storage.add_blobs((blob_hash, length, 0, 0), finished=True) async def store_fake_stream(self, stream_hash, blobs=None, file_name="fake_file", key="DEADBEEF"): blobs = blobs or [BlobInfo(1, 100, "DEADBEEF", 0, random_lbry_hash())] descriptor = StreamDescriptor( asyncio.get_event_loop(), self.blob_dir, file_name, key, file_name, blobs, stream_hash ) sd_blob = await descriptor.make_sd_blob() await self.storage.store_stream(sd_blob, descriptor) return descriptor async def make_and_store_fake_stream(self, blob_count=2, stream_hash=None): stream_hash = stream_hash or random_lbry_hash() blobs = [ BlobInfo(i + 1, 100, "DEADBEEF", 0, random_lbry_hash()) for i in range(blob_count) ] await self.store_fake_stream(stream_hash, blobs) class TestSQLiteStorage(StorageTest): async def test_setup(self): files = await self.storage.get_all_lbry_files() self.assertEqual(len(files), 0) blobs = await self.storage.get_all_blob_hashes() self.assertEqual(len(blobs), 0) async def test_store_blob(self): blob_hash = random_lbry_hash() await self.store_fake_blob(blob_hash) blob_hashes = await self.storage.get_all_blob_hashes() self.assertEqual(blob_hashes, [blob_hash]) async def test_delete_blob(self): blob_hash = random_lbry_hash() await self.store_fake_blob(blob_hash) blob_hashes = await self.storage.get_all_blob_hashes() self.assertEqual(blob_hashes, [blob_hash]) await self.storage.delete_blobs_from_db(blob_hashes) blob_hashes = await self.storage.get_all_blob_hashes() self.assertEqual(blob_hashes, []) async def test_supports_storage(self): claim_ids = [random_lbry_hash() for _ in range(10)] random_supports = [{ "txid": random_lbry_hash(), "nout": i, "address": f"addr{i}", "amount": f"{i}.0" } for i in range(20)] expected_supports = {} for idx, claim_id in enumerate(claim_ids): await self.storage.save_supports({claim_id: random_supports[idx*2:idx*2+2]}) for random_support in random_supports[idx*2:idx*2+2]: random_support['claim_id'] = claim_id expected_supports.setdefault(claim_id, []).append(random_support) supports = await self.storage.get_supports(claim_ids[0]) self.assertEqual(supports, expected_supports[claim_ids[0]]) all_supports = await self.storage.get_supports(*claim_ids) for support in all_supports: self.assertIn(support, expected_supports[support['claim_id']]) class StreamStorageTests(StorageTest): async def test_store_and_delete_stream(self): stream_hash = random_lbry_hash() descriptor = await self.store_fake_stream(stream_hash) files = await self.storage.get_all_lbry_files() self.assertListEqual(files, []) stream_hashes = await self.storage.get_all_stream_hashes() self.assertListEqual(stream_hashes, [stream_hash]) await self.storage.delete_stream(descriptor) files = await self.storage.get_all_lbry_files() self.assertListEqual(files, []) stream_hashes = await self.storage.get_all_stream_hashes() self.assertListEqual(stream_hashes, []) @unittest.SkipTest class FileStorageTests(StorageTest): async def test_store_file(self): download_directory = self.db_dir out = await self.storage.get_all_lbry_files() self.assertEqual(len(out), 0) stream_hash = random_lbry_hash() sd_hash = random_lbry_hash() blob1 = random_lbry_hash() blob2 = random_lbry_hash() await self.store_fake_blob(sd_hash) await self.store_fake_blob(blob1) await self.store_fake_blob(blob2) await self.store_fake_stream(stream_hash, sd_hash) await self.store_fake_stream_blob(stream_hash, blob1, 1) await self.store_fake_stream_blob(stream_hash, blob2, 2) blob_data_rate = 0 file_name = "test file" await self.storage.save_published_file( stream_hash, file_name, download_directory, blob_data_rate ) files = await self.storage.get_all_lbry_files() self.assertEqual(1, len(files)) @unittest.SkipTest class ContentClaimStorageTests(StorageTest): async def test_store_content_claim(self): download_directory = self.db_dir out = await self.storage.get_all_lbry_files() self.assertEqual(len(out), 0) stream_hash = random_lbry_hash() sd_hash = fake_claim_info['value']['stream']['source']['source'] # test that we can associate a content claim to a file # use the generated sd hash in the fake claim fake_outpoint = "%s:%i" % (fake_claim_info['txid'], fake_claim_info['nout']) await self.make_and_store_fake_stream(blob_count=2, stream_hash=stream_hash, sd_hash=sd_hash) blob_data_rate = 0 file_name = "test file" await self.storage.save_published_file( stream_hash, file_name, download_directory, blob_data_rate ) await self.storage.save_claims([fake_claim_info]) await self.storage.save_content_claim(stream_hash, fake_outpoint) stored_content_claim = await self.storage.get_content_claim(stream_hash) self.assertDictEqual(stored_content_claim, fake_claim_info) stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(fake_claim_info['claim_id'], stream_hash) self.assertListEqual(stream_hashes, []) # test that we can't associate a claim update with a new stream to the file second_stream_hash, second_sd_hash = random_lbry_hash(), random_lbry_hash() await self.make_and_store_fake_stream(blob_count=2, stream_hash=second_stream_hash, sd_hash=second_sd_hash) with self.assertRaisesRegex(Exception, "stream mismatch"): await self.storage.save_content_claim(second_stream_hash, fake_outpoint) # test that we can associate a new claim update containing the same stream to the file update_info = deepcopy(fake_claim_info) update_info['txid'] = "beef0000" * 12 update_info['nout'] = 0 second_outpoint = "%s:%i" % (update_info['txid'], update_info['nout']) await self.storage.save_claims([update_info]) await self.storage.save_content_claim(stream_hash, second_outpoint) update_info_result = await self.storage.get_content_claim(stream_hash) self.assertDictEqual(update_info_result, update_info) # test that we can't associate an update with a mismatching claim id invalid_update_info = deepcopy(fake_claim_info) invalid_update_info['txid'] = "beef0001" * 12 invalid_update_info['nout'] = 0 invalid_update_info['claim_id'] = "beef0002" * 5 invalid_update_outpoint = "%s:%i" % (invalid_update_info['txid'], invalid_update_info['nout']) with self.assertRaisesRegex(Exception, "mismatching claim ids when updating stream " "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef " "vs beef0002beef0002beef0002beef0002beef0002"): await self.storage.save_claims([invalid_update_info]) await self.storage.save_content_claim(stream_hash, invalid_update_outpoint) current_claim_info = await self.storage.get_content_claim(stream_hash) # this should still be the previous update self.assertDictEqual(current_claim_info, update_info) class UpdatePeersTest(StorageTest): async def test_update_get_peers(self): node_id = hashlib.sha384("1234".encode()).digest() args = (node_id, '73.186.148.72', 4444, None) fake_peer = make_kademlia_peer(*args) await self.storage.save_kademlia_peers([fake_peer]) peers = await self.storage.get_persisted_kademlia_peers() self.assertTupleEqual(args, peers[0]) ================================================ FILE: tests/unit/dht/__init__.py ================================================ ================================================ FILE: tests/unit/dht/protocol/__init__.py ================================================ ================================================ FILE: tests/unit/dht/protocol/test_data_store.py ================================================ import asyncio from unittest import mock, TestCase from lbry.dht.protocol.data_store import DictDataStore from lbry.dht.peer import PeerManager, make_kademlia_peer class DataStoreTests(TestCase): def setUp(self): self.loop = mock.Mock(spec=asyncio.BaseEventLoop) self.loop.time = lambda: 0.0 self.peer_manager = PeerManager(self.loop) self.data_store = DictDataStore(self.loop, self.peer_manager) def _test_add_peer_to_blob(self, blob=b'2' * 48, node_id=b'1' * 48, address='1.2.3.4', tcp_port=3333, udp_port=4444): peer = make_kademlia_peer(node_id, address, udp_port) peer.update_tcp_port(tcp_port) before = self.data_store.get_peers_for_blob(blob) self.data_store.add_peer_to_blob(peer, blob) self.assertListEqual(before + [peer], self.data_store.get_peers_for_blob(blob)) return peer def test_refresh_peer_to_blob(self): blob = b'f' * 48 self.assertListEqual([], self.data_store.get_peers_for_blob(blob)) peer = self._test_add_peer_to_blob(blob=blob, node_id=b'a' * 48, address='1.2.3.4') self.assertTrue(self.data_store.has_peers_for_blob(blob)) self.assertEqual(len(self.data_store.get_peers_for_blob(blob)), 1) self.assertEqual(self.data_store._data_store[blob][0][1], 0) self.loop.time = lambda: 100.0 self.assertEqual(self.data_store._data_store[blob][0][1], 0) self.data_store.add_peer_to_blob(peer, blob) self.assertEqual(self.data_store._data_store[blob][0][1], 100) def test_add_peer_to_blob(self, blob=b'f' * 48, peers=None): peers = peers or [ (b'a' * 48, '1.2.3.4'), (b'b' * 48, '1.2.3.5'), (b'c' * 48, '1.2.3.6'), ] self.assertListEqual([], self.data_store.get_peers_for_blob(blob)) peer_objects = [] for (node_id, address) in peers: peer_objects.append(self._test_add_peer_to_blob(blob=blob, node_id=node_id, address=address)) self.assertTrue(self.data_store.has_peers_for_blob(blob)) self.assertEqual(len(self.data_store.get_peers_for_blob(blob)), len(peers)) return peer_objects def test_get_storing_contacts(self, peers=None, blob1=b'd' * 48, blob2=b'e' * 48): peers = peers or [ (b'a' * 48, '1.2.3.4'), (b'b' * 48, '1.2.3.5'), (b'c' * 48, '1.2.3.6'), ] peer_objs1 = self.test_add_peer_to_blob(blob=blob1, peers=peers) self.assertEqual(len(peers), len(peer_objs1)) self.assertEqual(len(peers), len(self.data_store.get_storing_contacts())) peer_objs2 = self.test_add_peer_to_blob(blob=blob2, peers=peers) self.assertEqual(len(peers), len(peer_objs2)) self.assertEqual(len(peers), len(self.data_store.get_storing_contacts())) for o1, o2 in zip(peer_objs1, peer_objs2): self.assertIs(o1, o2) def test_remove_expired_peers(self): peers = [ (b'a' * 48, '1.2.3.4'), (b'b' * 48, '1.2.3.5'), (b'c' * 48, '1.2.3.6'), ] blob1 = b'd' * 48 blob2 = b'e' * 48 self.data_store.removed_expired_peers() # nothing should happen self.test_get_storing_contacts(peers, blob1, blob2) self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)), len(peers)) self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)), len(peers)) self.assertEqual(len(self.data_store.get_storing_contacts()), len(peers)) # expire the first peer from blob1 first = self.data_store._data_store[blob1][0][0] self.data_store._data_store[blob1][0] = (first, -86401) self.assertEqual(len(self.data_store.get_storing_contacts()), len(peers)) self.data_store.removed_expired_peers() self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)), len(peers) - 1) self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)), len(peers)) self.assertEqual(len(self.data_store.get_storing_contacts()), len(peers)) # expire the first peer from blob2 first = self.data_store._data_store[blob2][0][0] self.data_store._data_store[blob2][0] = (first, -86401) self.data_store.removed_expired_peers() self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)), len(peers) - 1) self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)), len(peers) - 1) self.assertEqual(len(self.data_store.get_storing_contacts()), len(peers) - 1) # expire the second and third peers from blob1 first = self.data_store._data_store[blob2][0][0] self.data_store._data_store[blob1][0] = (first, -86401) second = self.data_store._data_store[blob2][1][0] self.data_store._data_store[blob1][1] = (second, -86401) self.data_store.removed_expired_peers() self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)), 0) self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)), len(peers) - 1) self.assertEqual(len(self.data_store.get_storing_contacts()), len(peers) - 1) ================================================ FILE: tests/unit/dht/protocol/test_distance.py ================================================ import unittest from lbry.dht.protocol.distance import Distance class DistanceTests(unittest.TestCase): def test_invalid_key_length(self): self.assertRaises(ValueError, Distance, b'1' * 47) self.assertRaises(ValueError, Distance, b'1' * 49) self.assertRaises(ValueError, Distance, b'') self.assertRaises(ValueError, Distance(b'0' * 48), b'1' * 47) self.assertRaises(ValueError, Distance(b'0' * 48), b'1' * 49) self.assertRaises(ValueError, Distance(b'0' * 48), b'') ================================================ FILE: tests/unit/dht/protocol/test_kbucket.py ================================================ import struct import asyncio from lbry.utils import generate_id from lbry.dht.protocol.routing_table import KBucket from lbry.dht.peer import PeerManager, make_kademlia_peer from lbry.dht import constants from lbry.testcase import AsyncioTestCase def address_generator(address=(1, 2, 3, 4)): def increment(addr): value = struct.unpack("I", "".join([chr(x) for x in list(addr)[::-1]]).encode())[0] + 1 new_addr = [] for i in range(4): new_addr.append(value % 256) value >>= 8 return tuple(new_addr[::-1]) while True: yield "{}.{}.{}.{}".format(*address) address = increment(address) class TestKBucket(AsyncioTestCase): def setUp(self): self.loop = asyncio.get_event_loop() self.address_generator = address_generator() self.peer_manager = PeerManager(self.loop) self.kbucket = KBucket(self.peer_manager, 0, 2 ** constants.HASH_BITS, generate_id()) def test_add_peer(self): peer = make_kademlia_peer(constants.generate_id(2), "1.2.3.4", udp_port=4444) peer_update2 = make_kademlia_peer(constants.generate_id(2), "1.2.3.4", udp_port=4445) self.assertListEqual([], self.kbucket.peers) # add the peer self.kbucket.add_peer(peer) self.assertListEqual([peer], self.kbucket.peers) # re-add it self.kbucket.add_peer(peer) self.assertListEqual([peer], self.kbucket.peers) self.assertEqual(self.kbucket.peers[0].udp_port, 4444) # add a new peer object with the same id and address but a different port self.kbucket.add_peer(peer_update2) self.assertListEqual([peer_update2], self.kbucket.peers) self.assertEqual(self.kbucket.peers[0].udp_port, 4445) # modify the peer object to have a different port peer_update2.udp_port = 4444 self.kbucket.add_peer(peer_update2) self.assertListEqual([peer_update2], self.kbucket.peers) self.assertEqual(self.kbucket.peers[0].udp_port, 4444) self.kbucket.peers.clear() # Test if contacts can be added to empty list # Add k contacts to bucket for i in range(constants.K): peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444) self.assertTrue(self.kbucket.add_peer(peer)) self.assertEqual(peer, self.kbucket.peers[i]) # Test if contact is not added to full list peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444) self.assertFalse(self.kbucket.add_peer(peer)) # Test if an existing contact is updated correctly if added again existing_peer = self.kbucket.peers[0] self.assertTrue(self.kbucket.add_peer(existing_peer)) self.assertEqual(existing_peer, self.kbucket.peers[-1]) # def testGetContacts(self): # # try and get 2 contacts from empty list # result = self.kbucket.getContacts(2) # self.assertFalse(len(result) != 0, "Returned list should be empty; returned list length: %d" % # (len(result))) # # # Add k-2 contacts # node_ids = [] # if constants.k >= 2: # for i in range(constants.k-2): # node_ids.append(generate_id()) # tmpContact = self.contact_manager.make_contact(node_ids[-1], next(self.address_generator), 4444, 0, # None) # self.kbucket.addContact(tmpContact) # else: # # add k contacts # for i in range(constants.k): # node_ids.append(generate_id()) # tmpContact = self.contact_manager.make_contact(node_ids[-1], next(self.address_generator), 4444, 0, # None) # self.kbucket.addContact(tmpContact) # # # try to get too many contacts # # requested count greater than bucket size; should return at most k contacts # contacts = self.kbucket.getContacts(constants.k+3) # self.assertTrue(len(contacts) <= constants.k, # 'Returned list should not have more than k entries!') # # # verify returned contacts in list # for node_id, i in zip(node_ids, range(constants.k-2)): # self.assertFalse(self.kbucket._contacts[i].id != node_id, # "Contact in position %s not same as added contact" % (str(i))) # # # try to get too many contacts # # requested count one greater than number of contacts # if constants.k >= 2: # result = self.kbucket.getContacts(constants.k-1) # self.assertFalse(len(result) != constants.k-2, # "Too many contacts in returned list %s - should be %s" % # (len(result), constants.k-2)) # else: # result = self.kbucket.getContacts(constants.k-1) # # if the count is <= 0, it should return all of it's contats # self.assertFalse(len(result) != constants.k, # "Too many contacts in returned list %s - should be %s" % # (len(result), constants.k-2)) # result = self.kbucket.getContacts(constants.k-3) # self.assertFalse(len(result) != constants.k-3, # "Too many contacts in returned list %s - should be %s" % # (len(result), constants.k-3)) def test_remove_peer(self): # try remove contact from empty list peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444) self.assertRaises(ValueError, self.kbucket.remove_peer, peer) added = [] # Add couple contacts for i in range(constants.K - 2): peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444) self.assertTrue(self.kbucket.add_peer(peer)) added.append(peer) while added: peer = added.pop() self.assertIn(peer, self.kbucket.peers) self.kbucket.remove_peer(peer) self.assertNotIn(peer, self.kbucket.peers) ================================================ FILE: tests/unit/dht/protocol/test_protocol.py ================================================ import asyncio import binascii from lbry.testcase import AsyncioTestCase from tests import dht_mocks from lbry.dht.serialization.bencoding import bencode, bdecode from lbry.dht import constants from lbry.dht.protocol.protocol import KademliaProtocol from lbry.dht.peer import PeerManager, make_kademlia_peer class TestProtocol(AsyncioTestCase): async def test_ping(self): loop = asyncio.get_event_loop() with dht_mocks.mock_network_loop(loop): node_id1 = constants.generate_id() peer1 = KademliaProtocol( loop, PeerManager(loop), node_id1, '1.2.3.4', 4444, 3333 ) peer2 = KademliaProtocol( loop, PeerManager(loop), constants.generate_id(), '1.2.3.5', 4444, 3333 ) await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444)) await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444)) peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444) result = await peer2.get_rpc_peer(peer).ping() self.assertEqual(result, b'pong') peer1.stop() peer2.stop() peer1.disconnect() peer2.disconnect() async def test_update_token(self): loop = asyncio.get_event_loop() with dht_mocks.mock_network_loop(loop): node_id1 = constants.generate_id() peer1 = KademliaProtocol( loop, PeerManager(loop), node_id1, '1.2.3.4', 4444, 3333 ) peer2 = KademliaProtocol( loop, PeerManager(loop), constants.generate_id(), '1.2.3.5', 4444, 3333 ) await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444)) await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444)) peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444) self.assertEqual(None, peer2.peer_manager.get_node_token(peer.node_id)) await peer2.get_rpc_peer(peer).find_value(b'1' * 48) self.assertNotEqual(None, peer2.peer_manager.get_node_token(peer.node_id)) peer1.stop() peer2.stop() peer1.disconnect() peer2.disconnect() async def test_store_to_peer(self): loop = asyncio.get_event_loop() with dht_mocks.mock_network_loop(loop): node_id1 = constants.generate_id() peer1 = KademliaProtocol( loop, PeerManager(loop), node_id1, '1.2.3.4', 4444, 3333 ) peer2 = KademliaProtocol( loop, PeerManager(loop), constants.generate_id(), '1.2.3.5', 4444, 3333 ) await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444)) await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444)) peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444) peer2_from_peer1 = make_kademlia_peer( peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port ) peer2_from_peer1.update_tcp_port(3333) peer3 = make_kademlia_peer( constants.generate_id(), '1.2.3.6', udp_port=4444 ) store_result = await peer2.store_to_peer(b'2' * 48, peer) self.assertEqual(store_result[0], peer.node_id) self.assertTrue(store_result[1]) self.assertTrue(peer1.data_store.has_peers_for_blob(b'2' * 48)) self.assertFalse(peer1.data_store.has_peers_for_blob(b'3' * 48)) self.assertListEqual([peer2_from_peer1], peer1.data_store.get_storing_contacts()) peer1.data_store.completed_blobs.add(binascii.hexlify(b'2' * 48).decode()) find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48) self.assertEqual(len(find_value_response[b'contacts']), 0) self.assertSetEqual( {b'2' * 48, b'token', b'protocolVersion', b'contacts', b'p'}, set(find_value_response.keys()) ) self.assertEqual(2, len(find_value_response[b'2' * 48])) self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp()) self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response) find_value_page_above_pages_response = peer1.node_rpc.find_value(peer3, b'2' * 48, page=10) self.assertNotIn(b'2' * 48, find_value_page_above_pages_response) peer1.stop() peer2.stop() peer1.disconnect() peer2.disconnect() async def _make_protocol(self, other_peer, node_id, address, udp_port, tcp_port): proto = KademliaProtocol( self.loop, PeerManager(self.loop), node_id, address, udp_port, tcp_port ) await self.loop.create_datagram_endpoint(lambda: proto, (address, 4444)) proto.start() return proto, make_kademlia_peer(node_id, address, udp_port=udp_port) async def test_add_peer_after_handle_request(self): with dht_mocks.mock_network_loop(self.loop): node_id1 = constants.generate_id() node_id2 = constants.generate_id() node_id3 = constants.generate_id() node_id4 = constants.generate_id() peer1 = KademliaProtocol( self.loop, PeerManager(self.loop), node_id1, '1.2.3.4', 4444, 3333 ) await self.loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444)) peer1.start() peer2, peer_2_from_peer_1 = await self._make_protocol(peer1, node_id2, '1.2.3.5', 4444, 3333) peer3, peer_3_from_peer_1 = await self._make_protocol(peer1, node_id3, '1.2.3.6', 4444, 3333) peer4, peer_4_from_peer_1 = await self._make_protocol(peer1, node_id4, '1.2.3.7', 4444, 3333) # peers who reply should be added await peer1.get_rpc_peer(peer_2_from_peer_1).ping() await asyncio.sleep(0.5) self.assertListEqual([peer_2_from_peer_1], peer1.routing_table.get_peers()) peer1.routing_table.remove_peer(peer_2_from_peer_1) # peers not known by be good/bad should be enqueued to maybe-ping peer1_from_peer3 = peer3.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444)) self.assertEqual(0, len(peer1.ping_queue._pending_contacts)) pong = await peer1_from_peer3.ping() self.assertEqual(b'pong', pong) self.assertEqual(1, len(peer1.ping_queue._pending_contacts)) peer1.ping_queue._pending_contacts.clear() # peers who are already good should be added peer1_from_peer4 = peer4.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444)) peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444) peer1.peer_manager.report_last_replied('1.2.3.7', 4444) self.assertEqual(0, len(peer1.ping_queue._pending_contacts)) pong = await peer1_from_peer4.ping() self.assertEqual(b'pong', pong) await asyncio.sleep(0.5) self.assertEqual(1, len(peer1.routing_table.get_peers())) self.assertEqual(0, len(peer1.ping_queue._pending_contacts)) peer1.routing_table.buckets[0].peers.clear() # peers who are known to be bad recently should not be added or maybe-pinged peer1_from_peer4 = peer4.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444)) peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444) peer1.peer_manager.report_failure('1.2.3.7', 4444) peer1.peer_manager.report_failure('1.2.3.7', 4444) self.assertEqual(0, len(peer1.ping_queue._pending_contacts)) pong = await peer1_from_peer4.ping() self.assertEqual(b'pong', pong) self.assertEqual(0, len(peer1.routing_table.get_peers())) self.assertEqual(0, len(peer1.ping_queue._pending_contacts)) for p in [peer1, peer2, peer3, peer4]: p.stop() p.disconnect() ================================================ FILE: tests/unit/dht/protocol/test_routing_table.py ================================================ import asyncio from lbry.testcase import AsyncioTestCase from tests import dht_mocks from lbry.dht import constants from lbry.dht.node import Node from lbry.dht.peer import PeerManager, make_kademlia_peer expected_ranges = [ ( 0, 2462625387274654950767440006258975862817483704404090416746768337765357610718575663213391640930307227550414249394176 ), ( 2462625387274654950767440006258975862817483704404090416746768337765357610718575663213391640930307227550414249394176, 4925250774549309901534880012517951725634967408808180833493536675530715221437151326426783281860614455100828498788352 ), ( 4925250774549309901534880012517951725634967408808180833493536675530715221437151326426783281860614455100828498788352, 9850501549098619803069760025035903451269934817616361666987073351061430442874302652853566563721228910201656997576704 ), ( 9850501549098619803069760025035903451269934817616361666987073351061430442874302652853566563721228910201656997576704, 19701003098197239606139520050071806902539869635232723333974146702122860885748605305707133127442457820403313995153408 ), ( 19701003098197239606139520050071806902539869635232723333974146702122860885748605305707133127442457820403313995153408, 39402006196394479212279040100143613805079739270465446667948293404245721771497210611414266254884915640806627990306816 ) ] class TestRouting(AsyncioTestCase): async def test_fill_one_bucket(self): loop = asyncio.get_event_loop() peer_addresses = [ (constants.generate_id(1), '1.2.3.1'), (constants.generate_id(2), '1.2.3.2'), (constants.generate_id(3), '1.2.3.3'), (constants.generate_id(4), '1.2.3.4'), (constants.generate_id(5), '1.2.3.5'), (constants.generate_id(6), '1.2.3.6'), (constants.generate_id(7), '1.2.3.7'), (constants.generate_id(8), '1.2.3.8'), (constants.generate_id(9), '1.2.3.9'), ] with dht_mocks.mock_network_loop(loop): nodes = { i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address) for i, (node_id, address) in enumerate(peer_addresses) } node_1 = nodes[0] contact_cnt = 0 for i in range(1, len(peer_addresses)): self.assertEqual(len(node_1.protocol.routing_table.get_peers()), contact_cnt) node = nodes[i] peer = make_kademlia_peer( node.protocol.node_id, node.protocol.external_ip, udp_port=node.protocol.udp_port ) added = await node_1.protocol._add_peer(peer) self.assertTrue(added) contact_cnt += 1 self.assertEqual(len(node_1.protocol.routing_table.get_peers()), 8) self.assertEqual(node_1.protocol.routing_table.buckets_with_contacts(), 1) for node in nodes.values(): node.protocol.stop() async def test_cant_add_peer_without_a_node_id_gracefully(self): loop = asyncio.get_event_loop() node = Node(loop, PeerManager(loop), constants.generate_id(), 4444, 4444, 3333, '1.2.3.4') bad_peer = make_kademlia_peer(None, '1.2.3.4', 5555) with self.assertLogs(level='WARNING') as logged: self.assertFalse(await node.protocol._add_peer(bad_peer)) self.assertEqual(1, len(logged.output)) self.assertTrue(logged.output[0].endswith('Tried adding a peer with no node id!')) async def test_split_buckets(self): loop = asyncio.get_event_loop() peer_addresses = [ (constants.generate_id(1), '1.2.3.1'), ] for i in range(2, 200): peer_addresses.append((constants.generate_id(i), f'1.2.3.{i}')) with dht_mocks.mock_network_loop(loop): nodes = { i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address) for i, (node_id, address) in enumerate(peer_addresses) } node_1 = nodes[0] for i in range(1, len(peer_addresses)): node = nodes[i] peer = make_kademlia_peer( node.protocol.node_id, node.protocol.external_ip, udp_port=node.protocol.udp_port ) # set all of the peers to good (as to not attempt pinging stale ones during split) node_1.protocol.peer_manager.report_last_replied(peer.address, peer.udp_port) node_1.protocol.peer_manager.report_last_replied(peer.address, peer.udp_port) await node_1.protocol._add_peer(peer) # check that bucket 0 is always the one covering the local node id self.assertEqual(True, node_1.protocol.routing_table.buckets[0].key_in_range(node_1.protocol.node_id)) self.assertEqual(40, len(node_1.protocol.routing_table.get_peers())) self.assertEqual(len(expected_ranges), len(node_1.protocol.routing_table.buckets)) covered = 0 for (expected_min, expected_max), bucket in zip(expected_ranges, node_1.protocol.routing_table.buckets): self.assertEqual(expected_min, bucket.range_min) self.assertEqual(expected_max, bucket.range_max) covered += bucket.range_max - bucket.range_min self.assertEqual(2**384, covered) for node in nodes.values(): node.stop() # from binascii import hexlify, unhexlify # # from twisted.trial import unittest # from twisted.internet import defer # from lbry.dht import constants # from lbry.dht.routingtable import TreeRoutingTable # from lbry.dht.contact import ContactManager # from lbry.dht.distance import Distance # from lbry.utils import generate_id # # # class FakeRPCProtocol: # """ Fake RPC protocol; allows lbry.dht.contact.Contact objects to "send" RPCs """ # def sendRPC(self, *args, **kwargs): # return defer.succeed(None) # # # class TreeRoutingTableTest(unittest.TestCase): # """ Test case for the RoutingTable class """ # def setUp(self): # self.contact_manager = ContactManager() # self.nodeID = generate_id(b'node1') # self.protocol = FakeRPCProtocol() # self.routingTable = TreeRoutingTable(self.nodeID) # # def test_distance(self): # """ Test to see if distance method returns correct result""" # d = Distance(bytes((170,) * 48)) # result = d(bytes((85,) * 48)) # expected = int(hexlify(bytes((255,) * 48)), 16) # self.assertEqual(result, expected) # # @defer.inlineCallbacks # def test_add_contact(self): # """ Tests if a contact can be added and retrieved correctly """ # # Create the contact # contact_id = generate_id(b'node2') # contact = self.contact_manager.make_contact(contact_id, '127.0.0.1', 9182, self.protocol) # # Now add it... # yield self.routingTable.addContact(contact) # # ...and request the closest nodes to it (will retrieve it) # closest_nodes = self.routingTable.findCloseNodes(contact_id) # self.assertEqual(len(closest_nodes), 1) # self.assertIn(contact, closest_nodes) # # @defer.inlineCallbacks # def test_get_contact(self): # """ Tests if a specific existing contact can be retrieved correctly """ # contact_id = generate_id(b'node2') # contact = self.contact_manager.make_contact(contact_id, '127.0.0.1', 9182, self.protocol) # # Now add it... # yield self.routingTable.addContact(contact) # # ...and get it again # same_contact = self.routingTable.getContact(contact_id) # self.assertEqual(contact, same_contact, 'getContact() should return the same contact') # # @defer.inlineCallbacks # def test_add_parent_node_as_contact(self): # """ # Tests the routing table's behaviour when attempting to add its parent node as a contact # """ # # Create a contact with the same ID as the local node's ID # contact = self.contact_manager.make_contact(self.nodeID, '127.0.0.1', 9182, self.protocol) # # Now try to add it # yield self.routingTable.addContact(contact) # # ...and request the closest nodes to it using FIND_NODE # closest_nodes = self.routingTable.findCloseNodes(self.nodeID, constants.k) # self.assertNotIn(contact, closest_nodes, 'Node added itself as a contact') # # @defer.inlineCallbacks # def test_remove_contact(self): # """ Tests contact removal """ # # Create the contact # contact_id = generate_id(b'node2') # contact = self.contact_manager.make_contact(contact_id, '127.0.0.1', 9182, self.protocol) # # Now add it... # yield self.routingTable.addContact(contact) # # Verify addition # self.assertEqual(len(self.routingTable._buckets[0]), 1, 'Contact not added properly') # # Now remove it # self.routingTable.removeContact(contact) # self.assertEqual(len(self.routingTable._buckets[0]), 0, 'Contact not removed properly') # # @defer.inlineCallbacks # def test_split_bucket(self): # """ Tests if the the routing table correctly dynamically splits k-buckets """ # self.assertEqual(self.routingTable._buckets[0].rangeMax, 2**384, # 'Initial k-bucket range should be 0 <= range < 2**384') # # Add k contacts # for i in range(constants.k): # node_id = generate_id(b'remote node %d' % i) # contact = self.contact_manager.make_contact(node_id, '127.0.0.1', 9182, self.protocol) # yield self.routingTable.addContact(contact) # # self.assertEqual(len(self.routingTable._buckets), 1, # 'Only k nodes have been added; the first k-bucket should now ' # 'be full, but should not yet be split') # # Now add 1 more contact # node_id = generate_id(b'yet another remote node') # contact = self.contact_manager.make_contact(node_id, '127.0.0.1', 9182, self.protocol) # yield self.routingTable.addContact(contact) # self.assertEqual(len(self.routingTable._buckets), 2, # 'k+1 nodes have been added; the first k-bucket should have been ' # 'split into two new buckets') # self.assertNotEqual(self.routingTable._buckets[0].rangeMax, 2**384, # 'K-bucket was split, but its range was not properly adjusted') # self.assertEqual(self.routingTable._buckets[1].rangeMax, 2**384, # 'K-bucket was split, but the second (new) bucket\'s ' # 'max range was not set properly') # self.assertEqual(self.routingTable._buckets[0].rangeMax, # self.routingTable._buckets[1].rangeMin, # 'K-bucket was split, but the min/max ranges were ' # 'not divided properly') # # @defer.inlineCallbacks # def test_full_split(self): # """ # Test that a bucket is not split if it is full, but the new contact is not closer than the kth closest contact # """ # # self.routingTable._parentNodeID = bytes(48 * b'\xff') # # node_ids = [ # b"100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"ff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # b"010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" # ] # # # Add k contacts # for nodeID in node_ids: # # self.assertEquals(nodeID, node_ids[i].decode('hex')) # contact = self.contact_manager.make_contact(unhexlify(nodeID), '127.0.0.1', 9182, self.protocol) # yield self.routingTable.addContact(contact) # self.assertEqual(len(self.routingTable._buckets), 2) # self.assertEqual(len(self.routingTable._buckets[0]._contacts), 8) # self.assertEqual(len(self.routingTable._buckets[1]._contacts), 2) # # # try adding a contact who is further from us than the k'th known contact # nodeID = b'020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' # nodeID = unhexlify(nodeID) # contact = self.contact_manager.make_contact(nodeID, '127.0.0.1', 9182, self.protocol) # self.assertFalse(self.routingTable._shouldSplit(self.routingTable._kbucketIndex(contact.id), contact.id)) # yield self.routingTable.addContact(contact) # self.assertEqual(len(self.routingTable._buckets), 2) # self.assertEqual(len(self.routingTable._buckets[0]._contacts), 8) # self.assertEqual(len(self.routingTable._buckets[1]._contacts), 2) # self.assertNotIn(contact, self.routingTable._buckets[0]._contacts) # self.assertNotIn(contact, self.routingTable._buckets[1]._contacts) # # class KeyErrorFixedTest(unittest.TestCase): # """ Basic tests case for boolean operators on the Contact class """ # # def setUp(self): # own_id = (2 ** constants.key_bits) - 1 # # carefully chosen own_id. here's the logic # # we want a bunch of buckets (k+1, to be exact), and we want to make sure own_id # # is not in bucket 0. so we put own_id at the end so we can keep splitting by adding to the # # end # # self.table = lbry.dht.routingtable.OptimizedTreeRoutingTable(own_id) # # def fill_bucket(self, bucket_min): # bucket_size = lbry.dht.constants.k # for i in range(bucket_min, bucket_min + bucket_size): # self.table.addContact(lbry.dht.contact.Contact(long(i), '127.0.0.1', 9999, None)) # # def overflow_bucket(self, bucket_min): # bucket_size = lbry.dht.constants.k # self.fill_bucket(bucket_min) # self.table.addContact( # lbry.dht.contact.Contact(long(bucket_min + bucket_size + 1), # '127.0.0.1', 9999, None)) # # def testKeyError(self): # # # find middle, so we know where bucket will split # bucket_middle = self.table._buckets[0].rangeMax / 2 # # # fill last bucket # self.fill_bucket(self.table._buckets[0].rangeMax - lbry.dht.constants.k - 1) # # -1 in previous line because own_id is in last bucket # # # fill/overflow 7 more buckets # bucket_start = 0 # for i in range(0, lbry.dht.constants.k): # self.overflow_bucket(bucket_start) # bucket_start += bucket_middle / (2 ** i) # # # replacement cache now has k-1 entries. # # adding one more contact to bucket 0 used to cause a KeyError, but it should work # self.table.addContact( # lbry.dht.contact.Contact(long(lbry.dht.constants.k + 2), '127.0.0.1', 9999, None)) # # # import math # # print "" # # for i, bucket in enumerate(self.table._buckets): # # print "Bucket " + str(i) + " (2 ** " + str( # # math.log(bucket.rangeMin, 2) if bucket.rangeMin > 0 else 0) + " <= x < 2 ** "+str( # # math.log(bucket.rangeMax, 2)) + ")" # # for c in bucket.getContacts(): # # print " contact " + str(c.id) # # for key, bucket in self.table._replacementCache.items(): # # print "Replacement Cache for Bucket " + str(key) # # for c in bucket: # # print " contact " + str(c.id) ================================================ FILE: tests/unit/dht/serialization/__init__.py ================================================ ================================================ FILE: tests/unit/dht/serialization/test_bencoding.py ================================================ import unittest from lbry.dht.serialization.bencoding import _bencode, bencode, bdecode, DecodeError class EncodeDecodeTest(unittest.TestCase): def test_fail_with_not_dict(self): with self.assertRaises(TypeError): bencode(1) with self.assertRaises(TypeError): bencode(b'derp') with self.assertRaises(TypeError): bencode('derp') with self.assertRaises(TypeError): bencode([b'derp']) with self.assertRaises(TypeError): bencode([object()]) with self.assertRaises(TypeError): bencode({b'derp': object()}) def test_fail_bad_type(self): with self.assertRaises(DecodeError): bdecode(b'd4le', True) def test_integer(self): self.assertEqual(_bencode(42), b'i42e') self.assertEqual(bdecode(b'i42e', True), 42) def test_bytes(self): self.assertEqual(_bencode(b''), b'0:') self.assertEqual(_bencode(b'spam'), b'4:spam') self.assertEqual(_bencode(b'4:spam'), b'6:4:spam') self.assertEqual(_bencode(bytearray(b'spam')), b'4:spam') self.assertEqual(bdecode(b'0:', True), b'') self.assertEqual(bdecode(b'4:spam', True), b'spam') self.assertEqual(bdecode(b'6:4:spam', True), b'4:spam') def test_string(self): self.assertEqual(_bencode(''), b'0:') self.assertEqual(_bencode('spam'), b'4:spam') self.assertEqual(_bencode('4:spam'), b'6:4:spam') def test_list(self): self.assertEqual(_bencode([b'spam', 42]), b'l4:spami42ee') self.assertEqual(bdecode(b'l4:spami42ee', True), [b'spam', 42]) def test_dict(self): self.assertEqual(bencode({b'foo': 42, b'bar': b'spam'}), b'd3:bar4:spam3:fooi42ee') self.assertEqual(bdecode(b'd3:bar4:spam3:fooi42ee'), {b'foo': 42, b'bar': b'spam'}) def test_mixed(self): self.assertEqual(_bencode( [[b'abc', b'127.0.0.1', 1919], [b'def', b'127.0.0.1', 1921]]), b'll3:abc9:127.0.0.1i1919eel3:def9:127.0.0.1i1921eee' ) self.assertEqual(bdecode( b'll3:abc9:127.0.0.1i1919eel3:def9:127.0.0.1i1921eee', True), [[b'abc', b'127.0.0.1', 1919], [b'def', b'127.0.0.1', 1921]] ) def test_decode_error(self): self.assertRaises(DecodeError, bdecode, b'abcdefghijklmnopqrstuvwxyz', True) self.assertRaises(DecodeError, bdecode, b'', True) self.assertRaises(DecodeError, bdecode, b'l4:spami42ee') ================================================ FILE: tests/unit/dht/serialization/test_datagram.py ================================================ import binascii import unittest from lbry.dht.error import DecodeError from lbry.dht.serialization.bencoding import _bencode from lbry.dht.serialization.datagram import RequestDatagram, ResponseDatagram, decode_datagram, ErrorDatagram from lbry.dht.serialization.datagram import _decode_datagram from lbry.dht.serialization.datagram import REQUEST_TYPE, RESPONSE_TYPE, ERROR_TYPE from lbry.dht.serialization.datagram import make_compact_address, decode_compact_address class TestDatagram(unittest.TestCase): def test_ping_request_datagram(self): self.assertRaises(ValueError, RequestDatagram.make_ping, b'1' * 48, b'1' * 21) self.assertRaises(ValueError, RequestDatagram.make_ping, b'1' * 47, b'1' * 20) self.assertEqual(20, len(RequestDatagram.make_ping(b'1' * 48).rpc_id)) serialized = RequestDatagram.make_ping(b'1' * 48, b'1' * 20).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, REQUEST_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.method, b'ping') self.assertListEqual(decoded.args, [{b'protocolVersion': 1}]) def test_ping_response(self): self.assertRaises(ValueError, ResponseDatagram, RESPONSE_TYPE, b'1' * 21, b'1' * 48, b'pong') self.assertRaises(ValueError, ResponseDatagram, RESPONSE_TYPE, b'1' * 20, b'1' * 49, b'pong') self.assertRaises(ValueError, ResponseDatagram, 5, b'1' * 20, b'1' * 48, b'pong') serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, b'pong').bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, RESPONSE_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.response, b'pong') def test_find_node_request_datagram(self): self.assertRaises(ValueError, RequestDatagram.make_find_node, b'1' * 49, b'2' * 48, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_find_node, b'1' * 48, b'2' * 49, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_find_node, b'1' * 48, b'2' * 48, b'1' * 21) self.assertEqual(20, len(RequestDatagram.make_find_node(b'1' * 48, b'2' * 48).rpc_id)) serialized = RequestDatagram.make_find_node(b'1' * 48, b'2' * 48, b'1' * 20).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, REQUEST_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.method, b'findNode') self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1}]) def test_find_node_response(self): closest_response = [(b'3' * 48, '1.2.3.4', 1234)] expected = [[b'3' * 48, b'1.2.3.4', 1234]] serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, closest_response).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, RESPONSE_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.response, expected) def test_find_value_request(self): self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 49, b'2' * 48, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 49, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 48, b'1' * 21) self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 48, b'1' * 20, -1) self.assertEqual(20, len(RequestDatagram.make_find_value(b'1' * 48, b'2' * 48).rpc_id)) # default page argument serialized = RequestDatagram.make_find_value(b'1' * 48, b'2' * 48, b'1' * 20).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, REQUEST_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.method, b'findValue') self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1, b'p': 0}]) # nondefault page argument serialized = RequestDatagram.make_find_value(b'1' * 48, b'2' * 48, b'1' * 20, 1).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, REQUEST_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.method, b'findValue') self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1, b'p': 1}]) def test_find_value_response_without_pages_field(self): found_value_response = {b'2' * 48: [b'\x7f\x00\x00\x01']} serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, found_value_response).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, RESPONSE_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertDictEqual(decoded.response, found_value_response) def test_find_value_response_with_pages_field(self): found_value_response = {b'2' * 48: [b'\x7f\x00\x00\x01'], b'p': 1} serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, found_value_response).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, RESPONSE_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertDictEqual(decoded.response, found_value_response) def test_store_request(self): self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 47, b'2' * 48, b'3' * 48, 3333, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 48, b'2' * 49, b'3' * 48, 3333, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 48, b'2' * 48, b'3' * 47, 3333, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 48, b'2' * 48, b'3' * 48, -3333, b'1' * 20) self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 48, b'2' * 48, b'3' * 48, 3333, b'1' * 21) serialized = RequestDatagram.make_store(b'1' * 48, b'2' * 48, b'3' * 48, 3333, b'1' * 20).bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, REQUEST_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.method, b'store') def test_error_datagram(self): serialized = ErrorDatagram(ERROR_TYPE, b'1' * 20, b'1' * 48, b'FakeErrorType', b'more info').bencode() decoded = decode_datagram(serialized) self.assertEqual(decoded.packet_type, ERROR_TYPE) self.assertEqual(decoded.rpc_id, b'1' * 20) self.assertEqual(decoded.node_id, b'1' * 48) self.assertEqual(decoded.exception_type, 'FakeErrorType') self.assertEqual(decoded.response, 'more info') def test_invalid_datagram_type(self): serialized = b'di0ei5ei1e20:11111111111111111111i2e48:11111111111111111111' \ b'1111111111111111111111111111i3e13:FakeErrorTypei4e9:more infoe' self.assertRaises(ValueError, decode_datagram, serialized) self.assertRaises(DecodeError, decode_datagram, _bencode([1, 2, 3, 4])) def test_optional_field_backwards_compatible(self): datagram = decode_datagram(_bencode({ 0: 0, 1: b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc', 2: b'111111111111111111111111111111111111111111111111', 3: b'ping', 4: [{b'protocolVersion': 1}], 5: b'should not error' })) self.assertEqual(datagram.packet_type, REQUEST_TYPE) self.assertEqual(b'ping', datagram.method) def test_str_or_int_keys(self): datagram = decode_datagram(_bencode({ b'0': 0, b'1': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc', b'2': b'111111111111111111111111111111111111111111111111', b'3': b'ping', b'4': [{b'protocolVersion': 1}], b'5': b'should not error' })) self.assertEqual(datagram.packet_type, REQUEST_TYPE) self.assertEqual(b'ping', datagram.method) def test_mixed_str_or_int_keys(self): # datagram, _ = _bencode({ # b'0': 0, # 1: b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc', # b'2': b'111111111111111111111111111111111111111111111111', # 3: b'ping', # b'4': [{b'protocolVersion': 1}], # b'5': b'should not error' # })) encoded = binascii.unhexlify(b"64313a3069306569316532303a0abcb5269d6cfc1e87a08e920bf39fe9df8e92fc313a3234383a313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131693365343a70696e67313a346c6431353a70726f746f636f6c56657273696f6e6931656565313a3531363a73686f756c64206e6f74206572726f7265") self.assertDictEqual( { 'packet_type': 0, 'rpc_id': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc', 'node_id': b'111111111111111111111111111111111111111111111111', 'method': b'ping', 'args': [{b'protocolVersion': 1}] }, _decode_datagram(encoded)[0] ) class TestCompactAddress(unittest.TestCase): def test_encode_decode(self, address='1.2.3.4', port=4444, node_id=b'1' * 48): decoded = decode_compact_address(make_compact_address(node_id, address, port)) self.assertEqual((node_id, address, port), decoded) def test_errors(self): self.assertRaises(ValueError, make_compact_address, b'1' * 48, '1.2.3.4', 0) self.assertRaises(ValueError, make_compact_address, b'1' * 48, '1.2.3.4', 65536) self.assertRaises( ValueError, decode_compact_address, b'\x01\x02\x03\x04\x00\x00111111111111111111111111111111111111111111111111' ) self.assertRaises(ValueError, make_compact_address, b'1' * 48, '1.2.3.4.5', 4444) self.assertRaises(ValueError, make_compact_address, b'1' * 47, '1.2.3.4', 4444) self.assertRaises( ValueError, decode_compact_address, b'\x01\x02\x03\x04\x11\\11111111111111111111111111111111111111111111111' ) ================================================ FILE: tests/unit/dht/test_blob_announcer.py ================================================ import contextlib import logging import typing import binascii import socket import asyncio from lbry.testcase import AsyncioTestCase from tests import dht_mocks from lbry.dht.protocol.distance import Distance from lbry.conf import Config from lbry.dht import constants from lbry.dht.node import Node from lbry.dht.peer import PeerManager, make_kademlia_peer from lbry.dht.blob_announcer import BlobAnnouncer from lbry.extras.daemon.storage import SQLiteStorage class TestBlobAnnouncer(AsyncioTestCase): async def setup_node(self, peer_addresses, address, node_id): self.nodes: typing.Dict[int, Node] = {} self.advance = dht_mocks.get_time_accelerator(self.loop) self.instant_advance = dht_mocks.get_time_accelerator(self.loop) self.conf = Config() self.peer_manager = PeerManager(self.loop) self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address) await self.node.start_listening(address) await asyncio.gather(*[self.add_peer(node_id, address) for node_id, address in peer_addresses]) for first_peer in self.nodes.values(): for second_peer in self.nodes.values(): if first_peer == second_peer: continue self.add_peer_to_routing_table(first_peer, second_peer) self.add_peer_to_routing_table(second_peer, first_peer) await self.advance(0.1) # just to make pings go through self.node.joined.set() self.node._refresh_task = self.loop.create_task(self.node.refresh_node()) self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time) await self.storage.open() self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage) async def add_peer(self, node_id, address, add_to_routing_table=True): #print('add', node_id.hex()[:8], address) n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address) await n.start_listening(address) self.nodes.update({len(self.nodes): n}) if add_to_routing_table: self.add_peer_to_routing_table(self.node, n) def add_peer_to_routing_table(self, adder, being_added): adder.protocol.add_peer( make_kademlia_peer( being_added.protocol.node_id, being_added.protocol.external_ip, being_added.protocol.udp_port ) ) @contextlib.asynccontextmanager async def _test_network_context(self, peer_count=200): self.peer_addresses = [ (constants.generate_id(i), socket.inet_ntoa(int(i + 0x01000001).to_bytes(length=4, byteorder='big'))) for i in range(1, peer_count + 1) ] try: with dht_mocks.mock_network_loop(self.loop): await self.setup_node(self.peer_addresses, '1.2.3.1', constants.generate_id(1000)) yield finally: self.blob_announcer.stop() self.node.stop() for n in self.nodes.values(): n.stop() async def chain_peer(self, node_id, address): previous_last_node = self.nodes[len(self.nodes) - 1] await self.add_peer(node_id, address, False) last_node = self.nodes[len(self.nodes) - 1] peer = last_node.protocol.get_rpc_peer( make_kademlia_peer( previous_last_node.protocol.node_id, previous_last_node.protocol.external_ip, previous_last_node.protocol.udp_port ) ) await peer.ping() return last_node async def test_announce_blobs(self): blob1 = binascii.hexlify(b'1' * 48).decode() blob2 = binascii.hexlify(b'2' * 48).decode() async with self._test_network_context(peer_count=100): await self.storage.add_blobs((blob1, 1024, 0, True), (blob2, 1024, 0, True), finished=True) await self.storage.add_blobs( *((constants.generate_id(value).hex(), 1024, 0, True) for value in range(1000, 1090)), finished=True) await self.storage.db.execute("update blob set next_announce_time=0, should_announce=1") to_announce = await self.storage.get_blobs_to_announce() self.assertEqual(92, len(to_announce)) self.blob_announcer.start(batch_size=10) # so it covers batching logic # takes 60 seconds to start, but we advance 120 to ensure it processed all batches ongoing_announcements = asyncio.ensure_future(self.blob_announcer.wait()) await self.instant_advance(60.0) await ongoing_announcements to_announce = await self.storage.get_blobs_to_announce() self.assertEqual(0, len(to_announce)) self.blob_announcer.stop() # as routing table pollution will cause some peers to be hard to reach, we add a tolerance for CI tolerance = 0.8 # at least 80% of the announcements are within the top K for blob in await self.storage.get_all_blob_hashes(): distance = Distance(bytes.fromhex(blob)) candidates = list(self.nodes.values()) candidates.sort(key=lambda sorting_node: distance(sorting_node.protocol.node_id)) has_it = 0 for index, node in enumerate(candidates[:constants.K], start=1): if node.protocol.data_store.get_peers_for_blob(bytes.fromhex(blob)): has_it += 1 else: logging.warning("blob %s wasnt found between the best K (%s)", blob[:8], node.protocol.node_id.hex()[:8]) self.assertGreaterEqual(has_it, int(tolerance * constants.K)) # test that we can route from a poorly connected peer all the way to the announced blob current = len(self.nodes) await self.chain_peer(constants.generate_id(current + 1), '1.2.3.10') await self.chain_peer(constants.generate_id(current + 2), '1.2.3.11') await self.chain_peer(constants.generate_id(current + 3), '1.2.3.12') await self.chain_peer(constants.generate_id(current + 4), '1.2.3.13') last = await self.chain_peer(constants.generate_id(current + 5), '1.2.3.14') search_q, peer_q = asyncio.Queue(), asyncio.Queue() search_q.put_nowait(blob1) _, task = last.accumulate_peers(search_q, peer_q) found_peers = await asyncio.wait_for(peer_q.get(), 1.0) task.cancel() self.assertEqual(1, len(found_peers)) self.assertEqual(self.node.protocol.node_id, found_peers[0].node_id) self.assertEqual(self.node.protocol.external_ip, found_peers[0].address) self.assertEqual(self.node.protocol.peer_port, found_peers[0].tcp_port) async def test_popular_blob(self): peer_count = 150 blob_hash = constants.generate_id(99999) async with self._test_network_context(peer_count=peer_count): total_seen = set() announced_to = self.nodes.pop(0) for i, node in enumerate(self.nodes.values()): self.add_peer_to_routing_table(announced_to, node) peer = node.protocol.get_rpc_peer( make_kademlia_peer( announced_to.protocol.node_id, announced_to.protocol.external_ip, announced_to.protocol.udp_port ) ) response = await peer.store(blob_hash) self.assertEqual(response, b'OK') peers_for_blob = await peer.find_value(blob_hash, 0) if i == 0: self.assertNotIn(blob_hash, peers_for_blob) self.assertEqual(peers_for_blob[b'p'], 0) else: self.assertEqual(len(peers_for_blob[blob_hash]), min(i, constants.K)) self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i + 1) if i - 1 > constants.K: self.assertEqual(len(peers_for_blob[b'contacts']), constants.K) self.assertEqual(peers_for_blob[b'p'], (i // (constants.K + 1)) + 1) seen = set(peers_for_blob[blob_hash]) self.assertEqual(len(seen), constants.K) self.assertEqual(len(peers_for_blob[blob_hash]), len(seen)) for pg in range(1, peers_for_blob[b'p']): page_x = await peer.find_value(blob_hash, pg) self.assertNotIn(b'contacts', page_x) page_x_set = set(page_x[blob_hash]) self.assertEqual(len(page_x[blob_hash]), len(page_x_set)) self.assertGreater(len(page_x_set), 0) self.assertSetEqual(seen.intersection(page_x_set), set()) seen.intersection_update(page_x_set) total_seen.update(page_x_set) else: self.assertEqual(len(peers_for_blob[b'contacts']), 8) # we always add 8 on first page self.assertEqual(len(total_seen), peer_count - 2) ================================================ FILE: tests/unit/dht/test_node.py ================================================ import asyncio import time import unittest import typing from lbry.testcase import AsyncioTestCase from tests import dht_mocks from lbry.conf import Config from lbry.dht import constants from lbry.dht.node import Node from lbry.dht.peer import PeerManager, make_kademlia_peer from lbry.extras.daemon.storage import SQLiteStorage class TestBootstrapNode(AsyncioTestCase): TIMEOUT = 10.0 # do not increase. Hitting a timeout is a real failure async def test_bootstrap_node_adds_all_peers(self): loop = asyncio.get_event_loop() loop.set_debug(False) with dht_mocks.mock_network_loop(loop): advance = dht_mocks.get_time_accelerator(loop) self.bootstrap_node = Node(self.loop, PeerManager(loop), constants.generate_id(), 4444, 4444, 3333, '1.2.3.4', is_bootstrap_node=True) self.bootstrap_node.start('1.2.3.4', []) self.bootstrap_node.protocol.ping_queue._default_delay = 0 self.addCleanup(self.bootstrap_node.stop) # start the nodes nodes = {} futs = [] for i in range(100): nodes[i] = Node(loop, PeerManager(loop), constants.generate_id(i), 4444, 4444, 3333, f'1.3.3.{i}') nodes[i].start(f'1.3.3.{i}', [('1.2.3.4', 4444)]) self.addCleanup(nodes[i].stop) futs.append(nodes[i].joined.wait()) await asyncio.gather(*futs) while self.bootstrap_node.protocol.ping_queue.busy: await advance(1) self.assertEqual(100, len(self.bootstrap_node.protocol.routing_table.get_peers())) class TestNodePingQueueDiscover(AsyncioTestCase): async def test_ping_queue_discover(self): loop = asyncio.get_event_loop() loop.set_debug(False) peer_addresses = [ (constants.generate_id(1), '1.2.3.1'), (constants.generate_id(2), '1.2.3.2'), (constants.generate_id(3), '1.2.3.3'), (constants.generate_id(4), '1.2.3.4'), (constants.generate_id(5), '1.2.3.5'), (constants.generate_id(6), '1.2.3.6'), (constants.generate_id(7), '1.2.3.7'), (constants.generate_id(8), '1.2.3.8'), (constants.generate_id(9), '1.2.3.9'), ] with dht_mocks.mock_network_loop(loop): advance = dht_mocks.get_time_accelerator(loop) # start the nodes nodes: typing.Dict[int, Node] = { i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address) for i, (node_id, address) in enumerate(peer_addresses) } for i, n in nodes.items(): n.start(peer_addresses[i][1], []) await advance(1) node_1 = nodes[0] # ping 8 nodes from node_1, this will result in a delayed return ping futs = [] for i in range(1, len(peer_addresses)): node = nodes[i] assert node.protocol.node_id != node_1.protocol.node_id peer = make_kademlia_peer( node.protocol.node_id, node.protocol.external_ip, udp_port=node.protocol.udp_port ) futs.append(node_1.protocol.get_rpc_peer(peer).ping()) await advance(3) replies = await asyncio.gather(*tuple(futs)) self.assertTrue(all(map(lambda reply: reply == b"pong", replies))) # run for long enough for the delayed pings to have been sent by node 1 await advance(1000) # verify all of the previously pinged peers have node_1 in their routing tables for n in nodes.values(): peers = n.protocol.routing_table.get_peers() if n is node_1: self.assertEqual(8, len(peers)) # TODO: figure out why this breaks # else: # self.assertEqual(1, len(peers)) # self.assertEqual((peers[0].node_id, peers[0].address, peers[0].udp_port), # (node_1.protocol.node_id, node_1.protocol.external_ip, node_1.protocol.udp_port)) # run long enough for the refresh loop to run await advance(3600) # verify all the nodes know about each other for n in nodes.values(): if n is node_1: continue peers = n.protocol.routing_table.get_peers() self.assertEqual(8, len(peers)) self.assertSetEqual( {n_id[0] for n_id in peer_addresses if n_id[0] != n.protocol.node_id}, {c.node_id for c in peers} ) self.assertSetEqual( {n_addr[1] for n_addr in peer_addresses if n_addr[1] != n.protocol.external_ip}, {c.address for c in peers} ) # teardown for n in nodes.values(): n.stop() class TestTemporarilyLosingConnection(AsyncioTestCase): @unittest.SkipTest async def test_losing_connection(self): async def wait_for(check_ok, insist, timeout=20): start = time.time() while time.time() - start < timeout: if check_ok(): break await asyncio.sleep(0) else: insist() loop = self.loop loop.set_debug(False) peer_addresses = [ ('1.2.3.4', 40000+i) for i in range(10) ] node_ids = [constants.generate_id(i) for i in range(10)] nodes = [ Node( loop, PeerManager(loop), node_id, udp_port, udp_port, 3333, address, storage=SQLiteStorage(Config(), ":memory:", self.loop, self.loop.time) ) for node_id, (address, udp_port) in zip(node_ids, peer_addresses) ] dht_network = {peer_addresses[i]: node.protocol for i, node in enumerate(nodes)} num_seeds = 3 with dht_mocks.mock_network_loop(loop, dht_network): for i, n in enumerate(nodes): await n._storage.open() self.addCleanup(n.stop) n.start(peer_addresses[i][0], peer_addresses[:num_seeds]) await asyncio.gather(*[n.joined.wait() for n in nodes]) node = nodes[-1] advance = dht_mocks.get_time_accelerator(loop) await advance(500) # Join the network, assert that at least the known peers are in RT self.assertTrue(node.joined.is_set()) self.assertTrue(len(node.protocol.routing_table.get_peers()) >= num_seeds) # Refresh, so that the peers are persisted self.assertFalse(len(await node._storage.get_persisted_kademlia_peers()) > num_seeds) await advance(4000) self.assertTrue(len(await node._storage.get_persisted_kademlia_peers()) > num_seeds) # We lost internet connection - all the peers stop responding dht_network.pop((node.protocol.external_ip, node.protocol.udp_port)) # The peers are cleared on refresh from RT and storage await advance(4000) self.assertListEqual([], await node._storage.get_persisted_kademlia_peers()) await wait_for( lambda: len(node.protocol.routing_table.get_peers()) == 0, lambda: self.assertListEqual(node.protocol.routing_table.get_peers(), []) ) # Reconnect dht_network[(node.protocol.external_ip, node.protocol.udp_port)] = node.protocol # Check that node reconnects at least to them await advance(1000) await wait_for( lambda: len(node.protocol.routing_table.get_peers()) >= num_seeds, lambda: self.assertGreaterEqual(len(node.protocol.routing_table.get_peers()), num_seeds) ) ================================================ FILE: tests/unit/dht/test_peer.py ================================================ import asyncio import unittest from lbry.utils import generate_id from lbry.dht.peer import PeerManager, make_kademlia_peer, is_valid_public_ipv4 from lbry.testcase import AsyncioTestCase class PeerTest(AsyncioTestCase): def setUp(self): self.loop = asyncio.get_event_loop() self.peer_manager = PeerManager(self.loop) self.node_ids = [generate_id(), generate_id(), generate_id()] self.first_contact = make_kademlia_peer(self.node_ids[1], '1.0.0.1', udp_port=1024) self.second_contact = make_kademlia_peer(self.node_ids[0], '1.0.0.2', udp_port=1024) def test_peer_is_good_unknown_peer(self): # Scenario: peer replied, but caller doesn't know the node_id. # Outcome: We can't say it's good or bad. # (yes, we COULD tell the node id, but not here. It would be # a side effect and the caller is responsible to discover it) peer = make_kademlia_peer(None, '1.2.3.4', 4444) self.peer_manager.report_last_requested('1.2.3.4', 4444) self.peer_manager.report_last_replied('1.2.3.4', 4444) self.assertIsNone(self.peer_manager.peer_is_good(peer)) def test_make_contact_error_cases(self): self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', 100000) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4.5', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], 'this is not an ip', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', -1000) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', 0) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', 1023) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', 70000) self.assertRaises(ValueError, make_kademlia_peer, b'not valid node id', '1.2.3.4', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '0.0.0.0', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '10.0.0.1', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '100.64.0.1', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '127.0.0.1', 1024) self.assertIsNotNone(make_kademlia_peer(self.node_ids[1], '127.0.0.1', 1024, allow_localhost=True)) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.0.1', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '172.16.0.1', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '169.254.1.1', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.0.0.2', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.0.2.2', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.88.99.2', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '198.18.1.1', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '198.51.100.2', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '198.51.100.2', 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '203.0.113.4', 1024) for i in range(32): self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], f"{224 + i}.0.0.0", 1024) self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '255.255.255.255', 1024) self.assertRaises( ValueError, make_kademlia_peer, self.node_ids[1], 'beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', 1024 ) self.assertRaises( ValueError, make_kademlia_peer, self.node_ids[1], '2001:db8::ff00:42:8329', 1024 ) def test_is_valid_ipv4(self): self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef')) self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', True)) self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329')) self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329', True)) self.assertFalse(is_valid_public_ipv4('127.0.0.1')) self.assertTrue(is_valid_public_ipv4('127.0.0.1', True)) self.assertFalse(is_valid_public_ipv4('172.16.0.1')) self.assertFalse(is_valid_public_ipv4('172.16.0.1', True)) self.assertTrue(is_valid_public_ipv4('1.2.3.4')) self.assertTrue(is_valid_public_ipv4('1.2.3.4', True)) self.assertFalse(is_valid_public_ipv4('derp')) self.assertFalse(is_valid_public_ipv4('derp', True)) def test_boolean(self): self.assertNotEqual(self.first_contact, self.second_contact) self.assertEqual( self.second_contact, make_kademlia_peer(self.node_ids[0], '1.0.0.2', udp_port=1024) ) def test_compact_ip(self): self.assertEqual(b'\x01\x00\x00\x01', self.first_contact.compact_ip()) self.assertEqual(b'\x01\x00\x00\x02', self.second_contact.compact_ip()) @unittest.SkipTest class TestContactLastReplied(unittest.TestCase): def setUp(self): self.clock = task.Clock() self.contact_manager = ContactManager(self.clock.seconds) self.contact = self.contact_manager.make_contact(generate_id(), "127.0.0.1", 4444, None) self.clock.advance(3600) self.assertIsNone(self.contact.contact_is_good) def test_stale_replied_to_us(self): self.contact.update_last_replied() self.assertIs(self.contact.contact_is_good, True) def test_stale_requested_from_us(self): self.contact.update_last_requested() self.assertIsNone(self.contact.contact_is_good) def test_stale_then_fail(self): self.contact.update_last_failed() self.assertIsNone(self.contact.contact_is_good) self.clock.advance(1) self.contact.update_last_failed() self.assertIs(self.contact.contact_is_good, False) def test_good_turned_stale(self): self.contact.update_last_replied() self.assertIs(self.contact.contact_is_good, True) self.clock.advance(constants.checkRefreshInterval - 1) self.assertIs(self.contact.contact_is_good, True) self.clock.advance(1) self.assertIsNone(self.contact.contact_is_good) def test_good_then_fail(self): self.contact.update_last_replied() self.assertIs(self.contact.contact_is_good, True) self.clock.advance(1) self.contact.update_last_failed() self.assertIs(self.contact.contact_is_good, True) self.clock.advance(59) self.assertIs(self.contact.contact_is_good, True) self.contact.update_last_failed() self.assertIs(self.contact.contact_is_good, False) for _ in range(7200): self.clock.advance(60) self.assertIs(self.contact.contact_is_good, False) def test_good_then_fail_then_good(self): # it replies self.contact.update_last_replied() self.assertIs(self.contact.contact_is_good, True) self.clock.advance(1) # it fails twice in a row self.contact.update_last_failed() self.clock.advance(1) self.contact.update_last_failed() self.assertIs(self.contact.contact_is_good, False) self.clock.advance(1) # it replies self.contact.update_last_replied() self.clock.advance(1) self.assertIs(self.contact.contact_is_good, True) # it goes stale self.clock.advance(constants.checkRefreshInterval - 2) self.assertIs(self.contact.contact_is_good, True) self.clock.advance(1) self.assertIsNone(self.contact.contact_is_good) @unittest.SkipTest class TestContactLastRequested(unittest.TestCase): def setUp(self): self.clock = task.Clock() self.contact_manager = ContactManager(self.clock.seconds) self.contact = self.contact_manager.make_contact(generate_id(), "127.0.0.1", 4444, None) self.clock.advance(1) self.contact.update_last_replied() self.clock.advance(3600) self.assertIsNone(self.contact.contact_is_good) def test_previous_replied_then_requested(self): # it requests self.contact.update_last_requested() self.assertIs(self.contact.contact_is_good, True) # it goes stale self.clock.advance(constants.checkRefreshInterval - 1) self.assertIs(self.contact.contact_is_good, True) self.clock.advance(1) self.assertIsNone(self.contact.contact_is_good) def test_previous_replied_then_requested_then_failed(self): # it requests self.contact.update_last_requested() self.assertIs(self.contact.contact_is_good, True) self.clock.advance(1) # it fails twice in a row self.contact.update_last_failed() self.clock.advance(1) self.contact.update_last_failed() self.assertIs(self.contact.contact_is_good, False) self.clock.advance(1) # it requests self.contact.update_last_requested() self.clock.advance(1) self.assertIs(self.contact.contact_is_good, False) # it goes stale self.clock.advance((constants.refreshTimeout / 4) - 2) self.assertIs(self.contact.contact_is_good, False) self.clock.advance(1) self.assertIs(self.contact.contact_is_good, False) ================================================ FILE: tests/unit/lbrynet_daemon/__init__.py ================================================ ================================================ FILE: tests/unit/lbrynet_daemon/test_Daemon.py ================================================ import unittest from unittest import mock import json from lbry.extras.daemon.storage import SQLiteStorage from lbry.extras.daemon.componentmanager import ComponentManager from lbry.extras.daemon.components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT from lbry.extras.daemon.components import HASH_ANNOUNCER_COMPONENT from lbry.extras.daemon.components import UPNP_COMPONENT, BLOB_COMPONENT from lbry.extras.daemon.components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT from lbry.extras.daemon.daemon import Daemon as LBRYDaemon from lbry.wallet import WalletManager, Wallet from lbry.conf import Config from tests import test_utils # from tests.mocks import mock_conf_settings, FakeNetwork, FakeFileManager # from tests.mocks import ExchangeRateManager as DummyExchangeRateManager # from tests.mocks import BTCLBCFeed, USDBTCFeed from tests.test_utils import is_android def get_test_daemon(conf: Config, with_fee=False): conf.data_dir = '/tmp' rates = { 'BTCLBC': {'spot': 3.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1}, 'USDBTC': {'spot': 2.0, 'ts': test_utils.DEFAULT_ISO_TIME + 2} } component_manager = ComponentManager( conf, skip_components=[ DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, HASH_ANNOUNCER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, RATE_LIMITER_COMPONENT], file_manager=FakeFileManager ) daemon = LBRYDaemon(conf, component_manager=component_manager) daemon.payment_rate_manager = OnlyFreePaymentsManager() daemon.wallet_manager = mock.Mock(spec=WalletManager) daemon.wallet_manager.wallet = mock.Mock(spec=Wallet) daemon.wallet_manager.use_encryption = False daemon.wallet_manager.network = FakeNetwork() daemon.storage = mock.Mock(spec=SQLiteStorage) market_feeds = [BTCLBCFeed(), USDBTCFeed()] daemon.exchange_rate_manager = DummyExchangeRateManager(market_feeds, rates) daemon.stream_manager = component_manager.get_component(FILE_MANAGER_COMPONENT) metadata = { "author": "fake author", "language": "en", "content_type": "fake/format", "description": "fake description", "license": "fake license", "license_url": "fake license url", "nsfw": False, "sources": { "lbry_sd_hash": 'd2b8b6e907dde95245fe6d144d16c2fdd60c4e0c6463ec98' 'b85642d06d8e9414e8fcfdcb7cb13532ec5454fb8fe7f280' }, "thumbnail": "fake thumbnail", "title": "fake title", "ver": "0.0.3" } if with_fee: metadata.update( {"fee": {"USD": {"address": "bQ6BGboPV2SpTMEP7wLNiAcnsZiH8ye6eA", "amount": 0.75}}}) migrated = smart_decode(json.dumps(metadata)) daemon._resolve = daemon.resolve = lambda *_: defer.succeed( {"test": {'claim': {'value': migrated.claim_dict}}}) return daemon @unittest.SkipTest class TestCostEst(unittest.TestCase): def setUp(self): test_utils.reset_time(self) def test_fee_and_generous_data(self): size = 10000000 correct_result = 4.5 daemon = get_test_daemon(Config(is_generous_host=True), with_fee=True) result = yield f2d(daemon.get_est_cost("test", size)) self.assertEqual(result, correct_result) def test_generous_data_and_no_fee(self): size = 10000000 correct_result = 0.0 daemon = get_test_daemon(Config(is_generous_host=True)) result = yield f2d(daemon.get_est_cost("test", size)) self.assertEqual(result, correct_result) @unittest.SkipTest class TestJsonRpc(unittest.TestCase): def setUp(self): async def noop(): return None test_utils.reset_time(self) self.test_daemon = get_test_daemon(Config()) self.test_daemon.wallet_manager.get_best_blockhash = noop def test_status(self): status = yield f2d(self.test_daemon.jsonrpc_status()) self.assertDictContainsSubset({'is_running': False}, status) def test_help(self): result = self.test_daemon.jsonrpc_help(command='status') self.assertSubstring('daemon status', result['help']) if is_android(): test_help.skip = "Test cannot pass on Android because PYTHONOPTIMIZE removes the docstrings." @unittest.SkipTest class TestFileListSorting(unittest.TestCase): def setUp(self): test_utils.reset_time(self) self.test_daemon = get_test_daemon(Config()) self.test_daemon.file_manager.lbry_files = self._get_fake_lbry_files() self.test_points_paid = [ 2.5, 4.8, 5.9, 5.9, 5.9, 6.1, 7.1, 8.2, 8.4, 9.1 ] self.test_file_names = [ 'add.mp3', 'any.mov', 'day.tiff', 'decade.odt', 'different.json', 'hotel.bmp', 'might.bmp', 'physical.json', 'remember.mp3', 'than.ppt' ] self.test_authors = [ 'ashlee27', 'bfrederick', 'brittanyhicks', 'davidsonjeffrey', 'heidiherring', 'jlewis', 'kswanson', 'michelle50', 'richard64', 'xsteele' ] return f2d(self.test_daemon.component_manager.start()) def test_sort_by_points_paid_no_direction_specified(self): sort_options = ['points_paid'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(self.test_points_paid, [f['points_paid'] for f in file_list]) def test_sort_by_points_paid_ascending(self): sort_options = ['points_paid,asc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(self.test_points_paid, [f['points_paid'] for f in file_list]) def test_sort_by_points_paid_descending(self): sort_options = ['points_paid, desc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(list(reversed(self.test_points_paid)), [f['points_paid'] for f in file_list]) def test_sort_by_file_name_no_direction_specified(self): sort_options = ['file_name'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(self.test_file_names, [f['file_name'] for f in file_list]) def test_sort_by_file_name_ascending(self): sort_options = ['file_name,\nasc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(self.test_file_names, [f['file_name'] for f in file_list]) def test_sort_by_file_name_descending(self): sort_options = ['\tfile_name,\n\tdesc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(list(reversed(self.test_file_names)), [f['file_name'] for f in file_list]) def test_sort_by_multiple_criteria(self): expected = [ 'file_name=different.json, points_paid=9.1', 'file_name=physical.json, points_paid=8.4', 'file_name=any.mov, points_paid=8.2', 'file_name=hotel.bmp, points_paid=7.1', 'file_name=add.mp3, points_paid=6.1', 'file_name=decade.odt, points_paid=5.9', 'file_name=might.bmp, points_paid=5.9', 'file_name=than.ppt, points_paid=5.9', 'file_name=remember.mp3, points_paid=4.8', 'file_name=day.tiff, points_paid=2.5' ] format_result = lambda f: f"file_name={f['file_name']}, points_paid={f['points_paid']}" sort_options = ['file_name,asc', 'points_paid,desc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(expected, [format_result(r) for r in file_list]) # Check that the list is not sorted as expected when sorted only by file_name. sort_options = ['file_name,asc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertNotEqual(expected, [format_result(r) for r in file_list]) # Check that the list is not sorted as expected when sorted only by points_paid. sort_options = ['points_paid,desc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertNotEqual(expected, [format_result(r) for r in file_list]) # Check that the list is not sorted as expected when not sorted at all. file_list = yield f2d(self.test_daemon.jsonrpc_file_list()['items']) self.assertNotEqual(expected, [format_result(r) for r in file_list]) def test_sort_by_nested_field(self): extract_authors = lambda file_list: [f['metadata']['author'] for f in file_list] sort_options = ['metadata.author'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(self.test_authors, extract_authors(file_list)) # Check that the list matches the expected in reverse when sorting in descending order. sort_options = ['metadata.author,desc'] file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) self.assertEqual(list(reversed(self.test_authors)), extract_authors(file_list)) # Check that the list is not sorted as expected when not sorted at all. file_list = yield f2d(self.test_daemon.jsonrpc_file_list()['items']) self.assertNotEqual(self.test_authors, extract_authors(file_list)) def test_invalid_sort_produces_meaningful_errors(self): sort_options = ['meta.author'] expected_message = "Failed to get 'meta.author', key 'meta' was not found." with self.assertRaisesRegex(Exception, expected_message): yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) sort_options = ['metadata.foo.bar'] expected_message = "Failed to get 'metadata.foo.bar', key 'foo' was not found." with self.assertRaisesRegex(Exception, expected_message): yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options)['items']) @staticmethod def _get_fake_lbry_files(): faked_lbry_files = [] for metadata in FAKED_LBRY_FILES: lbry_file = mock.Mock(spec=ManagedEncryptedFileDownloader) for attribute in metadata: setattr(lbry_file, attribute, metadata[attribute]) async def get_total_bytes(): return 0 lbry_file.get_total_bytes = get_total_bytes async def status(): return EncryptedFileStatusReport( 'file_name', 1, 1, 'completed' ) lbry_file.status = status faked_lbry_files.append(lbry_file) return faked_lbry_files FAKED_LBRY_FILES = ( { 'channel_claim_id': '3aace03b007d108c668d201533b7b07ab2981d47', 'channel_name': '@ashlee27', 'claim_id': 'cb63e644b6629467c031d0097d52ab6e0f1a5bf8', 'claim_name': 'very-skill-place-growth', 'completed': True, 'download_directory': '/usually', 'download_path': '/usually/any.mov', 'file_name': 'any.mov', 'key': b'>a\x11}\xec\xc2j\x1c\xe9\xc5l]\xfc\x16s|', 'metadata': {'author': 'ashlee27', 'nsfw': True}, 'mime_type': 'multipart/signed', 'nout': 7197, 'outpoint': 'c5633a5932f9c8e3e5b9799c07251b236e3aec078b0546614f24a932b6b133f6', 'points_paid': 8.2, 'sd_hash': '3354ecf502870f6f6d59d21188755c1361c2cffaeb458764c179c136b26c4795083acfd93b3920218870b1a9c22535ef', 'stopped': True, 'stream_hash': 'c8f58a686726116c15a8de7f33b4f0d72504777c7dd0c48ba94d7bbea23d9c82b2f977081cd7c49d25d6a2841b232e1d', 'stream_name': 'down.txt', 'suggested_file_name': 'down.txt', 'txid': '1c02986bfdb77b1c338e60b60c4c9febc59130af2e225f51665067c3d3419a35', 'written_bytes': 6838, }, { 'channel_claim_id': 'dade35ea84001858d7cf10f50be3b5fea3e57fb7', 'channel_name': '@richard64', 'claim_id': '1c01096727da90140d333197fa8aaf88893f6ea8', 'claim_name': 'room-tonight-produce-good', 'completed': False, 'download_directory': '/ability', 'download_path': '/ability/day.tiff', 'file_name': 'day.tiff', 'key': b'`\x86j\xba\x97\x0c\xe4L\xad\x06nC\x8b]\xd6&', 'metadata': {'author': 'richard64', 'nsfw': False}, 'mime_type': 'multipart/related', 'nout': 9678, 'outpoint': '0138083012ce4ff5a6e4c0ec2fc08e11a52ebd2306d70a20f36424011f7c1330', 'points_paid': 2.5, 'sd_hash': '9e98f5e4bd4393b45a41839fe72a4df1f94b029e2339f8b14b9eaa9fec2be5245b160a59cfcc80fa86c1f91da67d5581', 'stopped': False, 'stream_hash': '7403ff9319292bdb022d66d0b88401775c6bb355fc0a75fe01452950fb19642ba58d492d34e24d8bf58375e6c2fca16f', 'stream_name': 'including.mp3', 'suggested_file_name': 'including.mp3', 'txid': '345bfc7b4a0c042b14474ac2cecf099236aec5a6730943630ffb176e7b421121', 'written_bytes': 8438, }, { 'channel_claim_id': '59766071ea2df38b4a751834a77246bf8ff8071d', 'channel_name': '@bfrederick', 'claim_id': '1c08967d515ac2a5fd3cb4e477fde18bddde22e2', 'claim_name': 'at-first-skill-agency', 'completed': True, 'download_directory': '/agree', 'download_path': '/agree/different.json', 'file_name': 'different.json', 'key': b'\xc06\xb9\x8e\x00S\xdbX\x1cC\xbd+\xfc\xea\xc96', 'metadata': {'author': 'bfrederick', 'nsfw': True}, 'mime_type': 'image/svg+xml', 'nout': 2975, 'outpoint': '62bb992e11c562e8064aec094e2b4eefb422e50b13ae49dc10f440a60f8e99bc', 'points_paid': 9.1, 'sd_hash': '3d809caf1266ec1ab78cc046a62f388434b6c59f85844500f59a1c75b4303b9ea27c532a231556e8b9776c544677bdf7', 'stopped': True, 'stream_hash': '9ecb8cf7dca7260f90666f05c88882017c786d31b572f3cba9447099ca9b49cdcb93f801db2249b7d32ff44ca6ffd69c', 'stream_name': 'north.doc', 'suggested_file_name': 'north.doc', 'txid': '98e12bce3a5db96e3513f1ff45afca8b69d61324117d6e839075ac512dd86251', 'written_bytes': 1929, }, { 'channel_claim_id': '046c8a762cd158a6e5b112d7d9c9e4b778f27388', 'channel_name': '@heidiherring', 'claim_id': '3d7573264601af7b5402cd54a66c13f2f93f296f', 'claim_name': 'drop-hot-military-drive', 'completed': False, 'download_directory': '/letter', 'download_path': '/letter/than.ppt', 'file_name': 'than.ppt', 'key': b'\n\xa7\xb3\x05\xab\x8e\xcc\n\xdcn\xd9\x81\xf3m\xf1t', 'metadata': {'author': 'heidiherring', 'nsfw': True}, 'mime_type': 'text/javascript', 'nout': 3452, 'outpoint': '5e8b55ffe59774366804c384f632f728f769bad566c784c6a23f1081724d00ea', 'points_paid': 5.9, 'sd_hash': 'c945b0acaf1c97dd7f262cf73cc3813ab6552f695ab00a445ca07a2a4da43bf3bb020cc7e338b484405d0865ef836480', 'stopped': False, 'stream_hash': '692ea08506d13422c875ce9d49fb9fe90b828d259d46c53adffa68a045e3852d4763b90ef94ef3864a1164bcab7eefa0', 'stream_name': 'few.css', 'suggested_file_name': 'few.css', 'txid': '9ef901facbf8bc133cfe67793fe4423c048753dab8370d987f6f552ed6483bbb', 'written_bytes': 9498, }, { 'channel_claim_id': 'adc87fa84d601aa1760d0f4585f02e60bc82c703', 'channel_name': '@xsteele', 'claim_id': '9022e1fd14646f6a4f6708566fbb6f6ac10ba3d5', 'claim_name': 'mean-television-miss-yourself', 'completed': True, 'download_directory': '/its', 'download_path': '/its/add.mp3', 'file_name': 'add.mp3', 'key': b'\xdc\xd1\xf1i!\x85\xc6\xc8\\\xe0\xd7\xc0\xceN<l', 'metadata': {'author': 'xsteele', 'nsfw': False}, 'mime_type': 'message/imdn+xml', 'nout': 7924, 'outpoint': '4fd8a071fd00050006a666de076595d7a61e04dac0ce8bf9ef90024d2415ba30', 'points_paid': 6.1, 'sd_hash': 'ac28c50337bd16b4a753a2ae6bdad25cbb93270b83f593ec238b8237ce4ddf30aff676eb7025211d970ab5a5cca204c7', 'stopped': True, 'stream_hash': 'a0a9aa762fb6599f94e7098d70cc14ced34d08c210863d62b6d1e9c7eb523d98d0d8e3c80ddc03ce68b51f99e88024b5', 'stream_name': 'picture.csv', 'suggested_file_name': 'picture.csv', 'txid': 'b4df85f9be396f2d9a3b9172f826be9899608d827cabc5196863ec27d3a32f82', 'written_bytes': 9220, }, { 'channel_claim_id': '79d17bbcb93b31c20fe395190dc199d871268ef1', 'channel_name': '@michelle50', 'claim_id': 'daf4c15cd3da305e7b29b0028cf801c61bd67e30', 'claim_name': 'card-oil-since-take', 'completed': True, 'download_directory': '/lawyer', 'download_path': '/lawyer/hotel.bmp', 'file_name': 'hotel.bmp', 'key': b'\xda<-\x11-\xbb\xe3u\x80\xffX\x01N\xfc\x01)', 'metadata': {'author': 'michelle50', 'nsfw': True}, 'mime_type': 'image/png', 'nout': 5576, 'outpoint': '27a62194fbf658327899431ecf251866bd0eec4da24d0b3feb8c440c4ea3ac1a', 'points_paid': 7.1, 'sd_hash': '20e4fc4513d7ea6f270e2021f7057c78e175754561e7db94e10c41d6d74ca639bb3c4d6e38dc2817ce303629c901d1a2', 'stopped': False, 'stream_hash': '781785e554fadb275ba75ee58cc5db0063f4d9cf2a1f1c4053a586ddce20089197d42e6640398cec75c8623a7c38ae0b', 'stream_name': 'paper.jpeg', 'suggested_file_name': 'paper.jpeg', 'txid': '8647c42f762694237804eeed4cbde776490a4f3b8b293ca41f550488098f883e', 'written_bytes': 7382, }, { 'channel_claim_id': '7e4cee485713909665c21246ba22159e0a20a820', 'channel_name': '@jlewis', 'claim_id': '7bc402e4bc6a8b1c1aa2184e8e082eb1d0353db3', 'claim_name': 'heavy-street-meeting-and', 'completed': True, 'download_directory': '/personal', 'download_path': '/personal/remember.mp3', 'file_name': 'remember.mp3', 'key': b'\x1c\x9d%\x1e\xe4\xab\xb9\x0c\xac<\x86\xc7P;\xfdO', 'metadata': {'author': 'jlewis', 'nsfw': True}, 'mime_type': 'video/x-matroska', 'nout': 8116, 'outpoint': 'e2da970db0edb37680519a58de53dc088e6d26d5c4a37ae37d5c0f1901f30197', 'points_paid': 4.8, 'sd_hash': 'f42913f4bfba90f157b4744b55e4043d79d2d658dc08aa306b34a3ae4a1c1c37759fd9f0b4b8181f539bd60373746954', 'stopped': True, 'stream_hash': '4051a577422fe2b444c9c572a0a1b3f731e0ed2e5eb3b9a3aaa4ce1b0ec694cd7786e224c94a126fc9a868f1b93cb2e1', 'stream_name': 'feel.html', 'suggested_file_name': 'feel.html', 'txid': 'c88e0c86ebbca20d0dafed682711a0b2e02e80637515c25eb26f2a589385bfe2', 'written_bytes': 9337, }, { 'channel_claim_id': '13aa3c28c0c8bb08a679a010f63bd3f4b5234e73', 'channel_name': '@brittanyhicks', 'claim_id': '3c9c02bf1bfcedb2654f9003c464df9059a8e6b0', 'claim_name': 'cold-music-admit-technology', 'completed': True, 'download_directory': '/nor', 'download_path': '/nor/might.bmp', 'file_name': 'might.bmp', 'key': b'\rh\xb3jqR\\\xdb\xb9\xa0a\xa4J\xa4\xacs', 'metadata': {'author': 'brittanyhicks', 'nsfw': True}, 'mime_type': 'application/xop+xml', 'nout': 8338, 'outpoint': 'a68a9dfa301292e1d0fe60a9bb0bcefa3e4e26630269064c8d2dd0f578427a10', 'points_paid': 5.9, 'sd_hash': '48fef5178b93b542495d19d76407692802ab529d989539b203a1cb38ce35ec2d4e9ea7d31eac660f715c39b69cd574ec', 'stopped': True, 'stream_hash': '2052889a9447ea73d743ec2c8c71678bf60616c01cd05d0a4d34a1aa92ee334585771a28f42cfe1b4124645352325946', 'stream_name': 'shoulder.js', 'suggested_file_name': 'shoulder.js', 'txid': 'b3f5f9db4c40157f348b9cf7dcb4ae3c53fe5e43481a4b66b2cc2334ae5ad2cb', 'written_bytes': 9736, }, { 'channel_claim_id': 'a18b45f2131fd79fea6bb493d94349c9734ef211', 'channel_name': '@kswanson', 'claim_id': '1223727010f0b4b9f6f45ca95cad0bfb3ce759a0', 'claim_name': 'often-speech-provide-run', 'completed': True, 'download_directory': '/member', 'download_path': '/member/physical.json', 'file_name': 'physical.json', 'key': b"'\xa7\x9b!\t\x86\xe2q\x15S\x9c\x92S@7;", 'metadata': {'author': 'kswanson', 'nsfw': True}, 'mime_type': 'multipart/form-data', 'nout': 7028, 'outpoint': '818a4265723d7682cff4cc89d9b3433af48636ba42d2ca1e65eef8b7f9bef0ad', 'points_paid': 8.4, 'sd_hash': 'c808d997ff914c4986e420c4b2547ab030082da28ffebe2a0844ad6325c9f276fad5a003b18dcd015397e41b71d172e2', 'stopped': False, 'stream_hash': '2d457dda5ed01009b3812ff60bd24cbc2a0cb1361f566433d71dbce7d757977deac7f5aca62a60ec63eaa1b401194da5', 'stream_name': 'country.avi', 'suggested_file_name': 'country.avi', 'txid': '42db2b952c578afcb8f640c2a12e563ba1a31b18fc8357d2f04f5de6c8515fba', 'written_bytes': 9688, }, { 'channel_claim_id': '84a06ce77cde8ed1511e268fcfdebd8feb1333e2', 'channel_name': '@davidsonjeffrey', 'claim_id': '8fb403f0bb0695530935a0991a7eb7218c46eed9', 'claim_name': 'option-company-glass-this', 'completed': True, 'download_directory': '/environment', 'download_path': '/environment/decade.odt', 'file_name': 'decade.odt', 'key': b')\xa3h\x12\xf2\xd5RPkWojN\x08%\x0e', 'metadata': {'author': 'davidsonjeffrey', 'nsfw': True}, 'mime_type': 'video/webm', 'nout': 8810, 'outpoint': '04da67fe9c6d129812e16045c02f1f670d3e329e7a9c0872712aaa74876becdd', 'points_paid': 5.9, 'sd_hash': '394eb1e0caf0d7dbeb80d435631534dc716229fac035aebe2af1729af5cbbad1c4fa503ce7fa7cc01e5366d1ce9d9d07', 'stopped': False, 'stream_hash': '1904d27ab8c784b7ae770980f004522e36089e86e3ce95d3005c3829cf4ad1571c5fe248a3f67a54521e07290a9e7466', 'stream_name': 'score.wav', 'suggested_file_name': 'score.wav', 'txid': 'd2f8ecfac4491e1de186b43a5e561413304769a1683612a16633dd3e725ff1e0', 'written_bytes': 7929, }, ) ================================================ FILE: tests/unit/lbrynet_daemon/test_allowed_origin.py ================================================ import unittest from aiohttp import ClientSession from aiohttp.test_utils import make_mocked_request as request from aiohttp.web import HTTPForbidden from lbry.testcase import AsyncioTestCase from lbry.conf import Config from lbry.extras.daemon.security import is_request_allowed as allowed, ensure_request_allowed as ensure from lbry.extras.daemon.components import ( DATABASE_COMPONENT, DISK_SPACE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT, LIBTORRENT_COMPONENT, BACKGROUND_DOWNLOADER_COMPONENT, TRACKER_ANNOUNCER_COMPONENT ) from lbry.extras.daemon.daemon import Daemon class TestAllowedOrigin(unittest.TestCase): def test_allowed_origin_default(self): conf = Config() # lack of Origin is always allowed self.assertTrue(allowed(request('GET', '/'), conf)) # deny all other Origins self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf)) self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf)) self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf)) def test_allowed_origin_star(self): conf = Config(allowed_origin='*') # everything is allowed self.assertTrue(allowed(request('GET', '/'), conf)) self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'null'}), conf)) self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf)) self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf)) def test_allowed_origin_specified(self): conf = Config(allowed_origin='localhost') # no origin and only localhost are allowed self.assertTrue(allowed(request('GET', '/'), conf)) self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf)) self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf)) self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf)) def test_ensure_default(self): conf = Config() ensure(request('GET', '/'), conf) with self.assertLogs() as log: with self.assertRaises(HTTPForbidden): ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf) self.assertIn("'localhost' are not allowed", log.output[0]) def test_ensure_specific(self): conf = Config(allowed_origin='localhost') ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf) with self.assertLogs() as log: with self.assertRaises(HTTPForbidden): ensure(request('GET', '/', headers={'Origin': 'hackers.com'}), conf) self.assertIn("'hackers.com' are not allowed", log.output[0]) self.assertIn("'allowed_origin' limits requests to: 'localhost'", log.output[0]) class TestAccessHeaders(AsyncioTestCase): async def asyncSetUp(self): conf = Config(allowed_origin='localhost') conf.data_dir = '/tmp' conf.share_usage_data = False conf.api = 'localhost:5299' conf.components_to_skip = ( DATABASE_COMPONENT, DISK_SPACE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT, LIBTORRENT_COMPONENT, BACKGROUND_DOWNLOADER_COMPONENT, TRACKER_ANNOUNCER_COMPONENT ) Daemon.component_attributes = {} self.daemon = Daemon(conf) await self.daemon.start() self.addCleanup(self.daemon.stop) async def test_headers(self): async with ClientSession() as session: # OPTIONS async with session.options('http://localhost:5299') as resp: self.assertEqual(resp.headers['Access-Control-Allow-Origin'], 'localhost') self.assertEqual(resp.headers['Access-Control-Allow-Methods'], 'localhost') self.assertEqual(resp.headers['Access-Control-Allow-Headers'], 'localhost') # GET status = {'method': 'status', 'params': []} async with session.get('http://localhost:5299/lbryapi', json=status) as resp: self.assertEqual(resp.headers['Access-Control-Allow-Origin'], 'localhost') self.assertEqual(resp.headers['Access-Control-Allow-Methods'], 'localhost') self.assertEqual(resp.headers['Access-Control-Allow-Headers'], 'localhost') ================================================ FILE: tests/unit/lbrynet_daemon/test_exchange_rate_manager.py ================================================ import asyncio from decimal import Decimal from time import time from lbry.schema.claim import Claim from lbry.extras.daemon.exchange_rate_manager import ( ExchangeRate, ExchangeRateManager, CurrencyConversionError, BittrexUSDFeed, BittrexBTCFeed, CoinExBTCFeed ) from lbry.testcase import AsyncioTestCase, FakeExchangeRateManager, get_fake_exchange_rate_manager from lbry.error import InvalidExchangeRateResponseError class ExchangeRateTests(AsyncioTestCase): def test_invalid_rates(self): with self.assertRaises(ValueError): ExchangeRate('USDBTC', 0, time()) with self.assertRaises(ValueError): ExchangeRate('USDBTC', -1, time()) def test_fee_converts_to_lbc(self): fee = Claim().stream.fee fee.usd = Decimal(10.0) fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" manager = get_fake_exchange_rate_manager() result = manager.convert_currency(fee.currency, "LBC", fee.amount) self.assertEqual(20.0, result) def test_missing_feed(self): fee = Claim().stream.fee fee.usd = Decimal(1.0) fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" manager = FakeExchangeRateManager([BittrexBTCFeed()], {'BTCLBC': 1.0}) with self.assertRaises(CurrencyConversionError): manager.convert_currency(fee.currency, "LBC", fee.amount) def test_bittrex_feed_response(self): feed = BittrexBTCFeed() out = feed.get_rate_from_response({ "symbol": "LBC-BTC", "lastTradeRate": "0.00000323", "bidRate": "0.00000322", "askRate": "0.00000327" }) self.assertEqual(1.0 / 0.00000323, out) with self.assertRaises(InvalidExchangeRateResponseError): feed.get_rate_from_response({}) with self.assertRaises(InvalidExchangeRateResponseError): feed.get_rate_from_response({ "success": True, "result": [] }) class BadMarketFeed(BittrexUSDFeed): def get_response(self): raise InvalidExchangeRateResponseError(self.name, 'bad stuff') class ExchangeRateManagerTests(AsyncioTestCase): async def test_get_rate_failure_retrieved(self): manager = ExchangeRateManager([BadMarketFeed]) manager.start() await manager.wait() for feed in manager.market_feeds: # no rate but it tried self.assertFalse(feed.has_rate) self.assertTrue(feed.event.is_set()) self.addCleanup(manager.stop) async def test_median_rate_used(self): manager = ExchangeRateManager([BittrexBTCFeed, CoinExBTCFeed]) for feed in manager.market_feeds: feed.last_check = time() bittrex, coinex = manager.market_feeds bittrex.rate = ExchangeRate(bittrex.market, 1.0, time()) coinex.rate = ExchangeRate(coinex.market, 2.0, time()) coinex.rate = ExchangeRate(coinex.market, 3.0, time()) self.assertEqual(14.0, manager.convert_currency("BTC", "LBC", Decimal(7.0))) coinex.rate.spot = 4.0 self.assertEqual(17.5, manager.convert_currency("BTC", "LBC", Decimal(7.0))) ================================================ FILE: tests/unit/lbrynet_daemon/test_mime_types.py ================================================ import unittest from lbry.schema import mime_types class TestMimeTypes(unittest.TestCase): def test_mp4_video(self): self.assertEqual("video/mp4", mime_types.guess_media_type("test.mp4")[0]) self.assertEqual("video/mp4", mime_types.guess_media_type("test.MP4")[0]) def test_x_ext_(self): self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.lbry")[0]) self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.LBRY")[0]) def test_octet_stream(self): self.assertEqual("application/octet-stream", mime_types.guess_media_type("test.")[0]) self.assertEqual("application/octet-stream", mime_types.guess_media_type("test")[0]) ================================================ FILE: tests/unit/schema/__init__.py ================================================ ================================================ FILE: tests/unit/schema/test_claim_from_bytes.py ================================================ from unittest import TestCase from binascii import unhexlify from lbry.schema import Claim class TestOldJSONSchemaCompatibility(TestCase): def test_old_json_schema_v1(self): claim = Claim.from_bytes( b'{"fee": {"LBC": {"amount": 1.0, "address": "bPwGA9h7uijoy5uAvzVPQw9QyLoYZehHJo"}}, "d' b'escription": "10MB test file to measure download speed on Lbry p2p-network.", "licens' b'e": "None", "author": "root", "language": "English", "title": "10MB speed test file",' b' "sources": {"lbry_sd_hash": "bbd1f68374ff9a1044a90d7dd578ce41979211c386caf19e6f49653' b'6db5f2c96b58fe2c7a6677b331419a117873b539f"}, "content-type": "application/octet-strea' b'm", "thumbnail": "/home/robert/lbry/speed.jpg"}' ) stream = claim.stream self.assertEqual(stream.title, '10MB speed test file') self.assertEqual(stream.description, '10MB test file to measure download speed on Lbry p2p-network.') self.assertEqual(stream.license, 'None') self.assertEqual(stream.author, 'root') self.assertEqual(stream.langtags, ['en']) self.assertEqual(stream.source.media_type, 'application/octet-stream') self.assertEqual(stream.thumbnail.url, '/home/robert/lbry/speed.jpg') self.assertEqual( stream.source.sd_hash, 'bbd1f68374ff9a1044a90d7dd578ce41979211c386caf19e' '6f496536db5f2c96b58fe2c7a6677b331419a117873b539f' ) self.assertEqual(stream.fee.address, 'bPwGA9h7uijoy5uAvzVPQw9QyLoYZehHJo') self.assertEqual(stream.fee.lbc, 1) self.assertEqual(stream.fee.dewies, 100000000) self.assertEqual(stream.fee.currency, 'LBC') with self.assertRaisesRegex(ValueError, 'USD can only be returned for USD fees.'): print(stream.fee.usd) def test_old_json_schema_v2(self): claim = Claim.from_bytes( b'{"license": "Creative Commons Attribution 3.0 United States", "fee": {"LBC": {"amount' b'": 10, "address": "bFro33qBKxnL1AsjUU9N4AQHp9V62Nhc5L"}}, "ver": "0.0.2", "descriptio' b'n": "Force P0 State for Nividia Cards! (max mining performance)", "language": "en", "' b'author": "Mii", "title": "Nividia P0", "sources": {"lbry_sd_hash": "c5ffee0fa5168e166' b'81b519d9d85446e8d1d818a616bd55540aa7374d2321b51abf2ac3dae1443a03dadcc8f7affaa62"}, "n' b'sfw": false, "license_url": "https://creativecommons.org/licenses/by/3.0/us/legalcode' b'", "content-type": "application/x-msdownload"}' ) stream = claim.stream self.assertEqual(stream.title, 'Nividia P0') self.assertEqual(stream.description, 'Force P0 State for Nividia Cards! (max mining performance)') self.assertEqual(stream.license, 'Creative Commons Attribution 3.0 United States') self.assertEqual(stream.license_url, 'https://creativecommons.org/licenses/by/3.0/us/legalcode') self.assertEqual(stream.author, 'Mii') self.assertEqual(stream.langtags, ['en']) self.assertEqual(stream.source.media_type, 'application/x-msdownload') self.assertEqual( stream.source.sd_hash, 'c5ffee0fa5168e16681b519d9d85446e8d1d818a616bd555' '40aa7374d2321b51abf2ac3dae1443a03dadcc8f7affaa62' ) self.assertEqual(stream.fee.address, 'bFro33qBKxnL1AsjUU9N4AQHp9V62Nhc5L') self.assertEqual(stream.fee.lbc, 10) self.assertEqual(stream.fee.dewies, 1000000000) self.assertEqual(stream.fee.currency, 'LBC') with self.assertRaisesRegex(ValueError, 'USD can only be returned for USD fees.'): print(stream.fee.usd) def test_old_json_schema_v3(self): claim = Claim.from_bytes( b'{"ver": "0.0.3", "description": "asd", "license": "Creative Commons Attribution 4.0 I' b'nternational", "author": "sgb", "title": "ads", "language": "en", "sources": {"lbry_s' b'd_hash": "d83db664c6d7d570aa824300f4869e0bfb560e765efa477aebf566467f8d3a57f4f8c704cab' b'1308eb75ff8b7e84e3caf"}, "content_type": "video/mp4", "nsfw": false}' ) stream = claim.stream self.assertEqual(stream.title, 'ads') self.assertEqual(stream.description, 'asd') self.assertEqual(stream.license, 'Creative Commons Attribution 4.0 International') self.assertEqual(stream.author, 'sgb') self.assertEqual(stream.langtags, ['en']) self.assertEqual(stream.source.media_type, 'video/mp4') self.assertEqual( stream.source.sd_hash, 'd83db664c6d7d570aa824300f4869e0bfb560e765efa477a' 'ebf566467f8d3a57f4f8c704cab1308eb75ff8b7e84e3caf' ) class TestTypesV1Compatibility(TestCase): def test_signed_claim_made_by_ytsync(self): claim = Claim.from_bytes(unhexlify( b'080110011aee04080112a604080410011a2b4865726520617265203520526561736f6e73204920e29da4e' b'fb88f204e657874636c6f7564207c20544c4722920346696e64206f7574206d6f72652061626f7574204e' b'657874636c6f75643a2068747470733a2f2f6e657874636c6f75642e636f6d2f0a0a596f752063616e206' b'6696e64206d65206f6e20746865736520736f6369616c733a0a202a20466f72756d733a2068747470733a' b'2f2f666f72756d2e6865617679656c656d656e742e696f2f0a202a20506f64636173743a2068747470733' b'a2f2f6f6666746f706963616c2e6e65740a202a2050617472656f6e3a2068747470733a2f2f7061747265' b'6f6e2e636f6d2f7468656c696e757867616d65720a202a204d657263683a2068747470733a2f2f7465657' b'37072696e672e636f6d2f73746f7265732f6f6666696369616c2d6c696e75782d67616d65720a202a2054' b'77697463683a2068747470733a2f2f7477697463682e74762f786f6e64616b0a202a20547769747465723' b'a2068747470733a2f2f747769747465722e636f6d2f7468656c696e757867616d65720a0a2e2e2e0a6874' b'7470733a2f2f7777772e796f75747562652e636f6d2f77617463683f763d4672546442434f535f66632a0' b'f546865204c696e75782047616d6572321c436f7079726967687465642028636f6e746163742061757468' b'6f722938004a2968747470733a2f2f6265726b2e6e696e6a612f7468756d626e61696c732f46725464424' b'34f535f666352005a001a41080110011a30040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc22' b'f0bff70c4fe0b91fd36da9a375e3e1c171db825bf5d1f32209766964656f2f6d70342a5c080110031a406' b'2b2dd4c45e364030fbfad1a6fefff695ebf20ea33a5381b947753e2a0ca359989a5cc7d15e5392a0d354c' b'0b68498382b2701b22c03beb8dcb91089031b871e72214feb61536c007cdf4faeeaab4876cb397feaf6b51' )) stream = claim.stream self.assertEqual(stream.title, 'Here are 5 Reasons I ❤️ Nextcloud | TLG') self.assertEqual( stream.description, 'Find out more about Nextcloud: https://nextcloud.com/\n\nYou can find me on these soci' 'als:\n * Forums: https://forum.heavyelement.io/\n * Podcast: https://offtopical.net\n ' '* Patreon: https://patreon.com/thelinuxgamer\n * Merch: https://teespring.com/stores/o' 'fficial-linux-gamer\n * Twitch: https://twitch.tv/xondak\n * Twitter: https://twitter.' 'com/thelinuxgamer\n\n...\nhttps://www.youtube.com/watch?v=FrTdBCOS_fc' ) self.assertEqual(stream.license, 'Copyrighted (contact author)') self.assertEqual(stream.author, 'The Linux Gamer') self.assertEqual(stream.langtags, ['en']) self.assertEqual(stream.source.media_type, 'video/mp4') self.assertEqual(stream.thumbnail.url, 'https://berk.ninja/thumbnails/FrTdBCOS_fc') self.assertEqual( stream.source.sd_hash, '040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc' '22f0bff70c4fe0b91fd36da9a375e3e1c171db825bf5d1f3' ) # certificate for above channel cert = Claim.from_bytes(unhexlify( b'08011002225e0801100322583056301006072a8648ce3d020106052b8104000a034200043878b1edd4a13' b'73149909ef03f4339f6da9c2bd2214c040fd2e530463ffe66098eca14fc70b50ff3aefd106049a815f595' b'ed5a13eda7419ad78d9ed7ae473f17' )) channel = cert.channel self.assertEqual( channel.public_key, '033878b1edd4a1373149909ef03f4339f6da9c2bd2214c040fd2e530463ffe6609' ) def test_unsigned_with_fee(self): claim = Claim.from_bytes(unhexlify( b'080110011ad6010801127c080410011a08727067206d69646922046d6964692a08727067206d696469322' b'e437265617469766520436f6d6d6f6e73204174747269627574696f6e20342e3020496e7465726e617469' b'6f6e616c38004224080110011a19553f00bc139bbf40de425f94d51fffb34c1bea6d9171cd374c2500007' b'0414a0052005a001a54080110011a301f41eb0312aa7e8a5ce49349bc77d811da975833719d751523b19f' b'123fc3d528d6a94e3446ccddb7b9329f27a9cad7e3221c6170706c69636174696f6e2f782d7a69702d636' b'f6d70726573736564' )) stream = claim.stream self.assertEqual(stream.title, 'rpg midi') self.assertEqual(stream.description, 'midi') self.assertEqual(stream.license, 'Creative Commons Attribution 4.0 International') self.assertEqual(stream.author, 'rpg midi') self.assertEqual(stream.langtags, ['en']) self.assertEqual(stream.source.media_type, 'application/x-zip-compressed') self.assertEqual( stream.source.sd_hash, '1f41eb0312aa7e8a5ce49349bc77d811da975833719d7515' '23b19f123fc3d528d6a94e3446ccddb7b9329f27a9cad7e3' ) self.assertEqual(stream.fee.address, 'bJUQ9MxS9N6M29zsA5GTpVSDzsnPjMBBX9') self.assertEqual(stream.fee.lbc, 15) self.assertEqual(stream.fee.dewies, 1500000000) self.assertEqual(stream.fee.currency, 'LBC') with self.assertRaisesRegex(ValueError, 'USD can only be returned for USD fees.'): print(stream.fee.usd) ================================================ FILE: tests/unit/schema/test_mime_types.py ================================================ import unittest import tempfile import os from lbry.schema.mime_types import guess_media_type class MediaTypeTests(unittest.TestCase): def test_guess_media_type_from_path_only(self): kind = guess_media_type('/tmp/test.mkv') self.assertEqual(kind, ('video/x-matroska', 'video')) def test_defaults_for_no_extension(self): kind = guess_media_type('/tmp/test') self.assertEqual(kind, ('application/octet-stream', 'binary')) def test_defaults_for_unknown_extension(self): kind = guess_media_type('/tmp/test.unk') self.assertEqual(kind, ('application/x-ext-unk', 'binary')) def test_spoofed_unknown(self): with tempfile.TemporaryDirectory() as temp_dir: file = os.path.join(temp_dir, 'spoofed_unknown.txt') with open(file, 'wb') as fd: bytes_lz4 = bytearray([0x04,0x22,0x4d,0x18]) fd.write(bytes_lz4) fd.close() kind = guess_media_type(file) self.assertEqual(kind, ('application/x-ext-lz4', 'binary')) def test_spoofed_known(self): with tempfile.TemporaryDirectory() as temp_dir: file = os.path.join(temp_dir, 'spoofed_known.avi') with open(file, 'wb') as fd: bytes_zip = bytearray([0x50,0x4b,0x03,0x06]) fd.write(bytes_zip) fd.close() kind = guess_media_type(file) self.assertEqual(kind, ('application/zip', 'binary')) def test_spoofed_synonym(self): with tempfile.TemporaryDirectory() as temp_dir: file = os.path.join(temp_dir, 'spoofed_known.cbz') with open(file, 'wb') as fd: bytes_zip = bytearray([0x50,0x4b,0x03,0x06]) fd.write(bytes_zip) fd.close() kind = guess_media_type(file) self.assertEqual(kind, ('application/vnd.comicbook+zip', 'document')) ================================================ FILE: tests/unit/schema/test_models.py ================================================ from unittest import TestCase from decimal import Decimal from lbry.schema.claim import Claim, Stream, Collection from lbry.error import InputValueIsNoneError class TestClaimContainerAwareness(TestCase): def test_stream_claim(self): stream = Stream() claim = stream.claim self.assertEqual(claim.claim_type, Claim.STREAM) claim = Claim.from_bytes(claim.to_bytes()) self.assertEqual(claim.claim_type, Claim.STREAM) self.assertIsNotNone(claim.stream) with self.assertRaisesRegex(ValueError, 'Claim is not a channel.'): print(claim.channel) class TestFee(TestCase): def test_amount_setters(self): stream = Stream() stream.fee.lbc = Decimal('1.01') self.assertEqual(stream.fee.lbc, Decimal('1.01')) self.assertEqual(stream.fee.dewies, 101000000) self.assertEqual(stream.fee.currency, 'LBC') stream.fee.dewies = 203000000 self.assertEqual(stream.fee.lbc, Decimal('2.03')) self.assertEqual(stream.fee.dewies, 203000000) self.assertEqual(stream.fee.currency, 'LBC') with self.assertRaisesRegex(ValueError, 'USD can only be returned for USD fees.'): print(stream.fee.usd) with self.assertRaisesRegex(ValueError, 'Pennies can only be returned for USD fees.'): print(stream.fee.pennies) stream.fee.usd = Decimal('1.01') self.assertEqual(stream.fee.usd, Decimal('1.01')) self.assertEqual(stream.fee.pennies, 101) self.assertEqual(stream.fee.currency, 'USD') stream.fee.pennies = 203 self.assertEqual(stream.fee.usd, Decimal('2.03')) self.assertEqual(stream.fee.pennies, 203) self.assertEqual(stream.fee.currency, 'USD') with self.assertRaisesRegex(ValueError, 'LBC can only be returned for LBC fees.'): print(stream.fee.lbc) with self.assertRaisesRegex(ValueError, 'Dewies can only be returned for LBC fees.'): print(stream.fee.dewies) class TestLanguages(TestCase): def test_language_successful_parsing(self): stream = Stream() stream.languages.append('en') self.assertEqual(stream.languages[0].langtag, 'en') self.assertEqual(stream.languages[0].language, 'en') self.assertEqual(stream.langtags, ['en']) stream.languages.append('en-US') self.assertEqual(stream.languages[1].langtag, 'en-US') self.assertEqual(stream.languages[1].language, 'en') self.assertEqual(stream.languages[1].region, 'US') self.assertEqual(stream.langtags, ['en', 'en-US']) stream.languages.append('en-Latn-US') self.assertEqual(stream.languages[2].langtag, 'en-Latn-US') self.assertEqual(stream.languages[2].language, 'en') self.assertEqual(stream.languages[2].script, 'Latn') self.assertEqual(stream.languages[2].region, 'US') self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US']) stream.languages.append('es-419') self.assertEqual(stream.languages[3].langtag, 'es-419') self.assertEqual(stream.languages[3].language, 'es') self.assertIsNone(stream.languages[3].script) self.assertEqual(stream.languages[3].region, '419') self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US', 'es-419']) stream = Stream() stream.languages.extend(['en-Latn-US', 'es-ES', 'de-DE']) self.assertEqual(stream.languages[0].language, 'en') self.assertEqual(stream.languages[1].language, 'es') self.assertEqual(stream.languages[2].language, 'de') def test_language_error_parsing(self): stream = Stream() with self.assertRaisesRegex(ValueError, "Enum Language has no value defined for name 'zz'"): stream.languages.append('zz') with self.assertRaisesRegex(ValueError, "Enum Script has no value defined for name 'Zabc'"): stream.languages.append('en-Zabc') with self.assertRaisesRegex(ValueError, "Enum Country has no value defined for name 'ZZ'"): stream.languages.append('en-Zzzz-ZZ') with self.assertRaisesRegex(AssertionError, "Failed to parse language tag: en-Zzz-US"): stream.languages.append('en-Zzz-US') class TestTags(TestCase): def test_normalize_tags(self): claim = Claim() claim.channel.update(tags=['Anime', 'anime', ' aNiMe', 'maNGA ']) self.assertCountEqual(claim.channel.tags, ['anime', 'manga']) claim.channel.update(tags=['Juri', 'juRi']) self.assertCountEqual(claim.channel.tags, ['anime', 'manga', 'juri']) claim.channel.update(tags='Anime') self.assertCountEqual(claim.channel.tags, ['anime', 'manga', 'juri']) claim.channel.update(clear_tags=True) self.assertEqual(len(claim.channel.tags), 0) claim.channel.update(tags='Anime') self.assertEqual(claim.channel.tags, ['anime']) class TestCollection(TestCase): def test_collection(self): collection = Collection() collection.update(claims=['abc123', 'def123']) self.assertListEqual(collection.claims.ids, ['abc123', 'def123']) collection.update(claims=['abc123', 'bbb123']) self.assertListEqual(collection.claims.ids, ['abc123', 'def123', 'abc123', 'bbb123']) collection.update(clear_claims=True, claims=['bbb987', 'bb']) self.assertListEqual(collection.claims.ids, ['bbb987', 'bb']) self.assertEqual(collection.to_dict(), {'claims': ['bbb987', 'bb']}) collection.update(clear_claims=True) self.assertListEqual(collection.claims.ids, []) class TestLocations(TestCase): def test_location_successful_parsing(self): # from simple string stream = Stream() stream.locations.append('US') self.assertEqual(stream.locations[0].country, 'US') # from full string stream = Stream() stream.locations.append('US:NH:Manchester:03101:42.990605:-71.460989') self.assertEqual(stream.locations[0].country, 'US') self.assertEqual(stream.locations[0].state, 'NH') self.assertEqual(stream.locations[0].city, 'Manchester') self.assertEqual(stream.locations[0].code, '03101') self.assertEqual(stream.locations[0].latitude, '42.990605') self.assertEqual(stream.locations[0].longitude, '-71.460989') # from partial string stream = Stream() stream.locations.append('::Manchester:03101:') self.assertIsNone(stream.locations[0].country) self.assertEqual(stream.locations[0].state, '') self.assertEqual(stream.locations[0].city, 'Manchester') self.assertEqual(stream.locations[0].code, '03101') self.assertIsNone(stream.locations[0].latitude) self.assertIsNone(stream.locations[0].longitude) # from partial string lat/long stream = Stream() stream.locations.append('::::42.990605:-71.460989') self.assertIsNone(stream.locations[0].country) self.assertEqual(stream.locations[0].state, '') self.assertEqual(stream.locations[0].city, '') self.assertEqual(stream.locations[0].code, '') self.assertEqual(stream.locations[0].latitude, '42.990605') self.assertEqual(stream.locations[0].longitude, '-71.460989') # from short circuit lat/long stream = Stream() stream.locations.append('42.990605:-71.460989') self.assertIsNone(stream.locations[0].country) self.assertEqual(stream.locations[0].state, '') self.assertEqual(stream.locations[0].city, '') self.assertEqual(stream.locations[0].code, '') self.assertEqual(stream.locations[0].latitude, '42.990605') self.assertEqual(stream.locations[0].longitude, '-71.460989') # from json string stream = Stream() stream.locations.append('{"country": "ES"}') self.assertEqual(stream.locations[0].country, 'ES') # from dict stream = Stream() stream.locations.append({"country": "UA"}) self.assertEqual(stream.locations[0].country, 'UA') class TestStreamUpdating(TestCase): def test_stream_update(self): stream = Stream() # each of these values is set differently inside of .update() stream.update( title="foo", thumbnail_url="somescheme:some/path", file_name="file-name" ) self.assertEqual(stream.title, "foo") self.assertEqual(stream.thumbnail.url, "somescheme:some/path") self.assertEqual(stream.source.name, "file-name") with self.assertRaises(InputValueIsNoneError): stream.update(title=None) with self.assertRaises(InputValueIsNoneError): stream.update(file_name=None) with self.assertRaises(InputValueIsNoneError): stream.update(thumbnail_url=None) ================================================ FILE: tests/unit/schema/test_tags.py ================================================ import unittest from lbry.schema.tags import normalize_tag, clean_tags class TestTagNormalization(unittest.TestCase): def assertNormalizedTag(self, clean, dirty): self.assertEqual(clean, normalize_tag(dirty)) def test_normalize_tag(self): tag = self.assertNormalizedTag tag('', ' \t #!~') tag('tag', 'T\'ag') tag('t ag', '\tT \nAG ') tag('tag hash', '#tag~#hash!') def test_clean_tags(self): self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t'])) cleaned = clean_tags(['fOo', '!taG', 'FoO']) self.assertIn('tag', cleaned) self.assertIn('foo', cleaned) self.assertEqual(len(cleaned), 2) ================================================ FILE: tests/unit/schema/test_url.py ================================================ import unittest from lbry.schema.url import URL claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e" class TestURLParsing(unittest.TestCase): segments = 'stream', 'channel' fields = 'name', 'claim_id', 'amount_order' def _assert_url(self, url_string, strictly=True, **kwargs): url = URL.parse(url_string) if strictly: if url_string.startswith('lbry://'): self.assertEqual(url_string, str(url)) else: self.assertEqual(f'lbry://{url_string}', str(url)) present = {} for key in kwargs: for segment_name in self.segments: if key.startswith(segment_name): present[segment_name] = True break for segment_name in self.segments: segment = getattr(url, segment_name) if segment_name not in present: self.assertIsNone(segment) else: for field in self.fields: self.assertEqual( getattr(segment, field), kwargs.get(f'{segment_name}_{field}', None) ) def _fail_url(self, url): with self.assertRaisesRegex(ValueError, 'Invalid LBRY URL'): URL.parse(url) def test_parser_valid_urls(self): url = self._assert_url # stream url('test', stream_name='test') url('test*1', stream_name='test*1') url('test$1', stream_name='test', stream_amount_order='1') url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id, strictly=False) url(f'test:{claim_id}', stream_name='test', stream_claim_id=claim_id) # channel url('@test', channel_name='@test') url('@test$1', channel_name='@test', channel_amount_order='1') url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id, strictly=False) url(f'@test:{claim_id}', channel_name='@test', channel_claim_id=claim_id) # channel/stream url('lbry://@test/stuff', channel_name='@test', stream_name='stuff') url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff') url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff', strictly=False) url(f'lbry://@test:{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff') # combined legacy and new url('@test:1/stuff#2', channel_claim_id='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False) # unicode regex edges _url = lambda name: url(name, stream_name=name) _url('\uD799') _url('\uE000') _url('\uFFFD') def test_parser_invalid_urls(self): fail = self._fail_url fail("lbry://") fail("lbry://\u0000") fail("lbry://\u0008") fail("lbry://\u000b") fail("lbry://\u000c") fail("lbry://\u000e") fail("lbry://\u001f") fail("lbry://\uD800") fail("lbry://\uDFFF") fail("lbry://\uDFFE") fail("lbry://\uFFFF") fail("lbry://;") fail("lbry://no\ttab") fail("lbry://no space") fail("lbry://no\rcr") fail("lbry://no\new\nline") fail("lbry://\"") fail("lbry://\\") fail("lbry:///") fail("lbry://<") and fail("lbry://>") fail("lbry://{") and fail("lbry://}") fail("lbry://[") and fail("lbry://]") fail("lbry://%") fail("lbry://|") fail("lbry://^") fail("lbry://~") fail("lbry://`") fail("lbry://test:3$1") fail("lbry://test$1:1") fail("lbry://test#x") fail("lbry://test#x/page") fail("lbry://test$") fail("lbry://test#") fail("lbry://test:") fail("lbry://test$x") fail("lbry://test:x") fail("lbry://@test@") fail("lbry://@test:") fail("lbry://test@") fail("lbry://tes@t") fail(f"lbry://test:1#{claim_id}") fail("lbry://test$0") fail("lbry://test/path") fail("lbry://test:1:1:1") fail("whatever/lbry://test") fail("lbry://lbry://test") fail("lbry://@/what") fail("lbry://abc:0x123") fail("lbry://abc:0x123/page") fail("lbry://@test1#ABCDEF/fakepath") fail("lbry://@test1$1/fakepath?arg1&arg2&arg3") ================================================ FILE: tests/unit/stream/__init__.py ================================================ ================================================ FILE: tests/unit/stream/test_managed_stream.py ================================================ import os import shutil import unittest from unittest import mock import asyncio from lbry.blob.blob_file import MAX_BLOB_SIZE from lbry.blob_exchange.serialization import BlobResponse from lbry.blob_exchange.server import BlobServerProtocol from lbry.dht.node import Node from lbry.dht.peer import make_kademlia_peer from lbry.extras.daemon.storage import StoredContentClaim from lbry.schema import Claim from lbry.stream.managed_stream import ManagedStream from lbry.stream.descriptor import StreamDescriptor from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase class TestManagedStream(BlobExchangeTestBase): async def create_stream(self, blob_count: int = 10, file_name='test_file'): self.stream_bytes = b'' for _ in range(blob_count): self.stream_bytes += os.urandom(MAX_BLOB_SIZE - 1) # create the stream file_path = os.path.join(self.server_dir, file_name) with open(file_path, 'wb') as f: f.write(self.stream_bytes) descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path) descriptor.suggested_file_name = file_name descriptor.stream_hash = descriptor.get_stream_hash() self.sd_hash = descriptor.sd_hash = descriptor.calculate_sd_hash() await descriptor.make_sd_blob() return descriptor async def setup_stream(self, blob_count: int = 10): await self.create_stream(blob_count) self.stream = ManagedStream( self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir ) async def test_client_sanitizes_file_name(self): illegal_name = 't<?t_f:|<' descriptor = await self.create_stream(file_name=illegal_name) descriptor.suggested_file_name = illegal_name self.stream = ManagedStream( self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir ) await self._test_transfer_stream(10, skip_setup=True) self.assertTrue(self.stream.completed) self.assertEqual(self.stream.file_name, 'tt_f') self.assertTrue(self.stream.output_file_exists) self.assertTrue(os.path.isfile(self.stream.full_path)) self.assertEqual(self.stream.full_path, os.path.join(self.client_dir, 'tt_f')) self.assertTrue(os.path.isfile(os.path.join(self.client_dir, 'tt_f'))) async def test_empty_name_fallback(self): descriptor = await self.create_stream(file_name=" ") descriptor.suggested_file_name = " " claim = Claim() claim.stream.source.name = "cool.mp4" self.stream = ManagedStream( self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir, claim=StoredContentClaim(serialized=claim.to_bytes().hex()) ) await self._test_transfer_stream(10, skip_setup=True) self.assertTrue(self.stream.completed) self.assertEqual(self.stream.suggested_file_name, "cool.mp4") self.assertEqual(self.stream.stream_name, "cool.mp4") self.assertEqual(self.stream.mime_type, "video/mp4") async def test_status_file_completed(self): await self._test_transfer_stream(10) self.assertTrue(self.stream.output_file_exists) self.assertTrue(self.stream.completed) with open(self.stream.full_path, 'w+b') as outfile: outfile.truncate(1) self.assertTrue(self.stream.output_file_exists) self.assertFalse(self.stream.completed) async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True, skip_setup=False): if not skip_setup: await self.setup_stream(blob_count) mock_node = mock.Mock(spec=Node) def _mock_accumulate_peers(q1, q2): async def _task(): pass q2.put_nowait([self.server_from_client]) return q2, self.loop.create_task(_task()) mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers self.stream.downloader.node = mock_node await self.stream.save_file() await self.stream.finished_write_attempt.wait() self.assertTrue(os.path.isfile(self.stream.full_path)) if stop_when_done: await self.stream.stop() self.assertTrue(os.path.isfile(self.stream.full_path)) with open(self.stream.full_path, 'rb') as f: self.assertEqual(f.read(), self.stream_bytes) await asyncio.sleep(0.01) async def test_transfer_stream(self): await self._test_transfer_stream(10) self.assertEqual(self.stream.status, "finished") self.assertFalse(self.stream._running.is_set()) async def test_delayed_stop(self): await self._test_transfer_stream(10, stop_when_done=False) self.assertEqual(self.stream.status, "finished") self.assertTrue(self.stream._running.is_set()) await asyncio.sleep(0.5) self.assertTrue(self.stream._running.is_set()) await asyncio.sleep(2) self.assertEqual(self.stream.status, "finished") self.assertFalse(self.stream._running.is_set()) @unittest.SkipTest async def test_transfer_hundred_blob_stream(self): await self._test_transfer_stream(100) async def test_transfer_stream_bad_first_peer_good_second(self): await self.setup_stream(2) mock_node = mock.Mock(spec=Node) bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334, allow_localhost=True) def _mock_accumulate_peers(q1, q2): async def _task(): pass q2.put_nowait([bad_peer]) self.loop.call_later(1, q2.put_nowait, [self.server_from_client]) return q2, self.loop.create_task(_task()) mock_node.accumulate_peers = _mock_accumulate_peers self.stream.downloader.node = mock_node await self.stream.save_file() await self.stream.finished_writing.wait() self.assertTrue(os.path.isfile(self.stream.full_path)) with open(self.stream.full_path, 'rb') as f: self.assertEqual(f.read(), self.stream_bytes) await self.stream.stop() # self.assertIs(self.server_from_client.tcp_last_down, None) # self.assertIsNot(bad_peer.tcp_last_down, None) async def test_client_chunked_response(self): self.server.stop_server() class ChunkedServerProtocol(BlobServerProtocol): def send_response(self, responses): to_send = [] while responses: to_send.append(responses.pop()) for byte in BlobResponse(to_send).serialize(): self.transport.write(bytes([byte])) self.server.server_protocol_class = ChunkedServerProtocol self.server.start_server(33333, '127.0.0.1') self.assertEqual(0, len(self.client_blob_manager.completed_blob_hashes)) await asyncio.wait_for(self._test_transfer_stream(10), timeout=2) self.assertEqual(11, len(self.client_blob_manager.completed_blob_hashes)) async def test_create_and_decrypt_one_blob_stream(self, blobs=1, corrupt=False): descriptor = await self.create_stream(blobs) # copy blob files shutil.copy(os.path.join(self.server_blob_manager.blob_dir, self.sd_hash), os.path.join(self.client_blob_manager.blob_dir, self.sd_hash)) self.stream = ManagedStream(self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir) for blob_info in descriptor.blobs[:-1]: shutil.copy(os.path.join(self.server_blob_manager.blob_dir, blob_info.blob_hash), os.path.join(self.client_blob_manager.blob_dir, blob_info.blob_hash)) if corrupt and blob_info.length == MAX_BLOB_SIZE: with open(os.path.join(self.client_blob_manager.blob_dir, blob_info.blob_hash), "rb+") as handle: handle.truncate() handle.flush() await self.stream.save_file() await self.stream.finished_writing.wait() if corrupt: return self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file"))) with open(os.path.join(self.client_dir, "test_file"), "rb") as f: decrypted = f.read() self.assertEqual(decrypted, self.stream_bytes) self.assertTrue(self.client_blob_manager.get_blob(self.sd_hash).get_is_verified()) self.assertEqual( True, self.client_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified() ) # # # its all blobs + sd blob - last blob, which is the same size as descriptor.blobs # self.assertEqual(len(descriptor.blobs), len(await downloader_storage.get_all_finished_blobs())) # self.assertEqual( # [descriptor.sd_hash, descriptor.blobs[0].blob_hash], await downloader_storage.get_blobs_to_announce() # ) # # await downloader_storage.close() # await self.storage.close() async def test_create_and_decrypt_multi_blob_stream(self): await self.test_create_and_decrypt_one_blob_stream(10) # async def test_create_truncate_and_handle_stream(self): # # The purpose of this test is just to make sure it can finish even if a blob is corrupt/truncated # await asyncio.wait_for(self.test_create_and_decrypt_one_blob_stream(corrupt=True), timeout=5) ================================================ FILE: tests/unit/stream/test_reflector.py ================================================ import os import asyncio import tempfile import shutil from lbry.testcase import AsyncioTestCase from lbry.conf import Config from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_manager import BlobManager from lbry.stream.stream_manager import StreamManager from lbry.stream.reflector.server import ReflectorServer class TestReflector(AsyncioTestCase): async def asyncSetUp(self): self.loop = asyncio.get_event_loop() self.key = b'deadbeef' * 4 self.cleartext = os.urandom(20000000) tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) self.conf = Config() self.storage = SQLiteStorage(self.conf, os.path.join(tmp_dir, "lbrynet.sqlite")) await self.storage.open() self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.conf) self.addCleanup(self.blob_manager.stop) self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None) server_tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(server_tmp_dir)) self.server_conf = Config() self.server_storage = SQLiteStorage(self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite")) await self.server_storage.open() self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_conf) self.addCleanup(self.server_blob_manager.stop) download_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(download_dir)) # create the stream file_path = os.path.join(tmp_dir, "test_file") with open(file_path, 'wb') as f: f.write(self.cleartext) self.stream_manager.config.reflect_streams = False self.stream = await self.stream_manager.create(file_path) async def _test_reflect_stream(self, response_chunk_size=50, partial_needs=False): reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size, partial_needs=partial_needs) reflector.start_server(5566, '127.0.0.1') if partial_needs: server_blob = self.server_blob_manager.get_blob(self.stream.sd_hash) client_blob = self.blob_manager.get_blob(self.stream.sd_hash) with client_blob.reader_context() as handle: server_blob.set_length(client_blob.get_length()) writer = server_blob.get_blob_writer('nobody', 0) writer.write(handle.read()) self.server_blob_manager.blob_completed(server_blob) await reflector.started_listening.wait() self.addCleanup(reflector.stop_server) self.assertEqual(0, self.stream.reflector_progress) sent = await self.stream.upload_to_reflector('127.0.0.1', 5566) self.assertEqual(100, self.stream.reflector_progress) if partial_needs: self.assertFalse(self.stream.is_fully_reflected) send_more = await self.stream.upload_to_reflector('127.0.0.1', 5566) self.assertGreater(len(send_more), 0) sent.extend(send_more) sent.append(self.stream.sd_hash) self.assertSetEqual( set(sent), set(map(lambda b: b.blob_hash, self.stream.descriptor.blobs[:-1] + [self.blob_manager.get_blob(self.stream.sd_hash)])) ) send_more = await self.stream.upload_to_reflector('127.0.0.1', 5566) self.assertEqual(len(send_more), 0) self.assertTrue(self.stream.is_fully_reflected) server_sd_blob = self.server_blob_manager.get_blob(self.stream.sd_hash) self.assertTrue(server_sd_blob.get_is_verified()) self.assertEqual(server_sd_blob.length, server_sd_blob.length) for blob in self.stream.descriptor.blobs[:-1]: server_blob = self.server_blob_manager.get_blob(blob.blob_hash) self.assertTrue(server_blob.get_is_verified()) self.assertEqual(server_blob.length, blob.length) sent = await self.stream.upload_to_reflector('127.0.0.1', 5566) self.assertListEqual(sent, []) async def test_reflect_stream(self): return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=50), 3) async def test_reflect_stream_but_reflector_changes_its_mind(self): return await asyncio.wait_for(self._test_reflect_stream(partial_needs=True), 3) async def test_reflect_stream_small_response_chunks(self): return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=30), 3) async def test_announces(self): to_announce = await self.storage.get_blobs_to_announce() self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce") self.assertNotIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob set to announce") async def test_result_from_disconnect_mid_sd_transfer(self): stop = asyncio.Event() incoming = asyncio.Event() reflector = ReflectorServer( self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming ) reflector.start_server(5566, '127.0.0.1') await reflector.started_listening.wait() self.addCleanup(reflector.stop_server) self.assertEqual(0, self.stream.reflector_progress) reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566)) await incoming.wait() stop.set() # this used to raise (and then propagate) a CancelledError self.assertListEqual(await reflect_task, []) self.assertFalse(self.stream.is_fully_reflected) self.assertFalse(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified()) async def test_result_from_disconnect_after_sd_transfer(self): stop = asyncio.Event() incoming = asyncio.Event() not_incoming = asyncio.Event() reflector = ReflectorServer( self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming, not_incoming_event=not_incoming ) reflector.start_server(5566, '127.0.0.1') await reflector.started_listening.wait() self.addCleanup(reflector.stop_server) self.assertEqual(0, self.stream.reflector_progress) reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566)) await incoming.wait() await not_incoming.wait() stop.set() sent = await reflect_task self.assertListEqual([self.stream.sd_hash], sent) self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified()) self.assertFalse(self.stream.is_fully_reflected) async def test_result_from_disconnect_after_data_transfer(self): stop = asyncio.Event() incoming = asyncio.Event() not_incoming = asyncio.Event() reflector = ReflectorServer( self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming, not_incoming_event=not_incoming ) reflector.start_server(5566, '127.0.0.1') await reflector.started_listening.wait() self.addCleanup(reflector.stop_server) self.assertEqual(0, self.stream.reflector_progress) reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566)) await incoming.wait() await not_incoming.wait() await incoming.wait() await not_incoming.wait() stop.set() sent = await reflect_task self.assertListEqual([self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash], sent) self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified()) self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()) self.assertFalse(self.stream.is_fully_reflected) async def test_result_from_disconnect_mid_data_transfer(self): stop = asyncio.Event() incoming = asyncio.Event() not_incoming = asyncio.Event() reflector = ReflectorServer( self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming, not_incoming_event=not_incoming ) reflector.start_server(5566, '127.0.0.1') await reflector.started_listening.wait() self.addCleanup(reflector.stop_server) self.assertEqual(0, self.stream.reflector_progress) reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566)) await incoming.wait() await not_incoming.wait() await incoming.wait() stop.set() self.assertListEqual(await reflect_task, [self.stream.sd_hash]) self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified()) self.assertFalse( self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified() ) self.assertFalse(self.stream.is_fully_reflected) async def test_delete_file_during_reflector_upload(self): stop = asyncio.Event() incoming = asyncio.Event() not_incoming = asyncio.Event() reflector = ReflectorServer( self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming, not_incoming_event=not_incoming ) reflector.start_server(5566, '127.0.0.1') await reflector.started_listening.wait() self.addCleanup(reflector.stop_server) self.assertEqual(0, self.stream.reflector_progress) reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566)) await incoming.wait() await not_incoming.wait() await incoming.wait() await self.stream_manager.delete(self.stream, delete_file=True) # this used to raise OSError when it can't read the deleted blob for the upload sent = await reflect_task self.assertListEqual([self.stream.sd_hash], sent) self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified()) self.assertFalse( self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified() ) self.assertFalse(self.stream.is_fully_reflected) ================================================ FILE: tests/unit/stream/test_stream_descriptor.py ================================================ import os import asyncio import tempfile import shutil import json from lbry.blob.blob_file import BlobFile from lbry.testcase import AsyncioTestCase from lbry.conf import Config from lbry.error import InvalidStreamDescriptorError from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_manager import BlobManager from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name class TestStreamDescriptor(AsyncioTestCase): async def asyncSetUp(self): self.loop = asyncio.get_event_loop() self.key = b'deadbeef' * 4 self.cleartext = os.urandom(20000000) self.tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(self.tmp_dir)) self.conf = Config() self.storage = SQLiteStorage(self.conf, ":memory:") await self.storage.open() self.blob_manager = BlobManager(self.loop, self.tmp_dir, self.storage, self.conf) self.file_path = os.path.join(self.tmp_dir, "test_file") with open(self.file_path, 'wb') as f: f.write(self.cleartext) self.descriptor = await StreamDescriptor.create_stream(self.loop, self.tmp_dir, self.file_path, key=self.key) self.sd_hash = self.descriptor.calculate_sd_hash() self.sd_dict = json.loads(self.descriptor.as_json()) def _write_sd(self): with open(os.path.join(self.tmp_dir, self.sd_hash), 'wb') as f: f.write(json.dumps(self.sd_dict, sort_keys=True).encode()) async def _test_invalid_sd(self): self._write_sd() with self.assertRaises(InvalidStreamDescriptorError): await self.blob_manager.get_stream_descriptor(self.sd_hash) async def test_load_sd_blob(self): self._write_sd() descriptor = await self.blob_manager.get_stream_descriptor(self.sd_hash) self.assertEqual(descriptor.calculate_sd_hash(), self.sd_hash) async def test_missing_terminator(self): self.sd_dict['blobs'].pop() await self._test_invalid_sd() async def test_terminator_not_at_end(self): terminator = self.sd_dict['blobs'].pop() self.sd_dict['blobs'] = [terminator] + self.sd_dict['blobs'] await self._test_invalid_sd() async def test_terminator_has_blob_hash(self): self.sd_dict['blobs'][-1]['blob_hash'] = '1' * 96 await self._test_invalid_sd() async def test_blob_order(self): terminator = self.sd_dict['blobs'].pop() self.sd_dict['blobs'].reverse() self.sd_dict['blobs'].append(terminator) await self._test_invalid_sd() async def test_skip_blobs(self): self.sd_dict['blobs'][-2]['blob_num'] = self.sd_dict['blobs'][-2]['blob_num'] + 1 await self._test_invalid_sd() async def test_invalid_stream_hash(self): self.sd_dict['blobs'][-2]['blob_hash'] = '1' * 96 await self._test_invalid_sd() async def test_zero_length_blob(self): self.sd_dict['blobs'][-2]['length'] = 0 await self._test_invalid_sd() def test_sanitize_file_name(self): self.assertEqual(sanitize_file_name(' t/-?t|.g.ext '), 't-t.g.ext') self.assertEqual(sanitize_file_name('end_dot .'), 'end_dot') self.assertEqual(sanitize_file_name('.file\0\0'), '.file') self.assertEqual(sanitize_file_name('test n\16ame.ext'), 'test name.ext') self.assertEqual(sanitize_file_name('COM8.ext', default_file_name='default1'), 'default1.ext') self.assertEqual(sanitize_file_name('LPT2', default_file_name='default2'), 'default2') self.assertEqual(sanitize_file_name('', default_file_name=''), '') class TestRecoverOldStreamDescriptors(AsyncioTestCase): async def test_old_key_sort_sd_blob(self): loop = asyncio.get_event_loop() tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) self.conf = Config() storage = SQLiteStorage(self.conf, ":memory:") await storage.open() blob_manager = BlobManager(loop, tmp_dir, storage, self.conf) sd_bytes = b'{"stream_name": "4f62616d6120446f6e6b65792d322e73746c", "blobs": [{"length": 1153488, "blob_num' \ b'": 0, "blob_hash": "9fa32a249ce3f2d4e46b78599800f368b72f2a7f22b81df443c7f6bdbef496bd61b4c0079c7' \ b'3d79c8bb9be9a6bf86592", "iv": "0bf348867244019c9e22196339016ea6"}, {"length": 0, "blob_num": 1,' \ b' "iv": "9f36abae16955463919b07ed530a3d18"}], "stream_type": "lbryfile", "key": "a03742b87628aa7' \ b'228e48f1dcd207e48", "suggested_file_name": "4f62616d6120446f6e6b65792d322e73746c", "stream_hash' \ b'": "b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e1' \ b'60207"}' sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2' stream_hash = 'b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e160207' blob = blob_manager.get_blob(sd_hash) blob.set_length(len(sd_bytes)) writer = blob.get_blob_writer() writer.write(sd_bytes) await blob.verified.wait() descriptor = await StreamDescriptor.from_stream_descriptor_blob( loop, blob_manager.blob_dir, blob ) self.assertEqual(stream_hash, descriptor.get_stream_hash()) self.assertEqual(sd_hash, descriptor.calculate_old_sort_sd_hash()) self.assertNotEqual(sd_hash, descriptor.calculate_sd_hash()) async def test_decode_corrupt_blob_raises_proper_exception_and_deletes_corrupt_file(self): loop = asyncio.get_event_loop() tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2' with open(os.path.join(tmp_dir, sd_hash), 'wb') as handle: handle.write(b'doesnt work') blob = BlobFile(loop, sd_hash, blob_directory=tmp_dir) self.assertTrue(blob.file_exists) self.assertIsNotNone(blob.length) with self.assertRaises(InvalidStreamDescriptorError): await StreamDescriptor.from_stream_descriptor_blob( loop, tmp_dir, blob ) self.assertFalse(blob.file_exists) # fixme: this is an emergency PR, please move this to blob_file tests later self.assertIsNone(blob.length) ================================================ FILE: tests/unit/stream/test_stream_manager.py ================================================ import os import shutil import binascii from unittest import mock import asyncio import json from decimal import Decimal from lbry.file.file_manager import FileManager from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase from lbry.testcase import get_fake_exchange_rate_manager from lbry.utils import generate_id from lbry.error import InsufficientFundsError from lbry.error import KeyFeeAboveMaxAllowedError, ResolveError, DownloadSDTimeoutError, DownloadDataTimeoutError from lbry.wallet import WalletManager, Wallet, Ledger, Transaction, Input, Output, Database from lbry.wallet.constants import CENT, NULL_HASH32 from lbry.wallet.network import ClientSession from lbry.conf import Config from lbry.extras.daemon.analytics import AnalyticsManager from lbry.stream.stream_manager import StreamManager from lbry.stream.descriptor import StreamDescriptor from lbry.dht.node import Node from lbry.dht.protocol.protocol import KademliaProtocol from lbry.dht.protocol.routing_table import TreeRoutingTable from lbry.schema.claim import Claim def get_mock_node(peer=None): def mock_accumulate_peers(q1: asyncio.Queue, q2: asyncio.Queue): async def _task(): pass if peer: q2.put_nowait([peer]) return q2, asyncio.create_task(_task()) mock_node = mock.Mock(spec=Node) mock_node.protocol = mock.Mock(spec=KademliaProtocol) mock_node.protocol.routing_table = mock.Mock(spec=TreeRoutingTable) mock_node.protocol.routing_table.get_peers = lambda: [] mock_node.accumulate_peers = mock_accumulate_peers mock_node.joined = asyncio.Event() mock_node.joined.set() return mock_node def get_output(amount=CENT, pubkey_hash=NULL_HASH32): return Transaction() \ .add_outputs([Output.pay_pubkey_hash(amount, pubkey_hash)]) \ .outputs[0] def get_input(): return Input.spend(get_output()) def get_transaction(txo=None): return Transaction() \ .add_inputs([get_input()]) \ .add_outputs([txo or Output.pay_pubkey_hash(CENT, NULL_HASH32)]) def get_claim_transaction(claim_name, claim=b''): return get_transaction( Output.pay_claim_name_pubkey_hash(CENT, claim_name, claim, NULL_HASH32) ) async def get_mock_wallet(sd_hash, storage, wallet_dir, balance=10.0, fee=None): claim = Claim() if fee: if fee['currency'] == 'LBC': claim.stream.fee.lbc = Decimal(fee['amount']) elif fee['currency'] == 'USD': claim.stream.fee.usd = Decimal(fee['amount']) claim.stream.title = "33rpm" claim.stream.languages.append("en") claim.stream.source.sd_hash = sd_hash claim.stream.source.media_type = "image/png" tx = get_claim_transaction("33rpm", claim.to_bytes()) tx.height = 514081 txo = tx.outputs[0] txo.meta.update({ "permanent_url": "33rpm#c49566d631226492317d06ad7fdbe1ed32925124", }) class FakeHeaders: def estimated_timestamp(self, height): return 1984 def __init__(self, height): self.height = height def __getitem__(self, item): return {'timestamp': 1984} wallet = Wallet() ledger = Ledger({ 'db': Database(os.path.join(wallet_dir, 'blockchain.db')), 'headers': FakeHeaders(514082), 'tx_cache_size': 10000 }) await ledger.db.open() wallet.generate_account(ledger) manager = WalletManager() manager.config = Config() manager.config.save_files = True manager.config.transaction_cache_size = 10000 manager.wallets.append(wallet) manager.ledgers[Ledger] = ledger manager.ledger.network.client = ClientSession( network=manager.ledger.network, server=('fakespv.lbry.com', 50001) ) async def mock_resolve(*args, **kwargs): result = {txo.meta['permanent_url']: txo} await storage.save_claim_from_output(ledger, txo) return result manager.ledger.resolve = mock_resolve async def get_balance(*_): return balance manager.get_balance = get_balance return manager, txo.meta['permanent_url'] class TestStreamManager(BlobExchangeTestBase): async def asyncSetUp(self): await super().asyncSetUp() self.client_config.share_usage_data = True async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False): file_path = os.path.join(self.server_dir, "test_file") with open(file_path, 'wb') as f: f.write(os.urandom(20000000)) descriptor = await StreamDescriptor.create_stream( self.loop, self.server_blob_manager.blob_dir, file_path, old_sort=old_sort ) self.sd_hash = descriptor.sd_hash self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, self.client_wallet_dir, balance, fee) analytics_manager = AnalyticsManager( self.client_config, binascii.hexlify(generate_id()).decode(), binascii.hexlify(generate_id()).decode() ) self.stream_manager = StreamManager( self.loop, self.client_config, self.client_blob_manager, self.mock_wallet, self.client_storage, get_mock_node(self.server_from_client), analytics_manager ) self.file_manager = FileManager( self.loop, self.client_config, self.mock_wallet, self.client_storage, analytics_manager ) self.file_manager.source_managers['stream'] = self.stream_manager self.exchange_rate_manager = get_fake_exchange_rate_manager() async def _test_time_to_first_bytes(self, check_post, error=None, after_setup=None): await self.setup_stream_manager() if after_setup: after_setup() checked_analytics_event = False async def _check_post(event): check_post(event) nonlocal checked_analytics_event checked_analytics_event = True self.stream_manager.analytics_manager._post = _check_post if error: with self.assertRaises(error): await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) else: await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) await asyncio.sleep(0) self.assertTrue(checked_analytics_event) async def test_time_to_first_bytes(self): def check_post(event): self.assertEqual(event['event'], 'Time To First Bytes') total_duration = event['properties']['total_duration'] resolve_duration = event['properties']['resolve_duration'] head_blob_duration = event['properties']['head_blob_duration'] sd_blob_duration = event['properties']['sd_blob_duration'] self.assertFalse(event['properties']['added_fixed_peers']) self.assertEqual(event['properties']['wallet_server'], "fakespv.lbry.com:50001") self.assertGreaterEqual(total_duration, resolve_duration + head_blob_duration + sd_blob_duration) await self._test_time_to_first_bytes(check_post) async def test_fixed_peer_delay_dht_peers_found(self): self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)] server_from_client = None self.server_from_client, server_from_client = server_from_client, self.server_from_client def after_setup(): self.stream_manager.node.protocol.routing_table.get_peers = lambda: [server_from_client] def check_post(event): self.assertEqual(event['event'], 'Time To First Bytes') total_duration = event['properties']['total_duration'] resolve_duration = event['properties']['resolve_duration'] head_blob_duration = event['properties']['head_blob_duration'] sd_blob_duration = event['properties']['sd_blob_duration'] self.assertEqual(event['event'], 'Time To First Bytes') self.assertEqual(event['properties']['tried_peers_count'], 1) self.assertEqual(event['properties']['active_peer_count'], 1) self.assertEqual(event['properties']['connection_failures_count'], 0) self.assertTrue(event['properties']['use_fixed_peers']) self.assertTrue(event['properties']['added_fixed_peers']) self.assertEqual(event['properties']['fixed_peer_delay'], self.client_config.fixed_peer_delay) self.assertGreaterEqual(total_duration, resolve_duration + head_blob_duration + sd_blob_duration) await self._test_time_to_first_bytes(check_post, after_setup=after_setup) async def test_tcp_connection_failure_analytics(self): self.client_config.download_timeout = 3.0 def after_setup(): self.server.stop_server() def check_post(event): self.assertEqual(event['event'], 'Time To First Bytes') self.assertIsNone(event['properties']['head_blob_duration']) self.assertIsNone(event['properties']['sd_blob_duration']) self.assertFalse(event['properties']['added_fixed_peers']) self.assertEqual(event['properties']['connection_failures_count'], 1) self.assertEqual( event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout.' ) await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError, after_setup=after_setup) async def test_override_fixed_peer_delay_dht_disabled(self): self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)] self.client_config.components_to_skip = ['dht', 'hash_announcer'] self.client_config.fixed_peer_delay = 9001.0 self.server_from_client = None def check_post(event): total_duration = event['properties']['total_duration'] resolve_duration = event['properties']['resolve_duration'] head_blob_duration = event['properties']['head_blob_duration'] sd_blob_duration = event['properties']['sd_blob_duration'] self.assertEqual(event['event'], 'Time To First Bytes') self.assertEqual(event['properties']['tried_peers_count'], 1) self.assertEqual(event['properties']['active_peer_count'], 1) self.assertTrue(event['properties']['use_fixed_peers']) self.assertTrue(event['properties']['added_fixed_peers']) self.assertEqual(event['properties']['fixed_peer_delay'], 0.0) self.assertGreaterEqual(total_duration, resolve_duration + head_blob_duration + sd_blob_duration) start = self.loop.time() await self._test_time_to_first_bytes(check_post) self.assertLess(self.loop.time() - start, 3) async def test_no_peers_timeout(self): # FIXME: the download should ideally fail right away if there are no peers # to initialize the shortlist and fixed peers are disabled self.server_from_client = None self.client_config.download_timeout = 3.0 def check_post(event): self.assertEqual(event['event'], 'Time To First Bytes') self.assertEqual(event['properties']['error'], 'DownloadSDTimeoutError') self.assertEqual(event['properties']['tried_peers_count'], 0) self.assertEqual(event['properties']['active_peer_count'], 0) self.assertFalse(event['properties']['use_fixed_peers']) self.assertFalse(event['properties']['added_fixed_peers']) self.assertIsNone(event['properties']['fixed_peer_delay']) self.assertEqual( event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout.' ) start = self.loop.time() await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError) duration = self.loop.time() - start self.assertLessEqual(duration, 5) self.assertGreaterEqual(duration, 3.0) async def test_download_stop_resume_delete(self): await self.setup_stream_manager() received = [] expected_events = ['Time To First Bytes', 'Download Finished'] async def check_post(event): received.append(event['event']) self.stream_manager.analytics_manager._post = check_post self.assertDictEqual(self.stream_manager.streams, {}) stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) stream_hash = stream.stream_hash self.assertDictEqual(self.stream_manager.streams, {stream.sd_hash: stream}) self.assertTrue(stream.running) self.assertFalse(stream.finished) self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file"))) stored_status = await self.client_storage.run_and_return_one_or_none( "select status from file where stream_hash=?", stream_hash ) self.assertEqual(stored_status, "running") await stream.stop() self.assertFalse(stream.finished) self.assertFalse(stream.running) self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file"))) stored_status = await self.client_storage.run_and_return_one_or_none( "select status from file where stream_hash=?", stream_hash ) self.assertEqual(stored_status, "stopped") stream.downloader.node = self.stream_manager.node await stream.save_file() await stream.finished_writing.wait() await asyncio.sleep(0) self.assertTrue(stream.finished) self.assertFalse(stream.running) self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file"))) stored_status = await self.client_storage.run_and_return_one_or_none( "select status from file where stream_hash=?", stream_hash ) self.assertEqual(stored_status, "finished") await self.stream_manager.delete(stream, True) self.assertDictEqual(self.stream_manager.streams, {}) self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file"))) stored_status = await self.client_storage.run_and_return_one_or_none( "select status from file where stream_hash=?", stream_hash ) self.assertIsNone(stored_status) self.assertListEqual(expected_events, received) async def _test_download_error_on_start(self, expected_error, timeout=None): error = None try: await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager, timeout) except Exception as err: if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8 raise error = err self.assertEqual(expected_error, type(error)) async def _test_download_error_analytics_on_start(self, expected_error, error_message, timeout=None): received = [] async def check_post(event): self.assertEqual("Time To First Bytes", event['event']) self.assertEqual(event['properties']['error_message'], error_message) received.append(event['properties']['error']) self.stream_manager.analytics_manager._post = check_post await self._test_download_error_on_start(expected_error, timeout) await asyncio.sleep(0) self.assertListEqual([expected_error.__name__], received) async def test_insufficient_funds(self): fee = { 'currency': 'LBC', 'amount': 11.0, 'address': 'bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF', 'version': '_0_0_1' } await self.setup_stream_manager(10.0, fee) await self._test_download_error_on_start(InsufficientFundsError, "") async def test_fee_above_max_allowed(self): fee = { 'currency': 'USD', 'amount': 51.0, 'address': 'bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF', 'version': '_0_0_1' } await self.setup_stream_manager(1000000.0, fee) await self._test_download_error_on_start(KeyFeeAboveMaxAllowedError, "") async def test_resolve_error(self): await self.setup_stream_manager() self.uri = "fake" await self._test_download_error_on_start(ResolveError) async def test_download_sd_timeout(self): self.server.stop_server() await self.setup_stream_manager() await self._test_download_error_analytics_on_start( DownloadSDTimeoutError, f'Failed to download sd blob {self.sd_hash} within timeout.', timeout=1 ) async def test_download_data_timeout(self): await self.setup_stream_manager() with open(os.path.join(self.server_dir, self.sd_hash), 'r') as sdf: head_blob_hash = json.loads(sdf.read())['blobs'][0]['blob_hash'] self.server_blob_manager.delete_blob(head_blob_hash) await self._test_download_error_analytics_on_start( DownloadDataTimeoutError, f'Failed to download data blobs for sd hash {self.sd_hash} within timeout.', timeout=1 ) async def test_unexpected_error(self): await self.setup_stream_manager() err_msg = f"invalid blob directory '{self.client_dir}'" shutil.rmtree(self.client_dir) await self._test_download_error_analytics_on_start( OSError, err_msg, timeout=1 ) os.mkdir(self.client_dir) # so the test cleanup doesn't error async def test_non_head_data_timeout(self): await self.setup_stream_manager() with open(os.path.join(self.server_dir, self.sd_hash), 'r') as sdf: last_blob_hash = json.loads(sdf.read())['blobs'][-2]['blob_hash'] self.server_blob_manager.delete_blob(last_blob_hash) self.client_config.blob_download_timeout = 0.1 stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) await stream.started_writing.wait() self.assertEqual('running', stream.status) self.assertIsNotNone(stream.full_path) self.assertGreater(stream.written_bytes, 0) await stream.finished_write_attempt.wait() self.assertEqual('stopped', stream.status) self.assertIsNone(stream.full_path) self.assertEqual(0, stream.written_bytes) await self.stream_manager.stop() await self.stream_manager.start() self.assertEqual(1, len(self.stream_manager.streams)) stream = list(self.stream_manager.streams.values())[0] self.assertEqual('stopped', stream.status) self.assertIsNone(stream.full_path) self.assertEqual(0, stream.written_bytes) async def test_download_then_recover_stream_on_startup(self, old_sort=False): expected_analytics_events = [ 'Time To First Bytes', 'Download Finished' ] received_events = [] async def check_post(event): received_events.append(event['event']) await self.setup_stream_manager(old_sort=old_sort) self.stream_manager.analytics_manager._post = check_post self.assertDictEqual(self.stream_manager.streams, {}) stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) await stream.finished_writing.wait() await asyncio.sleep(0) await self.stream_manager.stop() self.client_blob_manager.stop() # partial removal, only sd blob is missing. # in this case, we recover the sd blob while the other blobs are kept untouched as 'finished' os.remove(os.path.join(self.client_blob_manager.blob_dir, stream.sd_hash)) await self.client_blob_manager.setup() await self.stream_manager.start() self.assertEqual(1, len(self.stream_manager.streams)) self.assertListEqual([self.sd_hash], list(self.stream_manager.streams.keys())) for blob_hash in [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]: blob_status = await self.client_storage.get_blob_status(blob_hash) self.assertEqual('finished', blob_status) self.assertEqual('finished', self.stream_manager.streams[self.sd_hash].status) sd_blob = self.client_blob_manager.get_blob(stream.sd_hash) self.assertTrue(sd_blob.file_exists) self.assertTrue(sd_blob.get_is_verified()) self.assertListEqual(expected_analytics_events, received_events) # full removal, check that status is preserved (except sd blob, which was written) self.client_blob_manager.stop() os.remove(os.path.join(self.client_blob_manager.blob_dir, stream.sd_hash)) for blob in stream.descriptor.blobs[:-1]: os.remove(os.path.join(self.client_blob_manager.blob_dir, blob.blob_hash)) await self.client_blob_manager.setup() await self.stream_manager.start() for blob_hash in [b.blob_hash for b in stream.descriptor.blobs[:-1]]: blob_status = await self.client_storage.get_blob_status(blob_hash) self.assertEqual('pending', blob_status) # sd blob was recovered sd_blob = self.client_blob_manager.get_blob(stream.sd_hash) self.assertTrue(sd_blob.file_exists) self.assertTrue(sd_blob.get_is_verified()) self.assertListEqual(expected_analytics_events, received_events) # db reflects that too blob_status = await self.client_storage.get_blob_status(stream.sd_hash) self.assertEqual('finished', blob_status) def test_download_then_recover_old_sort_stream_on_startup(self): return self.test_download_then_recover_stream_on_startup(old_sort=True) ================================================ FILE: tests/unit/test_cli.py ================================================ import os import tempfile import shutil import contextlib import logging import pathlib from io import StringIO from unittest import TestCase from unittest.mock import patch from types import SimpleNamespace from contextlib import asynccontextmanager import docopt from lbry.testcase import AsyncioTestCase from lbry.extras.cli import normalize_value, main, setup_logging, ensure_directory_exists from lbry.extras.system_info import get_platform from lbry.extras.daemon.daemon import Daemon from lbry.conf import Config from lbry.extras import cli @asynccontextmanager async def get_logger(argv, **conf_options): # loggly requires loop, so we do this in async function logger = logging.getLogger('test-root-logger') temp_dir = tempfile.mkdtemp() temp_config = os.path.join(temp_dir, 'settings.yml') try: # create a config (to be loaded on startup) _conf = Config.create_from_arguments(SimpleNamespace(config=temp_config)) with _conf.update_config(): for opt_name, opt_value in conf_options.items(): setattr(_conf, opt_name, opt_value) # do what happens on startup argv.extend(['--data-dir', temp_dir]) argv.extend(['--wallet-dir', temp_dir]) argv.extend(['--config', temp_config]) parser = cli.get_argument_parser() args, command_args = parser.parse_known_args(argv) conf: Config = Config.create_from_arguments(args) setup_logging(logger, args, conf) yield logger finally: shutil.rmtree(temp_dir, ignore_errors=True) for mod in cli.LOG_MODULES: log = logger.getChild(mod) log.setLevel(logging.NOTSET) while log.handlers: h = log.handlers[0] log.removeHandler(log.handlers[0]) h.close() class CLILoggingTest(AsyncioTestCase): async def test_verbose_logging(self): async with get_logger(["start", "--quiet"], share_usage_data=False) as log: log = log.getChild("lbry") self.assertTrue(log.isEnabledFor(logging.INFO)) self.assertFalse(log.isEnabledFor(logging.DEBUG)) self.assertEqual(len(log.handlers), 1) self.assertIsInstance(log.handlers[0], logging.handlers.RotatingFileHandler) async with get_logger(["start", "--verbose"]) as log: self.assertTrue(log.getChild("lbry").isEnabledFor(logging.DEBUG)) self.assertTrue(log.getChild("lbry").isEnabledFor(logging.INFO)) self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG)) async with get_logger(["start", "--verbose", "lbry.extras", "lbry.wallet", "torba.client"]) as log: self.assertTrue(log.getChild("lbry.extras").isEnabledFor(logging.DEBUG)) self.assertTrue(log.getChild("lbry.wallet").isEnabledFor(logging.DEBUG)) self.assertTrue(log.getChild("torba.client").isEnabledFor(logging.DEBUG)) self.assertFalse(log.getChild("lbry").isEnabledFor(logging.DEBUG)) self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG)) async def test_quiet(self): async with get_logger(["start"]) as log: # default is loud log = log.getChild("lbry") self.assertEqual(len(log.handlers), 2) self.assertIs(type(log.handlers[1]), logging.StreamHandler) async with get_logger(["start", "--quiet"]) as log: log = log.getChild("lbry") self.assertEqual(len(log.handlers), 1) self.assertIsNot(type(log.handlers[0]), logging.StreamHandler) class CLITest(AsyncioTestCase): @staticmethod def shell(argv): actual_output = StringIO() with contextlib.redirect_stdout(actual_output): with contextlib.redirect_stderr(actual_output): try: main(argv) except SystemExit as e: print(e.args[0]) return actual_output.getvalue().strip() def test_guess_type(self): self.assertEqual('0.3.8', normalize_value('0.3.8')) self.assertEqual('0.3', normalize_value('0.3')) self.assertEqual(3, normalize_value('3')) self.assertEqual(3, normalize_value(3)) self.assertEqual( 'VdNmakxFORPSyfCprAD/eDDPk5TY9QYtSA==', normalize_value('VdNmakxFORPSyfCprAD/eDDPk5TY9QYtSA==') ) self.assertTrue(normalize_value('TRUE')) self.assertTrue(normalize_value('true')) self.assertTrue(normalize_value('TrUe')) self.assertFalse(normalize_value('FALSE')) self.assertFalse(normalize_value('false')) self.assertFalse(normalize_value('FaLsE')) self.assertTrue(normalize_value(True)) self.assertEqual('3', normalize_value('3', key="uri")) self.assertEqual('0.3', normalize_value('0.3', key="uri")) self.assertEqual('True', normalize_value('True', key="uri")) self.assertEqual('False', normalize_value('False', key="uri")) self.assertEqual('3', normalize_value('3', key="file_name")) self.assertEqual('3', normalize_value('3', key="name")) self.assertEqual('3', normalize_value('3', key="download_directory")) self.assertEqual('3', normalize_value('3', key="channel_name")) self.assertEqual('3', normalize_value('3', key="claim_name")) self.assertEqual(3, normalize_value('3', key="some_other_thing")) def test_help(self): self.assertIn('lbrynet [-v] [--api HOST:PORT]', self.shell(['--help'])) # start is special command, with separate help handling self.assertIn('--share-usage-data', self.shell(['start', '--help'])) # publish is ungrouped command, returns usage only implicitly self.assertIn('publish (<name> | --name=<name>)', self.shell(['publish'])) # publish is ungrouped command, with explicit --help self.assertIn('Create or replace a stream claim at a given name', self.shell(['publish', '--help'])) # account is a group, returns help implicitly self.assertIn('Return the balance of an account', self.shell(['account'])) # account is a group, with explicit --help self.assertIn('Return the balance of an account', self.shell(['account', '--help'])) # account add is a grouped command, returns usage implicitly self.assertIn('account_add (<account_name> | --account_name=<account_name>)', self.shell(['account', 'add'])) # account add is a grouped command, with explicit --help self.assertIn('Add a previously created account from a seed,', self.shell(['account', 'add', '--help'])) def test_help_error_handling(self): # person tries `help` command, then they get help even though that's invalid command self.assertIn('--config FILE', self.shell(['help'])) # help for invalid command, with explicit --help self.assertIn('--config FILE', self.shell(['nonexistant', '--help'])) # help for invalid command, implicit self.assertIn('--config FILE', self.shell(['nonexistant'])) def test_version_command(self): self.assertEqual( "lbrynet {lbrynet_version}".format(**get_platform()), self.shell(['--version']) ) def test_valid_command_daemon_not_started(self): self.assertEqual( "Could not connect to daemon. Are you sure it's running?", self.shell(["publish", 'asd']) ) def test_deprecated_command_daemon_not_started(self): actual_output = StringIO() with contextlib.redirect_stdout(actual_output): main(["channel", "new", "@foo", "1.0"]) self.assertEqual( actual_output.getvalue().strip(), "channel_new is deprecated, using channel_create.\n" "Could not connect to daemon. Are you sure it's running?" ) @patch.object(Daemon, 'start', spec=Daemon, wraps=Daemon.start) def test_keyboard_interrupt_handling(self, mock_daemon_start): def side_effect(): raise KeyboardInterrupt mock_daemon_start.side_effect = side_effect self.shell(["start", "--no-logging"]) mock_daemon_start.assert_called_once() class DaemonDocsTests(TestCase): def test_can_parse_api_method_docs(self): failures = [] for name, fn in Daemon.callable_methods.items(): try: docopt.docopt(fn.__doc__, ()) except docopt.DocoptLanguageError as err: failures.append(f"invalid docstring for {name}, {err.args[0]}") except docopt.DocoptExit: pass if failures: self.fail("\n" + "\n".join(failures)) class EnsureDirectoryExistsTests(TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.temp_dir) def test_when_parent_dir_does_not_exist_then_dir_is_created_with_parent(self): dir_path = os.path.join(self.temp_dir, "parent_dir", "dir") ensure_directory_exists(dir_path) self.assertTrue(os.path.exists(dir_path)) def test_when_non_writable_dir_exists_then_raise(self): dir_path = os.path.join(self.temp_dir, "dir") pathlib.Path(dir_path).mkdir(mode=0o555) # creates a non-writable, readable and executable dir with self.assertRaises(PermissionError): ensure_directory_exists(dir_path) def test_when_dir_exists_and_writable_then_no_raise(self): dir_path = os.path.join(self.temp_dir, "dir") pathlib.Path(dir_path).mkdir(mode=0o777) # creates a writable, readable and executable dir try: ensure_directory_exists(dir_path) except (FileExistsError, PermissionError) as err: self.fail(f"{type(err).__name__} was raised") def test_when_non_dir_file_exists_at_path_then_raise(self): file_path = os.path.join(self.temp_dir, "file.extension") pathlib.Path(file_path).touch() with self.assertRaises(FileExistsError): ensure_directory_exists(file_path) ================================================ FILE: tests/unit/test_conf.py ================================================ import os import sys import types import tempfile import unittest import argparse import lbry.wallet from lbry.conf import Config, BaseConfig, String, Integer, Toggle, Servers, Strings, StringChoice, NOT_SET from lbry.error import InvalidCurrencyError class TestConfig(BaseConfig): test_str = String('str help', 'the default', previous_names=['old_str']) test_int = Integer('int help', 9) test_false_toggle = Toggle('toggle help', False) test_true_toggle = Toggle('toggle help', True) servers = Servers('servers help', [('localhost', 80)]) strings = Strings('cheese', ['string']) string_choice = StringChoice("one of string", ["a", "b", "c"], "a") class ConfigurationTests(unittest.TestCase): @unittest.skipIf('darwin' not in sys.platform, 'skipping mac only test') def test_mac_defaults(self): c = Config() self.assertEqual(c.data_dir, os.path.expanduser("~/Library/Application Support/LBRY")) self.assertEqual(c.wallet_dir, os.path.expanduser('~/.lbryum')) self.assertEqual(c.download_dir, os.path.expanduser('~/Downloads')) self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml')) self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi') self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log')) @unittest.skipIf('win32' not in sys.platform, 'skipping windows only test') def test_windows_defaults(self): c = Config() prefix = os.path.join(r"C:\Users", os.getlogin(), r"AppData\Local\lbry") self.assertEqual(c.data_dir, os.path.join(prefix, 'lbrynet')) self.assertEqual(c.wallet_dir, os.path.join(prefix, 'lbryum')) self.assertEqual(c.download_dir, os.path.join(r"C:\Users", os.getlogin(), "Downloads")) self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml')) self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi') self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log')) @unittest.skipIf('linux' not in sys.platform, 'skipping linux only test') def test_linux_defaults(self): c = Config() self.assertEqual(c.data_dir, os.path.expanduser('~/.local/share/lbry/lbrynet')) self.assertEqual(c.wallet_dir, os.path.expanduser('~/.local/share/lbry/lbryum')) self.assertEqual(c.download_dir, os.path.expanduser('~/Downloads')) self.assertEqual(c.config, os.path.expanduser('~/.local/share/lbry/lbrynet/daemon_settings.yml')) self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi') self.assertEqual(c.log_file_path, os.path.expanduser('~/.local/share/lbry/lbrynet/lbrynet.log')) def test_search_order(self): c = TestConfig() c.runtime = {'test_str': 'runtime'} c.arguments = {'test_str': 'arguments'} c.environment = {'test_str': 'environment'} c.persisted = {'test_str': 'persisted'} self.assertEqual(c.test_str, 'runtime') c.runtime = {} self.assertEqual(c.test_str, 'arguments') c.arguments = {} self.assertEqual(c.test_str, 'environment') c.environment = {} self.assertEqual(c.test_str, 'persisted') c.persisted = {} self.assertEqual(c.test_str, 'the default') def test_is_set(self): c = TestConfig() self.assertEqual(c.test_str, 'the default') self.assertFalse(TestConfig.test_str.is_set(c)) c.test_str = 'new value' self.assertEqual(c.test_str, 'new value') self.assertTrue(TestConfig.test_str.is_set(c)) def test_is_set_to_default(self): c = TestConfig() self.assertEqual(TestConfig.test_str.default, 'the default') self.assertFalse(TestConfig.test_str.is_set(c)) self.assertFalse(TestConfig.test_str.is_set_to_default(c)) c.test_str = 'new value' self.assertTrue(TestConfig.test_str.is_set(c)) self.assertFalse(TestConfig.test_str.is_set_to_default(c)) c.test_str = 'the default' self.assertTrue(TestConfig.test_str.is_set(c)) self.assertTrue(TestConfig.test_str.is_set_to_default(c)) def test_arguments(self): parser = argparse.ArgumentParser() TestConfig.contribute_to_argparse(parser) args = parser.parse_args([]) c = TestConfig.create_from_arguments(args) self.assertEqual(c.test_str, 'the default') self.assertTrue(c.test_true_toggle) self.assertFalse(c.test_false_toggle) self.assertEqual(c.servers, [('localhost', 80)]) self.assertEqual(c.strings, ['string']) args = parser.parse_args(['--test-str', 'blah']) c = TestConfig.create_from_arguments(args) self.assertEqual(c.test_str, 'blah') self.assertTrue(c.test_true_toggle) self.assertFalse(c.test_false_toggle) args = parser.parse_args(['--test-true-toggle']) c = TestConfig.create_from_arguments(args) self.assertTrue(c.test_true_toggle) self.assertFalse(c.test_false_toggle) args = parser.parse_args(['--test-false-toggle']) c = TestConfig.create_from_arguments(args) self.assertTrue(c.test_true_toggle) self.assertTrue(c.test_false_toggle) args = parser.parse_args(['--no-test-true-toggle']) c = TestConfig.create_from_arguments(args) self.assertFalse(c.test_true_toggle) self.assertFalse(c.test_false_toggle) args = parser.parse_args(['--servers=localhost:1', '--servers=192.168.0.1:2']) c = TestConfig.create_from_arguments(args) self.assertEqual(c.servers, [('localhost', 1), ('192.168.0.1', 2)]) args = parser.parse_args(['--strings=cheddar', '--strings=mozzarella']) c = TestConfig.create_from_arguments(args) self.assertEqual(c.strings, ['cheddar', 'mozzarella']) def test_environment(self): c = TestConfig() self.assertEqual(c.test_str, 'the default') c.set_environment({'LBRY_TEST_STR': 'from environ'}) self.assertEqual(c.test_str, 'from environ') self.assertEqual(c.test_int, 9) c.set_environment({'LBRY_TEST_INT': '1'}) self.assertEqual(c.test_int, 1) def test_persisted(self): with tempfile.TemporaryDirectory() as temp_dir: c = TestConfig.create_from_arguments( types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml')) ) # settings.yml doesn't exist on file system self.assertFalse(c.persisted.exists) self.assertEqual(c.test_str, 'the default') self.assertEqual(c.modify_order, [c.runtime]) with c.update_config(): self.assertEqual(c.modify_order, [c.runtime, c.persisted]) c.test_str = 'original' self.assertEqual(c.modify_order, [c.runtime]) # share_usage_data has been saved to settings file self.assertTrue(c.persisted.exists) with open(c.config, 'r') as fd: self.assertEqual(fd.read(), 'test_str: original\n') # load the settings file and check share_usage_data is false c = TestConfig.create_from_arguments( types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml')) ) self.assertTrue(c.persisted.exists) self.assertEqual(c.test_str, 'original') # setting in runtime overrides config self.assertNotIn('test_str', c.runtime) c.test_str = 'from runtime' self.assertIn('test_str', c.runtime) self.assertEqual(c.test_str, 'from runtime') # without context manager NOT_SET only clears it in runtime location c.test_str = NOT_SET self.assertNotIn('test_str', c.runtime) self.assertEqual(c.test_str, 'original') # clear it in persisted as well by using context manager self.assertIn('test_str', c.persisted) with c.update_config(): c.test_str = NOT_SET self.assertNotIn('test_str', c.persisted) self.assertEqual(c.test_str, 'the default') with open(c.config, 'r') as fd: self.assertEqual(fd.read(), '{}\n') def test_persisted_upgrade(self): with tempfile.TemporaryDirectory() as temp_dir: config = os.path.join(temp_dir, 'settings.yml') with open(config, 'w') as fd: fd.write('old_str: old stuff\n') c = TestConfig.create_from_arguments( types.SimpleNamespace(config=config) ) self.assertEqual(c.test_str, 'old stuff') self.assertNotIn('old_str', c.persisted) with open(config, 'w') as fd: fd.write('test_str: old stuff\n') def test_validation(self): c = TestConfig() with self.assertRaisesRegex(AssertionError, 'must be a string'): c.test_str = 9 with self.assertRaisesRegex(AssertionError, 'must be an integer'): c.test_int = 'hi' with self.assertRaisesRegex(AssertionError, 'must be a true/false'): c.test_true_toggle = 'hi' c.test_false_toggle = 'hi' def test_file_extension_validation(self): with self.assertRaisesRegex(AssertionError, "'.json' is not supported"): TestConfig.create_from_arguments( types.SimpleNamespace(config=os.path.join('settings.json')) ) def test_serialize_deserialize(self): with tempfile.TemporaryDirectory() as temp_dir: c = TestConfig.create_from_arguments( types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml')) ) self.assertEqual(c.servers, [('localhost', 80)]) with c.update_config(): c.servers = [('localhost', 8080)] with open(c.config, 'r+') as fd: self.assertEqual(fd.read(), 'servers:\n- localhost:8080\n') fd.write('servers:\n - localhost:5566\n') c = TestConfig.create_from_arguments( types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml')) ) self.assertEqual(c.servers, [('localhost', 5566)]) def test_max_key_fee_from_yaml(self): with tempfile.TemporaryDirectory() as temp_dir: config = os.path.join(temp_dir, 'settings.yml') with open(config, 'w') as fd: fd.write('max_key_fee: {currency: USD, amount: 1}\n') c = Config.create_from_arguments( types.SimpleNamespace(config=config) ) self.assertEqual(c.max_key_fee['currency'], 'USD') self.assertEqual(c.max_key_fee['amount'], 1) with self.assertRaises(InvalidCurrencyError): c.max_key_fee = {'currency': 'BCH', 'amount': 1} with c.update_config(): c.max_key_fee = {'currency': 'BTC', 'amount': 1} with open(config, 'r') as fd: self.assertEqual(fd.read(), 'max_key_fee:\n amount: 1\n currency: BTC\n') with c.update_config(): c.max_key_fee = None with open(config, 'r') as fd: self.assertEqual(fd.read(), 'max_key_fee: null\n') def test_max_key_fee_from_args(self): parser = argparse.ArgumentParser() Config.contribute_to_argparse(parser) # default args = parser.parse_args([]) c = Config.create_from_arguments(args) self.assertEqual(c.max_key_fee, {'amount': 50.0, 'currency': 'USD'}) # disabled args = parser.parse_args(['--no-max-key-fee']) c = Config.create_from_arguments(args) self.assertIsNone(c.max_key_fee) args = parser.parse_args(['--max-key-fee', 'null']) c = Config.create_from_arguments(args) self.assertIsNone(c.max_key_fee) # set args = parser.parse_args(['--max-key-fee', '1.0', 'BTC']) c = Config.create_from_arguments(args) self.assertEqual(c.max_key_fee, {'amount': 1.0, 'currency': 'BTC'}) def test_string_choice(self): with self.assertRaisesRegex(ValueError, "No valid values provided"): StringChoice("no valid values", [], "") with self.assertRaisesRegex(ValueError, "Default value must be one of"): StringChoice("invalid default", ["a"], "b") c = TestConfig() self.assertEqual("a", c.string_choice) # default c.string_choice = "b" self.assertEqual("b", c.string_choice) with self.assertRaisesRegex(ValueError, "Setting 'string_choice' value must be one of"): c.string_choice = "d" parser = argparse.ArgumentParser() TestConfig.contribute_to_argparse(parser) args = parser.parse_args(['--string-choice', 'c']) c = TestConfig.create_from_arguments(args) self.assertEqual("c", c.string_choice) def test_known_hubs_list(self): with tempfile.TemporaryDirectory() as temp_dir: hubs = Config(config=os.path.join(temp_dir, 'settings.yml'), wallet_dir=temp_dir).known_hubs self.assertEqual(hubs.serialized, {}) self.assertEqual(list(hubs), []) self.assertFalse(hubs) hubs.set('new.hub.io:99', {'jurisdiction': 'us'}) self.assertTrue(hubs) self.assertFalse(hubs.exists) hubs.save() self.assertTrue(hubs.exists) hubs = Config(config=os.path.join(temp_dir, 'settings.yml'), wallet_dir=temp_dir).known_hubs self.assertEqual(list(hubs), [('new.hub.io', 99)]) self.assertEqual(hubs.serialized, {'new.hub.io:99': {'jurisdiction': 'us'}}) hubs.set('any.hub.io:99', {}) hubs.set('oth.hub.io:99', {'jurisdiction': 'other'}) self.assertEqual(list(hubs), [('new.hub.io', 99), ('any.hub.io', 99), ('oth.hub.io', 99)]) self.assertEqual(hubs.filter(), { ('new.hub.io', 99): {'jurisdiction': 'us'}, ('oth.hub.io', 99): {'jurisdiction': 'other'}, ('any.hub.io', 99): {} }) self.assertEqual(hubs.filter(foo="bar"), {}) self.assertEqual(hubs.filter(jurisdiction="us"), { ('new.hub.io', 99): {'jurisdiction': 'us'} }) self.assertEqual(hubs.filter(jurisdiction="us", match_none=True), { ('new.hub.io', 99): {'jurisdiction': 'us'}, ('any.hub.io', 99): {} }) ================================================ FILE: tests/unit/test_utils.py ================================================ import unittest from lbry import utils class UtilsTestCase(unittest.TestCase): def test_get_colliding_prefix_bits(self): self.assertEqual( 0, utils.get_colliding_prefix_bits(0xffffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big"))) self.assertEqual( 1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big"))) self.assertEqual( 8, utils.get_colliding_prefix_bits(0x00ffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big"))) self.assertEqual( 8, utils.get_colliding_prefix_bits(0x00ffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big"))) self.assertEqual( 1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big"))) self.assertEqual( 1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big"))) ================================================ FILE: tests/unit/torrent/__init__.py ================================================ ================================================ FILE: tests/unit/torrent/test_tracker.py ================================================ import asyncio import random from lbry.testcase import AsyncioTestCase from lbry.dht.peer import KademliaPeer from lbry.torrent.tracker import CompactIPv4Peer, TrackerClient, enqueue_tracker_search, UDPTrackerServerProtocol, encode_peer class UDPTrackerClientTestCase(AsyncioTestCase): async def asyncSetUp(self): self.client_servers_list = [] self.servers = {} self.client = TrackerClient(b"\x00" * 48, 4444, lambda: self.client_servers_list, timeout=1) await self.client.start() self.addCleanup(self.client.stop) await self.add_server() async def add_server(self, port=None, add_to_client=True): port = port or len(self.servers) + 59990 assert port not in self.servers server = UDPTrackerServerProtocol() self.servers[port] = server transport, _ = await self.loop.create_datagram_endpoint(lambda: server, local_addr=("127.0.0.1", port)) self.addCleanup(transport.close) if add_to_client: self.client_servers_list.append(("127.0.0.1", port)) async def test_announce(self): info_hash = random.getrandbits(160).to_bytes(20, "big", signed=False) announcement = (await self.client.get_peer_list(info_hash))[0] self.assertEqual(announcement.seeders, 1) self.assertEqual(announcement.peers, [CompactIPv4Peer(int.from_bytes(bytes([127, 0, 0, 1]), "big", signed=False), 4444)]) async def test_announce_many_info_hashes_to_many_servers_with_bad_one_and_dns_error(self): await asyncio.gather(*[self.add_server() for _ in range(3)]) self.client_servers_list.append(("no.it.does.not.exist", 7070)) self.client_servers_list.append(("127.0.0.2", 7070)) info_hashes = [random.getrandbits(160).to_bytes(20, "big", signed=False) for _ in range(5)] await self.client.announce_many(*info_hashes) for server in self.servers.values(): self.assertDictEqual( server.peers, { info_hash: [encode_peer("127.0.0.1", self.client.announce_port)] for info_hash in info_hashes }) async def test_announce_using_helper_function(self): info_hash = random.getrandbits(160).to_bytes(20, "big", signed=False) queue = asyncio.Queue() enqueue_tracker_search(info_hash, queue) peers = await queue.get() self.assertEqual(peers, [KademliaPeer('127.0.0.1', None, None, 4444, allow_localhost=True)]) async def test_error(self): info_hash = random.getrandbits(160).to_bytes(20, "big", signed=False) await self.client.get_peer_list(info_hash) list(self.servers.values())[0].known_conns.clear() self.client.results.clear() with self.assertRaises(Exception) as err: await self.client.get_peer_list(info_hash) self.assertEqual(err.exception.args[0], b'Connection ID missmatch.\x00') async def test_multiple_servers(self): await asyncio.gather(*[self.add_server() for _ in range(10)]) info_hash = random.getrandbits(160).to_bytes(20, "big", signed=False) await self.client.get_peer_list(info_hash) for server in self.servers.values(): self.assertEqual(server.peers, {info_hash: [encode_peer("127.0.0.1", self.client.announce_port)]}) async def test_multiple_servers_with_bad_one(self): await asyncio.gather(*[self.add_server() for _ in range(10)]) self.client_servers_list.append(("127.0.0.2", 7070)) info_hash = random.getrandbits(160).to_bytes(20, "big", signed=False) await self.client.get_peer_list(info_hash) for server in self.servers.values(): self.assertEqual(server.peers, {info_hash: [encode_peer("127.0.0.1", self.client.announce_port)]}) async def test_multiple_servers_with_different_peers_across_helper_function(self): # this is how the downloader uses it await asyncio.gather(*[self.add_server() for _ in range(10)]) info_hash = random.getrandbits(160).to_bytes(20, "big", signed=False) fake_peers = [] for server in self.servers.values(): for _ in range(10): peer = (f"127.0.0.{random.randint(1, 255)}", random.randint(2000, 65500)) fake_peers.append(peer) server.add_peer(info_hash, *peer) peer_q = asyncio.Queue() enqueue_tracker_search(info_hash, peer_q) await asyncio.sleep(0) await asyncio.gather(*self.client.tasks.values()) self.assertEqual(11, peer_q.qsize()) ================================================ FILE: tests/unit/wallet/__init__.py ================================================ ================================================ FILE: tests/unit/wallet/key_fixtures.py ================================================ expected_ids = [ b'948adae2a128c0bd1fa238117fd0d9690961f26e', b'cd9f4f2adde7de0a53ab6d326bb6a62b489876dd', b'c479e02a74a809ffecff60255d1c14f4081a197a', b'4bab2fb2c424f31f170b15ec53c4a596db9d6710', b'689cb7c621f57b7c398e7e04ed9a5098ab8389e9', b'75116d6a689a0f9b56fe7cfec9cbbd0e16814288', b'2439f0993fb298497dd7f317b9737c356f664a86', b'32f1cb4799008cf5496bb8cafdaf59d5dabec6af', b'fa29aa536353904e9cc813b0cf18efcc09e5ad13', b'37df34002f34d7875428a2977df19be3f4f40a31', b'8c8a72b5d2747a3e7e05ed85110188769d5656c3', b'e5c8ef10c5bdaa79c9a237a096f50df4dcac27f0', b'4d5270dc100fba85974665c20cd0f95d4822e8d1', b'e76b07da0cdd59915475cd310599544b9744fa34', b'6f009bccf8be99707161abb279d8ccf8fd953721', b'f32f08b722cc8607c3f7f192b4d5f13a74c85785', b'46f4430a5c91b9b799e9be6b47ac7a749d8d9f30', b'ebbf9850abe0aae2d09e7e3ebd6b51f01282f39b', b'5f6655438f8ddc6b2f6ea8197c8babaffc9f5c09', b'e194e70ee8711b0ed765608121e4cceb551cdf28' ] expected_privkeys = [ b'95557ee9a2bb7665e67e45246658b5c839f7dcd99b6ebc800eeebccd28bf134a', b'689b6921f65647a8e4fc1497924730c92ad4ad183f10fac2bdee65cc8fb6dcf9', b'977ee018b448c530327b7e927cc3645ca4cb152c5dd98e1bd917c52fd46fc80a', b'3c7fb05b0ab4da8b292e895f574f8213cadfe81b84ded7423eab61c5f884c8ae', b'b21fc7be1e69182827538683a48ac9d95684faf6c1c6deabb6e513d8c76afcc9', b'a5021734dbbf1d090b15509ba00f2c04a3d5afc19939b4594ca0850d4190b923', b'07dfe0aa94c1b948dc935be1f8179f3050353b46f3a3134e77c70e66208be72d', b'c331b2fb82cd91120b0703ee312042a854a51a8d945aa9e70fb14d68b0366fe1', b'3aa59ec4d8f1e7ce2775854b5e82433535b6e3503f9a8e7c4e60aac066d44718', b'ccc8b4ca73b266b4a0c89a9d33c4ec7532b434c9294c26832355e5e2bee2e005', b'280c074d8982e56d70c404072252c309694a6e5c05457a6abbe8fc225c2dfd52', b'546cee26da713a3a64b2066d5e3a52b7c1d927396d1ba8a3d9f6e3e973398856', b'7fbc4615d5e819eee22db440c5bcc4ff25bb046841c41a192003a6d9abfbafbf', b'5b63f13011cab965feea3a41fac2d7a877aa710ab20e2a9a1708474e3c05c050', b'394b36f528947557d317fd40a4adde5514c8745a5f64185421fa2c0c4a158938', b'8f101c8f5290ae6c0dd76d210b7effacd7f12db18f3befab711f533bde084c76', b'6637a656f897a66080fbe60027d32c3f4ebc0e3b5f96123a33f932a091b039c2', b'2815aa6667c042a3a4565fb789890cd33e380d047ed712759d097d479df71051', b'120e761c6382b07a9548650a20b3b9dd74b906093260fa6f92f790ba71f79e8d', b'823c8a613ea539f730a968518993195174bf973ed75c734b6898022867165d7b' ] expected_hardened_privkeys = [ b'abdba45b0459e7804beb68edb899e58a5c2636bf67d096711904001406afbd4c', b'c9e804d4b8fdd99ef6ab2b0ca627a57f4283c28e11e9152ad9d3f863404d940e', b'4cf87d68ae99711261f8cb8e1bde83b8703ff5d689ef70ce23106d1e6e8ed4bd', b'dbf8d578c77f9bf62bb2ad40975e253af1e1d44d53abf84a22d2be29b9488f7f', b'633bb840505521ffe39cb89a04fb8bff3298d6b64a5d8f170aca1e456d6f89b9', b'92e80a38791bd8ba2105b9867fd58ac2cc4fb9853e18141b7fee1884bc5aae69', b'd3663339af1386d05dd90ee20f627661ae87ddb1db0c2dc73fd8a4485930d0e7', b'09a448303452d241b8a25670b36cc758975b97e88f62b6f25cd9084535e3c13a', b'ee22eb77df05ff53e9c2ba797c1f2ebf97ec4cf5a99528adec94972674aeabed', b'935facccb6120659c5b7c606a457c797e5a10ce4a728346e1a3a963251169651', b'8ac9b4a48da1def375640ca03bc6711040dfd4eea7106d42bb4c2de83d7f595e', b'51ecd3f7565c2b86d5782dbde2175ab26a7b896022564063fafe153588610be9', b'04918252f6b6f51cd75957289b56a324b45cc085df80839137d740f9ada6c062', b'2efbd0c839af971e3769c26938d776990ebf097989df4861535a7547a2701483', b'85c6e31e6b27bd188291a910f4a7faba7fceb3e09df72884b10907ecc1491cd0', b'05e245131885bebda993a31bb14ac98b794062a50af639ad22010aed1e533a54', b'ddca42cf7db93f3a3f0723d5fee4c21bf60b7afac35d5c30eb34bd91b35cc609', b'324a5c16030e0c3947e4dcd2b5057fd3a4d5bed96b23e3b476b2af0ab76369c9', b'da63c41cdb398cdcd93e832f3e198528afbb4065821b026c143cec910d8362f0' ] ================================================ FILE: tests/unit/wallet/server/__init__.py ================================================ ================================================ FILE: tests/unit/wallet/server/test_migration.py ================================================ # import unittest # from shutil import rmtree # from tempfile import mkdtemp # # from lbry.wallet.server.history import History # from lbry.wallet.server.storage import LevelDB # # # # dumped from a real history database. Aside from the state, all records are <hashX><flush_count>: <value> # STATE_RECORD = (b'state\x00\x00', b"{'flush_count': 21497, 'comp_flush_count': -1, 'comp_cursor': -1, 'db_version': 0}") # UNMIGRATED_RECORDS = { # '00538b2cbe4a5f1be2dc320241': 'f5ed500142ee5001', # '00538b48def1904014880501f2': 'b9a52a01baa52a01', # '00538cdcf989b74de32c5100ca': 'c973870078748700', # '00538d42d5df44603474284ae1': 'f5d9d802', # '00538d42d5df44603474284ae2': '75dad802', # '00538ebc879dac6ddbee9e0029': '3ca42f0042a42f00', # '00538ed1d391327208748200bc': '804e7d00af4e7d00', # '00538f3de41d9e33affa0300c2': '7de8810086e88100', # '00539007f87792d98422c505a5': '8c5a7202445b7202', # '0053902cf52ee9682d633b0575': 'eb0f64026c106402', # '005390e05674571551632205a2': 'a13d7102e13d7102', # '0053914ef25a9ceed927330584': '78096902960b6902', # '005391768113f69548f37a01b1': 'a5b90b0114ba0b01', # '005391a289812669e5b44c02c2': '33da8a016cdc8a01', # } # # # class TestHistoryDBMigration(unittest.TestCase): # def test_migrate_flush_count_from_16_to_32_bits(self): # self.history = History() # tmpdir = mkdtemp() # self.addCleanup(lambda: rmtree(tmpdir)) # LevelDB.import_module() # db = LevelDB(tmpdir, 'hist', True) # with db.write_batch() as batch: # for key, value in UNMIGRATED_RECORDS.items(): # batch.put(bytes.fromhex(key), bytes.fromhex(value)) # batch.put(*STATE_RECORD) # self.history.db = db # self.history.read_state() # self.assertEqual(21497, self.history.flush_count) # self.assertEqual(0, self.history.db_version) # self.assertTrue(self.history.needs_migration) # self.history.migrate() # self.assertFalse(self.history.needs_migration) # self.assertEqual(1, self.history.db_version) # for idx, (key, value) in enumerate(sorted(db.iterator())): # if key == b'state\x00\x00': # continue # key, counter = key[:-4], key[-4:] # expected_value = UNMIGRATED_RECORDS[key.hex() + counter.hex()[-4:]] # self.assertEqual(value.hex(), expected_value) # # # if __name__ == '__main__': # unittest.main() ================================================ FILE: tests/unit/wallet/test_account.py ================================================ import asyncio from binascii import hexlify from lbry.testcase import AsyncioTestCase from lbry.wallet import ( Wallet, Ledger, Database, Headers, Account, SingleKey, HierarchicalDeterministic, DeterministicChannelKeyManager ) class TestAccount(AsyncioTestCase): async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) await self.ledger.db.open() async def asyncTearDown(self): await self.ledger.db.close() async def test_generate_account(self): account = Account.generate(self.ledger, Wallet(), 'lbryum') self.assertEqual(account.ledger, self.ledger) self.assertIsNotNone(account.seed) self.assertEqual(account.public_key.ledger, self.ledger) self.assertEqual(account.private_key.public_key, account.public_key) self.assertEqual(account.public_key.ledger, self.ledger) self.assertEqual(account.private_key.public_key, account.public_key) addresses = await account.receiving.get_addresses() self.assertEqual(len(addresses), 0) addresses = await account.change.get_addresses() self.assertEqual(len(addresses), 0) await account.ensure_address_gap() addresses = await account.receiving.get_addresses() self.assertEqual(len(addresses), 20) addresses = await account.change.get_addresses() self.assertEqual(len(addresses), 6) async def test_unused_address_on_account_creation_does_not_cause_a_race(self): account = Account.generate(self.ledger, Wallet(), 'lbryum') await account.ledger.db.db.executescript("update pubkey_address set used_times=10") await account.receiving.address_generator_lock.acquire() delayed1 = asyncio.ensure_future(account.receiving.ensure_address_gap()) delayed = asyncio.ensure_future(account.receiving.get_or_create_usable_address()) await asyncio.sleep(0) # wallet being created and queried at the same time account.receiving.address_generator_lock.release() await delayed1 await delayed async def test_generate_keys_over_batch_threshold_saves_it_properly(self): account = Account.generate(self.ledger, Wallet(), 'lbryum') async with account.receiving.address_generator_lock: await account.receiving._generate_keys(0, 200) records = await account.receiving.get_address_records() self.assertEqual(len(records), 201) async def test_ensure_address_gap(self): account = Account.generate(self.ledger, Wallet(), 'lbryum') self.assertIsInstance(account.receiving, HierarchicalDeterministic) async with account.receiving.address_generator_lock: await account.receiving._generate_keys(4, 7) await account.receiving._generate_keys(0, 3) await account.receiving._generate_keys(8, 11) records = await account.receiving.get_address_records() self.assertListEqual( [r['pubkey'].n for r in records], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] ) # we have 12, but default gap is 20 new_keys = await account.receiving.ensure_address_gap() self.assertEqual(len(new_keys), 8) records = await account.receiving.get_address_records() self.assertListEqual( [r['pubkey'].n for r in records], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] ) # case #1: no new addresses needed empty = await account.receiving.ensure_address_gap() self.assertEqual(len(empty), 0) # case #2: only one new addressed needed records = await account.receiving.get_address_records() await self.ledger.db.set_address_history(records[0]['address'], 'a:1:') new_keys = await account.receiving.ensure_address_gap() self.assertEqual(len(new_keys), 1) # case #3: 20 addresses needed await self.ledger.db.set_address_history(new_keys[0], 'a:1:') new_keys = await account.receiving.ensure_address_gap() self.assertEqual(len(new_keys), 20) async def test_get_or_create_usable_address(self): account = Account.generate(self.ledger, Wallet(), 'lbryum') keys = await account.receiving.get_addresses() self.assertEqual(len(keys), 0) address = await account.receiving.get_or_create_usable_address() self.assertIsNotNone(address) keys = await account.receiving.get_addresses() self.assertEqual(len(keys), 20) async def test_generate_account_from_seed(self): account = Account.from_dict( self.ledger, Wallet(), { "seed": "carbon smart garage balance margin twelve chest sword toas" "t envelope bottom stomach absent" } ) self.assertEqual( account.private_key.extended_key_string(), 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7DRNLEoB8' 'HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe' ) self.assertEqual( account.public_key.extended_key_string(), 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMmDgp66FxH' 'uDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9' ) address = await account.receiving.ensure_address_gap() self.assertEqual(address[0], 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx') private_key = await self.ledger.get_private_key_for_address( account.wallet, 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx' ) self.assertEqual( private_key.extended_key_string(), 'xprv9vwXVierUTT4hmoe3dtTeBfbNv1ph2mm8RWXARU6HsZjBaAoFaS2FRQu4fptR' 'AyJWhJW42dmsEaC1nKnVKKTMhq3TVEHsNj1ca3ciZMKktT' ) private_key = await self.ledger.get_private_key_for_address( account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX' ) self.assertIsNone(private_key) async def test_load_and_save_account(self): account_data = { 'name': 'Main Account', 'modified_on': 123, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7DRNLEoB8' 'HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMmDgp66FxH' 'uDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', 'certificates': {}, 'address_generator': { 'name': 'deterministic-chain', 'receiving': {'gap': 17, 'maximum_uses_per_address': 2}, 'change': {'gap': 10, 'maximum_uses_per_address': 2} } } account = Account.from_dict(self.ledger, Wallet(), account_data) await account.ensure_address_gap() addresses = await account.receiving.get_addresses() self.assertEqual(len(addresses), 17) addresses = await account.change.get_addresses() self.assertEqual(len(addresses), 10) account_data['ledger'] = 'lbc_mainnet' self.assertDictEqual(account_data, account.to_dict()) async def test_save_max_gap(self): account = Account.generate( self.ledger, Wallet(), 'lbryum', { 'name': 'deterministic-chain', 'receiving': {'gap': 3, 'maximum_uses_per_address': 2}, 'change': {'gap': 4, 'maximum_uses_per_address': 2} } ) self.assertEqual(account.receiving.gap, 3) self.assertEqual(account.change.gap, 4) await account.save_max_gap() self.assertEqual(account.receiving.gap, 20) self.assertEqual(account.change.gap, 6) # doesn't fail for single-address account account2 = Account.generate(self.ledger, Wallet(), 'lbryum', {'name': 'single-address'}) await account2.save_max_gap() def test_merge_diff(self): account_data = { 'name': 'My Account', 'modified_on': 123, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': 'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp' '5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna', 'public_key': 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6mKUMJFc7' 'UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', 'address_generator': { 'name': 'deterministic-chain', 'receiving': {'gap': 5, 'maximum_uses_per_address': 2}, 'change': {'gap': 5, 'maximum_uses_per_address': 2} } } account = Account.from_dict(self.ledger, Wallet(), account_data) self.assertEqual(account.name, 'My Account') self.assertEqual(account.modified_on, 123) self.assertEqual(account.change.gap, 5) self.assertEqual(account.change.maximum_uses_per_address, 2) self.assertEqual(account.receiving.gap, 5) self.assertEqual(account.receiving.maximum_uses_per_address, 2) account_data['name'] = 'Changed Name' account_data['address_generator']['change']['gap'] = 6 account_data['address_generator']['change']['maximum_uses_per_address'] = 7 account_data['address_generator']['receiving']['gap'] = 8 account_data['address_generator']['receiving']['maximum_uses_per_address'] = 9 account.merge(account_data) # no change because modified_on is not newer self.assertEqual(account.name, 'My Account') account_data['modified_on'] = 200.00 account.merge(account_data) self.assertEqual(account.name, 'Changed Name') self.assertEqual(account.change.gap, 6) self.assertEqual(account.change.maximum_uses_per_address, 7) self.assertEqual(account.receiving.gap, 8) self.assertEqual(account.receiving.maximum_uses_per_address, 9) class TestSingleKeyAccount(AsyncioTestCase): async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) await self.ledger.db.open() self.account = Account.generate(self.ledger, Wallet(), "torba", {'name': 'single-address'}) async def asyncTearDown(self): await self.ledger.db.close() async def test_generate_account(self): account = self.account self.assertEqual(account.ledger, self.ledger) self.assertIsNotNone(account.seed) self.assertEqual(account.public_key.ledger, self.ledger) self.assertEqual(account.private_key.public_key, account.public_key) addresses = await account.receiving.get_addresses() self.assertEqual(len(addresses), 0) addresses = await account.change.get_addresses() self.assertEqual(len(addresses), 0) await account.ensure_address_gap() addresses = await account.receiving.get_addresses() self.assertEqual(len(addresses), 1) self.assertEqual(addresses[0], account.public_key.address) addresses = await account.change.get_addresses() self.assertEqual(len(addresses), 1) self.assertEqual(addresses[0], account.public_key.address) addresses = await account.get_addresses() self.assertEqual(len(addresses), 1) self.assertEqual(addresses[0], account.public_key.address) async def test_ensure_address_gap(self): account = self.account self.assertIsInstance(account.receiving, SingleKey) addresses = await account.receiving.get_addresses() self.assertListEqual(addresses, []) # we have 12, but default gap is 20 new_keys = await account.receiving.ensure_address_gap() self.assertEqual(len(new_keys), 1) self.assertEqual(new_keys[0], account.public_key.address) records = await account.receiving.get_address_records() pubkey = records[0].pop('pubkey') self.assertListEqual(records, [{ 'chain': 0, 'account': account.public_key.address, 'address': account.public_key.address, 'history': None, 'used_times': 0 }]) self.assertEqual( pubkey.extended_key_string(), account.public_key.extended_key_string() ) # case #1: no new addresses needed empty = await account.receiving.ensure_address_gap() self.assertEqual(len(empty), 0) # case #2: after use, still no new address needed records = await account.receiving.get_address_records() await self.ledger.db.set_address_history(records[0]['address'], 'a:1:') empty = await account.receiving.ensure_address_gap() self.assertEqual(len(empty), 0) async def test_get_or_create_usable_address(self): account = self.account addresses = await account.receiving.get_addresses() self.assertEqual(len(addresses), 0) address1 = await account.receiving.get_or_create_usable_address() self.assertIsNotNone(address1) await self.ledger.db.set_address_history(address1, 'a:1:b:2:c:3:') records = await account.receiving.get_address_records() self.assertEqual(records[0]['used_times'], 3) address2 = await account.receiving.get_or_create_usable_address() self.assertEqual(address1, address2) keys = await account.receiving.get_addresses() self.assertEqual(len(keys), 1) async def test_generate_account_from_seed(self): account = Account.from_dict( self.ledger, Wallet(), { "seed": "carbon smart garage balance margin twelve chest sword toas" "t envelope bottom stomach absent", 'address_generator': {'name': 'single-address'} } ) self.assertEqual( account.private_key.extended_key_string(), 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', ) self.assertEqual( account.public_key.extended_key_string(), 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EM' 'mDgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', ) address = await account.receiving.ensure_address_gap() self.assertEqual(address[0], account.public_key.address) private_key = await self.ledger.get_private_key_for_address( account.wallet, address[0] ) self.assertEqual( private_key.extended_key_string(), 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', ) invalid_key = await self.ledger.get_private_key_for_address( account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX' ) self.assertIsNone(invalid_key) self.assertEqual( hexlify(private_key.wif()), b'1cef6c80310b1bcbcfa3176ea809ac840f48cda634c475d402e6bd68d5bb3827d601' ) async def test_load_and_save_account(self): account_data = { 'name': 'My Account', 'modified_on': 123, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EM' 'mDgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', 'address_generator': {'name': 'single-address'}, 'certificates': {} } account = Account.from_dict(self.ledger, Wallet(), account_data) await account.ensure_address_gap() addresses = await account.receiving.get_addresses() self.assertEqual(len(addresses), 1) addresses = await account.change.get_addresses() self.assertEqual(len(addresses), 1) self.maxDiff = None account_data['ledger'] = 'lbc_mainnet' self.assertDictEqual(account_data, account.to_dict()) class AccountEncryptionTests(AsyncioTestCase): password = "password" init_vector = b'0000000000000000' unencrypted_account = { 'name': 'My Account', 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7DRNLEo' 'B8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6mKUMJFc7' 'UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', 'address_generator': {'name': 'single-address'} } encrypted_account = { 'name': 'My Account', 'seed': "MDAwMDAwMDAwMDAwMDAwMJ4e4W4pE6nQtPiD6MujNIQ7aFPhUBl63GwPziAgGN" "MBTMoaSjZfyyvw7ELMCqAYTWJ61aV7K4lmd2hR11g9dpdnnpCb9f9j3zLZHRv7+" "bIkZ//trah9AIkmrc/ZvNkC0Q==", 'encrypted': True, 'private_key': 'MDAwMDAwMDAwMDAwMDAwMLkWikOLScA/ZxlFSGU7dl8pqVjgdpu1S3MWQF3IJ5H' 'OXPAQcgnhHldVq98uP7Q8JqSWOv1p4gpxGSYnA4w5Gbuh0aUD4hmV70m7nVTj7T' '15+Pu30DCspndru59pee/S+mShoK68q7t7r32leaVIfzw=', 'public_key': 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6mKUMJFc7' 'UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', 'address_generator': {'name': 'single-address'} } async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) def test_encrypt_wallet(self): account = Account.from_dict(self.ledger, Wallet(), self.unencrypted_account) account.init_vectors = { 'seed': self.init_vector, 'private_key': self.init_vector } self.assertFalse(account.encrypted) self.assertIsNotNone(account.private_key) account.encrypt(self.password) self.assertTrue(account.encrypted) self.assertEqual(account.seed, self.encrypted_account['seed']) self.assertEqual(account.private_key_string, self.encrypted_account['private_key']) self.assertIsNone(account.private_key) self.assertEqual(account.to_dict()['seed'], self.encrypted_account['seed']) self.assertEqual(account.to_dict()['private_key'], self.encrypted_account['private_key']) account.decrypt(self.password) self.assertEqual(account.init_vectors['private_key'], self.init_vector) self.assertEqual(account.init_vectors['seed'], self.init_vector) self.assertEqual(account.seed, self.unencrypted_account['seed']) self.assertEqual(account.private_key.extended_key_string(), self.unencrypted_account['private_key']) self.assertEqual(account.to_dict(encrypt_password=self.password)['seed'], self.encrypted_account['seed']) self.assertEqual(account.to_dict(encrypt_password=self.password)['private_key'], self.encrypted_account['private_key']) self.assertFalse(account.encrypted) def test_decrypt_wallet(self): account = Account.from_dict(self.ledger, Wallet(), self.encrypted_account) self.assertTrue(account.encrypted) account.decrypt(self.password) self.assertEqual(account.init_vectors['private_key'], self.init_vector) self.assertEqual(account.init_vectors['seed'], self.init_vector) self.assertFalse(account.encrypted) self.assertEqual(account.seed, self.unencrypted_account['seed']) self.assertEqual(account.private_key.extended_key_string(), self.unencrypted_account['private_key']) self.assertEqual(account.to_dict(encrypt_password=self.password)['seed'], self.encrypted_account['seed']) self.assertEqual(account.to_dict(encrypt_password=self.password)['private_key'], self.encrypted_account['private_key']) self.assertEqual(account.to_dict()['seed'], self.unencrypted_account['seed']) self.assertEqual(account.to_dict()['private_key'], self.unencrypted_account['private_key']) def test_encrypt_decrypt_read_only_account(self): account_data = self.unencrypted_account.copy() del account_data['seed'] del account_data['private_key'] account = Account.from_dict(self.ledger, Wallet(), account_data) encrypted = account.to_dict('password') self.assertFalse(encrypted['seed']) self.assertFalse(encrypted['private_key']) account.encrypt('password') account.decrypt('password') ================================================ FILE: tests/unit/wallet/test_bcd_data_stream.py ================================================ import unittest from lbry.wallet.bcd_data_stream import BCDataStream class TestBCDataStream(unittest.TestCase): def test_write_read(self): s = BCDataStream() s.write_string(b'a'*252) s.write_string(b'b'*254) s.write_string(b'c'*(0xFFFF + 1)) # s.write_string(b'd'*(0xFFFFFFFF + 1)) s.write_boolean(True) s.write_boolean(False) s.reset() self.assertEqual(s.read_string(), b'a'*252) self.assertEqual(s.read_string(), b'b'*254) self.assertEqual(s.read_string(), b'c'*(0xFFFF + 1)) # self.assertEqual(s.read_string(), b'd'*(0xFFFFFFFF + 1)) self.assertTrue(s.read_boolean()) self.assertFalse(s.read_boolean()) ================================================ FILE: tests/unit/wallet/test_bip32.py ================================================ from binascii import unhexlify, hexlify from lbry.testcase import AsyncioTestCase from lbry.wallet.bip32 import PublicKey, PrivateKey, from_extended_key_string from lbry.wallet import Ledger, Database, Headers from tests.unit.wallet.key_fixtures import expected_ids, expected_privkeys, expected_hardened_privkeys class BIP32Tests(AsyncioTestCase): def test_pubkey_validation(self): with self.assertRaisesRegex(TypeError, 'chain code must be raw bytes'): PublicKey(None, None, 1, None, None, None) with self.assertRaisesRegex(ValueError, 'invalid chain code'): PublicKey(None, None, b'abcd', None, None, None) with self.assertRaisesRegex(ValueError, 'invalid child number'): PublicKey(None, None, b'abcd'*8, -1, None, None) with self.assertRaisesRegex(ValueError, 'invalid depth'): PublicKey(None, None, b'abcd'*8, 0, 256, None) with self.assertRaisesRegex(TypeError, 'pubkey must be raw bytes'): PublicKey(None, None, b'abcd'*8, 0, 255, None) with self.assertRaisesRegex(ValueError, 'pubkey must be 33 bytes'): PublicKey(None, b'abcd', b'abcd'*8, 0, 255, None) with self.assertRaisesRegex(ValueError, 'invalid pubkey prefix byte'): PublicKey( None, unhexlify('33d1a3dc8155673bc1e2214fa493ccc82d57961b66054af9b6b653ac28eeef3ffe'), b'abcd'*8, 0, 255, None ) pubkey = PublicKey( # success None, unhexlify('03d1a3dc8155673bc1e2214fa493ccc82d57961b66054af9b6b653ac28eeef3ffe'), b'abcd'*8, 0, 1, None ) with self.assertRaisesRegex(ValueError, 'invalid BIP32 public key child number'): pubkey.child(-1) for i in range(20): new_key = pubkey.child(i) self.assertIsInstance(new_key, PublicKey) self.assertEqual(hexlify(new_key.identifier()), expected_ids[i]) async def test_private_key_validation(self): with self.assertRaisesRegex(TypeError, 'private key must be raw bytes'): PrivateKey(None, None, b'abcd'*8, 0, 255) with self.assertRaisesRegex(ValueError, 'private key must be 32 bytes'): PrivateKey(None, b'abcd', b'abcd'*8, 0, 255) private_key = PrivateKey( Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:'), }), unhexlify('2423f3dc6087d9683f73a684935abc0ccd8bc26370588f56653128c6a6f0bf7c'), b'abcd'*8, 0, 1 ) ec_point = private_key.ec_point() self.assertEqual( ec_point[0], 30487144161998778625547553412379759661411261804838752332906558028921886299019 ) self.assertEqual( ec_point[1], 86198965946979720220333266272536217633917099472454294641561154971209433250106 ) self.assertEqual('bUDcmraBp2zCV3QWmVVeQaEgepbs1b2gC9', private_key.address) with self.assertRaisesRegex(ValueError, 'invalid BIP32 private key child number'): private_key.child(-1) self.assertIsInstance(private_key.child(PrivateKey.HARDENED), PrivateKey) async def test_private_key_derivation(self): private_key = PrivateKey( Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:'), }), unhexlify('2423f3dc6087d9683f73a684935abc0ccd8bc26370588f56653128c6a6f0bf7c'), b'abcd'*8, 0, 1 ) for i in range(20): new_privkey = private_key.child(i) self.assertIsInstance(new_privkey, PrivateKey) self.assertEqual(hexlify(new_privkey.private_key_bytes), expected_privkeys[i]) for i in range(PrivateKey.HARDENED + 1, private_key.HARDENED + 20): new_privkey = private_key.child(i) self.assertIsInstance(new_privkey, PrivateKey) self.assertEqual(hexlify(new_privkey.private_key_bytes), expected_hardened_privkeys[i - 1 - PrivateKey.HARDENED]) async def test_from_extended_keys(self): ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:'), }) self.assertIsInstance( from_extended_key_string( ledger, 'xprv9s21ZrQH143K2dyhK7SevfRG72bYDRNv25yKPWWm6dqApNxm1Zb1m5gGcBWYfbsPjTr2v5joit8Af2Zp5P' '6yz3jMbycrLrRMpeAJxR8qDg8', ), PrivateKey ) self.assertIsInstance( from_extended_key_string( ledger, 'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f' 'iW44g14WF52fYC5J483wqQ5ZP', ), PublicKey ) ================================================ FILE: tests/unit/wallet/test_claim_proofs.py ================================================ import unittest from binascii import hexlify, unhexlify from lbry.wallet.claim_proofs import get_hash_for_outpoint, verify_proof from lbry.crypto.hash import double_sha256 class ClaimProofsTestCase(unittest.TestCase): def test_verify_proof(self): claim1_name = 97 # 'a' claim1_txid = 'bd9fa7ffd57d810d4ce14de76beea29d847b8ac34e8e536802534ecb1ca43b68' claim1_outpoint = 0 claim1_height = 10 claim1_node_hash = get_hash_for_outpoint( unhexlify(claim1_txid)[::-1], claim1_outpoint, claim1_height) claim2_name = 98 # 'b' claim2_txid = 'ad9fa7ffd57d810d4ce14de76beea29d847b8ac34e8e536802534ecb1ca43b68' claim2_outpoint = 1 claim2_height = 5 claim2_node_hash = get_hash_for_outpoint( unhexlify(claim2_txid)[::-1], claim2_outpoint, claim2_height) to_hash1 = claim1_node_hash hash1 = double_sha256(to_hash1) to_hash2 = bytes((claim1_name,)) + hash1 + bytes((claim2_name,)) + claim2_node_hash root_hash = double_sha256(to_hash2) proof = { 'last takeover height': claim1_height, 'txhash': claim1_txid, 'nOut': claim1_outpoint, 'nodes': [ {'children': [ {'character': 97}, { 'character': 98, 'nodeHash': hexlify(claim2_node_hash[::-1]) } ]}, {'children': []}, ] } out = verify_proof(proof, hexlify(root_hash[::-1]), 'a') self.assertTrue(out) ================================================ FILE: tests/unit/wallet/test_coinselection.py ================================================ from types import GeneratorType from lbry.testcase import AsyncioTestCase from lbry.wallet import Ledger, Database, Headers from lbry.wallet.coinselection import CoinSelector, MAXIMUM_TRIES from lbry.constants import CENT from tests.unit.wallet.test_transaction import get_output as utxo NULL_HASH = b'\x00'*32 def search(*args, **kwargs): selection = CoinSelector(*args[1:], **kwargs).select(args[0], 'branch_and_bound') return [o.txo.amount for o in selection] if selection else selection class BaseSelectionTestCase(AsyncioTestCase): async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:'), }) await self.ledger.db.open() async def asyncTearDown(self): await self.ledger.db.close() def estimates(self, *args): txos = args[0] if isinstance(args[0], (GeneratorType, list)) else args return [txo.get_estimator(self.ledger) for txo in txos] class TestCoinSelectionTests(BaseSelectionTestCase): def test_empty_coins(self): self.assertListEqual(CoinSelector(0, 0).select([]), []) def test_skip_binary_search_if_total_not_enough(self): fee = utxo(CENT).get_estimator(self.ledger).fee big_pool = self.estimates(utxo(CENT+fee) for _ in range(100)) selector = CoinSelector(101 * CENT, 0) self.assertListEqual(selector.select(big_pool), []) self.assertEqual(selector.tries, 0) # Never tried. # check happy path selector = CoinSelector(100 * CENT, 0) self.assertEqual(len(selector.select(big_pool)), 100) self.assertEqual(selector.tries, 201) def test_exact_match(self): fee = utxo(CENT).get_estimator(self.ledger).fee utxo_pool = self.estimates( utxo(CENT + fee), utxo(CENT), utxo(CENT - fee) ) selector = CoinSelector(CENT, 0) match = selector.select(utxo_pool) self.assertListEqual([CENT + fee], [c.txo.amount for c in match]) self.assertTrue(selector.exact_match) def test_random_draw(self): utxo_pool = self.estimates( utxo(2 * CENT), utxo(3 * CENT), utxo(4 * CENT) ) selector = CoinSelector(CENT, 0, '\x00') match = selector.select(utxo_pool) self.assertListEqual([2 * CENT], [c.txo.amount for c in match]) self.assertFalse(selector.exact_match) def test_pick(self): utxo_pool = self.estimates( utxo(1*CENT), utxo(1*CENT), utxo(3*CENT), utxo(5*CENT), utxo(10*CENT), ) selector = CoinSelector(3*CENT, 0) match = selector.select(utxo_pool) self.assertListEqual([5*CENT], [c.txo.amount for c in match]) def test_confirmed_strategies(self): utxo_pool = self.estimates( utxo(11*CENT, height=5), utxo(11*CENT, height=0), utxo(11*CENT, height=-2), utxo(11*CENT, height=5), ) match = CoinSelector(20*CENT, 0).select(utxo_pool, "only_confirmed") self.assertListEqual([5, 5], [c.txo.tx_ref.height for c in match]) match = CoinSelector(25*CENT, 0).select(utxo_pool, "only_confirmed") self.assertListEqual([], [c.txo.tx_ref.height for c in match]) match = CoinSelector(20*CENT, 0).select(utxo_pool, "prefer_confirmed") self.assertListEqual([5, 5], [c.txo.tx_ref.height for c in match]) match = CoinSelector(25*CENT, 0, '\x00').select(utxo_pool, "prefer_confirmed") self.assertListEqual([5, 0, -2], [c.txo.tx_ref.height for c in match]) class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase): # Bitcoin implementation: # https://github.com/bitcoin/bitcoin/blob/master/src/wallet/coinselection.cpp # # Bitcoin implementation tests: # https://github.com/bitcoin/bitcoin/blob/master/src/wallet/test/coinselector_tests.cpp # # Branch and Bound coin selection white paper: # https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf def make_hard_case(self, utxos): target = 0 utxo_pool = [] for i in range(utxos): amount = 1 << (utxos+i) target += amount utxo_pool.append(utxo(amount)) utxo_pool.append(utxo(amount + (1 << (utxos-1-i)))) return self.estimates(utxo_pool), target def test_branch_and_bound_coin_selection(self): self.ledger.fee_per_byte = 0 utxo_pool = self.estimates( utxo(1 * CENT), utxo(2 * CENT), utxo(3 * CENT), utxo(4 * CENT) ) # Select 1 Cent self.assertListEqual([1 * CENT], search(utxo_pool, 1 * CENT, 0.5 * CENT)) # Select 2 Cent self.assertListEqual([2 * CENT], search(utxo_pool, 2 * CENT, 0.5 * CENT)) # Select 5 Cent self.assertListEqual([3 * CENT, 2 * CENT], search(utxo_pool, 5 * CENT, 0.5 * CENT)) # Select 11 Cent, not possible self.assertListEqual([], search(utxo_pool, 11 * CENT, 0.5 * CENT)) # Select 10 Cent utxo_pool += self.estimates(utxo(5 * CENT)) self.assertListEqual( [4 * CENT, 3 * CENT, 2 * CENT, 1 * CENT], search(utxo_pool, 10 * CENT, 0.5 * CENT) ) # Negative effective value # Select 10 Cent but have 1 Cent not be possible because too small # TODO: bitcoin has [5, 3, 2] self.assertListEqual( [4 * CENT, 3 * CENT, 2 * CENT, 1 * CENT], search(utxo_pool, 10 * CENT, 5000) ) # Select 0.25 Cent, not possible self.assertListEqual(search(utxo_pool, 0.25 * CENT, 0.5 * CENT), []) # Iteration exhaustion test utxo_pool, target = self.make_hard_case(17) selector = CoinSelector(target, 0) self.assertListEqual(selector.select(utxo_pool, 'branch_and_bound'), []) self.assertEqual(selector.tries, MAXIMUM_TRIES) # Should exhaust utxo_pool, target = self.make_hard_case(14) self.assertIsNotNone(search(utxo_pool, target, 0)) # Should not exhaust # Test same value early bailout optimization utxo_pool = self.estimates([ utxo(7 * CENT), utxo(7 * CENT), utxo(7 * CENT), utxo(7 * CENT), utxo(2 * CENT) ] + [utxo(5 * CENT)]*50000) self.assertListEqual( [7 * CENT, 7 * CENT, 7 * CENT, 7 * CENT, 2 * CENT], search(utxo_pool, 30 * CENT, 5000) ) # Select 1 Cent with pool of only greater than 5 Cent utxo_pool = self.estimates(utxo(i * CENT) for i in range(5, 21)) for _ in range(100): self.assertListEqual(search(utxo_pool, 1 * CENT, 2 * CENT), []) ================================================ FILE: tests/unit/wallet/test_database.py ================================================ import sys import os import unittest import sqlite3 import tempfile import asyncio from concurrent.futures.thread import ThreadPoolExecutor from lbry.wallet import ( Wallet, Account, Ledger, Database, Headers, Transaction, Input ) from lbry.wallet.constants import COIN from lbry.wallet.database import query, interpolate, constraints_to_sql, AIOSQLite from lbry.crypto.hash import sha256 from lbry.testcase import AsyncioTestCase from tests.unit.wallet.test_transaction import get_output, NULL_HASH class TestAIOSQLite(AsyncioTestCase): async def asyncSetUp(self): self.db = await AIOSQLite.connect(':memory:') await self.db.executescript(""" pragma foreign_keys=on; create table parent (id integer primary key, name); create table child (id integer primary key, parent_id references parent); """) await self.db.execute("insert into parent values (1, 'test')") await self.db.execute("insert into child values (2, 1)") @staticmethod def delete_item(transaction): transaction.execute('delete from parent where id=1') async def test_foreign_keys_integrity_error(self): self.assertListEqual([(1, 'test')], await self.db.execute_fetchall("select * from parent")) with self.assertRaises(sqlite3.IntegrityError): await self.db.run(self.delete_item) self.assertListEqual([(1, 'test')], await self.db.execute_fetchall("select * from parent")) await self.db.executescript("pragma foreign_keys=off;") await self.db.run(self.delete_item) self.assertListEqual([], await self.db.execute_fetchall("select * from parent")) async def test_run_without_foreign_keys(self): self.assertListEqual([(1, 'test')], await self.db.execute_fetchall("select * from parent")) await self.db.run_with_foreign_keys_disabled(self.delete_item) self.assertListEqual([], await self.db.execute_fetchall("select * from parent")) async def test_integrity_error_when_foreign_keys_disabled_and_skipped(self): await self.db.executescript("pragma foreign_keys=off;") self.assertListEqual([(1, 'test')], await self.db.execute_fetchall("select * from parent")) with self.assertRaises(sqlite3.IntegrityError): await self.db.run_with_foreign_keys_disabled(self.delete_item) self.assertListEqual([(1, 'test')], await self.db.execute_fetchall("select * from parent")) class TestQueryBuilder(unittest.TestCase): def test_dot(self): self.assertTupleEqual( constraints_to_sql({'txo.position': 18}), ('txo.position = :txo_position0', {'txo_position0': 18}) ) self.assertTupleEqual( constraints_to_sql({'txo.position#6': 18}), ('txo.position = :txo_position6', {'txo_position6': 18}) ) def test_any(self): self.assertTupleEqual( constraints_to_sql({ 'ages__any': { 'txo.age__gt': 18, 'txo.age__lt': 38 } }), ('(txo.age > :ages__any0_txo_age__gt0 OR txo.age < :ages__any0_txo_age__lt0)', { 'ages__any0_txo_age__gt0': 18, 'ages__any0_txo_age__lt0': 38 }) ) def test_in(self): self.assertTupleEqual( constraints_to_sql({'txo.age__in#2': [18, 38]}), ('txo.age IN (:txo_age__in2_0, :txo_age__in2_1)', { 'txo_age__in2_0': 18, 'txo_age__in2_1': 38 }) ) self.assertTupleEqual( constraints_to_sql({'txo.name__in': ('abc123', 'def456')}), ('txo.name IN (:txo_name__in0_0, :txo_name__in0_1)', { 'txo_name__in0_0': 'abc123', 'txo_name__in0_1': 'def456' }) ) self.assertTupleEqual( constraints_to_sql({'txo.name__in': {'abc123'}}), ('txo.name = :txo_name__in0', { 'txo_name__in0': 'abc123', }) ) self.assertTupleEqual( constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}), ('txo.age IN (SELECT age from ages_table)', {}) ) def test_not_in(self): self.assertTupleEqual( constraints_to_sql({'txo.age__not_in': [18, 38]}), ('txo.age NOT IN (:txo_age__not_in0_0, :txo_age__not_in0_1)', { 'txo_age__not_in0_0': 18, 'txo_age__not_in0_1': 38 }) ) self.assertTupleEqual( constraints_to_sql({'txo.name__not_in': ('abc123', 'def456')}), ('txo.name NOT IN (:txo_name__not_in0_0, :txo_name__not_in0_1)', { 'txo_name__not_in0_0': 'abc123', 'txo_name__not_in0_1': 'def456' }) ) self.assertTupleEqual( constraints_to_sql({'txo.name__not_in': ('abc123',)}), ('txo.name != :txo_name__not_in0', { 'txo_name__not_in0': 'abc123', }) ) self.assertTupleEqual( constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}), ('txo.age NOT IN (SELECT age from ages_table)', {}) ) def test_in_invalid(self): with self.assertRaisesRegex(ValueError, 'list, set or string'): constraints_to_sql({'ages__in': 9}) def test_query(self): self.assertTupleEqual( query("select * from foo"), ("select * from foo", {}) ) self.assertTupleEqual( query( "select * from foo", a__not='b', b__in='select * from blah where c=:$c', d__any={'one__like': 'o', 'two': 2}, limit=10, order_by='b', **{'$c': 3}), ( "select * from foo WHERE a != :a__not0 AND " "b IN (select * from blah where c=:$c) AND " "(one LIKE :d__any0_one__like0 OR two = :d__any0_two0) ORDER BY b LIMIT 10", {'a__not0': 'b', 'd__any0_one__like0': 'o', 'd__any0_two0': 2, '$c': 3} ) ) def test_query_order_by(self): self.assertTupleEqual( query("select * from foo", order_by='foo'), ("select * from foo ORDER BY foo", {}) ) self.assertTupleEqual( query("select * from foo", order_by=['foo', 'bar']), ("select * from foo ORDER BY foo, bar", {}) ) with self.assertRaisesRegex(ValueError, 'order_by must be string or list'): query("select * from foo", order_by={'foo': 'bar'}) def test_query_limit_offset(self): self.assertTupleEqual( query("select * from foo", limit=10), ("select * from foo LIMIT 10", {}) ) self.assertTupleEqual( query("select * from foo", offset=10), ("select * from foo OFFSET 10", {}) ) self.assertTupleEqual( query("select * from foo", limit=20, offset=10), ("select * from foo LIMIT 20 OFFSET 10", {}) ) def test_query_interpolation(self): self.maxDiff = None # tests that interpolation replaces longer keys first self.assertEqual( interpolate(*query( "select * from foo", a__not='b', b__in='select * from blah where c=:$c', d__any={'one__like': 'o', 'two': 2}, a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order ahash=sha256(b'hello world'), limit=10, order_by='b', **{'$c': 3}) ), "select * from foo WHERE a != 'b' AND " "b IN (select * from blah where c=3) AND " "(one LIKE 'o' OR two = 2) AND " "a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 " "AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' " "ORDER BY b LIMIT 10" ) class TestQueries(AsyncioTestCase): async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) await self.ledger.headers.open() self.wallet = Wallet() await self.ledger.db.open() async def asyncTearDown(self): await self.ledger.db.close() async def create_account(self, wallet=None): account = Account.generate(self.ledger, wallet or self.wallet) await account.ensure_address_gap() return account async def create_tx_from_nothing(self, my_account, height): to_address = await my_account.receiving.get_or_create_usable_address() to_hash = Ledger.address_to_hash160(to_address) tx = Transaction(height=height, is_verified=True) \ .add_inputs([self.txi(self.txo(1, sha256(str(height).encode())))]) \ .add_outputs([self.txo(1, to_hash)]) await self.ledger.db.insert_transaction(tx) await self.ledger.db.save_transaction_io(tx, to_address, to_hash, '') return tx async def create_tx_from_txo(self, txo, to_account, height): from_hash = txo.script.values['pubkey_hash'] from_address = self.ledger.hash160_to_address(from_hash) to_address = await to_account.receiving.get_or_create_usable_address() to_hash = Ledger.address_to_hash160(to_address) tx = Transaction(height=height, is_verified=True) \ .add_inputs([self.txi(txo)]) \ .add_outputs([self.txo(1, to_hash)]) await self.ledger.db.insert_transaction(tx) await self.ledger.db.save_transaction_io(tx, from_address, from_hash, '') await self.ledger.db.save_transaction_io(tx, to_address, to_hash, '') return tx async def create_tx_to_nowhere(self, txo, height): from_hash = txo.script.values['pubkey_hash'] from_address = self.ledger.hash160_to_address(from_hash) to_hash = NULL_HASH tx = Transaction(height=height, is_verified=True) \ .add_inputs([self.txi(txo)]) \ .add_outputs([self.txo(1, to_hash)]) await self.ledger.db.insert_transaction(tx) await self.ledger.db.save_transaction_io(tx, from_address, from_hash, '') return tx def txo(self, amount, address): return get_output(int(amount*COIN), address) def txi(self, txo): return Input.spend(txo) async def test_large_tx_doesnt_hit_variable_limits(self): # SQLite is usually compiled with 999 variables limit: https://www.sqlite.org/limits.html # This can be removed when there is a better way. See: https://github.com/lbryio/lbry-sdk/issues/2281 fetchall = self.ledger.db.db.execute_fetchall def check_parameters_length(sql, parameters, read_only=False): self.assertLess(len(parameters or []), 999) return fetchall(sql, parameters, read_only) self.ledger.db.db.execute_fetchall = check_parameters_length account = await self.create_account() tx = await self.create_tx_from_nothing(account, 0) for height in range(1, 1200): tx = await self.create_tx_from_txo(tx.outputs[0], account, height=height) variable_limit = self.ledger.db.MAX_QUERY_VARIABLES for limit in range(variable_limit - 2, variable_limit + 2): txs = await self.ledger.get_transactions( accounts=self.wallet.accounts, limit=limit, order_by='height asc') self.assertEqual(len(txs), limit) inputs, outputs, last_tx = set(), set(), txs[0] for tx in txs[1:]: self.assertEqual(len(tx.inputs), 1) self.assertEqual(tx.inputs[0].txo_ref.tx_ref.id, last_tx.id) self.assertEqual(len(tx.outputs), 1) last_tx = tx async def test_queries(self): wallet1 = Wallet() account1 = await self.create_account(wallet1) self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account1])) wallet2 = Wallet() account2 = await self.create_account(wallet2) account3 = await self.create_account(wallet2) self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account2])) self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2, account3])) self.assertEqual(0, await self.ledger.db.get_utxo_count()) self.assertListEqual([], await self.ledger.db.get_utxos()) self.assertEqual(0, await self.ledger.db.get_txo_count()) self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1)) self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2)) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1])) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2])) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3])) tx1 = await self.create_tx_from_nothing(account1, 1) self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account1])) self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account2])) self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account1])) self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1])) self.assertEqual(0, await self.ledger.db.get_txo_count(accounts=[account2])) self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet1)) self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2)) self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account1])) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2])) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3])) tx2 = await self.create_tx_from_txo(tx1.outputs[0], account2, 2) tx2b = await self.create_tx_from_nothing(account3, 2) self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1])) self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account2])) self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account3])) self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account1])) self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1])) self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account2])) self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2])) self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account3])) self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account3])) self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1)) self.assertEqual(10**8+10**8, await self.ledger.db.get_balance(wallet=wallet2)) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1])) self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account2])) self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3])) tx3 = await self.create_tx_to_nowhere(tx2.outputs[0], 3) self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1])) self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account2])) self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account1])) self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1])) self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account2])) self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2])) self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1)) self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet2)) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1])) self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2])) self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3])) txs = await self.ledger.db.get_transactions(accounts=[account1, account2]) self.assertListEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs]) self.assertListEqual([3, 2, 1], [tx.height for tx in txs]) txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts, include_is_my_output=True) self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs]) self.assertEqual(txs[0].inputs[0].is_my_input, True) self.assertEqual(txs[0].outputs[0].is_my_output, False) self.assertEqual(txs[1].inputs[0].is_my_input, False) self.assertEqual(txs[1].outputs[0].is_my_output, True) txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2], include_is_my_output=True) self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs]) self.assertEqual(txs[0].inputs[0].is_my_input, True) self.assertEqual(txs[0].outputs[0].is_my_output, False) self.assertEqual(txs[1].inputs[0].is_my_input, False) self.assertEqual(txs[1].outputs[0].is_my_output, True) self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account2])) tx = await self.ledger.db.get_transaction(txid=tx2.id) self.assertEqual(tx.id, tx2.id) self.assertIsNone(tx.inputs[0].is_my_input) self.assertIsNone(tx.outputs[0].is_my_output) tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id, include_is_my_output=True) self.assertTrue(tx.inputs[0].is_my_input) self.assertFalse(tx.outputs[0].is_my_output) tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id, include_is_my_output=True) self.assertFalse(tx.inputs[0].is_my_input) self.assertTrue(tx.outputs[0].is_my_output) # height 0 sorted to the top with the rest in descending order tx4 = await self.create_tx_from_nothing(account1, 0) txos = await self.ledger.db.get_txos() self.assertListEqual([0, 3, 2, 2, 1], [txo.tx_ref.height for txo in txos]) self.assertListEqual([tx4.id, tx3.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos]) txs = await self.ledger.db.get_transactions(accounts=[account1, account2]) self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs]) self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs]) async def test_empty_history(self): self.assertEqual((None, []), await self.ledger.get_local_status_and_history('')) class TestUpgrade(AsyncioTestCase): def setUp(self) -> None: self.path = tempfile.mktemp() def tearDown(self) -> None: os.remove(self.path) def get_version(self): with sqlite3.connect(self.path) as conn: versions = conn.execute('select version from version').fetchall() assert len(versions) == 1 return versions[0][0] def get_tables(self): with sqlite3.connect(self.path) as conn: sql = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;" return [col[0] for col in conn.execute(sql).fetchall()] def add_address(self, address): with sqlite3.connect(self.path) as conn: conn.execute(""" INSERT INTO account_address (address, account, chain, n, pubkey, chain_code, depth) VALUES (?, 'account1', 0, 0, 'pubkey', 'chain_code', 0) """, (address,)) def get_addresses(self): with sqlite3.connect(self.path) as conn: sql = "SELECT address FROM account_address ORDER BY address;" return [col[0] for col in conn.execute(sql).fetchall()] async def test_reset_on_version_change(self): self.ledger = Ledger({ 'db': Database(self.path), 'headers': Headers(':memory:') }) # initial open, pre-version enabled db self.ledger.db.SCHEMA_VERSION = None self.assertListEqual(self.get_tables(), []) await self.ledger.db.open() self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo']) self.assertListEqual(self.get_addresses(), []) self.add_address('address1') await self.ledger.db.close() # initial open after version enabled self.ledger.db.SCHEMA_VERSION = '1.0' await self.ledger.db.open() self.assertEqual(self.get_version(), '1.0') self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version']) self.assertListEqual(self.get_addresses(), []) # address1 deleted during version upgrade self.add_address('address2') await self.ledger.db.close() # nothing changes self.assertEqual(self.get_version(), '1.0') self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version']) await self.ledger.db.open() self.assertEqual(self.get_version(), '1.0') self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version']) self.assertListEqual(self.get_addresses(), ['address2']) await self.ledger.db.close() # upgrade version, database reset self.ledger.db.SCHEMA_VERSION = '1.1' self.ledger.db.CREATE_TABLES_QUERY += """ create table if not exists foo (bar text); """ await self.ledger.db.open() self.assertEqual(self.get_version(), '1.1') self.assertListEqual(self.get_tables(), ['account_address', 'foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version']) self.assertListEqual(self.get_addresses(), []) # all tables got reset await self.ledger.db.close() class TestSQLiteRace(AsyncioTestCase): max_misuse_attempts = 120000 def setup_db(self): self.db = sqlite3.connect(":memory:", isolation_level=None) self.db.executescript( "create table test1 (id text primary key not null, val text);\n" + "create table test2 (id text primary key not null, val text);\n" + "\n".join(f"insert into test1 values ({v}, NULL);" for v in range(1000)) ) async def asyncSetUp(self): self.executor = ThreadPoolExecutor(1) await self.loop.run_in_executor(self.executor, self.setup_db) async def asyncTearDown(self): await self.loop.run_in_executor(self.executor, self.db.close) self.executor.shutdown() async def test_binding_param_0_error(self): # test real param 0 binding errors for supported_type in [str, int, bytes]: await self.loop.run_in_executor( self.executor, self.db.executemany, "insert into test2 values (?, NULL)", [(supported_type(1), ), (supported_type(2), )] ) await self.loop.run_in_executor( self.executor, self.db.execute, "delete from test2 where id in (1, 2)" ) for unsupported_type in [lambda x: (x, ), lambda x: [x], lambda x: {x}]: try: await self.loop.run_in_executor( self.executor, self.db.executemany, "insert into test2 (id, val) values (?, NULL)", [(unsupported_type(1), ), (unsupported_type(2), )] ) self.assertTrue(False) except sqlite3.InterfaceError as err: self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.") async def test_unhandled_sqlite_misuse(self): # test SQLITE_MISUSE being incorrectly raised as a param 0 binding error attempts = 0 python_version = sys.version.split('\n')[0].rstrip(' ') try: while attempts < self.max_misuse_attempts: f1 = asyncio.wrap_future( self.loop.run_in_executor( self.executor, self.db.executemany, "update test1 set val='derp' where id=?", ((str(i),) for i in range(2)) ) ) f2 = asyncio.wrap_future( self.loop.run_in_executor( self.executor, self.db.executemany, "update test2 set val='derp' where id=?", ((str(i),) for i in range(2)) ) ) attempts += 1 await asyncio.gather(f1, f2) print(f"\nsqlite3 {sqlite3.version}/python {python_version} " f"did not raise SQLITE_MISUSE within {attempts} attempts of the race condition") self.assertTrue(False, 'this test failing means either the sqlite race conditions ' 'have been fixed in cpython or the test max_attempts needs to be increased') except sqlite3.InterfaceError as err: self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.") print(f"\nsqlite3 {sqlite3.version}/python {python_version} raised SQLITE_MISUSE " f"after {attempts} attempts of the race condition") @unittest.SkipTest async def test_fetchall_prevents_sqlite_misuse(self): # test that calling fetchall sufficiently avoids the race attempts = 0 def executemany_fetchall(query, params): self.db.executemany(query, params).fetchall() while attempts < self.max_misuse_attempts: f1 = asyncio.wrap_future( self.loop.run_in_executor( self.executor, executemany_fetchall, "update test1 set val='derp' where id=?", ((str(i),) for i in range(2)) ) ) f2 = asyncio.wrap_future( self.loop.run_in_executor( self.executor, executemany_fetchall, "update test2 set val='derp' where id=?", ((str(i),) for i in range(2)) ) ) attempts += 1 await asyncio.gather(f1, f2) ================================================ FILE: tests/unit/wallet/test_dewies.py ================================================ import unittest from lbry.wallet.dewies import lbc_to_dewies as l2d, dewies_to_lbc as d2l class TestDeweyConversion(unittest.TestCase): def test_good_output(self): self.assertEqual(d2l(1), "0.00000001") self.assertEqual(d2l(10**7), "0.1") self.assertEqual(d2l(2*10**8), "2.0") self.assertEqual(d2l(2*10**17), "2000000000.0") def test_good_input(self): self.assertEqual(l2d("0.00000001"), 1) self.assertEqual(l2d("0.1"), 10**7) self.assertEqual(l2d("1.0"), 10**8) self.assertEqual(l2d("2.00000000"), 2*10**8) self.assertEqual(l2d("2000000000.0"), 2*10**17) def test_bad_input(self): with self.assertRaises(ValueError): l2d("1") with self.assertRaises(ValueError): l2d("-1.0") with self.assertRaises(ValueError): l2d("10000000000.0") with self.assertRaises(ValueError): l2d("1.000000000") with self.assertRaises(ValueError): l2d("-0") with self.assertRaises(ValueError): l2d("1") with self.assertRaises(ValueError): l2d(".1") with self.assertRaises(ValueError): l2d("1e-7") ================================================ FILE: tests/unit/wallet/test_hash.py ================================================ from unittest import TestCase, mock from lbry.crypto.crypt import aes_decrypt, aes_encrypt, better_aes_decrypt, better_aes_encrypt from lbry.error import InvalidPasswordError class TestAESEncryptDecrypt(TestCase): message = 'The Times 03/Jan/2009 Chancellor on brink of second bailout for banks' expected = 'ZmZmZmZmZmZmZmZmZmZmZjlrKptoKD+MFwDxcg3XtCD9qz8UWhEhq/TVJT5+Mtp2a8sE' \ 'CaO6WQj7fYsWGu2Hvbc0qYqxdN0HeTsiO+cZRo3eJISgr3F+rXFYi5oSBlD2' password = 'bubblegum' @mock.patch('os.urandom', side_effect=lambda i: b'd'*i) def test_encrypt_iv_f(self, _): self.assertEqual( aes_encrypt(self.password, self.message), 'ZGRkZGRkZGRkZGRkZGRkZKBP/4pR+47hLHbHyvDJm9aRKDuoBdTG8SrFvHqfagK6Co1VrHUOd' 'oF+6PGSxru3+VR63ybkXLNM75s/qVw+dnKVAkI8OfoVnJvGRSc49e38' ) @mock.patch('os.urandom', side_effect=lambda i: b'f'*i) def test_encrypt_iv_d(self, _): self.assertEqual( aes_encrypt(self.password, self.message), 'ZmZmZmZmZmZmZmZmZmZmZjlrKptoKD+MFwDxcg3XtCD9qz8UWhEhq/TVJT5+Mtp2a8sE' 'CaO6WQj7fYsWGu2Hvbc0qYqxdN0HeTsiO+cZRo3eJISgr3F+rXFYi5oSBlD2' ) self.assertTupleEqual( aes_decrypt(self.password, self.expected), (self.message, b'f' * 16) ) def test_encrypt_decrypt(self): self.assertEqual( aes_decrypt('bubblegum', aes_encrypt('bubblegum', self.message))[0], self.message ) def test_decrypt_error(self): with self.assertRaises(InvalidPasswordError): aes_decrypt('notbubblegum', aes_encrypt('bubblegum', self.message)) def test_edge_case_invalid_password_valid_padding_invalid_unicode(self): with self.assertRaises(InvalidPasswordError): aes_decrypt( 'notbubblegum', 'gy3/mNq3FWB/xAXirOQnlAqQLuvhLGXZaeGBUIg1w6yY4PDLDT7BU83XOfBsJol' 'uWU5zEU4+upOFH35HDqyV8EMQhcKSufN9WkT1izEbFtweBUTK8nTSkV7NBppE1Jaz' ) def test_better_encrypt_decrypt(self): self.assertEqual( b'valuable value', better_aes_decrypt( 'super secret', better_aes_encrypt('super secret', b'valuable value'))) @mock.patch('os.urandom', side_effect=lambda i: b'd'*i) def test_better_decrypt_error(self, _): with self.assertRaises(InvalidPasswordError): better_aes_decrypt( 'super secret but wrong', better_aes_encrypt('super secret', b'valuable value') ) def test_edge_case_invalid_password_valid_everything(self): value = b'czo4MTkyOjE2OjE6VrwsN8FSJlegxHVEQePoyjWT1k8yAXBCUbbGCFKcsNY=' self.assertEqual(b'valuable value', better_aes_decrypt('super secret', value)) self.assertNotEqual(b'valuable value', better_aes_decrypt('super secret but wrong', value)) ================================================ FILE: tests/unit/wallet/test_headers.py ================================================ import os import asyncio import tempfile from binascii import unhexlify from lbry.wallet.util import ArithUint256 from lbry.testcase import AsyncioTestCase from lbry.wallet.ledger import Headers as _Headers class Headers(_Headers): checkpoints = {} def block_bytes(blocks): return blocks * Headers.header_size class TestHeaders(AsyncioTestCase): async def test_deserialize(self): self.maxDiff = None h = Headers(':memory:') await h.open() await h.connect(0, HEADERS) self.assertEqual(await h.get(0), { 'bits': 520159231, 'block_height': 0, 'claim_trie_root': b'0000000000000000000000000000000000000000000000000000000000000001', 'merkle_root': b'b8211c82c3d15bcd78bba57005b86fed515149a53a425eb592c07af99fe559cc', 'nonce': 1287, 'prev_block_hash': b'0000000000000000000000000000000000000000000000000000000000000000', 'timestamp': 1446058291, 'version': 1 }) self.assertEqual(await h.get(10), { 'bits': 509349720, 'block_height': 10, 'merkle_root': b'f4d8fded6a181d4a8a2817a0eb423cc0f414af29490004a620e66c35c498a554', 'claim_trie_root': b'0000000000000000000000000000000000000000000000000000000000000001', 'nonce': 75838, 'prev_block_hash': b'fdab1b38bcf236bc85b6bcd52fe8ec19bcb0b6c7352e913de05fa5a4e5ae8d55', 'timestamp': 1466646593, 'version': 536870912 }) async def test_connect_from_genesis(self): headers = Headers(':memory:') await headers.open() self.assertEqual(headers.height, -1) await headers.connect(0, HEADERS) self.assertEqual(headers.height, 19) async def test_connect_from_middle(self): headers_temporary_file = tempfile.mktemp() self.addCleanup(os.remove, headers_temporary_file) with open(headers_temporary_file, 'w+b') as headers_file: headers_file.write(HEADERS[:block_bytes(10)]) h = Headers(headers_temporary_file) await h.open() self.assertEqual(h.height, 9) await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)]) self.assertEqual(h.height, 19) def test_target_calculation(self): # see: https://github.com/lbryio/lbrycrd/blob/master/src/test/lbry_tests.cpp # 1 test block 1 difficulty, should be a max retarget self.assertEqual( 0x1f00e146, Headers(':memory').get_next_block_target( max_target=ArithUint256(Headers.max_target), previous={'timestamp': 1386475638}, current={'timestamp': 1386475638, 'bits': 0x1f00ffff} ).compact ) # test max retarget (difficulty increase) self.assertEqual( 0x1f008ccc, Headers(':memory').get_next_block_target( max_target=ArithUint256(Headers.max_target), previous={'timestamp': 1386475638}, current={'timestamp': 1386475638, 'bits': 0x1f00a000} ).compact ) # test min retarget (difficulty decrease) self.assertEqual( 0x1f00f000, Headers(':memory').get_next_block_target( max_target=ArithUint256(Headers.max_target), previous={'timestamp': 1386475638}, current={'timestamp': 1386475638 + 60*20, 'bits': 0x1f00a000} ).compact ) # test to see if pow limit is not exceeded self.assertEqual( 0x1f00ffff, Headers(':memory').get_next_block_target( max_target=ArithUint256(Headers.max_target), previous={'timestamp': 1386475638}, current={'timestamp': 1386475638 + 600, 'bits': 0x1f00ffff} ).compact ) def test_get_proof_of_work_hash(self): # see: https://github.com/lbryio/lbrycrd/blob/master/src/test/lbry_tests.cpp self.assertEqual( Headers.header_hash_to_pow_hash(Headers.hash_header(b"test string")), b"485f3920d48a0448034b0852d1489cfa475341176838c7d36896765221be35ce" ) self.assertEqual( Headers.header_hash_to_pow_hash(Headers.hash_header(b"a"*70)), b"eb44af2f41e7c6522fb8be4773661be5baa430b8b2c3a670247e9ab060608b75" ) self.assertEqual( Headers.header_hash_to_pow_hash(Headers.hash_header(b"d"*140)), b"74044747b7c1ff867eb09a84d026b02d8dc539fb6adcec3536f3dfa9266495d9" ) async def test_bounds(self): headers = Headers(':memory:') await headers.open() await headers.connect(0, HEADERS) self.assertEqual(19, headers.height) with self.assertRaises(IndexError): _ = await headers.get(3001) with self.assertRaises(IndexError): _ = await headers.get(-1) self.assertIsNotNone(await headers.get(19)) self.assertIsNotNone(await headers.get(0)) async def test_repair(self): headers = Headers(':memory:') await headers.open() await headers.connect(0, HEADERS[:block_bytes(11)]) self.assertEqual(10, headers.height) await headers.repair() self.assertEqual(10, headers.height) # corrupt the middle of it headers.io.seek(block_bytes(8)) headers.io.write(b"wtf") await headers.repair() self.assertEqual(7, headers.height) self.assertEqual(8, len(headers)) # corrupt by appending headers.io.seek(block_bytes(len(headers))) headers.io.write(b"appending") await headers.repair() self.assertEqual(7, headers.height) await headers.connect(len(headers), HEADERS[block_bytes(8):]) self.assertEqual(19, headers.height) # verify from middle await headers.repair(start_height=10) self.assertEqual(19, headers.height) async def test_do_not_estimate_unconfirmed(self): headers = Headers(':memory:') await headers.open() self.assertIsNone(headers.estimated_timestamp(-1)) self.assertIsNone(headers.estimated_timestamp(0)) self.assertIsNotNone(headers.estimated_timestamp(1)) async def test_dont_estimate_whats_there(self): headers = Headers(':memory:') await headers.open() estimated = headers.estimated_timestamp(10) await headers.connect(0, HEADERS) real_time = (await headers.get(10))['timestamp'] after_downloading_header_estimated = headers.estimated_timestamp(10) self.assertNotEqual(estimated, after_downloading_header_estimated) self.assertEqual(after_downloading_header_estimated, real_time) async def test_misalignment_triggers_repair_on_open(self): headers_temporary_file = tempfile.mktemp() self.addCleanup(os.remove, headers_temporary_file) with open(headers_temporary_file, 'w+b') as headers_file: headers_file.write(HEADERS) headers = Headers(headers_temporary_file) with self.assertLogs(level='WARN') as cm: await headers.open() await headers.close() self.assertEqual(cm.output, []) with open(headers_temporary_file, 'w+b') as headers_file: headers_file.seek(0) headers_file.truncate() headers_file.write(HEADERS[:block_bytes(10)]) headers_file.write(b'ops') headers_file.write(HEADERS[block_bytes(10):]) await headers.open() self.assertEqual( cm.output, [ 'WARNING:lbry.wallet.header:Reader file size doesnt match header size. ' 'Repairing, might take a while.', 'WARNING:lbry.wallet.header:Header file corrupted at height 9, truncating ' 'it.' ] ) async def test_concurrency(self): BLOCKS = 19 headers_temporary_file = tempfile.mktemp() headers = Headers(headers_temporary_file) await headers.open() self.addCleanup(os.remove, headers_temporary_file) async def writer(): for block_index in range(BLOCKS): await headers.connect(block_index, HEADERS[block_bytes(block_index):block_bytes(block_index + 1)]) async def reader(): for block_index in range(BLOCKS): while len(headers) <= block_index: await asyncio.sleep(0.000001) assert (await headers.get(block_index))['block_height'] == block_index reader_task = asyncio.create_task(reader()) await writer() await reader_task await headers.close() HEADERS = unhexlify( b'010000000000000000000000000000000000000000000000000000000000000000000000cc59e59ff97ac092b55e4' b'23aa5495151ed6fb80570a5bb78cd5bd1c3821c21b801000000000000000000000000000000000000000000000000' b'0000000000000033193156ffff001f070500000000002063f4346a4db34fdfce29a70f5e8d11f065f6b91602b7036' b'c7f22f3a03b28899cba888e2f9c037f831046f8ad09f6d378f79c728d003b177a64d29621f481da5d010000000000' b'00000000000000000000000000000000000000000000000000003c406b5746e1001f5b4f000000000020246cb8584' b'3ac936d55388f2ff288b011add5b1b20cca9cfd19a403ca2c9ecbde09d8734d81b5f2eb1b653caf17491544ddfbc7' b'2f2f4c0c3f22a3362db5ba9d4701000000000000000000000000000000000000000000000000000000000000003d4' b'06b57ffff001f4ff20000000000200044e1258b865d262587c28ff98853bc52bb31266230c1c648cc9004047a5428' b'e285dbf24334585b9a924536a717160ee185a86d1eeb7b19684538685eca761a01000000000000000000000000000' b'000000000000000000000000000000000003d406b5746e1001fce9c010000000020bbf8980e3f7604896821203bf6' b'2f97f311124da1fbb95bf523fcfdb356ad19c9d83cf1408debbd631950b7a95b0c940772119cd8a615a3d44601568' b'713fec80c01000000000000000000000000000000000000000000000000000000000000003e406b573dc6001fec7b' b'0000000000201a650b9b7b9d132e257ff6b336ba7cd96b1796357c4fc8dd7d0bd1ff1de057d547638e54178dbdddf' b'2e81a3b7566860e5264df6066755f9760a893f5caecc5790100000000000000000000000000000000000000000000' b'0000000000000000003e406b5773ae001fcf770000000000206d694b93a2bb5ac23a13ed6749a789ca751cf73d598' b'2c459e0cd9d5d303da74cec91627e0dba856b933983425d7f72958e8f974682632a0fa2acee9cfd81940101000000' b'000000000000000000000000000000000000000000000000000000003e406b578399001f225c010000000020b5780' b'8c188b7315583cf120fe89de923583bc7a8ebff03189145b86bf859b21ba3c4a19948a1263722c45c5601fd10a7ae' b'a7cf73bfa45e060508f109155e80ab010000000000000000000000000000000000000000000000000000000000000' b'03f406b571787001f0816070000000020a6a5b330e816242d54c8586ba9b6d63c19d921171ef3d4525b8ffc635742' b'e83a0fc2da46cf0de0057c1b9fc93d997105ff6cf2c8c43269b446c1dbf5ac18be8c0100000000000000000000000' b'00000000000000000000000000000000000000040406b570ae1761edd8f030000000020b8447f415279dffe8a09af' b'e6f6d5e335a2f6911fce8e1d1866723d5e5e8a53067356a733f87e592ea133328792dd9d676ed83771c8ff0f51992' b'8ce752f159ba6010000000000000000000000000000000000000000000000000000000000000040406b57139d681e' b'd40d000000000020558daee5a4a55fe03d912e35c7b6b0bc19ece82fd5bcb685bc36f2bc381babfd54a598c4356ce' b'620a604004929af14f4c03c42eba017288a4a1d186aedfdd8f4010000000000000000000000000000000000000000' b'000000000000000000000041406b57580f5c1e3e280100000000200381bfc0b2f10c9a3c0fc2dc8ad06388aff8ea5' b'a9f7dba6a945073b021796197364b79f33ff3f3a7ccb676fc0a37b7d831bd5942a05eac314658c6a7e4c4b1a40100' b'00000000000000000000000000000000000000000000000000000000000041406b574303511ec0ae0100000000202' b'aae02063ae0f1025e6acecd5e8e2305956ecaefd185bb47a64ea2ae953233891df3d4c1fc547ab3bbca027c8bbba7' b'44c051add8615d289b567f97c64929dcf201000000000000000000000000000000000000000000000000000000000' b'0000042406b578c4a471e04ee00000000002016603ef45d5a7c02bfbb30f422016746872ff37f8b0b5824a0f70caa' b'668eea5415aad300e70f7d8755d93645d1fd21eda9c40c5d0ed797acd0e07ace34585aaf010000000000000000000' b'000000000000000000000000000000000000000000042406b577bbc3e1ea163000000000020cad8863b312914f2fd' b'2aad6e9420b64859039effd67ac4681a7cf60e42b09b7e7bafa1e8d5131f477785d8338294da0f998844a85b39d24' b'26e839b370e014e3b010000000000000000000000000000000000000000000000000000000000000042406b573935' b'371e20e900000000002053d5e608ce5a12eda5931f86ee81198fdd231fea64cf096e9aeae321cf2efbe241e888d5a' b'af495e4c2a9f11b932db979d7483aeb446f479179b0c0b8d24bfa0e01000000000000000000000000000000000000' b'0000000000000000000000000045406b573c95301e34af0a0000000020df0e494c02ff79e3929bc1f2491077ec4f6' b'a607d7a1a5e1be96536642c98f86e533febd715f8a234028fd52046708551c6b6ac415480a6568aaa35cb94dc7203' b'01000000000000000000000000000000000000000000000000000000000000004f406b57c4c02a1ec54d230000000' b'020341f7d8e7d242e5e46343c40840c44f07e7e7306eb2355521b51502e8070e569485ba7eec4efdff0fc755af6e7' b'3e38b381a88b0925a68193a25da19d0f616e9f0100000000000000000000000000000000000000000000000000000' b'00000000050406b575be8251e1f61010000000020cd399f8078166ca5f0bdd1080ab1bb22d3c271b9729b6000b44f' b'4592cc9fab08c00ebab1e7cd88677e3b77c1598c7ac58660567f49f3a30ec46a48a1ae7652fe01000000000000000' b'0000000000000000000000000000000000000000000000052406b57d55b211e6f53090000000020c6c14ed4a53bbb' b'4f181acf2bbfd8b74d13826732f2114140ca99ca371f7dd87c51d18a05a1a6ffa37c041877fa33c2229a45a0ab66b' b'5530f914200a8d6639a6f010000000000000000000000000000000000000000000000000000000000000055406b57' b'0d5b1d1eff1c0900' ) ================================================ FILE: tests/unit/wallet/test_ledger.py ================================================ import os from unittest import TestCase from binascii import hexlify from lbry.testcase import AsyncioTestCase from lbry.wallet import Wallet, Account, Transaction, Output, Input, Ledger, Database, Headers from tests.unit.wallet.test_transaction import get_transaction, get_output from tests.unit.wallet.test_headers import HEADERS, block_bytes class MockNetwork: def __init__(self, history, transaction): self.history = history self.transaction = transaction self.address = None self.get_history_called = [] self.get_transaction_called = [] self.is_connected = False def retriable_call(self, function, *args, **kwargs): return function(*args, **kwargs) async def get_history(self, address): self.get_history_called.append(address) self.address = address return self.history async def get_merkle(self, txid, height): return {'merkle': ['abcd01'], 'pos': 1} async def get_transaction(self, tx_hash, _=None): self.get_transaction_called.append(tx_hash) return self.transaction[tx_hash] async def get_transaction_and_merkle(self, tx_hash, known_height=None): tx = await self.get_transaction(tx_hash) merkle = {'block_height': -1} if known_height: merkle = await self.get_merkle(tx_hash, known_height) return tx, merkle async def get_transaction_batch(self, txids, restricted): return { txid: await self.get_transaction_and_merkle(txid) for txid in txids } class LedgerTestCase(AsyncioTestCase): async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) self.ledger.headers.checkpoints = {} await self.ledger.headers.open() self.account = Account.generate(self.ledger, Wallet(), "lbryum") await self.ledger.db.open() async def asyncTearDown(self): await self.ledger.db.close() def make_header(self, **kwargs): header = { 'bits': 486604799, 'block_height': 0, 'merkle_root': b'4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', 'nonce': 2083236893, 'prev_block_hash': b'0000000000000000000000000000000000000000000000000000000000000000', 'timestamp': 1231006505, 'claim_trie_root': b'4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', 'version': 1 } header.update(kwargs) header['merkle_root'] = header['merkle_root'].ljust(64, b'a') header['prev_block_hash'] = header['prev_block_hash'].ljust(64, b'0') return self.ledger.headers.serialize(header) def add_header(self, **kwargs): serialized = self.make_header(**kwargs) self.ledger.headers.io.seek(0, os.SEEK_END) self.ledger.headers.io.write(serialized) self.ledger.headers._size = self.ledger.headers.io.seek(0, os.SEEK_END) // self.ledger.headers.header_size class TestUtils(TestCase): def test_valid_address(self): self.assertTrue(Ledger.is_script_address("rCz6yb1p33oYHToGZDzTjX7nFKaU3kNgBd")) class TestSynchronization(LedgerTestCase): async def test_update_history(self): txid1 = '252bda9b22cc902ca2aa2de3548ee8baf06b8501ff7bfb3b0b7d980dbd1bf792' txid2 = 'ab9c0654dd484ac20437030f2034e25dcb29fc507e84b91138f80adc3af738f9' txid3 = 'a2ae3d1db3c727e7d696122cab39ee20a7f81856dab7019056dd539f38c548a0' txid4 = '047cf1d53ef68f0fd586d46f90c09ff8e57a4180f67e7f4b8dd0135c3741e828' account = Account.generate(self.ledger, Wallet(), "torba") address = await account.receiving.get_or_create_usable_address() address_details = await self.ledger.db.get_address(address=address) self.assertIsNone(address_details['history']) self.add_header(block_height=0, merkle_root=b'abcd04') self.add_header(block_height=1, merkle_root=b'abcd04') self.add_header(block_height=2, merkle_root=b'abcd04') self.add_header(block_height=3, merkle_root=b'abcd04') self.ledger.network = MockNetwork([ {'tx_hash': txid1, 'height': 0}, {'tx_hash': txid2, 'height': 1}, {'tx_hash': txid3, 'height': 2}, ], { txid1: hexlify(get_transaction(get_output(1)).raw), txid2: hexlify(get_transaction(get_output(2)).raw), txid3: hexlify(get_transaction(get_output(3)).raw), }) await self.ledger.update_history(address, '') self.assertListEqual(self.ledger.network.get_history_called, [address]) self.assertListEqual(self.ledger.network.get_transaction_called, [txid1, txid2, txid3]) address_details = await self.ledger.db.get_address(address=address) self.assertEqual( address_details['history'], f'{txid1}:0:' f'{txid2}:1:' f'{txid3}:2:' ) self.ledger.network.get_history_called = [] self.ledger.network.get_transaction_called = [] self.assertEqual(0, len(self.ledger._tx_cache)) await self.ledger.update_history(address, '') self.assertListEqual(self.ledger.network.get_history_called, [address]) self.assertListEqual(self.ledger.network.get_transaction_called, []) self.ledger.network.history.append({'tx_hash': txid4, 'height': 3}) self.ledger.network.transaction[txid4] = hexlify(get_transaction(get_output(4)).raw) self.ledger.network.get_history_called = [] self.ledger.network.get_transaction_called = [] await self.ledger.update_history(address, '') self.assertListEqual(self.ledger.network.get_history_called, [address]) self.assertListEqual(self.ledger.network.get_transaction_called, [txid4]) address_details = await self.ledger.db.get_address(address=address) self.assertEqual( address_details['history'], f'{txid1}:0:' f'{txid2}:1:' f'{txid3}:2:' f'{txid4}:3:' ) class MocHeaderNetwork(MockNetwork): def __init__(self, responses): super().__init__(None, None) self.responses = responses async def get_headers(self, height, blocks): return self.responses[height] class BlockchainReorganizationTests(LedgerTestCase): async def test_1_block_reorganization(self): self.ledger.network = MocHeaderNetwork({ 10: {'height': 10, 'count': 5, 'hex': hexlify( HEADERS[block_bytes(10):block_bytes(15)] )}, 15: {'height': 15, 'count': 0, 'hex': b''} }) headers = self.ledger.headers await headers.connect(0, HEADERS[:block_bytes(10)]) self.add_header(block_height=len(headers)) self.assertEqual(10, headers.height) await self.ledger.receive_header([{ 'height': 11, 'hex': hexlify(self.make_header(block_height=11)) }]) async def test_3_block_reorganization(self): self.ledger.network = MocHeaderNetwork({ 10: {'height': 10, 'count': 5, 'hex': hexlify( HEADERS[block_bytes(10):block_bytes(15)] )}, 11: {'height': 11, 'count': 1, 'hex': hexlify(self.make_header(block_height=11))}, 12: {'height': 12, 'count': 1, 'hex': hexlify(self.make_header(block_height=12))}, 15: {'height': 15, 'count': 0, 'hex': b''} }) headers = self.ledger.headers await headers.connect(0, HEADERS[:block_bytes(10)]) self.add_header(block_height=len(headers)) self.add_header(block_height=len(headers)) self.add_header(block_height=len(headers)) self.assertEqual(headers.height, 12) await self.ledger.receive_header([{ 'height': 13, 'hex': hexlify(self.make_header(block_height=13)) }]) class BasicAccountingTests(LedgerTestCase): async def test_empty_state(self): self.assertEqual(await self.account.get_balance(), 0) async def test_balance(self): address = await self.account.receiving.get_or_create_usable_address() hash160 = self.ledger.address_to_hash160(address) tx = Transaction(is_verified=True)\ .add_outputs([Output.pay_pubkey_hash(100, hash160)]) await self.ledger.db.insert_transaction(tx) await self.ledger.db.save_transaction_io( tx, address, hash160, f'{tx.id}:1:' ) self.assertEqual(await self.account.get_balance(), 100) tx = Transaction(is_verified=True)\ .add_outputs([Output.pay_claim_name_pubkey_hash(100, 'foo', b'', hash160)]) await self.ledger.db.insert_transaction(tx) await self.ledger.db.save_transaction_io( tx, address, hash160, f'{tx.id}:1:' ) self.assertEqual(await self.account.get_balance(), 100) # claim names don't count towards balance self.assertEqual(await self.account.get_balance(include_claims=True), 200) async def test_get_utxo(self): address = yield self.account.receiving.get_or_create_usable_address() hash160 = self.ledger.address_to_hash160(address) tx = Transaction(is_verified=True)\ .add_outputs([Output.pay_pubkey_hash(100, hash160)]) await self.ledger.db.save_transaction_io( 'insert', tx, address, hash160, f'{tx.id}:1:' ) utxos = await self.account.get_utxos() self.assertEqual(len(utxos), 1) tx = Transaction(is_verified=True)\ .add_inputs([Input.spend(utxos[0])]) await self.ledger.db.save_transaction_io( 'insert', tx, address, hash160, f'{tx.id}:1:' ) self.assertEqual(await self.account.get_balance(include_claims=True), 0) utxos = await self.account.get_utxos() self.assertEqual(len(utxos), 0) ================================================ FILE: tests/unit/wallet/test_mnemonic.py ================================================ import unittest from binascii import hexlify from lbry.wallet.mnemonic import Mnemonic class TestMnemonic(unittest.TestCase): def test_mnemonic_to_seed(self): seed = Mnemonic.mnemonic_to_seed(mnemonic='foobar', passphrase='torba') self.assertEqual( hexlify(seed), b'475a419db4e991cab14f08bde2d357e52b3e7241f72c6d8a2f92782367feeee9f403dc6a37c26a3f02ab9' b'dec7f5063161eb139cea00da64cd77fba2f07c49ddc' ) def test_make_seed_decode_encode(self): iters = 10 m = Mnemonic('en') for _ in range(iters): seed = m.make_seed() i = m.mnemonic_decode(seed) self.assertEqual(m.mnemonic_encode(i), seed) ================================================ FILE: tests/unit/wallet/test_schema_signing.py ================================================ from binascii import unhexlify from lbry.testcase import AsyncioTestCase from lbry.wallet.constants import CENT, NULL_HASH32 from lbry.wallet.bip32 import PrivateKey, KeyPath from lbry.wallet.mnemonic import Mnemonic from lbry.wallet import Ledger, Database, Headers, Transaction, Input, Output from lbry.schema.claim import Claim from lbry.crypto.hash import sha256 def get_output(amount=CENT, pubkey_hash=NULL_HASH32): return Transaction() \ .add_outputs([Output.pay_pubkey_hash(amount, pubkey_hash)]) \ .outputs[0] def get_input(): return Input.spend(get_output()) def get_tx(): return Transaction().add_inputs([get_input()]) async def get_channel(claim_name='@foo'): seed = Mnemonic.mnemonic_to_seed(Mnemonic().make_seed(), '') key = PrivateKey.from_seed(Ledger, seed) channel_key = key.child(KeyPath.CHANNEL).child(0) channel_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(), b'abc') channel_txo.set_channel_private_key(channel_key) get_tx().add_outputs([channel_txo]) return channel_txo def get_stream(claim_name='foo'): stream_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(), b'abc') get_tx().add_outputs([stream_txo]) return stream_txo class TestSigningAndValidatingClaim(AsyncioTestCase): async def test_successful_create_sign_and_validate(self): channel = await get_channel() stream = get_stream() stream.sign(channel) self.assertTrue(stream.is_signed_by(channel)) async def test_fail_to_validate_on_wrong_channel(self): stream = get_stream() stream.sign(await get_channel()) self.assertFalse(stream.is_signed_by(await get_channel())) async def test_fail_to_validate_altered_claim(self): channel = await get_channel() stream = get_stream() stream.sign(channel) self.assertTrue(stream.is_signed_by(channel)) stream.claim.stream.title = 'hello' self.assertFalse(stream.is_signed_by(channel)) async def test_valid_private_key_for_cert(self): channel = await get_channel() self.assertTrue(channel.is_channel_private_key(channel.private_key)) async def test_fail_to_load_wrong_private_key_for_cert(self): channel = await get_channel() self.assertFalse(channel.is_channel_private_key((await get_channel()).private_key)) class TestValidatingOldSignatures(AsyncioTestCase): def test_signed_claim_made_by_ytsync(self): stream_tx = Transaction(unhexlify( b'0100000001eb2a756e15bde95db3d2ae4a6e9b2796a699087890644607b5b04a5f15b67062010000006a4' b'7304402206444b920bd318a07d9b982e30eb66245fdaaa6c9866e1f6e5900161d9b0ffd70022036464714' b'4f1830898a2042aa0d6cef95a243799cc6e36630a58d411e2f9111f00121029b15f9a00a7c3f21b10bd4b' b'98ab23a9e895bd9160e21f71317862bf55fbbc89effffffff0240420f0000000000fd1503b52268657265' b'2d6172652d352d726561736f6e732d692d6e657874636c6f75642d746c674dd302080110011aee0408011' b'2a604080410011a2b4865726520617265203520526561736f6e73204920e29da4efb88f204e657874636c' b'6f7564207c20544c4722920346696e64206f7574206d6f72652061626f7574204e657874636c6f75643a2' b'068747470733a2f2f6e657874636c6f75642e636f6d2f0a0a596f752063616e2066696e64206d65206f6e' b'20746865736520736f6369616c733a0a202a20466f72756d733a2068747470733a2f2f666f72756d2e686' b'5617679656c656d656e742e696f2f0a202a20506f64636173743a2068747470733a2f2f6f6666746f7069' b'63616c2e6e65740a202a2050617472656f6e3a2068747470733a2f2f70617472656f6e2e636f6d2f74686' b'56c696e757867616d65720a202a204d657263683a2068747470733a2f2f746565737072696e672e636f6d' b'2f73746f7265732f6f6666696369616c2d6c696e75782d67616d65720a202a205477697463683a2068747' b'470733a2f2f7477697463682e74762f786f6e64616b0a202a20547769747465723a2068747470733a2f2f' b'747769747465722e636f6d2f7468656c696e757867616d65720a0a2e2e2e0a68747470733a2f2f7777772' b'e796f75747562652e636f6d2f77617463683f763d4672546442434f535f66632a0f546865204c696e7578' b'2047616d6572321c436f7079726967687465642028636f6e7461637420617574686f722938004a2968747' b'470733a2f2f6265726b2e6e696e6a612f7468756d626e61696c732f4672546442434f535f666352005a00' b'1a41080110011a30040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc22f0bff70c4fe0b91fd36' b'da9a375e3e1c171db825bf5d1f32209766964656f2f6d70342a5c080110031a4062b2dd4c45e364030fbf' b'ad1a6fefff695ebf20ea33a5381b947753e2a0ca359989a5cc7d15e5392a0d354c0b68498382b2701b22c' b'03beb8dcb91089031b871e72214feb61536c007cdf4faeeaab4876cb397feaf6b516d7576a914f4f43f6f' b'7a472bbf27fa3630329f771135fc445788ac86ff0600000000001976a914cef0fe3eeaf04416f0c3ff3e7' b'8a598a081e70ee788ac00000000' )) stream = stream_tx.outputs[0] channel_tx = Transaction(unhexlify( b'010000000192a1e1e3f66b8ca05a021cfa5fb6645ebc066b46639ccc9b3781fa588a88da65010000006a4' b'7304402206be09a355f6abea8a10b5512180cd258460b42d516b5149431ffa3230a02533a0220325e83c6' b'176b295d633b18aad67adb4ad766d13152536ac04583f86d14645c9901210269c63bc8bac8143ef02f972' b'4a4ab35b12bdfa65ee1ad8c0db3d6511407a4cc2effffffff0240420f000000000091b50e405468654c69' b'6e757847616d65724c6408011002225e0801100322583056301006072a8648ce3d020106052b8104000a0' b'34200043878b1edd4a1373149909ef03f4339f6da9c2bd2214c040fd2e530463ffe66098eca14fc70b50f' b'f3aefd106049a815f595ed5a13eda7419ad78d9ed7ae473f176d7576a914994dad5f21c384ff526749b87' b'6d9d017d257b69888ac00dd6d00000000001976a914979202508a44f0e8290cea80787c76f98728845388' b'ac00000000' )) channel = channel_tx.outputs[0] ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) self.assertTrue(stream.is_signed_by(channel, ledger)) def test_another_signed_claim_made_by_ytsync(self): stream_tx = Transaction(unhexlify( b'010000000185870fabdd6bd2d57749afebc0b239e8d0ebeb6f3647d6cfcabd5ea2200ac632010000006b4' b'83045022100877c86de154e39f21959bc2157865071924adb7930a7a8910714f27398cd2689022074270f' b'074ae260fff319d5e0c030691821bc75b82ff0179898ac3eaeda4123eb01210200328f7f001f22ea25d72' b'ba37379e3065020c4d8371d9199dc4e3770084e26b9ffffffff0240420f0000000000fdcc05b527746865' b'2d637269746963616c2d6e6565642d666f722d696e646570656e64656e742d6d656469614d85050191bba' b'd064bdc455b9ebddeeb559686b13f027615384ec7c9d981c3c21a6e3d723a654e86bd707d21174c4f697f' b'5080cf367a3b2dfc059e6cc14a962631df69b9886f4d8b97cb339b14633966fd5ac7d75edacdf30ac5010' b'a90010a304af34d1c1467ebfc8785e2a49c7d5bec3cc6db94db858f1dcf95e4256564fba586d6e01f496d' b'f2a34344e021d2725ffd12197468652d637269746963616c2d6e6565642d666f722e6d703418ee97eac10' b'22209766964656f2f6d70343230ba13e6b667a9acef7e1b1caa88b9eb1d4680dea84b1d3e838266595805' b'ab3343855c20af35012f942ce0d5111ce080331a1f436f7079726967687465642028636f6e74616374207' b'075626c69736865722928e2e3c98d065a0908800f10b80818f314423954686520437269746963616c204e' b'65656420666f7220496e646570656e64656e74204d65646961207c20476c656e6e20477265656e77616c6' b'44af006496e636c7564657320616e20696e74726f64756374696f6e20627920546f6d20576f6f64732e20' b'5265636f7264656420696e204c616b65204a61636b736f6e2c2054657861732c206f6e20446563656d626' b'57220342c20323032312e0a0a526f6e205061756c27732074776f2063616d706169676e7320666f722070' b'7265736964656e7420283230303820616e64203230313229207765726520776174657273686564206d6f6' b'd656e747320666f72206c6962657274792d6d696e6465642070656f706c652061726f756e642074686520' b'776f726c642e205468652022526f6e205061756c205265766f6c7574696f6e22e2809463656e746572656' b'42061726f756e642068697320756e64696c75746564206d657373616765206f662070656163652c207072' b'6f70657274792c20616e64206d61726b657473e280946368616e6765642074686520776179206d696c6c6' b'96f6e732074686f756768742061626f75742074686520416d65726963616e20656d7069726520616e6420' b'74686520416d65726963616e2066696e616e6369616c2073797374656d2e2044722e205061756c2773206' b'66f637573206f6e2063656e7472616c2062616e6b696e6720616e6420666f726569676e20706f6c696379' b'2063617567687420706f6c6974696369616e7320616e642070756e64697473206f66662067756172642c2' b'0666f7263696e67207468656d20746f20736372616d626c6520666f72206578706c616e6174696f6e7320' b'6f66206f7572204d6964646c65204561737420706f6c69637920616e6420536f766965742d7374796c652' b'063656e7472616c20706c616e6e696e6720617420746865204665642e20506f6c697469637320696e2041' b'6d657269636120686173206e6f74206265656e207468652073616d652073696e636520746865202247697' b'56c69616e69206d6f6d656e742220616e642022456e6420746865204665642e222054686520526f6e2050' b'61756c205265766f6c7574696f6e2077617320626f7468206120706f6c69746963616c20616e642063756' b'c747572616c207068656e6f6d656e6f6e2e0a0a303a303020496e74726f64756374696f6e20627920546f' b'6d20576f6f64730a343a323720476c656e6e20477265656e77616c640a2e2e2e0a68747470733a2f2f777' b'7772e796f75747562652e636f6d2f77617463683f763d4e4b70706d52467673453052292a276874747073' b'3a2f2f7468756d626e61696c732e6c6272792e636f6d2f4e4b70706d5246767345305a046e6577735a096' b'3617468656472616c5a0f636f72706f72617465206d656469615a08637269746963616c5a0f676c656e6e' b'20677265656e77616c645a0b696e646570656e64656e745a0a6a6f75726e616c69736d5a056d656469615' b'a056d697365735a08706f6c69746963735a0a70726f706167616e64615a08726f6e207061756c5a057472' b'757468620208016d7576a9140969964db5b5744e2d2d0de797f5904efc80d02188acc8814200000000001' b'976a91439086597f9cfc066f4749b8bb245bf561714fda888ac00000000' )) stream = stream_tx.outputs[0] channel_tx = Transaction(unhexlify( b'01000000011d47b91b409b317e427adb87ec4b0bfc9fad2abf6ec3296f41918e4b3cb9d4e7010000006a4' b'7304402205e53ef7fc643ed00f0240dd1c3302b82141f481ed071cbcdd6b6ec6166ffd4e002203eb28ce6' b'39f80253f66ff3bf45288a60133d7f5625217d1ecf3b57da440b559f012103b852d61074eb995b702a800' b'f284e937ece4fea7f023beb70e6b0d1bff36d64b9ffffffff0240420f0000000000fdde01b506406d6973' b'65734db801001299010a583056301006072a8648ce3d020106052b8104000a034200047ddb1d639d7bdd0' b'953d9ab0bf9e971a632f85f9823c1d85780aa3e0a702b503c2962d00f67360e803514bf5864710925aacb' b'effd9597532c7e60eb21b4e3fd03223d2a3b68747470733a2f2f7468756d626e61696c732e6c6272792e6' b'36f6d2f62616e6e65722d55436d54362d43684b7061694956753266684549734e7451420a6d697365736d' b'656469614ad401466561747572656420766964656f732066726f6d20746865204d6973657320496e73746' b'9747574652e20546865204d6973657320496e737469747574652070726f6d6f7465732041757374726961' b'6e2065636f6e6f6d6963732c2066726565646f6d2c20616e6420706561636520696e20746865206c69626' b'572616c20696e74656c6c65637475616c20747261646974696f6e206f66204c756477696720766f6e204d' b'69736573207468726f7567682072657365617263682c207075626c697368696e672c20616e64206564756' b'36174696f6e2e52362a3468747470733a2f2f7468756d626e61696c732e6c6272792e636f6d2f55436d54' b'362d43684b7061694956753266684549734e74516d7576a914cd77ded2400e6569f03a2580244bb395f95' b'f91fc88ac344ab701000000001976a914cabdbfce726d2fda92ffe0041a4303f6c6c34cda88ac00000000' )) channel = channel_tx.outputs[0] ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) self.assertTrue(stream.is_signed_by(channel, ledger)) def test_claim_signed_using_ecdsa_validates_with_coincurve(self): channel_tx = Transaction(unhexlify( "0100000001b91d829283c0d80cb8113d5f36b6da3dfe9df3e783f158bfb3fd1b2b178d7fc9010000006b48" "3045022100f4e2b4ee38388c3d3a62f4b12fdd413f6f140168e85884bbeb33a3f2d3159ef502201721200f" "4a4f3b87484d4f47c9054e31cd3ba451dd3886a7f9f854893e7c8cf90121023f9e906e0c120f3bf74feb40" "f01ddeafbeb1856d91938c3bef25bed06767247cffffffff0200e1f5050000000081b505406368616e4c5d" "00125a0a583056301006072a8648ce3d020106052b8104000a03420004d7fa13fd8e57f3a0b878eaaf3d17" "9144d25ddbe4a3e4440a661f51b4134c6a13c9c98678ff8411932e60fd97d7baf03ea67ebcc21097230cfb" "2241348aadb55e6d7576a9149c6d700f89c77f0e8c650ba05656f8f2392782d388acf47c95350000000019" "76a914d9502233e0e1fc76e13e36c546f704c3124d5eaa88ac00000000" )) channel = channel_tx.outputs[0] stream_tx = Transaction(unhexlify( "010000000116a1d90763f2e3a2348c7fb438a23f232b15e3ffe3f058c3b2ab52c8bed8dcb5010000006b48" "30450221008f38561b3a16944c63b4f4f1562f1efe1b2060f31d249e234003ee5e3461756f02205773c99e" "83c968728e4f2433a13871c6ad23f6c10368ac52fa62a09f3f7ef5fd012102597f39845b98e2415b777aa0" "3849d346d287af7970deb05f11214b3418ae9d82ffffffff0200e1f50500000000fd0c01b505636c61696d" "4ce8012e6e40fa5fee1b915af3b55131dcbcebee34ab9148292b084ce3741f2e0db49783f3d854ac885f2b" "6304a76ef7048046e338dd414ba4c64e8468651768ffaaf550c8560637ac8c477ea481ac2a9264097240f4" "ab0a90010a8d010a3056bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454" "f4edd1373e2b64ee2e68350d916e120b746d706c69647879363171180322186170706c69636174696f6e2f" "6f637465742d73747265616d3230f293f5acf4310562d4a41f6620167fe6d83761a98d36738908ce5c8776" "1642710e55352a396276a42eda92ff5856f46f6d7576a91434bd3dc4c45cc0635eb2ad5da658727e5442ca" "0f88ace82f902f000000001976a91427b27c89eaebf68d063c107241584c07e5a6ccc688ac00000000" )) stream = stream_tx.outputs[0] ledger = Ledger({'db': Database(':memory:'), 'headers': Headers(':memory:')}) self.assertTrue(stream.is_signed_by(channel, ledger)) class TestValidateSignContent(AsyncioTestCase): async def test_sign_some_content(self): some_content = "MEANINGLESS CONTENT AEE3353320".encode() timestamp_str = "1630564175" channel = await get_channel() signature = channel.sign_data(some_content, timestamp_str) pieces = [timestamp_str.encode(), channel.claim_hash, some_content] self.assertTrue(Output.is_signature_valid( unhexlify(signature.encode()), sha256(b''.join(pieces)), channel.claim.channel.public_key_bytes )) ================================================ FILE: tests/unit/wallet/test_script.py ================================================ import unittest from binascii import hexlify, unhexlify from lbry.wallet.bcd_data_stream import BCDataStream from lbry.wallet.script import ( InputScript, OutputScript, Template, ParseError, tokenize, push_data, PUSH_SINGLE, PUSH_INTEGER, PUSH_MANY, OP_HASH160, OP_EQUAL ) def parse(opcodes, source): template = Template('test', opcodes) s = BCDataStream() for t in source: if isinstance(t, bytes): s.write_many(push_data(t)) elif isinstance(t, int): s.write_uint8(t) else: raise ValueError() s.reset() return template.parse(tokenize(s)) class TestScriptTemplates(unittest.TestCase): def test_push_data(self): self.assertDictEqual(parse( (PUSH_SINGLE('script_hash'),), (b'abcdef',) ), { 'script_hash': b'abcdef' } ) self.assertDictEqual(parse( (PUSH_SINGLE('first'), PUSH_INTEGER('rating')), (b'Satoshi', (1000).to_bytes(2, 'little')) ), { 'first': b'Satoshi', 'rating': 1000, } ) self.assertDictEqual(parse( (OP_HASH160, PUSH_SINGLE('script_hash'), OP_EQUAL), (OP_HASH160, b'abcdef', OP_EQUAL) ), { 'script_hash': b'abcdef' } ) def test_push_data_many(self): self.assertDictEqual(parse( (PUSH_MANY('names'),), (b'amit',) ), { 'names': [b'amit'] } ) self.assertDictEqual(parse( (PUSH_MANY('names'),), (b'jeremy', b'amit', b'victor') ), { 'names': [b'jeremy', b'amit', b'victor'] } ) self.assertDictEqual(parse( (OP_HASH160, PUSH_MANY('names'), OP_EQUAL), (OP_HASH160, b'grin', b'jack', OP_EQUAL) ), { 'names': [b'grin', b'jack'] } ) def test_push_data_mixed(self): self.assertDictEqual(parse( (PUSH_SINGLE('CEO'), PUSH_MANY('Devs'), PUSH_SINGLE('CTO'), PUSH_SINGLE('State')), (b'jeremy', b'lex', b'amit', b'victor', b'jack', b'grin', b'NH') ), { 'CEO': b'jeremy', 'CTO': b'grin', 'Devs': [b'lex', b'amit', b'victor', b'jack'], 'State': b'NH' } ) def test_push_data_many_separated(self): self.assertDictEqual(parse( (PUSH_MANY('Chiefs'), OP_HASH160, PUSH_MANY('Devs')), (b'jeremy', b'grin', OP_HASH160, b'lex', b'jack') ), { 'Chiefs': [b'jeremy', b'grin'], 'Devs': [b'lex', b'jack'] } ) def test_push_data_many_not_separated(self): with self.assertRaisesRegex(ParseError, 'consecutive PUSH_MANY'): parse((PUSH_MANY('Chiefs'), PUSH_MANY('Devs')), (b'jeremy', b'grin', b'lex', b'jack')) class TestRedeemPubKeyHash(unittest.TestCase): def redeem_pubkey_hash(self, sig, pubkey): # this checks that factory function correctly sets up the script src1 = InputScript.redeem_pubkey_hash(unhexlify(sig), unhexlify(pubkey)) self.assertEqual(src1.template.name, 'pubkey_hash') self.assertEqual(hexlify(src1.values['signature']), sig) self.assertEqual(hexlify(src1.values['pubkey']), pubkey) # now we test that it will round trip src2 = InputScript(src1.source) self.assertEqual(src2.template.name, 'pubkey_hash') self.assertEqual(hexlify(src2.values['signature']), sig) self.assertEqual(hexlify(src2.values['pubkey']), pubkey) return hexlify(src1.source) def test_redeem_pubkey_hash_1(self): self.assertEqual( self.redeem_pubkey_hash( b'30450221009dc93f25184a8d483745cd3eceff49727a317c9bfd8be8d3d04517e9cdaf8dd502200e' b'02dc5939cad9562d2b1f303f185957581c4851c98d497af281118825e18a8301', b'025415a06514230521bff3aaface31f6db9d9bbc39bf1ca60a189e78731cfd4e1b' ), b'4830450221009dc93f25184a8d483745cd3eceff49727a317c9bfd8be8d3d04517e9cdaf8dd502200e02d' b'c5939cad9562d2b1f303f185957581c4851c98d497af281118825e18a830121025415a06514230521bff3' b'aaface31f6db9d9bbc39bf1ca60a189e78731cfd4e1b' ) class TestRedeemScriptHash(unittest.TestCase): def redeem_script_hash(self, sigs, pubkeys): # this checks that factory function correctly sets up the script src1 = InputScript.redeem_multi_sig_script_hash( [unhexlify(sig) for sig in sigs], [unhexlify(pubkey) for pubkey in pubkeys] ) subscript1 = src1.values['script'] self.assertEqual(src1.template.name, 'script_hash+multi_sig') self.assertListEqual([hexlify(v) for v in src1.values['signatures']], sigs) self.assertListEqual([hexlify(p) for p in subscript1.values['pubkeys']], pubkeys) self.assertEqual(subscript1.values['signatures_count'], len(sigs)) self.assertEqual(subscript1.values['pubkeys_count'], len(pubkeys)) # now we test that it will round trip src2 = InputScript(src1.source) subscript2 = src2.values['script'] self.assertEqual(src2.template.name, 'script_hash+multi_sig') self.assertListEqual([hexlify(v) for v in src2.values['signatures']], sigs) self.assertListEqual([hexlify(p) for p in subscript2.values['pubkeys']], pubkeys) self.assertEqual(subscript2.values['signatures_count'], len(sigs)) self.assertEqual(subscript2.values['pubkeys_count'], len(pubkeys)) return hexlify(src1.source) def test_redeem_script_hash_1(self): self.assertEqual( self.redeem_script_hash([ b'3045022100fec82ed82687874f2a29cbdc8334e114af645c45298e85bb1efe69fcf15c617a0220575' b'e40399f9ada388d8e522899f4ec3b7256896dd9b02742f6567d960b613f0401', b'3044022024890462f731bd1a42a4716797bad94761fc4112e359117e591c07b8520ea33b02201ac68' b'9e35c4648e6beff1d42490207ba14027a638a62663b2ee40153299141eb01', b'30450221009910823e0142967a73c2d16c1560054d71c0625a385904ba2f1f53e0bc1daa8d02205cd' b'70a89c6cf031a8b07d1d5eb0d65d108c4d49c2d403f84fb03ad3dc318777a01' ], [ b'0372ba1fd35e5f1b1437cba0c4ebfc4025b7349366f9f9c7c8c4b03a47bd3f68a4', b'03061d250182b2db1ba144167fd8b0ef3fe0fc3a2fa046958f835ffaf0dfdb7692', b'02463bfbc1eaec74b5c21c09239ae18dbf6fc07833917df10d0b43e322810cee0c', b'02fa6a6455c26fb516cfa85ea8de81dd623a893ffd579ee2a00deb6cdf3633d6bb', b'0382910eae483ce4213d79d107bfc78f3d77e2a31ea597be45256171ad0abeaa89' ]), b'00483045022100fec82ed82687874f2a29cbdc8334e114af645c45298e85bb1efe69fcf15c617a0220575e' b'40399f9ada388d8e522899f4ec3b7256896dd9b02742f6567d960b613f0401473044022024890462f731bd' b'1a42a4716797bad94761fc4112e359117e591c07b8520ea33b02201ac689e35c4648e6beff1d42490207ba' b'14027a638a62663b2ee40153299141eb014830450221009910823e0142967a73c2d16c1560054d71c0625a' b'385904ba2f1f53e0bc1daa8d02205cd70a89c6cf031a8b07d1d5eb0d65d108c4d49c2d403f84fb03ad3dc3' b'18777a014cad53210372ba1fd35e5f1b1437cba0c4ebfc4025b7349366f9f9c7c8c4b03a47bd3f68a42103' b'061d250182b2db1ba144167fd8b0ef3fe0fc3a2fa046958f835ffaf0dfdb76922102463bfbc1eaec74b5c2' b'1c09239ae18dbf6fc07833917df10d0b43e322810cee0c2102fa6a6455c26fb516cfa85ea8de81dd623a89' b'3ffd579ee2a00deb6cdf3633d6bb210382910eae483ce4213d79d107bfc78f3d77e2a31ea597be45256171' b'ad0abeaa8955ae' ) class TestPayPubKeyHash(unittest.TestCase): def pay_pubkey_hash(self, pubkey_hash): # this checks that factory function correctly sets up the script src1 = OutputScript.pay_pubkey_hash(unhexlify(pubkey_hash)) self.assertEqual(src1.template.name, 'pay_pubkey_hash') self.assertEqual(hexlify(src1.values['pubkey_hash']), pubkey_hash) # now we test that it will round trip src2 = OutputScript(src1.source) self.assertEqual(src2.template.name, 'pay_pubkey_hash') self.assertEqual(hexlify(src2.values['pubkey_hash']), pubkey_hash) return hexlify(src1.source) def test_pay_pubkey_hash_1(self): self.assertEqual( self.pay_pubkey_hash(b'64d74d12acc93ba1ad495e8d2d0523252d664f4d'), b'76a91464d74d12acc93ba1ad495e8d2d0523252d664f4d88ac' ) class TestPayScriptHash(unittest.TestCase): def pay_script_hash(self, script_hash): # this checks that factory function correctly sets up the script src1 = OutputScript.pay_script_hash(unhexlify(script_hash)) self.assertEqual(src1.template.name, 'pay_script_hash') self.assertEqual(hexlify(src1.values['script_hash']), script_hash) # now we test that it will round trip src2 = OutputScript(src1.source) self.assertEqual(src2.template.name, 'pay_script_hash') self.assertEqual(hexlify(src2.values['script_hash']), script_hash) return hexlify(src1.source) def test_pay_pubkey_hash_1(self): self.assertEqual( self.pay_script_hash(b'63d65a2ee8c44426d06050cfd71c0f0ff3fc41ac'), b'a91463d65a2ee8c44426d06050cfd71c0f0ff3fc41ac87' ) class TestPayClaimNamePubkeyHash(unittest.TestCase): def pay_claim_name_pubkey_hash(self, name, claim, pubkey_hash): # this checks that factory function correctly sets up the script src1 = OutputScript.pay_claim_name_pubkey_hash( name, unhexlify(claim), unhexlify(pubkey_hash)) self.assertEqual(src1.template.name, 'claim_name+pay_pubkey_hash') self.assertEqual(src1.values['claim_name'], name) self.assertEqual(hexlify(src1.values['claim']), claim) self.assertEqual(hexlify(src1.values['pubkey_hash']), pubkey_hash) # now we test that it will round trip src2 = OutputScript(src1.source) self.assertEqual(src2.template.name, 'claim_name+pay_pubkey_hash') self.assertEqual(src2.values['claim_name'], name) self.assertEqual(hexlify(src2.values['claim']), claim) self.assertEqual(hexlify(src2.values['pubkey_hash']), pubkey_hash) return hexlify(src1.source) def test_pay_claim_name_pubkey_hash_1(self): self.assertEqual( self.pay_claim_name_pubkey_hash( # name b'cats', # claim b'080110011a7808011230080410011a084d616361726f6e6922002a003214416c6c20726967687473' b'2072657365727665642e38004a0052005a001a42080110011a30add80aaf02559ba09853636a0658' b'c42b727cb5bb4ba8acedb4b7fe656065a47a31878dbf9912135ddb9e13806cc1479d220a696d6167' b'652f6a7065672a5c080110031a404180cc0fa4d3839ee29cca866baed25fafb43fca1eb3b608ee88' b'9d351d3573d042c7b83e2e643db0d8e062a04e6e9ae6b90540a2f95fe28638d0f18af4361a1c2214' b'f73de93f4299fb32c32f949e02198a8e91101abd', # pub key b'be16e4b0f9bd8f6d47d02b3a887049c36d3b84cb' ), b'b504636174734cdc080110011a7808011230080410011a084d616361726f6e6922002a003214416c6c207' b'269676874732072657365727665642e38004a0052005a001a42080110011a30add80aaf02559ba0985363' b'6a0658c42b727cb5bb4ba8acedb4b7fe656065a47a31878dbf9912135ddb9e13806cc1479d220a696d616' b'7652f6a7065672a5c080110031a404180cc0fa4d3839ee29cca866baed25fafb43fca1eb3b608ee889d35' b'1d3573d042c7b83e2e643db0d8e062a04e6e9ae6b90540a2f95fe28638d0f18af4361a1c2214f73de93f4' b'299fb32c32f949e02198a8e91101abd6d7576a914be16e4b0f9bd8f6d47d02b3a887049c36d3b84cb88ac' ) ================================================ FILE: tests/unit/wallet/test_stream_controller.py ================================================ from lbry.wallet.stream import StreamController from lbry.wallet.tasks import TaskGroup from lbry.testcase import AsyncioTestCase class StreamControllerTestCase(AsyncioTestCase): def test_non_unique_events(self): events = [] controller = StreamController() controller.stream.listen(on_data=events.append) controller.add("yo") controller.add("yo") self.assertListEqual(events, ["yo", "yo"]) def test_unique_events(self): events = [] controller = StreamController(merge_repeated_events=True) controller.stream.listen(on_data=events.append) controller.add("yo") controller.add("yo") self.assertListEqual(events, ["yo"]) class TaskGroupTestCase(AsyncioTestCase): async def test_cancel_sets_it_done(self): group = TaskGroup() group.cancel() self.assertTrue(group.done.is_set()) ================================================ FILE: tests/unit/wallet/test_transaction.py ================================================ import os import unittest import tempfile import shutil from binascii import hexlify, unhexlify from itertools import cycle from lbry.testcase import AsyncioTestCase from lbry.wallet.constants import CENT, COIN, NULL_HASH32 from lbry.wallet import Wallet, Account, Ledger, Database, Headers, Transaction, Output, Input NULL_HASH = b'\x00'*32 FEE_PER_BYTE = 50 FEE_PER_CHAR = 200000 def get_output(amount=CENT, pubkey_hash=NULL_HASH32, height=-2): return Transaction(height=height) \ .add_outputs([Output.pay_pubkey_hash(amount, pubkey_hash)]) \ .outputs[0] def get_input(amount=CENT, pubkey_hash=NULL_HASH): return Input.spend(get_output(amount, pubkey_hash)) def get_transaction(txo=None): return Transaction() \ .add_inputs([get_input()]) \ .add_outputs([txo or Output.pay_pubkey_hash(CENT, NULL_HASH32)]) def get_claim_transaction(claim_name, claim=b''): return get_transaction( Output.pay_claim_name_pubkey_hash(CENT, claim_name, claim, NULL_HASH32) ) class TestSizeAndFeeEstimation(AsyncioTestCase): async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:'), 'fee_per_name_char': 200_000 }) await self.ledger.db.open() async def asyncTearDown(self): await self.ledger.db.close() def test_output_size_and_fee(self): txo = get_output() self.assertEqual(txo.size, 46) self.assertEqual(txo.get_fee(self.ledger), 46 * FEE_PER_BYTE) claim_name = 'verylongname' tx = get_claim_transaction(claim_name, b'0'*4000) base_size = tx.size - tx.inputs[0].size - tx.outputs[0].size txo = tx.outputs[0] self.assertEqual(tx.size, 4225) self.assertEqual(tx.base_size, base_size) self.assertEqual(txo.size, 4067) self.assertEqual(txo.get_fee(self.ledger), len(claim_name) * FEE_PER_CHAR) # fee based on total bytes is the larger fee claim_name = 'a' tx = get_claim_transaction(claim_name, b'0'*4000) base_size = tx.size - tx.inputs[0].size - tx.outputs[0].size txo = tx.outputs[0] self.assertEqual(tx.size, 4214) self.assertEqual(tx.base_size, base_size) self.assertEqual(txo.size, 4056) self.assertEqual(txo.get_fee(self.ledger), txo.size * FEE_PER_BYTE) def test_input_size_and_fee(self): txi = get_input() self.assertEqual(txi.size, 148) self.assertEqual(txi.get_fee(self.ledger), 148 * FEE_PER_BYTE) def test_transaction_size_and_fee(self): tx = get_transaction() self.assertEqual(tx.size, 204) self.assertEqual(tx.base_size, tx.size - tx.inputs[0].size - tx.outputs[0].size) self.assertEqual(tx.get_base_fee(self.ledger), FEE_PER_BYTE * tx.base_size) class TestAccountBalanceImpactFromTransaction(unittest.TestCase): def test_is_my_output_not_set(self): tx = get_transaction() with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"): _ = tx.net_account_balance tx.inputs[0].txo_ref.txo.is_my_output = True with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"): _ = tx.net_account_balance tx.outputs[0].is_my_output = True # all inputs/outputs are set now so it should work _ = tx.net_account_balance def test_paying_from_my_account_to_other_account(self): tx = Transaction() \ .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) tx.inputs[0].txo_ref.txo.is_my_output = True tx.outputs[0].is_my_output = False tx.outputs[1].is_my_output = True self.assertEqual(tx.net_account_balance, -200*CENT) def test_paying_from_other_account_to_my_account(self): tx = Transaction() \ .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) tx.inputs[0].txo_ref.txo.is_my_output = False tx.outputs[0].is_my_output = True tx.outputs[1].is_my_output = False self.assertEqual(tx.net_account_balance, 190*CENT) def test_paying_from_my_account_to_my_account(self): tx = Transaction() \ .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) tx.inputs[0].txo_ref.txo.is_my_output = True tx.outputs[0].is_my_output = True tx.outputs[1].is_my_output = True self.assertEqual(tx.net_account_balance, -10*CENT) # lost to fee class TestTransactionSerialization(unittest.TestCase): def test_genesis_transaction(self): raw = unhexlify( "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1f0" "4ffff001d010417696e736572742074696d657374616d7020737472696e67ffffffff01000004bfc91b8e" "001976a914345991dbf57bfb014b87006acdfafbfc5fe8292f88ac00000000" ) tx = Transaction(raw) self.assertEqual(tx.version, 1) self.assertEqual(tx.locktime, 0) self.assertEqual(len(tx.inputs), 1) self.assertEqual(len(tx.outputs), 1) coinbase = tx.inputs[0] self.assertTrue(coinbase.txo_ref.is_null) self.assertEqual(coinbase.txo_ref.position, 0xFFFFFFFF) self.assertEqual(coinbase.sequence, 0xFFFFFFFF) self.assertIsNotNone(coinbase.coinbase) self.assertIsNone(coinbase.script) self.assertEqual( hexlify(coinbase.coinbase), b'04ffff001d010417696e736572742074696d657374616d7020737472696e67' ) out = tx.outputs[0] self.assertEqual(out.amount, 40000000000000000) self.assertEqual(out.position, 0) self.assertTrue(out.script.is_pay_pubkey_hash) self.assertFalse(out.script.is_pay_script_hash) self.assertFalse(out.script.is_claim_involved) tx._reset() self.assertEqual(tx.raw, raw) def test_coinbase_transaction(self): raw = unhexlify( "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff200" "34d520504f89ac55a086032d217bf0700000d2f6e6f64655374726174756d2f0000000001a03489850800" "00001976a914cfab870d6deea54ca94a41912a75484649e52f2088ac00000000" ) tx = Transaction(raw) self.assertEqual(tx.version, 1) self.assertEqual(tx.locktime, 0) self.assertEqual(len(tx.inputs), 1) self.assertEqual(len(tx.outputs), 1) coinbase = tx.inputs[0] self.assertTrue(coinbase.txo_ref.is_null) self.assertEqual(coinbase.txo_ref.position, 0xFFFFFFFF) self.assertEqual(coinbase.sequence, 0) self.assertIsNotNone(coinbase.coinbase) self.assertIsNone(coinbase.script) self.assertEqual( hexlify(coinbase.coinbase), b'034d520504f89ac55a086032d217bf0700000d2f6e6f64655374726174756d2f' ) out = tx.outputs[0] self.assertEqual(out.amount, 36600100000) self.assertEqual(out.position, 0) self.assertTrue(out.script.is_pay_pubkey_hash) self.assertFalse(out.script.is_pay_script_hash) self.assertFalse(out.script.is_claim_involved) tx._reset() self.assertEqual(tx.raw, raw) def test_claim_transaction(self): raw = unhexlify( "01000000012433e1b327603843b083344dbae5306ff7927f87ebbc5ae9eb50856c5b53fd1d000000006a4" "7304402201a91e1023d11c383a11e26bf8f9034087b15d8ada78fa565e0610455ffc8505e0220038a63a6" "ecb399723d4f1f78a20ddec0a78bf8fb6c75e63e166ef780f3944fbf0121021810150a2e4b088ec51b20c" "be1b335962b634545860733367824d5dc3eda767dffffffff028096980000000000fdff00b50463617473" "4cdc080110011a7808011230080410011a084d616361726f6e6922002a003214416c6c207269676874732" "072657365727665642e38004a0052005a001a42080110011a30add80aaf02559ba09853636a0658c42b72" "7cb5bb4ba8acedb4b7fe656065a47a31878dbf9912135ddb9e13806cc1479d220a696d6167652f6a70656" "72a5c080110031a404180cc0fa4d3839ee29cca866baed25fafb43fca1eb3b608ee889d351d3573d042c7" "b83e2e643db0d8e062a04e6e9ae6b90540a2f95fe28638d0f18af4361a1c2214f73de93f4299fb32c32f9" "49e02198a8e91101abd6d7576a914be16e4b0f9bd8f6d47d02b3a887049c36d3b84cb88ac0cd2520b0000" "00001976a914f521178feb733a719964e1da4a9efb09dcc39cfa88ac00000000" ) tx = Transaction(raw) self.assertEqual(tx.id, '666c3d15de1d6949a4fe717126c368e274b36957dce29fd401138c1e87e92a62') self.assertEqual(tx.version, 1) self.assertEqual(tx.locktime, 0) self.assertEqual(len(tx.inputs), 1) self.assertEqual(len(tx.outputs), 2) txin = tx.inputs[0] self.assertEqual( txin.txo_ref.id, '1dfd535b6c8550ebe95abceb877f92f76f30e5ba4d3483b043386027b3e13324:0' ) self.assertEqual(txin.txo_ref.position, 0) self.assertEqual(txin.sequence, 0xFFFFFFFF) self.assertIsNone(txin.coinbase) self.assertEqual(txin.script.template.name, 'pubkey_hash') self.assertEqual( hexlify(txin.script.values['pubkey']), b'021810150a2e4b088ec51b20cbe1b335962b634545860733367824d5dc3eda767d' ) self.assertEqual( hexlify(txin.script.values['signature']), b'304402201a91e1023d11c383a11e26bf8f9034087b15d8ada78fa565e0610455ffc8505e0220038a63a6' b'ecb399723d4f1f78a20ddec0a78bf8fb6c75e63e166ef780f3944fbf01' ) # Claim out0 = tx.outputs[0] self.assertEqual(out0.amount, 10000000) self.assertEqual(out0.position, 0) self.assertTrue(out0.script.is_pay_pubkey_hash) self.assertTrue(out0.script.is_claim_name) self.assertTrue(out0.script.is_claim_involved) self.assertEqual(out0.script.values['claim_name'], b'cats') self.assertEqual( hexlify(out0.script.values['pubkey_hash']), b'be16e4b0f9bd8f6d47d02b3a887049c36d3b84cb' ) # Change out1 = tx.outputs[1] self.assertEqual(out1.amount, 189977100) self.assertEqual(out1.position, 1) self.assertTrue(out1.script.is_pay_pubkey_hash) self.assertFalse(out1.script.is_claim_involved) self.assertEqual( hexlify(out1.script.values['pubkey_hash']), b'f521178feb733a719964e1da4a9efb09dcc39cfa' ) tx._reset() self.assertEqual(tx.raw, raw) def test_redeem_scripthash_transaction(self): raw = unhexlify( "0200000001409223c2405238fdc516d4f2e8aa57637ce52d3b1ac42b26f1accdcda9697e79010000008a4" "730440220033d5286f161da717d9d1bc3c2bc28da7636b38fc0c6aefb1e0864212f05282c02205df3ce13" "5e79c76d44489212f77ad4e3a838562e601e6377704fa6206a6ae44f012102261773e7eebe9da80a5653d" "865cc600362f8e7b2b598661139dd902b5b01ea101f03aaf30ab17576a914a3328f18ac1892a6667f713d" "7020ff3437d973c888acfeffffff0180ed3e17000000001976a914353352b7ce1e3c9c05ffcd6ae97609d" "e2999744488accdf50a00" ) tx = Transaction(raw) self.assertEqual(tx.id, 'e466881128889d1cc4110627753051c22e72a81d11229a1a1337da06940bebcf') self.assertEqual(tx.version, 2) self.assertEqual(tx.locktime, 718285,) self.assertEqual(len(tx.inputs), 1) self.assertEqual(len(tx.outputs), 1) txin = tx.inputs[0] self.assertEqual( txin.txo_ref.id, '797e69a9cdcdacf1262bc41a3b2de57c6357aae8f2d416c5fd385240c2239240:1' ) self.assertEqual(txin.txo_ref.position, 1) self.assertEqual(txin.sequence, 4294967294) self.assertIsNone(txin.coinbase) self.assertEqual(txin.script.template.name, 'script_hash+timelock') self.assertEqual( hexlify(txin.script.values['signature']), b'30440220033d5286f161da717d9d1bc3c2bc28da7636b38fc0c6aefb1e0864212f' b'05282c02205df3ce135e79c76d44489212f77ad4e3a838562e601e6377704fa620' b'6a6ae44f01' ) self.assertEqual( hexlify(txin.script.values['pubkey']), b'02261773e7eebe9da80a5653d865cc600362f8e7b2b598661139dd902b5b01ea10' ) script = txin.script.values['script'] self.assertEqual(script.template.name, 'timelock') self.assertEqual(script.values['height'], 717738) self.assertEqual(hexlify(script.values['pubkey_hash']), b'a3328f18ac1892a6667f713d7020ff3437d973c8') class TestTransactionSigning(AsyncioTestCase): async def asyncSetUp(self): self.ledger = Ledger({ 'db': Database(':memory:'), 'headers': Headers(':memory:') }) await self.ledger.db.open() async def asyncTearDown(self): await self.ledger.db.close() async def test_sign(self): account = Account.from_dict( self.ledger, Wallet(), { "seed": "carbon smart garage balance margin twelve chest sword toas" "t envelope bottom stomach absent" } ) await account.ensure_address_gap() address1, address2 = await account.receiving.get_addresses(limit=2) pubkey_hash1 = self.ledger.address_to_hash160(address1) pubkey_hash2 = self.ledger.address_to_hash160(address2) tx = Transaction() \ .add_inputs([Input.spend(get_output(int(2*COIN), pubkey_hash1))]) \ .add_outputs([Output.pay_pubkey_hash(int(1.9*COIN), pubkey_hash2)]) await tx.sign([account]) self.assertEqual( hexlify(tx.inputs[0].script.values['signature']), b'304402200dafa26ad7cf38c5a971c8a25ce7d85a076235f146126762296b1223c42ae21e022020ef9eeb8' b'398327891008c5c0be4357683f12cb22346691ff23914f457bf679601' ) class TransactionIOBalancing(AsyncioTestCase): async def asyncSetUp(self): wallet_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, wallet_dir) self.ledger = Ledger({ 'db': Database(os.path.join(wallet_dir, 'blockchain.db')), 'headers': Headers(':memory:'), }) await self.ledger.db.open() self.account = Account.from_dict( self.ledger, Wallet(), { "seed": "carbon smart garage balance margin twelve chest sword " "toast envelope bottom stomach absent" } ) addresses = await self.account.ensure_address_gap() self.pubkey_hash = [self.ledger.address_to_hash160(a) for a in addresses] self.hash_cycler = cycle(self.pubkey_hash) async def asyncTearDown(self): await self.ledger.db.close() def txo(self, amount, address=None): return get_output(int(amount*COIN), address or next(self.hash_cycler)) def txi(self, txo): return Input.spend(txo) def tx(self, inputs, outputs): return Transaction.create(inputs, outputs, [self.account], self.account) async def create_utxos(self, amounts): utxos = [self.txo(amount) for amount in amounts] self.funding_tx = Transaction(is_verified=True) \ .add_inputs([self.txi(self.txo(sum(amounts)+0.1))]) \ .add_outputs(utxos) await self.ledger.db.insert_transaction(self.funding_tx) for utxo in utxos: await self.ledger.db.save_transaction_io( self.funding_tx, self.ledger.hash160_to_address(utxo.script.values['pubkey_hash']), utxo.script.values['pubkey_hash'], '' ) return utxos @staticmethod def inputs(tx): return [round(i.amount/COIN, 2) for i in tx.inputs] @staticmethod def outputs(tx): return [round(o.amount/COIN, 2) for o in tx.outputs] async def test_basic_use_cases(self): self.ledger.fee_per_byte = int(.01*CENT) # available UTXOs for filling missing inputs utxos = await self.create_utxos([ 1, 1, 3, 5, 10 ]) # pay 3 coins (3.02 w/ fees) tx = await self.tx( [], # inputs [self.txo(3)] # outputs ) # best UTXO match is 5 (as UTXO 3 will be short 0.02 to cover fees) self.assertListEqual(self.inputs(tx), [5]) # a change of 1.98 is added to reach balance self.assertListEqual(self.outputs(tx), [3, 1.98]) await self.ledger.release_outputs(utxos) # pay 2.98 coins (3.00 w/ fees) tx = await self.tx( [], # inputs [self.txo(2.98)] # outputs ) # best UTXO match is 3 and no change is needed self.assertListEqual(self.inputs(tx), [3]) self.assertListEqual(self.outputs(tx), [2.98]) await self.ledger.release_outputs(utxos) # supplied input and output, but input is not enough to cover output tx = await self.tx( [self.txi(self.txo(10))], # inputs [self.txo(11)] # outputs ) # additional input is chosen (UTXO 3) self.assertListEqual([10, 3], self.inputs(tx)) # change is now needed to consume extra input self.assertListEqual([11, 1.96], self.outputs(tx)) await self.ledger.release_outputs(utxos) # liquidating a UTXO tx = await self.tx( [self.txi(self.txo(10))], # inputs [] # outputs ) self.assertListEqual([10], self.inputs(tx)) # missing change added to consume the amount self.assertListEqual([9.98], self.outputs(tx)) await self.ledger.release_outputs(utxos) # liquidating at a loss, requires adding extra inputs tx = await self.tx( [self.txi(self.txo(0.01))], # inputs [] # outputs ) # UTXO 1 is added to cover some of the fee self.assertListEqual([0.01, 1], self.inputs(tx)) # change is now needed to consume extra input self.assertListEqual([0.97], self.outputs(tx)) async def test_basic_use_cases_sqlite(self): self.ledger.coin_selection_strategy = 'sqlite' self.ledger.fee_per_byte = int(0.01*CENT) # available UTXOs for filling missing inputs utxos = await self.create_utxos([ 1, 1, 3, 5, 10 ]) self.assertEqual(5, len(await self.ledger.get_utxos())) # pay 3 coins (3.07 w/ fees) tx = await self.tx( [], # inputs [self.txo(3)] # outputs ) await self.ledger.db.db.run(self.ledger.db._transaction_io, tx, tx.outputs[0].get_address(self.ledger), tx.id) self.assertListEqual(self.inputs(tx), [1.0, 1.0, 3.0]) # a change of 1.95 is added to reach balance self.assertListEqual(self.outputs(tx), [3, 1.95]) # utxos: 1.95, 3, 5, 10 self.assertEqual(2, len(await self.ledger.get_utxos())) # pay 4.946 coins (5.00 w/ fees) tx = await self.tx( [], # inputs [self.txo(4.946)] # outputs ) self.assertEqual(1, len(await self.ledger.get_utxos())) self.assertListEqual(self.inputs(tx), [5.0]) self.assertEqual(2, len(tx.outputs)) self.assertEqual(494600000, tx.outputs[0].amount) # utxos: 3, 1.95, 4.946, 10 await self.ledger.release_outputs(utxos) # supplied input and output, but input is not enough to cover output tx = await self.tx( [self.txi(self.txo(10))], # inputs [self.txo(11)] # outputs ) # additional input is chosen (UTXO 1) self.assertListEqual([10, 1.0, 1.0], self.inputs(tx)) # change is now needed to consume extra input self.assertListEqual([11, 0.95], self.outputs(tx)) await self.ledger.release_outputs(utxos) # liquidating a UTXO tx = await self.tx( [self.txi(self.txo(10))], # inputs [] # outputs ) self.assertListEqual([10], self.inputs(tx)) # missing change added to consume the amount self.assertListEqual([9.98], self.outputs(tx)) await self.ledger.release_outputs(utxos) # liquidating at a loss, requires adding extra inputs tx = await self.tx( [self.txi(self.txo(0.01))], # inputs [] # outputs ) # UTXO 1 is added to cover some of the fee self.assertListEqual([0.01, 1], self.inputs(tx)) # change is now needed to consume extra input self.assertListEqual([0.97], self.outputs(tx)) ================================================ FILE: tests/unit/wallet/test_utils.py ================================================ import unittest from lbry.wallet.util import ArithUint256 from lbry.wallet.util import coins_to_satoshis as c2s, satoshis_to_coins as s2c class TestCoinValueParsing(unittest.TestCase): def test_good_output(self): self.assertEqual(s2c(1), "0.00000001") self.assertEqual(s2c(10**7), "0.1") self.assertEqual(s2c(2*10**8), "2.0") self.assertEqual(s2c(2*10**17), "2000000000.0") def test_good_input(self): self.assertEqual(c2s("0.00000001"), 1) self.assertEqual(c2s("0.1"), 10**7) self.assertEqual(c2s("1.0"), 10**8) self.assertEqual(c2s("2.00000000"), 2*10**8) self.assertEqual(c2s("2000000000.0"), 2*10**17) def test_bad_input(self): with self.assertRaises(ValueError): c2s("1") with self.assertRaises(ValueError): c2s("-1.0") with self.assertRaises(ValueError): c2s("10000000000.0") with self.assertRaises(ValueError): c2s("1.000000000") with self.assertRaises(ValueError): c2s("-0") with self.assertRaises(ValueError): c2s("1") with self.assertRaises(ValueError): c2s(".1") with self.assertRaises(ValueError): c2s("1e-7") class TestArithUint256(unittest.TestCase): def test_arithunit256(self): # https://github.com/bitcoin/bitcoin/blob/master/src/test/arith_uint256_tests.cpp from_compact = ArithUint256.from_compact eq = self.assertEqual eq(from_compact(0).value, 0) eq(from_compact(0x00123456).value, 0) eq(from_compact(0x01003456).value, 0) eq(from_compact(0x02000056).value, 0) eq(from_compact(0x03000000).value, 0) eq(from_compact(0x04000000).value, 0) eq(from_compact(0x00923456).value, 0) eq(from_compact(0x01803456).value, 0) eq(from_compact(0x02800056).value, 0) eq(from_compact(0x03800000).value, 0) eq(from_compact(0x04800000).value, 0) # Make sure that we don't generate compacts with the 0x00800000 bit set uint = ArithUint256(0x80) eq(uint.compact, 0x02008000) uint = from_compact(0x01123456) eq(uint.value, 0x12) eq(uint.compact, 0x01120000) uint = from_compact(0x01fedcba) eq(uint.value, 0x7e) eq(uint.negative, 0x01fe0000) uint = from_compact(0x02123456) eq(uint.value, 0x1234) eq(uint.compact, 0x02123400) uint = from_compact(0x03123456) eq(uint.value, 0x123456) eq(uint.compact, 0x03123456) uint = from_compact(0x04123456) eq(uint.value, 0x12345600) eq(uint.compact, 0x04123456) uint = from_compact(0x04923456) eq(uint.value, 0x12345600) eq(uint.negative, 0x04923456) uint = from_compact(0x05009234) eq(uint.value, 0x92340000) eq(uint.compact, 0x05009234) uint = from_compact(0x20123456) eq(uint.value, 0x1234560000000000000000000000000000000000000000000000000000000000) eq(uint.compact, 0x20123456) ================================================ FILE: tests/unit/wallet/test_wallet.py ================================================ import json import jsonschema import os import tempfile from binascii import hexlify import lbry.schema.types.v2 as schema_v2 from unittest import TestCase, mock from lbry.testcase import AsyncioTestCase from lbry.wallet import ( Ledger, RegTestLedger, WalletManager, Account, Wallet, WalletStorage, TimestampedPreferences ) class TestWalletCreation(AsyncioTestCase): async def asyncSetUp(self): self.manager = WalletManager() config = {'data_path': '/tmp/wallet'} self.main_ledger = self.manager.get_or_create_ledger(Ledger.get_id(), config) self.test_ledger = self.manager.get_or_create_ledger(RegTestLedger.get_id(), config) def test_create_wallet_and_accounts(self): wallet = Wallet() self.assertEqual(wallet.name, 'Wallet') self.assertListEqual(wallet.accounts, []) account1 = wallet.generate_account(self.main_ledger) wallet.generate_account(self.main_ledger) wallet.generate_account(self.test_ledger) self.assertEqual(wallet.default_account, account1) self.assertEqual(len(wallet.accounts), 3) def test_load_and_save_wallet(self): wallet_dict = { 'version': 1, 'name': 'Main Wallet', 'preferences': {}, 'accounts': [ { 'certificates': {}, 'name': 'An Account', 'ledger': 'lbc_mainnet', 'modified_on': 123, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm' 'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', 'address_generator': { 'name': 'deterministic-chain', 'receiving': {'gap': 17, 'maximum_uses_per_address': 3}, 'change': {'gap': 10, 'maximum_uses_per_address': 3} } } ] } storage = WalletStorage(default=wallet_dict) wallet = Wallet.from_storage(storage, self.manager) self.assertEqual(wallet.name, 'Main Wallet') self.assertEqual( hexlify(wallet.hash), b'869acc4660dde0f13784ed743796adf89562cdf79fdfc9e5c6dbea98d62ccf90' ) self.assertEqual(len(wallet.accounts), 1) account = wallet.default_account self.assertIsInstance(account, Account) self.maxDiff = None self.assertDictEqual(wallet_dict, wallet.to_dict()) encrypted = wallet.pack('password') decrypted = Wallet.unpack('password', encrypted) self.assertEqual(decrypted['accounts'][0]['name'], 'An Account') def test_wallet_file_schema(self): wallet_dict = { 'version': 1, 'name': 'Main Wallet', 'preferences': {}, 'accounts': [ { 'certificates': {'x': 'y'}, 'name': 'Account 1', 'ledger': 'lbc_mainnet', 'modified_on': 123, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm' 'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', 'address_generator': { 'name': 'deterministic-chain', 'receiving': {'gap': 17, 'maximum_uses_per_address': 3}, 'change': {'gap': 10, 'maximum_uses_per_address': 3} } }, { 'certificates': {'a': 'b'}, 'name': 'Account 2', 'ledger': 'lbc_mainnet', 'modified_on': 123, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': True, 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm' 'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', 'address_generator': { 'name': 'single-address', } }, ] } storage = WalletStorage(default=wallet_dict) wallet = Wallet.from_storage(storage, self.manager) self.assertDictEqual(wallet_dict, wallet.to_dict()) with open(os.path.join(*schema_v2.__path__, 'wallet.json')) as f: wallet_schema = json.load(f) jsonschema.validate(schema=wallet_schema, instance=wallet.to_dict()) def test_no_password_but_encryption_preferred(self): wallet_dict = { 'version': 1, 'name': 'Main Wallet', 'preferences': { "encrypt-on-disk": { "ts": 1571762543.351794, "value": True }, }, 'accounts': [ { 'certificates': {}, 'name': 'An Account', 'ledger': 'lbc_mainnet', 'modified_on': 123, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm' 'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', 'address_generator': { 'name': 'deterministic-chain', 'receiving': {'gap': 17, 'maximum_uses_per_address': 3}, 'change': {'gap': 10, 'maximum_uses_per_address': 3} } } ] } storage = WalletStorage(default=wallet_dict) wallet = Wallet.from_storage(storage, self.manager) self.assertEqual( hexlify(wallet.hash), b'8cc6341885e6ad46f72a17364c65f8441f09e79996c55202196b399c75f8d751' ) self.assertFalse(wallet.is_encrypted) def test_read_write(self): manager = WalletManager() config = {'data_path': '/tmp/wallet'} ledger = manager.get_or_create_ledger(Ledger.get_id(), config) with tempfile.NamedTemporaryFile(suffix='.json') as wallet_file: wallet_file.write(b'{"version": 1}') wallet_file.seek(0) # create and write wallet to a file wallet = manager.import_wallet(wallet_file.name) account = wallet.generate_account(ledger) wallet.save() # read wallet from file wallet_storage = WalletStorage(wallet_file.name) wallet = Wallet.from_storage(wallet_storage, manager) self.assertEqual(account.public_key.address, wallet.default_account.public_key.address) def test_merge(self): wallet1 = Wallet() wallet1.preferences['one'] = 1 wallet1.preferences['conflict'] = 1 wallet1.generate_account(self.main_ledger) wallet2 = Wallet() wallet2.preferences['two'] = 2 wallet2.preferences['conflict'] = 2 # will be more recent wallet2.generate_account(self.main_ledger) self.assertEqual(len(wallet1.accounts), 1) self.assertEqual(wallet1.preferences, {'one': 1, 'conflict': 1}) added, _ = wallet1.merge(self.manager, 'password', wallet2.pack('password')) self.assertEqual(added[0].id, wallet2.default_account.id) self.assertEqual(len(wallet1.accounts), 2) self.assertEqual(wallet1.accounts[1].id, wallet2.default_account.id) self.assertEqual(wallet1.preferences, {'one': 1, 'two': 2, 'conflict': 2}) class TestTimestampedPreferences(TestCase): def test_init(self): p = TimestampedPreferences() p['one'] = 1 p2 = TimestampedPreferences(p.data) self.assertEqual(p2['one'], 1) def test_hash(self): p = TimestampedPreferences() self.assertEqual( hexlify(p.hash), b'44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a' ) with mock.patch('time.time', mock.Mock(return_value=12345)): p['one'] = 1 self.assertEqual( hexlify(p.hash), b'c9e82bf4cb099dd0125f78fa381b21a8131af601917eb531e1f5f980f8f3da66' ) def test_merge(self): p1 = TimestampedPreferences() p2 = TimestampedPreferences() with mock.patch('time.time', mock.Mock(return_value=10)): p1['one'] = 1 p1['conflict'] = 1 with mock.patch('time.time', mock.Mock(return_value=20)): p2['two'] = 2 p2['conflict'] = 2 # conflict in p2 overrides conflict in p1 p1.merge(p2.data) self.assertEqual(p1, {'one': 1, 'two': 2, 'conflict': 2}) # have a newer conflict in p1 so it is not overridden this time with mock.patch('time.time', mock.Mock(return_value=21)): p1['conflict'] = 1 p1.merge(p2.data) self.assertEqual(p1, {'one': 1, 'two': 2, 'conflict': 1}) ================================================ FILE: tox.ini ================================================ [testenv] usedevelop = true deps = coverage extras = test hub changedir = {toxinidir}/tests setenv = HOME=/tmp ELASTIC_HOST={env:ELASTIC_HOST:localhost} commands = orchstr8 download blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs} claims: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.claims {posargs} takeovers: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.takeovers {posargs} transactions: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.transactions {posargs} datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs} other: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.other {posargs}