Repository: platformio/platformio-core Branch: develop Commit: 73d9f3dbea80 Files: 350 Total size: 1.5 MB Directory structure: gitextract_4sox4ymo/ ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE.md │ └── workflows/ │ ├── core.yml │ ├── deployment.yml │ ├── docs.yml │ ├── examples.yml │ └── projects.yml ├── .gitignore ├── .gitmodules ├── .pylintrc ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── HISTORY.rst ├── LICENSE ├── Makefile ├── README.rst ├── SECURITY.md ├── platformio/ │ ├── __init__.py │ ├── __main__.py │ ├── account/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── client.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ ├── destroy.py │ │ │ ├── forgot.py │ │ │ ├── login.py │ │ │ ├── logout.py │ │ │ ├── password.py │ │ │ ├── register.py │ │ │ ├── show.py │ │ │ ├── token.py │ │ │ └── update.py │ │ ├── org/ │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ └── commands/ │ │ │ ├── __init__.py │ │ │ ├── add.py │ │ │ ├── create.py │ │ │ ├── destroy.py │ │ │ ├── list.py │ │ │ ├── remove.py │ │ │ └── update.py │ │ ├── team/ │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ └── commands/ │ │ │ ├── __init__.py │ │ │ ├── add.py │ │ │ ├── create.py │ │ │ ├── destroy.py │ │ │ ├── list.py │ │ │ ├── remove.py │ │ │ └── update.py │ │ └── validate.py │ ├── app.py │ ├── assets/ │ │ ├── schema/ │ │ │ └── library.json │ │ └── system/ │ │ └── 99-platformio-udev.rules │ ├── builder/ │ │ ├── __init__.py │ │ ├── main.py │ │ └── tools/ │ │ ├── __init__.py │ │ ├── pioasm.py │ │ ├── piobuild.py │ │ ├── piohooks.py │ │ ├── pioino.py │ │ ├── piointegration.py │ │ ├── piolib.py │ │ ├── piomaxlen.py │ │ ├── piomisc.py │ │ ├── pioplatform.py │ │ ├── pioproject.py │ │ ├── piosize.py │ │ ├── piotarget.py │ │ ├── piotest.py │ │ └── pioupload.py │ ├── cache.py │ ├── check/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── defect.py │ │ └── tools/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── clangtidy.py │ │ ├── cppcheck.py │ │ └── pvsstudio.py │ ├── cli.py │ ├── commands/ │ │ ├── __init__.py │ │ ├── boards.py │ │ ├── ci.py │ │ ├── device/ │ │ │ └── __init__.py │ │ ├── lib.py │ │ ├── platform.py │ │ ├── settings.py │ │ ├── update.py │ │ └── upgrade.py │ ├── compat.py │ ├── debug/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── config/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── blackmagic.py │ │ │ ├── factory.py │ │ │ ├── generic.py │ │ │ ├── jlink.py │ │ │ ├── mspdebug.py │ │ │ ├── native.py │ │ │ ├── qemu.py │ │ │ └── renode.py │ │ ├── exception.py │ │ ├── helpers.py │ │ └── process/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── client.py │ │ ├── gdb.py │ │ └── server.py │ ├── dependencies.py │ ├── device/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── finder.py │ │ ├── list/ │ │ │ ├── __init__.py │ │ │ ├── command.py │ │ │ └── util.py │ │ └── monitor/ │ │ ├── __init__.py │ │ ├── command.py │ │ ├── filters/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── hexlify.py │ │ │ ├── log2file.py │ │ │ ├── send_on_enter.py │ │ │ └── time.py │ │ └── terminal.py │ ├── exception.py │ ├── fs.py │ ├── home/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── rpc/ │ │ │ ├── __init__.py │ │ │ ├── handlers/ │ │ │ │ ├── __init__.py │ │ │ │ ├── account.py │ │ │ │ ├── app.py │ │ │ │ ├── base.py │ │ │ │ ├── ide.py │ │ │ │ ├── misc.py │ │ │ │ ├── os.py │ │ │ │ ├── piocore.py │ │ │ │ ├── platform.py │ │ │ │ ├── project.py │ │ │ │ └── registry.py │ │ │ └── server.py │ │ └── run.py │ ├── http.py │ ├── maintenance.py │ ├── package/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ ├── exec.py │ │ │ ├── install.py │ │ │ ├── list.py │ │ │ ├── outdated.py │ │ │ ├── pack.py │ │ │ ├── publish.py │ │ │ ├── search.py │ │ │ ├── show.py │ │ │ ├── uninstall.py │ │ │ ├── unpublish.py │ │ │ └── update.py │ │ ├── download.py │ │ ├── exception.py │ │ ├── lockfile.py │ │ ├── manager/ │ │ │ ├── __init__.py │ │ │ ├── _download.py │ │ │ ├── _install.py │ │ │ ├── _legacy.py │ │ │ ├── _registry.py │ │ │ ├── _symlink.py │ │ │ ├── _uninstall.py │ │ │ ├── _update.py │ │ │ ├── base.py │ │ │ ├── core.py │ │ │ ├── library.py │ │ │ ├── platform.py │ │ │ └── tool.py │ │ ├── manifest/ │ │ │ ├── __init__.py │ │ │ ├── parser.py │ │ │ └── schema.py │ │ ├── meta.py │ │ ├── pack.py │ │ ├── unpack.py │ │ ├── vcsclient.py │ │ └── version.py │ ├── platform/ │ │ ├── __init__.py │ │ ├── _packages.py │ │ ├── _run.py │ │ ├── base.py │ │ ├── board.py │ │ ├── exception.py │ │ └── factory.py │ ├── proc.py │ ├── project/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ ├── config.py │ │ │ ├── init.py │ │ │ └── metadata.py │ │ ├── config.py │ │ ├── exception.py │ │ ├── helpers.py │ │ ├── integration/ │ │ │ ├── __init__.py │ │ │ ├── generator.py │ │ │ └── tpls/ │ │ │ ├── clion/ │ │ │ │ └── .gitignore.tpl │ │ │ ├── codeblocks/ │ │ │ │ └── platformio.cbp.tpl │ │ │ ├── eclipse/ │ │ │ │ ├── .cproject.tpl │ │ │ │ ├── .project.tpl │ │ │ │ └── .settings/ │ │ │ │ ├── PlatformIO Debugger.launch.tpl │ │ │ │ ├── language.settings.xml.tpl │ │ │ │ └── org.eclipse.cdt.core.prefs.tpl │ │ │ ├── emacs/ │ │ │ │ ├── .ccls.tpl │ │ │ │ └── .gitignore.tpl │ │ │ ├── netbeans/ │ │ │ │ └── nbproject/ │ │ │ │ ├── configurations.xml.tpl │ │ │ │ ├── private/ │ │ │ │ │ ├── configurations.xml.tpl │ │ │ │ │ ├── launcher.properties.tpl │ │ │ │ │ └── private.xml.tpl │ │ │ │ └── project.xml.tpl │ │ │ ├── qtcreator/ │ │ │ │ ├── .gitignore.tpl │ │ │ │ ├── Makefile.tpl │ │ │ │ ├── platformio.cflags.tpl │ │ │ │ ├── platformio.config.tpl │ │ │ │ ├── platformio.creator.tpl │ │ │ │ ├── platformio.cxxflags.tpl │ │ │ │ ├── platformio.files.tpl │ │ │ │ └── platformio.includes.tpl │ │ │ ├── sublimetext/ │ │ │ │ ├── .ccls.tpl │ │ │ │ └── platformio.sublime-project.tpl │ │ │ ├── vim/ │ │ │ │ ├── .ccls.tpl │ │ │ │ └── .gitignore.tpl │ │ │ ├── visualstudio/ │ │ │ │ ├── platformio.vcxproj.filters.tpl │ │ │ │ └── platformio.vcxproj.tpl │ │ │ └── vscode/ │ │ │ ├── .gitignore.tpl │ │ │ └── .vscode/ │ │ │ ├── c_cpp_properties.json.tpl │ │ │ ├── extensions.json.tpl │ │ │ └── launch.json.tpl │ │ ├── options.py │ │ └── savedeps.py │ ├── public.py │ ├── registry/ │ │ ├── __init__.py │ │ ├── access/ │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ ├── commands/ │ │ │ │ ├── __init__.py │ │ │ │ ├── grant.py │ │ │ │ ├── list.py │ │ │ │ ├── private.py │ │ │ │ ├── public.py │ │ │ │ └── revoke.py │ │ │ └── validate.py │ │ ├── client.py │ │ └── mirror.py │ ├── remote/ │ │ ├── __init__.py │ │ ├── ac/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── process.py │ │ │ ├── psync.py │ │ │ └── serial.py │ │ ├── cli.py │ │ ├── client/ │ │ │ ├── __init__.py │ │ │ ├── agent_list.py │ │ │ ├── agent_service.py │ │ │ ├── async_base.py │ │ │ ├── base.py │ │ │ ├── device_list.py │ │ │ ├── device_monitor.py │ │ │ ├── run_or_test.py │ │ │ └── update_core.py │ │ ├── factory/ │ │ │ ├── __init__.py │ │ │ ├── client.py │ │ │ └── ssl.py │ │ └── projectsync.py │ ├── run/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── helpers.py │ │ └── processor.py │ ├── system/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ ├── completion.py │ │ │ ├── info.py │ │ │ └── prune.py │ │ ├── completion.py │ │ └── prune.py │ ├── telemetry.py │ ├── test/ │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── exception.py │ │ ├── helpers.py │ │ ├── reports/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── json.py │ │ │ ├── junit.py │ │ │ └── stdout.py │ │ ├── result.py │ │ └── runners/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── doctest.py │ │ ├── factory.py │ │ ├── googletest.py │ │ ├── readers/ │ │ │ ├── __init__.py │ │ │ ├── native.py │ │ │ └── serial.py │ │ └── unity.py │ └── util.py ├── scripts/ │ ├── docspregen.py │ ├── fixsymlink.py │ └── install_devplatforms.py ├── setup.py ├── tests/ │ ├── __init__.py │ ├── commands/ │ │ ├── __init__.py │ │ ├── pkg/ │ │ │ ├── __init__.py │ │ │ ├── test_exec.py │ │ │ ├── test_install.py │ │ │ ├── test_list.py │ │ │ ├── test_outdated.py │ │ │ ├── test_search.py │ │ │ ├── test_show.py │ │ │ ├── test_uninstall.py │ │ │ └── test_update.py │ │ ├── test_account_org_team.py │ │ ├── test_boards.py │ │ ├── test_check.py │ │ ├── test_ci.py │ │ ├── test_init.py │ │ ├── test_lib.py │ │ ├── test_lib_complex.py │ │ ├── test_platform.py │ │ ├── test_run.py │ │ ├── test_settings.py │ │ └── test_test.py │ ├── conftest.py │ ├── misc/ │ │ ├── __init__.py │ │ ├── ino2cpp/ │ │ │ ├── __init__.py │ │ │ ├── examples/ │ │ │ │ ├── basic/ │ │ │ │ │ └── basic.ino │ │ │ │ ├── multifiles/ │ │ │ │ │ ├── bar.ino │ │ │ │ │ └── foo.pde │ │ │ │ └── strmultilines/ │ │ │ │ └── main.ino │ │ │ └── test_ino2cpp.py │ │ ├── test_maintenance.py │ │ └── test_misc.py │ ├── package/ │ │ ├── __init__.py │ │ ├── test_manager.py │ │ ├── test_manifest.py │ │ ├── test_meta.py │ │ └── test_pack.py │ ├── project/ │ │ ├── __init__.py │ │ ├── test_config.py │ │ ├── test_metadata.py │ │ └── test_savedeps.py │ └── test_examples.py └── tox.ini ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/FUNDING.yml ================================================ custom: https://platformio.org/donate ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ What kind of issue is this? - [ ] **Question**. This issue tracker is not the place for questions. If you want to ask how to do something, or to understand why something isn't working the way you expect it to, use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support) - [ ] **PlatformIO IDE**. All issues related to PlatformIO IDE should be reported to the [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) repository - [ ] **Development Platform or Board**. All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms should be reported to appropriate repository related to your hardware https://github.com/topics/platformio-platform - [ ] **Feature Request**. Start by telling us what problem you’re trying to solve. Often a solution already exists! Don’t send pull requests to implement new features without first getting our support. Sometimes we leave features out on purpose to keep the project small. - [ ] **PlatformIO Core**. If you’ve found a bug, please provide an information below. *You can erase any parts of this template not applicable to your Issue.* ------------------------------------------------------------------ ### Configuration **Operating system**: **PlatformIO Version** (`platformio --version`): ### Description of problem #### Steps to Reproduce 1. 2. 3. ### Actual Results ### Expected Results ### If problems with PlatformIO Build System: **The content of `platformio.ini`:** ```ini Insert here... ``` **Source file to reproduce issue:** ```cpp Insert here... ``` ### Additional info ================================================ FILE: .github/workflows/core.yml ================================================ name: Core on: [push, pull_request] jobs: build: strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ["3.11", "3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 with: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install tox - name: Run "codespell" on Linux if: startsWith(matrix.os, 'ubuntu') run: | python -m pip install codespell make codespell - name: Core System Info run: | tox -e py - name: Integration Tests if: ${{ matrix.python-version == '3.11' }} run: | tox -e testcore - name: Slack Notification uses: homoluctus/slatify@master if: failure() with: type: ${{ job.status }} job_name: '*Core*' commit: true url: ${{ secrets.SLACK_BUILD_WEBHOOK }} token: ${{ secrets.SLACK_GITHUB_TOKEN }} ================================================ FILE: .github/workflows/deployment.yml ================================================ name: Deployment on: push: branches: - "master" - "release/**" jobs: deployment: runs-on: ubuntu-latest environment: production steps: - uses: actions/checkout@v6 with: submodules: "recursive" - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip pip install tox build - name: Deployment Tests env: TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }} TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }} TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }} run: | tox -e testcore - name: Build Python distributions run: python -m build - name: Publish package to PyPI if: ${{ github.ref == 'refs/heads/master' }} uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} ================================================ FILE: .github/workflows/docs.yml ================================================ name: Docs on: [push, pull_request] jobs: build: name: Build Docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 with: submodules: "recursive" - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip pip install tox - name: Build docs run: | tox -e docs - name: Slack Notification uses: homoluctus/slatify@master if: failure() with: type: ${{ job.status }} job_name: '*Docs*' commit: true url: ${{ secrets.SLACK_BUILD_WEBHOOK }} token: ${{ secrets.SLACK_GITHUB_TOKEN }} - name: Preserve Docs if: ${{ github.event_name == 'push' }} run: | tar -czvf docs.tar.gz -C docs/_build html rtdpage - name: Save artifact if: ${{ github.event_name == 'push' }} uses: actions/upload-artifact@v4 with: name: docs path: ./docs.tar.gz deploy: name: Deploy Docs needs: build runs-on: ubuntu-latest env: DOCS_REPO: platformio/platformio-docs DOCS_DIR: platformio-docs LATEST_DOCS_DIR: latest-docs RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }} if: ${{ github.event_name == 'push' }} steps: - name: Download artifact uses: actions/download-artifact@v4 with: name: docs - name: Unpack artifact run: | mkdir ./${{ env.LATEST_DOCS_DIR }} tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }} - name: Delete Artifact uses: geekyeggo/delete-artifact@v5 with: name: docs - name: Select Docs type id: get-destination-dir run: | if [[ ${{ env.RELEASE_BUILD }} == true ]]; then echo "::set-output name=dst_dir::stable" else echo "::set-output name=dst_dir::latest" fi - name: Checkout latest Docs continue-on-error: true uses: actions/checkout@v6 with: repository: ${{ env.DOCS_REPO }} path: ${{ env.DOCS_DIR }} ref: gh-pages - name: Synchronize Docs run: | rm -rf ${{ env.DOCS_DIR }}/.git rm -rf ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} mkdir -p ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} cp -rf ${{ env.LATEST_DOCS_DIR }}/html/* ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} if [[ ${{ env.RELEASE_BUILD }} == false ]]; then rm -rf ${{ env.DOCS_DIR }}/page mkdir -p ${{ env.DOCS_DIR }}/page cp -rf ${{ env.LATEST_DOCS_DIR }}/rtdpage/* ${{ env.DOCS_DIR }}/page fi - name: Validate Docs run: | if [ -z "$(ls -A ${{ env.DOCS_DIR }})" ]; then echo "Docs folder is empty. Aborting!" exit 1 fi - name: Deploy to Github Pages uses: peaceiris/actions-gh-pages@v4 with: personal_token: ${{ secrets.DEPLOY_GH_DOCS_TOKEN }} external_repository: ${{ env.DOCS_REPO }} publish_dir: ./${{ env.DOCS_DIR }} commit_message: Sync Docs ================================================ FILE: .github/workflows/examples.yml ================================================ name: Examples on: [push, pull_request] jobs: build: strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} env: PIO_INSTALL_DEVPLATFORM_OWNERNAMES: "platformio" PIO_INSTALL_DEVPLATFORM_NAMES: "aceinna_imu,atmelavr,atmelmegaavr,atmelsam,espressif32,espressif8266,nordicnrf52,raspberrypi,ststm32,teensy" steps: - name: Free Disk Space uses: endersonmenezes/free-disk-space@v3 with: remove_android: true remove_dotnet: true remove_haskell: true # Faster cleanup remove_packages_one_command: true rm_cmd: "rmz" - uses: actions/checkout@v6 with: submodules: "recursive" - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip pip install tox - name: Run on Linux if: startsWith(matrix.os, 'ubuntu') run: | # Free space sudo apt clean # docker rmi $(docker image ls -aq) df -h tox -e testexamples - name: Run on macOS if: startsWith(matrix.os, 'macos') run: | df -h tox -e testexamples - name: Run on Windows if: startsWith(matrix.os, 'windows') env: PLATFORMIO_CORE_DIR: C:/pio PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH run: | tox -e testexamples - name: Slack Notification uses: homoluctus/slatify@master if: failure() with: type: ${{ job.status }} job_name: '*Examples*' commit: true url: ${{ secrets.SLACK_BUILD_WEBHOOK }} token: ${{ secrets.SLACK_GITHUB_TOKEN }} ================================================ FILE: .github/workflows/projects.yml ================================================ name: Projects on: [push, pull_request] jobs: build: strategy: fail-fast: false matrix: project: - marlin: repository: "MarlinFirmware/Marlin" folder: "Marlin" config_dir: "Marlin" env_name: "mega2560" - smartknob: repository: "scottbez1/smartknob" folder: "smartknob" config_dir: "smartknob" env_name: "view" - espurna: repository: "xoseperez/espurna" folder: "espurna" config_dir: "espurna/code" env_name: "nodemcu-lolin" - OpenMQTTGateway: repository: "1technophile/OpenMQTTGateway" folder: "OpenMQTTGateway" config_dir: "OpenMQTTGateway" env_name: "esp32-m5atom-lite" os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 with: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: 3.11 - name: Install PlatformIO run: pip install -U . - name: Check out ${{ matrix.project.repository }} uses: actions/checkout@v6 with: submodules: "recursive" repository: ${{ matrix.project.repository }} path: ${{ matrix.project.folder }} - name: Compile ${{ matrix.project.repository }} run: pio run -d ${{ matrix.project.config_dir }} -e ${{ matrix.project.env_name }} ================================================ FILE: .gitignore ================================================ *.egg-info *.pyc __pycache__ .tox docs/_build dist build .cache coverage.xml .coverage htmlcov .pytest_cache ================================================ FILE: .gitmodules ================================================ [submodule "examples"] path = examples url = https://github.com/platformio/platformio-examples.git [submodule "docs"] path = docs url = https://github.com/platformio/platformio-docs.git branch = develop ================================================ FILE: .pylintrc ================================================ [REPORTS] output-format=colorized [MESSAGES CONTROL] disable= missing-docstring, duplicate-code, invalid-name, too-few-public-methods, consider-using-f-string, cyclic-import, use-dict-literal ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Code of Conduct See https://piolabs.com/legal/code-of-conduct.html ================================================ FILE: CONTRIBUTING.md ================================================ Contributing ------------ To get started, sign the Contributor License Agreement. 1. Fork the repository on GitHub 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git` 3. Run `pip install tox` 4. Go to the root of the PlatformIO Core project where `tox.ini` is located (``cd platformio-core``) and run `tox -e py39`. You can replace `py39` with your own Python version. For example, `py311` means Python 3.11. 5. Activate current development environment: * Windows: `.tox\py39\Scripts\activate` * Bash/ZSH: `source .tox/py39/bin/activate` * Fish: `source .tox/py39/bin/activate.fish` 6. Make changes to code, documentation, etc. 7. Lint source code `make before-commit` 8. Run the tests `make test` 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html) 10. Commit changes to your forked repository 11. Submit a Pull Request on GitHub ================================================ FILE: HISTORY.rst ================================================ Release Notes ============= .. |PIOCONF| replace:: `"platformio.ini" `__ configuration file .. |LIBRARYJSON| replace:: `library.json `__ .. |LDF| replace:: `LDF `__ .. |INTERPOLATION| replace:: `Interpolation of Values `__ .. |UNITTESTING| replace:: `Unit Testing `__ .. |DEBUGGING| replace:: `Debugging `__ .. |STATICCODEANALYSIS| replace:: `Static Code Analysis `__ .. |PIOHOME| replace:: `PIO Home `__ .. _release_notes_6: PlatformIO Core 6 ----------------- Unlock the true potential of embedded software development with PlatformIO's collaborative ecosystem, embracing declarative principles, test-driven methodologies, and modern toolchains for unrivaled success. 6.1.19 (2026-02-04) ~~~~~~~~~~~~~~~~~~~ * Added support for Python 3.14 * Upgraded the `Doctest `__ testing framework to version 2.4.12, the `GoogleTest `__ to version 1.17.0, and the `Unity `__ to version 2.6.1, incorporating the latest features and improvements for enhanced testing capabilities * Enhanced compatibility with the CCLS language server, improving integration with editors like `Emacs `__, `Sublime Text `__, and `Vim `__ (`issue #5186 `_) * Improved error messages for package installation to make it easier to understand when a package is missing or incompatible (`pull #5336 `_). * Fixed a regression issue where custom build flags were not properly reflected in the `compile_commands.json `__ file, ensuring accurate compilation database generation * Fixed an issue where fully-qualified serial port URLs (e.g., ``rfc2217://host:port``) were incorrectly treated as wildcard patterns (`issue #5225 `_) * Fixed an issue where the toolchain path in static analysis was not handled correctly if it contained spaces (`pull #5351 `_) * Fixed installation failure when the executable path contains spaces for ``postinstall`` scripts and handling both list and string command formats (`pull #5366 `_) * Fixed cleanup of the ``.pio/libdeps`` folder so that leftover libraries are properly removed when the `lib_deps `__ option is empty (`issue #5110 `_) 6.1.18 (2025-03-11) ~~~~~~~~~~~~~~~~~~~ * Resolved a regression issue that prevented |PIOHOME| from opening external links (`issue #5084 `_) 6.1.17 (2025-02-13) ~~~~~~~~~~~~~~~~~~~ * Introduced the `PLATFORMIO_RUN_JOBS `__ environment variable, allowing manual override of the number of parallel build jobs (`issue #5077 `_) * Added support for ``tar.xz`` tarball dependencies (`pull #4974 `_) * Ensured that dependencies of private libraries are no longer unnecessarily re-installed, optimizing dependency management and reducing redundant operations (`issue #4987 `_) * Resolved an issue where the ``compiledb`` target failed to properly escape compiler executable paths containing spaces (`issue #4998 `_) * Resolved an issue with incorrect path resolution when linking static libraries via the `build_flags `__ option (`issue #5004 `_) * Resolved an issue where the ``--project-dir`` flag did not function correctly with the `pio check `__ and `pio debug `__ commands (`issue #5029 `_) * Resolved an issue where the |LDF| occasionally excluded bundled platform libraries from the dependency graph (`pull #4941 `_) 6.1.16 (2024-09-26) ~~~~~~~~~~~~~~~~~~~ * Added support for Python 3.13 * Introduced the `PLATFORMIO_SYSTEM_TYPE `__ environment variable, enabling manual override of the detected system type for greater flexibility and control in custom build environments * Enhanced internet connection checks by falling back to HTTPS protocol when HTTP (port 80) fails (`issue #4980 `_) * Upgraded the build engine to the latest version of SCons (4.8.1) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) * Upgraded the `Doctest `__ testing framework to version 2.4.11, the `GoogleTest `__ to version 1.15.2, and the `Unity `__ to version 2.6.0, incorporating the latest features and improvements for enhanced testing capabilities * Corrected an issue where the incorrect public class was imported for the ``DoctestTestRunner`` (`issue #4949 `_) 6.1.15 (2024-04-25) ~~~~~~~~~~~~~~~~~~~ * Resolved an issue where the |LDF| couldn't locate a library dependency declared via version control system repository (`issue #4885 `_) * Resolved an issue related to the inaccurate detection of the Clang compiler (`pull #4897 `_) 6.1.14 (2024-03-21) ~~~~~~~~~~~~~~~~~~~ * Introduced the ``--json-output`` option to the `pio test `__ command, enabling users to generate test results in the JSON format * Upgraded the build engine to the latest version of SCons (4.7.0) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) * Broadened version support for the ``pyelftools`` dependency, enabling compatibility with lower versions and facilitating integration with a wider range of third-party tools (`issue #4834 `_) * Addressed an issue where passing a relative path (``--project-dir``) to the `pio project init `__ command resulted in an error (`issue #4847 `_) * Enhanced |STATICCODEANALYSIS| to accommodate scenarios where custom ``src_dir`` or ``include_dir`` are located outside the project folder (`pull #4874 `_) * Corrected the validation of ``symlink://`` `package specifications `__ , resolving an issue that caused the package manager to repeatedly reinstall dependencies (`pull #4870 `_) * Resolved an issue related to the relative package path in the `pio pkg publish `__ command * Resolved an issue where the |LDF| selected an incorrect library version (`issue #4860 `_) * Resolved an issue with the ``hexlify`` filter in the `device monitor `__ command, ensuring proper representation of characters with Unicode code points higher than 127 (`issue #4732 `_) 6.1.13 (2024-01-12) ~~~~~~~~~~~~~~~~~~~ * Expanded support for SCons variables declared in the legacy format ``${SCONS_VARNAME}`` (`issue #4828 `_) 6.1.12 (2024-01-10) ~~~~~~~~~~~~~~~~~~~ * Added support for Python 3.12 * Introduced the capability to launch the debug server in a separate process (`issue #4722 `_) * Introduced a warning during the verification of MCU maximum RAM usage, signaling when the allocated RAM surpasses 100% (`issue #4791 `_) * Drastically enhanced the speed of project building when operating in verbose mode (`issue #4783 `_) * Upgraded the build engine to the latest version of SCons (4.6.0) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) * Enhanced the handling of built-in variables in |PIOCONF| during |INTERPOLATION| (`issue #4695 `_) * Enhanced PIP dependency declarations for improved reliability and extended support to include Python 3.6 (`issue #4819 `_) * Implemented automatic installation of missing dependencies when utilizing a SOCKS proxy (`issue #4822 `_) * Implemented a fail-safe mechanism to terminate a debugging session if an unknown CLI option is passed (`issue #4699 `_) * Rectified an issue where ``${platformio.name}`` erroneously represented ``None`` as the default `project name `__ (`issue #4717 `_) * Resolved an issue where the ``COMPILATIONDB_INCLUDE_TOOLCHAIN`` setting was not correctly applying to private libraries (`issue #4762 `_) * Resolved an issue where ``get_systype()`` inaccurately returned the architecture when executed within a Docker container on a 64-bit kernel with a 32-bit userspace (`issue #4777 `_) * Resolved an issue with incorrect handling of the ``check_src_filters`` option when used in multiple environments (`issue #4788 `_) * Resolved an issue where running `pio project metadata `__ resulted in duplicated "include" entries (`issue #4723 `_) * Resolved an issue where native debugging failed on the host machine (`issue #4745 `_) * Resolved an issue where custom debug configurations were being inadvertently overwritten in VSCode's ``launch.json`` (`issue #4810 `_) 6.1.11 (2023-08-31) ~~~~~~~~~~~~~~~~~~~ * Resolved a possible issue that may cause generated projects for `PlatformIO IDE for VSCode `__ to fail to launch a debug session because of a missing "objdump" binary when GDB is not part of the toolchain package * Resolved a regression issue that resulted in the malfunction of the Memory Inspection feature within |PIOHOME| 6.1.10 (2023-08-11) ~~~~~~~~~~~~~~~~~~~ * Resolved an issue that caused generated projects for `PlatformIO IDE for VSCode `__ to break when the ``-iprefix`` compiler flag was used * Resolved an issue encountered while utilizing the `pio pkg exec `__ command on the Windows platform to execute Python scripts from a package * Implemented a crucial improvement to the `pio run `__ command, guaranteeing that the ``monitor`` target is not executed if any of the preceding targets, such as ``upload``, encounter failures * `Cppcheck `__ v2.11 with new checks, CLI commands and various analysis improvements * Resolved a critical issue that arose on macOS ARM platforms due to the Python "requests" module, leading to a "ModuleNotFoundError: No module named 'chardet'" (`issue #4702 `_) 6.1.9 (2023-07-06) ~~~~~~~~~~~~~~~~~~ * Rectified a regression bug that occurred when the ``-include`` flag was passed via the `build_flags `__ option as a relative path and subsequently expanded (`issue #4683 `_) * Resolved an issue that resulted in unresolved absolute toolchain paths when generating the `Compilation database "compile_commands.json" `__ (`issue #4684 `_) 6.1.8 (2023-07-05) ~~~~~~~~~~~~~~~~~~ * Added a new ``--lint`` option to the `pio project config `__ command, enabling users to efficiently perform linting on the |PIOCONF| * Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information * Expanded the functionality of the |LIBRARYJSON| manifest by allowing the use of the underscore symbol in the `keywords `__ field * Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) * Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) * Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) * Updated the `pio project metadata `__ command to return C/C++ flags as parsed Unix shell arguments when dumping project build metadata * Resolved a critical issue related to the usage of the ``-include`` flag within the `build_flags `__ option, specifically when employing dynamic variables (`issue #4682 `_) * Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated `__ 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ * Introduced a new ``--sample-code`` option to the `pio project init `__ command, which allows users to include sample code in the newly created project * Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) * Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) * Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process * Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) * Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) * Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to * Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow * Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 `_, `issue #4493 `_, `issue #4607 `_) * Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) * Resolved an issue where organization details could not be updated using the `pio org update `__ command * Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) * Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) * Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) * Resolved an issue where multiple targets were not executed sequentially (`issue #4604 `_) * Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ * Added support for Python 3.11 * Added a new `name `__ configuration option to customize a project name (`pull #4498 `_) * Made assets (templates, ``99-platformio-udev.rules``) part of Python's module (`issue #4458 `_) * Updated `Clang-Tidy `__ check tool to v15.0.5 with new diagnostics and bugfixes * Removed dependency on the "zeroconf" package and install it only when a user lists mDNS devices (issue with zeroconf's LGPL license) * Show the real error message instead of "Can not remove temporary directory" when |PIOCONF| is broken (`issue #4480 `_) * Fixed an issue with an incorrect test summary when a testcase name includes a colon (`issue #4508 `_) * Fixed an issue when `extends `__ did not override options in the right order (`issue #4462 `_) * Fixed an issue when `pio pkg list `__ and `pio pkg uninstall `__ commands fail if there are circular dependencies in the |LIBRARYJSON| manifests (`issue #4475 `_) 6.1.5 (2022-11-01) ~~~~~~~~~~~~~~~~~~ * Added a new `enable_proxy_strict_ssl `__ setting to disable the proxy server certificate verification (`issue #4432 `_) * Documented `PlatformIO Core Proxy Configuration `__ * Speeded up device port finder by avoiding loading board HWIDs from development platforms * Improved caching of build metadata in debug mode * Fixed an issue when `pio pkg install --storage-dir `__ command requires PlatformIO project (`issue #4410 `_) 6.1.4 (2022-08-12) ~~~~~~~~~~~~~~~~~~ * Added support for accepting the original FileNode environment in a "callback" function when using `Build Middlewares `__ (`pull #4380 `_) * Improved device port finder when using dual channel UART converter (`issue #4367 `_) * Improved project dependency resolving when using the `pio project init --ide `__ command * Upgraded build engine to the SCons 4.4.0 (`release notes `__) * Keep custom "unwantedRecommendations" when generating projects for VSCode (`issue #4383 `_) * Do not resolve project dependencies for the ``cleanall`` target (`issue #4344 `_) * Warn about calling "env.BuildSources" in a POST-type script (`issue #4385 `_) * Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) * Fixed an issue when the "cleanall" target removes dependencies from all working environments (`issue #4386 `_) 6.1.3 (2022-07-18) ~~~~~~~~~~~~~~~~~~ * Fixed a regression bug when opening device monitor without any filters (`issue #4363 `_) 6.1.2 (2022-07-18) ~~~~~~~~~~~~~~~~~~ * Export a ``PIO_UNIT_TESTING`` macro to the project source files and dependent libraries in the |UNITTESTING| mode * Improved detection of Windows architecture (`issue #4353 `_) * Warn about unknown `device monitor filters `__ (`issue #4362 `_) * Fixed a regression bug when `libArchive `__ option declared in the |LIBRARYJSON| manifest was ignored (`issue #4351 `_) * Fixed an issue when the `pio pkg publish `__ command didn't work with Python 3.6 (`issue #4352 `_) 6.1.1 (2022-07-11) ~~~~~~~~~~~~~~~~~~ * Added new ``monitor_encoding`` project configuration option to configure `Device Monitor `__ (`issue #4350 `_) * Allowed specifying project environments for `pio ci `__ command (`issue #4347 `_) * Show "TimeoutError" only in the verbose mode when can not find a serial port * Fixed an issue when a serial port was not automatically detected if the board has predefined HWIDs * Fixed an issue with endless scanning of project dependencies (`issue #4349 `_) * Fixed an issue with |LDF| when incompatible libraries were used for the working project environment with the missed framework (`pull #4346 `_) 6.1.0 (2022-07-06) ~~~~~~~~~~~~~~~~~~ * **Device Manager** - Automatically reconnect device monitor if a connection fails - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection - Handle device monitor disconnects more gracefully (`issue #3939 `_) - Improved a serial port finder for `Black Magic Probe `__ (`issue #4023 `_) - Improved a serial port finder for a board with predefined HWIDs - Replaced ``monitor_flags`` with independent project configuration options: `monitor_parity `__, `monitor_eol `__, `monitor_raw `__, `monitor_echo `__ - Fixed an issue when the monitor filters were not applied in their order (`issue #4320 `_) * **Unit Testing** - Updated "Getting Started" documentation for `GoogleTest `__ testing and mocking framework - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled - Do not immediately terminate a testing program when results are received - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) - Fixed an issue when testing results were wrong in the verbose mode (`issue #4336 `_) * **Build System** - Significantly improved support for `Pre & Post Actions `__ * Allowed to declare actions in the `PRE-type scripts `__ even if the target is not ready yet * Allowed library maintainers to use Pre & Post Actions in the library `extraScript `__ - Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) - Added new `pio run --monitor-port `__ option to specify custom device monitor port to the ``monitor`` target (`issue #4337 `_) - Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) - Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) - Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ - Fixed an issue with the |LDF| when recursively scanning dependencies in the ``chain`` mode - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) * **Package Management** - Fixed an issue when library dependencies were installed for the incompatible project environment (`issue #4338 `_) * **Miscellaneous** - Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ * Control |UNITTESTING| verbosity with a new multilevel `pio test -v `__ command option (`issue #4276 `_) * Follow symbolic links during searching for the unit test suites (`issue #4288 `_) * Show a warning when testing an empty project without a test suite (`issue #4278 `_) * Improved support for `Asking for input (prompts) `_ * Fixed an issue when the `build_src_flags `__ option was applied outside the project scope (`issue #4277 `_) * Fixed an issue with debugging assembly files without preprocessor (".s") 6.0.1 (2022-05-17) ~~~~~~~~~~~~~~~~~~ * Improved support for the renamed configuration options (`issue #4270 `_) * Fixed an issue when calling the built-in `pio device monitor `__ filters * Fixed an issue when using |INTERPOLATION| and merging str+int options (`issue #4271 `_) 6.0.0 (2022-05-16) ~~~~~~~~~~~~~~~~~~ Please check the `Migration guide from 5.x to 6.0 `__. * **Package Management** - New unified Package Management CLI (``pio pkg``): * `pio pkg exec `_ - run command from package tool (`issue #4163 `_) * `pio pkg install `_ - install the project dependencies or custom packages * `pio pkg list `__ - list installed packages * `pio pkg outdated `__ - check for project outdated packages * `pio pkg search `__ - search for packages * `pio pkg show `__ - show package information * `pio pkg uninstall `_ - uninstall the project dependencies or custom packages * `pio pkg update `__ - update the project dependencies or custom packages - Package Manifest * Added support for `"scripts" `__ (`issue #485 `_) * Added support for `multi-licensed `__ packages using SPDX Expressions (`issue #4037 `_) * Added support for `"dependencies" `__ declared in a "tool" package manifest - Added support for `symbolic links `__ allowing pointing the local source folder to the Package Manager (`issue #3348 `_) - Automatically install dependencies of the local (private) project libraries (`issue #2910 `_) - Improved detection of a package type from the tarball archive (`issue #3828 `_) - Ignore files according to the patterns declared in ".gitignore" when using the `pio package pack `__ command (`issue #4188 `_) - Dropped automatic updates of global libraries and development platforms (`issue #4179 `_) - Dropped support for the "pythonPackages" field in "platform.json" manifest in favor of `Extra Python Dependencies `__ - Fixed an issue when manually removed dependencies from the |PIOCONF| were not uninstalled from the storage (`issue #3076 `_) * **Unit Testing** - Refactored from scratch |UNITTESTING| solution and its documentation - New: `Test Hierarchy `_ (`issue #4135 `_) - New: `Doctest `__ testing framework (`issue #4240 `_) - New: `GoogleTest `__ testing and mocking framework (`issue #3572 `_) - New: `Semihosting `__ (`issue #3516 `_) - New: Hardware `Simulators `__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions) - New: ``test`` `build configuration `__ - Added support for a `custom testing framework `_ - Added support for a custom `testing command `__ - Added support for a `custom Unity library `__ (`issue #3980 `_) - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port `__ option (`issue #4229 `_) - List available project tests with a new `pio test --list-tests `__ option - Pass extra arguments to the testing program with a new `pio test --program-arg `__ option (`issue #3132 `_) - Generate reports in JUnit and JSON formats using the `pio test `__ command (`issue #2891 `_) - Provide more information when the native program crashed on a host (errored with a non-zero return code) (`issue #3429 `_) - Improved automatic detection of a testing serial port (`issue #4076 `_) - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 `_) - Renamed the "test_build_project_src" project configuration option to the `test_build_src `__ - Removed the "test_transport" option in favor of the `Custom "unity_config.h" `_ * **Static Code Analysis** - Updated analysis tools: * `Cppcheck `__ v2.7 with various checker improvements and fixed false positives * `PVS-Studio `__ v7.18 with improved and updated semantic analysis system - Added support for the custom `Clang-Tidy `__ configuration file (`issue #4186 `_) - Added ability to override a tool version using the `platform_packages `__ option (`issue #3798 `_) - Fixed an issue with improper handling of defects that don't specify a source file (`issue #4237 `_) * **Build System** - Show project dependency licenses when building in the verbose mode - Fixed an issue when |LDF| ignores the project `lib_deps `__ while resolving library dependencies (`issue #3598 `_) - Fixed an issue with calling an extra script located outside a project (`issue #4220 `_) - Fixed an issue when GCC preprocessor was applied to the ".s" assembly files on case-sensitive OS such as Window OS (`issue #3917 `_) - Fixed an issue when |LDF| ignores `build_src_flags `__ in the "deep+" mode (`issue #4253 `_) * **Integration** - Added a new build variable (``COMPILATIONDB_INCLUDE_TOOLCHAIN``) to include toolchain paths in the compilation database (`issue #3735 `_) - Changed a default path for compilation database `compile_commands.json `__ to the project root - Enhanced integration for Qt Creator (`issue #3046 `_) * **Project Configuration** - Extended |INTERPOLATION| with ``${this}`` pattern (`issue #3953 `_) - Embed environment name of the current section in the |PIOCONF| using ``${this.__env__}`` pattern - Renamed the "src_build_flags" project configuration option to the `build_src_flags `__ - Renamed the "src_filter" project configuration option to the `build_src_filter `__ * **Miscellaneous** - Pass extra arguments to the `native `__ program with a new `pio run --program-arg `__ option (`issue #4246 `_) - Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 `_) - Finally removed all tracks to the Python 2.7, the Python 3.6 is the minimum supported version. .. _release_notes_5: PlatformIO Core 5 ----------------- See `PlatformIO Core 5.0 history `__. .. _release_notes_4: PlatformIO Core 4 ----------------- See `PlatformIO Core 4.0 history `__. PlatformIO Core 3 ----------------- See `PlatformIO Core 3.0 history `__. PlatformIO Core 2 ----------------- See `PlatformIO Core 2.0 history `__. PlatformIO Core 1 ----------------- See `PlatformIO Core 1.0 history `__. PlatformIO Core Preview ----------------------- See `PlatformIO Core Preview history `__. ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. ================================================ FILE: Makefile ================================================ lint: pylint --rcfile=./.pylintrc ./tests pylint --rcfile=./.pylintrc ./platformio isort: isort ./platformio isort ./tests format: black ./platformio black ./tests codespell: codespell --skip "./build,./docs/_build" -L "AtLeast,TRE,ans,dout,homestate,ser" test: pytest --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py before-commit: codespell isort format lint clean-docs: rm -rf docs/_build clean: clean-docs find . -name \*.pyc -delete find . -name __pycache__ -delete rm -rf .cache rm -rf build rm -rf htmlcov rm -f .coverage profile: # Usage $ > make PIOARGS="boards" profile python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof pack: python setup.py sdist publish: python setup.py sdist upload ================================================ FILE: README.rst ================================================ PlatformIO Core =============== .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg :target: https://docs.platformio.org/en/latest/core/index.html :alt: CI Build for PlatformIO Core .. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg :target: https://docs.platformio.org?utm_source=github&utm_medium=core :alt: CI Build for Docs .. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg :target: https://github.com/platformio/platformio-examples :alt: CI Build for dev-platform examples .. image:: https://github.com/platformio/platformio-core/workflows/Projects/badge.svg :target: https://docs.platformio.org/en/latest/tutorials/index.html#projects :alt: CI Build for the Community Projects .. image:: https://img.shields.io/pypi/v/platformio.svg :target: https://pypi.python.org/pypi/platformio/ :alt: Latest Version .. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg :alt: PlatformIO Labs :target: https://piolabs.com/?utm_source=github&utm_medium=core **Quick Links:** `Homepage `_ | `PlatformIO IDE `_ | `Registry `_ | `Project Examples `__ | `Docs `_ | `Donate `_ | `Contact Us `_ **Social:** `LinkedIn `_ | `Twitter `_ | `Facebook `_ | `Community Forums `_ .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core `PlatformIO `_: Your Gateway to Embedded Software Development Excellence. Unlock the true potential of embedded software development with PlatformIO's collaborative ecosystem, embracing declarative principles, test-driven methodologies, and modern toolchains for unrivaled success. * Open source, maximum permissive Apache 2.0 license * Cross-platform IDE and Unified Debugger * Static Code Analyzer and Remote Unit Testing * Multi-platform and Multi-architecture Build System * Firmware File Explorer and Memory Inspection Get Started ----------- * `What is PlatformIO? `_ * `PlatformIO IDE `_ * `PlatformIO Core (CLI) `_ * `Project Examples `__ Solutions --------- * `Library Management `_ * `Desktop IDEs Integration `_ * `Continuous Integration `_ **Advanced** * `Debugging `_ * `Unit Testing `_ * `Static Code Analysis `_ * `Remote Development `_ Registry -------- * `Libraries `_ * `Development Platforms `_ * `Development Tools `_ Contributing ------------ See `contributing guidelines `_. Telemetry / Privacy Policy -------------------------- Share minimal diagnostics and usage information to help us make PlatformIO better. It is enabled by default. For more information see: * `Telemetry Setting `_ License ------- Copyright (c) 2014-present PlatformIO The PlatformIO is licensed under the permissive Apache 2.0 license, so you can use it in both commercial and personal projects with confidence. .. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md :alt: SWUbanner ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions We are committed to ensuring the security and protection of PlatformIO Core. To this end, we support only the following versions: | Version | Supported | | ------- | ------------------ | | 6.1.x | :white_check_mark: | | < 6.1 | :x: | Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. Therefore, it is important that all developers use only supported versions of the PlatformIO Core. ## Reporting a Vulnerability We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately. To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including: - A description of the vulnerability and how it can be exploited - Steps to reproduce the vulnerability - Any additional information that can help us understand and reproduce the vulnerability Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed. If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion. If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively. Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy. Thank you for helping us keep our systems and data secure. ================================================ FILE: platformio/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. VERSION = (6, 1, "20a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" __description__ = ( "Your Gateway to Embedded Software Development Excellence. " "Unlock the true potential of embedded software development " "with PlatformIO's collaborative ecosystem, embracing " "declarative principles, test-driven methodologies, and " "modern toolchains for unrivaled success." ) __url__ = "https://platformio.org" __author__ = "PlatformIO Labs" __email__ = "contact@piolabs.com" __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO Labs" __accounts_api__ = "https://api.accounts.platformio.org" __registry_mirror_hosts__ = [ "registry.platformio.org", "registry.nm1.platformio.org", ] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __check_internet_hosts__ = [ "185.199.110.153", # Github.com "88.198.170.159", # platformio.org "github.com", ] + __registry_mirror_hosts__ ================================================ FILE: platformio/__main__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import traceback import click from platformio import __version__, exception, maintenance from platformio.cli import PlatformioCLI from platformio.compat import IS_CYGWIN, ensure_python3 @click.command( cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"]) ) @click.version_option(__version__, prog_name="PlatformIO Core") @click.option("--force", "-f", is_flag=True, help="DEPRECATED", hidden=True) @click.option("--caller", "-c", help="Caller ID (service)") @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters") @click.pass_context def cli(ctx, force, caller, no_ansi): # pylint: disable=unused-argument try: if ( no_ansi or str( os.getenv("PLATFORMIO_NO_ANSI", os.getenv("PLATFORMIO_DISABLE_COLOR")) ).lower() == "true" ): # pylint: disable=protected-access click._compat.isatty = lambda stream: False elif ( str( os.getenv("PLATFORMIO_FORCE_ANSI", os.getenv("PLATFORMIO_FORCE_COLOR")) ).lower() == "true" ): # pylint: disable=protected-access click._compat.isatty = lambda stream: True except: # pylint: disable=bare-except pass maintenance.on_cmd_start(ctx, caller) @cli.result_callback() @click.pass_context def process_result(*_, **__): maintenance.on_cmd_end() def configure(): if IS_CYGWIN: raise exception.CygwinEnvDetected() # https://urllib3.readthedocs.org # /en/latest/security.html#insecureplatformwarning try: import urllib3 # pylint: disable=import-outside-toplevel urllib3.disable_warnings() except (AttributeError, ImportError): pass # Handle IOError issue with VSCode's Terminal (Windows) click_echo_origin = [click.echo, click.secho] def _safe_echo(origin, *args, **kwargs): try: click_echo_origin[origin](*args, **kwargs) except IOError: (sys.stderr.write if kwargs.get("err") else sys.stdout.write)( "%s\n" % (args[0] if args else "") ) click.echo = lambda *args, **kwargs: _safe_echo(0, *args, **kwargs) click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) def main(argv=None): exit_code = 0 prev_sys_argv = sys.argv[:] if argv: assert isinstance(argv, list) sys.argv = argv try: ensure_python3(raise_exception=True) configure() cli() # pylint: disable=no-value-for-parameter except SystemExit as exc: if exc.code and str(exc.code).isdigit(): exit_code = int(exc.code) except Exception as exc: # pylint: disable=broad-except if not isinstance(exc, exception.ReturnErrorCode): maintenance.on_platformio_exception(exc) error_str = f"{exc.__class__.__name__}: " if isinstance(exc, exception.PlatformioException): error_str += str(exc) else: error_str += traceback.format_exc() error_str += """ ============================================================ An unexpected error occurred. Further steps: * Verify that you have the latest version of PlatformIO using `python -m pip install -U platformio` command * Try to find answer in FAQ Troubleshooting section https://docs.platformio.org/page/faq/index.html * Report this problem to the developers https://github.com/platformio/platformio-core/issues ============================================================ """ click.secho(error_str, fg="red", err=True) exit_code = int(str(exc)) if str(exc).isdigit() else 1 maintenance.on_platformio_exit() sys.argv = prev_sys_argv return exit_code def debug_gdb_main(): return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]) if __name__ == "__main__": sys.exit(main()) ================================================ FILE: platformio/account/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/account/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.commands.destroy import account_destroy_cmd from platformio.account.commands.forgot import account_forgot_cmd from platformio.account.commands.login import account_login_cmd from platformio.account.commands.logout import account_logout_cmd from platformio.account.commands.password import account_password_cmd from platformio.account.commands.register import account_register_cmd from platformio.account.commands.show import account_show_cmd from platformio.account.commands.token import account_token_cmd from platformio.account.commands.update import account_update_cmd @click.group( "account", commands=[ account_destroy_cmd, account_forgot_cmd, account_login_cmd, account_logout_cmd, account_password_cmd, account_register_cmd, account_show_cmd, account_token_cmd, account_update_cmd, ], short_help="Manage PlatformIO account", ) def cli(): pass ================================================ FILE: platformio/account/client.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time from platformio import __accounts_api__, app from platformio.exception import PlatformioException, UserSideException from platformio.http import HTTPClient, HTTPClientError class AccountError(PlatformioException): MESSAGE = "{0}" class AccountNotAuthorized(AccountError, UserSideException): MESSAGE = "You are not authorized! Please log in to PlatformIO Account." class AccountAlreadyAuthorized(AccountError, UserSideException): MESSAGE = "You are already authorized with {0} account." class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): super().__init__(__accounts_api__) @staticmethod def get_refresh_token(): try: return app.get_state_item("account").get("auth").get("refresh_token") except Exception as exc: raise AccountNotAuthorized() from exc @staticmethod def delete_local_session(): app.delete_state_item("account") @staticmethod def delete_local_state(key): account = app.get_state_item("account") if not account or key not in account: return del account[key] app.set_state_item("account", account) def fetch_json_data(self, *args, **kwargs): try: return super().fetch_json_data(*args, **kwargs) except HTTPClientError as exc: raise AccountError(exc) from exc def fetch_authentication_token(self): if os.environ.get("PLATFORMIO_AUTH_TOKEN"): return os.environ.get("PLATFORMIO_AUTH_TOKEN") auth = app.get_state_item("account", {}).get("auth", {}) if auth.get("access_token") and auth.get("access_token_expire"): if auth.get("access_token_expire") > time.time(): return auth.get("access_token") if auth.get("refresh_token"): try: data = self.fetch_json_data( "post", "/v1/login", headers={ "Authorization": "Bearer %s" % auth.get("refresh_token") }, ) app.set_state_item("account", data) return data.get("auth").get("access_token") except AccountError: self.delete_local_session() raise AccountNotAuthorized() def login(self, username, password): try: self.fetch_authentication_token() except: # pylint:disable=bare-except pass else: raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) data = self.fetch_json_data( "post", "/v1/login", data={"username": username, "password": password}, ) app.set_state_item("account", data) return data def login_with_code(self, client_id, code, redirect_uri): try: self.fetch_authentication_token() except: # pylint:disable=bare-except pass else: raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) result = self.fetch_json_data( "post", "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, ) app.set_state_item("account", result) return result def logout(self): refresh_token = self.get_refresh_token() self.delete_local_session() try: self.fetch_json_data( "post", "/v1/logout", data={"refresh_token": refresh_token}, ) except AccountError: pass return True def change_password(self, old_password, new_password): return self.fetch_json_data( "post", "/v1/password", data={"old_password": old_password, "new_password": new_password}, x_with_authorization=True, ) def registration( self, username, email, password, firstname, lastname ): # pylint: disable=too-many-arguments,too-many-positional-arguments try: self.fetch_authentication_token() except: # pylint:disable=bare-except pass else: raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) return self.fetch_json_data( "post", "/v1/registration", data={ "username": username, "email": email, "password": password, "firstname": firstname, "lastname": lastname, }, ) def auth_token(self, password, regenerate): return self.fetch_json_data( "post", "/v1/token", data={"password": password, "regenerate": 1 if regenerate else 0}, x_with_authorization=True, ).get("auth_token") def forgot_password(self, username): return self.fetch_json_data( "post", "/v1/forgot", data={"username": username}, ) def get_profile(self): return self.fetch_json_data( "get", "/v1/profile", x_with_authorization=True, ) def update_profile(self, profile, current_password): profile["current_password"] = current_password self.delete_local_state("summary") response = self.fetch_json_data( "put", "/v1/profile", data=profile, x_with_authorization=True, ) return response def get_account_info(self, offline=False): account = app.get_state_item("account") or {} if ( account.get("summary") and account["summary"].get("expire_at", 0) > time.time() ): return account["summary"] if offline and account.get("email"): return { "profile": { "email": account.get("email"), "username": account.get("username"), } } result = self.fetch_json_data( "get", "/v1/summary", x_with_authorization=True, ) account["summary"] = dict( profile=result.get("profile"), packages=result.get("packages"), subscriptions=result.get("subscriptions"), user_id=result.get("user_id"), expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL, ) app.set_state_item("account", account) return result def get_logged_username(self): return self.get_account_info(offline=True).get("profile").get("username") def destroy_account(self): return self.fetch_json_data( "delete", "/v1/account", x_with_authorization=True, ) def create_org(self, orgname, email, displayname): return self.fetch_json_data( "post", "/v1/orgs", data={"orgname": orgname, "email": email, "displayname": displayname}, x_with_authorization=True, ) def get_org(self, orgname): return self.fetch_json_data( "get", "/v1/orgs/%s" % orgname, x_with_authorization=True, ) def list_orgs(self): return self.fetch_json_data( "get", "/v1/orgs", x_with_authorization=True, ) def update_org(self, orgname, data): return self.fetch_json_data( "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v}, x_with_authorization=True, ) def destroy_org(self, orgname): return self.fetch_json_data( "delete", "/v1/orgs/%s" % orgname, x_with_authorization=True, ) def add_org_owner(self, orgname, username): return self.fetch_json_data( "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, x_with_authorization=True, ) def list_org_owners(self, orgname): return self.fetch_json_data( "get", "/v1/orgs/%s/owners" % orgname, x_with_authorization=True, ) def remove_org_owner(self, orgname, username): return self.fetch_json_data( "delete", "/v1/orgs/%s/owners" % orgname, params={"username": username}, x_with_authorization=True, ) def create_team(self, orgname, teamname, description): return self.fetch_json_data( "post", "/v1/orgs/%s/teams" % orgname, data={"name": teamname, "description": description}, x_with_authorization=True, ) def destroy_team(self, orgname, teamname): return self.fetch_json_data( "delete", "/v1/orgs/%s/teams/%s" % (orgname, teamname), x_with_authorization=True, ) def get_team(self, orgname, teamname): return self.fetch_json_data( "get", "/v1/orgs/%s/teams/%s" % (orgname, teamname), x_with_authorization=True, ) def list_teams(self, orgname): return self.fetch_json_data( "get", "/v1/orgs/%s/teams" % orgname, x_with_authorization=True, ) def update_team(self, orgname, teamname, data): return self.fetch_json_data( "put", "/v1/orgs/%s/teams/%s" % (orgname, teamname), data={k: v for k, v in data.items() if v}, x_with_authorization=True, ) def add_team_member(self, orgname, teamname, username): return self.fetch_json_data( "post", "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), data={"username": username}, x_with_authorization=True, ) def remove_team_member(self, orgname, teamname, username): return self.fetch_json_data( "delete", "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), params={"username": username}, x_with_authorization=True, ) ================================================ FILE: platformio/account/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/account/commands/destroy.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient, AccountNotAuthorized @click.command("destroy", short_help="Destroy account") def account_destroy_cmd(): client = AccountClient() click.confirm( "Are you sure you want to delete the %s user account?\n" "Warning! All linked data will be permanently removed and can not be restored." % client.get_logged_username(), abort=True, ) client.destroy_account() try: client.logout() except AccountNotAuthorized: pass click.secho( "User account has been destroyed.", fg="green", ) ================================================ FILE: platformio/account/commands/forgot.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient @click.command("forgot", short_help="Forgot password") @click.option("--username", prompt="Username or email") def account_forgot_cmd(username): client = AccountClient() client.forgot_password(username) click.secho( "If this account is registered, we will send the " "further instructions to your email.", fg="green", ) ================================================ FILE: platformio/account/commands/login.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient @click.command("login", short_help="Log in to PlatformIO Account") @click.option("-u", "--username", prompt="Username or email") @click.option("-p", "--password", prompt=True, hide_input=True) def account_login_cmd(username, password): client = AccountClient() client.login(username, password) click.secho("Successfully logged in!", fg="green") ================================================ FILE: platformio/account/commands/logout.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient @click.command("logout", short_help="Log out of PlatformIO Account") def account_logout_cmd(): client = AccountClient() client.logout() click.secho("Successfully logged out!", fg="green") ================================================ FILE: platformio/account/commands/password.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient @click.command("password", short_help="Change password") @click.option("--old-password", prompt=True, hide_input=True) @click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True) def account_password_cmd(old_password, new_password): client = AccountClient() client.change_password(old_password, new_password) click.secho("Password successfully changed!", fg="green") ================================================ FILE: platformio/account/commands/register.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import ( validate_email, validate_password, validate_username, ) @click.command("register", short_help="Create new PlatformIO Account") @click.option( "-u", "--username", prompt=True, callback=lambda _, __, value: validate_username(value), ) @click.option( "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value) ) @click.option( "-p", "--password", prompt=True, hide_input=True, confirmation_prompt=True, callback=lambda _, __, value: validate_password(value), ) @click.option("--firstname", prompt=True) @click.option("--lastname", prompt=True) def account_register_cmd(username, email, password, firstname, lastname): client = AccountClient() client.registration(username, email, password, firstname, lastname) click.secho( "An account has been successfully created. " "Please check your mail to activate your account and verify your email address.", fg="green", ) ================================================ FILE: platformio/account/commands/show.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from tabulate import tabulate from platformio import util from platformio.account.client import AccountClient @click.command("show", short_help="PlatformIO Account information") @click.option("--offline", is_flag=True) @click.option("--json-output", is_flag=True) def account_show_cmd(offline, json_output): client = AccountClient() info = client.get_account_info(offline) if json_output: click.echo(json.dumps(info)) return click.echo() if info.get("profile"): print_profile(info["profile"]) if info.get("packages"): print_packages(info["packages"]) if info.get("subscriptions"): print_subscriptions(info["subscriptions"]) click.echo() def print_profile(profile): click.secho("Profile", fg="cyan", bold=True) click.echo("=" * len("Profile")) data = [] if profile.get("username"): data.append(("Username:", profile["username"])) if profile.get("email"): data.append(("Email:", profile["email"])) if profile.get("firstname"): data.append(("First name:", profile["firstname"])) if profile.get("lastname"): data.append(("Last name:", profile["lastname"])) click.echo(tabulate(data, tablefmt="plain")) def print_packages(packages): click.echo() click.secho("Packages", fg="cyan") click.echo("=" * len("Packages")) for package in packages: click.echo() click.secho(package.get("name"), bold=True) click.echo("-" * len(package.get("name"))) if package.get("description"): click.echo(package.get("description")) data = [] expire = "-" if "subscription" in package: expire = util.parse_datetime( package["subscription"].get("end_at") or package["subscription"].get("next_bill_at") ).strftime("%Y-%m-%d") data.append(("Expire:", expire)) services = [] for key in package: if not key.startswith("service."): continue if isinstance(package[key], dict): services.append(package[key].get("title")) else: services.append(package[key]) if services: data.append(("Services:", ", ".join(services))) click.echo(tabulate(data, tablefmt="plain")) def print_subscriptions(subscriptions): click.echo() click.secho("Subscriptions", fg="cyan") click.echo("=" * len("Subscriptions")) for subscription in subscriptions: click.echo() click.secho(subscription.get("product_name"), bold=True) click.echo("-" * len(subscription.get("product_name"))) data = [("State:", subscription.get("status"))] begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c") data.append(("Start date:", begin_at or "-")) end_at = subscription.get("end_at") if end_at: end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c") data.append(("End date:", end_at or "-")) next_bill_at = subscription.get("next_bill_at") if next_bill_at: next_bill_at = util.parse_datetime( subscription.get("next_bill_at") ).strftime("%c") data.append(("Next payment:", next_bill_at or "-")) data.append( ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-") ) data.append( ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-") ) click.echo(tabulate(data, tablefmt="plain")) ================================================ FILE: platformio/account/commands/token.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from platformio.account.client import AccountClient @click.command("token", short_help="Get or regenerate Authentication Token") @click.option("-p", "--password", prompt=True, hide_input=True) @click.option("--regenerate", is_flag=True) @click.option("--json-output", is_flag=True) def account_token_cmd(password, regenerate, json_output): client = AccountClient() auth_token = client.auth_token(password, regenerate) if json_output: click.echo(json.dumps({"status": "success", "result": auth_token})) return click.secho("Personal Authentication Token: %s" % auth_token, fg="green") ================================================ FILE: platformio/account/commands/update.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient, AccountNotAuthorized from platformio.account.validate import validate_email, validate_username @click.command("update", short_help="Update profile information") @click.option("--current-password", prompt=True, hide_input=True) @click.option("--username") @click.option("--email") @click.option("--firstname") @click.option("--lastname") def account_update_cmd(current_password, **kwargs): client = AccountClient() profile = client.get_profile() new_profile = profile.copy() if not any(kwargs.values()): for field in profile: new_profile[field] = click.prompt( field.replace("_", " ").capitalize(), default=profile[field] ) if field == "email": validate_email(new_profile[field]) if field == "username": validate_username(new_profile[field]) else: new_profile.update({key: value for key, value in kwargs.items() if value}) client.update_profile(new_profile, current_password) click.secho("Profile successfully updated!", fg="green") username_changed = new_profile["username"] != profile["username"] email_changed = new_profile["email"] != profile["email"] if not username_changed and not email_changed: return None try: client.logout() except AccountNotAuthorized: pass if email_changed: click.secho( "Please check your mail to verify your new email address and re-login. ", fg="yellow", ) return None click.secho("Please re-login.", fg="yellow") return None ================================================ FILE: platformio/account/org/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/account/org/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.org.commands.add import org_add_cmd from platformio.account.org.commands.create import org_create_cmd from platformio.account.org.commands.destroy import org_destroy_cmd from platformio.account.org.commands.list import org_list_cmd from platformio.account.org.commands.remove import org_remove_cmd from platformio.account.org.commands.update import org_update_cmd @click.group( "account", commands=[ org_add_cmd, org_create_cmd, org_destroy_cmd, org_list_cmd, org_remove_cmd, org_update_cmd, ], short_help="Manage organizations", ) def cli(): pass ================================================ FILE: platformio/account/org/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/account/org/commands/add.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient @click.command("add", short_help="Add a new owner to organization") @click.argument( "orgname", ) @click.argument( "username", ) def org_add_cmd(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) return click.secho( "The new owner `%s` has been successfully added to the `%s` organization." % (username, orgname), fg="green", ) ================================================ FILE: platformio/account/org/commands/create.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import validate_email, validate_orgname @click.command("create", short_help="Create a new organization") @click.argument( "orgname", callback=lambda _, __, value: validate_orgname(value), ) @click.option( "--email", callback=lambda _, __, value: validate_email(value) if value else value ) @click.option( "--displayname", ) def org_create_cmd(orgname, email, displayname): client = AccountClient() client.create_org(orgname, email, displayname) return click.secho( "The organization `%s` has been successfully created." % orgname, fg="green", ) ================================================ FILE: platformio/account/org/commands/destroy.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient @click.command("destroy", short_help="Destroy organization") @click.argument("orgname") def org_destroy_cmd(orgname): client = AccountClient() click.confirm( "Are you sure you want to delete the `%s` organization account?\n" "Warning! All linked data will be permanently removed and can not be restored." % orgname, abort=True, ) client.destroy_org(orgname) return click.secho( "Organization `%s` has been destroyed." % orgname, fg="green", ) ================================================ FILE: platformio/account/org/commands/list.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from tabulate import tabulate from platformio.account.client import AccountClient @click.command("list", short_help="List organizations and their members") @click.option("--json-output", is_flag=True) def org_list_cmd(json_output): client = AccountClient() orgs = client.list_orgs() if json_output: return click.echo(json.dumps(orgs)) if not orgs: return click.echo("You do not have any organization") for org in orgs: click.echo() click.secho(org.get("orgname"), fg="cyan") click.echo("-" * len(org.get("orgname"))) data = [] if org.get("displayname"): data.append(("Display Name:", org.get("displayname"))) if org.get("email"): data.append(("Email:", org.get("email"))) data.append( ( "Owners:", ", ".join((owner.get("username") for owner in org.get("owners"))), ) ) click.echo(tabulate(data, tablefmt="plain")) return click.echo() ================================================ FILE: platformio/account/org/commands/remove.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient @click.command("remove", short_help="Remove an owner from organization") @click.argument( "orgname", ) @click.argument( "username", ) def org_remove_cmd(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) return click.secho( "The `%s` owner has been successfully removed from the `%s` organization." % (username, orgname), fg="green", ) ================================================ FILE: platformio/account/org/commands/update.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import validate_email, validate_orgname @click.command("update", short_help="Update organization") @click.argument("cur_orgname") @click.option( "--orgname", callback=lambda _, __, value: validate_orgname(value) if value else value, help="A new orgname", ) @click.option( "--email", callback=lambda _, __, value: validate_email(value) if value else value, ) @click.option("--displayname") def org_update_cmd(cur_orgname, **kwargs): client = AccountClient() org = client.get_org(cur_orgname) new_org = { key: value if value is not None else org[key] for key, value in kwargs.items() } if not any(kwargs.values()): for key in kwargs: new_org[key] = click.prompt(key.capitalize(), default=org[key]) if key == "email": validate_email(new_org[key]) if key == "orgname": validate_orgname(new_org[key]) client.update_org(cur_orgname, new_org) return click.secho( "The organization `%s` has been successfully updated." % cur_orgname, fg="green", ) ================================================ FILE: platformio/account/team/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/account/team/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.team.commands.add import team_add_cmd from platformio.account.team.commands.create import team_create_cmd from platformio.account.team.commands.destroy import team_destroy_cmd from platformio.account.team.commands.list import team_list_cmd from platformio.account.team.commands.remove import team_remove_cmd from platformio.account.team.commands.update import team_update_cmd @click.group( "team", commands=[ team_add_cmd, team_create_cmd, team_destroy_cmd, team_list_cmd, team_remove_cmd, team_update_cmd, ], short_help="Manage organization teams", ) def cli(): pass ================================================ FILE: platformio/account/team/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/account/team/commands/add.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import validate_orgname_teamname @click.command("add", short_help="Add a new member to team") @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.argument( "username", ) def team_add_cmd(orgname_teamname, username): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() client.add_team_member(orgname, teamname, username) return click.secho( "The new member %s has been successfully added to the %s team." % (username, teamname), fg="green", ) ================================================ FILE: platformio/account/team/commands/create.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import validate_orgname_teamname @click.command("create", short_help="Create a new team") @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.option( "--description", ) def team_create_cmd(orgname_teamname, description): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() client.create_team(orgname, teamname, description) return click.secho( "The team %s has been successfully created." % teamname, fg="green", ) ================================================ FILE: platformio/account/team/commands/destroy.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import validate_orgname_teamname @click.command("destroy", short_help="Destroy a team") @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) def team_destroy_cmd(orgname_teamname): orgname, teamname = orgname_teamname.split(":", 1) click.confirm( click.style( "Are you sure you want to destroy the %s team?" % teamname, fg="yellow" ), abort=True, ) client = AccountClient() client.destroy_team(orgname, teamname) return click.secho( "The team %s has been successfully destroyed." % teamname, fg="green", ) ================================================ FILE: platformio/account/team/commands/list.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from tabulate import tabulate from platformio.account.client import AccountClient @click.command("list", short_help="List teams") @click.argument("orgname", required=False) @click.option("--json-output", is_flag=True) def team_list_cmd(orgname, json_output): client = AccountClient() data = {} if not orgname: for item in client.list_orgs(): teams = client.list_teams(item.get("orgname")) data[item.get("orgname")] = teams else: teams = client.list_teams(orgname) data[orgname] = teams if json_output: return click.echo(json.dumps(data[orgname] if orgname else data)) if not any(data.values()): return click.secho("You do not have any teams.", fg="yellow") for org_name, teams in data.items(): for team in teams: click.echo() click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan") click.echo("-" * len("%s:%s" % (org_name, team.get("name")))) table_data = [] if team.get("description"): table_data.append(("Description:", team.get("description"))) table_data.append( ( "Members:", ( ", ".join( (member.get("username") for member in team.get("members")) ) if team.get("members") else "-" ), ) ) click.echo(tabulate(table_data, tablefmt="plain")) return click.echo() ================================================ FILE: platformio/account/team/commands/remove.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import validate_orgname_teamname @click.command("remove", short_help="Remove a member from team") @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.argument("username") def team_remove_cmd(orgname_teamname, username): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() client.remove_team_member(orgname, teamname, username) return click.secho( "The %s member has been successfully removed from the %s team." % (username, teamname), fg="green", ) ================================================ FILE: platformio/account/team/commands/update.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.account.validate import validate_orgname_teamname, validate_teamname @click.command("update", short_help="Update team") @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.option( "--name", callback=lambda _, __, value: validate_teamname(value) if value else value, help="A new team name", ) @click.option( "--description", ) def team_update_cmd(orgname_teamname, **kwargs): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() team = client.get_team(orgname, teamname) new_team = { key: value if value is not None else team[key] for key, value in kwargs.items() } if not any(kwargs.values()): for key in kwargs: new_team[key] = click.prompt(key.capitalize(), default=team[key]) if key == "name": validate_teamname(new_team[key]) client.update_team(orgname, teamname, new_team) return click.secho( "The team %s has been successfully updated." % teamname, fg="green", ) ================================================ FILE: platformio/account/validate.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re import click def validate_username(value, field="username"): value = str(value).strip() if value else None if not value or not re.match( r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I ): raise click.BadParameter( "Invalid %s format. " "%s must contain only alphanumeric characters " "or single hyphens, cannot begin or end with a hyphen, " "and must not be longer than 38 characters." % (field.lower(), field.capitalize()) ) return value def validate_orgname(value): return validate_username(value, "Organization name") def validate_email(value): value = str(value).strip() if value else None if not value or not re.match( r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I ): raise click.BadParameter("Invalid email address") return value def validate_password(value): value = str(value).strip() if value else None if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): raise click.BadParameter( "Invalid password format. " "Password must contain at least 8 characters" " including a number and a lowercase letter" ) return value def validate_teamname(value): value = str(value).strip() if value else None if not value or not re.match( r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I ): raise click.BadParameter( "Invalid team name format. " "Team name must only contain alphanumeric characters, " "single hyphens, underscores, spaces. It can not " "begin or end with a hyphen or a underscore and must" " not be longer than 20 characters." ) return value def validate_orgname_teamname(value): value = str(value).strip() if value else None if not value or ":" not in value: raise click.BadParameter( "Please specify organization and team name using the following" " format - orgname:teamname. For example, mycompany:DreamTeam" ) orgname, teamname = value.split(":", 1) validate_orgname(orgname) validate_teamname(teamname) return value ================================================ FILE: platformio/app.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import getpass import hashlib import json import os import platform import socket import time import uuid from platformio import __version__, exception, fs, proc from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.config import ProjectConfig from platformio.project.helpers import get_default_projects_dir def projects_dir_validate(projects_dir): assert os.path.isdir(projects_dir) return os.path.abspath(projects_dir) DEFAULT_SETTINGS = { "check_platformio_interval": { "description": "Check for the new PlatformIO Core interval (days)", "value": 7, }, "check_prune_system_threshold": { "description": "Check for pruning unnecessary data threshold (megabytes)", "value": 1024, }, "enable_cache": { "description": "Enable caching for HTTP API requests", "value": True, }, "enable_telemetry": { "description": ("Telemetry service (Yes/No)"), "value": True, }, "force_verbose": { "description": "Force verbose output when processing environments", "value": False, }, "projects_dir": { "description": "Default location for PlatformIO projects (PlatformIO Home)", "value": get_default_projects_dir(), "validator": projects_dir_validate, }, "enable_proxy_strict_ssl": { "description": "Verify the proxy server certificate against the list of supplied CAs", "value": True, }, } SESSION_VARS = { "command_ctx": None, "caller_id": None, "custom_project_conf": None, "pause_telemetry": False, } def resolve_state_path(conf_option_dir, file_name, ensure_dir_exists=True): state_dir = ProjectConfig.get_instance().get("platformio", conf_option_dir) if ensure_dir_exists and not os.path.isdir(state_dir): os.makedirs(state_dir) return os.path.join(state_dir, file_name) class State: def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: self.path = resolve_state_path("core_dir", "appstate.json") self._storage = {} self._lockfile = None self.modified = False def __enter__(self): try: self._lock_state_file() if os.path.isfile(self.path): self._storage = fs.load_json(self.path) assert isinstance(self._storage, dict) except ( AssertionError, ValueError, UnicodeDecodeError, exception.InvalidJSONFile, ): self._storage = {} return self def __exit__(self, type_, value, traceback): if self.modified: try: with open(self.path, mode="w", encoding="utf8") as fp: fp.write(json.dumps(self._storage)) except IOError as exc: raise exception.HomeDirPermissionsError( os.path.dirname(self.path) ) from exc self._unlock_state_file() def _lock_state_file(self): if not self.lock: return self._lockfile = LockFile(self.path) try: self._lockfile.acquire() except IOError as exc: raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) from exc def _unlock_state_file(self): if hasattr(self, "_lockfile") and self._lockfile: self._lockfile.release() def __del__(self): self._unlock_state_file() # Dictionary Proxy def as_dict(self): return self._storage def keys(self): return self._storage.keys() def get(self, key, default=True): return self._storage.get(key, default) def update(self, *args, **kwargs): self.modified = True return self._storage.update(*args, **kwargs) def clear(self): return self._storage.clear() def __getitem__(self, key): return self._storage[key] def __setitem__(self, key, value): self.modified = True self._storage[key] = value def __delitem__(self, key): self.modified = True del self._storage[key] def __contains__(self, item): return item in self._storage def sanitize_setting(name, value): if name not in DEFAULT_SETTINGS: raise exception.InvalidSettingName(name) defdata = DEFAULT_SETTINGS[name] try: if "validator" in defdata: value = defdata["validator"](value) elif isinstance(defdata["value"], bool): if not isinstance(value, bool): value = str(value).lower() in ("true", "yes", "y", "1") elif isinstance(defdata["value"], int): value = int(value) except Exception as exc: raise exception.InvalidSettingValue(value, name) from exc return value def get_state_item(name, default=None): with State() as state: return state.get(name, default) def set_state_item(name, value): with State(lock=True) as state: state[name] = value state.modified = True def delete_state_item(name): with State(lock=True) as state: if name in state: del state[name] def get_setting(name): _env_name = "PLATFORMIO_SETTING_%s" % name.upper() if _env_name in os.environ: return sanitize_setting(name, os.getenv(_env_name)) with State() as state: if "settings" in state and name in state["settings"]: return state["settings"][name] return DEFAULT_SETTINGS[name]["value"] def set_setting(name, value): with State(lock=True) as state: if "settings" not in state: state["settings"] = {} state["settings"][name] = sanitize_setting(name, value) state.modified = True def reset_settings(): with State(lock=True) as state: if "settings" in state: del state["settings"] def get_session_var(name, default=None): return SESSION_VARS.get(name, default) def set_session_var(name, value): assert name in SESSION_VARS SESSION_VARS[name] = value def is_disabled_progressbar(): return os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true" def get_cid(): cid = get_state_item("cid") if cid: return cid uid = None if os.getenv("GITHUB_USER"): uid = os.getenv("GITHUB_USER") elif os.getenv("GITPOD_GIT_USER_NAME"): uid = os.getenv("GITPOD_GIT_USER_NAME") if not uid: uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) if IS_WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) set_state_item("created_at", int(time.time())) return cid def get_project_id(project_dir): return hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest() def get_user_agent(): data = [ "PlatformIO/%s" % __version__, "CI/%d" % int(proc.is_ci()), "Container/%d" % int(proc.is_container()), ] if get_session_var("caller_id"): data.append("Caller/%s" % get_session_var("caller_id")) if os.getenv("PLATFORMIO_IDE"): data.append("IDE/%s" % os.getenv("PLATFORMIO_IDE")) data.append("Python/%s" % platform.python_version()) data.append("Platform/%s" % platform.platform()) if not get_setting("enable_telemetry"): data.append("Telemetry/0") return " ".join(data) def get_host_id(): h = hashlib.sha1(hashlib_encode_data(get_cid())) try: username = getpass.getuser() h.update(hashlib_encode_data(username)) except: # pylint: disable=bare-except pass return h.hexdigest() def get_host_name(): return str(socket.gethostname())[:255] ================================================ FILE: platformio/assets/schema/library.json ================================================ { "$id": "https://example.com/library.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "library.json schema", "type": "object", "properties": { "name": { "type": "string", "maxLength": 50, "description": "A name of a library.\nMust be unique in the PlatformIO Registry\nShould be slug style for simplicity, consistency, and compatibility. Example: HelloWorld\nCan contain a-z, digits, and dashes (but not start/end with them)\nConsecutive dashes and [:;/,@<>] chars are not allowed.", "required": true }, "version": { "type": "string", "maxLength": 20, "description": "A version of a current library source code. Can contain a-z, digits, dots or dash and should be Semantic Versioning compatible.", "required": true }, "description": { "type": "string", "maxLength": 255, "description": "The field helps users to identify and search for your library with a brief description. Describe the hardware devices (sensors, boards and etc.) which are suitable with it.", "required": true }, "keywords": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "array", "items": { "type": "string", "maxLength": 255 } } ], "description": "Used for search by keyword. Helps to make your library easier to discover without people needing to know its name.\nThe keyword should be lowercased, can contain a-z, digits and dash (but not start/end with them). A list from the keywords can be specified with separator , or declared as Array.", "required": true }, "homepage": { "type": "string", "maxLength": 255, "description": "Home page of a library (if is different from repository url).", "required": false }, "repository": { "type": "object", "properties": { "type": { "enum": [ "git", "hg", "svn" ], "description": "only “git”, “hg” or “svn” are supported" }, "url": { "type": "string" }, "branch": { "type": "string", "description": "if is not specified, default branch will be used. This field will be ignored if tag/release exists with the value of version." } }, "description": "The repository in which the source code can be found.", "required": false }, "authors": { "anyOf": [ { "type": "object", "properties": { "name": { "type": "string", "required": true, "description": "Full name" }, "email": { "type": "string" }, "url": { "type": "string", "description": "An author’s contact page" }, "maintainer": { "type": "boolean", "description": "Specify “maintainer” status" } } }, { "type": "array", "items": { "type": "object", "properties": { "name": { "type": "string", "required": true, "description": "Full name" }, "email": { "type": "string" }, "url": { "type": "string", "description": "An author’s contact page" }, "maintainer": { "type": "boolean", "description": "Specify “maintainer” status" } } } } ], "description": "An author contact information\nIf authors field is not defined, PlatformIO will try to fetch data from VCS provider (Github, Gitlab, etc) if repository is declared.", "required": false }, "license": { "type": "string", "description": "A SPDX license ID or SPDX Expression. You can check the full list of SPDX license IDs (see “Identifier” column).", "required": false }, "frameworks": { "anyOf": [ { "type": "string", "description": "espidf, freertos, *, etc'" }, { "type": "array", "items": { "type": "string", "description": "espidf, freertos, *, etc'" } } ], "description": "A list with compatible frameworks. The available framework names are defined in the Frameworks section.\nIf the library is compatible with the all frameworks, then do not declare this field or you use *", "required": false }, "platforms": { "anyOf": [ { "type": "string", "description": "atmelavr, espressif8266, *, etc'" }, { "type": "array", "items": { "type": "string", "description": "atmelavr, espressif8266, *, etc'" } } ], "description": "A list with compatible development platforms. The available platform name are defined in Development Platforms section.\nIf the library is compatible with the all platforms, then do not declare this field or use *.\nPlatformIO does not check platforms for compatibility in default mode. See Compatibility Mode for details. If you need a strict checking for compatible platforms for a library, please set libCompatMode to strict.", "required": false }, "headers": { "anyOf": [ { "type": "string", "description": "MyLibrary.h" }, { "type": "array", "items": { "type": "string", "description": "FooCore.h, FooFeature.h" } } ], "description": "A list of header files that can be included in a project source files using #include <...> directive.", "required": false }, "examples": { "type": "array", "items": { "type": "object", "properties": { "name": { "type": "string" }, "base": { "type": "string" }, "files": { "type": "array", "items": { "type": "string" } } } }, "description": "A list of example patterns.", "required": "false" }, "dependencies": { "anyOf": [ { "type": "object", "properties": { "owner": { "type": "string", "description": "an owner name (username) from the PlatformIO Registry" }, "name": { "type": "string", "description": "library name" }, "version": { "type": "string", "description": "Version Requirements or Package Specifications" }, "frameworks": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "project compatible Frameworks" }, "platforms": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": " project compatible Development Platforms" } } }, { "type": "array", "items": { "type": "object", "properties": { "owner": { "type": "string", "description": "an owner name (username) from the PlatformIO Registry" }, "name": { "type": "string", "description": "library name" }, "version": { "type": "string", "description": "Version Requirements or Package Specifications" }, "frameworks": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "project compatible Frameworks" }, "platforms": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": " project compatible Development Platforms" } } } } ], "description": "A list of dependent libraries that will be automatically installed.", "required": false }, "export": { "type": "object", "properties": { "include": { "type": "array", "items": { "type": "string" }, "description": "Export only files that matched declared patterns.\n* - matches everything\n? - matches any single character\n[seq] - matches any character in seq\n[!seq] - matches any character not in seq" }, "exclude": { "type": "array", "items": { "type": "string" }, "description": "Exclude the directories and files which match with exclude patterns." } }, "description": "This option is useful if you need to exclude extra data (test code, docs, images, PDFs, etc). It allows one to reduce the size of the final archive.\nTo check which files will be included in the final packages, please use pio pkg pack command.", "required": false }, "scripts": { "type": "object", "properties": { "postinstall": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "runs a script AFTER the package has been installed.\nRun a custom Python script located in the package “scripts” folder AFTER the package is installed. Please note that you don’t need to specify a Python interpreter for Python scripts" }, "preuninstall": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "runs a script BEFORE the package is removed.\nRun a custom Bash script BEFORE the package is uninstalled. The script is declared as a list of command arguments and is located at the root of a package" } }, "description": "Execute custom scripts during the special Package Management CLI life cycle events", "required": false }, "build": { "type": "object", "properties": { "flags": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "Extra flags to control preprocessing, compilation, assembly, and linking processes. More details build_flags.\nKeep in mind when operating with the -I flag (directories to be searched for header files). The path should be relative to the root directory where the library.json manifest is located." }, "unflags": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "Remove base/initial flags which were set by development platform. More details build_unflags." }, "includeDir": { "type": "string", "description": "Custom directory to be searched for header files. A default value is include and means that folder is located at the root of a library.\nThe Library Dependency Finder (LDF) will pick a library automatically only when a project or other dependent libraries include any header file located in includeDir or srcDir.", "required": false }, "srcDir": { "type": "string", "description": "Custom location of library source code. A default value is src and means that folder is located in the root of a library.", "required": "false" }, "srcFilter": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "Specify which source files should be included/excluded from build process. The path in filter should be relative to the srcDir option of a library.\nSee syntax for build_src_filter.\nPlease note that you can generate source filter “on-the-fly” using extraScript", "required": false }, "extraScript": { "type": "string", "description": "Launch extra script before a build process.", "required": "false" }, "libArchive": { "type": "boolean", "description": "Create an archive (*.a, static library) from the object files and link it into a firmware (program). This is default behavior of PlatformIO Build System (\"libArchive\": true).\nSetting \"libArchive\": false will instruct PlatformIO Build System to link object files directly (in-line). This could be useful if you need to override weak symbols defined in framework or other libraries.\nYou can disable library archiving globally using lib_archive option in “platformio.ini” (Project Configuration File).", "required": "false" }, "libLDFMode": { "anyOf": [ { "enum": [ "off" ], "description": "“Manual mode”, does not process source files of a project and dependencies. Builds only the libraries that are specified in manifests (library.json, module.json) or using lib_deps option." }, { "enum": [ "chain" ], "description": "[DEFAULT] Parses ALL C/C++ source files of the project and follows only by nested includes (#include ..., chain...) from the libraries. It also parses C, CC, CPP files from libraries which have the same name as included header file. Does not evaluate C/C++ Preprocessor conditional syntax." }, { "enum": [ "deep" ], "description": "Parses ALL C/C++ source files of the project and parses ALL C/C++ source files of the each found dependency (recursively). Does not evaluate C/C++ Preprocessor conditional syntax." }, { "enum": [ "chain+" ], "description": "The same behavior as for the chain but evaluates C/C++ Preprocessor conditional syntax." }, { "enum": [ "deep+" ], "description": "The same behavior as for the deep but evaluates C/C++ Preprocessor conditional syntax." } ], "description": "Specify Library Dependency Finder Mode. See Dependency Finder Mode for details.", "required": false }, "libCompatMode": { "type": "string", "description": "Specify Library Compatibility Mode. See Compatibility Mode for details.", "required": false }, "builder": { "anyOf": [ { "enum": [ "PlatformIOLibBuilder" ], "description": "Default Builder" }, { "enum": [ "ArduinoLibBuilder" ] }, { "enum": [ "MbedLibBuilder" ] } ], "description": "Override default PlatformIOLibBuilder with another builder.", "required": false } }, "required": false } } } ================================================ FILE: platformio/assets/system/99-platformio-udev.rules ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ##################################################################################### # # INSTALLATION # # Please visit > https://docs.platformio.org/en/latest/core/installation/udev-rules.html # ##################################################################################### # # Boards # # CP210X USB UART ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea[67][013]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="80a9", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FT231XS USB UART ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FX2348N USB UART ATTRS{idVendor}=="0843", ATTRS{idProduct}=="5740", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Prolific Technology, Inc. PL2303 Serial Port ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # QinHeng Electronics HL-340 USB-Serial adapter ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # QinHeng Electronics CH343 USB-Serial adapter ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d3", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # QinHeng Electronics CH9102 USB-Serial adapter ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d4", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Arduino boards ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][023]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Arduino SAM-BA ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{MTP_NO_PROBE}="1" # Digistump boards ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Maple with DFU ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # USBtiny ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # USBasp V2.0 ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Teensy boards ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1" SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666" KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666" # TI Stellaris Launchpad ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI MSP430 Launchpad ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # GD32V DFU Bootloader ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FireBeetle-ESP32 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7522", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Wio Terminal ATTRS{idVendor}=="2886", ATTRS{idProduct}=="[08]02d", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Raspberry Pi Pico ATTRS{idVendor}=="2e8a", ATTRS{idProduct}=="[01]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # AIR32F103 ATTRS{idVendor}=="0d28", ATTRS{idProduct}=="0204", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # STM32 virtual COM port ATTRS{idVendor}=="0483", ATTRS{idProduct}=="5740", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # # Debuggers # # Black Magic Probe SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # opendous and estick ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Original FT232/FT245/FT2232/FT232H/FT4232 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="60[01][104]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # DISTORTEC JTAG-lock-pick Tiny 2 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TUMPA, TUMPA Lite ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a9[89]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # XDS100v2 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE) ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca[01]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI/Luminary Stellaris Evaluation Board FTDI (several) ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd[9a]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # egnite Turtelizer 2 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Section5 ICEbear ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c14[01]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Amontec JTAGkey and JTAGkey-tiny ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI ICDI ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # STLink probes ATTRS{idVendor}=="0483", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Hilscher NXHX Boards ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Hitex probes ATTRS{idVendor}=="0640", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Altera USB Blaster ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Amontec JTAGkey-HiSpeed ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # SEGGER J-Link ATTRS{idVendor}=="1366", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Raisonance RLink ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Debug Board for Neo1973 ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Olimex probes ATTRS{idVendor}=="15ba", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # USBprog with OpenOCD firmware ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Marvell Sheevaplug ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Keil Software, Inc. ULink ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # CMSIS-DAP compatible adapters ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Atmel AVR Dragon ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2107", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Espressif USB JTAG/serial debug unit ATTRS{idVendor}=="303a", ATTRS{idProduct}=="1001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Zephyr framework USB CDC-ACM ATTRS{idVendor}=="2fe3", ATTRS{idProduct}=="0100", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" ================================================ FILE: platformio/builder/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/builder/main.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import sys from time import time import click from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import DEFAULT_TARGETS # pylint: disable=import-error from SCons.Script import AllowSubstExceptions # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import Default # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import app, fs from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir AllowSubstExceptions(NameError) # append CLI arguments to build environment clivars = Variables(None) clivars.AddVariables( ("BUILD_SCRIPT",), ("PROJECT_CONFIG",), ("PIOENV",), ("PIOTEST_RUNNING_NAME",), ("UPLOAD_PORT",), ("PROGRAM_ARGS",), ) DEFAULT_ENV_OPTIONS = dict( tools=[ "ar", "cc", "c++", "link", "piohooks", "pioasm", "piobuild", "pioproject", "pioplatform", "piotest", "piotarget", "piolib", "pioupload", "piosize", "pioino", "piomisc", "piointegration", "piomaxlen", ], toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")], variables=clivars, # Propagating External Environment ENV=os.environ, UNIX_TIME=int(time()), BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"), BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"), LIBPATH=["$BUILD_DIR"], PROGNAME="program", PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PROG_PATH="$PROGPATH", # deprecated PYTHONEXE=get_pythonexe_path(), ) # Declare command verbose messages command_strings = dict( ARCOM="Archiving", LINKCOM="Linking", RANLIBCOM="Indexing", ASCOM="Compiling", ASPPCOM="Compiling", CCCOM="Compiling", CXXCOM="Compiling", ) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): for name, value in command_strings.items(): DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) env.SConscriptChdir(False) # Load variables from CLI env.Replace( **{ key: PlatformBase.decode_scons_arg(env[key]) for key in list(clivars.keys()) if key in env } ) # Setup project optional directories config = env.GetProjectConfig() app.set_session_var("custom_project_conf", config.path) env.Replace( PROJECT_DIR=get_project_dir(), PROJECT_CORE_DIR=config.get("platformio", "core_dir"), PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"), PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"), PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"), PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"), PROJECT_SRC_DIR=config.get("platformio", "src_dir"), PROJECTSRC_DIR="$PROJECT_SRC_DIR", # legacy for dev/platform PROJECT_TEST_DIR=config.get("platformio", "test_dir"), PROJECT_DATA_DIR=config.get("platformio", "data_dir"), PROJECTDATA_DIR="$PROJECT_DATA_DIR", # legacy for dev/platform PROJECT_BUILD_DIR=config.get("platformio", "build_dir"), BUILD_TYPE=env.GetBuildType(), BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"), LIBSOURCE_DIRS=[ config.get("platformio", "lib_dir"), os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), config.get("platformio", "globallib_dir"), ], ) if int(ARGUMENTS.get("ISATTY", 0)): # pylint: disable=protected-access click._compat.isatty = lambda stream: True if env.subst("$BUILD_CACHE_DIR"): if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")): os.makedirs(env.subst("$BUILD_CACHE_DIR")) env.CacheDir("$BUILD_CACHE_DIR") if not int(ARGUMENTS.get("PIOVERBOSE", 0)): click.echo("Verbose mode can be enabled via `-v, --verbose` option") if not os.path.isdir(env.subst("$BUILD_DIR")): os.makedirs(env.subst("$BUILD_DIR")) # Dynamically load dependent tools if "compiledb" in COMMAND_LINE_TARGETS: env.Tool("compilation_db") env.LoadProjectOptions() env.LoadPioPlatform() env.SConsignFile( os.path.join( "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]), ) ) env.SConscript(env.GetExtraScripts("pre"), exports="env") if env.IsCleanTarget(): env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0))) env.Exit(0) env.SConscript("$BUILD_SCRIPT") if "UPLOAD_FLAGS" in env: env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"]) if env.GetProjectOption("upload_command"): env.Replace(UPLOADCMD=env.GetProjectOption("upload_command")) env.SConscript(env.GetExtraScripts("post"), exports="env") ############################################################################## # Checking program size if env.get("SIZETOOL") and not ( set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS) ): env.Depends("upload", "checkprogsize") # Replace platform's "size" target with our _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"] Default(None) Default(_new_targets) Default("checkprogsize") if "compiledb" in COMMAND_LINE_TARGETS: env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH")) # Print configured protocols env.AddPreAction( "upload", env.VerboseAction( lambda source, target, env: env.PrintUploadInfo(), "Configuring upload protocol...", ), ) AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) env.ProcessDelayedActions() ############################################################################## if "envdump" in COMMAND_LINE_TARGETS: click.echo(env.Dump()) env.Exit(0) if env.IsIntegrationDump(): projenv = None try: Import("projenv") except: # pylint: disable=bare-except projenv = env data = projenv.DumpIntegrationData(env) # dump to file for the further reading by project.helpers.load_build_metadata with open( projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")), mode="w", encoding="utf8", ) as fp: json.dump(data, fp) click.echo("\n%s\n" % json.dumps(data)) # pylint: disable=undefined-variable env.Exit(0) if "sizedata" in COMMAND_LINE_TARGETS: AlwaysBuild( env.Alias( "sizedata", DEFAULT_TARGETS, env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."), ) ) Default("sizedata") # issue #4604: process targets sequentially for index, target in enumerate( [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:] ): env.Depends(target, COMMAND_LINE_TARGETS[index]) ================================================ FILE: platformio/builder/tools/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/builder/tools/pioasm.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import SCons.Tool.asm # pylint: disable=import-error # # Resolve https://github.com/platformio/platformio-core/issues/3917 # Avoid forcing .S to bare assembly on Windows OS # if ".S" in SCons.Tool.asm.ASSuffixes: SCons.Tool.asm.ASSuffixes.remove(".S") if ".S" not in SCons.Tool.asm.ASPPSuffixes: SCons.Tool.asm.ASPPSuffixes.append(".S") generate = SCons.Tool.asm.generate exists = SCons.Tool.asm.exists ================================================ FILE: platformio/builder/tools/piobuild.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import fnmatch import os import sys from SCons import Builder, Util # pylint: disable=import-error from SCons.Node import FS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error from platformio import __version__, fs from platformio.compat import IS_MACOS, string_types from platformio.package.version import pepver_to_semver from platformio.proc import where_is_program SRC_HEADER_EXT = ["h", "hpp"] SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] SRC_C_EXT = ["c"] SRC_CXX_EXT = ["cc", "cpp", "cxx", "c++"] SRC_BUILD_EXT = SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep] def scons_patched_match_splitext(path, suffixes=None): """Patch SCons Builder, append $OBJSUFFIX to the end of each target""" tokens = Util.splitext(path) if suffixes and tokens[1] and tokens[1] in suffixes: return (path, tokens[1]) return tokens def GetBuildType(env): modes = [] if ( set(["__debug", "sizedata"]) # sizedata = for memory inspection & set(COMMAND_LINE_TARGETS) or env.GetProjectOption("build_type") == "debug" ): modes.append("debug") if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test": modes.append("test") return ", ".join(modes or ["release"]) def BuildProgram(env): env.ProcessProgramDeps() env.ProcessCompileDbToolchainOption() env.ProcessProjectDeps() # append into the beginning a main LD script if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) # enable "cyclic reference" for linker if ( env.get("LIBS") and env.GetCompilerType() == "gcc" and (env.PioPlatform().is_embedded() or not IS_MACOS) ): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"]) env.Replace(PIOMAINPROG=program) AlwaysBuild( env.Alias( "checkprogsize", program, env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"), ) ) print("Building in %s mode" % env["BUILD_TYPE"]) return program def ProcessProgramDeps(env): def _append_pio_macros(): core_version = pepver_to_semver(__version__) env.AppendUnique( CPPDEFINES=[ ( "PLATFORMIO", int( "{0:02d}{1:02d}{2:02d}".format( core_version.major, core_version.minor, core_version.patch ) ), ) ] ) _append_pio_macros() env.PrintConfiguration() # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): env.ProcessFlags(env.BoardConfig().get("build.extra_flags")) # apply user flags env.ProcessFlags(env.get("BUILD_FLAGS")) # process framework scripts env.BuildFrameworks(env.get("PIOFRAMEWORK")) if "debug" in env["BUILD_TYPE"]: env.ConfigureDebugTarget() # remove specified flags env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) def ProcessCompileDbToolchainOption(env): if "compiledb" not in COMMAND_LINE_TARGETS: return # Resolve absolute path of toolchain for cmd in ("CC", "CXX", "AS"): if cmd not in env: continue if os.path.isabs(env[cmd]) or '"' in env[cmd]: continue env[cmd] = where_is_program(env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")) if " " in env[cmd]: # issue #4998: Space in compilator path env[cmd] = f'"{env[cmd]}"' if env.get("COMPILATIONDB_INCLUDE_TOOLCHAIN"): print("Warning! `COMPILATIONDB_INCLUDE_TOOLCHAIN` is scoping") for scope, includes in env.DumpIntegrationIncludes().items(): if scope in ("toolchain",): env.Append(CPPPATH=includes) def ProcessProjectDeps(env): plb = env.ConfigureProjectLibBuilder() # prepend project libs to the beginning of list env.Prepend(LIBS=plb.build()) # prepend extra linker related options from libs env.PrependUnique( **{ key: plb.env.get(key) for key in ("LIBS", "LIBPATH", "LINKFLAGS") if plb.env.get(key) } ) if "test" in env["BUILD_TYPE"]: build_files_before_nums = len(env.get("PIOBUILDFILES", [])) plb.env.BuildSources( "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1: sys.stderr.write( "Error: Nothing to build. Please put your test suites " "to the '%s' folder\n" % env.subst("$PROJECT_TEST_DIR") ) env.Exit(1) if "test" not in env["BUILD_TYPE"] or env.GetProjectOption("test_build_src"): plb.env.BuildSources( "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") ) if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: sys.stderr.write( "Error: Nothing to build. Please put your source code files " "to the '%s' folder\n" % env.subst("$PROJECT_SRC_DIR") ) env.Exit(1) def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches if not isinstance(flags, list): flags = [flags] result = {} for raw in flags: for key, value in env.ParseFlags(str(raw)).items(): if key not in result: result[key] = [] result[key].extend(value) cppdefines = [] for item in result["CPPDEFINES"]: if not Util.is_Sequence(item): cppdefines.append(item) continue name, value = item[:2] if '"' in value: value = value.replace('"', '\\"') elif value.isdigit(): value = int(value) elif value.replace(".", "", 1).isdigit(): value = float(value) cppdefines.append((name, value)) result["CPPDEFINES"] = cppdefines # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): for i, p in enumerate(result.get(k, [])): p = env.subst(p) if os.path.isdir(p): result[k][i] = os.path.abspath(p) # fix relative LIBs for i, l in enumerate(result.get("LIBS", [])): if isinstance(l, FS.File): result["LIBS"][i] = os.path.abspath(l.get_path()) # fix relative path for "-include" for i, f in enumerate(result.get("CCFLAGS", [])): if isinstance(f, tuple) and f[0] == "-include": result["CCFLAGS"][i] = (f[0], env.subst(f[1].get_path())) return result def ProcessFlags(env, flags): # pylint: disable=too-many-branches if not flags: return env.Append(**env.ParseFlagsExtended(flags)) # Cancel any previous definition of name, either built in or # provided with a -U option // Issue #191 undefines = [ u for u in env.get("CCFLAGS", []) if isinstance(u, string_types) and u.startswith("-U") ] if undefines: for undef in undefines: env["CCFLAGS"].remove(undef) if undef[2:] in env["CPPDEFINES"]: env["CPPDEFINES"].remove(undef[2:]) env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines)) def ProcessUnFlags(env, flags): if not flags: return parsed = env.ParseFlagsExtended(flags) unflag_scopes = tuple(set(["ASPPFLAGS"] + list(parsed.keys()))) for scope in unflag_scopes: for unflags in parsed.values(): for unflag in unflags: for current in list(env.get(scope, [])): conditions = [ unflag == current, not isinstance(unflag, (tuple, list)) and isinstance(current, (tuple, list)) and unflag == current[0], ] if any(conditions): env[scope].remove(current) def StringifyMacro(env, value): # pylint: disable=unused-argument return '\\"%s\\"' % value.replace('"', '\\\\\\"') def MatchSourceFiles(env, src_dir, src_filter=None, src_exts=None): src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT src_exts = src_exts or (SRC_BUILD_EXT + SRC_HEADER_EXT) return fs.match_src_files(env.subst(src_dir), src_filter, src_exts) def CollectBuildFiles( env, variant_dir, src_dir, src_filter=None, duplicate=False ): # pylint: disable=too-many-locals sources = [] variants = [] src_dir = env.subst(src_dir) if src_dir.endswith(os.sep): src_dir = src_dir[:-1] for item in env.MatchSourceFiles(src_dir, src_filter, SRC_BUILD_EXT): _reldir = os.path.dirname(item) _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir if _var_dir not in variants: variants.append(_var_dir) env.VariantDir(_var_dir, _src_dir, duplicate) sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) middlewares = env.get("__PIO_BUILD_MIDDLEWARES") if not middlewares: return sources new_sources = [] for node in sources: new_node = node for callback, pattern in middlewares: if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): continue if callback.__code__.co_argcount == 2: new_node = callback(env, new_node) else: new_node = callback(new_node) if not new_node: break if new_node: new_sources.append(new_node) return new_sources def AddBuildMiddleware(env, callback, pattern=None): env.Append(__PIO_BUILD_MIDDLEWARES=[(callback, pattern)]) def BuildFrameworks(env, frameworks): if not frameworks: return if "BOARD" not in env: sys.stderr.write( "Please specify `board` in `platformio.ini` to use " "with '%s' framework\n" % ", ".join(frameworks) ) env.Exit(1) supported_frameworks = env.BoardConfig().get("frameworks", []) for name in frameworks: if name == "arduino": # Arduino IDE appends .o to the end of filename Builder.match_splitext = scons_patched_match_splitext if "nobuild" not in COMMAND_LINE_TARGETS: env.ConvertInoToCpp() if name in supported_frameworks: SConscript(env.GetFrameworkScript(name), exports="env") else: sys.stderr.write("Error: This board doesn't support %s framework!\n" % name) env.Exit(1) def BuildLibrary(env, variant_dir, src_dir, src_filter=None, nodes=None): env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) nodes = nodes or env.CollectBuildFiles(variant_dir, src_dir, src_filter) return env.StaticLibrary(env.subst(variant_dir), nodes) def BuildSources(env, variant_dir, src_dir, src_filter=None): if env.get("PIOMAINPROG"): sys.stderr.write( "Error: The main program is already constructed and the inline " "source files are not allowed. Please use `env.BuildLibrary(...)` " "or PRE-type script instead." ) env.Exit(1) nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) DefaultEnvironment().Append( PIOBUILDFILES=[ env.Object(node) if isinstance(node, FS.File) else node for node in nodes ] ) def exists(_): return True def generate(env): env.AddMethod(GetBuildType) env.AddMethod(BuildProgram) env.AddMethod(ProcessProgramDeps) env.AddMethod(ProcessCompileDbToolchainOption) env.AddMethod(ProcessProjectDeps) env.AddMethod(ParseFlagsExtended) env.AddMethod(ProcessFlags) env.AddMethod(ProcessUnFlags) env.AddMethod(StringifyMacro) env.AddMethod(MatchSourceFiles) env.AddMethod(CollectBuildFiles) env.AddMethod(AddBuildMiddleware) env.AddMethod(BuildFrameworks) env.AddMethod(BuildLibrary) env.AddMethod(BuildSources) return env ================================================ FILE: platformio/builder/tools/piohooks.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. def AddActionWrapper(handler): def wraps(env, files, action): if not isinstance(files, (list, tuple, set)): files = [files] known_nodes = [] unknown_files = [] for item in files: nodes = env.arg2nodes(item, env.fs.Entry) if nodes and nodes[0].exists(): known_nodes.extend(nodes) else: unknown_files.append(item) if unknown_files: env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_files, action)]}) if known_nodes: return handler(known_nodes, action) return [] return wraps def ProcessDelayedActions(env): for func, nodes, action in env.get("_PIO_DELAYED_ACTIONS", []): func(nodes, action) def generate(env): env.Replace(**{"_PIO_DELAYED_ACTIONS": []}) env.AddMethod(AddActionWrapper(env.AddPreAction), "AddPreAction") env.AddMethod(AddActionWrapper(env.AddPostAction), "AddPostAction") env.AddMethod(ProcessDelayedActions) def exists(_): return True ================================================ FILE: platformio/builder/tools/pioino.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import atexit import glob import io import os import re import tempfile import click from platformio.compat import get_filesystem_encoding, get_locale_encoding class InoToCPPConverter: PROTOTYPE_RE = re.compile( r"""^( (?:template\<.*\>\s*)? # template ([a-z_\d\&]+\*?\s+){1,2} # return type ([a-z_\d]+\s*) # name of prototype \([a-z_,\.\*\&\[\]\s\d]*\) # arguments )\s*(\{|;) # must end with `{` or `;` """, re.X | re.M | re.I, ) DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" def __init__(self, env): self.env = env self._main_ino = None self._safe_encoding = None def read_safe_contents(self, path): error_reported = False for encoding in ( "utf-8", None, get_filesystem_encoding(), get_locale_encoding(), "latin-1", ): try: with io.open(path, encoding=encoding) as fp: contents = fp.read() self._safe_encoding = encoding return contents except UnicodeDecodeError: if not error_reported: error_reported = True click.secho( "Unicode decode error has occurred, please remove invalid " "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" % path, fg="yellow", err=True, ) return "" def write_safe_contents(self, path, contents): with io.open( path, "w", encoding=self._safe_encoding, errors="backslashreplace" ) as fp: return fp.write(contents) def is_main_node(self, contents): return self.DETECTMAIN_RE.search(contents) def convert(self, nodes): contents = self.merge(nodes) if not contents: return None return self.process(contents) def merge(self, nodes): assert nodes lines = [] for node in nodes: contents = self.read_safe_contents(node.get_path()) _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] if self.is_main_node(contents): lines = _lines + lines self._main_ino = node.get_path() else: lines.extend(_lines) if not self._main_ino: self._main_ino = nodes[0].get_path() return "\n".join(["#include "] + lines) if lines else None def process(self, contents): out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp" assert self._gcc_preprocess(contents, out_file) contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) self.write_safe_contents(out_file, self.append_prototypes(contents)) return out_file def _gcc_preprocess(self, contents, out_file): tmp_path = tempfile.mkstemp()[1] self.write_safe_contents(tmp_path, contents) self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( out_file, tmp_path ), "Converting " + os.path.basename(out_file[:-4]), ) ) atexit.register(_delete_file, tmp_path) return os.path.isfile(out_file) def _join_multiline_strings(self, contents): if "\\\n" not in contents: return contents newlines = [] linenum = 0 stropen = False for line in contents.split("\n"): _linenum = self._parse_preproc_line_num(line) if _linenum is not None: linenum = _linenum else: linenum += 1 if line.endswith("\\"): if line.startswith('"'): stropen = True newlines.append(line[:-1]) continue if stropen: newlines[len(newlines) - 1] += line[:-1] continue elif stropen and line.endswith(('",', '";')): newlines[len(newlines) - 1] += line stropen = False newlines.append( '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/")) ) continue newlines.append(line) return "\n".join(newlines) @staticmethod def _parse_preproc_line_num(line): if not line.startswith("#"): return None tokens = line.split(" ", 3) if len(tokens) > 2 and tokens[1].isdigit(): return int(tokens[1]) return None def _parse_prototypes(self, contents): prototypes = [] reserved_keywords = set(["if", "else", "while"]) for match in self.PROTOTYPE_RE.finditer(contents): if ( set([match.group(2).strip(), match.group(3).strip()]) & reserved_keywords ): continue prototypes.append(match) return prototypes def _get_total_lines(self, contents): total = 0 if contents.endswith("\n"): contents = contents[:-1] for line in contents.split("\n")[::-1]: linenum = self._parse_preproc_line_num(line) if linenum is not None: return total + linenum total += 1 return total def append_prototypes(self, contents): prototypes = self._parse_prototypes(contents) or [] # skip already declared prototypes declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";") prototypes = [m for m in prototypes if m.group(1).strip() not in declared] if not prototypes: return contents prototype_names = set(m.group(3).strip() for m in prototypes) split_pos = prototypes[0].start() match_ptrs = re.search( self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), contents[:split_pos], re.M, ) if match_ptrs: split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1 result = [] result.append(contents[:split_pos].strip()) result.append("%s;" % ";\n".join([m.group(1) for m in prototypes])) result.append( '#line %d "%s"' % ( self._get_total_lines(contents[:split_pos]), self._main_ino.replace("\\", "/"), ) ) result.append(contents[split_pos:].strip()) return "\n".join(result) def FindInoNodes(env): src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR")) return env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( os.path.join(src_dir, "*.pde") ) def ConvertInoToCpp(env): ino_nodes = env.FindInoNodes() if not ino_nodes: return c = InoToCPPConverter(env) out_file = c.convert(ino_nodes) atexit.register(_delete_file, out_file) def _delete_file(path): try: if os.path.isfile(path): os.remove(path) except: # pylint: disable=bare-except pass def generate(env): env.AddMethod(FindInoNodes) env.AddMethod(ConvertInoToCpp) def exists(_): return True ================================================ FILE: platformio/builder/tools/piointegration.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import glob import os import SCons.Defaults # pylint: disable=import-error import SCons.Subst # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio.proc import exec_command, where_is_program def IsIntegrationDump(_): return set(["__idedata", "idedata"]) & set(COMMAND_LINE_TARGETS) def DumpIntegrationIncludes(env): result = dict(build=[], compatlib=[], toolchain=[]) # `env`(project) CPPPATH result["build"].extend( [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])] ) # installed libs for lb in env.GetLibBuilders(): result["compatlib"].extend( [os.path.abspath(inc) for inc in lb.get_include_dirs()] ) # includes from toolchains p = env.PioPlatform() for pkg in p.get_installed_packages(with_optional=False): if p.get_package_type(pkg.metadata.name) != "toolchain": continue toolchain_dir = glob.escape(pkg.path) toolchain_incglobs = [ os.path.join(toolchain_dir, "*", "include", "c++", "*"), os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"), os.path.join(toolchain_dir, "*", "include*"), ] for g in toolchain_incglobs: result["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)]) return result def get_gcc_defines(env): items = [] try: sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command( "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True ) except OSError: return items if result["returncode"] != 0: return items for line in result["out"].split("\n"): tokens = line.strip().split(" ", 2) if not tokens or tokens[0] != "#define": continue if len(tokens) > 2: items.append("%s=%s" % (tokens[1], tokens[2])) else: items.append(tokens[1]) return items def dump_defines(env): defines = [] # global symbols for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])): item = item.strip() if item: defines.append(env.subst(item).replace('\\"', '"')) # special symbol for Atmel AVR MCU if env["PIOPLATFORM"] == "atmelavr": board_mcu = env.get("BOARD_MCU") if not board_mcu and "BOARD" in env: board_mcu = env.BoardConfig().get("build.mcu") if board_mcu: defines.append( str( "__AVR_%s__" % board_mcu.upper() .replace("ATMEGA", "ATmega") .replace("ATTINY", "ATtiny") ) ) # built-in GCC marcos # if env.GetCompilerType() == "gcc": # defines.extend(get_gcc_defines(env)) return defines def dump_svd_path(env): svd_path = env.GetProjectOption("debug_svd_path") if svd_path: return os.path.abspath(svd_path) if "BOARD" not in env: return None try: svd_path = env.BoardConfig().get("debug.svd_path") assert svd_path except (AssertionError, KeyError): return None # custom path to SVD file if os.path.isfile(svd_path): return svd_path # default file from ./platform/misc/svd folder p = env.PioPlatform() if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)): return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path)) return None def _split_flags_string(env, s): args = env.subst_list(s, SCons.Subst.SUBST_CMD)[0] return [str(arg) for arg in args] def DumpIntegrationData(*args): projenv, globalenv = args[0:2] # pylint: disable=unbalanced-tuple-unpacking data = { "build_type": globalenv.GetBuildType(), "env_name": globalenv["PIOENV"], "libsource_dirs": [ globalenv.subst(item) for item in globalenv.GetLibSourceDirs() ], "defines": dump_defines(projenv), "includes": projenv.DumpIntegrationIncludes(), "cc_flags": _split_flags_string(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS"), "cxx_flags": _split_flags_string(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), "cc_path": where_is_program( globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}") ), "cxx_path": where_is_program( globalenv.subst("$CXX"), globalenv.subst("${ENV['PATH']}") ), "gdb_path": where_is_program( globalenv.subst("$GDB"), globalenv.subst("${ENV['PATH']}") ), "prog_path": globalenv.subst("$PROGPATH"), "svd_path": dump_svd_path(globalenv), "compiler_type": globalenv.GetCompilerType(), "targets": globalenv.DumpTargets(), "extra": dict( flash_images=[ {"offset": item[0], "path": globalenv.subst(item[1])} for item in globalenv.get("FLASH_EXTRA_IMAGES", []) ] ), } for key in ("IDE_EXTRA_DATA", "INTEGRATION_EXTRA_DATA"): data["extra"].update(globalenv.get(key, {})) return data def exists(_): return True def generate(env): env["IDE_EXTRA_DATA"] = {} # legacy support env["INTEGRATION_EXTRA_DATA"] = {} env.AddMethod(IsIntegrationDump) env.AddMethod(DumpIntegrationIncludes) env.AddMethod(DumpIntegrationData) return env ================================================ FILE: platformio/builder/tools/piolib.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-instance-attributes, too-many-public-methods # pylint: disable=assignment-from-no-return, unused-argument, too-many-lines import hashlib import io import os import re import sys import click import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs from platformio.builder.tools import piobuild from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, ) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) from platformio.package.meta import PackageCompatibility, PackageItem, PackageSpec from platformio.project.options import ProjectOptions class LibBuilderFactory: @staticmethod def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" if os.path.isfile(os.path.join(path, "library.json")): clsname = "PlatformIOLibBuilder" else: used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks) if common_frameworks: clsname = "%sLibBuilder" % list(common_frameworks)[0].capitalize() elif used_frameworks: clsname = "%sLibBuilder" % used_frameworks[0].capitalize() obj = globals()[clsname](env, path, verbose=verbose) # Handle PlatformIOLibBuilder.manifest.build.builder # pylint: disable=protected-access if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get( "builder" ): obj = globals()[obj._manifest.get("build", {}).get("builder")]( env, path, verbose=verbose ) assert isinstance(obj, LibBuilderBase) return obj @staticmethod def get_used_frameworks(env, path): if any( os.path.isfile(os.path.join(path, fname)) for fname in ("library.properties", "keywords.txt") ): return ["arduino"] if os.path.isfile(os.path.join(path, "module.json")): return ["mbed"] include_re = re.compile( r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', flags=re.MULTILINE ) # check source files for root, _, files in os.walk(path, followlinks=True): if "mbed_lib.json" in files: return ["mbed"] for fname in files: if not fs.path_endswith_ext( fname, piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT ): continue with io.open( os.path.join(root, fname), encoding="utf8", errors="ignore" ) as fp: content = fp.read() if not content: continue if "Arduino.h" in content and include_re.search(content): return ["arduino"] if "mbed.h" in content and include_re.search(content): return ["mbed"] return [] class LibBuilderBase: CLASSIC_SCANNER = SCons.Scanner.C.CScanner() CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner() # Max depth of nested includes: # -1 = unlimited # 0 - disabled nesting # >0 - number of allowed nested includes CCONDITIONAL_SCANNER_DEPTH = 99 PARSE_SRC_BY_H_NAME = True _INCLUDE_DIRS_CACHE = None def __init__(self, env, path, manifest=None, verbose=False): self.env = env.Clone() self.envorigin = env.Clone() self.path = os.path.abspath(env.subst(path)) self.verbose = verbose try: self._manifest = manifest if manifest else self.load_manifest() except ManifestParserError: click.secho( "Warning! Ignoring broken library manifest in " + self.path, fg="yellow" ) self._manifest = {} self.is_dependent = False self.is_built = False self.depbuilders = [] self._deps_are_processed = False self._circular_deps = [] self._processed_search_files = [] # pass a macro to the projenv + libs if "test" in env["BUILD_TYPE"]: self.env.Append(CPPDEFINES=["PIO_UNIT_TESTING"]) # reset source filter, could be overridden with extra script self.env["SRC_FILTER"] = "" # process extra options and append to build environment self.process_extra_options() def __repr__(self): return "%s(%r)" % (self.__class__, self.path) def __contains__(self, child_path): return self.is_common_builder(self.path, child_path) def is_common_builder(self, root_path, child_path): if IS_WINDOWS: root_path = root_path.lower() child_path = child_path.lower() if root_path == child_path: return True if ( os.path.commonprefix([root_path + os.path.sep, child_path]) == root_path + os.path.sep ): return True # try to resolve paths root_path = os.path.realpath(root_path) child_path = os.path.realpath(child_path) return ( os.path.commonprefix([root_path + os.path.sep, child_path]) == root_path + os.path.sep ) @property def name(self): return self._manifest.get("name", os.path.basename(self.path)) @property def version(self): return self._manifest.get("version") @property def dependent(self): """Backward compatibility with ESP-IDF""" return self.is_dependent @property def dependencies(self): return self._manifest.get("dependencies") @property def src_filter(self): return piobuild.SRC_FILTER_DEFAULT + [ "-" % os.sep, "-" % os.sep, "-" % os.sep, "-" % os.sep, ] @property def include_dir(self): for name in ("include", "Include"): d = os.path.join(self.path, name) if os.path.isdir(d): return d return None @property def src_dir(self): for name in ("src", "Src"): d = os.path.join(self.path, name) if os.path.isdir(d): return d return self.path def get_include_dirs(self): items = [] include_dir = self.include_dir if include_dir: items.append(include_dir) items.append(self.src_dir) return items @property def build_dir(self): lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] return os.path.join( "$BUILD_DIR", "lib%s" % lib_hash, os.path.basename(self.path) ) @property def build_flags(self): return None @property def build_unflags(self): return None @property def extra_script(self): return None @property def lib_archive(self): return self.env.GetProjectOption("lib_archive") @property def lib_ldf_mode(self): return self.env.GetProjectOption("lib_ldf_mode") @staticmethod def validate_ldf_mode(mode): ldf_modes = ProjectOptions["env.lib_ldf_mode"].type.choices if isinstance(mode, string_types): mode = mode.strip().lower() if mode in ldf_modes: return mode try: return ldf_modes[int(mode)] except (IndexError, ValueError): pass return ProjectOptions["env.lib_ldf_mode"].default @property def lib_compat_mode(self): return self.env.GetProjectOption("lib_compat_mode") @staticmethod def validate_compat_mode(mode): compat_modes = ProjectOptions["env.lib_compat_mode"].type.choices if isinstance(mode, string_types): mode = mode.strip().lower() if mode in compat_modes: return mode try: return compat_modes[int(mode)] except (IndexError, ValueError): pass return ProjectOptions["env.lib_compat_mode"].default def is_platforms_compatible(self, platforms): return True def is_frameworks_compatible(self, frameworks): return True def load_manifest(self): return {} def process_extra_options(self): with fs.cd(self.path): self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(True) self.env.SConscript( os.path.abspath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) self.env.SConscriptChdir(False) self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): if not self.dependencies or self._deps_are_processed: return self._deps_are_processed = True for dependency in self.dependencies: found = False for lb in self.env.GetLibBuilders(): if not lb.is_dependency_compatible(dependency): continue found = True if lb not in self.depbuilders: self.depend_on(lb) break if not found and self.verbose: sys.stderr.write( "Warning: Ignored `%s` dependency for `%s` " "library\n" % (dependency["name"], self.name) ) def is_dependency_compatible(self, dependency): pkg = PackageItem(self.path) qualifiers = {"name": self.name, "version": self.version} if pkg.metadata: qualifiers = {"name": pkg.metadata.name, "version": pkg.metadata.version} if pkg.metadata.spec and pkg.metadata.spec.owner: qualifiers["owner"] = pkg.metadata.spec.owner dep_qualifiers = { k: v for k, v in dependency.items() if k in ("owner", "name", "version") } if ( "version" in dep_qualifiers and not PackageSpec(dep_qualifiers["version"]).requirements ): del dep_qualifiers["version"] return PackageCompatibility.from_dependency(dep_qualifiers).is_compatible( PackageCompatibility(**qualifiers) ) def get_search_files(self): return [ os.path.join(self.src_dir, item) for item in self.env.MatchSourceFiles( self.src_dir, self.src_filter, piobuild.SRC_BUILD_EXT ) ] def get_implicit_includes( # pylint: disable=too-many-branches self, search_files=None ): # all include directories if not LibBuilderBase._INCLUDE_DIRS_CACHE: LibBuilderBase._INCLUDE_DIRS_CACHE = [ self.env.Dir(d) for d in ProjectAsLibBuilder( self.envorigin, "$PROJECT_DIR", export_projenv=False ).get_include_dirs() ] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( [self.env.Dir(d) for d in lb.get_include_dirs()] ) # append self include directories include_dirs = [self.env.Dir(d) for d in self.get_include_dirs()] include_dirs.extend(LibBuilderBase._INCLUDE_DIRS_CACHE) result = [] search_files = search_files or [] while search_files: node = self.env.File(search_files.pop(0)) if node.get_abspath() in self._processed_search_files: continue self._processed_search_files.append(node.get_abspath()) try: assert "+" in self.lib_ldf_mode candidates = LibBuilderBase.CCONDITIONAL_SCANNER( node, self.env, tuple(include_dirs), depth=self.CCONDITIONAL_SCANNER_DEPTH, ) except Exception as exc: # pylint: disable=broad-except if self.verbose and "+" in self.lib_ldf_mode: sys.stderr.write( "Warning! Classic Pre Processor is used for `%s`, " "advanced has failed with `%s`\n" % (node.get_abspath(), exc) ) candidates = LibBuilderBase.CLASSIC_SCANNER( node, self.env, tuple(include_dirs) ) # print(node.get_abspath(), [c.get_abspath() for c in candidates]) for item in candidates: item_path = item.get_abspath() # process internal files recursively if ( item_path not in self._processed_search_files and item_path not in search_files and item_path in self ): search_files.append(item_path) if item not in result: result.append(item) if not self.PARSE_SRC_BY_H_NAME: continue if not fs.path_endswith_ext(item_path, piobuild.SRC_HEADER_EXT): continue item_fname = item_path[: item_path.rindex(".")] for ext in piobuild.SRC_C_EXT + piobuild.SRC_CXX_EXT: if not os.path.isfile("%s.%s" % (item_fname, ext)): continue item_c_node = self.env.File("%s.%s" % (item_fname, ext)) if item_c_node not in result: result.append(item_c_node) return result def search_deps_recursive(self, search_files=None): self.process_dependencies() # when LDF is disabled if self.lib_ldf_mode == "off": return if self.lib_ldf_mode.startswith("deep"): search_files = self.get_search_files() lib_inc_map = {} for inc in self.get_implicit_includes(search_files): inc_path = inc.get_abspath() for lb in self.env.GetLibBuilders(): if inc_path in lb: if lb not in lib_inc_map: lib_inc_map[lb] = [] lib_inc_map[lb].append(inc_path) break for lb, lb_search_files in lib_inc_map.items(): self.depend_on(lb, search_files=lb_search_files) def depend_on(self, lb, search_files=None, recursive=True): def _already_depends(_lb): if self in _lb.depbuilders: return True for __lb in _lb.depbuilders: if _already_depends(__lb): return True return False # assert isinstance(lb, LibBuilderBase) if self != lb: if _already_depends(lb): if self.verbose: sys.stderr.write( "Warning! Circular dependencies detected " "between `%s` and `%s`\n" % (self.path, lb.path) ) self._circular_deps.append(lb) elif lb not in self.depbuilders: self.depbuilders.append(lb) lb.is_dependent = True LibBuilderBase._INCLUDE_DIRS_CACHE = None if recursive: lb.search_deps_recursive(search_files) def build(self): libs = [] shared_scopes = ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS") for lb in self.depbuilders: libs.extend(lb.build()) # copy shared information to self env self.env.PrependUnique( **{ scope: lb.env.get(scope) for scope in shared_scopes if lb.env.get(scope) } ) for lb in self._circular_deps: self.env.PrependUnique(CPPPATH=lb.get_include_dirs()) if self.is_built: return libs self.is_built = True self.env.PrependUnique(CPPPATH=self.get_include_dirs()) self.env.ProcessCompileDbToolchainOption() if self.lib_ldf_mode == "off": for lb in self.env.GetLibBuilders(): if self == lb or not lb.is_built: continue self.env.PrependUnique( **{ scope: lb.env.get(scope) for scope in shared_scopes if lb.env.get(scope) } ) do_not_archive = not self.lib_archive if not do_not_archive: nodes = self.env.CollectBuildFiles( self.build_dir, self.src_dir, self.src_filter ) if nodes: libs.append( self.env.BuildLibrary( self.build_dir, self.src_dir, self.src_filter, nodes ) ) else: do_not_archive = True if do_not_archive: self.env.BuildSources(self.build_dir, self.src_dir, self.src_filter) return libs class UnknownLibBuilder(LibBuilderBase): pass class ArduinoLibBuilder(LibBuilderBase): def load_manifest(self): manifest_path = os.path.join(self.path, "library.properties") if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() @property def include_dir(self): if not all( os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src") ): return None return os.path.join(self.path, "include") def get_include_dirs(self): include_dirs = super().get_include_dirs() if os.path.isdir(os.path.join(self.path, "src")): return include_dirs if os.path.isdir(os.path.join(self.path, "utility")): include_dirs.append(os.path.join(self.path, "utility")) return include_dirs @property def src_filter(self): src_dir = os.path.join(self.path, "src") if os.path.isdir(src_dir): # pylint: disable=no-member src_filter = LibBuilderBase.src_filter.fget(self) for root, _, files in os.walk(src_dir, followlinks=True): found = False for fname in files: if fname.lower().endswith("asm"): found = True break if not found: continue rel_path = root.replace(src_dir, "") if rel_path.startswith(os.path.sep): rel_path = rel_path[1:] + os.path.sep src_filter.append("-<%s*.[aA][sS][mM]>" % rel_path) return src_filter src_filter = [] is_utility = os.path.isdir(os.path.join(self.path, "utility")) for ext in piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT: # arduino ide ignores files with .asm or .ASM extensions if ext.lower() == "asm": continue src_filter.append("+<*.%s>" % ext) if is_utility: src_filter.append("+" % (os.path.sep, ext)) return src_filter @property def dependencies(self): # do not include automatically all libraries for build # chain+ will decide later return None @property def lib_ldf_mode(self): # pylint: disable=no-member if not self._manifest.get("dependencies"): return LibBuilderBase.lib_ldf_mode.fget(self) missing = object() global_value = self.env.GetProjectConfig().getraw( "env:" + self.env["PIOENV"], "lib_ldf_mode", missing ) if global_value != missing: return LibBuilderBase.lib_ldf_mode.fget(self) # automatically enable C++ Preprocessing in runtime # (Arduino IDE has this behavior) return "chain+" def is_frameworks_compatible(self, frameworks): return PackageCompatibility(frameworks=frameworks).is_compatible( PackageCompatibility(frameworks=["arduino", "energia"]) ) def is_platforms_compatible(self, platforms): return PackageCompatibility(platforms=platforms).is_compatible( PackageCompatibility(platforms=self._manifest.get("platforms")) ) @property def build_flags(self): ldflags = [ LibBuilderBase.build_flags.fget(self), # pylint: disable=no-member self._manifest.get("ldflags"), ] if self._manifest.get("precompiled") in ("true", "full"): # add to LDPATH {build.mcu} folder board_config = self.env.BoardConfig() for key in ("build.mcu", "build.cpu"): libpath = os.path.join(self.src_dir, board_config.get(key, "")) if not os.path.isdir(libpath): continue self.env.PrependUnique(LIBPATH=libpath) break ldflags = [flag for flag in ldflags if flag] # remove empty return " ".join(ldflags) if ldflags else None class MbedLibBuilder(LibBuilderBase): def load_manifest(self): manifest_path = os.path.join(self.path, "module.json") if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() @property def src_dir(self): if os.path.isdir(os.path.join(self.path, "source")): return os.path.join(self.path, "source") return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member def get_include_dirs(self): include_dirs = super().get_include_dirs() if self.path not in include_dirs: include_dirs.append(self.path) # library with module.json for p in self._manifest.get("extraIncludes", []): include_dirs.append(os.path.join(self.path, p)) # old mbed library without manifest, add to CPPPATH all folders if not self._manifest: for root, _, __ in os.walk(self.path): part = root.replace(self.path, "").lower() if any(s in part for s in ("%s." % os.path.sep, "test", "example")): continue if root not in include_dirs: include_dirs.append(root) return include_dirs def is_frameworks_compatible(self, frameworks): return PackageCompatibility(frameworks=frameworks).is_compatible( PackageCompatibility(frameworks=["mbed"]) ) def process_extra_options(self): self._process_mbed_lib_confs() return super().process_extra_options() def _process_mbed_lib_confs(self): mbed_lib_paths = [ os.path.join(root, "mbed_lib.json") for root, _, files in os.walk(self.path) if "mbed_lib.json" in files ] if not mbed_lib_paths: return None mbed_config_path = None for p in self.env.get("CPPPATH"): mbed_config_path = os.path.join(self.env.subst(p), "mbed_config.h") if os.path.isfile(mbed_config_path): break mbed_config_path = None if not mbed_config_path: return None macros = {} for mbed_lib_path in mbed_lib_paths: macros.update(self._mbed_lib_conf_parse_macros(mbed_lib_path)) self._mbed_conf_append_macros(mbed_config_path, macros) return True @staticmethod def _mbed_normalize_macro(macro): name = macro value = None if "=" in macro: name, value = macro.split("=", 1) return dict(name=name, value=value) def _mbed_lib_conf_parse_macros(self, mbed_lib_path): macros = {} cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) manifest = fs.load_json(mbed_lib_path) # default macros for macro in manifest.get("macros", []): macro = self._mbed_normalize_macro(macro) macros[macro["name"]] = macro # configuration items for key, options in manifest.get("config", {}).items(): if "value" not in options: continue macros[key] = dict( name=options.get("macro_name"), value=options.get("value") ) # overrode items per target for target, options in manifest.get("target_overrides", {}).items(): if target != "*" and "TARGET_" + target not in cppdefines: continue for macro in options.get("target.macros_add", []): macro = self._mbed_normalize_macro(macro) macros[macro["name"]] = macro for key, value in options.items(): if not key.startswith("target.") and key in macros: macros[key]["value"] = value # normalize macro names for key, macro in macros.items(): if not macro["name"]: macro["name"] = key if "." not in macro["name"]: macro["name"] = "%s.%s" % (manifest.get("name"), macro["name"]) macro["name"] = re.sub( r"[^a-z\d]+", "_", macro["name"], flags=re.I ).upper() macro["name"] = "MBED_CONF_" + macro["name"] if isinstance(macro["value"], bool): macro["value"] = 1 if macro["value"] else 0 return {macro["name"]: macro["value"] for macro in macros.values()} def _mbed_conf_append_macros(self, mbed_config_path, macros): lines = [] with open(mbed_config_path, encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if line == "#endif": lines.append("// PlatformIO Library Dependency Finder (LDF)") lines.extend( [ "#define %s %s" % (name, value if value is not None else "") for name, value in macros.items() ] ) lines.append("") if not line.startswith("#define"): lines.append(line) continue tokens = line.split() if len(tokens) < 2 or tokens[1] not in macros: lines.append(line) lines.append("") with open(mbed_config_path, mode="w", encoding="utf8") as fp: fp.write("\n".join(lines)) class PlatformIOLibBuilder(LibBuilderBase): def load_manifest(self): manifest_path = os.path.join(self.path, "library.json") if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() def _has_arduino_manifest(self): return os.path.isfile(os.path.join(self.path, "library.properties")) @property def include_dir(self): if "includeDir" in self._manifest.get("build", {}): with fs.cd(self.path): return os.path.abspath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member def get_include_dirs(self): include_dirs = super().get_include_dirs() # backwards compatibility with PlatformIO 2.0 if ( "build" not in self._manifest and self._has_arduino_manifest() and not os.path.isdir(os.path.join(self.path, "src")) and os.path.isdir(os.path.join(self.path, "utility")) ): include_dirs.append(os.path.join(self.path, "utility")) for path in self.env.get("CPPPATH", []): if path not in include_dirs and path not in self.envorigin.get( "CPPPATH", [] ): include_dirs.append(self.env.subst(path)) return include_dirs @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): with fs.cd(self.path): return os.path.abspath(self._manifest.get("build").get("srcDir")) return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member @property def src_filter(self): # pylint: disable=no-member if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") if self.env["SRC_FILTER"]: return self.env["SRC_FILTER"] if self._has_arduino_manifest(): return ArduinoLibBuilder.src_filter.fget(self) return LibBuilderBase.src_filter.fget(self) @property def build_flags(self): if "flags" in self._manifest.get("build", {}): return self._manifest.get("build").get("flags") return LibBuilderBase.build_flags.fget(self) # pylint: disable=no-member @property def build_unflags(self): if "unflags" in self._manifest.get("build", {}): return self._manifest.get("build").get("unflags") return LibBuilderBase.build_unflags.fget(self) # pylint: disable=no-member @property def extra_script(self): if "extraScript" in self._manifest.get("build", {}): return self._manifest.get("build").get("extraScript") return LibBuilderBase.extra_script.fget(self) # pylint: disable=no-member @property def lib_archive(self): missing = object() global_value = self.env.GetProjectConfig().getraw( "env:" + self.env["PIOENV"], "lib_archive", missing ) if global_value != missing: return self.env.GetProjectConfig().get( "env:" + self.env["PIOENV"], "lib_archive" ) # pylint: disable=no-member return self._manifest.get("build", {}).get( "libArchive", LibBuilderBase.lib_archive.fget(self) ) @property def lib_ldf_mode(self): # pylint: disable=no-member return self.validate_ldf_mode( self._manifest.get("build", {}).get( "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self) ) ) @property def lib_compat_mode(self): # pylint: disable=no-member return self.validate_compat_mode( self._manifest.get("build", {}).get( "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) ) ) def is_platforms_compatible(self, platforms): return PackageCompatibility(platforms=platforms).is_compatible( PackageCompatibility(platforms=self._manifest.get("platforms")) ) def is_frameworks_compatible(self, frameworks): return PackageCompatibility(frameworks=frameworks).is_compatible( PackageCompatibility(frameworks=self._manifest.get("frameworks")) ) class ProjectAsLibBuilder(LibBuilderBase): def __init__(self, env, *args, **kwargs): export_projenv = kwargs.get("export_projenv", True) if "export_projenv" in kwargs: del kwargs["export_projenv"] # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter if export_projenv: env.Export(dict(projenv=self.env)) def __contains__(self, child_path): for root_path in (self.include_dir, self.src_dir, self.test_dir): if root_path and self.is_common_builder(root_path, child_path): return True return False @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") return include_dir if os.path.isdir(include_dir) else None @property def src_dir(self): return self.env.subst("$PROJECT_SRC_DIR") @property def test_dir(self): return self.env.subst("$PROJECT_TEST_DIR") def get_search_files(self): items = [] build_type = self.env["BUILD_TYPE"] # project files if "test" not in build_type or self.env.GetProjectOption("test_build_src"): items.extend(super().get_search_files()) # test files if "test" in build_type: items.extend( [ os.path.join("$PROJECT_TEST_DIR", item) for item in self.env.MatchSourceFiles( "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) ] ) return items @property def lib_ldf_mode(self): mode = LibBuilderBase.lib_ldf_mode.fget(self) # pylint: disable=no-member if not mode.startswith("chain"): return mode # parse all project files return "deep+" if "+" in mode else "deep" @property def src_filter(self): # pylint: disable=no-member return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self) @property def build_flags(self): # pylint: disable=no-member return self.env.get("SRC_BUILD_FLAGS") or LibBuilderBase.build_flags.fget(self) @property def dependencies(self): return self.env.GetProjectOption("lib_deps", []) def process_extra_options(self): with fs.cd(self.path): self.env.ProcessFlags(self.build_flags) self.env.ProcessUnFlags(self.build_unflags) def install_dependencies(self): def _is_builtin(spec): for lb in self.env.GetLibBuilders(): if lb.name == spec: return True return False not_found_specs = [] for spec in self.dependencies: # check if built-in library if _is_builtin(spec): continue found = False for storage_dir in self.env.GetLibSourceDirs(): lm = LibraryPackageManager(storage_dir) if lm.get_package(spec): found = True break if not found: not_found_specs.append(spec) did_install = False lm = LibraryPackageManager( self.env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) ) for spec in not_found_specs: try: lm.install(spec) did_install = True except ( HTTPClientError, UnknownPackageError, InternetConnectionError, ) as exc: click.secho("Warning! %s" % exc, fg="yellow") # reset cache if did_install: DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) def process_dependencies(self): # pylint: disable=too-many-branches found_lbs = [] for spec in self.dependencies: found = False for storage_dir in self.env.GetLibSourceDirs(): if found: break lm = LibraryPackageManager(storage_dir) pkg = lm.get_package(spec) if not pkg: continue for lb in self.env.GetLibBuilders(): if pkg.path != lb.path: continue if lb not in self.depbuilders: self.depend_on(lb, recursive=False) found_lbs.append(lb) found = True break if found: continue # look for built-in libraries by a name # which don't have package manifest for lb in self.env.GetLibBuilders(): if lb.name != spec: continue if lb not in self.depbuilders: self.depend_on(lb) found = True break # process library dependencies for lb in found_lbs: lb.search_deps_recursive() def build(self): self.is_built = True # do not build Project now result = super().build() self.env.PrependUnique(CPPPATH=self.get_include_dirs()) return result def GetLibSourceDirs(env): items = env.GetProjectOption("lib_extra_dirs", []) items.extend(env["LIBSOURCE_DIRS"]) return [ env.subst(fs.expanduser(item) if item.startswith("~") else item) for item in items ] def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): compat_mode = lb.lib_compat_mode if lb.name in env.GetProjectOption("lib_ignore", []): if verbose: sys.stderr.write("Ignored library %s\n" % lb.path) return None if compat_mode == "strict" and not lb.is_platforms_compatible(env["PIOPLATFORM"]): if verbose: sys.stderr.write("Platform incompatible library %s\n" % lb.path) return False if compat_mode in ("soft", "strict") and not lb.is_frameworks_compatible( env.get("PIOFRAMEWORK", "__noframework__") ): if verbose: sys.stderr.write("Framework incompatible library %s\n" % lb.path) return False return True def GetLibBuilders(_): # pylint: disable=too-many-branches env = DefaultEnvironment() if env.get("__PIO_LIB_BUILDERS", None) is not None: return sorted( env["__PIO_LIB_BUILDERS"], key=lambda lb: 0 if lb.is_dependent else 1, ) env.Replace(__PIO_LIB_BUILDERS=[]) verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) found_incompat = False for storage_dir in env.GetLibSourceDirs(): storage_dir = os.path.abspath(storage_dir) if not os.path.isdir(storage_dir): continue for item in sorted(os.listdir(storage_dir)): lib_dir = os.path.join(storage_dir, item) if item == "__cores__": continue if LibraryPackageManager.is_symlink(lib_dir): lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir) if not lib_dir or not os.path.isdir(lib_dir): continue try: lb = LibBuilderFactory.new(env, lib_dir) except exception.InvalidJSONFile: if verbose: sys.stderr.write( "Skip library with broken manifest: %s\n" % lib_dir ) continue if env.IsCompatibleLibBuilder(lb): env.Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True for lb in env.get("EXTRA_LIB_BUILDERS", []): if env.IsCompatibleLibBuilder(lb): env.Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True if verbose and found_incompat: sys.stderr.write( 'More details about "Library Compatibility Mode": ' "https://docs.platformio.org/page/librarymanager/ldf.html#" "ldf-compat-mode\n" ) return env["__PIO_LIB_BUILDERS"] def ConfigureProjectLibBuilder(env): _pm_storage = {} def _get_lib_license(pkg): storage_dir = os.path.dirname(os.path.dirname(pkg.path)) if storage_dir not in _pm_storage: _pm_storage[storage_dir] = LibraryPackageManager(storage_dir) try: return (_pm_storage[storage_dir].load_manifest(pkg) or {}).get("license") except MissingPackageManifestError: pass return None def _correct_found_libs(lib_builders): # build full dependency graph found_lbs = [lb for lb in lib_builders if lb.is_dependent] for lb in lib_builders: if lb in found_lbs: lb.search_deps_recursive(lb.get_search_files()) # refill found libs after recursive search found_lbs = [lb for lb in lib_builders if lb.is_dependent] for lb in lib_builders: for deplb in lb.depbuilders[:]: if deplb not in found_lbs: lb.depbuilders.remove(deplb) def _print_deps_tree(root, level=0): margin = "| " * (level) for lb in root.depbuilders: title = lb.name pkg = PackageItem(lb.path) if pkg.metadata: title += " @ %s" % pkg.metadata.version elif lb.version: title += " @ %s" % lb.version click.echo("%s|-- %s" % (margin, title), nl=False) if int(ARGUMENTS.get("PIOVERBOSE", 0)): click.echo( " (License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False ) if pkg.metadata and pkg.metadata.spec.external: click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False) click.echo("Path: %s" % lb.path, nl=False) click.echo(")", nl=False) click.echo("") if lb.verbose and lb.depbuilders: _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") if "test" in env["BUILD_TYPE"]: project.env.ConfigureTestTarget() ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf") click.echo( "LDF Modes: Finder ~ %s, Compatibility ~ %s" % (ldf_mode, project.lib_compat_mode) ) project.install_dependencies() lib_builders = env.GetLibBuilders() click.echo("Found %d compatible libraries" % len(lib_builders)) click.echo("Scanning dependencies...") project.search_deps_recursive() if ldf_mode.startswith("chain") and project.depbuilders: _correct_found_libs(lib_builders) if project.depbuilders: click.echo("Dependency Graph") _print_deps_tree(project) else: click.echo("No dependencies") return project def exists(_): return True def generate(env): env.AddMethod(GetLibSourceDirs) env.AddMethod(IsCompatibleLibBuilder) env.AddMethod(GetLibBuilders) env.AddMethod(ConfigureProjectLibBuilder) return env ================================================ FILE: platformio/builder/tools/piomaxlen.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import os import re from SCons.Platform import TempFileMunge # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Subst import quote_spaces # pylint: disable=import-error from platformio.compat import IS_WINDOWS, hashlib_encode_data # There are the next limits depending on a platform: # - Windows = 8191 # - Unix = 131072 # We need ~512 characters for compiler and temporary file paths MAX_LINE_LENGTH = (8191 if IS_WINDOWS else 131072) - 512 WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)") def tempfile_arg_esc_func(arg): arg = quote_spaces(arg) if not IS_WINDOWS: return arg # GCC requires double Windows slashes, let's use UNIX separator return WINPATHSEP_RE.sub(r"/\1", arg) def long_sources_hook(env, sources): _sources = str(sources).replace("\\", "/") if len(str(_sources)) < MAX_LINE_LENGTH: return sources # fix space in paths data = [] for line in _sources.split(".o "): line = line.strip() if not line.endswith(".o"): line += ".o" data.append('"%s"' % line) return '@"%s"' % _file_long_data(env, " ".join(data)) def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not os.path.isdir(build_dir): os.makedirs(build_dir) tmp_file = os.path.join( build_dir, "longcmd-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest() ) if os.path.isfile(tmp_file): return tmp_file with open(tmp_file, mode="w", encoding="utf8") as fp: fp.write(data) return tmp_file def exists(env): return "compiledb" not in COMMAND_LINE_TARGETS and not env.IsIntegrationDump() def generate(env): if not exists(env): return env kwargs = dict( _long_sources_hook=long_sources_hook, TEMPFILE=TempFileMunge, MAXLINELENGTH=MAX_LINE_LENGTH, TEMPFILEARGESCFUNC=tempfile_arg_esc_func, TEMPFILESUFFIX=".tmp", TEMPFILEDIR="$BUILD_DIR", ) for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"): kwargs[name] = "${TEMPFILE('%s','$%sSTR')}" % (env.get(name), name) kwargs["ARCOM"] = env.get("ARCOM", "").replace( "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" ) env.Replace(**kwargs) return env ================================================ FILE: platformio/builder/tools/piomisc.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys from platformio import fs, util from platformio.proc import exec_command @util.memoized() def GetCompilerType(env): # pylint: disable=too-many-return-statements CC = env.subst("$CC") if CC.endswith("-gcc"): return "gcc" if os.path.basename(CC) == "clang": return "clang" try: sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command([CC, "-v"], env=sysenv) except OSError: return None if result["returncode"] != 0: return None output = "".join([result["out"], result["err"]]).lower() if "clang version" in output: return "clang" if "gcc" in output: return "gcc" return None def GetActualLDScript(env): def _lookup_in_ldpath(script): for d in env.get("LIBPATH", []): path = os.path.join(env.subst(d), script) if os.path.isfile(path): return path return None script = None script_in_next = False for f in env.get("LINKFLAGS", []): raw_script = None if f == "-T": script_in_next = True continue if script_in_next: script_in_next = False raw_script = f elif f.startswith("-Wl,-T"): raw_script = f[6:] else: continue script = env.subst(raw_script.replace('"', "").strip()) if os.path.isfile(script): return script path = _lookup_in_ldpath(script) if path: return path if script: sys.stderr.write( "Error: Could not find '%s' LD script in LDPATH '%s'\n" % (script, env.subst("$LIBPATH")) ) env.Exit(1) if not script and "LDSCRIPT_PATH" in env: path = _lookup_in_ldpath(env["LDSCRIPT_PATH"]) if path: return path sys.stderr.write("Error: Could not find LD script\n") env.Exit(1) def ConfigureDebugTarget(env): def _cleanup_debug_flags(scope): if scope not in env: return unflags = ["-Os", "-g"] for level in [0, 1, 2, 3]: for flag in ("O", "g", "ggdb"): unflags.append("-%s%d" % (flag, level)) env[scope] = [f for f in env.get(scope, []) if f not in unflags] env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): _cleanup_debug_flags(scope) debug_flags = env.ParseFlags( env.get("PIODEBUGFLAGS") if env.get("PIODEBUGFLAGS") and not env.GetProjectOptions(as_dict=True).get("debug_build_flags") else env.GetProjectOption("debug_build_flags") ) env.MergeFlags(debug_flags) optimization_flags = [ f for f in debug_flags.get("CCFLAGS", []) if f.startswith(("-O", "-g")) ] if optimization_flags: env.AppendUnique( ASFLAGS=[ # skip -O flags for assembler f for f in optimization_flags if f.startswith("-g") ], LINKFLAGS=optimization_flags, ) def GetExtraScripts(env, scope): items = [] for item in env.GetProjectOption("extra_scripts", []): if scope == "post" and ":" not in item: items.append(item) elif item.startswith("%s:" % scope): items.append(item[len(scope) + 1 :]) if not items: return items with fs.cd(env.subst("$PROJECT_DIR")): return [os.path.abspath(env.subst(item)) for item in items] def generate(env): env.AddMethod(GetCompilerType) env.AddMethod(GetActualLDScript) env.AddMethod(ConfigureDebugTarget) env.AddMethod(GetExtraScripts) # backward-compatibility with Zephyr build script env.AddMethod(ConfigureDebugTarget, "ConfigureDebugFlags") def exists(_): return True ================================================ FILE: platformio/builder/tools/pioplatform.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import fs, util from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.package.meta import PackageItem from platformio.package.version import get_original_version from platformio.platform.exception import UnknownBoard from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @util.memoized() def _PioPlatform(): env = DefaultEnvironment() return PlatformFactory.from_env(env["PIOENV"], targets=COMMAND_LINE_TARGETS) def PioPlatform(_): return _PioPlatform() def BoardConfig(env, board=None): with fs.cd(env.subst("$PROJECT_DIR")): try: p = env.PioPlatform() board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" return p.board_config(board) except (AssertionError, UnknownBoard) as exc: sys.stderr.write("Error: %s\n" % str(exc)) env.Exit(1) return None def GetFrameworkScript(env, framework): p = env.PioPlatform() assert p.frameworks and framework in p.frameworks script_path = env.subst(p.frameworks[framework]["script"]) if not os.path.isfile(script_path): script_path = os.path.join(p.get_dir(), script_path) return script_path def LoadPioPlatform(env): p = env.PioPlatform() # Ensure real platform name env["PIOPLATFORM"] = p.name # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH for pkg in p.get_installed_packages(): type_ = p.get_package_type(pkg.metadata.name) if type_ not in ("toolchain", "uploader", "debugger"): continue env.PrependENVPath( "PATH", ( os.path.join(pkg.path, "bin") if os.path.isdir(os.path.join(pkg.path, "bin")) else pkg.path ), ) if ( not IS_WINDOWS and os.path.isdir(os.path.join(pkg.path, "lib")) and type_ != "toolchain" ): env.PrependENVPath( "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH", os.path.join(pkg.path, "lib"), ) # Platform specific LD Scripts if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")): env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: return # update board manifest with overridden data from INI config board_config = env.BoardConfig() for option, value in env.GetProjectOptions(): if not option.startswith("board_"): continue option = option.lower()[6:] try: if isinstance(board_config.get(option), bool): value = str(value).lower() in ("1", "yes", "true") elif isinstance(board_config.get(option), int): value = int(value) except KeyError: pass board_config.update(option, value) # load default variables from board config for option_meta in ProjectOptions.values(): if not option_meta.buildenvvar or option_meta.buildenvvar in env: continue data_path = ( option_meta.name[6:] if option_meta.name.startswith("board_") else option_meta.name.replace("_", ".") ) try: env[option_meta.buildenvvar] = board_config.get(data_path) except KeyError: pass if "build.ldscript" in board_config: env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript")) def PrintConfiguration(env): # pylint: disable=too-many-statements platform = env.PioPlatform() pkg_metadata = PackageItem(platform.get_dir()).metadata board_config = env.BoardConfig() if "BOARD" in env else None def _get_configuration_data(): return ( None if not board_config else [ "CONFIGURATION:", "https://docs.platformio.org/page/boards/%s/%s.html" % (platform.name, board_config.id), ] ) def _get_plaform_data(): data = [ "PLATFORM: %s (%s)" % ( platform.title, pkg_metadata.version if pkg_metadata else platform.version, ) ] if ( int(ARGUMENTS.get("PIOVERBOSE", 0)) and pkg_metadata and pkg_metadata.spec.external ): data.append("(%s)" % pkg_metadata.spec.uri) if board_config: data.extend([">", board_config.get("name")]) return data def _get_hardware_data(): data = ["HARDWARE:"] mcu = env.subst("$BOARD_MCU") f_cpu = env.subst("$BOARD_F_CPU") if mcu: data.append(mcu.upper()) if f_cpu: f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()])) data.append("%dMHz," % (f_cpu / 1000000)) if not board_config: return data ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") data.append( "%s RAM, %s Flash" % (fs.humanize_file_size(ram), fs.humanize_file_size(flash)) ) return data def _get_debug_data(): debug_tools = ( board_config.get("debug", {}).get("tools") if board_config else None ) if not debug_tools: return None data = [ "DEBUG:", "Current", "(%s)" % board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")), ] onboard = [] external = [] for key, value in debug_tools.items(): if value.get("onboard"): onboard.append(key) else: external.append(key) if onboard: data.extend(["On-board", "(%s)" % ", ".join(sorted(onboard))]) if external: data.extend(["External", "(%s)" % ", ".join(sorted(external))]) return data def _get_packages_data(): data = [] for item in platform.dump_used_packages(): original_version = get_original_version(item["version"]) info = "%s @ %s" % (item["name"], item["version"]) extra = [] if original_version: extra.append(original_version) if "src_url" in item and int(ARGUMENTS.get("PIOVERBOSE", 0)): extra.append(item["src_url"]) if extra: info += " (%s)" % ", ".join(extra) data.append(info) if not data: return None return ["PACKAGES:"] + ["\n - %s" % d for d in sorted(data)] for data in ( _get_configuration_data(), _get_plaform_data(), _get_hardware_data(), _get_debug_data(), _get_packages_data(), ): if data and len(data) > 1: print(" ".join(data)) def exists(_): return True def generate(env): env.AddMethod(PioPlatform) env.AddMethod(BoardConfig) env.AddMethod(GetFrameworkScript) env.AddMethod(LoadPioPlatform) env.AddMethod(PrintConfiguration) return env ================================================ FILE: platformio/builder/tools/pioproject.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.compat import MISSING from platformio.project.config import ProjectConfig def GetProjectConfig(env): return ProjectConfig.get_instance(env["PROJECT_CONFIG"]) def GetProjectOptions(env, as_dict=False): return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict) def GetProjectOption(env, option, default=MISSING): return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default) def LoadProjectOptions(env): config = env.GetProjectConfig() section = "env:" + env["PIOENV"] for option in config.options(section): option_meta = config.find_option_meta(section, option) if ( not option_meta or not option_meta.buildenvvar or option_meta.buildenvvar in env ): continue env[option_meta.buildenvvar] = config.get(section, option) def exists(_): return True def generate(env): env.AddMethod(GetProjectConfig) env.AddMethod(GetProjectOptions) env.AddMethod(GetProjectOption) env.AddMethod(LoadProjectOptions) return env ================================================ FILE: platformio/builder/tools/piosize.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-locals import json import sys from os import environ, makedirs, remove from os.path import isdir, join, splitdrive from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile from platformio.compat import IS_WINDOWS from platformio.proc import exec_command def _run_tool(cmd, env, tool_args): sysenv = environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) tmp_file = join(build_dir, "size-data-longcmd.txt") with open(tmp_file, mode="w", encoding="utf8") as fp: fp.write("\n".join(tool_args)) cmd.append("@" + tmp_file) result = exec_command(cmd, env=sysenv) remove(tmp_file) return result def _get_symbol_locations(env, elf_path, addrs): if not addrs: return {} cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path] result = _run_tool(cmd, env, addrs) locations = [line for line in result["out"].split("\n") if line] assert len(addrs) == len(locations) return dict(zip(addrs, [loc.strip() for loc in locations])) def _get_demangled_names(env, mangled_names): if not mangled_names: return {} result = _run_tool( [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names ) demangled_names = [line for line in result["out"].split("\n") if line] assert len(mangled_names) == len(demangled_names) return dict( zip( mangled_names, [dn.strip().replace("::__FUNCTION__", "") for dn in demangled_names], ) ) def _collect_sections_info(env, elffile): sections = {} for section in elffile.iter_sections(): if section.is_null() or section.name.startswith(".debug"): continue section_type = section["sh_type"] section_flags = describe_sh_flags(section["sh_flags"]) section_size = section.data_size section_data = { "name": section.name, "size": section_size, "start_addr": section["sh_addr"], "type": section_type, "flags": section_flags, } sections[section.name] = section_data sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data) sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data) return sections def _collect_symbols_info(env, elffile, elf_path, sections): symbols = [] symbol_section = elffile.get_section_by_name(".symtab") if symbol_section.is_null(): sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?") env.Exit(1) sysenv = environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) symbol_addrs = [] mangled_names = [] for s in symbol_section.iter_symbols(): symbol_info = s.entry["st_info"] symbol_addr = s["st_value"] symbol_size = s["st_size"] symbol_type = symbol_info["type"] if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr): continue symbol = { "addr": symbol_addr, "bind": symbol_info["bind"], "name": s.name, "type": symbol_type, "size": symbol_size, "section": env.pioSizeDetermineSection(sections, symbol_addr), } if s.name.startswith("_Z"): mangled_names.append(s.name) symbol_addrs.append(hex(symbol_addr)) symbols.append(symbol) symbol_locations = _get_symbol_locations(env, elf_path, symbol_addrs) demangled_names = _get_demangled_names(env, mangled_names) for symbol in symbols: if symbol["name"].startswith("_Z"): symbol["demangled_name"] = demangled_names.get(symbol["name"]) location = symbol_locations.get(hex(symbol["addr"])) if not location or "?" in location: continue if IS_WINDOWS: drive, tail = splitdrive(location) location = join(drive.upper(), tail) symbol["file"] = location symbol["line"] = 0 if ":" in location: file_, line = location.rsplit(":", 1) if line.isdigit(): symbol["file"] = file_ symbol["line"] = int(line) return symbols def pioSizeDetermineSection(_, sections, symbol_addr): for section, info in sections.items(): if not info.get("in_flash", False) and not info.get("in_ram", False): continue if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]): return section return "unknown" def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address): return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE" def pioSizeIsRamSection(_, section): return ( section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS") and section.get("flags", "") == "WA" ) def pioSizeIsFlashSection(_, section): return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "") def pioSizeCalculateFirmwareSize(_, sections): flash_size = ram_size = 0 for section_info in sections.values(): if section_info.get("in_flash", False): flash_size += section_info.get("size", 0) if section_info.get("in_ram", False): ram_size += section_info.get("size", 0) return ram_size, flash_size def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument data = {"device": {}, "memory": {}, "version": 1} board = env.BoardConfig() if board: data["device"] = { "mcu": board.get("build.mcu", ""), "cpu": board.get("build.cpu", ""), "frequency": board.get("build.f_cpu"), "flash": int(board.get("upload.maximum_size", 0)), "ram": int(board.get("upload.maximum_ram_size", 0)), } if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"): data["device"]["frequency"] = int(data["device"]["frequency"][0:-1]) elf_path = env.subst("$PIOMAINPROG") with open(elf_path, "rb") as fp: elffile = ELFFile(fp) if not elffile.has_dwarf_info(): sys.stderr.write("Elf file doesn't contain DWARF information") env.Exit(1) sections = _collect_sections_info(env, elffile) firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections) data["memory"]["total"] = { "ram_size": firmware_ram, "flash_size": firmware_flash, "sections": sections, } files = {} for symbol in _collect_symbols_info(env, elffile, elf_path, sections): file_path = symbol.get("file") or "unknown" if not files.get(file_path, {}): files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0} symbol_size = symbol.get("size", 0) section = sections.get(symbol.get("section", ""), {}) if not section: continue if section.get("in_ram", False): files[file_path]["ram_size"] += symbol_size if section.get("in_flash", False): files[file_path]["flash_size"] += symbol_size files[file_path]["symbols"].append(symbol) data["memory"]["files"] = [] for k, v in files.items(): file_data = {"path": k} file_data.update(v) data["memory"]["files"].append(file_data) with open( join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8" ) as fp: fp.write(json.dumps(data)) def exists(_): return True def generate(env): env.AddMethod(pioSizeIsRamSection) env.AddMethod(pioSizeIsFlashSection) env.AddMethod(pioSizeCalculateFirmwareSize) env.AddMethod(pioSizeDetermineSection) env.AddMethod(pioSizeIsValidSymbol) env.AddMethod(DumpSizeData) return env ================================================ FILE: platformio/builder/tools/piotarget.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from platformio import compat, fs def VerboseAction(_, act, actstr): if int(ARGUMENTS.get("PIOVERBOSE", 0)): return act return Action(act, actstr) def IsCleanTarget(env): return env.GetOption("clean") def CleanProject(env, fullclean=False): def _relpath(path): if compat.IS_WINDOWS: prefix = os.getcwd()[:2].lower() if ( ":" not in prefix or not path.lower().startswith(prefix) or os.path.relpath(path).startswith("..") ): return path return os.path.relpath(path) def _clean_dir(path): clean_rel_path = _relpath(path) print(f"Removing {clean_rel_path}") fs.rmtree(path) build_dir = env.subst("$BUILD_DIR") libdeps_dir = env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) if os.path.isdir(build_dir): _clean_dir(build_dir) else: print("Build environment is clean") if fullclean and os.path.isdir(libdeps_dir): _clean_dir(libdeps_dir) print("Done cleaning") def AddTarget( # pylint: disable=too-many-arguments,too-many-positional-arguments env, name, dependencies, actions, title=None, description=None, group="General", always_build=True, ): if "__PIO_TARGETS" not in env: env["__PIO_TARGETS"] = {} assert name not in env["__PIO_TARGETS"] env["__PIO_TARGETS"][name] = dict( name=name, title=title, description=description, group=group ) target = env.Alias(name, dependencies, actions) if always_build: AlwaysBuild(target) return target def AddPlatformTarget(env, *args, **kwargs): return env.AddTarget(group="Platform", *args, **kwargs) def AddCustomTarget(env, *args, **kwargs): return env.AddTarget(group="Custom", *args, **kwargs) def DumpTargets(env): targets = env.get("__PIO_TARGETS") or {} # pre-fill default targets if embedded dev-platform if env.PioPlatform().is_embedded() and not any( t["group"] == "Platform" for t in targets.values() ): targets["upload"] = dict(name="upload", group="Platform", title="Upload") return list(targets.values()) def exists(_): return True def generate(env): env.AddMethod(VerboseAction) env.AddMethod(IsCleanTarget) env.AddMethod(CleanProject) env.AddMethod(AddTarget) env.AddMethod(AddPlatformTarget) env.AddMethod(AddCustomTarget) env.AddMethod(DumpTargets) return env ================================================ FILE: platformio/builder/tools/piotest.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from platformio.builder.tools import piobuild from platformio.test.result import TestSuite from platformio.test.runners.factory import TestRunnerFactory def ConfigureTestTarget(env): env.Append( CPPDEFINES=["UNIT_TEST"], # deprecated, use PIO_UNIT_TESTING PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piobuild.SRC_BUILD_EXT], ) env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"]) if "PIOTEST_RUNNING_NAME" in env: test_name = env["PIOTEST_RUNNING_NAME"] while True: test_name = os.path.dirname(test_name) # parent dir # skip nested tests (user's side issue?) if not test_name or os.path.basename(test_name).startswith("test_"): break env.Prepend( PIOTEST_SRC_FILTER=[ f"+<{test_name}{os.path.sep}*.{ext}>" for ext in piobuild.SRC_BUILD_EXT ], CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)], ) env.Prepend( PIOTEST_SRC_FILTER=[f"+<$PIOTEST_RUNNING_NAME{os.path.sep}>"], CPPPATH=[os.path.join("$PROJECT_TEST_DIR", "$PIOTEST_RUNNING_NAME")], ) test_runner = TestRunnerFactory.new( TestSuite(env["PIOENV"], env.get("PIOTEST_RUNNING_NAME", "*")), env.GetProjectConfig(), ) test_runner.configure_build_env(env) def generate(env): env.AddMethod(ConfigureTestTarget) def exists(_): return True ================================================ FILE: platformio/builder/tools/pioupload.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import os import re import sys from shutil import copyfile from time import sleep from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException from platformio import exception, fs from platformio.device.finder import SerialPortFinder, find_mbed_disk, is_pattern_port from platformio.device.list.util import list_serial_ports from platformio.proc import exec_command def FlushSerialBuffer(env, port): s = Serial(env.subst(port)) s.flushInput() s.setDTR(False) s.setRTS(False) sleep(0.1) s.setDTR(True) s.setRTS(True) s.close() def TouchSerialPort(env, port, baudrate): port = env.subst(port) print("Forcing reset using %dbps open/close on port %s" % (baudrate, port)) try: s = Serial(port=port, baudrate=baudrate) s.setDTR(False) s.close() except: # pylint: disable=bare-except pass sleep(0.4) # DO NOT REMOVE THAT (required by SAM-BA based boards) def WaitForNewSerialPort(env, before): print("Waiting for the new upload port...") prev_port = env.subst("$UPLOAD_PORT") new_port = None elapsed = 0 before = [p["port"] for p in before] while elapsed < 5 and new_port is None: now = [p["port"] for p in list_serial_ports()] for p in now: if p not in before: new_port = p break before = now sleep(0.25) elapsed += 0.25 if not new_port: for p in now: if prev_port == p: new_port = p break try: s = Serial(new_port) s.close() except SerialException: sleep(1) if not new_port: sys.stderr.write( "Error: Couldn't find a board on the selected port. " "Check that you have the correct port selected. " "If it is correct, try pressing the board's reset " "button after initiating the upload.\n" ) env.Exit(1) return new_port def AutodetectUploadPort(*args, **kwargs): env = args[0] initial_port = env.subst("$UPLOAD_PORT") upload_protocol = env.subst("$UPLOAD_PROTOCOL") if initial_port and not is_pattern_port(initial_port): print(env.subst("Using manually specified: $UPLOAD_PORT")) return if upload_protocol == "mbed" or ( "mbed" in env.subst("$PIOFRAMEWORK") and not upload_protocol ): env.Replace(UPLOAD_PORT=find_mbed_disk(initial_port)) else: try: fs.ensure_udev_rules() except exception.InvalidUdevRules as exc: sys.stderr.write("\n%s\n\n" % exc) env.Replace( UPLOAD_PORT=SerialPortFinder( board_config=env.BoardConfig() if "BOARD" in env else None, upload_protocol=upload_protocol, prefer_gdb_port="blackmagic" in upload_protocol, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)), ).find(initial_port) ) if env.subst("$UPLOAD_PORT"): print(env.subst("Auto-detected: $UPLOAD_PORT")) else: sys.stderr.write( "Error: Please specify `upload_port` for environment or use " "global `--upload-port` option.\n" "For some development platforms it can be a USB flash " "drive (i.e. /media//)\n" ) env.Exit(1) def UploadToDisk(_, target, source, env): assert "UPLOAD_PORT" in env progname = env.subst("$PROGNAME") for ext in ("bin", "hex"): fpath = os.path.join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext)) if not os.path.isfile(fpath): continue copyfile( fpath, os.path.join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)) ) print( "Firmware has been successfully uploaded.\n" "(Some boards may require manual hard reset)" ) def CheckUploadSize(_, target, source, env): check_conditions = [ env.get("BOARD"), env.get("SIZETOOL") or env.get("SIZECHECKCMD"), ] if not all(check_conditions): return program_max_size = int(env.BoardConfig().get("upload.maximum_size", 0)) data_max_size = int(env.BoardConfig().get("upload.maximum_ram_size", 0)) if program_max_size == 0: return def _configure_defaults(): env.Replace( SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+", ) def _get_size_output(): cmd = env.get("SIZECHECKCMD") if not cmd: return None if not isinstance(cmd, list): cmd = cmd.split() cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg] sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command(env.subst(cmd), env=sysenv) if result["returncode"] != 0: return None return result["out"].strip() def _calculate_size(output, pattern): if not output or not pattern: return -1 size = 0 regexp = re.compile(pattern) for line in output.split("\n"): line = line.strip() if not line: continue match = regexp.search(line) if not match: continue size += sum(int(value) for value in match.groups()) return size def _format_availale_bytes(value, total): percent_raw = float(value) / float(total) blocks_per_progress = 10 used_blocks = min( int(round(blocks_per_progress * percent_raw)), blocks_per_progress ) return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format( "=" * used_blocks, blocks_per_progress, percent_raw, value, total ) if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"): _configure_defaults() output = _get_size_output() program_size = _calculate_size(output, env.get("SIZEPROGREGEXP")) data_size = _calculate_size(output, env.get("SIZEDATAREGEXP")) print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"') if data_max_size and data_size > -1: print("RAM: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: print("Flash: %s" % _format_availale_bytes(program_size, program_max_size)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): print(output) if data_max_size and data_size > data_max_size: sys.stderr.write( "Warning! The data size (%d bytes) is greater " "than maximum allowed (%s bytes)\n" % (data_size, data_max_size) ) if program_size > program_max_size: sys.stderr.write( "Error: The program size (%d bytes) is greater " "than maximum allowed (%s bytes)\n" % (program_size, program_max_size) ) env.Exit(1) def PrintUploadInfo(env): configured = env.subst("$UPLOAD_PROTOCOL") available = [configured] if configured else [] if "BOARD" in env: available.extend(env.BoardConfig().get("upload", {}).get("protocols", [])) if available: print("AVAILABLE: %s" % ", ".join(sorted(set(available)))) if configured: print("CURRENT: upload_protocol = %s" % configured) def exists(_): return True def generate(env): env.AddMethod(FlushSerialBuffer) env.AddMethod(TouchSerialPort) env.AddMethod(WaitForNewSerialPort) env.AddMethod(AutodetectUploadPort) env.AddMethod(UploadToDisk) env.AddMethod(CheckUploadSize) env.AddMethod(PrintUploadInfo) return env ================================================ FILE: platformio/cache.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import os from time import time from platformio import app, fs from platformio.compat import hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import get_project_cache_dir class ContentCache: def __init__(self, namespace=None): self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content") self._db_path = os.path.join(self.cache_dir, "db.data") self._lockfile = None if not os.path.isdir(self.cache_dir): os.makedirs(self.cache_dir) def __enter__(self): # cleanup obsolete items self.delete() return self def __exit__(self, type_, value, traceback): pass @staticmethod def key_from_args(*args): h = hashlib.sha1() for arg in args: if arg: h.update(hashlib_encode_data(arg)) return h.hexdigest() def get_cache_path(self, key): assert "/" not in key and "\\" not in key key = str(key) assert len(key) > 3 return os.path.join(self.cache_dir, key) def get(self, key): cache_path = self.get_cache_path(key) if not os.path.isfile(cache_path): return None with open(cache_path, "r", encoding="utf8") as fp: return fp.read() def set(self, key, data, valid): if not app.get_setting("enable_cache"): return False cache_path = self.get_cache_path(key) if os.path.isfile(cache_path): self.delete(key) if not data: return False tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} assert valid.endswith(tuple(tdmap)) expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) if not self._lock_dbindex(): return False if not os.path.isdir(os.path.dirname(cache_path)): os.makedirs(os.path.dirname(cache_path)) try: with open(cache_path, mode="w", encoding="utf8") as fp: fp.write(data) with open(self._db_path, mode="a", encoding="utf8") as fp: fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path))) except UnicodeError: if os.path.isfile(cache_path): try: os.remove(cache_path) except OSError: pass return self._unlock_dbindex() def delete(self, keys=None): """Keys=None, delete expired items""" if not os.path.isfile(self._db_path): return None if not keys: keys = [] if not isinstance(keys, list): keys = [keys] paths_for_delete = [self.get_cache_path(k) for k in keys] found = False newlines = [] with open(self._db_path, encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if "=" not in line: continue expire, fname = line.split("=") path = os.path.join(self.cache_dir, fname) try: if ( time() < int(expire) and os.path.isfile(path) and path not in paths_for_delete ): newlines.append(line) continue except ValueError: pass found = True if os.path.isfile(path): try: os.remove(path) if not os.listdir(os.path.dirname(path)): fs.rmtree(os.path.dirname(path)) except OSError: pass if found and self._lock_dbindex(): with open(self._db_path, mode="w", encoding="utf8") as fp: fp.write("\n".join(newlines) + "\n") self._unlock_dbindex() return True def clean(self): if not os.path.isdir(self.cache_dir): return fs.rmtree(self.cache_dir) def _lock_dbindex(self): self._lockfile = LockFile(self.cache_dir) try: self._lockfile.acquire() except: # pylint: disable=bare-except return False return True def _unlock_dbindex(self): if self._lockfile: self._lockfile.release() return True # # Helpers # def cleanup_content_cache(namespace=None): with ContentCache(namespace) as cc: cc.clean() ================================================ FILE: platformio/check/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/check/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-arguments,too-many-locals,too-many-branches # pylint: disable=redefined-builtin,too-many-statements import json import os import shutil from collections import Counter from time import time import click from tabulate import tabulate from platformio import app, exception, fs, util from platformio.check.defect import DefectItem from platformio.check.tools import CheckToolFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, get_project_dir @click.command("check", short_help="Static Code Analysis") @click.option("-e", "--environment", multiple=True) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--pattern", multiple=True, hidden=True) @click.option("-f", "--src-filters", multiple=True) @click.option("--flags", multiple=True) @click.option( "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) ) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--json-output", is_flag=True) @click.option( "--fail-on-defect", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()), ) @click.option("--skip-packages", is_flag=True) def cli( # pylint: disable=too-many-positional-arguments environment, project_dir, project_conf, src_filters, pattern, flags, severity, silent, verbose, json_output, fail_on_defect, skip_packages, ): app.set_session_var("custom_project_conf", project_conf) # find project directory on upper level if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) results = [] with fs.cd(project_dir): config = ProjectConfig.get_instance(project_conf) config.validate(environment) default_envs = config.default_envs() for envname in config.envs(): skipenv = any( [ environment and envname not in environment, not environment and default_envs and envname not in default_envs, ] ) env_options = config.items(env=envname, as_dict=True) env_dump = [] for k, v in env_options.items(): if k not in ("platform", "framework", "board"): continue env_dump.append( "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) ) default_src_filters = [] for d in ( config.get("platformio", "src_dir"), config.get("platformio", "include_dir"), ): try: default_src_filters.append("+<%s>" % os.path.relpath(d)) except ValueError as exc: # On Windows if sources are located on a different logical drive if not json_output and not silent: click.echo( "Error: Project cannot be analyzed! The project folder `%s`" " is located on a different logical drive\n" % d ) raise exception.ReturnErrorCode(1) from exc env_src_filters = ( src_filters or pattern or env_options.get( "check_src_filters", env_options.get("check_patterns", default_src_filters), ) ) tool_options = dict( verbose=verbose, silent=silent, src_filters=env_src_filters, flags=flags or env_options.get("check_flags"), severity=( [DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent else severity or config.get("env:" + envname, "check_severity") ), skip_packages=skip_packages or env_options.get("check_skip_packages"), platform_packages=env_options.get("platform_packages"), ) for tool in config.get("env:" + envname, "check_tool"): if skipenv: results.append({"env": envname, "tool": tool}) continue if not silent and not json_output: print_processing_header(tool, envname, env_dump) ct = CheckToolFactory.new( tool, os.getcwd(), config, envname, tool_options ) result = {"env": envname, "tool": tool, "duration": time()} rc = ct.check( on_defect_callback=( None if (json_output or verbose) else lambda defect: click.echo(repr(defect)) ) ) result["defects"] = ct.get_defects() result["duration"] = time() - result["duration"] result["succeeded"] = rc == 0 if fail_on_defect: result["succeeded"] = rc == 0 and not any( DefectItem.SEVERITY_LABELS[d.severity] in fail_on_defect for d in result["defects"] ) result["stats"] = collect_component_stats(result) results.append(result) if verbose: click.echo("\n".join(repr(d) for d in result["defects"])) if not json_output and not silent: if rc != 0: click.echo( "Error: %s failed to perform check! Please " "examine tool output in verbose mode." % tool ) elif not result["defects"]: click.echo("No defects found") print_processing_footer(result) if json_output: click.echo(json.dumps(results_to_json(results))) elif not silent: print_check_summary(results, verbose=verbose) # Reset custom project config app.set_session_var("custom_project_conf", None) command_failed = any(r.get("succeeded") is False for r in results) if command_failed: raise exception.ReturnErrorCode(1) def results_to_json(raw): results = [] for item in raw: if item.get("succeeded") is None: continue item.update( { "succeeded": bool(item.get("succeeded")), "defects": [d.as_dict() for d in item.get("defects", [])], } ) results.append(item) return results def print_processing_header(tool, envname, envdump): click.echo( "Checking %s > %s (%s)" % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) ) terminal_width = shutil.get_terminal_size().columns click.secho("-" * terminal_width, bold=True) def print_processing_footer(result): is_failed = not result.get("succeeded") util.print_labeled_bar( "[%s] Took %.2f seconds" % ( ( click.style("FAILED", fg="red", bold=True) if is_failed else click.style("PASSED", fg="green", bold=True) ), result["duration"], ), is_error=is_failed, ) def collect_component_stats(result): components = {} def _append_defect(component, defect): if not components.get(component): components[component] = Counter() components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1}) for defect in result.get("defects", []): component = os.path.dirname(defect.file) or defect.file _append_defect(component, defect) if component.lower().startswith(get_project_dir().lower()): while os.sep in component: component = os.path.dirname(component) _append_defect(component, defect) return components def print_defects_stats(results): if not results: return component_stats = {} for r in results: for k, v in r.get("stats", {}).items(): if not component_stats.get(k): component_stats[k] = Counter() component_stats[k].update(r["stats"][k]) if not component_stats: return severity_labels = list(DefectItem.SEVERITY_LABELS.values()) severity_labels.reverse() tabular_data = [] for k, v in component_stats.items(): tool_defect = [v.get(s, 0) for s in severity_labels] tabular_data.append([k] + tool_defect) total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]] tabular_data.sort() tabular_data.append([]) # Empty line as delimiter tabular_data.append(total) headers = ["Component"] headers.extend([label.upper() for label in severity_labels]) headers = [click.style(h, bold=True) for h in headers] click.echo(tabulate(tabular_data, headers=headers, numalign="center")) click.echo() def print_check_summary(results, verbose=False): click.echo() tabular_data = [] succeeded_nums = 0 failed_nums = 0 duration = 0 print_defects_stats(results) for result in results: duration += result.get("duration", 0) if result.get("succeeded") is False: failed_nums += 1 status_str = click.style("FAILED", fg="red") elif result.get("succeeded") is None: status_str = "IGNORED" if not verbose: continue else: succeeded_nums += 1 status_str = click.style("PASSED", fg="green") tabular_data.append( ( click.style(result["env"], fg="cyan"), result["tool"], status_str, util.humanize_duration_time(result.get("duration")), ) ) click.echo( tabulate( tabular_data, headers=[ click.style(s, bold=True) for s in ("Environment", "Tool", "Status", "Duration") ], ), err=failed_nums, ) util.print_labeled_bar( "%s%d succeeded in %s" % ( "%d failed, " % failed_nums if failed_nums else "", succeeded_nums, util.humanize_duration_time(duration), ), is_error=failed_nums, fg="red" if failed_nums else "green", ) ================================================ FILE: platformio/check/defect.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import click from platformio.exception import PlatformioException from platformio.project.helpers import get_project_dir # pylint: disable=too-many-instance-attributes, redefined-builtin # pylint: disable=too-many-arguments class DefectItem: SEVERITY_HIGH = 1 SEVERITY_MEDIUM = 2 SEVERITY_LOW = 4 SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} def __init__( # pylint: disable=too-many-positional-arguments self, severity, category, message, file=None, line=0, column=0, id=None, callstack=None, cwe=None, ): assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW) self.severity = severity self.category = category self.message = message self.line = int(line) self.column = int(column) self.callstack = callstack self.cwe = cwe self.id = id self.file = file or "unknown" if file.lower().startswith(get_project_dir().lower()): self.file = os.path.relpath(file, get_project_dir()) def __repr__(self): defect_color = None if self.severity == self.SEVERITY_HIGH: defect_color = "red" elif self.severity == self.SEVERITY_MEDIUM: defect_color = "yellow" format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" return format_str.format( severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color), category=click.style(self.category.lower(), fg=defect_color), file=click.style(self.file, bold=True), message=self.message, line=self.line, id="%s" % "[%s]" % self.id if self.id else "", ) def __or__(self, defect): return self.severity | defect.severity @staticmethod def severity_to_int(label): for key, value in DefectItem.SEVERITY_LABELS.items(): if label == value: return key raise PlatformioException("Unknown severity label -> %s" % label) def as_dict(self): return { "severity": self.SEVERITY_LABELS[self.severity], "category": self.category, "message": self.message, "file": os.path.abspath(self.file), "line": self.line, "column": self.column, "callstack": self.callstack, "id": self.id, "cwe": self.cwe, } ================================================ FILE: platformio/check/tools/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio import exception from platformio.check.tools.clangtidy import ClangtidyCheckTool from platformio.check.tools.cppcheck import CppcheckCheckTool from platformio.check.tools.pvsstudio import PvsStudioCheckTool class CheckToolFactory: @staticmethod def new(tool, project_dir, config, envname, options): cls = None if tool == "cppcheck": cls = CppcheckCheckTool elif tool == "clangtidy": cls = ClangtidyCheckTool elif tool == "pvs-studio": cls = PvsStudioCheckTool else: raise exception.PlatformioException("Unknown check tool `%s`" % tool) return cls(project_dir, config, envname, options) ================================================ FILE: platformio/check/tools/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import tempfile import click from platformio import fs, proc from platformio.check.defect import DefectItem from platformio.package.manager.core import get_core_package_dir from platformio.package.meta import PackageSpec from platformio.project.helpers import load_build_metadata class CheckToolBase: # pylint: disable=too-many-instance-attributes def __init__(self, project_dir, config, envname, options): self.config = config self.envname = envname self.options = options self.project_dir = project_dir self.cc_flags = [] self.cxx_flags = [] self.cpp_includes = [] self.cpp_defines = [] self.toolchain_defines = [] self._tmp_files = [] self.cc_path = None self.cxx_path = None self._defects = [] self._on_defect_callback = None self._bad_input = False self._load_cpp_data() # detect all defects by default if not self.options.get("severity"): self.options["severity"] = [ DefectItem.SEVERITY_LOW, DefectItem.SEVERITY_MEDIUM, DefectItem.SEVERITY_HIGH, ] # cast to severity by ids self.options["severity"] = [ s if isinstance(s, int) else DefectItem.severity_to_int(s) for s in self.options["severity"] ] def _load_cpp_data(self): data = load_build_metadata(self.project_dir, self.envname) if not data: return self.cc_flags = data.get("cc_flags", []) self.cxx_flags = data.get("cxx_flags", []) self.cpp_includes = self._dump_includes(data.get("includes", {})) self.cpp_defines = data.get("defines", []) self.cc_path = data.get("cc_path") self.cxx_path = data.get("cxx_path") self.toolchain_defines = self._get_toolchain_defines() def get_tool_dir(self, pkg_name): for spec in self.options["platform_packages"] or []: spec = PackageSpec(spec) if spec.name == pkg_name: return get_core_package_dir(pkg_name, spec=spec) return get_core_package_dir(pkg_name) def get_flags(self, tool): result = [] flags = self.options.get("flags") or [] for flag in flags: if ":" not in flag or flag.startswith("-"): result.extend([f for f in flag.split(" ") if f]) elif flag.startswith("%s:" % tool): result.extend([f for f in flag.split(":", 1)[1].split(" ") if f]) return result def _get_toolchain_defines(self): def _extract_defines(language, includes_file): build_flags = self.cxx_flags if language == "c++" else self.cc_flags defines = [] cmd = 'echo | "%s" -x %s %s %s -dM -E -' % ( self.cc_path, language, " ".join( [f for f in build_flags if f.startswith(("-m", "-f", "-std"))] ), includes_file, ) result = proc.exec_command(cmd, shell=True) if result["returncode"] != 0: click.echo("Warning: Failed to extract toolchain defines!") if self.options.get("verbose"): click.echo(result["out"]) click.echo(result["err"]) for line in result["out"].split("\n"): tokens = line.strip().split(" ", 2) if not tokens or tokens[0] != "#define": continue if len(tokens) > 2: defines.append("%s=%s" % (tokens[1], tokens[2])) else: defines.append(tokens[1]) return defines incflags_file = self._long_includes_hook(self.cpp_includes) return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")} def _create_tmp_file(self, data): with tempfile.NamedTemporaryFile("w", delete=False) as fp: fp.write(data) self._tmp_files.append(fp.name) return fp.name def _long_includes_hook(self, includes): data = [] for inc in includes: data.append('-I"%s"' % fs.to_unix_path(inc)) return '@"%s"' % self._create_tmp_file(" ".join(data)) @staticmethod def _dump_includes(includes_map): result = [] for includes in includes_map.values(): for include in includes: if include not in result: result.append(include) return result @staticmethod def is_flag_set(flag, flags): return any(flag in f for f in flags) def get_defects(self): return self._defects def configure_command(self): raise NotImplementedError def on_tool_output(self, line): line = self.tool_output_filter(line) if not line: return defect = self.parse_defect(line) if not isinstance(defect, DefectItem): if self.options.get("verbose"): click.echo(line) return if defect.severity not in self.options["severity"]: return self._defects.append(defect) if self._on_defect_callback: self._on_defect_callback(defect) @staticmethod def tool_output_filter(line): return line @staticmethod def parse_defect(raw_line): return raw_line def clean_up(self): for f in self._tmp_files: if os.path.isfile(f): os.remove(f) @staticmethod def is_check_successful(cmd_result): return cmd_result["returncode"] == 0 def execute_check_cmd(self, cmd): result = proc.exec_command( cmd, stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), ) if not self.is_check_successful(result): click.echo( "\nError: Failed to execute check command! Exited with code %d." % result["returncode"] ) if self.options.get("verbose"): click.echo(result["out"]) click.echo(result["err"]) self._bad_input = True return result @staticmethod def get_project_target_files(project_dir, src_filters): c_extension = (".c",) cpp_extensions = (".cc", ".cpp", ".cxx", ".ino") header_extensions = (".h", ".hh", ".hpp", ".hxx") result = {"c": [], "c++": [], "headers": []} def _add_file(path): if path.endswith(header_extensions): result["headers"].append(os.path.abspath(path)) elif path.endswith(c_extension): result["c"].append(os.path.abspath(path)) elif path.endswith(cpp_extensions): result["c++"].append(os.path.abspath(path)) src_filters = normalize_src_filters(src_filters) for f in fs.match_src_files(project_dir, src_filters): _add_file(f) return result def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback cmd = self.configure_command() if cmd: if self.options.get("verbose"): click.echo(" ".join(cmd)) self.execute_check_cmd(cmd) else: if self.options.get("verbose"): click.echo("Error: Couldn't configure command") self._bad_input = True self.clean_up() return self._bad_input # # Helpers # def normalize_src_filters(src_filters): def _normalize(src_filters): return ( src_filters if src_filters.startswith(("+<", "-<")) else "+<%s>" % src_filters ) if isinstance(src_filters, (list, tuple)): return " ".join([_normalize(f) for f in src_filters]) return _normalize(src_filters) ================================================ FILE: platformio/check/tools/clangtidy.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re from os.path import join from platformio.check.defect import DefectItem from platformio.check.tools.base import CheckToolBase class ClangtidyCheckTool(CheckToolBase): def tool_output_filter(self, line): # pylint: disable=arguments-differ if not self.options.get("verbose") and "[clang-diagnostic-error]" in line: return "" if "[CommonOptionsParser]" in line: self._bad_input = True return line if any(d in line for d in ("note: ", "error: ", "warning: ")): return line return "" def parse_defect(self, raw_line): # pylint: disable=arguments-differ match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line) if not match: return raw_line file_, line, column, category, message, defect_id = match.groups() severity = DefectItem.SEVERITY_LOW if category == "error": severity = DefectItem.SEVERITY_HIGH elif category == "warning": severity = DefectItem.SEVERITY_MEDIUM return DefectItem(severity, category, message, file_, line, column, defect_id) @staticmethod def is_check_successful(cmd_result): # Note: Clang-Tidy returns 1 for not critical compilation errors, # so 0 and 1 are only acceptable values return cmd_result["returncode"] < 2 def configure_command(self): tool_path = join(self.get_tool_dir("tool-clangtidy"), "clang-tidy") cmd = [tool_path, "--quiet"] flags = self.get_flags("clangtidy") if not ( self.is_flag_set("--checks", flags) or self.is_flag_set("--config", flags) ): cmd.append("--checks=*") project_files = self.get_project_target_files( self.project_dir, self.options["src_filters"] ) src_files = [] for items in project_files.values(): src_files.extend(items) cmd.extend(flags + src_files + ["--"]) cmd.extend( ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]] ) includes = [] for inc in self.cpp_includes: if self.options.get("skip_packages") and inc.lower().startswith( self.config.get("platformio", "packages_dir").lower() ): continue includes.append(inc) cmd.extend(["-I%s" % inc for inc in includes]) return cmd ================================================ FILE: platformio/check/tools/cppcheck.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import click from platformio import proc from platformio.check.defect import DefectItem from platformio.check.tools.base import CheckToolBase class CppcheckCheckTool(CheckToolBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._field_delimiter = "<&PIO&>" self._buffer = "" self.defect_fields = [ "severity", "message", "file", "line", "column", "callstack", "cwe", "id", ] def tool_output_filter(self, line): # pylint: disable=arguments-differ if ( not self.options.get("verbose") and "--suppress=unmatchedSuppression:" in line ): return "" if any( msg in line for msg in ( "No C or C++ source files found", "unrecognized command line option", "there was an internal error", ) ): self._bad_input = True return line def parse_defect(self, raw_line): # pylint: disable=arguments-differ if self._field_delimiter not in raw_line: return None self._buffer += raw_line if any(f not in self._buffer for f in self.defect_fields): return None args = {} for field in self._buffer.split(self._field_delimiter): field = field.strip().replace('"', "") name, value = field.split("=", 1) args[name] = value args["category"] = args["severity"] if args["severity"] == "error": args["severity"] = DefectItem.SEVERITY_HIGH elif args["severity"] == "warning": args["severity"] = DefectItem.SEVERITY_MEDIUM else: args["severity"] = DefectItem.SEVERITY_LOW # Skip defects found in third-party software, but keep in mind that such defects # might break checking process so defects from project files are not reported breaking_defect_ids = ("preprocessorErrorDirective", "syntaxError") if ( args.get("file", "") .lower() .startswith(self.config.get("platformio", "packages_dir").lower()) ): if args["id"] in breaking_defect_ids: if self.options.get("verbose"): click.echo( "Error: Found a breaking defect '%s' in %s:%s\n" "Please note: check results might not be valid!\n" "Try adding --skip-packages" % (args.get("message"), args.get("file"), args.get("line")) ) click.echo() self._bad_input = True self._buffer = "" return None self._buffer = "" return DefectItem(**args) def configure_command(self, language, src_file): # pylint: disable=arguments-differ tool_path = os.path.join(self.get_tool_dir("tool-cppcheck"), "cppcheck") cmd = [ tool_path, "--addon-python=%s" % proc.get_pythonexe_path(), "--error-exitcode=3", "--verbose" if self.options.get("verbose") else "--quiet", ] cmd.append( '--template="%s"' % self._field_delimiter.join( ["{0}={{{0}}}".format(f) for f in self.defect_fields] ) ) flags = self.get_flags("cppcheck") if not flags: # by default user can suppress reporting individual defects # directly in code // cppcheck-suppress warningID cmd.append("--inline-suppr") if not self.is_flag_set("--platform", flags): cmd.append("--platform=unspecified") if not self.is_flag_set("--enable", flags): enabled_checks = [ "warning", "style", "performance", "portability", "unusedFunction", ] cmd.append("--enable=%s" % ",".join(enabled_checks)) if not self.is_flag_set("--language", flags): cmd.append("--language=" + language) build_flags = self.cxx_flags if language == "c++" else self.cc_flags if not self.is_flag_set("--std", flags): # Try to guess the standard version from the build flags for flag in build_flags: if "-std" in flag: cmd.append("-" + self.convert_language_standard(flag)) cmd.extend( ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]] ) cmd.extend(flags) cmd.extend( "--include=" + inc for inc in self.get_forced_includes(build_flags, self.cpp_includes) ) cmd.append("--includes-file=%s" % self._generate_inc_file()) cmd.append('"%s"' % src_file) return cmd @staticmethod def get_forced_includes(build_flags, includes): def _extract_filepath(flag, include_options, build_flags): path = "" for option in include_options: if not flag.startswith(option): continue if flag.split(option)[1].strip(): path = flag.split(option)[1].strip() elif build_flags.index(flag) + 1 < len(build_flags): path = build_flags[build_flags.index(flag) + 1] return path def _search_include_dir(filepath, include_paths): for inc_path in include_paths: path = os.path.join(inc_path, filepath) if os.path.isfile(path): return path return "" result = [] include_options = ("-include", "-imacros") for f in build_flags: if f.startswith(include_options): filepath = _extract_filepath(f, include_options, build_flags) if not os.path.isabs(filepath): filepath = _search_include_dir(filepath, includes) if os.path.isfile(filepath): result.append(filepath) return result def _generate_src_file(self, src_files): return self._create_tmp_file("\n".join(src_files)) def _generate_inc_file(self): result = [] for inc in self.cpp_includes: if self.options.get("skip_packages") and inc.lower().startswith( self.config.get("platformio", "packages_dir").lower() ): continue result.append(inc) return self._create_tmp_file("\n".join(result)) def clean_up(self): super().clean_up() # delete temporary dump files generated by addons if not self.is_flag_set("--addon", self.get_flags("cppcheck")): return for files in self.get_project_target_files( self.project_dir, self.options["src_filters"] ).values(): for f in files: dump_file = f + ".dump" if os.path.isfile(dump_file): os.remove(dump_file) @staticmethod def is_check_successful(cmd_result): # Cppcheck is configured to return '3' if a defect is found return cmd_result["returncode"] in (0, 3) @staticmethod def convert_language_standard(flag): cpp_standards_map = { "0x": "11", "1y": "14", "1z": "17", "2a": "20", } standard = flag[-2:] # Note: GNU extensions are not supported and converted to regular standards return flag.replace("gnu", "c").replace( standard, cpp_standards_map.get(standard, standard) ) def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback project_files = self.get_project_target_files( self.project_dir, self.options["src_filters"] ) src_files_scope = ("c", "c++") if not any(project_files[t] for t in src_files_scope): click.echo("Error: Nothing to check.") return True for scope, files in project_files.items(): if scope not in src_files_scope: continue for src_file in files: cmd = self.configure_command(scope, src_file) if not cmd: self._bad_input = True continue if self.options.get("verbose"): click.echo(" ".join(cmd)) self.execute_check_cmd(cmd) self.clean_up() return self._bad_input ================================================ FILE: platformio/check/tools/pvsstudio.py ================================================ # Copyright (c) 2020-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import tempfile from xml.etree.ElementTree import fromstring import click from platformio import proc from platformio.check.defect import DefectItem from platformio.check.tools.base import CheckToolBase from platformio.compat import IS_WINDOWS class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._tmp_dir = tempfile.mkdtemp(prefix="piocheck") self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i" self._tmp_output_file = self._generate_tmp_file_path() + ".pvs" self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg" self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd" self.tool_path = os.path.join( self.get_tool_dir("tool-pvs-studio"), "x64" if IS_WINDOWS else "bin", "pvs-studio", ) with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp: fp.write( "exclude-path = " + self.config.get("platformio", "packages_dir").replace("\\", "/") ) with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp: fp.write( " ".join( ['-I"%s"' % inc.replace("\\", "/") for inc in self.cpp_includes] ) ) def tool_output_filter(self, line): # pylint: disable=arguments-differ if any( err_msg in line.lower() for err_msg in ( "license was not entered", "license information is incorrect", ) ): self._bad_input = True return line def _process_defects(self, defects): for defect in defects: if not isinstance(defect, DefectItem): return if defect.severity not in self.options["severity"]: return self._defects.append(defect) if self._on_defect_callback: self._on_defect_callback(defect) def _demangle_report(self, output_file): converter_tool = os.path.join( self.get_tool_dir("tool-pvs-studio"), "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"), ) cmd = ( converter_tool, "-t", "xml", output_file, "-m", "cwe", "-m", "misra", "-a", # Enable all possible analyzers and defect levels "GA:1,2,3;64:1,2,3;OP:1,2,3;CS:1,2,3;MISRA:1,2,3", "--cerr", ) result = proc.exec_command(cmd) if result["returncode"] != 0: click.echo(result["err"]) self._bad_input = True return result["err"] def parse_defects(self, output_file): defects = [] report = self._demangle_report(output_file) if not report: self._bad_input = True return [] try: defects_data = fromstring(report) except: # pylint: disable=bare-except click.echo("Error: Couldn't decode generated report!") self._bad_input = True return [] for table in defects_data.iter("PVS-Studio_Analysis_Log"): message = table.find("Message").text category = table.find("ErrorType").text line = table.find("Line").text file_ = table.find("File").text defect_id = table.find("ErrorCode").text cwe = table.find("CWECode") cwe_id = None if cwe is not None: cwe_id = cwe.text.lower().replace("cwe-", "") misra = table.find("MISRA") if misra is not None: message += " [%s]" % misra.text severity = DefectItem.SEVERITY_LOW if category == "error": severity = DefectItem.SEVERITY_HIGH elif category == "warning": severity = DefectItem.SEVERITY_MEDIUM defects.append( DefectItem( severity, category, message, file_, line, id=defect_id, cwe=cwe_id ) ) return defects def configure_command(self, src_file): # pylint: disable=arguments-differ if os.path.isfile(self._tmp_output_file): os.remove(self._tmp_output_file) if not os.path.isfile(self._tmp_preprocessed_file): click.echo("Error: Missing preprocessed file for '%s'" % src_file) return "" cmd = [ self.tool_path, "--skip-cl-exe", "yes", "--language", "C" if src_file.endswith(".c") else "C++", "--preprocessor", "gcc", "--cfg", self._tmp_cfg_file, "--source-file", src_file, "--i-file", self._tmp_preprocessed_file, "--output-file", self._tmp_output_file, ] flags = self.get_flags("pvs-studio") if not self.is_flag_set("--platform", flags): cmd.append("--platform=arm") cmd.extend(flags) return cmd def _generate_tmp_file_path(self): # pylint: disable=protected-access return os.path.join(self._tmp_dir, next(tempfile._get_candidate_names())) def _prepare_preprocessed_file(self, src_file): if os.path.isfile(self._tmp_preprocessed_file): os.remove(self._tmp_preprocessed_file) flags = self.cxx_flags compiler = self.cxx_path if src_file.endswith(".c"): flags = self.cc_flags compiler = self.cc_path cmd = [ '"%s"' % compiler, '"%s"' % src_file, "-E", "-o", '"%s"' % self._tmp_preprocessed_file, ] cmd.extend([f for f in flags if f]) cmd.extend(['"-D%s"' % d.replace('"', '\\"') for d in self.cpp_defines]) cmd.append('@"%s"' % self._tmp_cmd_file) # Explicitly specify C++ as the language used in .ino files if src_file.endswith(".ino"): cmd.insert(1, "-xc++") result = proc.exec_command(" ".join(cmd), shell=True) if result["returncode"] != 0 or result["err"]: if self.options.get("verbose"): click.echo(" ".join(cmd)) click.echo(result["err"]) self._bad_input = True def clean_up(self): super().clean_up() if os.path.isdir(self._tmp_dir): shutil.rmtree(self._tmp_dir) @staticmethod def is_check_successful(cmd_result): return ( "license" not in cmd_result["err"].lower() and cmd_result["returncode"] == 0 ) def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback for scope, files in self.get_project_target_files( self.project_dir, self.options["src_filters"] ).items(): if scope not in ("c", "c++"): continue for src_file in files: self._prepare_preprocessed_file(src_file) cmd = self.configure_command(src_file) if self.options.get("verbose"): click.echo(" ".join(cmd)) if not cmd: self._bad_input = True continue result = self.execute_check_cmd(cmd) if result["returncode"] != 0: continue self._process_defects(self.parse_defects(self._tmp_output_file)) self.clean_up() return self._bad_input ================================================ FILE: platformio/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import importlib from pathlib import Path import click class PlatformioCLI(click.Group): leftover_args = [] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._pio_root_path = Path(__file__).parent self._pio_cmd_aliases = dict(package="pkg") def _find_pio_commands(self): def _to_module_path(p): return ( "platformio." + ".".join(p.relative_to(self._pio_root_path).parts)[:-3] ) result = {} for p in self._pio_root_path.rglob("cli.py"): # skip this module if p.parent == self._pio_root_path: continue cmd_name = p.parent.name result[self._pio_cmd_aliases.get(cmd_name, cmd_name)] = _to_module_path(p) # find legacy commands for p in (self._pio_root_path / "commands").iterdir(): if p.name.startswith("_"): continue if (p / "command.py").is_file(): result[p.name] = _to_module_path(p / "command.py") elif p.name.endswith(".py"): result[p.name[:-3]] = _to_module_path(p) return result @staticmethod def in_silence(): args = PlatformioCLI.leftover_args return args and any( [ args[0] == "debug" and "--interpreter" in " ".join(args), args[0] == "upgrade", "--json-output" in args, "--version" in args, ] ) @classmethod def reveal_cmd_path_args(cls, ctx): result = [] group = ctx.command args = cls.leftover_args[::] while args: cmd_name = args.pop(0) next_group = group.get_command(ctx, cmd_name) if next_group: group = next_group result.append(cmd_name) if not hasattr(group, "get_command"): break return result def invoke(self, ctx): PlatformioCLI.leftover_args = ctx.args if hasattr(ctx, "protected_args"): PlatformioCLI.leftover_args = ctx.protected_args + ctx.args return super().invoke(ctx) def list_commands(self, ctx): # pylint: disable=unused-argument return sorted(list(self._find_pio_commands())) def get_command(self, ctx, cmd_name): commands = self._find_pio_commands() if cmd_name not in commands: return self._handle_obsolate_command(ctx, cmd_name) module = importlib.import_module(commands[cmd_name]) return getattr(module, "cli") @staticmethod def _handle_obsolate_command(ctx, cmd_name): # pylint: disable=import-outside-toplevel if cmd_name == "init": from platformio.project.commands.init import project_init_cmd return project_init_cmd if cmd_name == "package": from platformio.package.cli import cli return cli raise click.UsageError('No such command "%s"' % cmd_name, ctx) ================================================ FILE: platformio/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/commands/boards.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import shutil import click from tabulate import tabulate from platformio import fs from platformio.package.manager.platform import PlatformPackageManager @click.command("boards", short_help="Board Explorer") @click.argument("query", required=False) @click.option("--installed", is_flag=True) @click.option("--json-output", is_flag=True) def cli(query, installed, json_output): # pylint: disable=R0912 if json_output: return _print_boards_json(query, installed) grpboards = {} for board in _get_boards(installed): if query and not any( query.lower() in str(board.get(k, "")).lower() for k in ("id", "name", "mcu", "vendor", "platform", "frameworks") ): continue if board["platform"] not in grpboards: grpboards[board["platform"]] = [] grpboards[board["platform"]].append(board) terminal_width = shutil.get_terminal_size().columns for platform, boards in sorted(grpboards.items()): click.echo("") click.echo("Platform: ", nl=False) click.secho(platform, bold=True) click.echo("=" * terminal_width) print_boards(boards) return True def print_boards(boards): click.echo( tabulate( [ ( click.style(b["id"], fg="cyan"), b["mcu"], "%dMHz" % (b["fcpu"] / 1000000), fs.humanize_file_size(b["rom"]), fs.humanize_file_size(b["ram"]), b["name"], ) for b in boards ], headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"], ) ) def _get_boards(installed=False): pm = PlatformPackageManager() return pm.get_installed_boards() if installed else pm.get_all_boards() def _print_boards_json(query, installed=False): result = [] for board in _get_boards(installed): if query: search_data = "%s %s" % (board["id"], json.dumps(board).lower()) if query.lower() not in search_data.lower(): continue result.append(board) click.echo(json.dumps(result)) ================================================ FILE: platformio/commands/ci.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import glob import os import shutil import tempfile import click from platformio import fs from platformio.exception import CIBuildEnvsEmpty from platformio.project.commands.init import project_init_cmd, validate_boards from platformio.project.config import ProjectConfig from platformio.run.cli import cli as cmd_run def validate_path(ctx, param, value): # pylint: disable=unused-argument invalid_path = None value = list(value) for i, p in enumerate(value): if p.startswith("~"): value[i] = fs.expanduser(p) value[i] = os.path.abspath(value[i]) if not glob.glob(value[i], recursive=True): invalid_path = p break try: assert invalid_path is None return value except AssertionError as exc: raise click.BadParameter("Found invalid path: %s" % invalid_path) from exc @click.command("ci", short_help="Continuous Integration") @click.argument("src", nargs=-1, callback=validate_path) @click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") @click.option("--exclude", multiple=True) @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option( "--build-dir", default=tempfile.mkdtemp, type=click.Path(file_okay=False, dir_okay=True, writable=True), ) @click.option("--keep-build-dir", is_flag=True) @click.option( "-c", "--project-conf", type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("-O", "--project-option", multiple=True) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-v", "--verbose", is_flag=True) @click.pass_context def cli( # pylint: disable=too-many-arguments,too-many-positional-arguments, too-many-branches ctx, src, lib, exclude, board, build_dir, keep_build_dir, project_conf, project_option, environments, verbose, ): if not src and os.getenv("PLATFORMIO_CI_SRC"): src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":")) if not src: raise click.BadParameter("Missing argument 'src'") try: if not keep_build_dir and os.path.isdir(build_dir): fs.rmtree(build_dir) if not os.path.isdir(build_dir): os.makedirs(build_dir) for dir_name, patterns in dict(lib=lib, src=src).items(): if not patterns: continue contents = [] for p in patterns: contents += glob.glob(p, recursive=True) _copy_contents(os.path.join(build_dir, dir_name), contents) if project_conf and os.path.isfile(project_conf): _copy_project_conf(build_dir, project_conf) elif not board: raise CIBuildEnvsEmpty() if exclude: _exclude_contents(build_dir, exclude) # initialise project ctx.invoke( project_init_cmd, project_dir=build_dir, boards=board, project_options=project_option, ) # process project ctx.invoke( cmd_run, project_dir=build_dir, environment=environments, verbose=verbose ) finally: if not keep_build_dir: fs.rmtree(build_dir) def _copy_contents(dst_dir, contents): # pylint: disable=too-many-branches items = {"dirs": set(), "files": set()} for path in contents: if os.path.isdir(path): items["dirs"].add(path) elif os.path.isfile(path): items["files"].add(path) dst_dir_name = os.path.basename(dst_dir) if dst_dir_name == "src" and len(items["dirs"]) == 1: if not os.path.isdir(dst_dir): shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) else: if not os.path.isdir(dst_dir): os.makedirs(dst_dir) for d in items["dirs"]: src_dst_dir = os.path.join(dst_dir, os.path.basename(d)) if not os.path.isdir(src_dst_dir): shutil.copytree(d, src_dst_dir, symlinks=True) if not items["files"]: return if dst_dir_name == "lib": dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir)) for f in items["files"]: dst_file = os.path.join(dst_dir, os.path.basename(f)) if f == dst_file: continue shutil.copyfile(f, dst_file) def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True) for path in contents: path = os.path.abspath(path) if os.path.isdir(path): fs.rmtree(path) elif os.path.isfile(path): os.remove(path) def _copy_project_conf(build_dir, project_conf): config = ProjectConfig(project_conf, parse_extra=False) if config.has_section("platformio"): config.remove_section("platformio") config.save(os.path.join(build_dir, "platformio.ini")) ================================================ FILE: platformio/commands/device/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-import from platformio.device.monitor.filters.base import ( DeviceMonitorFilterBase as DeviceMonitorFilter, ) ================================================ FILE: platformio/commands/lib.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-branches, too-many-locals import json import logging import os import click from platformio import exception, fs from platformio.cli import PlatformioCLI from platformio.package.commands.install import package_install_cmd from platformio.package.commands.list import package_list_cmd from platformio.package.commands.search import package_search_cmd from platformio.package.commands.show import package_show_cmd from platformio.package.commands.uninstall import package_uninstall_cmd from platformio.package.commands.update import package_update_cmd from platformio.package.exception import NotGlobalLibDir from platformio.package.manager.library import LibraryPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs" CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments" CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" def get_project_global_lib_dir(): return ProjectConfig.get_instance().get("platformio", "globallib_dir") def invoke_command(ctx, cmd, **kwargs): input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] for input_dir in input_dirs: cmd_kwargs = kwargs.copy() if is_platformio_project(input_dir): cmd_kwargs["project_dir"] = input_dir cmd_kwargs["environments"] = project_environments else: cmd_kwargs["global"] = True cmd_kwargs["storage_dir"] = input_dir ctx.invoke(cmd, **cmd_kwargs) @click.group(short_help="Library manager", hidden=True) @click.option( "-d", "--storage-dir", multiple=True, default=None, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), help="Manage custom library storage", ) @click.option( "-g", "--global", is_flag=True, help="Manage global PlatformIO library storage" ) @click.option( "-e", "--environment", multiple=True, help=( "Manage libraries for the specific project build environments " "declared in `platformio.ini`" ), ) @click.pass_context def cli(ctx, **options): in_silence = PlatformioCLI.in_silence() storage_cmds = ("install", "uninstall", "update", "list") # skip commands that don't need storage folder if ctx.invoked_subcommand not in storage_cmds or ( len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help") ): return storage_dirs = list(options["storage_dir"]) if options["global"]: storage_dirs.append(get_project_global_lib_dir()) if not storage_dirs: if is_platformio_project(): storage_dirs = [get_project_dir()] elif is_ci(): storage_dirs = [get_project_global_lib_dir()] click.secho( "Warning! Global library storage is used automatically. " "Please use `platformio lib --global %s` command to remove " "this warning." % ctx.invoked_subcommand, fg="yellow", ) if not storage_dirs: raise NotGlobalLibDir( get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand ) ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"] ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {} for storage_dir in storage_dirs: if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue with fs.cd(storage_dir): config = ProjectConfig.get_instance( os.path.join(storage_dir, "platformio.ini") ) config.validate(options["environment"], silent=in_silence) libdeps_dir = config.get("platformio", "libdeps_dir") for env in config.envs(): if options["environment"] and env not in options["environment"]: continue storage_dir = os.path.join(libdeps_dir, env) ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( "env:" + env, "lib_deps", [] ) @cli.command("install", short_help="Install library") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") @click.option( "--save/--no-save", is_flag=True, default=True, help="Save installed libraries into the `platformio.ini` dependency list" " (enabled by default)", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( "--interactive", is_flag=True, help="Deprecated! Please use a strict dependency specification (owner/libname)", ) @click.option( "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists" ) @click.pass_context def lib_install( # pylint: disable=too-many-arguments,too-many-positional-arguments,unused-argument ctx, libraries, save, silent, interactive, force ): click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg install` instead.\n", fg="yellow", ) return invoke_command( ctx, package_install_cmd, libraries=libraries, no_save=not save, force=force, silent=silent, ) @cli.command("uninstall", short_help="Remove libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") @click.option( "--save/--no-save", is_flag=True, default=True, help="Remove libraries from the `platformio.ini` dependency list and save changes" " (enabled by default)", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.pass_context def lib_uninstall(ctx, libraries, save, silent): click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg uninstall` instead.\n", fg="yellow", ) invoke_command( ctx, package_uninstall_cmd, libraries=libraries, no_save=not save, silent=silent, ) @cli.command("update", short_help="Update installed libraries") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") @click.option( "-c", "--only-check", is_flag=True, help="DEPRECATED. Please use `--dry-run` instead", ) @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) @click.pass_context def lib_update( # pylint: disable=too-many-arguments,too-many-positional-arguments ctx, libraries, only_check, dry_run, silent, json_output ): only_check = dry_run or only_check if only_check and not json_output: raise exception.UserSideException( "This command is deprecated, please use `pio pkg outdated` instead" ) if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg update` instead.\n", fg="yellow", ) return invoke_command( ctx, package_update_cmd, libraries=libraries, silent=silent, ) storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, []) lm = LibraryPackageManager(storage_dir) lm.set_log_level(logging.WARN if silent else logging.DEBUG) _libraries = libraries or lib_deps or lm.get_installed() result = [] for library in _libraries: spec = None pkg = None if isinstance(library, PackageItem): pkg = library else: spec = PackageSpec(library) pkg = lm.get_package(spec) if not pkg: continue outdated = lm.outdated(pkg, spec) if not outdated.is_outdated(allow_incompatible=True): continue manifest = lm.legacy_load_manifest(pkg) manifest["versionWanted"] = ( str(outdated.wanted) if outdated.wanted else None ) manifest["versionLatest"] = ( str(outdated.latest) if outdated.latest else None ) result.append(manifest) json_result[storage_dir] = result return click.echo( json.dumps( json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) ) @cli.command("list", short_help="List installed libraries") @click.option("--json-output", is_flag=True) @click.pass_context def lib_list(ctx, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg list` instead.\n", fg="yellow", ) return invoke_command(ctx, package_list_cmd, only_libraries=True) storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: lm = LibraryPackageManager(storage_dir) json_result[storage_dir] = lm.legacy_get_installed() return click.echo( json.dumps( json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) ) @cli.command("search", short_help="Search for a library") @click.argument("query", required=False, nargs=-1) @click.option("--json-output", is_flag=True) @click.option("--page", type=click.INT, default=1) @click.option("--id", multiple=True) @click.option("-o", "--owner", multiple=True) @click.option("-n", "--name", multiple=True) @click.option("-a", "--author", multiple=True) @click.option("-k", "--keyword", multiple=True) @click.option("-f", "--framework", multiple=True) @click.option("-p", "--platform", multiple=True) @click.option("-i", "--header", multiple=True) @click.option( "--noninteractive", is_flag=True, help="Do not prompt, automatically paginate with delay", ) @click.pass_context def lib_search( # pylint: disable=unused-argument ctx, query, json_output, page, noninteractive, **filters ): if not query: query = [] if not isinstance(query, list): query = list(query) for key, values in filters.items(): for value in values: query.append('%s:"%s"' % (key, value)) if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg search` instead.\n", fg="yellow", ) query.append("type:library") return ctx.invoke(package_search_cmd, query=" ".join(query), page=page) regclient = LibraryPackageManager().get_registry_client_instance() result = regclient.fetch_json_data( "get", "/v2/lib/search", params=dict(query=" ".join(query), page=page), x_cache_valid="1d", ) return click.echo(json.dumps(result)) @cli.command("builtin", short_help="List built-in libraries") @click.option("--storage", multiple=True) @click.option("--json-output", is_flag=True) def lib_builtin(storage, json_output): items = LibraryPackageManager.get_builtin_libs(storage) if json_output: return click.echo(json.dumps(items)) for storage_ in items: if not storage_["items"]: continue click.secho(storage_["name"], fg="green") click.echo("*" * len(storage_["name"])) click.echo() for item in sorted(storage_["items"], key=lambda i: i["name"]): print_lib_item(item) return True @cli.command("show", short_help="Show detailed info about a library") @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) @click.pass_context def lib_show(ctx, library, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg show` instead.\n", fg="yellow", ) return ctx.invoke(package_show_cmd, pkg_type="library", spec=library) lm = LibraryPackageManager() lm.set_log_level(logging.ERROR if json_output else logging.DEBUG) lib_id = lm.reveal_registry_package_id(library) regclient = lm.get_registry_client_instance() lib = regclient.fetch_json_data( "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h" ) return click.echo(json.dumps(lib)) @cli.command("register", short_help="Deprecated") @click.argument("config_url") def lib_register(config_url): # pylint: disable=unused-argument raise exception.UserSideException( "This command is deprecated. Please use `pio pkg publish` command." ) @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease visit " "https://registry.platformio.org\n", fg="yellow", ) return None regclient = LibraryPackageManager().get_registry_client_instance() result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h") return click.echo(json.dumps(result)) def print_lib_item(item): click.secho(item["name"], fg="cyan") click.echo("=" * len(item["name"])) if "id" in item: click.secho("#ID: %d" % item["id"], bold=True) if "description" in item or "url" in item: click.echo(item.get("description", item.get("url", ""))) click.echo() for key in ("version", "homepage", "license", "keywords"): if key not in item or not item[key]: continue if isinstance(item[key], list): click.echo("%s: %s" % (key.capitalize(), ", ".join(item[key]))) else: click.echo("%s: %s" % (key.capitalize(), item[key])) for key in ("frameworks", "platforms"): if key not in item: continue click.echo( "Compatible %s: %s" % ( key, ", ".join( [i["title"] if isinstance(i, dict) else i for i in item[key]] ), ) ) if "authors" in item or "authornames" in item: click.echo( "Authors: %s" % ", ".join( item.get( "authornames", [a.get("name", "") for a in item.get("authors", [])] ) ) ) if "__src_url" in item: click.secho("Source: %s" % item["__src_url"]) click.echo() ================================================ FILE: platformio/commands/platform.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import logging import os import click from platformio.exception import UserSideException from platformio.package.commands.install import package_install_cmd from platformio.package.commands.list import package_list_cmd from platformio.package.commands.search import package_search_cmd from platformio.package.commands.show import package_show_cmd from platformio.package.commands.uninstall import package_uninstall_cmd from platformio.package.commands.update import package_update_cmd from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.package.version import get_original_version from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory @click.group(short_help="Platform manager", hidden=True) def cli(): pass @cli.command("search", short_help="Search for development platform") @click.argument("query", required=False) @click.option("--json-output", is_flag=True) @click.pass_context def platform_search(ctx, query, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg search` instead.\n", fg="yellow", ) query = query or "" return ctx.invoke(package_search_cmd, query=f"type:platform {query}".strip()) platforms = [] for platform in _get_registry_platforms(): if query == "all": query = "" search_data = json.dumps(platform) if query and query.lower() not in search_data.lower(): continue platforms.append( _get_registry_platform_data( platform["name"], with_boards=False, expose_packages=False ) ) click.echo(json.dumps(platforms)) return None @cli.command("frameworks", short_help="List supported frameworks, SDKs") @click.argument("query", required=False) @click.option("--json-output", is_flag=True) def platform_frameworks(query, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease visit https://docs.platformio.org" "/en/latest/frameworks/index.html\n", fg="yellow", ) return regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] for framework in regclient.fetch_json_data( "get", "/v2/frameworks", x_cache_valid="1d" ): if query == "all": query = "" search_data = json.dumps(framework) if query and query.lower() not in search_data.lower(): continue framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"] framework["platforms"] = [ platform["name"] for platform in _get_registry_platforms() if framework["name"] in platform["frameworks"] ] frameworks.append(framework) frameworks = sorted(frameworks, key=lambda manifest: manifest["name"]) click.echo(json.dumps(frameworks)) @cli.command("list", short_help="List installed development platforms") @click.option("--json-output", is_flag=True) @click.pass_context def platform_list(ctx, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg list` instead.\n", fg="yellow", ) return ctx.invoke(package_list_cmd, **{"global": True, "only_platforms": True}) platforms = [] pm = PlatformPackageManager() for pkg in pm.get_installed(): platforms.append( _get_installed_platform_data(pkg, with_boards=False, expose_packages=False) ) platforms = sorted(platforms, key=lambda manifest: manifest["name"]) click.echo(json.dumps(platforms)) return None @cli.command("show", short_help="Show details about development platform") @click.argument("platform") @click.option("--json-output", is_flag=True) @click.pass_context def platform_show(ctx, platform, json_output): # pylint: disable=too-many-branches if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg show` instead.\n", fg="yellow", ) return ctx.invoke(package_show_cmd, pkg_type="platform", spec=platform) data = _get_platform_data(platform) if not data: raise UnknownPlatform(platform) return click.echo(json.dumps(data)) @cli.command("install", short_help="Install new development platform") @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") @click.option("--with-package", multiple=True) @click.option("--without-package", multiple=True) @click.option("--skip-default-package", is_flag=True) @click.option("--with-all-packages", is_flag=True) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( "-f", "--force", is_flag=True, help="Reinstall/redownload dev/platform and its packages if exist", ) @click.pass_context def platform_install( # pylint: disable=too-many-arguments,too-many-positional-arguments ctx, platforms, with_package, without_package, skip_default_package, with_all_packages, silent, force, ): click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg install` instead.\n", fg="yellow", ) ctx.invoke( package_install_cmd, **{ "global": True, "platforms": platforms, "skip_dependencies": ( not with_all_packages and (with_package or without_package or skip_default_package) ), "silent": silent, "force": force, }, ) @cli.command("uninstall", short_help="Uninstall development platform") @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") @click.pass_context def platform_uninstall(ctx, platforms): click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg uninstall` instead.\n", fg="yellow", ) ctx.invoke( package_uninstall_cmd, **{ "global": True, "platforms": platforms, }, ) @cli.command("update", short_help="Update installed development platforms") @click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]") @click.option( "-p", "--only-packages", is_flag=True, help="Update only the platform packages" ) @click.option( "-c", "--only-check", is_flag=True, help="DEPRECATED. Please use `--dry-run` instead", ) @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) @click.pass_context def platform_update( # pylint: disable=too-many-locals,too-many-arguments,too-many-positional-arguments ctx, platforms, only_check, dry_run, silent, json_output, **_ ): only_check = dry_run or only_check if only_check and not json_output: raise UserSideException( "This command is deprecated, please use `pio pkg outdated` instead" ) if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg update` instead.\n", fg="yellow", ) return ctx.invoke( package_update_cmd, **{ "global": True, "platforms": platforms, "silent": silent, }, ) pm = PlatformPackageManager() pm.set_log_level(logging.WARN if silent else logging.DEBUG) platforms = platforms or pm.get_installed() result = [] for platform in platforms: spec = None pkg = None if isinstance(platform, PackageItem): pkg = platform else: spec = PackageSpec(platform) pkg = pm.get_package(spec) if not pkg: continue outdated = pm.outdated(pkg, spec) if ( not outdated.is_outdated(allow_incompatible=True) and not PlatformFactory.new(pkg).are_outdated_packages() ): continue data = _get_installed_platform_data( pkg, with_boards=False, expose_packages=False ) if outdated.is_outdated(allow_incompatible=True): data["versionLatest"] = str(outdated.latest) if outdated.latest else None result.append(data) click.echo(json.dumps(result)) return True # # Helpers # def _get_registry_platforms(): regclient = PlatformPackageManager().get_registry_client_instance() return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d") def _get_platform_data(*args, **kwargs): try: return _get_installed_platform_data(*args, **kwargs) except UnknownPlatform: return _get_registry_platform_data(*args, **kwargs) def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): p = PlatformFactory.new(platform) data = dict( name=p.name, title=p.title, description=p.description, version=p.version, homepage=p.homepage, url=p.homepage, repository=p.repository_url, license=p.license, forDesktop=not p.is_embedded(), frameworks=sorted(list(p.frameworks) if p.frameworks else []), packages=list(p.packages) if p.packages else [], ) # if dump to API # del data['version'] # return data # overwrite VCS version and add extra fields manifest = PlatformPackageManager().legacy_load_manifest( os.path.dirname(p.manifest_path) ) assert manifest for key in manifest: if key == "version" or key.startswith("__"): data[key] = manifest[key] if with_boards: data["boards"] = [c.get_brief_data() for c in p.get_boards().values()] if not data["packages"] or not expose_packages: return data data["packages"] = [] installed_pkgs = { pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages() } for name, options in p.packages.items(): item = dict( name=name, type=p.get_package_type(name), requirements=options.get("version"), optional=options.get("optional") is True, ) if name in installed_pkgs: for key, value in installed_pkgs[name].items(): if key not in ("url", "version", "description"): continue item[key] = value if key == "version": item["originalVersion"] = get_original_version(value) data["packages"].append(item) return data def _get_registry_platform_data( # pylint: disable=unused-argument platform, with_boards=True, expose_packages=True ): _data = None for p in _get_registry_platforms(): if p["name"] == platform: _data = p break if not _data: return None data = dict( ownername=_data.get("ownername"), name=_data["name"], title=_data["title"], description=_data["description"], homepage=_data["homepage"], repository=_data["repository"], url=_data["url"], license=_data["license"], forDesktop=_data["forDesktop"], frameworks=_data["frameworks"], packages=_data["packages"], versions=_data.get("versions"), ) if with_boards: data["boards"] = [ board for board in PlatformPackageManager().get_registered_boards() if board["platform"] == _data["name"] ] return data ================================================ FILE: platformio/commands/settings.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from tabulate import tabulate from platformio import app from platformio.compat import string_types def format_value(raw): if isinstance(raw, bool): return "Yes" if raw else "No" if isinstance(raw, string_types): return raw return str(raw) @click.group(short_help="Manage system settings") def cli(): pass @cli.command("get", short_help="Get existing setting/-s") @click.argument("name", required=False) def settings_get(name): tabular_data = [] for key, options in sorted(app.DEFAULT_SETTINGS.items()): if name and name != key: continue raw_value = app.get_setting(key) formatted_value = format_value(raw_value) if raw_value != options["value"]: default_formatted_value = format_value(options["value"]) formatted_value += "%s" % ( "\n" if len(default_formatted_value) > 10 else " " ) formatted_value += "[%s]" % click.style( default_formatted_value, fg="yellow" ) tabular_data.append( (click.style(key, fg="cyan"), formatted_value, options["description"]) ) click.echo( tabulate( tabular_data, headers=["Name", "Current value [Default]", "Description"] ) ) @cli.command("set", short_help="Set new value for the setting") @click.argument("name") @click.argument("value") @click.pass_context def settings_set(ctx, name, value): app.set_setting(name, value) click.secho("The new value for the setting has been set!", fg="green") ctx.invoke(settings_get, name=name) @cli.command("reset", short_help="Reset settings to default") @click.pass_context def settings_reset(ctx): app.reset_settings() click.secho("The settings have been reset!", fg="green") ctx.invoke(settings_get) ================================================ FILE: platformio/commands/update.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click @click.command( "update", short_help="Update installed platforms, packages and libraries", hidden=True, ) @click.option("--core-packages", is_flag=True, help="Update only the core packages") @click.option( "-c", "--only-check", is_flag=True, help="DEPRECATED. Please use `--dry-run` instead", ) @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) def cli(*_, **__): click.secho( "This command is deprecated and will be removed in the next releases. \n" "Please use `pio pkg update` instead.", fg="yellow", ) ================================================ FILE: platformio/commands/upgrade.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import re import subprocess import click from platformio import VERSION, __version__, app, exception from platformio.dependencies import get_pip_dependencies from platformio.http import fetch_remote_content from platformio.package.manager.core import update_core_packages from platformio.proc import get_pythonexe_path PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json" DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip" DEVELOP_INIT_SCRIPT_URL = ( "https://raw.githubusercontent.com/platformio/platformio-core" "/develop/platformio/__init__.py" ) @click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version") @click.option("--dev", is_flag=True, help="Use development branch") @click.option("--only-dependencies", is_flag=True) @click.option("--verbose", "-v", is_flag=True) def cli(dev, only_dependencies, verbose): if only_dependencies: return upgrade_pip_dependencies(verbose) update_core_packages() if not dev and __version__ == get_latest_version(): return click.secho( "You're up-to-date!\nPlatformIO %s is currently the " "newest version available." % __version__, fg="green", ) click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow") python_exe = get_pythonexe_path() to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio" try: # PIO Core subprocess.run( [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec], check=True, stdout=subprocess.PIPE if not verbose else None, ) # PyPI dependencies subprocess.run( [python_exe, "-m", "platformio", "upgrade", "--only-dependencies"], check=False, stdout=subprocess.PIPE, ) # Check version output = subprocess.run( [python_exe, "-m", "platformio", "--version"], check=True, stdout=subprocess.PIPE, ).stdout.decode() assert "version" in output actual_version = output.split("version", 1)[1].strip() click.secho( "PlatformIO has been successfully upgraded to %s" % actual_version, fg="green", ) click.echo("Release notes: ", nl=False) click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") if app.get_session_var("caller_id"): click.secho( "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" ) except (AssertionError, subprocess.CalledProcessError) as exc: click.secho( "\nWarning!!! Could not automatically upgrade the PlatformIO Core.", fg="red", ) click.secho( "Please upgrade it manually using the following command:\n", fg="red", ) click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan") raise exception.ReturnErrorCode(1) from exc return True def upgrade_pip_dependencies(verbose): subprocess.run( [ get_pythonexe_path(), "-m", "pip", "install", "--upgrade", "pip", *get_pip_dependencies(), ], check=True, stdout=subprocess.PIPE if not verbose else None, ) def get_latest_version(): try: if not str(VERSION[2]).isdigit(): try: return get_develop_latest_version() except: # pylint: disable=bare-except pass return get_pypi_latest_version() except Exception as exc: raise exception.GetLatestVersionError() from exc def get_develop_latest_version(): version = None content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL) for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): continue match = re.match(r"VERSION\s*=\s*\(([^\)]+)\)", line) if not match: continue version = match.group(1) for c in (" ", "'", '"'): version = version.replace(c, "") version = ".".join(version.split(",")) assert version return version def get_pypi_latest_version(): content = fetch_remote_content(PYPI_JSON_URL) return json.loads(content)["info"]["version"] ================================================ FILE: platformio/compat.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-import,no-name-in-module import importlib.util import inspect import locale import os import shlex import sys from platformio.exception import UserSideException if sys.version_info >= (3, 7): from asyncio import create_task as aio_create_task from asyncio import get_running_loop as aio_get_running_loop else: from asyncio import ensure_future as aio_create_task from asyncio import get_event_loop as aio_get_running_loop if sys.version_info >= (3, 8): from shlex import join as shlex_join else: def shlex_join(split_command): return " ".join(shlex.quote(arg) for arg in split_command) if sys.version_info >= (3, 9): from asyncio import to_thread as aio_to_thread else: try: from starlette.concurrency import run_in_threadpool as aio_to_thread except ImportError: pass PY2 = sys.version_info[0] == 2 # DO NOT REMOVE IT. ESP8266/ESP32 depend on it PY36 = sys.version_info[0:2] == (3, 6) IS_CYGWIN = sys.platform.startswith("cygwin") IS_WINDOWS = WINDOWS = sys.platform.startswith("win") IS_MACOS = sys.platform.startswith("darwin") MISSING = object() string_types = (str,) def is_bytes(x): return isinstance(x, (bytes, memoryview, bytearray)) def isascii(text): if sys.version_info >= (3, 7): return text.isascii() for c in text or "": if ord(c) > 127: return False return True def is_terminal(): try: return sys.stdout.isatty() except Exception: # pylint: disable=broad-except return False def ci_strings_are_equal(a, b): if a == b: return True if not a or not b: return False return a.strip().lower() == b.strip().lower() def hashlib_encode_data(data): if is_bytes(data): return data if not isinstance(data, string_types): data = str(data) return data.encode() def load_python_module(name, pathname): spec = importlib.util.spec_from_file_location(name, pathname) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def get_filesystem_encoding(): return sys.getfilesystemencoding() or sys.getdefaultencoding() def get_locale_encoding(): return locale.getpreferredencoding() def get_object_members(obj, ignore_private=True): members = inspect.getmembers(obj, lambda a: not inspect.isroutine(a)) if not ignore_private: return members return { item[0]: item[1] for item in members if not (item[0].startswith("__") and item[0].endswith("__")) } def ensure_python3(raise_exception=True): compatible = sys.version_info >= (3, 6) if not raise_exception or compatible: return compatible raise UserSideException( "Python 3.6 or later is required for this operation. \n" "Please check a migration guide:\n" "https://docs.platformio.org/en/latest/core/migration.html" "#drop-support-for-python-2-and-3-5" ) def path_to_unicode(path): """ Deprecated: Compatibility with dev-platforms, and custom device monitor filters """ return path def is_proxy_set(socks=False): for var in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"): value = os.getenv(var, os.getenv(var.lower())) if not value or (socks and not value.startswith("socks5://")): continue return True return False def click_launch(url, wait=False, locate=False) -> int: return _click_open_url(url, wait=wait, locate=locate) def _click_open_url( # pylint: disable=too-many-branches, too-many-return-statements, consider-using-with, import-outside-toplevel, unspecified-encoding url, wait=False, locate=False ): """ Issue https://github.com/pallets/click/issues/2868 Keep in sync with https://github.com/pallets/click/blob/main/src/click/_termui_impl.py """ import subprocess def _unquote_file(url) -> str: from urllib.parse import unquote if url.startswith("file://"): url = unquote(url[7:]) return url if IS_MACOS: args = ["open"] if wait: args.append("-W") if locate: args.append("-R") args.append(_unquote_file(url)) null = open("/dev/null", "w") try: return subprocess.Popen(args, stderr=null).wait() finally: null.close() elif IS_WINDOWS: if locate: url = _unquote_file(url) args = ["explorer", f"/select,{url}"] else: args = ["start"] if wait: args.append("/WAIT") args.append("") args.append(url) try: return subprocess.call(args, shell=True) except OSError: # Command not found return 127 elif IS_CYGWIN: if locate: url = _unquote_file(url) args = ["cygstart", os.path.dirname(url)] else: args = ["cygstart"] if wait: args.append("-w") args.append(url) try: return subprocess.call(args) except OSError: # Command not found return 127 try: if locate: url = os.path.dirname(_unquote_file(url)) or "." else: url = _unquote_file(url) c = subprocess.Popen(["xdg-open", url]) if wait: return c.wait() return 0 except OSError: if url.startswith(("http://", "https://")) and not locate and not wait: import webbrowser webbrowser.open(url) return 0 return 1 ================================================ FILE: platformio/debug/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/debug/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-arguments, too-many-locals # pylint: disable=too-many-branches, too-many-statements import asyncio import os import signal import subprocess import click from platformio import app, exception, fs, proc from platformio.compat import IS_WINDOWS from platformio.debug import helpers from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.process.gdb import GDBClientProcess from platformio.exception import ReturnErrorCode from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project from platformio.project.options import ProjectOptions @click.command( "debug", context_settings=dict(ignore_unknown_options=True), short_help="Unified Debugger", ) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--environment", "-e", metavar="") @click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type) @click.option("--verbose", "-v", is_flag=True) @click.option("--interface", type=click.Choice(["gdb"])) @click.argument("client_extra_args", nargs=-1, type=click.UNPROCESSED) @click.pass_context def cli( # pylint: disable=too-many-positional-arguments ctx, project_dir, project_conf, environment, load_mode, verbose, interface, client_extra_args, ): app.set_session_var("custom_project_conf", project_conf) if not interface and client_extra_args: raise click.UsageError("Please specify debugging interface") # use env variables from Eclipse or CLion for name in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): if is_platformio_project(project_dir): break if os.getenv(name): project_dir = os.getenv(name) with fs.cd(project_dir): project_config = ProjectConfig.get_instance(project_conf) project_config.validate(envs=[environment] if environment else None) env_name = environment or helpers.get_default_debug_env(project_config) if not interface: return helpers.predebug_project( ctx, os.getcwd(), project_config, env_name, False, verbose ) configure_args = ( ctx, project_config, env_name, load_mode, verbose, client_extra_args, ) if helpers.is_gdbmi_mode(): os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = "true" stream = helpers.GDBMIConsoleStream() with proc.capture_std_streams(stream): debug_config = _configure(*configure_args) stream.close() else: debug_config = _configure(*configure_args) _run(os.getcwd(), debug_config, client_extra_args) return None def _configure( ctx, project_config, env_name, load_mode, verbose, client_extra_args ): # pylint: disable=too-many-positional-arguments platform = PlatformFactory.from_env(env_name, autoinstall=True) debug_config = DebugConfigFactory.new( platform, project_config, env_name, ) if "--version" in client_extra_args: raise ReturnErrorCode( subprocess.run( [debug_config.client_executable_path, "--version"], check=True ).returncode ) try: fs.ensure_udev_rules() except exception.InvalidUdevRules as exc: click.echo(str(exc)) rebuild_prog = False preload = debug_config.load_cmds == ["preload"] load_mode = load_mode or debug_config.load_mode if load_mode == "always": rebuild_prog = preload or not helpers.has_debug_symbols( debug_config.program_path ) elif load_mode == "modified": rebuild_prog = helpers.is_prog_obsolete( debug_config.program_path ) or not helpers.has_debug_symbols(debug_config.program_path) if not (debug_config.program_path and os.path.isfile(debug_config.program_path)): rebuild_prog = True if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server debug_config.load_cmds = [] if rebuild_prog: click.echo("Preparing firmware for debugging...") helpers.predebug_project( ctx, os.getcwd(), project_config, env_name, preload, verbose ) # save SHA sum of newly created prog if load_mode == "modified": helpers.is_prog_obsolete(debug_config.program_path) if not os.path.isfile(debug_config.program_path): raise DebugInvalidOptionsError("Program/firmware is missed") return debug_config def _run(project_dir, debug_config, client_extra_args): try: loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) client = GDBClientProcess(project_dir, debug_config) coro = client.run(client_extra_args) try: signal.signal(signal.SIGINT, signal.SIG_IGN) loop.run_until_complete(coro) if IS_WINDOWS: client.close() # an issue with `asyncio` executor and STIDIN, # it cannot be closed gracefully proc.force_exit() finally: client.close() loop.close() ================================================ FILE: platformio/debug/config/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/debug/config/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os from platformio import fs, proc, util from platformio.compat import string_types from platformio.debug.exception import DebugInvalidOptionsError from platformio.project.config import ProjectConfig from platformio.project.helpers import load_build_metadata from platformio.project.options import ProjectOptions class DebugConfigBase: # pylint: disable=too-many-instance-attributes DEFAULT_PORT = None def __init__(self, platform, project_config, env_name): self.platform = platform self.project_config = project_config self.env_name = env_name self.env_options = project_config.items(env=env_name, as_dict=True) self.build_data = self._load_build_data() self.tool_name = None self.board_config = {} self.tool_settings = {} if "board" in self.env_options: self.board_config = platform.board_config(self.env_options["board"]) self.tool_name = self.board_config.get_debug_tool_name( self.env_options.get("debug_tool") ) self.tool_settings = ( self.board_config.get("debug", {}) .get("tools", {}) .get(self.tool_name, {}) ) self._load_cmds = None self._port = None self.server = self._configure_server() try: platform.configure_debug_session(self) except NotImplementedError: pass @staticmethod def cleanup_cmds(items): items = ProjectConfig.parse_multi_values(items) return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] @property def program_path(self): return self.build_data["prog_path"] @property def client_executable_path(self): return self.build_data["gdb_path"] @property def load_cmds(self): if self._load_cmds is not None: return self._load_cmds result = self.env_options.get("debug_load_cmds") if not result: result = self.tool_settings.get("load_cmds") if not result: # legacy result = self.tool_settings.get("load_cmd") if not result: result = ProjectOptions["env.debug_load_cmds"].default return self.cleanup_cmds(result) @load_cmds.setter def load_cmds(self, cmds): self._load_cmds = cmds @property def load_mode(self): result = self.env_options.get("debug_load_mode") if not result: result = self.tool_settings.get("load_mode") return result or ProjectOptions["env.debug_load_mode"].default @property def init_break(self): missed = object() result = self.env_options.get("debug_init_break", missed) if result != missed: return result result = None if not result: result = self.tool_settings.get("init_break") return result or ProjectOptions["env.debug_init_break"].default @property def init_cmds(self): return self.cleanup_cmds( self.env_options.get("debug_init_cmds", self.tool_settings.get("init_cmds")) ) @property def extra_cmds(self): return self.cleanup_cmds( self.env_options.get("debug_extra_cmds") ) + self.cleanup_cmds(self.tool_settings.get("extra_cmds")) @property def port(self): return ( self._port or self.env_options.get("debug_port") or self.tool_settings.get("port") or self.DEFAULT_PORT ) @port.setter def port(self, value): self._port = value @property def upload_protocol(self): return self.env_options.get( "upload_protocol", self.board_config.get("upload", {}).get("protocol") ) @property def speed(self): return self.env_options.get("debug_speed", self.tool_settings.get("speed")) @property def server_ready_pattern(self): return self.env_options.get( "debug_server_ready_pattern", (self.server or {}).get("ready_pattern") ) def _load_build_data(self): data = load_build_metadata( os.getcwd(), self.env_name, cache=True, build_type="debug" ) if not data: raise DebugInvalidOptionsError("Could not load a build configuration") return data def _configure_server(self): # user disabled server in platformio.ini if "debug_server" in self.env_options and not self.env_options.get( "debug_server" ): return None result = None # specific server per a system if isinstance(self.tool_settings.get("server", {}), list): for item in self.tool_settings["server"][:]: self.tool_settings["server"] = item if util.get_systype() in item.get("system", []): break # user overwrites debug server if self.env_options.get("debug_server"): result = { "cwd": None, "executable": None, "arguments": self.env_options.get("debug_server"), } result["executable"] = result["arguments"][0] result["arguments"] = result["arguments"][1:] elif "server" in self.tool_settings: result = self.tool_settings["server"] server_package = result.get("package") server_package_dir = ( self.platform.get_package_dir(server_package) if server_package else None ) if server_package and not server_package_dir: self.platform.install_package(server_package) server_package_dir = self.platform.get_package_dir(server_package) result.update( dict( cwd=server_package_dir if server_package else None, executable=result.get("executable"), arguments=[ ( a.replace("$PACKAGE_DIR", server_package_dir) if server_package_dir else a ) for a in result.get("arguments", []) ], ) ) return self.reveal_patterns(result) if result else None def get_init_script(self, debugger): try: return getattr(self, "%s_INIT_SCRIPT" % debugger.upper()) except AttributeError as exc: raise NotImplementedError from exc def reveal_patterns(self, source, recursive=True): program_path = self.program_path or "" patterns = { "PLATFORMIO_CORE_DIR": self.project_config.get("platformio", "core_dir"), "PYTHONEXE": proc.get_pythonexe_path(), "PROJECT_DIR": os.getcwd(), "PROG_PATH": program_path, "PROG_DIR": os.path.dirname(program_path), "PROG_NAME": os.path.basename(os.path.splitext(program_path)[0]), "DEBUG_PORT": self.port, "UPLOAD_PROTOCOL": self.upload_protocol, "INIT_BREAK": self.init_break or "", "LOAD_CMDS": "\n".join(self.load_cmds or []), } for key, value in patterns.items(): if key.endswith(("_DIR", "_PATH")): patterns[key] = fs.to_unix_path(value) def _replace(text): for key, value in patterns.items(): pattern = "$%s" % key text = text.replace(pattern, value or "") return text if isinstance(source, string_types): source = _replace(source) elif isinstance(source, (list, dict)): items = enumerate(source) if isinstance(source, list) else source.items() for key, value in items: if isinstance(value, string_types): source[key] = _replace(value) elif isinstance(value, (list, dict)) and recursive: source[key] = self.reveal_patterns(value, patterns) data = json.dumps(source) if any(("$" + key) in data for key in patterns): source = self.reveal_patterns(source, patterns) return source ================================================ FILE: platformio/debug/config/blackmagic.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.debug.config.base import DebugConfigBase from platformio.debug.exception import DebugInvalidOptionsError from platformio.device.finder import SerialPortFinder, is_pattern_port class BlackmagicDebugConfig(DebugConfigBase): GDB_INIT_SCRIPT = """ define pio_reset_halt_target set language c set *0xE000ED0C = 0x05FA0004 set $busy = (*0xE000ED0C & 0x4) while ($busy) set $busy = (*0xE000ED0C & 0x4) end set language auto end define pio_reset_run_target pio_reset_halt_target end target extended-remote $DEBUG_PORT monitor swdp_scan attach 1 set mem inaccessible-by-default off $LOAD_CMDS $INIT_BREAK set language c set *0xE000ED0C = 0x05FA0004 set $busy = (*0xE000ED0C & 0x4) while ($busy) set $busy = (*0xE000ED0C & 0x4) end set language auto """ @property def port(self): # pylint: disable=assignment-from-no-return initial_port = DebugConfigBase.port.fget(self) if initial_port and not is_pattern_port(initial_port): return initial_port port = SerialPortFinder( board_config=self.board_config, upload_protocol=self.tool_name, prefer_gdb_port=True, ).find(initial_port) if port: return port raise DebugInvalidOptionsError( "Please specify `debug_port` for the working environment" ) @port.setter def port(self, value): self._port = value ================================================ FILE: platformio/debug/config/factory.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import importlib import re from platformio.debug.config.generic import GenericDebugConfig from platformio.debug.config.native import NativeDebugConfig class DebugConfigFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) return "%sDebugConfig" % name.lower().capitalize() @classmethod def new(cls, platform, project_config, env_name): board_id = project_config.get("env:" + env_name, "board") config_cls = None tool_name = None if board_id: tool_name = platform.board_config( project_config.get("env:" + env_name, "board") ).get_debug_tool_name(project_config.get("env:" + env_name, "debug_tool")) try: mod = importlib.import_module("platformio.debug.config.%s" % tool_name) config_cls = getattr(mod, cls.get_clsname(tool_name)) except ModuleNotFoundError: config_cls = ( GenericDebugConfig if platform.is_embedded() else NativeDebugConfig ) return config_cls(platform, project_config, env_name) ================================================ FILE: platformio/debug/config/generic.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.debug.config.base import DebugConfigBase class GenericDebugConfig(DebugConfigBase): DEFAULT_PORT = ":3333" GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset halt end define pio_reset_run_target monitor reset end target extended-remote $DEBUG_PORT monitor init $LOAD_CMDS pio_reset_halt_target $INIT_BREAK """ ================================================ FILE: platformio/debug/config/jlink.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.debug.config.base import DebugConfigBase class JlinkDebugConfig(DebugConfigBase): DEFAULT_PORT = ":2331" GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset monitor halt end define pio_reset_run_target monitor clrbp monitor reset monitor go end target extended-remote $DEBUG_PORT monitor clrbp monitor speed auto pio_reset_halt_target $LOAD_CMDS $INIT_BREAK """ @property def server_ready_pattern(self): return super().server_ready_pattern or ("Waiting for GDB connection") ================================================ FILE: platformio/debug/config/mspdebug.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.debug.config.base import DebugConfigBase class MspdebugDebugConfig(DebugConfigBase): DEFAULT_PORT = ":2000" GDB_INIT_SCRIPT = """ define pio_reset_halt_target end define pio_reset_run_target end target remote $DEBUG_PORT monitor erase $LOAD_CMDS pio_reset_halt_target $INIT_BREAK """ ================================================ FILE: platformio/debug/config/native.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.compat import IS_WINDOWS from platformio.debug.config.base import DebugConfigBase class NativeDebugConfig(DebugConfigBase): GDB_INIT_SCRIPT = """ define pio_reset_halt_target end define pio_reset_run_target end define pio_restart_target end $INIT_BREAK """ + ("set startup-with-shell off" if not IS_WINDOWS else "") ================================================ FILE: platformio/debug/config/qemu.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.debug.config.base import DebugConfigBase class QemuDebugConfig(DebugConfigBase): DEFAULT_PORT = ":1234" GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor system_reset end define pio_reset_run_target monitor system_reset end target extended-remote $DEBUG_PORT $LOAD_CMDS pio_reset_halt_target $INIT_BREAK """ ================================================ FILE: platformio/debug/config/renode.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.debug.config.base import DebugConfigBase class RenodeDebugConfig(DebugConfigBase): DEFAULT_PORT = ":3333" GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor machine Reset $LOAD_CMDS monitor start end define pio_reset_run_target pio_reset_halt_target end target extended-remote $DEBUG_PORT $LOAD_CMDS $INIT_BREAK monitor start """ @property def server_ready_pattern(self): return super().server_ready_pattern or ( "GDB server with all CPUs started on port" ) ================================================ FILE: platformio/debug/exception.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.exception import PlatformioException, UserSideException class DebugError(PlatformioException): pass class DebugSupportError(DebugError, UserSideException): MESSAGE = ( "Currently, PlatformIO does not support debugging for `{0}`.\n" "Please request support at https://github.com/platformio/" "platformio-core/issues \nor visit -> https://docs.platformio.org" "/page/plus/debugging.html" ) class DebugInvalidOptionsError(DebugError, UserSideException): pass class DebugInitError(DebugError, UserSideException): pass ================================================ FILE: platformio/debug/helpers.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import sys import time from hashlib import sha1 from io import BytesIO from platformio.cli import PlatformioCLI from platformio.compat import is_bytes from platformio.debug.exception import DebugInvalidOptionsError from platformio.run.cli import cli as cmd_run from platformio.run.cli import print_processing_header from platformio.test.helpers import list_test_names from platformio.test.result import TestSuite from platformio.test.runners.base import TestRunnerOptions from platformio.test.runners.factory import TestRunnerFactory class GDBMIConsoleStream(BytesIO): # pylint: disable=too-few-public-methods STDOUT = sys.stdout def write(self, text): self.STDOUT.write(escape_gdbmi_stream("~", text)) self.STDOUT.flush() def is_gdbmi_mode(): return "--interpreter" in " ".join(PlatformioCLI.leftover_args) def escape_gdbmi_stream(prefix, stream): bytes_stream = False if is_bytes(stream): bytes_stream = True stream = stream.decode() if not stream: return b"" if bytes_stream else "" ends_nl = stream.endswith("\n") stream = re.sub(r"\\+", "\\\\\\\\", stream) stream = stream.replace('"', '\\"') stream = stream.replace("\n", "\\n") stream = '%s"%s"' % (prefix, stream) if ends_nl: stream += "\n" return stream.encode() if bytes_stream else stream def get_default_debug_env(config): default_envs = config.default_envs() all_envs = config.envs() for env in default_envs: if config.get("env:" + env, "build_type") == "debug": return env for env in all_envs: if config.get("env:" + env, "build_type") == "debug": return env return default_envs[0] if default_envs else all_envs[0] def predebug_project( ctx, project_dir, project_config, env_name, preload, verbose ): # pylint: disable=too-many-arguments,too-many-positional-arguments debug_testname = project_config.get("env:" + env_name, "debug_test") if debug_testname: test_names = list_test_names(project_config) if debug_testname not in test_names: raise DebugInvalidOptionsError( "Unknown test name `%s`. Valid names are `%s`" % (debug_testname, ", ".join(test_names)) ) print_processing_header(env_name, project_config, verbose) test_runner = TestRunnerFactory.new( TestSuite(env_name, debug_testname), project_config, TestRunnerOptions( verbose=3 if verbose else 0, without_building=False, without_debugging=False, without_uploading=not preload, without_testing=True, ), ) test_runner.start(ctx) else: ctx.invoke( cmd_run, project_dir=project_dir, project_conf=project_config.path, environment=[env_name], target=["__debug"] + (["upload"] if preload else []), verbose=verbose, ) if preload: time.sleep(5) def has_debug_symbols(prog_path): if not os.path.isfile(prog_path): return False matched = { b".debug_info": False, b".debug_abbrev": False, b" -Og": False, b" -g": False, # b"__PLATFORMIO_BUILD_DEBUG__": False, } with open(prog_path, "rb") as fp: last_data = b"" while True: data = fp.read(1024) if not data: break for pattern, found in matched.items(): if found: continue if pattern in last_data + data: matched[pattern] = True last_data = data return all(matched.values()) def is_prog_obsolete(prog_path): prog_hash_path = prog_path + ".sha1" if not os.path.isfile(prog_path): return True shasum = sha1() with open(prog_path, "rb") as fp: while True: data = fp.read(1024) if not data: break shasum.update(data) new_digest = shasum.hexdigest() old_digest = None if os.path.isfile(prog_hash_path): with open(prog_hash_path, encoding="utf8") as fp: old_digest = fp.read() if new_digest == old_digest: return False with open(prog_hash_path, mode="w", encoding="utf8") as fp: fp.write(new_digest) return True ================================================ FILE: platformio/debug/process/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/debug/process/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import asyncio import signal import subprocess import sys import time from platformio.compat import ( IS_WINDOWS, aio_create_task, aio_get_running_loop, get_locale_encoding, ) class DebugSubprocessProtocol(asyncio.SubprocessProtocol): def __init__(self, factory): self.factory = factory self._is_exited = False def connection_made(self, transport): self.factory.connection_made(transport) def pipe_data_received(self, fd, data): pipe_to_cb = [ self.factory.stdin_data_received, self.factory.stdout_data_received, self.factory.stderr_data_received, ] pipe_to_cb[fd](data) def connection_lost(self, exc): self.process_exited() def process_exited(self): if self._is_exited: return self.factory.process_exited() self._is_exited = True class DebugBaseProcess: STDOUT_CHUNK_SIZE = 2048 LOG_FILE = None def __init__(self): self.transport = None self._is_running = False self._last_activity = 0 self._exit_future = None self._stdin_read_task = None self._std_encoding = get_locale_encoding() async def spawn(self, *args, **kwargs): wait_until_exit = False if "wait_until_exit" in kwargs: wait_until_exit = kwargs["wait_until_exit"] del kwargs["wait_until_exit"] for pipe in ("stdin", "stdout", "stderr"): if pipe not in kwargs: kwargs[pipe] = subprocess.PIPE loop = aio_get_running_loop() await loop.subprocess_exec( lambda: DebugSubprocessProtocol(self), *args, **kwargs ) if wait_until_exit: self._exit_future = loop.create_future() await self._exit_future def is_running(self): return self._is_running def connection_made(self, transport): self._is_running = True self.transport = transport def connect_stdin_pipe(self): self._stdin_read_task = aio_create_task(self._read_stdin_pipe()) async def _read_stdin_pipe(self): loop = aio_get_running_loop() if IS_WINDOWS: while True: self.stdin_data_received( await loop.run_in_executor(None, sys.stdin.buffer.readline) ) else: reader = asyncio.StreamReader() protocol = asyncio.StreamReaderProtocol(reader) await loop.connect_read_pipe(lambda: protocol, sys.stdin) while True: self.stdin_data_received(await reader.readline()) def stdin_data_received(self, data): self._last_activity = time.time() if self.LOG_FILE: with open(self.LOG_FILE, "ab") as fp: fp.write(data) def stdout_data_received(self, data): self._last_activity = time.time() if self.LOG_FILE: with open(self.LOG_FILE, "ab") as fp: fp.write(data) while data: chunk = data[: self.STDOUT_CHUNK_SIZE] print(chunk.decode(self._std_encoding, "replace"), end="", flush=True) data = data[self.STDOUT_CHUNK_SIZE :] def stderr_data_received(self, data): self._last_activity = time.time() if self.LOG_FILE: with open(self.LOG_FILE, "ab") as fp: fp.write(data) print( data.decode(self._std_encoding, "replace"), end="", file=sys.stderr, flush=True, ) def process_exited(self): self._is_running = False self._last_activity = time.time() # Allow terminating via SIGINT/CTRL+C signal.signal(signal.SIGINT, signal.default_int_handler) if self._stdin_read_task: self._stdin_read_task.cancel() self._stdin_read_task = None if self._exit_future: self._exit_future.set_result(True) self._exit_future = None def terminate(self): if not self.is_running() or not self.transport: return try: self.transport.kill() self.transport.close() except: # pylint: disable=bare-except pass ================================================ FILE: platformio/debug/process/client.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import os import signal import tempfile from platformio import fs, proc from platformio.cache import ContentCache from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.debug.process.base import DebugBaseProcess from platformio.debug.process.server import DebugServerProcess from platformio.project.helpers import get_project_cache_dir class DebugClientProcess(DebugBaseProcess): def __init__(self, project_dir, debug_config): super().__init__() self.project_dir = project_dir self.debug_config = debug_config self._server_process = None self._session_id = None if not os.path.isdir(get_project_cache_dir()): os.makedirs(get_project_cache_dir()) self.working_dir = tempfile.mkdtemp( dir=get_project_cache_dir(), prefix=".piodebug-" ) self._target_is_running = False self._errors_buffer = b"" async def run(self): session_hash = ( self.debug_config.client_executable_path + self.debug_config.program_path ) self._session_id = hashlib.sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() if self.debug_config.server: self._server_process = DebugServerProcess(self.debug_config) self.debug_config.port = await self._server_process.run() def connection_made(self, transport): super().connection_made(transport) self._lock_session(transport.get_pid()) # Disable SIGINT and allow GDB's Ctrl+C interrupt signal.signal(signal.SIGINT, lambda *args, **kwargs: None) self.connect_stdin_pipe() def process_exited(self): if self._server_process: self._server_process.terminate() super().process_exited() def close(self): self._unlock_session() if self.working_dir and os.path.isdir(self.working_dir): fs.rmtree(self.working_dir) def __del__(self): self.close() def _kill_previous_session(self): assert self._session_id pid = None with ContentCache() as cc: pid = cc.get(self._session_id) cc.delete(self._session_id) if not pid: return if IS_WINDOWS: kill = ["Taskkill", "/PID", pid, "/F"] else: kill = ["kill", pid] try: proc.exec_command(kill) except: # pylint: disable=bare-except pass def _lock_session(self, pid): if not self._session_id: return with ContentCache() as cc: cc.set(self._session_id, str(pid), "1h") def _unlock_session(self): if not self._session_id: return with ContentCache() as cc: cc.delete(self._session_id) ================================================ FILE: platformio/debug/process/gdb.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import signal import time from platformio import telemetry from platformio.compat import aio_get_running_loop, is_bytes from platformio.debug import helpers from platformio.debug.exception import DebugInitError from platformio.debug.process.client import DebugClientProcess class GDBClientProcess(DebugClientProcess): PIO_SRC_NAME = ".pioinit" INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._target_is_running = False self._errors_buffer = b"" async def run(self, extra_args): # pylint: disable=arguments-differ await super().run() self.generate_init_script(os.path.join(self.working_dir, self.PIO_SRC_NAME)) gdb_path = self.debug_config.client_executable_path or "gdb" # start GDB client args = [ gdb_path, "-q", "--directory", self.working_dir, "--directory", self.project_dir, "-l", "10", ] args.extend(list(extra_args or [])) gdb_data_dir = self._get_data_dir(gdb_path) if gdb_data_dir: args.extend(["--data-directory", gdb_data_dir]) args.append(self.debug_config.program_path) await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True) @staticmethod def _get_data_dir(gdb_path): if "msp430" in gdb_path: return None gdb_data_dir = os.path.abspath( os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb") ) return gdb_data_dir if os.path.isdir(gdb_data_dir) else None def generate_init_script(self, dst): # default GDB init commands depending on debug tool commands = self.debug_config.get_init_script("gdb").split("\n") if self.debug_config.init_cmds: commands = self.debug_config.init_cmds commands.extend(self.debug_config.extra_cmds) if not any("define pio_reset_run_target" in cmd for cmd in commands): commands = [ "define pio_reset_run_target", " echo Warning! Undefined pio_reset_run_target command\\n", " monitor reset", "end", ] + commands if not any("define pio_reset_halt_target" in cmd for cmd in commands): commands = [ "define pio_reset_halt_target", " echo Warning! Undefined pio_reset_halt_target command\\n", " monitor reset halt", "end", ] + commands if not any("define pio_restart_target" in cmd for cmd in commands): commands += [ "define pio_restart_target", " pio_reset_halt_target", " $INIT_BREAK", " %s" % ("continue" if self.debug_config.init_break else "next"), "end", ] banner = [ "echo PlatformIO Unified Debugger -> https://bit.ly/pio-debug\\n", "echo PlatformIO: debug_tool = %s\\n" % self.debug_config.tool_name, "echo PlatformIO: Initializing remote target...\\n", ] footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] commands = banner + commands + footer with open(dst, mode="w", encoding="utf8") as fp: fp.write("\n".join(self.debug_config.reveal_patterns(commands))) def stdin_data_received(self, data): super().stdin_data_received(data) if b"-exec-run" in data: if self._target_is_running: token, _ = data.split(b"-", 1) self.stdout_data_received(token + b"^running\n") return if self.debug_config.platform.is_embedded(): data = data.replace(b"-exec-run", b"-exec-continue") if b"-exec-continue" in data: self._target_is_running = True if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): # Allow terminating via SIGINT/CTRL+C signal.signal(signal.SIGINT, signal.default_int_handler) self.transport.get_pipe_transport(0).write(b"pio_reset_run_target\n") self.transport.get_pipe_transport(0).write(data) def stdout_data_received(self, data): super().stdout_data_received(data) self._handle_error(data) # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: telemetry.log_debug_started(self.debug_config) self._auto_exec_continue() def console_log(self, msg): if helpers.is_gdbmi_mode(): msg = helpers.escape_gdbmi_stream("~", msg) self.stdout_data_received(msg if is_bytes(msg) else msg.encode()) def _auto_exec_continue(self): auto_exec_delay = 0.5 # in seconds if self._last_activity > (time.time() - auto_exec_delay): aio_get_running_loop().call_later(0.1, self._auto_exec_continue) return if not self.debug_config.init_break or self._target_is_running: return self.console_log( "PlatformIO: Resume the execution to `debug_init_break = %s`\n" % self.debug_config.init_break ) self.console_log( "PlatformIO: More configuration options -> https://bit.ly/pio-debug\n" ) if self.debug_config.platform.is_embedded(): self.transport.get_pipe_transport(0).write( b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" ) else: self.transport.get_pipe_transport(0).write( b"0-exec-run\n" if helpers.is_gdbmi_mode() else b"run\n" ) self._target_is_running = True def stderr_data_received(self, data): super().stderr_data_received(data) self._handle_error(data) def _handle_error(self, data): self._errors_buffer = (self._errors_buffer + data)[-8192:] # keep last 8 KBytes if not ( self.PIO_SRC_NAME.encode() in self._errors_buffer and b"Error in sourced" in self._errors_buffer ): return telemetry.log_debug_exception( DebugInitError(self._errors_buffer.decode()), self.debug_config ) self.transport.close() ================================================ FILE: platformio/debug/process/server.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import asyncio import os import re import time from platformio import fs from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode from platformio.debug.process.base import DebugBaseProcess from platformio.proc import where_is_program class DebugServerProcess(DebugBaseProcess): STD_BUFFER_SIZE = 1024 def __init__(self, debug_config): super().__init__() self.debug_config = debug_config self._ready = False self._std_buffer = {"out": b"", "err": b""} async def run(self): # pylint: disable=too-many-branches server = self.debug_config.server if not server: return None server_executable = server["executable"] if not server_executable: return None if server["cwd"]: server_executable = os.path.join(server["cwd"], server_executable) if ( IS_WINDOWS and not server_executable.endswith(".exe") and os.path.isfile(server_executable + ".exe") ): server_executable = server_executable + ".exe" if not os.path.isfile(server_executable): server_executable = where_is_program(server_executable) if not os.path.isfile(server_executable): raise DebugInvalidOptionsError( "Could not launch Debug Server '%s'. Please check that it " "is installed and is included in a system PATH\n" "See https://docs.platformio.org/page/plus/debugging.html" % server_executable ) openocd_pipe_allowed = all( [ not self.debug_config.env_options.get( "debug_port", self.debug_config.tool_settings.get("port") ), "gdb" in self.debug_config.client_executable_path, "openocd" in server_executable, ] ) if openocd_pipe_allowed: args = [] if server["cwd"]: args.extend(["-s", server["cwd"]]) args.extend( ["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"] ) args.extend(server["arguments"]) str_args = " ".join( [arg if arg.startswith("-") else '"%s"' % arg for arg in args] ) return fs.to_unix_path('| "%s" %s' % (server_executable, str_args)) env = os.environ.copy() # prepend server "lib" folder to LD path if ( not IS_WINDOWS and server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "lib")) ): ld_key = "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH" env[ld_key] = os.path.join(server["cwd"], "lib") if os.environ.get(ld_key): env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) # prepend BIN to PATH if server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "bin")): env["PATH"] = "%s%s%s" % ( os.path.join(server["cwd"], "bin"), os.pathsep, os.environ.get("PATH", os.environ.get("Path", "")), ) await self.spawn( *([server_executable] + server["arguments"]), cwd=server["cwd"], env=env ) await self._wait_until_ready() return self.debug_config.port async def _wait_until_ready(self): ready_pattern = self.debug_config.server_ready_pattern timeout = 60 if ready_pattern else 10 elapsed = 0 delay = 0.5 auto_ready_delay = 0.5 while not self._ready and self.is_running() and elapsed < timeout: await asyncio.sleep(delay) if not ready_pattern: self._ready = self._last_activity < (time.time() - auto_ready_delay) elapsed += delay def _check_ready_by_pattern(self, data): if self._ready: return self._ready ready_pattern = self.debug_config.server_ready_pattern if ready_pattern: if ready_pattern.startswith("^"): self._ready = re.match( ready_pattern, data.decode("utf-8", "ignore"), ) else: self._ready = ready_pattern.encode() in data return self._ready def stdout_data_received(self, data): super().stdout_data_received( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data ) self._std_buffer["out"] += data self._check_ready_by_pattern(self._std_buffer["out"]) self._std_buffer["out"] = self._std_buffer["out"][-1 * self.STD_BUFFER_SIZE :] def stderr_data_received(self, data): super().stderr_data_received(data) self._std_buffer["err"] += data self._check_ready_by_pattern(self._std_buffer["err"]) self._std_buffer["err"] = self._std_buffer["err"][-1 * self.STD_BUFFER_SIZE :] ================================================ FILE: platformio/dependencies.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.compat import is_proxy_set def get_core_dependencies(): return { "contrib-piohome": "~3.4.2", "contrib-pioremote": "~1.0.0", "tool-scons": "~4.40801.0", "tool-cppcheck": "~1.21100.0", "tool-clangtidy": "~1.150005.0", "tool-pvs-studio": "~7.18.0", } def get_pip_dependencies(): core = [ "bottle == 0.13.*", "click >=8.0.4, <8.4", # click 9.0 removes 'protected_args' attribute "colorama", "marshmallow == 3.*", "pyelftools >=0.27, <1", "pyserial == 3.5.*", # keep in sync "device/monitor/terminal.py" "requests%s == 2.*" % ("[socks]" if is_proxy_set(socks=True) else ""), "semantic_version == 2.10.*", "tabulate == 0.*", ] home = [ # PIO Home requirements "ajsonrpc == 1.2.*", "starlette >=0.19, <0.53", "uvicorn >=0.16, <0.42", "wsproto == 1.*", ] extra = [] # issue #4702; Broken "requests/charset_normalizer" on macOS ARM extra.append( 'chardet >= 3.0.2,<6; platform_system == "Darwin" and "arm" in platform_machine' ) # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ try: import ssl # pylint: disable=import-outside-toplevel if ssl.OPENSSL_VERSION.startswith("OpenSSL ") and ssl.OPENSSL_VERSION_INFO < ( 1, 1, 1, ): extra.append("urllib3<2") except ImportError: pass return core + home + extra ================================================ FILE: platformio/device/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/device/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.device.list.command import device_list_cmd from platformio.device.monitor.command import device_monitor_cmd @click.group( "device", commands=[ device_list_cmd, device_monitor_cmd, ], short_help="Device manager & Serial/Socket monitor", ) def cli(): pass ================================================ FILE: platformio/device/finder.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from fnmatch import fnmatch from functools import lru_cache import click import serial from platformio.compat import IS_WINDOWS from platformio.device.list.util import list_logical_devices, list_serial_ports from platformio.fs import get_platformio_udev_rules_path from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.util import retry BLACK_MAGIC_HWIDS = [ "1D50:6018", ] def parse_udev_rules_hwids(path): result = [] with open(path, mode="r", encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if not line or line.startswith("#"): continue attrs = {} for attr in line.split(","): attr = attr.replace("==", "=").replace('"', "").strip() if "=" not in attr: continue name, value = attr.split("=", 1) attrs[name] = value hwid = "%s:%s" % ( attrs.get("ATTRS{idVendor}", "*"), attrs.get("ATTRS{idProduct}", "*"), ) if hwid != "*:*": result.append(hwid.upper()) return result def is_pattern_port(port): if not port: return False return set(["*", "?", "[", "]"]) & set(port) def find_mbed_disk(initial_port): msdlabels = ("mbed", "nucleo", "frdm", "microbit") for item in list_logical_devices(): if item["path"].startswith("/net"): continue if ( initial_port and is_pattern_port(initial_port) and not fnmatch(item["path"], initial_port) ): continue mbed_pages = [os.path.join(item["path"], n) for n in ("mbed.htm", "mbed.html")] if any(os.path.isfile(p) for p in mbed_pages): return item["path"] if item["name"] and any(l in item["name"].lower() for l in msdlabels): return item["path"] return None def is_serial_port_ready(port, timeout=1): try: serial.Serial(port, timeout=timeout).close() return True except: # pylint: disable=bare-except pass return False class SerialPortFinder: def __init__( # pylint: disable=too-many-arguments,too-many-positional-arguments self, board_config=None, upload_protocol=None, ensure_ready=False, prefer_gdb_port=False, timeout=2, verbose=False, ): self.board_config = board_config self.upload_protocol = upload_protocol self.ensure_ready = ensure_ready self.prefer_gdb_port = prefer_gdb_port self.timeout = timeout self.verbose = verbose @staticmethod def normalize_board_hwid(value): if isinstance(value, (list, tuple)): value = ("%s:%s" % (value[0], value[1])).replace("0x", "") return value.upper() @staticmethod def match_serial_port(pattern): for item in list_serial_ports(): if fnmatch(item["port"], pattern): return item["port"] return None @staticmethod def match_device_hwid(patterns): if not patterns: return None for item in list_serial_ports(as_objects=True): if not item.vid or not item.pid: continue hwid = "{:04X}:{:04X}".format(item.vid, item.pid) for pattern in patterns: if fnmatch(hwid, pattern): return item return None def find(self, initial_port=None): if initial_port: # Treat any URL (contains '://') as a literal port if "://" in initial_port: return initial_port # Otherwise fall back to existing wildcard logic if not is_pattern_port(initial_port): return initial_port return self.match_serial_port(initial_port) if self.upload_protocol and self.upload_protocol.startswith("blackmagic"): return self._find_blackmagic_port() device = None if self.board_config and self.board_config.get("build.hwids", []): device = self._find_board_device() if not device: device = self._find_known_device() if device: return self._reveal_device_port(device) # pick the best PID:VID USB device port = best_port = None for item in list_serial_ports(): if self.ensure_ready and not is_serial_port_ready(item["port"]): continue port = item["port"] if "VID:PID" in item["hwid"]: best_port = port return best_port or port def _reveal_device_port(self, device): candidates = [] for item in list_serial_ports(as_objects=True): if item.vid == device.vid and item.pid == device.pid: candidates.append(item) if len(candidates) <= 1: return device.device for item in candidates: if ("GDB" if self.prefer_gdb_port else "UART") in item.description: return item.device candidates = sorted(candidates, key=lambda item: item.device) # first port is GDB? BlackMagic, ESP-Prog return candidates[0 if self.prefer_gdb_port else -1].device def _find_blackmagic_port(self): device = self.match_device_hwid(BLACK_MAGIC_HWIDS) if not device: return None port = self._reveal_device_port(device) if IS_WINDOWS and port.startswith("COM") and len(port) > 4: return "\\\\.\\%s" % port return port def _find_board_device(self): hwids = [ self.normalize_board_hwid(hwid) for hwid in self.board_config.get("build.hwids", []) ] try: @retry(timeout=self.timeout) def wrapper(): device = self.match_device_hwid(hwids) if device: return device raise retry.RetryNextException() return wrapper() except retry.RetryStopException: pass if self.verbose: click.secho( "TimeoutError: Could not automatically find serial port " "for the `%s` board based on the declared HWIDs=%s" % (self.board_config.get("name", "unknown"), hwids), fg="yellow", err=True, ) return None def _find_known_device(self): hwids = list(BLACK_MAGIC_HWIDS) # load from UDEV rules udev_rules_path = get_platformio_udev_rules_path() if os.path.isfile(udev_rules_path): hwids.extend(parse_udev_rules_hwids(udev_rules_path)) @lru_cache(maxsize=1) def _fetch_hwids_from_platforms(): """load from installed dev-platforms""" result = [] for platform in PlatformPackageManager().get_installed(): p = PlatformFactory.new(platform) for board_config in p.get_boards().values(): for board_hwid in board_config.get("build.hwids", []): board_hwid = self.normalize_board_hwid(board_hwid) if board_hwid not in result: result.append(board_hwid) return result try: @retry(timeout=self.timeout) def wrapper(): device = self.match_device_hwid(hwids) if not device: device = self.match_device_hwid(_fetch_hwids_from_platforms()) if device: return device raise retry.RetryNextException() return wrapper() except retry.RetryStopException: pass if self.verbose: click.secho( "TimeoutError: Could not automatically find serial port " "based on the known UART bridges", fg="yellow", err=True, ) return None ================================================ FILE: platformio/device/list/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/device/list/command.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from platformio.device.list.util import ( list_logical_devices, list_mdns_services, list_serial_ports, ) @click.command("list", short_help="List devices") @click.option("--serial", is_flag=True, help="List serial ports, default") @click.option("--logical", is_flag=True, help="List logical devices") @click.option("--mdns", is_flag=True, help="List multicast DNS services") @click.option("--json-output", is_flag=True) def device_list_cmd( # pylint: disable=too-many-branches serial, logical, mdns, json_output ): if not logical and not mdns: serial = True data = {} if serial: data["serial"] = list_serial_ports() if logical: data["logical"] = list_logical_devices() if mdns: data["mdns"] = list_mdns_services() single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: return click.echo(json.dumps(data[single_key] if single_key else data)) titles = { "serial": "Serial Ports", "logical": "Logical Devices", "mdns": "Multicast DNS Services", } for key, value in data.items(): if not single_key: click.secho(titles[key], bold=True) click.echo("=" * len(titles[key])) if key == "serial": for item in value: click.secho(item["port"], fg="cyan") click.echo("-" * len(item["port"])) click.echo("Hardware ID: %s" % item["hwid"]) click.echo("Description: %s" % item["description"]) click.echo("") if key == "logical": for item in value: click.secho(item["path"], fg="cyan") click.echo("-" * len(item["path"])) click.echo("Name: %s" % item["name"]) click.echo("") if key == "mdns": for item in value: click.secho(item["name"], fg="cyan") click.echo("-" * len(item["name"])) click.echo("Type: %s" % item["type"]) click.echo("IP: %s" % item["ip"]) click.echo("Port: %s" % item["port"]) if item["properties"]: click.echo( "Properties: %s" % ( "; ".join( [ "%s=%s" % (k, v) for k, v in item["properties"].items() ] ) ) ) click.echo("") if single_key: click.echo("") return True ================================================ FILE: platformio/device/list/util.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import re import time from glob import glob from platformio import __version__, exception, proc from platformio.compat import IS_MACOS, IS_WINDOWS def list_serial_ports(filter_hwid=False, as_objects=False): try: # pylint: disable=import-outside-toplevel from serial.tools.list_ports import comports except ImportError as exc: raise exception.GetSerialPortsError(os.name) from exc if as_objects: return comports() result = [] for p, d, h in comports(): if not p: continue if not filter_hwid or "VID:PID" in h: result.append({"port": p, "description": d, "hwid": h}) if filter_hwid: return result # fix for PySerial if not result and IS_MACOS: for p in glob("/dev/tty.*"): result.append({"port": p, "description": "n/a", "hwid": "n/a"}) return result def list_logical_devices(): items = [] if IS_WINDOWS: try: result = proc.exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"] ).get("out", "") devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") for line in result.split("\n"): match = devicenamere.match(line.strip()) if not match: continue items.append({"path": match.group(1) + "\\", "name": match.group(2)}) return items except WindowsError: # pylint: disable=undefined-variable pass # try "fsutil" result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") for device in re.findall(r"[A-Z]:\\", result): items.append({"path": device, "name": None}) return items result = proc.exec_command(["df"]).get("out") devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) for line in result.split("\n"): match = devicenamere.match(line.strip()) if not match: continue items.append({"path": match.group(1), "name": os.path.basename(match.group(1))}) return items def list_mdns_services(): try: import zeroconf # pylint: disable=import-outside-toplevel except ImportError: result = proc.exec_command( [proc.get_pythonexe_path(), "-m", "pip", "install", "zeroconf"] ) if result.get("returncode") != 0: print(result.get("err")) import zeroconf # pylint: disable=import-outside-toplevel class mDNSListener: def __init__(self): self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) self._found_types = [] self._found_services = [] def __enter__(self): zeroconf.ServiceBrowser( self._zc, [ "_http._tcp.local.", "_hap._tcp.local.", "_services._dns-sd._udp.local.", ], self, ) return self def __exit__(self, etype, value, traceback): self._zc.close() def add_service(self, zc, type_, name): try: assert zeroconf.service_type_name(name) assert str(name) except (AssertionError, UnicodeError, zeroconf.BadTypeInNameException): return if name not in self._found_types: self._found_types.append(name) zeroconf.ServiceBrowser(self._zc, name, self) if type_ in self._found_types: s = zc.get_service_info(type_, name) if s: self._found_services.append(s) def remove_service(self, zc, type_, name): pass def update_service(self, zc, type_, name): pass def get_services(self): return self._found_services items = [] with mDNSListener() as mdns: time.sleep(3) for service in mdns.get_services(): properties = None if service.properties: try: properties = { k.decode("utf8"): ( v.decode("utf8") if isinstance(v, bytes) else v ) for k, v in service.properties.items() } json.dumps(properties) except UnicodeDecodeError: properties = None items.append( { "type": service.type, "name": service.name, "ip": ", ".join(service.parsed_addresses()), "port": service.port, "properties": properties, } ) return items ================================================ FILE: platformio/device/monitor/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/device/monitor/command.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import click from platformio import exception, fs from platformio.device.finder import SerialPortFinder from platformio.device.monitor.filters.base import register_filters from platformio.device.monitor.terminal import get_available_filters, start_terminal from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.project.options import ProjectOptions @click.command("monitor", short_help="Monitor device (Serial/Socket)") @click.option("--port", "-p", help="Port, a number or a device name") @click.option( "-b", "--baud", type=ProjectOptions["env.monitor_speed"].type, help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, ) @click.option( "--parity", type=ProjectOptions["env.monitor_parity"].type, help="Enable parity checking [default=%s]" % ProjectOptions["env.monitor_parity"].default, ) @click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") @click.option("--xonxoff", is_flag=True, help="Enable software flow control") @click.option( "--rts", type=ProjectOptions["env.monitor_rts"].type, help="Set initial RTS line state", ) @click.option( "--dtr", type=ProjectOptions["env.monitor_dtr"].type, help="Set initial DTR line state", ) @click.option("--echo", is_flag=True, help="Enable local echo") @click.option( "--encoding", help=( "Set the encoding for the serial port " "(e.g. hexlify, Latin-1, UTF-8) [default=%s]" % ProjectOptions["env.monitor_encoding"].default ), ) @click.option( "-f", "--filter", "filters", multiple=True, help="Apply filters/text transformations", ) @click.option( "--eol", type=ProjectOptions["env.monitor_eol"].type, help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, ) @click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) @click.option( "--exit-char", type=int, default=3, show_default=True, help="ASCII code of special character that is used to exit " "the application [default=3 (Ctrl+C)]", ) @click.option( "--menu-char", type=int, default=20, help="ASCII code of special character that is used to " "control terminal (menu) [default=20 (DEC)]", ) @click.option( "--quiet", is_flag=True, help="Diagnostics: suppress non-error messages", ) @click.option( "--no-reconnect", is_flag=True, help="Disable automatic reconnection if the established connection fails", ) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", "--environment", help="Load configuration from `platformio.ini` and the specified environment", ) def device_monitor_cmd(**options): with fs.cd(options["project_dir"]): platform = None project_options = {} try: project_options = get_project_options(options["environment"]) if "platform" in project_options: platform = PlatformFactory.new(project_options["platform"]) except NotPlatformIOProjectError: pass options = apply_project_monitor_options(options, project_options) register_filters(platform=platform, options=options) options["port"] = SerialPortFinder( board_config=( platform.board_config(project_options.get("board")) if platform and project_options.get("board") else None ), upload_protocol=project_options.get("upload_protocol"), ensure_ready=True, ).find(initial_port=options["port"]) if options["menu_char"] == options["exit_char"]: raise exception.UserSideException( "--exit-char can not be the same as --menu-char" ) # check for unknown filters if options["filters"]: known_filters = set(get_available_filters()) unknown_filters = set(options["filters"]) - known_filters if unknown_filters: options["filters"] = list(known_filters & set(options["filters"])) click.secho( ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), fg="yellow", ) start_terminal(options) def get_project_options(environment=None): config = ProjectConfig.get_instance() config.validate(envs=[environment] if environment else None) environment = environment or config.get_default_env() return config.items(env=environment, as_dict=True) def apply_project_monitor_options(initial_options, project_options): for option_meta in ProjectOptions.values(): if option_meta.group != "monitor": continue cli_key = option_meta.name.split("_", 1)[1] if cli_key == "speed": cli_key = "baud" # value set from CLI, skip overriding if initial_options[cli_key] not in (None, (), []) and ( option_meta.type != click.BOOL or f"--{cli_key}" in sys.argv[1:] ): continue initial_options[cli_key] = project_options.get( option_meta.name, option_meta.default ) return initial_options ================================================ FILE: platformio/device/monitor/filters/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/device/monitor/filters/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import inspect import os from serial.tools import miniterm from platformio.compat import get_object_members, load_python_module from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig class DeviceMonitorFilterBase(miniterm.Transform): def __init__(self, options=None): """Called by PlatformIO to pass context""" super().__init__() self.options = options or {} self.project_dir = self.options.get("project_dir") self.environment = self.options.get("environment") self._running_terminal = None self.config = ProjectConfig.get_instance() if not self.environment: default_envs = self.config.default_envs() if default_envs: self.environment = default_envs[0] elif self.config.envs(): self.environment = self.config.envs()[0] def __call__(self): """Called by the miniterm library when the filter is actually used""" return self @property def NAME(self): raise NotImplementedError("Please declare NAME attribute for the filter class") def set_running_terminal(self, terminal): self._running_terminal = terminal def get_running_terminal(self): return self._running_terminal def register_filters(platform=None, options=None): # issue #4556: remove default colorize filter miniterm.TRANSFORMATIONS.pop("colorize", None) # project filters load_monitor_filters( ProjectConfig.get_instance().get("platformio", "monitor_dir"), prefix="filter_", options=options, ) # platform filters if platform: load_monitor_filters( os.path.join(platform.get_dir(), "monitor"), prefix="filter_", options=options, ) # load package filters pm = ToolPackageManager() for pkg in pm.get_installed(): load_monitor_filters( os.path.join(pkg.path, "monitor"), prefix="filter_", options=options ) # default filters load_monitor_filters(os.path.dirname(__file__), options=options) def load_monitor_filters(monitor_dir, prefix=None, options=None): if not os.path.isdir(monitor_dir): return for name in os.listdir(monitor_dir): if (prefix and not name.startswith(prefix)) or not name.endswith(".py"): continue path = os.path.join(monitor_dir, name) if not os.path.isfile(path): continue load_monitor_filter(path, options) def load_monitor_filter(path, options=None): name = os.path.basename(path) name = name[: name.find(".")] module = load_python_module("platformio.device.monitor.filters.%s" % name, path) for cls in get_object_members(module).values(): if ( not inspect.isclass(cls) or not issubclass(cls, DeviceMonitorFilterBase) or cls == DeviceMonitorFilterBase ): continue obj = cls(options) miniterm.TRANSFORMATIONS[obj.NAME] = obj return True ================================================ FILE: platformio/device/monitor/filters/hexlify.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import serial from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class Hexlify(DeviceMonitorFilterBase): NAME = "hexlify" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._counter = 0 def set_running_terminal(self, terminal): # force to Latin-1, issue #4732 if terminal.input_encoding == "UTF-8": terminal.set_rx_encoding("Latin-1") super().set_running_terminal(terminal) def rx(self, text): result = "" for c in serial.iterbytes(text): if (self._counter % 16) == 0: result += "\n{:04X} | ".format(self._counter) asciicode = ord(c) if asciicode <= 255: result += "{:02X} ".format(asciicode) else: result += "?? " self._counter += 1 return result ================================================ FILE: platformio/device/monitor/filters/log2file.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import io import os from datetime import datetime from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class LogToFile(DeviceMonitorFilterBase): NAME = "log2file" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._log_fp = None def __call__(self): if not os.path.isdir("logs"): os.makedirs("logs") log_file_name = os.path.join( "logs", "device-monitor-%s.log" % datetime.now().strftime("%y%m%d-%H%M%S") ) print("--- Logging an output to %s" % os.path.abspath(log_file_name)) # pylint: disable=consider-using-with self._log_fp = io.open(log_file_name, "w", encoding="utf-8") return self def __del__(self): if self._log_fp: self._log_fp.close() def rx(self, text): self._log_fp.write(text) self._log_fp.flush() return text ================================================ FILE: platformio/device/monitor/filters/send_on_enter.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class SendOnEnter(DeviceMonitorFilterBase): NAME = "send_on_enter" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._buffer = "" if self.options.get("eol") == "CR": self._eol = "\r" elif self.options.get("eol") == "LF": self._eol = "\n" else: self._eol = "\r\n" def tx(self, text): self._buffer += text if self._buffer.endswith(self._eol): text = self._buffer self._buffer = "" return text return "" ================================================ FILE: platformio/device/monitor/filters/time.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from datetime import datetime from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class Timestamp(DeviceMonitorFilterBase): NAME = "time" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._line_started = False def rx(self, text): if self._line_started and "\n" not in text: return text timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] if not self._line_started: self._line_started = True text = "%s > %s" % (timestamp, text) if text.endswith("\n"): self._line_started = False return text[:-1].replace("\n", "\n%s > " % timestamp) + "\n" return text.replace("\n", "\n%s > " % timestamp) ================================================ FILE: platformio/device/monitor/terminal.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import signal import sys import threading import click import serial from serial.tools import miniterm from platformio.exception import UserSideException class Terminal(miniterm.Miniterm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.pio_unexpected_exception = None def reader(self): try: super().reader() except Exception as exc: # pylint: disable=broad-except self.pio_unexpected_exception = exc def writer(self): try: super().writer() except Exception as exc: # pylint: disable=broad-except self.pio_unexpected_exception = exc def get_available_filters(): return sorted(miniterm.TRANSFORMATIONS.keys()) def start_terminal(options): retries = 0 is_port_valid = False while True: term = None try: term = new_terminal(options) is_port_valid = True options["port"] = term.serial.name if retries: click.echo("\t Connected!", err=True) elif not options["quiet"]: print_terminal_settings(term) retries = 0 # reset term.start() try: term.join(True) except KeyboardInterrupt: pass term.join() # cleanup term.console.cleanup() # restore original standard streams sys.stdin = sys.__stdin__ sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ term.close() if term.pio_unexpected_exception: click.secho( "Disconnected (%s)" % term.pio_unexpected_exception, fg="red", err=True, ) if not options["no_reconnect"]: raise UserSideException(term.pio_unexpected_exception) return except UserSideException as exc: if not is_port_valid: raise exc if not retries: click.echo("Reconnecting to %s " % options["port"], err=True, nl=False) signal.signal(signal.SIGINT, signal.SIG_DFL) else: click.echo(".", err=True, nl=False) retries += 1 threading.Event().wait(retries / 2) def new_terminal(options): term = Terminal( new_serial_instance(options), echo=options["echo"], eol=options["eol"].lower(), filters=list(reversed(options["filters"] or ["default"])), ) term.exit_character = chr(options["exit_char"]) term.menu_character = chr(options["menu_char"]) term.raw = options["raw"] term.set_rx_encoding(options["encoding"]) term.set_tx_encoding(options["encoding"]) for ts in (term.tx_transformations, term.rx_transformations): for t in ts: try: t.set_running_terminal(term) except AttributeError: pass return term def print_terminal_settings(terminal): click.echo( "--- Terminal on {p.name} | " "{p.baudrate} {p.bytesize}-{p.parity}-{p.stopbits}".format(p=terminal.serial) ) click.echo( "--- Available filters and text transformations: %s" % ", ".join(get_available_filters()) ) click.echo("--- More details at https://bit.ly/pio-monitor-filters") click.echo( "--- Quit: {} | Menu: {} | Help: {} followed by {}".format( miniterm.key_description(terminal.exit_character), miniterm.key_description(terminal.menu_character), miniterm.key_description(terminal.menu_character), miniterm.key_description("\x08"), ) ) def new_serial_instance(options): # pylint: disable=too-many-branches serial_instance = None port = options["port"] while serial_instance is None: # no port given on command line -> ask user now if port is None or port == "-": try: port = miniterm.ask_for_port() except KeyboardInterrupt as exc: click.echo("", err=True) raise UserSideException("User aborted and port is not given") from exc if not port: raise UserSideException("Port is not given") try: serial_instance = serial.serial_for_url( port, options["baud"], parity=options["parity"], rtscts=options["rtscts"], xonxoff=options["xonxoff"], do_not_open=True, ) if not hasattr(serial_instance, "cancel_read"): # enable timeout for alive flag polling if cancel_read is not available serial_instance.timeout = 1 if options["dtr"] is not None: if not options["quiet"]: click.echo( "--- forcing DTR {}".format( "active" if options["dtr"] else "inactive" ) ) serial_instance.dtr = options["dtr"] if options["rts"] is not None: if not options["quiet"]: click.echo( "--- forcing RTS {}".format( "active" if options["rts"] else "inactive" ) ) serial_instance.rts = options["rts"] if isinstance(serial_instance, serial.Serial): serial_instance.exclusive = True serial_instance.open() except serial.SerialException as exc: raise UserSideException(exc) from exc return serial_instance ================================================ FILE: platformio/exception.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. class PlatformioException(Exception): MESSAGE = None def __str__(self): # pragma: no cover if self.MESSAGE: # pylint: disable=not-an-iterable return self.MESSAGE.format(*self.args) return super().__str__() class ReturnErrorCode(PlatformioException): MESSAGE = "{0}" class UserSideException(PlatformioException): pass class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" # # UDEV Rules # class InvalidUdevRules(UserSideException): pass class MissedUdevRules(InvalidUdevRules): MESSAGE = ( "Warning! Please install `99-platformio-udev.rules`. \nMore details: " "https://docs.platformio.org/en/latest/core/installation/udev-rules.html" ) class OutdatedUdevRules(InvalidUdevRules): MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." "\nMore details: " "https://docs.platformio.org/en/latest/core/installation/udev-rules.html" ) # # Misc # class GetSerialPortsError(PlatformioException): MESSAGE = "No implementation for your platform ('{0}') available" class GetLatestVersionError(PlatformioException): MESSAGE = "Can not retrieve the latest PlatformIO version" class InvalidSettingName(UserSideException): MESSAGE = "Invalid setting with the name '{0}'" class InvalidSettingValue(UserSideException): MESSAGE = "Invalid value '{0}' for the setting '{1}'" class InvalidJSONFile(ValueError, UserSideException): MESSAGE = "Could not load broken JSON: {0}" class CIBuildEnvsEmpty(UserSideException): MESSAGE = ( "Can't find PlatformIO build environments.\n" "Please specify `--board` or path to `platformio.ini` with " "predefined environments using `--project-conf` option" ) class HomeDirPermissionsError(UserSideException): MESSAGE = ( "The directory `{0}` or its parent directory is not owned by the " "current user and PlatformIO can not store configuration data.\n" "Please check the permissions and owner of that directory.\n" "Otherwise, please remove manually `{0}` directory and PlatformIO " "will create new from the current user." ) class CygwinEnvDetected(PlatformioException): MESSAGE = ( "PlatformIO does not work within Cygwin environment. " "Use native Terminal instead." ) ================================================ FILE: platformio/fs.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import glob import hashlib import io import json import os import re import shutil import stat import sys import click from platformio import exception from platformio.compat import IS_WINDOWS class cd: def __init__(self, new_path): self.new_path = new_path self.prev_path = os.getcwd() def __enter__(self): os.chdir(self.new_path) def __exit__(self, etype, value, traceback): os.chdir(self.prev_path) def get_source_dir(): curpath = os.path.abspath(__file__) if not os.path.isfile(curpath): for p in sys.path: if os.path.isfile(os.path.join(p, __file__)): curpath = os.path.join(p, __file__) break return os.path.dirname(curpath) def get_assets_dir(): return os.path.join(get_source_dir(), "assets") def load_json(file_path): try: with open(file_path, mode="r", encoding="utf8") as f: return json.load(f) except ValueError as exc: raise exception.InvalidJSONFile(file_path) from exc def humanize_file_size(filesize): base = 1024 unit = 0 suffix = "B" filesize = float(filesize) if filesize < base: return "%d%s" % (filesize, suffix) for i, suffix in enumerate("KMGTPEZY"): unit = base ** (i + 2) if filesize >= unit: continue if filesize % (base ** (i + 1)): return "%.2f%sB" % ((base * filesize / unit), suffix) break return "%d%sB" % ((base * filesize / unit), suffix) def calculate_file_hashsum(algorithm, path): h = hashlib.new(algorithm) with io.open(path, "rb", buffering=0) as fp: while True: chunk = fp.read(io.DEFAULT_BUFFER_SIZE) if not chunk: break h.update(chunk) return h.hexdigest() def calculate_folder_size(path): assert os.path.isdir(path) result = 0 for root, __, files in os.walk(path): for f in files: file_path = os.path.join(root, f) if not os.path.islink(file_path): result += os.path.getsize(file_path) return result def get_platformio_udev_rules_path(): return os.path.abspath( os.path.join(get_assets_dir(), "system", "99-platformio-udev.rules") ) def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel def _rules_to_set(rules_path): result = set() with open(rules_path, encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if not line or line.startswith("#"): continue result.add(line) return result if "linux" not in get_systype(): return None installed_rules = [ "/etc/udev/rules.d/99-platformio-udev.rules", "/lib/udev/rules.d/99-platformio-udev.rules", ] if not any(os.path.isfile(p) for p in installed_rules): raise exception.MissedUdevRules origin_path = get_platformio_udev_rules_path() if not os.path.isfile(origin_path): return None origin_rules = _rules_to_set(origin_path) for rules_path in installed_rules: if not os.path.isfile(rules_path): continue current_rules = _rules_to_set(rules_path) if not origin_rules <= current_rules: raise exception.OutdatedUdevRules(rules_path) return True def path_endswith_ext(path, extensions): if not isinstance(extensions, (list, tuple)): extensions = [extensions] for ext in extensions: if path.endswith("." + ext): return True return False def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): def _add_candidate(items, item, src_dir): if not src_exts or path_endswith_ext(item, src_exts): items.add(os.path.relpath(item, src_dir)) def _find_candidates(pattern): candidates = set() for item in glob.glob( os.path.join(glob.escape(src_dir), pattern), recursive=True ): if not os.path.isdir(item): _add_candidate(candidates, item, src_dir) continue for root, dirs, files in os.walk(item, followlinks=followlinks): for d in dirs if not followlinks else []: if os.path.islink(os.path.join(root, d)): _add_candidate(candidates, os.path.join(root, d), src_dir) for f in files: _add_candidate(candidates, os.path.join(root, f), src_dir) return candidates src_filter = src_filter or "" if isinstance(src_filter, (list, tuple)): src_filter = " ".join(src_filter) result = set() # correct fs directory separator src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) for action, pattern in re.findall(r"(\+|\-)<([^>]+)>", src_filter): candidates = _find_candidates(pattern) if action == "+": result |= candidates else: result -= candidates return sorted(list(result)) def to_unix_path(path): if not IS_WINDOWS or not path: return path return path.replace("\\", "/") def expanduser(path): """ Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE """ if not IS_WINDOWS or not path.startswith("~") or "USERPROFILE" not in os.environ: return os.path.expanduser(path) return os.environ["USERPROFILE"] + path[1:] def change_filemtime(path, mtime): os.utime(path, (mtime, mtime)) def rmtree(path): def _onexc(func, path, _): try: st_mode = os.stat(path).st_mode if st_mode & stat.S_IREAD: os.chmod(path, st_mode | stat.S_IWRITE) func(path) except Exception as exc: # pylint: disable=broad-except click.secho( "%s \nPlease manually remove the file `%s`" % (str(exc), path), fg="red", err=True, ) # pylint: disable=unexpected-keyword-arg, deprecated-argument if sys.version_info < (3, 12): return shutil.rmtree(path, onerror=_onexc) return shutil.rmtree(path, onexc=_onexc) ================================================ FILE: platformio/home/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/home/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import mimetypes import socket import click from platformio.compat import IS_WINDOWS, click_launch from platformio.home.run import run_server from platformio.package.manager.core import get_core_package_dir @click.command("home", short_help="GUI to manage PlatformIO") @click.option("--port", type=int, default=8008, help="HTTP port, default=8008") @click.option( "--host", default="127.0.0.1", help=( "HTTP host, default=127.0.0.1. You can open PIO Home for inbound " "connections with --host=0.0.0.0" ), ) @click.option("--no-open", is_flag=True) @click.option( "--shutdown-timeout", default=0, type=int, help=( "Automatically shutdown server on timeout (in seconds) when no clients " "are connected. Default is 0 which means never auto shutdown" ), ) @click.option( "--session-id", help=( "A unique session identifier to keep PIO Home isolated from other instances " "and protect from 3rd party access" ), ) def cli(port, host, no_open, shutdown_timeout, session_id): # hook for `platformio-node-helpers` if host == "__do_not_start__": # download all dependent packages get_core_package_dir("contrib-piohome") return # Ensure PIO Home mimetypes are known mimetypes.add_type("text/html", ".html") mimetypes.add_type("text/css", ".css") mimetypes.add_type("application/javascript", ".js") home_url = "http://%s:%d%s" % ( host, port, ("/session/%s/" % session_id) if session_id else "/", ) click.echo( "\n".join( [ "", " ___I_", " /\\-_--\\ PlatformIO Home", "/ \\_-__\\", "|[]| [] | %s" % home_url, "|__|____|__%s" % ("_" * len(home_url)), ] ) ) click.echo("") click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url) if is_port_used(host, port): click.secho( "PlatformIO Home server is already started in another process.", fg="yellow" ) if not no_open: click_launch(home_url) return run_server( host=host, port=port, no_open=no_open, shutdown_timeout=shutdown_timeout, home_url=home_url, ) def is_port_used(host, port): socket.setdefaulttimeout(1) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if IS_WINDOWS: try: s.bind((host, port)) s.close() return False except (OSError, socket.error): pass else: try: s.connect((host, port)) s.close() except socket.error: return False return True ================================================ FILE: platformio/home/rpc/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/home/rpc/handlers/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/home/rpc/handlers/account.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from ajsonrpc.core import JSONRPC20DispatchException from platformio.account.client import AccountClient from platformio.home.rpc.handlers.base import BaseRPCHandler class AccountRPC(BaseRPCHandler): @staticmethod def call_client(method, *args, **kwargs): try: client = AccountClient() return getattr(client, method)(*args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( code=5000, message="PIO Account Call Error", data=str(exc) ) from exc ================================================ FILE: platformio/home/rpc/handlers/app.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from pathlib import Path from platformio import __version__, app, fs, util from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.project.helpers import is_platformio_project class AppRPC(BaseRPCHandler): IGNORE_STORAGE_KEYS = [ "cid", "coreVersion", "coreSystype", "coreCaller", "coreSettings", "homeDir", "projectsDir", ] @staticmethod def load_state(): with app.State( app.resolve_state_path("core_dir", "homestate.json"), lock=True ) as state: storage = state.get("storage", {}) # base data caller_id = app.get_session_var("caller_id") storage["cid"] = app.get_cid() storage["coreVersion"] = __version__ storage["coreSystype"] = util.get_systype() storage["coreCaller"] = str(caller_id).lower() if caller_id else None storage["coreSettings"] = { name: { "description": data["description"], "default_value": data["value"], "value": app.get_setting(name), } for name, data in app.DEFAULT_SETTINGS.items() } storage["homeDir"] = fs.expanduser("~") storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"] # skip non-existing recent projects storage["recentProjects"] = list( set( str(Path(p).resolve()) for p in storage.get("recentProjects", []) if is_platformio_project(p) ) ) state["storage"] = storage state.modified = False # skip saving extra fields return state.as_dict() @staticmethod def get_state(): return AppRPC.load_state() @staticmethod def save_state(state): with app.State( app.resolve_state_path("core_dir", "homestate.json"), lock=True ) as s: s.clear() s.update(state) storage = s.get("storage", {}) for k in AppRPC.IGNORE_STORAGE_KEYS: if k in storage: del storage[k] return True ================================================ FILE: platformio/home/rpc/handlers/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. class BaseRPCHandler: factory = None ================================================ FILE: platformio/home/rpc/handlers/ide.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time from pathlib import Path from ajsonrpc.core import JSONRPC20DispatchException from platformio.compat import aio_get_running_loop from platformio.home.rpc.handlers.base import BaseRPCHandler class IDERPC(BaseRPCHandler): COMMAND_TIMEOUT = 1.5 # in seconds def __init__(self): self._ide_queue = [] self._cmd_queue = {} async def listen_commands(self): f = aio_get_running_loop().create_future() self._ide_queue.append(f) self._process_commands() return await f async def send_command(self, command, params=None): cmd_id = f"ide-{command}-{time.time()}" self._cmd_queue[cmd_id] = { "method": command, "params": params, "time": time.time(), "future": aio_get_running_loop().create_future(), } self._process_commands() # in case if IDE agent has not been started aio_get_running_loop().call_later( self.COMMAND_TIMEOUT + 0.1, self._process_commands ) return await self._cmd_queue[cmd_id]["future"] def on_command_result(self, cmd_id, value): if cmd_id not in self._cmd_queue: return False if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs": value = [str(Path(p).resolve()) for p in value] self._cmd_queue[cmd_id]["future"].set_result(value) del self._cmd_queue[cmd_id] return True def _process_commands(self): for cmd_id in list(self._cmd_queue): cmd_data = self._cmd_queue[cmd_id] if cmd_data["future"].done(): del self._cmd_queue[cmd_id] continue if ( not self._ide_queue and (time.time() - cmd_data["time"]) > self.COMMAND_TIMEOUT ): cmd_data["future"].set_exception( JSONRPC20DispatchException( code=4005, message="PIO Home IDE agent is not started" ) ) continue while self._ide_queue: self._ide_queue.pop().set_result( { "id": cmd_id, "method": cmd_data["method"], "params": cmd_data["params"], } ) ================================================ FILE: platformio/home/rpc/handlers/misc.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import time from platformio.cache import ContentCache from platformio.compat import aio_create_task from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.os import OSRPC class MiscRPC(BaseRPCHandler): async def load_latest_tweets(self, data_url): cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" with ContentCache() as cc: cache_data = cc.get(cache_key) if cache_data: cache_data = json.loads(cache_data) # automatically update cache in background every 12 hours if cache_data["time"] < (time.time() - (3600 * 12)): aio_create_task( self._preload_latest_tweets(data_url, cache_key, cache_valid) ) return cache_data["result"] return await self._preload_latest_tweets(data_url, cache_key, cache_valid) @staticmethod async def _preload_latest_tweets(data_url, cache_key, cache_valid): result = json.loads((await OSRPC.fetch_content(data_url))) with ContentCache() as cc: cc.set( cache_key, json.dumps({"time": int(time.time()), "result": result}), cache_valid, ) return result ================================================ FILE: platformio/home/rpc/handlers/os.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import glob import io import os import shutil from functools import cmp_to_key from platformio import fs from platformio.cache import ContentCache from platformio.compat import aio_to_thread, click_launch from platformio.device.list.util import list_logical_devices from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.http import HTTPSession, ensure_internet_on class HTTPAsyncSession(HTTPSession): async def request( # pylint: disable=signature-differs,invalid-overridden-method self, *args, **kwargs ): func = super().request return await aio_to_thread(func, *args, **kwargs) class OSRPC(BaseRPCHandler): _http_session = None @classmethod async def fetch_content(cls, url, data=None, headers=None, cache_valid=None): if not headers: headers = { "User-Agent": ( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " "Safari/603.3.8" ) } cache_key = ContentCache.key_from_args(url, data) if cache_valid else None with ContentCache() as cc: if cache_key: result = cc.get(cache_key) if result is not None: return result # check internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) if not cls._http_session: cls._http_session = HTTPAsyncSession() if data: r = await cls._http_session.post(url, data=data, headers=headers) else: r = await cls._http_session.get(url, headers=headers) r.raise_for_status() result = r.text if cache_valid: with ContentCache() as cc: cc.set(cache_key, result, cache_valid) return result async def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith("http"): return await self.fetch_content(uri, data, headers, cache_valid) local_path = uri[7:] if uri.startswith("file://") else uri with io.open(local_path, encoding="utf-8") as fp: return fp.read() return None @staticmethod def open_url(url): return click_launch(url) @staticmethod def reveal_file(path): return click_launch(path, locate=True) @staticmethod def open_file(path): return click_launch(path) @staticmethod def call_path_module_func(name, args, **kwargs): return getattr(os.path, name)(*args, **kwargs) @staticmethod def get_path_separator(): return os.sep @staticmethod def is_file(path): return os.path.isfile(path) @staticmethod def is_dir(path): return os.path.isdir(path) @staticmethod def make_dirs(path): return os.makedirs(path) @staticmethod def get_file_mtime(path): return os.path.getmtime(path) @staticmethod def rename(src, dst): return os.rename(src, dst) @staticmethod def copy(src, dst): return shutil.copytree(src, dst, symlinks=True) @staticmethod def glob(pathnames, root=None): if not isinstance(pathnames, list): pathnames = [pathnames] result = set() for pathname in pathnames: result |= set( glob.glob( os.path.join(root, pathname) if root else pathname, recursive=True ) ) return list(result) @staticmethod def list_dir(path): def _cmp(x, y): if x[1] and not y[1]: return -1 if not x[1] and y[1]: return 1 if x[0].lower() > y[0].lower(): return 1 if x[0].lower() < y[0].lower(): return -1 return 0 items = [] if path.startswith("~"): path = fs.expanduser(path) if not os.path.isdir(path): return items for item in os.listdir(path): try: item_is_dir = os.path.isdir(os.path.join(path, item)) if item_is_dir: os.listdir(os.path.join(path, item)) items.append((item, item_is_dir)) except OSError: pass return sorted(items, key=cmp_to_key(_cmp)) @staticmethod def get_logical_devices(): return list_logical_devices() ================================================ FILE: platformio/home/rpc/handlers/piocore.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import asyncio import functools import io import json import os import sys import threading import click from ajsonrpc.core import JSONRPC20DispatchException from platformio import __main__, __version__, app, fs, proc, util from platformio.compat import ( IS_WINDOWS, aio_create_task, aio_get_running_loop, aio_to_thread, get_locale_encoding, is_bytes, ) from platformio.exception import PlatformioException from platformio.home.rpc.handlers.base import BaseRPCHandler class PIOCoreProtocol(asyncio.SubprocessProtocol): def __init__(self, exit_future, on_data_callback=None): self.exit_future = exit_future self.on_data_callback = on_data_callback self.stdout = "" self.stderr = "" self._is_exited = False self._encoding = get_locale_encoding() def pipe_data_received(self, fd, data): data = data.decode(self._encoding, "replace") pipe = ["stdin", "stdout", "stderr"][fd] if pipe == "stdout": self.stdout += data if pipe == "stderr": self.stderr += data if self.on_data_callback: self.on_data_callback(pipe=pipe, data=data) def connection_lost(self, exc): self.process_exited() def process_exited(self): if self._is_exited: return self.exit_future.set_result(True) self._is_exited = True class MultiThreadingStdStream: def __init__(self, parent_stream): self._buffers = {threading.get_ident(): parent_stream} def __getattr__(self, name): thread_id = threading.get_ident() self._ensure_thread_buffer(thread_id) return getattr(self._buffers[thread_id], name) def _ensure_thread_buffer(self, thread_id): if thread_id not in self._buffers: self._buffers[thread_id] = io.StringIO() def write(self, value): thread_id = threading.get_ident() self._ensure_thread_buffer(thread_id) return self._buffers[thread_id].write( value.decode() if is_bytes(value) else value ) def get_value_and_reset(self): result = "" try: result = self.getvalue() self.seek(0) self.truncate(0) except AttributeError: pass return result @util.memoized(expire="60s") def get_core_fullpath(): return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) class PIOCoreRPC(BaseRPCHandler): @staticmethod def version(): return __version__ async def exec(self, args, options=None): loop = aio_get_running_loop() exit_future = loop.create_future() data_callback = functools.partial( self._on_exec_data_received, exec_options=options ) if args[0] != "--caller" and app.get_session_var("caller_id"): args = ["--caller", app.get_session_var("caller_id")] + args transport, protocol = await loop.subprocess_exec( lambda: PIOCoreProtocol(exit_future, data_callback), get_core_fullpath(), *args, stdin=None, **options.get("spawn", {}), ) await exit_future transport.close() return { "stdout": protocol.stdout, "stderr": protocol.stderr, "returncode": transport.get_returncode(), } def _on_exec_data_received(self, exec_options, pipe, data): notification_method = exec_options.get(f"{pipe}NotificationMethod") if not notification_method: return aio_create_task( self.factory.notify_clients( method=notification_method, params=[data], actor="frontend", ) ) @staticmethod def setup_multithreading_std_streams(): if isinstance(sys.stdout, MultiThreadingStdStream): return PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout) PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr) sys.stdout = PIOCoreRPC.thread_stdout sys.stderr = PIOCoreRPC.thread_stderr @staticmethod async def call(args, options=None): for i, arg in enumerate(args): if not isinstance(arg, str): args[i] = str(arg) options = options or {} to_json = "--json-output" in args try: if options.get("force_subprocess"): result = await PIOCoreRPC._call_subprocess(args, options) return PIOCoreRPC._process_result(result, to_json) result = await PIOCoreRPC._call_inline(args, options) try: return PIOCoreRPC._process_result(result, to_json) except ValueError: # fall-back to subprocess method result = await PIOCoreRPC._call_subprocess(args, options) return PIOCoreRPC._process_result(result, to_json) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( code=5000, message="PIO Core Call Error", data=str(exc) ) from exc @staticmethod async def _call_subprocess(args, options): result = await aio_to_thread( proc.exec_command, [get_core_fullpath()] + args, cwd=options.get("cwd") or os.getcwd(), ) return (result["out"], result["err"], result["returncode"]) @staticmethod async def _call_inline(args, options): PIOCoreRPC.setup_multithreading_std_streams() def _thread_safe_call(args, cwd): with fs.cd(cwd): exit_code = __main__.main(["-c"] + args) return ( PIOCoreRPC.thread_stdout.get_value_and_reset(), PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code, ) return await aio_to_thread( _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd() ) @staticmethod def _process_result(result, to_json=False): out, err, code = result if out and is_bytes(out): out = out.decode(get_locale_encoding()) if err and is_bytes(err): err = err.decode(get_locale_encoding()) text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise PlatformioException(text) if not to_json: return text try: return json.loads(out) except ValueError as exc: click.secho("%s => `%s`" % (exc, out), fg="red", err=True) # if PIO Core prints unhandled warnings for line in out.split("\n"): line = line.strip() if not line: continue try: return json.loads(line) except ValueError: pass raise exc ================================================ FILE: platformio/home/rpc/handlers/platform.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os.path from platformio.compat import aio_to_thread from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory class PlatformRPC(BaseRPCHandler): async def fetch_platforms(self, search_query=None, page=0, force_installed=False): if force_installed: return { "items": await aio_to_thread( self._load_installed_platforms, search_query ) } search_result = await self.factory.manager.dispatcher["registry.call_client"]( method="list_packages", query=search_query, qualifiers={ "types": ["platform"], }, page=page, ) return { "page": search_result["page"], "limit": search_result["limit"], "total": search_result["total"], "items": [ { "id": item["id"], "ownername": item["owner"]["username"], "name": item["name"], "version": item["version"]["name"], "description": item["description"], "tier": item["tier"], } for item in search_result["items"] ], } @staticmethod def _load_installed_platforms(search_query=None): search_query = (search_query or "").strip() def _matchSearchQuery(p): content_blocks = [p.name, p.title, p.description] if p.frameworks: content_blocks.append(" ".join(p.frameworks.keys())) for board in p.get_boards().values(): board_data = board.get_brief_data() for key in ("id", "mcu", "vendor"): content_blocks.append(board_data.get(key)) return search_query in " ".join(content_blocks) items = [] pm = PlatformPackageManager() for pkg in pm.get_installed(): p = PlatformFactory.new(pkg) if search_query and not _matchSearchQuery(p): continue items.append( { "__pkg_path": pkg.path, "ownername": pkg.metadata.spec.owner if pkg.metadata.spec else None, "name": p.name, "version": str(pkg.metadata.version), "title": p.title, "description": p.description, } ) return items async def fetch_boards(self, platform_spec): spec = PackageSpec(platform_spec) if spec.owner: return await self.factory.manager.dispatcher["registry.call_client"]( method="get_package", typex="platform", owner=spec.owner, name=spec.name, extra_path="/boards", ) return await aio_to_thread(self._load_installed_boards, spec) @staticmethod def _load_installed_boards(platform_spec): p = PlatformFactory.new(platform_spec) return sorted( [b.get_brief_data() for b in p.get_boards().values()], key=lambda item: item["name"], ) async def fetch_examples(self, platform_spec): spec = PackageSpec(platform_spec) if spec.owner: return await self.factory.manager.dispatcher["registry.call_client"]( method="get_package", typex="platform", owner=spec.owner, name=spec.name, extra_path="/examples", ) return await aio_to_thread(self._load_installed_examples, spec) @staticmethod def _load_installed_examples(platform_spec): platform = PlatformFactory.new(platform_spec) platform_dir = platform.get_dir() parser = ManifestParserFactory.new_from_dir(platform_dir) result = parser.as_dict().get("examples") or [] for example in result: example["files"] = [ { "path": item, "url": ( "file://%s" + os.path.join(platform_dir, "examples", example["name"], item) ), } for item in example["files"] ] return result ================================================ FILE: platformio/home/rpc/handlers/project.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import time from pathlib import Path import semantic_version from ajsonrpc.core import JSONRPC20DispatchException from platformio import app, exception, fs from platformio.home.rpc.handlers.app import AppRPC from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import get_project_dir, is_platformio_project from platformio.project.integration.generator import ProjectGenerator from platformio.project.options import get_config_options_schema class ProjectRPC(BaseRPCHandler): @staticmethod def config_call(init_kwargs, method, *args): assert isinstance(init_kwargs, dict) assert "path" in init_kwargs if os.path.isdir(init_kwargs["path"]): project_dir = init_kwargs["path"] init_kwargs["path"] = os.path.join(init_kwargs["path"], "platformio.ini") elif os.path.isfile(init_kwargs["path"]): project_dir = os.path.dirname(init_kwargs["path"]) else: project_dir = get_project_dir() with fs.cd(project_dir): return getattr(ProjectConfig(**init_kwargs), method)(*args) @staticmethod def config_load(path): return ProjectConfig( path, parse_extra=False, expand_interpolations=False ).as_tuple() @staticmethod def config_dump(path, data): config = ProjectConfig(path, parse_extra=False, expand_interpolations=False) config.update(data, clear=True) return config.save() @staticmethod def config_update_description(path, text): config = ProjectConfig(path, parse_extra=False, expand_interpolations=False) if not config.has_section("platformio"): config.add_section("platformio") if text: config.set("platformio", "description", text) else: if config.has_option("platformio", "description"): config.remove_option("platformio", "description") if not config.options("platformio"): config.remove_section("platformio") return config.save() @staticmethod def get_config_schema(): return get_config_options_schema() @staticmethod def get_projects(): def _get_project_data(): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} config = ProjectConfig() data["envs"] = config.envs() data["description"] = config.get("platformio", "description") data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) libdeps_dir = config.get("platformio", "libdeps_dir") for section in config.sections(): if not section.startswith("env:"): continue data["envLibdepsDirs"].append(os.path.join(libdeps_dir, section[4:])) if config.has_option(section, "board"): data["boards"].append(config.get(section, "board")) data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", [])) # skip non existing folders and resolve full path for key in ("envLibdepsDirs", "libExtraDirs"): data[key] = [ fs.expanduser(d) if d.startswith("~") else os.path.abspath(d) for d in data[key] if os.path.isdir(d) ] return data def _path_to_name(path): return (os.path.sep).join(path.split(os.path.sep)[-2:]) result = [] pm = PlatformPackageManager() for project_dir in AppRPC.load_state()["storage"]["recentProjects"]: if not os.path.isdir(project_dir): continue data = {} boards = [] try: with fs.cd(project_dir): data = _get_project_data() except ProjectError: continue for board_id in data.get("boards", []): name = board_id try: name = pm.board_config(board_id)["name"] except exception.PlatformioException: pass boards.append({"id": board_id, "name": name}) result.append( { "path": project_dir, "name": _path_to_name(project_dir), "modified": int(os.path.getmtime(project_dir)), "boards": boards, "description": data.get("description"), "envs": data.get("envs", []), "envLibStorages": [ {"name": os.path.basename(d), "path": d} for d in data.get("envLibdepsDirs", []) ], "extraLibStorages": [ {"name": _path_to_name(d), "path": d} for d in data.get("libExtraDirs", []) ], } ) return result @staticmethod def get_project_examples(): result = [] pm = PlatformPackageManager() for pkg in pm.get_installed(): examples_dir = os.path.join(pkg.path, "examples") if not os.path.isdir(examples_dir): continue items = [] for project_dir, _, __ in os.walk(examples_dir): project_description = None try: config = ProjectConfig(os.path.join(project_dir, "platformio.ini")) config.validate(silent=True) project_description = config.get("platformio", "description") except ProjectError: continue path_tokens = project_dir.split(os.path.sep) items.append( { "name": "/".join( path_tokens[path_tokens.index("examples") + 1 :] ), "path": project_dir, "description": project_description, } ) manifest = pm.load_manifest(pkg) result.append( { "platform": { "title": manifest["title"], "version": manifest["version"], }, "items": sorted(items, key=lambda item: item["name"]), } ) return sorted(result, key=lambda data: data["platform"]["title"]) async def init(self, board, framework, project_dir): assert project_dir if not os.path.isdir(project_dir): os.makedirs(project_dir) args = ["init", "--board", board, "--sample-code"] if framework: args.extend(["--project-option", "framework = %s" % framework]) ide = app.get_session_var("caller_id") if ide in ProjectGenerator.get_supported_ides(): args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) return project_dir @staticmethod async def import_arduino(board, use_arduino_libs, arduino_project_dir): board = str(board) # don't import PIO Project if is_platformio_project(arduino_project_dir): return arduino_project_dir is_arduino_project = any( os.path.isfile( os.path.join( arduino_project_dir, "%s.%s" % (os.path.basename(arduino_project_dir), ext), ) ) for ext in ("ino", "pde") ) if not is_arduino_project: raise JSONRPC20DispatchException( code=4000, message="Not an Arduino project: %s" % arduino_project_dir ) state = AppRPC.load_state() project_dir = os.path.join( state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board ) if not os.path.isdir(project_dir): os.makedirs(project_dir) args = ["init", "--board", board] args.extend(["--project-option", "framework = arduino"]) if use_arduino_libs: args.extend( ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"] ) ide = app.get_session_var("caller_id") if ide in ProjectGenerator.get_supported_ides(): args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) with fs.cd(project_dir): config = ProjectConfig() src_dir = config.get("platformio", "src_dir") if os.path.isdir(src_dir): fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir, symlinks=True) return project_dir @staticmethod async def import_pio(project_dir): if not project_dir or not is_platformio_project(project_dir): raise JSONRPC20DispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir ) new_project_dir = os.path.join( AppRPC.load_state()["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir), ) shutil.copytree(project_dir, new_project_dir, symlinks=True) args = ["init"] ide = app.get_session_var("caller_id") if ide in ProjectGenerator.get_supported_ides(): args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": new_project_dir, "force_subprocess": True} ) return new_project_dir async def init_v2(self, configuration, options=None): project_dir = os.path.join(configuration["location"], configuration["name"]) if not os.path.isdir(project_dir): os.makedirs(project_dir) envclone = os.environ.copy() envclone["PLATFORMIO_FORCE_ANSI"] = "true" options = options or {} options["spawn"] = {"env": envclone, "cwd": project_dir} args = ["project", "init"] ide = app.get_session_var("caller_id") if ide in ProjectGenerator.get_supported_ides(): args.extend(["--ide", ide]) if configuration.get("example"): await self.factory.notify_clients( method=options.get("stdoutNotificationMethod"), params=["Copying example files...\n"], actor="frontend", ) await self._pre_init_example(configuration, project_dir) else: args.extend(self._pre_init_empty(configuration)) return await self.factory.manager.dispatcher["core.exec"](args, options=options) @staticmethod def _pre_init_empty(configuration): project_options = [] platform = configuration["platform"] board_id = configuration.get("board", {}).get("id") env_name = board_id or platform["name"] if configuration.get("description"): project_options.append(("description", configuration.get("description"))) try: v = semantic_version.Version(platform.get("version")) assert not v.prerelease project_options.append( ("platform", "{name} @ ^{version}".format(**platform)) ) except (AssertionError, ValueError): project_options.append( ("platform", "{name} @ {version}".format(**platform)) ) if board_id: project_options.append(("board", board_id)) if configuration.get("framework"): project_options.append(("framework", configuration["framework"]["name"])) args = ["-e", env_name, "--sample-code"] for name, value in project_options: args.extend(["-O", f"{name}={value}"]) return args async def _pre_init_example(self, configuration, project_dir): for item in configuration["example"]["files"]: p = Path(project_dir).joinpath(item["path"]) if not p.parent.is_dir(): p.parent.mkdir(parents=True) p.write_text( await self.factory.manager.dispatcher["os.request_content"]( item["url"] ), encoding="utf-8", ) return [] @staticmethod def configuration(project_dir, env): assert is_platformio_project(project_dir) with fs.cd(project_dir): config = ProjectConfig(os.path.join(project_dir, "platformio.ini")) platform = PlatformFactory.from_env(env, autoinstall=True) platform_pkg = PlatformPackageManager().get_package(platform.get_dir()) board_id = config.get(f"env:{env}", "board", None) # frameworks frameworks = [] for name in config.get(f"env:{env}", "framework", []): if name not in platform.frameworks: continue f_pkg_name = platform.frameworks[name].get("package") if not f_pkg_name: continue f_pkg = platform.get_package(f_pkg_name) if not f_pkg: continue f_manifest = platform.pm.load_manifest(f_pkg) frameworks.append( dict( name=name, title=f_manifest.get("title"), version=str(f_pkg.metadata.version), ) ) return dict( platform=dict( ownername=( platform_pkg.metadata.spec.owner if platform_pkg.metadata.spec else None ), name=platform.name, title=platform.title, version=str(platform_pkg.metadata.version), ), board=( platform.board_config(board_id).get_brief_data() if board_id else None ), frameworks=frameworks or None, ) ================================================ FILE: platformio/home/rpc/handlers/registry.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from ajsonrpc.core import JSONRPC20DispatchException from platformio.compat import aio_to_thread from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.registry.client import RegistryClient class RegistryRPC(BaseRPCHandler): @staticmethod async def call_client(method, *args, **kwargs): try: client = RegistryClient() return await aio_to_thread(getattr(client, method), *args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( code=5000, message="Registry Call Error", data=str(exc) ) from exc ================================================ FILE: platformio/home/rpc/server.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from urllib.parse import parse_qs import ajsonrpc.utils import click from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request from ajsonrpc.dispatcher import Dispatcher from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response from starlette.endpoints import WebSocketEndpoint from platformio.compat import aio_create_task, aio_get_running_loop from platformio.http import InternetConnectionError from platformio.proc import force_exit # Remove this line when PR is merged # https://github.com/pavlov99/ajsonrpc/pull/22 ajsonrpc.utils.is_invalid_params = lambda: False class JSONRPCServerFactoryBase: connection_nums = 0 shutdown_timer = None def __init__(self, shutdown_timeout=0): self.shutdown_timeout = shutdown_timeout self.manager = AsyncJSONRPCResponseManager( Dispatcher(), is_server_error_verbose=True ) self._clients = {} def __call__(self, *args, **kwargs): raise NotImplementedError def add_object_handler(self, handler, namespace): handler.factory = self self.manager.dispatcher.add_object(handler, prefix="%s." % namespace) def on_client_connect(self, connection, actor=None): self._clients[connection] = {"actor": actor} self.connection_nums += 1 if self.shutdown_timer: self.shutdown_timer.cancel() self.shutdown_timer = None def on_client_disconnect(self, connection): if connection in self._clients: del self._clients[connection] self.connection_nums -= 1 if self.connection_nums < 1: self.connection_nums = 0 if self.connection_nums == 0: self.shutdown_by_timeout() async def on_shutdown(self): pass def shutdown_by_timeout(self): if self.shutdown_timeout < 1: return def _auto_shutdown_server(): click.echo("Automatically shutdown server on timeout") force_exit() self.shutdown_timer = aio_get_running_loop().call_later( self.shutdown_timeout, _auto_shutdown_server ) async def notify_clients(self, method, params=None, actor=None): for client, options in self._clients.items(): if actor and options["actor"] != actor: continue request = JSONRPC20Request(method, params, is_notification=True) await client.send_text(self.manager.serialize(request.body)) return True class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase): def __call__(self, *args, **kwargs): ws = WebSocketJSONRPCServer(*args, **kwargs) ws.factory = self return ws class WebSocketJSONRPCServer(WebSocketEndpoint): encoding = "text" factory: WebSocketJSONRPCServerFactory = None async def on_connect(self, websocket): await websocket.accept() qs = parse_qs(self.scope.get("query_string", b"")) actors = qs.get(b"actor") self.factory.on_client_connect( # pylint: disable=no-member websocket, actor=actors[0].decode() if actors else None ) async def on_receive(self, websocket, data): aio_create_task(self._handle_rpc(websocket, data)) async def on_disconnect(self, websocket, close_code): self.factory.on_client_disconnect(websocket) # pylint: disable=no-member async def _handle_rpc(self, websocket, data): # pylint: disable=no-member response = await self.factory.manager.get_response_for_payload(data) if response.error and response.error.data: click.secho("Error: %s" % response.error.data, fg="red", err=True) if InternetConnectionError.MESSAGE in response.error.data: response = JSONRPC20Response( id=response.id, error=JSONRPC20Error( code=4008, message="No Internet Connection", data=response.error.data, ), ) await websocket.send_text(self.factory.manager.serialize(response.body)) ================================================ FILE: platformio/home/run.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from urllib.parse import urlparse import click import uvicorn from starlette.applications import Starlette from starlette.middleware import Middleware from starlette.responses import PlainTextResponse from starlette.routing import Mount, Route, WebSocketRoute from starlette.staticfiles import StaticFiles from starlette.status import HTTP_403_FORBIDDEN from platformio.compat import aio_get_running_loop from platformio.exception import PlatformioException from platformio.home.rpc.handlers.account import AccountRPC from platformio.home.rpc.handlers.app import AppRPC from platformio.home.rpc.handlers.ide import IDERPC from platformio.home.rpc.handlers.misc import MiscRPC from platformio.home.rpc.handlers.os import OSRPC from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.home.rpc.handlers.platform import PlatformRPC from platformio.home.rpc.handlers.project import ProjectRPC from platformio.home.rpc.handlers.registry import RegistryRPC from platformio.home.rpc.server import WebSocketJSONRPCServerFactory from platformio.package.manager.core import get_core_package_dir from platformio.proc import force_exit class ShutdownMiddleware: def __init__(self, app): self.app = app async def __call__(self, scope, receive, send): if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", ""): await shutdown_server() await self.app(scope, receive, send) async def shutdown_server(_=None): aio_get_running_loop().call_later(0.5, force_exit) return PlainTextResponse("Server has been shutdown!") async def protected_page(_): return PlainTextResponse( "Protected PlatformIO Home session", status_code=HTTP_403_FORBIDDEN ) def run_server(host, port, no_open, shutdown_timeout, home_url): contrib_dir = get_core_package_dir("contrib-piohome") if not os.path.isdir(contrib_dir): raise PlatformioException("Invalid path to PIO Home Contrib") ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout) ws_rpc_factory.add_object_handler(AccountRPC(), namespace="account") ws_rpc_factory.add_object_handler(AppRPC(), namespace="app") ws_rpc_factory.add_object_handler(IDERPC(), namespace="ide") ws_rpc_factory.add_object_handler(MiscRPC(), namespace="misc") ws_rpc_factory.add_object_handler(OSRPC(), namespace="os") ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core") ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project") ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform") ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry") path = urlparse(home_url).path routes = [ WebSocketRoute(path + "wsrpc", ws_rpc_factory, name="wsrpc"), Route(path + "__shutdown__", shutdown_server, methods=["POST"]), Mount(path, StaticFiles(directory=contrib_dir, html=True), name="static"), ] if path != "/": routes.append(Route("/", protected_page)) uvicorn.run( Starlette( middleware=[Middleware(ShutdownMiddleware)], routes=routes, on_startup=[ lambda: click.echo( "PIO Home has been started. Press Ctrl+C to shutdown." ), lambda: None if no_open else click.launch(home_url), ], ), host=host, port=port, log_level="warning", ) ================================================ FILE: platformio/http.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import socket from urllib.parse import urljoin import requests.adapters from urllib3.util.retry import Retry from platformio import __check_internet_hosts__, app, util from platformio.cache import ContentCache, cleanup_content_cache from platformio.compat import is_proxy_set from platformio.exception import PlatformioException, UserSideException __default_requests_timeout__ = (10, None) # (connect, read) class HTTPClientError(UserSideException): def __init__(self, message, response=None): super().__init__() self.message = message self.response = response def __str__(self): # pragma: no cover return self.message class InternetConnectionError(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" "PlatformIO needs the Internet connection to" " download dependent packages or to work with PlatformIO Account." ) class HTTPSession(requests.Session): def __init__(self, *args, **kwargs): self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None super().__init__(*args, **kwargs) self.headers.update({"User-Agent": app.get_user_agent()}) try: self.verify = app.get_setting("enable_proxy_strict_ssl") except PlatformioException: self.verify = True def request( # pylint: disable=signature-differs,arguments-differ self, method, url, *args, **kwargs ): # print("HTTPSession::request", self._x_base_url, method, url, args, kwargs) if "timeout" not in kwargs: kwargs["timeout"] = __default_requests_timeout__ return super().request( method, ( url if url.startswith("http") or not self._x_base_url else urljoin(self._x_base_url, url) ), *args, **kwargs ) class HTTPSessionIterator: def __init__(self, endpoints): if not isinstance(endpoints, list): endpoints = [endpoints] self.endpoints = endpoints self.endpoints_iter = iter(endpoints) # https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html self.retry = Retry( total=5, backoff_factor=1, # [0, 2, 4, 8, 16] secs # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], status_forcelist=[413, 500, 502, 503, 504], ) def __iter__(self): # pylint: disable=non-iterator-returned return self def __next__(self): base_url = next(self.endpoints_iter) session = HTTPSession(x_base_url=base_url) adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) session.mount(base_url, adapter) return session class HTTPClient: def __init__(self, endpoints): self._session_iter = HTTPSessionIterator(endpoints) self._session = None self._next_session() def __del__(self): if not self._session: return try: self._session.close() except: # pylint: disable=bare-except pass self._session = None def _next_session(self): if self._session: self._session.close() self._session = next(self._session_iter) @util.throttle(500) def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) headers = kwargs.get("headers", {}) with_authorization = ( kwargs.pop("x_with_authorization") if "x_with_authorization" in kwargs else False ) if with_authorization and "Authorization" not in headers: # pylint: disable=import-outside-toplevel from platformio.account.client import AccountClient headers["Authorization"] = ( "Bearer %s" % AccountClient().fetch_authentication_token() ) kwargs["headers"] = headers while True: try: return getattr(self._session, method)(path, **kwargs) except requests.exceptions.RequestException as exc: try: self._next_session() except Exception as exc2: raise HTTPClientError(str(exc2)) from exc def fetch_json_data(self, method, path, **kwargs): if method not in ("get", "head", "options"): cleanup_content_cache("http") cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None if not cache_valid: return self._parse_json_response(self.send_request(method, path, **kwargs)) cache_key = ContentCache.key_from_args( method, path, kwargs.get("params"), kwargs.get("data") ) with ContentCache("http") as cc: result = cc.get(cache_key) if result is not None: try: return json.loads(result) except json.JSONDecodeError: pass response = self.send_request(method, path, **kwargs) data = self._parse_json_response(response) cc.set(cache_key, response.text, cache_valid) return data @staticmethod def _parse_json_response(response, expected_codes=(200, 201, 202)): if response.status_code in expected_codes: try: return response.json() except ValueError: pass try: message = response.json()["message"] except (KeyError, ValueError): message = response.text raise HTTPClientError(message, response) # # Helpers # @util.memoized(expire="10s") def _internet_on(): timeout = 2 use_proxy = is_proxy_set() socket.setdefaulttimeout(timeout) for host in __check_internet_hosts__: try: if use_proxy: requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) return True # try to resolve `host` for both AF_INET and AF_INET6, and then try to connect # to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds: s = socket.create_connection((host, 80)) s.close() return True except: # pylint: disable=bare-except pass # falling back to HTTPs, issue #4980 for host in __check_internet_hosts__: try: requests.get("https://%s" % host, allow_redirects=False, timeout=timeout) except requests.exceptions.RequestException: pass return True return False def ensure_internet_on(raise_exception=False): result = _internet_on() if raise_exception and not result: raise InternetConnectionError() return result def fetch_remote_content(*args, **kwargs): with HTTPSession() as s: r = s.get(*args, **kwargs) r.raise_for_status() r.close() return r.text ================================================ FILE: platformio/maintenance.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil from time import time import click import semantic_version from platformio import __version__, app, exception, fs, telemetry from platformio.cache import cleanup_content_cache from platformio.cli import PlatformioCLI from platformio.commands.upgrade import get_latest_version from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on from platformio.package.manager.core import update_core_packages from platformio.package.version import pepver_to_semver from platformio.system.prune import calculate_unnecessary_system_data def on_cmd_start(ctx, caller): app.set_session_var("command_ctx", ctx) set_caller(caller) telemetry.on_cmd_start(ctx) if PlatformioCLI.in_silence(): return after_upgrade(ctx) def on_cmd_end(): if PlatformioCLI.in_silence(): return try: check_platformio_upgrade() check_prune_system() except ( HTTPClientError, InternetConnectionError, exception.GetLatestVersionError, ): click.secho( "Failed to check for PlatformIO upgrades. " "Please check your Internet connection.", fg="red", ) def on_platformio_exception(exc): telemetry.log_exception(exc) def on_platformio_exit(): telemetry.on_exit() def set_caller(caller=None): caller = caller or os.getenv("PLATFORMIO_CALLER") if not caller: if os.getenv("CODESPACES"): caller = "codespaces" elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): caller = "vscode" elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): caller = "gitpod" if caller: app.set_session_var("caller_id", caller) class Upgrader: def __init__(self, from_version, to_version): self.from_version = from_version self.to_version = to_version self._upgraders = [ (semantic_version.Version("6.1.8-a.1"), self._appstate_migration), ] def run(self, ctx): if self.from_version > self.to_version: return True result = [True] for version, callback in self._upgraders: if self.from_version >= version or self.to_version < version: continue result.append(callback(ctx)) return all(result) @staticmethod def _appstate_migration(_): state_path = app.resolve_state_path("core_dir", "appstate.json") if not os.path.isfile(state_path): return True app.delete_state_item("telemetry") created_at = app.get_state_item("created_at", None) if not created_at: state_stat = os.stat(state_path) app.set_state_item( "created_at", int( state_stat.st_birthtime if hasattr(state_stat, "st_birthtime") else state_stat.st_ctime ), ) return True def after_upgrade(ctx): terminal_width = shutil.get_terminal_size().columns last_version_str = app.get_state_item("last_version", "0.0.0") if last_version_str == __version__: return None if last_version_str == "0.0.0": app.set_state_item("last_version", __version__) return print_welcome_banner() last_version = pepver_to_semver(last_version_str) current_version = pepver_to_semver(__version__) if last_version > current_version and not last_version.prerelease: click.secho("*" * terminal_width, fg="yellow") click.secho( "Obsolete PIO Core v%s is used (previous was %s)" % (__version__, last_version_str), fg="yellow", ) click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") click.secho( "https://docs.platformio.org/en/latest/core" "/installation/troubleshooting.html", fg="cyan", ) click.secho("*" * terminal_width, fg="yellow") return None click.secho("Please wait while upgrading PlatformIO...", fg="yellow") # Update PlatformIO's Core packages cleanup_content_cache("http") update_core_packages() u = Upgrader(last_version, current_version) if u.run(ctx): app.set_state_item("last_version", __version__) click.secho( "PlatformIO has been successfully upgraded to %s!\n" % __version__, fg="green", ) telemetry.log_event( "pio_upgrade_core", { "label": "%s > %s" % (last_version_str, __version__), "from_version": last_version_str, "to_version": __version__, }, ) return print_welcome_banner() def print_welcome_banner(): terminal_width = shutil.get_terminal_size().columns click.echo("*" * terminal_width) click.echo("If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) click.echo( "- %s it on GitHub > %s" % ( click.style("star", fg="cyan"), click.style("https://github.com/platformio/platformio-core", fg="cyan"), ) ) click.echo( "- %s us on LinkedIn to stay up-to-date " "on the latest project news > %s" % ( click.style("follow", fg="cyan"), click.style("https://www.linkedin.com/company/platformio/", fg="cyan"), ) ) if not os.getenv("PLATFORMIO_IDE"): click.echo( "- %s PlatformIO IDE for embedded development > %s" % ( click.style("try", fg="cyan"), click.style("https://platformio.org/platformio-ide", fg="cyan"), ) ) click.echo("*" * terminal_width) click.echo("") def check_platformio_upgrade(): interval = int(app.get_setting("check_platformio_interval")) * 3600 * 24 check_state = app.get_state_item("last_check", {}) last_checked_time = check_state.get("platformio_upgrade", 0) if (time() - interval) < last_checked_time: return check_state["platformio_upgrade"] = int(time()) app.set_state_item("last_check", check_state) if not last_checked_time: return ensure_internet_on(raise_exception=True) # Update PlatformIO Core packages update_core_packages() latest_version = get_latest_version() if pepver_to_semver(latest_version) <= pepver_to_semver(__version__): return terminal_width = shutil.get_terminal_size().columns click.echo("") click.echo("*" * terminal_width) click.secho( "There is a new version %s of PlatformIO available.\n" "Please upgrade it via `" % latest_version, fg="yellow", nl=False, ) if os.path.join("Cellar", "platformio") in fs.get_source_dir(): click.secho("brew update && brew upgrade", fg="cyan", nl=False) click.secho("` command.", fg="yellow") else: click.secho("platformio upgrade", fg="cyan", nl=False) click.secho("` or `", fg="yellow", nl=False) click.secho("python -m pip install -U platformio", fg="cyan", nl=False) click.secho("` command.", fg="yellow") click.secho("Changes: ", fg="yellow", nl=False) click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") click.echo("*" * terminal_width) click.echo("") def check_prune_system(): interval = 30 * 3600 * 24 # 1 time per month check_state = app.get_state_item("last_check", {}) last_checked_time = check_state.get("prune_system", 0) if (time() - interval) < last_checked_time: return check_state["prune_system"] = int(time()) app.set_state_item("last_check", check_state) if not last_checked_time: return threshold_mb = int(app.get_setting("check_prune_system_threshold") or 0) if threshold_mb <= 0: return unnecessary_size = calculate_unnecessary_system_data() if (unnecessary_size / 1024) < threshold_mb: return terminal_width = shutil.get_terminal_size().columns click.echo() click.echo("*" * terminal_width) click.secho( "We found %s of unnecessary PlatformIO system data (temporary files, " "unnecessary packages, etc.).\nUse `pio system prune --dry-run` to list " "them or `pio system prune` to save disk space." % fs.humanize_file_size(unnecessary_size), fg="yellow", ) ================================================ FILE: platformio/package/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/package/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.package.commands.exec import package_exec_cmd from platformio.package.commands.install import package_install_cmd from platformio.package.commands.list import package_list_cmd from platformio.package.commands.outdated import package_outdated_cmd from platformio.package.commands.pack import package_pack_cmd from platformio.package.commands.publish import package_publish_cmd from platformio.package.commands.search import package_search_cmd from platformio.package.commands.show import package_show_cmd from platformio.package.commands.uninstall import package_uninstall_cmd from platformio.package.commands.unpublish import package_unpublish_cmd from platformio.package.commands.update import package_update_cmd @click.group( "pkg", commands=[ package_exec_cmd, package_install_cmd, package_list_cmd, package_outdated_cmd, package_pack_cmd, package_publish_cmd, package_search_cmd, package_show_cmd, package_uninstall_cmd, package_unpublish_cmd, package_update_cmd, ], short_help="Unified Package Manager", ) def cli(): pass ================================================ FILE: platformio/package/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/package/commands/exec.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import subprocess import click from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.exception import ReturnErrorCode, UserSideException from platformio.package.manager.tool import ToolPackageManager from platformio.proc import get_pythonexe_path, where_is_program @click.command("exec", short_help="Run command from package tool") @click.option("-p", "--package", metavar="SPECIFICATION") @click.option("-c", "--call", metavar=" [args...]") @click.argument("args", nargs=-1, type=click.UNPROCESSED) @click.pass_obj def package_exec_cmd(obj, package, call, args): if not call and not args: raise click.BadArgumentUsage("Please provide command name") pkg = None if package: pm = ToolPackageManager() pkg = pm.get_package(package) if not pkg: pkg = pm.install(package) else: executable = args[0] if args else call.split(" ")[0] pkg = find_pkg_by_executable(executable) if not pkg: raise UserSideException( "Could not find a package with '%s' executable file" % executable ) click.echo( "Using %s package" % click.style("%s@%s" % (pkg.metadata.name, pkg.metadata.version), fg="cyan") ) inject_pkg_to_environ(pkg) os.environ["PIO_PYTHON_EXE"] = get_pythonexe_path() # inject current python interpreter on Windows if args and args[0].endswith(".py"): args = [os.environ["PIO_PYTHON_EXE"]] + list(args) if not os.path.exists(args[1]): args[1] = where_is_program(args[1]) result = None try: run_options = dict(shell=call is not None, env=os.environ) force_click_stream = (obj or {}).get("force_click_stream") if force_click_stream: run_options.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) result = subprocess.run( # pylint: disable=subprocess-run-check call or args, **run_options ) if force_click_stream: click.echo(result.stdout.decode().strip(), err=result.returncode != 0) except Exception as exc: raise UserSideException(exc) from exc if result and result.returncode != 0: raise ReturnErrorCode(result.returncode) def find_pkg_by_executable(executable): exes = [executable] if IS_WINDOWS and not executable.endswith(".exe"): exes.append(f"{executable}.exe") for pkg in ToolPackageManager().get_installed(): for exe in exes: if os.path.exists(os.path.join(pkg.path, exe)) or os.path.exists( os.path.join(pkg.path, "bin", exe) ): return pkg return None def inject_pkg_to_environ(pkg): bin_dir = os.path.join(pkg.path, "bin") lib_dir = os.path.join(pkg.path, "lib") paths = [bin_dir, pkg.path] if os.path.isdir(bin_dir) else [pkg.path] if os.environ.get("PATH"): paths.append(os.environ.get("PATH")) os.environ["PATH"] = os.pathsep.join(paths) if IS_WINDOWS or not os.path.isdir(lib_dir) or "toolchain" in pkg.metadata.name: return lib_path_key = "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH" lib_paths = [lib_dir] if os.environ.get(lib_path_key): lib_paths.append(os.environ.get(lib_path_key)) os.environ[lib_path_key] = os.pathsep.join(lib_paths) ================================================ FILE: platformio/package/commands/install.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os from pathlib import Path import click from platformio import fs from platformio.package.exception import UnknownPackageError from platformio.package.manager.core import get_core_package_dir from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageCompatibility, PackageSpec from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies from platformio.test.result import TestSuite from platformio.test.runners.factory import TestRunnerFactory @click.command( "install", short_help="Install the project dependencies or custom packages" ) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) @click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) @click.option( "--no-save", is_flag=True, help="Prevent saving specified packages to `platformio.ini`", ) @click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies") @click.option("-g", "--global", is_flag=True, help="Install package globally") @click.option( "--storage-dir", default=None, type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists") @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") def package_install_cmd(**options): if options.get("global") or options.get("storage_dir"): install_global_dependencies(options) else: install_project_dependencies(options) def install_global_dependencies(options): pm = PlatformPackageManager(options.get("storage_dir")) tm = ToolPackageManager(options.get("storage_dir")) lm = LibraryPackageManager(options.get("storage_dir")) for obj in (pm, tm, lm): obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG) for spec in options.get("platforms"): pm.install( spec, skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) for spec in options.get("tools"): tm.install( spec, skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) for spec in options.get("libraries", []): lm.install( spec, skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) def install_project_dependencies(options): environments = options["environments"] with fs.cd(options["project_dir"]): config = ProjectConfig.get_instance() config.validate(environments) for env in config.envs(): if environments and env not in environments: continue if not options.get("silent"): click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not install_project_env_dependencies(env, options) if not options.get("silent") and already_up_to_date: click.secho("Already up-to-date.", fg="green") def install_project_env_dependencies(project_env, options=None): """Used in `pio run` -> Processor""" options = options or {} installed_conds = [] # custom platforms if options.get("platforms"): installed_conds.append( _install_project_env_custom_platforms(project_env, options) ) # custom tools if options.get("tools"): installed_conds.append(_install_project_env_custom_tools(project_env, options)) # custom libraries if options.get("libraries"): installed_conds.append( _install_project_env_custom_libraries(project_env, options) ) # declared dependencies if not installed_conds: installed_conds = [ _install_project_env_platform(project_env, options), _install_project_env_libraries(project_env, options), ] return any(installed_conds) def _install_project_env_platform(project_env, options): config = ProjectConfig.get_instance() pm = PlatformPackageManager() if options.get("silent"): pm.set_log_level(logging.WARN) spec = config.get(f"env:{project_env}", "platform") if not spec: return False already_up_to_date = not options.get("force") if not pm.get_package(spec): already_up_to_date = False PlatformPackageManager().install( spec, project_env=project_env, project_targets=options.get("project_targets"), skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) # ensure SCons is installed get_core_package_dir("tool-scons") return not already_up_to_date def _install_project_env_custom_platforms(project_env, options): already_up_to_date = not options.get("force") pm = PlatformPackageManager() if not options.get("silent"): pm.set_log_level(logging.DEBUG) for spec in options.get("platforms"): if not pm.get_package(spec): already_up_to_date = False pm.install( spec, project_env=project_env, project_targets=options.get("project_targets"), skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) return not already_up_to_date def _install_project_env_custom_tools(project_env, options): already_up_to_date = not options.get("force") tm = ToolPackageManager() if not options.get("silent"): tm.set_log_level(logging.DEBUG) specs_to_save = [] for tool in options.get("tools"): spec = PackageSpec(tool) if not tm.get_package(spec): already_up_to_date = False pkg = tm.install( spec, skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) specs_to_save.append(pkg_to_save_spec(pkg, spec)) if not options.get("no_save") and specs_to_save: save_project_dependencies( os.getcwd(), specs_to_save, scope="platform_packages", action="add", environments=[project_env], ) return not already_up_to_date def _install_project_env_libraries(project_env, options): already_up_to_date = not options.get("force") config = ProjectConfig.get_instance() compatibility_qualifiers = {} if config.get(f"env:{project_env}", "platform", None): try: p = PlatformFactory.new(config.get(f"env:{project_env}", "platform")) compatibility_qualifiers["platforms"] = [p.name] except UnknownPlatform: pass if config.get(f"env:{project_env}", "framework"): compatibility_qualifiers["frameworks"] = config.get( f"env:{project_env}", "framework" ) env_lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), project_env), compatibility=( PackageCompatibility(**compatibility_qualifiers) if compatibility_qualifiers else None ), ) private_lm = LibraryPackageManager( os.path.join(config.get("platformio", "lib_dir")) ) if options.get("silent"): env_lm.set_log_level(logging.WARN) private_lm.set_log_level(logging.WARN) lib_deps = config.get(f"env:{project_env}", "lib_deps") if "__test" in options.get("project_targets", []): test_runner = TestRunnerFactory.new( TestSuite(project_env, options.get("piotest_running_name", "*")), config ) lib_deps.extend(test_runner.EXTRA_LIB_DEPS or []) _uninstall_project_unused_libdeps(env_lm, lib_deps) for library in lib_deps: spec = PackageSpec(library) # skip built-in dependencies if not spec.external and not spec.owner: continue if not env_lm.get_package(spec): already_up_to_date = False env_lm.install( spec, skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) # install dependencies from the private libraries for pkg in private_lm.get_installed(): _install_project_private_library_deps(pkg, private_lm, env_lm, options) return not already_up_to_date def _uninstall_project_unused_libdeps(lm, lib_deps): lib_deps = set(lib_deps) storage_dir = Path(lm.package_dir) if not lib_deps: if storage_dir.exists(): fs.rmtree(str(storage_dir)) return integrity_dat = storage_dir / "integrity.dat" if integrity_dat.is_file(): prev_lib_deps = set( integrity_dat.read_text(encoding="utf-8").strip().split("\n") ) if lib_deps == prev_lib_deps: return if lm.log.getEffectiveLevel() < logging.WARN: click.secho("Removing unused dependencies...") for spec in set(prev_lib_deps) - set(lib_deps): try: lm.uninstall(spec) except UnknownPackageError: pass if not storage_dir.is_dir(): storage_dir.mkdir(parents=True) integrity_dat.write_text("\n".join(lib_deps), encoding="utf-8") def _install_project_private_library_deps(private_pkg, private_lm, env_lm, options): for dependency in private_lm.get_pkg_dependencies(private_pkg) or []: spec = private_lm.dependency_to_spec(dependency) # skip built-in dependencies if not spec.external and not spec.owner: continue pkg = private_lm.get_package(spec) if ( not pkg and not private_lm.get_package(spec) and not env_lm.get_package(spec) ): pkg = env_lm.install( spec, skip_dependencies=True, force=options.get("force"), ) if not pkg: continue _install_project_private_library_deps(pkg, private_lm, env_lm, options) def _install_project_env_custom_libraries(project_env, options): already_up_to_date = not options.get("force") config = ProjectConfig.get_instance() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), project_env) ) if not options.get("silent"): lm.set_log_level(logging.DEBUG) specs_to_save = [] for library in options.get("libraries") or []: spec = PackageSpec(library) if not lm.get_package(spec): already_up_to_date = False pkg = lm.install( spec, skip_dependencies=options.get("skip_dependencies"), force=options.get("force"), ) specs_to_save.append(pkg_to_save_spec(pkg, spec)) if not options.get("no_save") and specs_to_save: save_project_dependencies( os.getcwd(), specs_to_save, scope="lib_deps", action="add", environments=[project_env], ) return not already_up_to_date ================================================ FILE: platformio/package/commands/list.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from typing import List import click from platformio import fs from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @click.command("list", short_help="List installed packages") @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) @click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) @click.option("-g", "--global", is_flag=True, help="List globally installed packages") @click.option( "--storage-dir", default=None, type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("--only-platforms", is_flag=True, help="List only platform packages") @click.option("--only-tools", is_flag=True, help="List only tool packages") @click.option("--only-libraries", is_flag=True, help="List only library packages") @click.option("-v", "--verbose", is_flag=True) def package_list_cmd(**options): if options.get("global"): list_global_packages(options) else: list_project_packages(options) def humanize_package(pkg, spec=None, verbose=False): if spec and not isinstance(spec, PackageSpec): spec = PackageSpec(spec) data = [ click.style(pkg.metadata.name, fg="cyan"), click.style(f"@ {str(pkg.metadata.version)}", bold=True), ] extra_data = ["required: %s" % (spec.humanize() if spec else "Any")] if verbose: extra_data.append(pkg.path) data.append("(%s)" % ", ".join(extra_data)) return " ".join(data) def print_dependency_tree(pm, specs=None, filter_specs=None, level=0, verbose=False): filtered_pkgs = [ pm.get_package(spec) for spec in filter_specs or [] if pm.get_package(spec) ] candidates = {} if specs: for spec in specs: pkg = pm.get_package(spec) if not pkg: continue candidates[pkg.path] = (pkg, spec) else: candidates = {pkg.path: (pkg, pkg.metadata.spec) for pkg in pm.get_installed()} if not candidates: return candidates = sorted(candidates.values(), key=lambda item: item[0].metadata.name) for index, (pkg, spec) in enumerate(candidates): if filtered_pkgs and not _pkg_tree_contains(pm, pkg, filtered_pkgs): continue printed_pkgs = pm.memcache_get("__printed_pkgs", []) if printed_pkgs and pkg.path in printed_pkgs: continue printed_pkgs.append(pkg.path) pm.memcache_set("__printed_pkgs", printed_pkgs) click.echo( "%s%s %s" % ( "│ " * level, "├──" if index < len(candidates) - 1 else "└──", humanize_package( pkg, spec=spec, verbose=verbose, ), ) ) dependencies = pm.get_pkg_dependencies(pkg) if dependencies: print_dependency_tree( pm, specs=[pm.dependency_to_spec(item) for item in dependencies], filter_specs=filter_specs, level=level + 1, verbose=verbose, ) def _pkg_tree_contains(pm, root: PackageItem, children: List[PackageItem]): if root in children: return True for dependency in pm.get_pkg_dependencies(root) or []: pkg = pm.get_package(pm.dependency_to_spec(dependency)) if pkg and _pkg_tree_contains(pm, pkg, children): return True return False def list_global_packages(options): data = [ ("platforms", PlatformPackageManager(options.get("storage_dir"))), ("tools", ToolPackageManager(options.get("storage_dir"))), ("libraries", LibraryPackageManager(options.get("storage_dir"))), ] only_packages = any( options.get(typex) or options.get(f"only_{typex}") for (typex, _) in data ) for typex, pm in data: skip_conds = [ only_packages and not options.get(typex) and not options.get(f"only_{typex}"), not pm.get_installed(), ] if any(skip_conds): continue click.secho(typex.capitalize(), bold=True) print_dependency_tree( pm, filter_specs=options.get(typex), verbose=options.get("verbose") ) click.echo() def list_project_packages(options): environments = options["environments"] only_packages = any( options.get(typex) or options.get(f"only_{typex}") for typex in ("platforms", "tools", "libraries") ) only_platform_packages = any( options.get(typex) or options.get(f"only_{typex}") for typex in ("platforms", "tools") ) only_library_packages = options.get("libraries") or options.get("only_libraries") with fs.cd(options["project_dir"]): config = ProjectConfig.get_instance() config.validate(environments) for env in config.envs(): if environments and env not in environments: continue click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) found = False if not only_packages or only_platform_packages: _found = print_project_env_platform_packages(env, options) found = found or _found if not only_packages or only_library_packages: _found = print_project_env_library_packages(env, options) found = found or _found if not found: click.echo("No packages") if (not environments and len(config.envs()) > 1) or len(environments) > 1: click.echo() def print_project_env_platform_packages(project_env, options): try: p = PlatformFactory.from_env(project_env) except UnknownPlatform: return None click.echo( "Platform %s" % ( humanize_package( PlatformPackageManager().get_package(p.get_dir()), p.config.get(f"env:{project_env}", "platform"), verbose=options.get("verbose"), ) ) ) print_dependency_tree( p.pm, specs=[p.get_package_spec(name) for name in p.packages], filter_specs=options.get("tools"), ) click.echo() return True def print_project_env_library_packages(project_env, options): config = ProjectConfig.get_instance() lib_deps = config.get(f"env:{project_env}", "lib_deps") lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), project_env) ) if not lib_deps or not lm.get_installed(): return None click.echo("Libraries") print_dependency_tree( lm, lib_deps, filter_specs=options.get("libraries"), verbose=options.get("verbose"), ) return True ================================================ FILE: platformio/package/commands/outdated.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import click from tabulate import tabulate from platformio import fs from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig class OutdatedCandidate: def __init__(self, pm, pkg, spec, envs=None): self.pm = pm self.pkg = pkg self.spec = spec self.envs = envs or [] self.outdated = None if not isinstance(self.envs, list): self.envs = [self.envs] def __eq__(self, other): return all( [ self.pm.package_dir == other.pm.package_dir, self.pkg == other.pkg, self.spec == other.spec, ] ) def check(self): self.outdated = self.pm.outdated(self.pkg, self.spec) def is_outdated(self): if not self.outdated: self.check() return self.outdated.is_outdated(allow_incompatible=self.pm.pkg_type != "tool") @click.command("outdated", short_help="Check for outdated packages") @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) def package_outdated_cmd(project_dir, environments): with fs.cd(project_dir): candidates = fetch_outdated_candidates(environments, with_progress=True) print_outdated_candidates(candidates) def print_outdated_candidates(candidates): if not candidates: click.secho("Everything is up-to-date!", fg="green") return tabulate_data = [ ( click.style( candidate.pkg.metadata.name, fg=get_candidate_update_color(candidate.outdated), ), candidate.outdated.current, candidate.outdated.wanted, click.style(candidate.outdated.latest, fg="cyan"), candidate.pm.pkg_type.capitalize(), ", ".join(set(candidate.envs)), ) for candidate in candidates ] click.echo() click.secho("Semantic Versioning color legend:", bold=True) click.echo( tabulate( [ ( click.style("", fg="red"), "backward-incompatible updates", ), ( click.style("", fg="yellow"), "backward-compatible features", ), ( click.style("", fg="green"), "backward-compatible bug fixes", ), ], tablefmt="plain", ) ) click.echo() click.echo( tabulate( tabulate_data, headers=["Package", "Current", "Wanted", "Latest", "Type", "Environments"], ) ) def get_candidate_update_color(outdated): if outdated.update_increment_type == outdated.UPDATE_INCREMENT_MAJOR: return "red" if outdated.update_increment_type == outdated.UPDATE_INCREMENT_MINOR: return "yellow" if outdated.update_increment_type == outdated.UPDATE_INCREMENT_PATCH: return "green" return None def fetch_outdated_candidates(environments, with_progress=False): candidates = [] config = ProjectConfig.get_instance() config.validate(environments) def _add_candidate(data): new_candidate = OutdatedCandidate( data["pm"], data["pkg"], data["spec"], data["env"] ) for candidate in candidates: if candidate == new_candidate: candidate.envs.append(data["env"]) return candidates.append(new_candidate) # platforms for item in find_platform_candidates(config, environments): _add_candidate(item) # platform package dependencies for dep_item in find_platform_dependency_candidates(item["env"]): _add_candidate(dep_item) # libraries for item in find_library_candidates(config, environments): _add_candidate(item) result = [] if not with_progress: for candidate in candidates: if candidate.is_outdated(): result.append(candidate) return result with click.progressbar(candidates, label="Checking") as pb: for candidate in pb: if candidate.is_outdated(): result.append(candidate) return result def find_platform_candidates(config, environments): result = [] pm = PlatformPackageManager() for env in config.envs(): platform = config.get(f"env:{env}", "platform", None) if not platform or (environments and env not in environments): continue spec = PackageSpec(platform) pkg = pm.get_package(spec) if not pkg: continue result.append(dict(env=env, pm=pm, pkg=pkg, spec=spec)) return result def find_platform_dependency_candidates(env): result = [] p = PlatformFactory.from_env(env) for pkg in p.get_installed_packages(): result.append( dict( env=env, pm=p.pm, pkg=pkg, spec=p.get_package_spec(pkg.metadata.name), ) ) return sorted(result, key=lambda item: item["pkg"].metadata.name) def find_library_candidates(config, environments): result = [] for env in config.envs(): if environments and env not in environments: continue package_dir = os.path.join(config.get("platformio", "libdeps_dir") or "", env) lib_deps = [ item for item in config.get(f"env:{env}", "lib_deps", []) if "/" in item ] if not os.path.isdir(package_dir) or not lib_deps: continue pm = LibraryPackageManager(package_dir) for lib in lib_deps: spec = PackageSpec(lib) pkg = pm.get_package(spec) if not pkg: continue result.append(dict(env=env, pm=pm, pkg=pkg, spec=spec)) return sorted(result, key=lambda item: item["pkg"].metadata.name) ================================================ FILE: platformio/package/commands/pack.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import click from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError from platformio.package.pack import PackagePacker @click.command("pack", short_help="Create a tarball from a package") @click.argument( "package", default=os.getcwd, metavar="", type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "-o", "--output", help="A destination path (folder or a full path to file)" ) def package_pack_cmd(package, output): p = PackagePacker(package) archive_path = p.pack(output) # validate manifest try: ManifestSchema().load_manifest( ManifestParserFactory.new_from_archive(archive_path).as_dict() ) except ManifestValidationError as exc: os.remove(archive_path) raise exc click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") ================================================ FILE: platformio/package/commands/publish.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import tarfile import tempfile from datetime import datetime import click from tabulate import tabulate from platformio import fs from platformio.account.client import AccountClient from platformio.compat import isascii from platformio.exception import UserSideException from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.package.meta import PackageType from platformio.package.pack import PackagePacker from platformio.package.unpack import FileUnpacker, TARArchiver from platformio.registry.client import RegistryClient def validate_datetime(ctx, param, value): # pylint: disable=unused-argument if not value: return value try: datetime.strptime(value, "%Y-%m-%d %H:%M:%S") except ValueError as exc: raise click.BadParameter(exc) return value @click.command("publish", short_help="Publish a package to the registry") @click.argument( "package", default=os.getcwd, metavar="", type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "--owner", help="PIO Account username (can be organization username). " "Default is set to a username of the authorized PIO Account", ) @click.option( "--type", "typex", type=click.Choice(list(PackageType.items().values())), help="Custom package type", ) @click.option( "--released-at", callback=validate_datetime, help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", ) @click.option("--private", is_flag=True, help="Restricted access (not a public)") @click.option( "--notify/--no-notify", default=True, help="Notify by email when package is processed", ) @click.option( "--no-interactive", is_flag=True, help="Do not show interactive prompt", ) @click.option( "--non-interactive", is_flag=True, help="Do not show interactive prompt", hidden=True, ) def package_publish_cmd( # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals package, owner, typex, released_at, private, notify, no_interactive, non_interactive ): click.secho("Preparing a package...", fg="cyan") package = os.path.abspath(package) no_interactive = no_interactive or non_interactive owner = owner or AccountClient().get_logged_username() do_not_pack = ( not os.path.isdir(package) and isinstance(FileUnpacker.new_archiver(package), TARArchiver) and PackageType.from_archive(package) ) archive_path = None with tempfile.TemporaryDirectory() as tmp_dir: # pylint: disable=no-member # publish .tar.gz instantly without repacking if do_not_pack: archive_path = package else: with fs.cd(tmp_dir): p = PackagePacker(package) archive_path = p.pack() typex = typex or PackageType.from_archive(archive_path) manifest = ManifestSchema().load_manifest( ManifestParserFactory.new_from_archive(archive_path).as_dict() ) name = manifest.get("name") version = manifest.get("version") data = [ ("Type:", typex), ("Owner:", owner), ("Name:", name), ("Version:", version), ("Size:", fs.humanize_file_size(os.path.getsize(archive_path))), ] if manifest.get("system"): data.insert(len(data) - 1, ("System:", ", ".join(manifest.get("system")))) click.echo(tabulate(data, tablefmt="plain")) # check files containing non-ascii chars check_archive_file_names(archive_path) # look for duplicates check_package_duplicates(owner, typex, name, version, manifest.get("system")) if not no_interactive: click.confirm( "Are you sure you want to publish the %s %s to the registry?\n" % ( typex, click.style( "%s/%s@%s" % (owner, name, version), fg="cyan", ), ), abort=True, ) click.secho( "The package publishing may take some time depending " "on your Internet connection and the package size.", fg="yellow", ) click.echo("Publishing...") response = RegistryClient().publish_package( owner, typex, archive_path, released_at, private, notify ) if not do_not_pack: os.remove(archive_path) click.secho(response.get("message"), fg="green") def check_archive_file_names(archive_path): with tarfile.open(archive_path, mode="r:gz") as tf: for name in tf.getnames(): if not isascii(name) or not name.isprintable(): click.secho( f"Warning! The `{name}` file contains non-ASCII chars and can " "lead to the unpacking issues on a user machine", fg="yellow", ) def check_package_duplicates( owner, type, name, version, system ): # pylint: disable=redefined-builtin found = False items = ( RegistryClient() .list_packages(qualifiers=dict(types=[type], names=[name])) .get("items") ) if not items: return True # duplicated version by owner / system found = False for item in items: if item["owner"]["username"] != owner or item["version"]["name"] != version: continue if not system: found = True break published_systems = [] for f in item["version"]["files"]: published_systems.extend(f.get("system", [])) found = set(system).issubset(set(published_systems)) if found: raise UserSideException( "The package `%s/%s@%s` is already published in the registry" % (owner, name, version) ) other_owners = [ item["owner"]["username"] for item in items if item["owner"]["username"] != owner ] if other_owners: click.secho( "\nWarning! A package with the name `%s` is already published by the next " "owners: %s\n" % (name, ", ".join(other_owners)), fg="yellow", ) return True ================================================ FILE: platformio/package/commands/search.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import math import click from platformio import util from platformio.registry.client import RegistryClient @click.command("search", short_help="Search for packages") @click.argument("query") @click.option("-p", "--page", type=click.IntRange(min=1)) @click.option( "-s", "--sort", type=click.Choice(["relevance", "popularity", "trending", "added", "updated"]), ) def package_search_cmd(query, page, sort): client = RegistryClient() result = client.list_packages(query, page=page, sort=sort) if not result["total"]: click.secho("Nothing has been found by your request", fg="yellow") click.echo( "Try a less-specific search or use truncation (or wildcard) operator *" ) return print_search_result(result) def print_search_result(result): click.echo( "Found %d packages (page %d of %d)" % ( result["total"], result["page"], math.ceil(result["total"] / result["limit"]), ) ) for item in result["items"]: click.echo() print_search_item(item) def print_search_item(item): click.echo( "%s/%s" % ( click.style(item["owner"]["username"], fg="cyan"), click.style(item["name"], fg="cyan", bold=True), ) ) click.echo( "%s • %s • Published on %s" % ( ( item["type"].capitalize() if item["tier"] == "community" else click.style( ("%s %s" % (item["tier"], item["type"])).title(), bold=True ) ), item["version"]["name"], util.parse_datetime(item["version"]["released_at"]).strftime("%c"), ) ) click.echo(item["description"]) ================================================ FILE: platformio/package/commands/show.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from urllib.parse import quote import click from tabulate import tabulate from platformio import fs, util from platformio.exception import UserSideException from platformio.package.manager._registry import PackageManagerRegistryMixin from platformio.package.meta import PackageSpec, PackageType from platformio.registry.client import RegistryClient @click.command("show", short_help="Show package information") @click.argument("spec", metavar="[/][@]") @click.option( "-t", "--type", "pkg_type", type=click.Choice(list(PackageType.items().values())), help="Package type", ) def package_show_cmd(spec, pkg_type): spec = PackageSpec(spec) data = fetch_package_data(spec, pkg_type) if not data: raise UserSideException( "Could not find '%s' package in the PlatormIO Registry" % spec.humanize() ) click.echo() click.echo( "%s/%s" % ( click.style(data["owner"]["username"], fg="cyan"), click.style(data["name"], fg="cyan", bold=True), ) ) click.echo( "%s • %s • %s • Published on %s" % ( data["type"].capitalize(), data["version"]["name"], "Private" if data.get("private") else "Public", util.parse_datetime(data["version"]["released_at"]).strftime("%c"), ) ) # Description click.echo() click.echo(data["description"]) # Extra info click.echo() fields = [ ("homepage", "Homepage"), ("repository_url", "Repository"), ("license", "License"), ("popularity_rank", "Popularity"), ("stars_count", "Stars"), ("examples_count", "Examples"), ("version.unpacked_size", "Installed Size"), ("dependents_count", "Used By"), ("dependencies_count", "Dependencies"), ("platforms", "Compatible Platforms"), ("frameworks", "Compatible Frameworks"), ("keywords", "Keywords"), ] type_plural = "libraries" if data["type"] == "library" else (data["type"] + "s") extra = [ ( "Registry", click.style( "https://registry.platformio.org/%s/%s/%s" % (type_plural, data["owner"]["username"], quote(data["name"])), fg="blue", ), ) ] for key, title in fields: if "." in key: k1, k2 = key.split(".") value = data.get(k1, {}).get(k2) else: value = data.get(key) if not value: continue if isinstance(value, list): value = ", ".join(value) elif key.endswith("_size"): value = fs.humanize_file_size(value) extra.append((title, value)) click.echo(tabulate(extra)) # Versions click.echo("") table = tabulate( [ ( version["name"], fs.humanize_file_size(max(f["size"] for f in version["files"])), util.parse_datetime(version["released_at"]), ) for version in data["versions"] ], headers=["Version", "Size", "Published"], ) click.echo(table) click.echo("") def fetch_package_data(spec, pkg_type=None): assert isinstance(spec, PackageSpec) client = RegistryClient() if pkg_type and spec.owner and spec.name: return client.get_package( pkg_type, spec.owner, spec.name, version=spec.requirements ) qualifiers = {} if spec.id: qualifiers["ids"] = str(spec.id) if spec.name: qualifiers["names"] = spec.name.lower() if pkg_type: qualifiers["types"] = pkg_type if spec.owner: qualifiers["owners"] = spec.owner.lower() packages = client.list_packages(qualifiers=qualifiers)["items"] if not packages: return None if len(packages) > 1: PackageManagerRegistryMixin.print_multi_package_issue( click.echo, packages, spec ) return None return client.get_package( packages[0]["type"], packages[0]["owner"]["username"], packages[0]["name"], version=spec.requirements, ) ================================================ FILE: platformio/package/commands/uninstall.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import click from platformio import fs from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.project.config import ProjectConfig from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies @click.command( "uninstall", short_help="Uninstall the project dependencies or custom packages" ) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) @click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) @click.option( "--no-save", is_flag=True, help="Prevent removing specified packages from `platformio.ini`", ) @click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies") @click.option("-g", "--global", is_flag=True, help="Uninstall global packages") @click.option( "--storage-dir", default=None, type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") def package_uninstall_cmd(**options): if options.get("global"): uninstall_global_dependencies(options) else: uninstall_project_dependencies(options) def uninstall_global_dependencies(options): pm = PlatformPackageManager(options.get("storage_dir")) tm = ToolPackageManager(options.get("storage_dir")) lm = LibraryPackageManager(options.get("storage_dir")) for obj in (pm, tm, lm): obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG) for spec in options.get("platforms"): pm.uninstall( spec, skip_dependencies=options.get("skip_dependencies"), ) for spec in options.get("tools"): tm.uninstall( spec, skip_dependencies=options.get("skip_dependencies"), ) for spec in options.get("libraries", []): lm.uninstall( spec, skip_dependencies=options.get("skip_dependencies"), ) def uninstall_project_dependencies(options): environments = options["environments"] with fs.cd(options["project_dir"]): config = ProjectConfig.get_instance() config.validate(environments) for env in config.envs(): if environments and env not in environments: continue if not options["silent"]: click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not uninstall_project_env_dependencies(env, options) if not options["silent"] and already_up_to_date: click.secho("Already up-to-date.", fg="green") def uninstall_project_env_dependencies(project_env, options=None): options = options or {} uninstalled_conds = [] # custom platforms if options.get("platforms"): uninstalled_conds.append( _uninstall_project_env_custom_platforms(project_env, options) ) # custom tools if options.get("tools"): uninstalled_conds.append( _uninstall_project_env_custom_tools(project_env, options) ) # custom libraries if options.get("libraries"): uninstalled_conds.append( _uninstall_project_env_custom_libraries(project_env, options) ) # declared dependencies if not uninstalled_conds: uninstalled_conds = [ _uninstall_project_env_platform(project_env, options), _uninstall_project_env_libraries(project_env, options), ] return any(uninstalled_conds) def _uninstall_project_env_platform(project_env, options): config = ProjectConfig.get_instance() pm = PlatformPackageManager() if options.get("silent"): pm.set_log_level(logging.WARN) spec = config.get(f"env:{project_env}", "platform") if not spec: return None already_up_to_date = True if not pm.get_package(spec): return None PlatformPackageManager().uninstall( spec, project_env=project_env, skip_dependencies=options.get("skip_dependencies"), ) return not already_up_to_date def _uninstall_project_env_custom_platforms(project_env, options): already_up_to_date = True pm = PlatformPackageManager() if not options.get("silent"): pm.set_log_level(logging.DEBUG) for spec in options.get("platforms"): if pm.get_package(spec): already_up_to_date = False pm.uninstall( spec, project_env=project_env, skip_dependencies=options.get("skip_dependencies"), ) return not already_up_to_date def _uninstall_project_env_custom_tools(project_env, options): already_up_to_date = True tm = ToolPackageManager() if not options.get("silent"): tm.set_log_level(logging.DEBUG) specs_to_save = [] for tool in options.get("tools"): spec = PackageSpec(tool) if tm.get_package(spec): already_up_to_date = False pkg = tm.uninstall( spec, skip_dependencies=options.get("skip_dependencies"), ) specs_to_save.append(pkg_to_save_spec(pkg, spec)) if not options.get("no_save") and specs_to_save: save_project_dependencies( os.getcwd(), specs_to_save, scope="platform_packages", action="remove", environments=[project_env], ) return not already_up_to_date def _uninstall_project_env_libraries(project_env, options): already_up_to_date = True config = ProjectConfig.get_instance() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), project_env) ) if options.get("silent"): lm.set_log_level(logging.WARN) for library in config.get(f"env:{project_env}", "lib_deps"): spec = PackageSpec(library) # skip built-in dependencies if not spec.external and not spec.owner: continue if lm.get_package(spec): already_up_to_date = False lm.uninstall( spec, skip_dependencies=options.get("skip_dependencies"), ) return not already_up_to_date def _uninstall_project_env_custom_libraries(project_env, options): already_up_to_date = True config = ProjectConfig.get_instance() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), project_env) ) if not options.get("silent"): lm.set_log_level(logging.DEBUG) specs_to_save = [] for library in options.get("libraries") or []: spec = PackageSpec(library) if lm.get_package(spec): already_up_to_date = False pkg = lm.uninstall( spec, skip_dependencies=options.get("skip_dependencies"), ) specs_to_save.append(pkg_to_save_spec(pkg, spec)) if not options.get("no_save") and specs_to_save: save_project_dependencies( os.getcwd(), specs_to_save, scope="lib_deps", action="remove", environments=[project_env], ) return not already_up_to_date ================================================ FILE: platformio/package/commands/unpublish.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.account.client import AccountClient from platformio.package.meta import PackageSpec, PackageType from platformio.registry.client import RegistryClient @click.command("unpublish", short_help="Remove a pushed package from the registry") @click.argument( "package", required=True, metavar="[/][@]" ) @click.option( "--type", type=click.Choice(list(PackageType.items().values())), default="library", help="Package type, default is set to `library`", ) @click.option( "--undo", is_flag=True, help="Undo a remove, putting a version back into the registry", ) def package_unpublish_cmd(package, type, undo): # pylint: disable=redefined-builtin spec = PackageSpec(package) response = RegistryClient().unpublish_package( owner=spec.owner or AccountClient().get_logged_username(), type=type, name=spec.name, version=str(spec.requirements), undo=undo, ) click.secho(response.get("message"), fg="green") ================================================ FILE: platformio/package/commands/update.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import click from platformio import fs from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.project.config import ProjectConfig from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies @click.command( "update", short_help="Update the project dependencies or custom packages" ) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) @click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) @click.option( "--no-save", is_flag=True, help="Prevent saving specified packages to `platformio.ini`", ) @click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies") @click.option("-g", "--global", is_flag=True, help="Update global packages") @click.option( "--storage-dir", default=None, type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") def package_update_cmd(**options): if options.get("global"): update_global_dependencies(options) else: update_project_dependencies(options) def update_global_dependencies(options): pm = PlatformPackageManager(options.get("storage_dir")) tm = ToolPackageManager(options.get("storage_dir")) lm = LibraryPackageManager(options.get("storage_dir")) for obj in (pm, tm, lm): obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG) for spec in options.get("platforms"): pm.update( from_spec=spec, to_spec=spec, skip_dependencies=options.get("skip_dependencies"), ) for spec in options.get("tools"): tm.update( from_spec=spec, to_spec=spec, skip_dependencies=options.get("skip_dependencies"), ) for spec in options.get("libraries", []): lm.update( from_spec=spec, to_spec=spec, skip_dependencies=options.get("skip_dependencies"), ) def update_project_dependencies(options): environments = options["environments"] with fs.cd(options["project_dir"]): config = ProjectConfig.get_instance() config.validate(environments) for env in config.envs(): if environments and env not in environments: continue if not options["silent"]: click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not update_project_env_dependencies(env, options) if not options["silent"] and already_up_to_date: click.secho("Already up-to-date.", fg="green") def update_project_env_dependencies(project_env, options=None): options = options or {} updated_conds = [] # custom platforms if options.get("platforms"): updated_conds.append(_update_project_env_custom_platforms(project_env, options)) # custom tools if options.get("tools"): updated_conds.append(_update_project_env_custom_tools(project_env, options)) # custom libraries if options.get("libraries"): updated_conds.append(_update_project_env_custom_libraries(project_env, options)) # declared dependencies if not updated_conds: updated_conds = [ _update_project_env_platform(project_env, options), _update_project_env_libraries(project_env, options), ] return any(updated_conds) def _update_project_env_platform(project_env, options): config = ProjectConfig.get_instance() pm = PlatformPackageManager() if options.get("silent"): pm.set_log_level(logging.WARN) spec = config.get(f"env:{project_env}", "platform") if not spec: return None cur_pkg = pm.get_package(spec) if not cur_pkg: return None new_pkg = PlatformPackageManager().update( cur_pkg, to_spec=spec, project_env=project_env, skip_dependencies=options.get("skip_dependencies"), ) return cur_pkg != new_pkg def _update_project_env_custom_platforms(project_env, options): already_up_to_date = True pm = PlatformPackageManager() if not options.get("silent"): pm.set_log_level(logging.DEBUG) for spec in options.get("platforms"): cur_pkg = pm.get_package(spec) new_pkg = pm.update( cur_pkg, to_spec=spec, project_env=project_env, skip_dependencies=options.get("skip_dependencies"), ) if cur_pkg != new_pkg: already_up_to_date = False return not already_up_to_date def _update_project_env_custom_tools(project_env, options): already_up_to_date = True tm = ToolPackageManager() if not options.get("silent"): tm.set_log_level(logging.DEBUG) specs_to_save = [] for tool in options.get("tools"): spec = PackageSpec(tool) cur_pkg = tm.get_package(spec) new_pkg = tm.update( cur_pkg, to_spec=spec, skip_dependencies=options.get("skip_dependencies"), ) if cur_pkg != new_pkg: already_up_to_date = False specs_to_save.append(pkg_to_save_spec(new_pkg, spec)) if not options.get("no_save") and specs_to_save: save_project_dependencies( os.getcwd(), specs_to_save, scope="platform_packages", action="add", environments=[project_env], ) return not already_up_to_date def _update_project_env_libraries(project_env, options): already_up_to_date = True config = ProjectConfig.get_instance() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), project_env) ) if options.get("silent"): lm.set_log_level(logging.WARN) for library in config.get(f"env:{project_env}", "lib_deps"): spec = PackageSpec(library) # skip built-in dependencies if not spec.external and not spec.owner: continue cur_pkg = lm.get_package(spec) if cur_pkg: new_pkg = lm.update( cur_pkg, to_spec=spec, skip_dependencies=options.get("skip_dependencies"), ) if cur_pkg != new_pkg: already_up_to_date = False return not already_up_to_date def _update_project_env_custom_libraries(project_env, options): already_up_to_date = True config = ProjectConfig.get_instance() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), project_env) ) if not options.get("silent"): lm.set_log_level(logging.DEBUG) specs_to_save = [] for library in options.get("libraries") or []: spec = PackageSpec(library) cur_pkg = lm.get_package(spec) new_pkg = lm.update( cur_pkg, to_spec=spec, skip_dependencies=options.get("skip_dependencies"), ) if cur_pkg != new_pkg: already_up_to_date = False specs_to_save.append(pkg_to_save_spec(new_pkg, spec)) if not options.get("no_save") and specs_to_save: save_project_dependencies( os.getcwd(), specs_to_save, scope="lib_deps", action="add", environments=[project_env], ) return not already_up_to_date ================================================ FILE: platformio/package/download.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import io from email.utils import parsedate from os.path import getsize, join from time import mktime import click from platformio import fs from platformio.compat import is_terminal from platformio.http import HTTPSession from platformio.package.exception import PackageException class FileDownloader: def __init__(self, url, dest_dir=None): self._http_session = HTTPSession() self._http_response = None # make connection self._http_response = self._http_session.get( url, stream=True, ) if self._http_response.status_code not in (200, 203): raise PackageException( "Got the unrecognized status code '{0}' when downloaded {1}".format( self._http_response.status_code, url ) ) disposition = self._http_response.headers.get("content-disposition") if disposition and "filename=" in disposition: self._fname = ( disposition[disposition.index("filename=") + 9 :] .replace('"', "") .replace("'", "") ) else: self._fname = [p for p in url.split("/") if p][-1] self._fname = str(self._fname) self._destination = self._fname if dest_dir: self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): self._destination = destination def get_filepath(self): return self._destination def get_lmtime(self): return self._http_response.headers.get("last-modified") def get_size(self): if "content-length" not in self._http_response.headers: return -1 return int(self._http_response.headers["content-length"]) def start(self, with_progress=True, silent=False): label = "Downloading" file_size = self.get_size() itercontent = self._http_response.iter_content( chunk_size=io.DEFAULT_BUFFER_SIZE ) try: with open(self._destination, "wb") as fp: if file_size == -1 or not with_progress or silent: if not silent: click.echo(f"{label}...") for chunk in itercontent: fp.write(chunk) elif not is_terminal(): click.echo(f"{label} 0%", nl=False) print_percent_step = 10 printed_percents = 0 downloaded_size = 0 for chunk in itercontent: fp.write(chunk) downloaded_size += len(chunk) if (downloaded_size / file_size * 100) >= ( printed_percents + print_percent_step ): printed_percents += print_percent_step click.echo(f" {printed_percents}%", nl=False) click.echo("") else: with click.progressbar( length=file_size, iterable=itercontent, label=label, update_min_steps=min( 256 * 1024, file_size / 100 ), # every 256Kb or less ) as pb: for chunk in pb: pb.update(len(chunk)) fp.write(chunk) finally: self._http_response.close() self._http_session.close() if self.get_lmtime(): self._preserve_filemtime(self.get_lmtime()) return True def verify(self, checksum=None): _dlsize = getsize(self._destination) if self.get_size() != -1 and _dlsize != self.get_size(): raise PackageException( ( "The size ({0:d} bytes) of downloaded file '{1}' " "is not equal to remote size ({2:d} bytes)" ).format(_dlsize, self._fname, self.get_size()) ) if not checksum: return True checksum_len = len(checksum) hash_algo = None if checksum_len == 32: hash_algo = "md5" elif checksum_len == 40: hash_algo = "sha1" elif checksum_len == 64: hash_algo = "sha256" if not hash_algo: raise PackageException( "Could not determine checksum algorithm by %s" % checksum ) dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination) if checksum.lower() != dl_checksum.lower(): raise PackageException( "The checksum '{0}' of the downloaded file '{1}' " "does not match to the remote '{2}'".format( dl_checksum, self._fname, checksum ) ) return True def _preserve_filemtime(self, lmdate): lmtime = mktime(parsedate(lmdate)) fs.change_filemtime(self._destination, lmtime) def __del__(self): self._http_session.close() if self._http_response: self._http_response.close() ================================================ FILE: platformio/package/exception.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.exception import UserSideException class PackageException(UserSideException): pass class ManifestException(PackageException): pass class UnknownManifestError(ManifestException): pass class ManifestParserError(ManifestException): pass class ManifestValidationError(ManifestException): def __init__(self, messages, data, valid_data): super().__init__() self.messages = messages self.data = data self.valid_data = valid_data def __str__(self): return ( "Invalid manifest fields: %s. \nPlease check specification -> " "https://docs.platformio.org/page/librarymanager/config.html" % self.messages ) class MissingPackageManifestError(ManifestException): MESSAGE = "Could not find one of '{0}' manifest files in the package" class UnknownPackageError(PackageException): MESSAGE = "Could not find the package with '{0}' requirements" class IncompatiblePackageError(UnknownPackageError): MESSAGE = ( "Could not find a version of the package with '{0}' requirements " "compatible with the '{1}' system" ) class NotGlobalLibDir(PackageException): MESSAGE = ( "The `{0}` is not a PlatformIO project.\n\n" "To manage libraries in global storage `{1}`,\n" "please use `platformio lib --global {2}` or specify custom storage " "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" "Check `platformio lib --help` for details." ) ================================================ FILE: platformio/package/lockfile.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from time import sleep, time from platformio.exception import UserSideException LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour LOCKFILE_DELAY = 0.2 LOCKFILE_INTERFACE_FCNTL = 1 LOCKFILE_INTERFACE_MSVCRT = 2 try: import fcntl LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL except ImportError: try: import msvcrt LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT except ImportError: LOCKFILE_CURRENT_INTERFACE = None class LockFileExists(UserSideException): pass class LockFileTimeoutError(UserSideException): pass class LockFile: def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): self.timeout = timeout self.delay = delay self._lock_path = os.path.abspath(path) + ".lock" self._fp = None def _lock(self): if not LOCKFILE_CURRENT_INTERFACE and os.path.exists(self._lock_path): # remove stale lock if time() - os.path.getmtime(self._lock_path) > 10: try: os.remove(self._lock_path) except: # pylint: disable=bare-except pass else: raise LockFileExists self._fp = open( # pylint: disable=consider-using-with self._lock_path, mode="w", encoding="utf8" ) try: if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT: msvcrt.locking( # pylint: disable=used-before-assignment self._fp.fileno(), msvcrt.LK_NBLCK, 1 ) except (BlockingIOError, IOError) as exc: self._fp.close() self._fp = None raise LockFileExists from exc return True def _unlock(self): if not self._fp: return if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN) elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT: msvcrt.locking(self._fp.fileno(), msvcrt.LK_UNLCK, 1) self._fp.close() self._fp = None def acquire(self): elapsed = 0 while elapsed < self.timeout: try: return self._lock() except LockFileExists: sleep(self.delay) elapsed += self.delay raise LockFileTimeoutError() def release(self): self._unlock() if os.path.exists(self._lock_path): try: os.remove(self._lock_path) except: # pylint: disable=bare-except pass def __enter__(self): self.acquire() def __exit__(self, type_, value, traceback): self.release() def __del__(self): self.release() ================================================ FILE: platformio/package/manager/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/package/manager/_download.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import logging import os import tempfile import time import click from platformio import app, compat, util from platformio.package.download import FileDownloader from platformio.package.lockfile import LockFile class PackageManagerDownloadMixin: DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month def compute_download_path(self, *args): request_hash = hashlib.new("sha1") for arg in args: request_hash.update(compat.hashlib_encode_data(arg)) dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) return dl_path def get_download_usagedb_path(self): return os.path.join(self.get_download_dir(), "usage.db") def set_download_utime(self, path, utime=None): with app.State(self.get_download_usagedb_path(), lock=True) as state: state[os.path.basename(path)] = int(time.time() if not utime else utime) @util.memoized(DOWNLOAD_CACHE_EXPIRE) def cleanup_expired_downloads(self, _=None): with app.State(self.get_download_usagedb_path(), lock=True) as state: # remove outdated for fname in list(state.keys()): if state[fname] > (time.time() - self.DOWNLOAD_CACHE_EXPIRE): continue del state[fname] dl_path = os.path.join(self.get_download_dir(), fname) if os.path.isfile(dl_path): os.remove(dl_path) def download(self, url, checksum=None): silent = not self.log.isEnabledFor(logging.INFO) dl_path = self.compute_download_path(url, checksum or "") if os.path.isfile(dl_path): self.set_download_utime(dl_path) return dl_path with_progress = not app.is_disabled_progressbar() tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) try: with LockFile(dl_path): try: fd = FileDownloader(url) fd.set_destination(tmp_path) fd.start(with_progress=with_progress, silent=silent) except IOError as exc: raise_error = not silent if with_progress: try: fd = FileDownloader(url) fd.set_destination(tmp_path) fd.start(with_progress=False, silent=silent) except IOError: raise_error = True if raise_error: self.log.error( click.style( "Error: Please read https://bit.ly/package-manager-ioerror", fg="red", ) ) raise exc if checksum: fd.verify(checksum) os.close(tmp_fd) os.rename(tmp_path, dl_path) finally: if os.path.isfile(tmp_path): os.close(tmp_fd) os.remove(tmp_path) assert os.path.isfile(dl_path) self.set_download_utime(dl_path) return dl_path ================================================ FILE: platformio/package/manager/_install.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import os import shutil import tempfile import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException, UnknownPackageError from platformio.package.meta import PackageCompatibility, PackageItem from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory class PackageManagerInstallMixin: _INSTALL_HISTORY = None # avoid circle dependencies @staticmethod def unpack(src, dst): with_progress = not app.is_disabled_progressbar() try: with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=with_progress) except IOError as exc: if not with_progress: raise exc with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) def install(self, spec, skip_dependencies=False, force=False): try: self.lock() pkg = self._install(spec, skip_dependencies=skip_dependencies, force=force) self.memcache_reset() self.cleanup_expired_downloads() return pkg finally: self.unlock() def _install( self, spec, skip_dependencies=False, force=False, compatibility: PackageCompatibility = None, ): spec = self.ensure_spec(spec) # avoid circle dependencies if not self._INSTALL_HISTORY: self._INSTALL_HISTORY = {} if not force and spec in self._INSTALL_HISTORY: return self._INSTALL_HISTORY[spec] # check if package is already installed pkg = self.get_package(spec) # if a forced installation if pkg and force: self.uninstall(pkg) pkg = None if pkg: # avoid RecursionError for circular_dependencies self._INSTALL_HISTORY[spec] = pkg self.log.debug( click.style( "{name}@{version} is already installed".format( **pkg.metadata.as_dict() ), fg="yellow", ) ) # ensure package dependencies are installed if not skip_dependencies: self.install_dependencies(pkg, print_header=False) return pkg self.log.info("Installing %s" % click.style(spec.humanize(), fg="cyan")) if spec.external: pkg = self.install_from_uri(spec.uri, spec) else: pkg = self.install_from_registry( spec, search_qualifiers=( compatibility.to_search_qualifiers( ["platforms", "frameworks", "authors"] ) if compatibility else None ), ) if not pkg or not pkg.metadata: raise PackageException( "Could not install package '%s' for '%s' system" % (spec.humanize(), util.get_systype()) ) self.call_pkg_script(pkg, "postinstall") self.log.info( click.style( "{name}@{version} has been installed!".format(**pkg.metadata.as_dict()), fg="green", ) ) self.memcache_reset() # avoid RecursionError for circular_dependencies self._INSTALL_HISTORY[spec] = pkg if not skip_dependencies: self.install_dependencies(pkg) return pkg def install_dependencies(self, pkg, print_header=True): assert isinstance(pkg, PackageItem) dependencies = self.get_pkg_dependencies(pkg) if not dependencies: return if print_header: self.log.info("Resolving dependencies...") for dependency in dependencies: try: self.install_dependency(dependency) except UnknownPackageError: if dependency.get("owner"): self.log.warning( click.style( "Warning! Could not install `%s` dependency " "for the`%s` package" % (dependency, pkg.metadata.name), fg="yellow", ) ) def install_dependency(self, dependency): dependency_compatibility = PackageCompatibility.from_dependency(dependency) if self.compatibility and not dependency_compatibility.is_compatible( self.compatibility ): self.log.debug( click.style( "Skip incompatible `%s` dependency with `%s`" % (dependency, self.compatibility), fg="yellow", ) ) return None return self._install( spec=self.dependency_to_spec(dependency), compatibility=dependency_compatibility, ) def install_from_uri(self, uri, spec, checksum=None): spec = self.ensure_spec(spec) if spec.symlink: return self.install_symlink(spec) tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir()) vcs = None try: if uri.startswith("file://"): _uri = uri[7:] if os.path.isfile(_uri): self.unpack(_uri, tmp_dir) else: fs.rmtree(tmp_dir) shutil.copytree(_uri, tmp_dir, symlinks=True) elif uri.startswith(("http://", "https://")): dl_path = self.download(uri, checksum) assert os.path.isfile(dl_path) self.unpack(dl_path, tmp_dir) else: vcs = VCSClientFactory.new(tmp_dir, uri) assert vcs.export() root_dir = self.find_pkg_root(tmp_dir, spec) pkg_item = PackageItem( root_dir, self.build_metadata( root_dir, spec, vcs.get_current_revision() if vcs else None ), ) pkg_item.dump_meta() return self._install_tmp_pkg(pkg_item) finally: if os.path.isdir(tmp_dir): try: fs.rmtree(tmp_dir) except: # pylint: disable=bare-except pass def _install_tmp_pkg(self, tmp_pkg): assert isinstance(tmp_pkg, PackageItem) # validate package version and declared requirements if ( tmp_pkg.metadata.spec.requirements and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements ): raise PackageException( "Package version %s doesn't satisfy requirements %s based on %s" % ( tmp_pkg.metadata.version, tmp_pkg.metadata.spec.requirements, tmp_pkg.metadata, ) ) dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.get_safe_dirname()) ) # what to do with existing package? action = "overwrite" if tmp_pkg.metadata.spec.has_custom_name(): action = "overwrite" dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) ) elif dst_pkg.metadata: if dst_pkg.metadata.spec.external: if dst_pkg.metadata.spec.uri != tmp_pkg.metadata.spec.uri: action = "detach-existing" elif ( dst_pkg.metadata.version != tmp_pkg.metadata.version or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner ): action = ( "detach-existing" if tmp_pkg.metadata.version > dst_pkg.metadata.version else "detach-new" ) def _cleanup_dir(path): if os.path.isdir(path): fs.rmtree(path) if action == "detach-existing": target_dirname = "%s@%s" % ( tmp_pkg.get_safe_dirname(), dst_pkg.metadata.version, ) if dst_pkg.metadata.spec.uri: target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( compat.hashlib_encode_data(dst_pkg.metadata.spec.uri) ).hexdigest(), ) # move existing into the new place pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) shutil.copytree(dst_pkg.path, pkg_dir, symlinks=True) # move new source to the destination location _cleanup_dir(dst_pkg.path) shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) return PackageItem(dst_pkg.path) if action == "detach-new": target_dirname = "%s@%s" % ( tmp_pkg.get_safe_dirname(), tmp_pkg.metadata.version, ) if tmp_pkg.metadata.spec.external: target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( compat.hashlib_encode_data(tmp_pkg.metadata.spec.uri) ).hexdigest(), ) pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) shutil.copytree(tmp_pkg.path, pkg_dir, symlinks=True) return PackageItem(pkg_dir) # otherwise, overwrite existing _cleanup_dir(dst_pkg.path) shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) return PackageItem(dst_pkg.path) ================================================ FILE: platformio/package/manager/_legacy.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from platformio import fs from platformio.package.meta import PackageItem, PackageSpec class PackageManagerLegacyMixin: def build_legacy_spec(self, pkg_dir): # find src manifest src_manifest_name = ".piopkgmanager.json" src_manifest_path = None for name in os.listdir(pkg_dir): if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): continue src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) break if src_manifest_path: src_manifest = fs.load_json(src_manifest_path) return PackageSpec( name=src_manifest.get("name"), uri=src_manifest.get("url"), requirements=src_manifest.get("requirements"), ) # fall back to a package manifest manifest = self.load_manifest(pkg_dir) return PackageSpec(name=manifest.get("name")) def legacy_load_manifest(self, pkg): if not isinstance(pkg, PackageItem): assert os.path.isdir(pkg) pkg = PackageItem(pkg) manifest = self.load_manifest(pkg) manifest["__pkg_dir"] = pkg.path for key in ("name", "version"): if not manifest.get(key): manifest[key] = str(getattr(pkg.metadata, key)) if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: manifest["__src_url"] = pkg.metadata.spec.uri manifest["version"] = str(pkg.metadata.version) if pkg.metadata and pkg.metadata.spec.owner: manifest["ownername"] = pkg.metadata.spec.owner return manifest def legacy_get_installed(self): return [self.legacy_load_manifest(pkg) for pkg in self.get_installed()] ================================================ FILE: platformio/package/manager/_registry.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time import click from platformio import util from platformio.package.exception import IncompatiblePackageError, UnknownPackageError from platformio.package.meta import PackageSpec, PackageType from platformio.package.version import cast_version_to_semver from platformio.registry.client import RegistryClient from platformio.registry.mirror import RegistryFileMirrorIterator class PackageManagerRegistryMixin: def install_from_registry(self, spec, search_qualifiers=None): package = version = None if spec.owner and spec.name and not search_qualifiers: package = self.fetch_registry_package(spec) if not package: raise UnknownPackageError(spec.humanize()) version = self.pick_best_registry_version(package["versions"], spec) elif spec.id or spec.name: packages = self.search_registry_packages(spec, search_qualifiers) if not packages: raise UnknownPackageError(spec.humanize()) if len(packages) > 1: self.print_multi_package_issue(self.log.warning, packages, spec) package, version = self.find_best_registry_version(packages, spec) if not package or not version: raise UnknownPackageError(spec.humanize()) pkgfile = self.pick_compatible_pkg_file(version["files"]) if version else None if not pkgfile: if self.pkg_type == PackageType.TOOL: raise IncompatiblePackageError(spec.humanize(), util.get_systype()) raise UnknownPackageError(spec.humanize()) for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]): try: return self.install_from_uri( url, PackageSpec( owner=package["owner"]["username"], id=package["id"], name=package["name"], ), checksum or pkgfile["checksum"]["sha256"], ) except Exception as exc: # pylint: disable=broad-except self.log.warning( click.style("Warning! Package Mirror: %s" % exc, fg="yellow") ) self.log.warning( click.style("Looking for another mirror...", fg="yellow") ) return None def get_registry_client_instance(self): if not self._registry_client: self._registry_client = RegistryClient() return self._registry_client def search_registry_packages(self, spec, qualifiers=None): assert isinstance(spec, PackageSpec) qualifiers = qualifiers or {} if spec.id: qualifiers["ids"] = str(spec.id) else: qualifiers["types"] = self.pkg_type qualifiers["names"] = spec.name.lower() if spec.owner: qualifiers["owners"] = spec.owner.lower() return self.get_registry_client_instance().list_packages(qualifiers=qualifiers)[ "items" ] def fetch_registry_package(self, spec): assert isinstance(spec, PackageSpec) result = None regclient = self.get_registry_client_instance() if spec.owner and spec.name: result = regclient.get_package(self.pkg_type, spec.owner, spec.name) if not result and (spec.id or (spec.name and not spec.owner)): packages = self.search_registry_packages(spec) if packages: result = regclient.get_package( self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"] ) if not result: raise UnknownPackageError(spec.humanize()) return result def reveal_registry_package_id(self, spec): spec = self.ensure_spec(spec) if spec.id: return spec.id packages = self.search_registry_packages(spec) if not packages: raise UnknownPackageError(spec.humanize()) if len(packages) > 1: self.print_multi_package_issue(self.log.warning, packages, spec) self.log.info("") return packages[0]["id"] @staticmethod def print_multi_package_issue(print_func, packages, spec): print_func( click.style( "Warning! More than one package has been found by ", fg="yellow" ) + click.style(spec.humanize(), fg="cyan") + click.style(" requirements:", fg="yellow") ) for item in packages: print_func( " - {owner}/{name}@{version}".format( owner=click.style(item["owner"]["username"], fg="cyan"), name=item["name"], version=item["version"]["name"], ) ) print_func( click.style( "Please specify detailed REQUIREMENTS using package owner and version " "(shown above) to avoid name conflicts", fg="yellow", ) ) def find_best_registry_version(self, packages, spec): for package in packages: # find compatible version within the latest package versions version = self.pick_best_registry_version([package["version"]], spec) if version: return (package, version) # if the custom version requirements, check ALL package versions version = self.pick_best_registry_version( self.fetch_registry_package( PackageSpec( id=package["id"], owner=package["owner"]["username"], name=package["name"], ) ).get("versions"), spec, ) if version: return (package, version) time.sleep(1) return (None, None) def get_compatible_registry_versions(self, versions, spec=None, custom_system=None): assert not spec or isinstance(spec, PackageSpec) result = [] for version in versions: semver = cast_version_to_semver(version["name"]) if spec and spec.requirements and semver not in spec.requirements: continue if not any( self.is_system_compatible(f.get("system"), custom_system=custom_system) for f in version["files"] ): continue result.append(version) return result def pick_best_registry_version(self, versions, spec=None, custom_system=None): best = None for version in self.get_compatible_registry_versions( versions, spec, custom_system ): semver = cast_version_to_semver(version["name"]) if not best or (semver > cast_version_to_semver(best["name"])): best = version return best def pick_compatible_pkg_file(self, version_files, custom_system=None): for item in version_files: if self.is_system_compatible( item.get("system"), custom_system=custom_system ): return item return None ================================================ FILE: platformio/package/manager/_symlink.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os from platformio import fs from platformio.package.exception import PackageException from platformio.package.meta import PackageItem, PackageSpec class PackageManagerSymlinkMixin: @staticmethod def is_symlink(path): return path and path.endswith(".pio-link") and os.path.isfile(path) @classmethod def resolve_symlink(cls, path): assert cls.is_symlink(path) data = fs.load_json(path) spec = PackageSpec(**data["spec"]) assert spec.symlink pkg_dir = spec.uri[10:] if not os.path.isabs(pkg_dir): pkg_dir = os.path.normpath(os.path.join(data["cwd"], pkg_dir)) return (pkg_dir if os.path.isdir(pkg_dir) else None, spec) def get_symlinked_package(self, path): pkg_dir, spec = self.resolve_symlink(path) if not pkg_dir: return None pkg = PackageItem(os.path.realpath(pkg_dir)) if not pkg.metadata: pkg.metadata = self.build_metadata(pkg.path, spec) return pkg def install_symlink(self, spec): assert spec.symlink pkg_dir = spec.uri[10:] if not os.path.isdir(pkg_dir): raise PackageException( f"Can not create a symbolic link for `{pkg_dir}`, not a directory" ) link_path = os.path.join( self.package_dir, "%s.pio-link" % (spec.name or os.path.basename(os.path.abspath(pkg_dir))), ) with open(link_path, mode="w", encoding="utf-8") as fp: json.dump(dict(cwd=os.getcwd(), spec=spec.as_dict()), fp) return self.get_symlinked_package(link_path) def uninstall_symlink(self, spec): assert spec.symlink for name in os.listdir(self.package_dir): path = os.path.join(self.package_dir, name) if not self.is_symlink(path): continue pkg = self.get_symlinked_package(path) if pkg.metadata.spec.uri == spec.uri: os.remove(path) ================================================ FILE: platformio/package/manager/_uninstall.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import click from platformio import fs from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageSpec class PackageManagerUninstallMixin: def uninstall(self, spec, skip_dependencies=False): try: self.lock() return self._uninstall(spec, skip_dependencies) finally: self.unlock() def _uninstall(self, spec, skip_dependencies=False): pkg = self.get_package(spec) if not pkg or not pkg.metadata: raise UnknownPackageError(spec) uninstalled_pkgs = self.memcache_get("__uninstalled_pkgs", []) if uninstalled_pkgs and pkg.path in uninstalled_pkgs: return pkg uninstalled_pkgs.append(pkg.path) self.memcache_set("__uninstalled_pkgs", uninstalled_pkgs) self.log.info( "Removing %s @ %s" % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version) ) self.call_pkg_script(pkg, "preuninstall") # firstly, remove dependencies if not skip_dependencies: self.uninstall_dependencies(pkg) if pkg.metadata.spec.symlink: self.uninstall_symlink(pkg.metadata.spec) elif os.path.islink(pkg.path): os.unlink(pkg.path) else: fs.rmtree(pkg.path) self.memcache_reset() # unfix detached-package with the same name detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) if ( detached_pkg and "@" in detached_pkg.path and not os.path.isdir( os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) ) ): shutil.move( detached_pkg.path, os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), ) self.memcache_reset() self.log.info( click.style( "{name}@{version} has been removed!".format(**pkg.metadata.as_dict()), fg="green", ) ) return pkg def uninstall_dependencies(self, pkg): assert isinstance(pkg, PackageItem) dependencies = self.get_pkg_dependencies(pkg) if not dependencies: return self.log.info("Removing dependencies...") for dependency in dependencies: pkg = self.get_package(self.dependency_to_spec(dependency)) if not pkg: continue self._uninstall(pkg) ================================================ FILE: platformio/package/manager/_update.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import click from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec from platformio.package.vcsclient import VCSBaseException, VCSClientFactory class PackageManagerUpdateMixin: def outdated(self, pkg, spec=None): assert isinstance(pkg, PackageItem) assert pkg.metadata if spec and not isinstance(spec, PackageSpec): spec = PackageSpec(spec) if not os.path.isdir(pkg.path): return PackageOutdatedResult(current=pkg.metadata.version) # skip detached package to a specific version detached_conditions = [ "@" in pkg.path, pkg.metadata.spec and not pkg.metadata.spec.external, not spec, ] if all(detached_conditions): return PackageOutdatedResult(current=pkg.metadata.version, detached=True) latest = None wanted = None if pkg.metadata.spec.external: latest = self._fetch_vcs_latest_version(pkg) else: try: reg_pkg = self.fetch_registry_package(pkg.metadata.spec) latest = ( self.pick_best_registry_version(reg_pkg["versions"]) or {} ).get("name") if spec: wanted = ( self.pick_best_registry_version(reg_pkg["versions"], spec) or {} ).get("name") if not wanted: # wrong library latest = None except UnknownPackageError: pass return PackageOutdatedResult( current=pkg.metadata.version, latest=latest, wanted=wanted ) def _fetch_vcs_latest_version(self, pkg): vcs = None try: vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri, silent=True) except VCSBaseException: return None if not vcs.can_be_updated: return None vcs_revision = vcs.get_latest_revision() if not vcs_revision: return None return str( self.build_metadata( pkg.path, pkg.metadata.spec, vcs_revision=vcs_revision ).version ) def update( self, from_spec, to_spec=None, skip_dependencies=False, ): pkg = self.get_package(from_spec) if not pkg or not pkg.metadata: raise UnknownPackageError(from_spec) outdated = self.outdated(pkg, to_spec) if not outdated.is_outdated(allow_incompatible=False): self.log.debug( click.style( "{name}@{version} is already up-to-date".format( **pkg.metadata.as_dict() ), fg="yellow", ) ) return pkg self.log.info( "Updating %s @ %s" % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version) ) try: self.lock() return self._update(pkg, outdated, skip_dependencies) finally: self.unlock() def _update(self, pkg, outdated, skip_dependencies=False): if pkg.metadata.spec.external: vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri) assert vcs.update() pkg.metadata.version = self._fetch_vcs_latest_version(pkg) pkg.dump_meta() return pkg # uninstall existing version self.uninstall(pkg, skip_dependencies=True) return self.install( PackageSpec( id=pkg.metadata.spec.id, owner=pkg.metadata.spec.owner, name=pkg.metadata.spec.name, requirements=outdated.wanted or outdated.latest, ), skip_dependencies=skip_dependencies, ) ================================================ FILE: platformio/package/manager/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import subprocess from datetime import datetime import click import semantic_version from platformio import fs, util from platformio.cli import PlatformioCLI from platformio.compat import ci_strings_are_equal from platformio.package.exception import ManifestException, MissingPackageManifestError from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._legacy import PackageManagerLegacyMixin from platformio.package.manager._registry import PackageManagerRegistryMixin from platformio.package.manager._symlink import PackageManagerSymlinkMixin from platformio.package.manager._uninstall import PackageManagerUninstallMixin from platformio.package.manager._update import PackageManagerUpdateMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( PackageItem, PackageMetadata, PackageSpec, PackageType, ) from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_cache_dir class ClickLoggingHandler(logging.Handler): def emit(self, record): click.echo(self.format(record)) class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-instance-attributes PackageManagerDownloadMixin, PackageManagerRegistryMixin, PackageManagerSymlinkMixin, PackageManagerInstallMixin, PackageManagerUninstallMixin, PackageManagerUpdateMixin, PackageManagerLegacyMixin, ): _MEMORY_CACHE = {} def __init__(self, pkg_type, package_dir, compatibility=None): self.pkg_type = pkg_type self.package_dir = package_dir self.compatibility = compatibility self.log = self._setup_logger() self._MEMORY_CACHE = {} self._lockfile = None self._download_dir = None self._tmp_dir = None self._registry_client = None def __repr__(self): return ( f"{self.__class__.__name__} " ) def _setup_logger(self): logger = logging.getLogger(str(self.__class__.__name__).replace("Package", " ")) logger.setLevel(logging.INFO) formatter = logging.Formatter("%(name)s: %(message)s") sh = ClickLoggingHandler() sh.setFormatter(formatter) logger.handlers.clear() logger.addHandler(sh) return logger def set_log_level(self, level): self.log.setLevel(level) def lock(self): if self._lockfile: return self.ensure_dir_exists(os.path.dirname(self.package_dir)) self._lockfile = LockFile(self.package_dir) self.ensure_dir_exists(self.package_dir) self._lockfile.acquire() def unlock(self): if hasattr(self, "_lockfile") and self._lockfile: self._lockfile.release() self._lockfile = None def __del__(self): self.unlock() def memcache_get(self, key, default=None): return self._MEMORY_CACHE.get(key, default) def memcache_set(self, key, value): self._MEMORY_CACHE[key] = value def memcache_reset(self): self._MEMORY_CACHE.clear() @staticmethod def is_system_compatible(value, custom_system=None): if not value or "*" in value: return True return util.items_in_list(value, custom_system or util.get_systype()) @staticmethod def ensure_dir_exists(path): if not os.path.isdir(path): os.makedirs(path) assert os.path.isdir(path) return path @staticmethod def ensure_spec(spec): return spec if isinstance(spec, PackageSpec) else PackageSpec(spec) @property def manifest_names(self): raise NotImplementedError def get_download_dir(self): if not self._download_dir: self._download_dir = self.ensure_dir_exists( os.path.join(get_project_cache_dir(), "downloads") ) return self._download_dir def get_tmp_dir(self): if not self._tmp_dir: self._tmp_dir = self.ensure_dir_exists( os.path.join(get_project_cache_dir(), "tmp") ) return self._tmp_dir def find_pkg_root(self, path, spec): # pylint: disable=unused-argument if self.manifest_exists(path): return path for root, _, _ in os.walk(path): if self.manifest_exists(root): return root raise MissingPackageManifestError(", ".join(self.manifest_names)) def get_manifest_path(self, pkg_dir): if not os.path.isdir(pkg_dir): return None for name in self.manifest_names: manifest_path = os.path.join(pkg_dir, name) if os.path.isfile(manifest_path): return manifest_path return None def manifest_exists(self, pkg_dir): return self.get_manifest_path(pkg_dir) def load_manifest(self, src): path = src.path if isinstance(src, PackageItem) else src cache_key = "load_manifest-%s" % path result = self.memcache_get(cache_key) if result: return result candidates = ( [os.path.join(path, name) for name in self.manifest_names] if os.path.isdir(path) else [path] ) for item in candidates: if not os.path.isfile(item): continue try: result = ManifestParserFactory.new_from_file(item).as_dict() self.memcache_set(cache_key, result) return result except ManifestException as exc: if not PlatformioCLI.in_silence(): self.log.warning(click.style(str(exc), fg="yellow")) raise MissingPackageManifestError(", ".join(self.manifest_names)) @staticmethod def generate_rand_version(): return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") def build_metadata(self, pkg_dir, spec, vcs_revision=None): manifest = self.load_manifest(pkg_dir) metadata = PackageMetadata( type=self.pkg_type, name=manifest.get("name"), version=manifest.get("version"), spec=spec, ) if not metadata.name or spec.has_custom_name(): metadata.name = spec.name if vcs_revision: metadata.version = "%s+sha.%s" % ( metadata.version if metadata.version else "0.0.0", vcs_revision, ) if not metadata.version: metadata.version = self.generate_rand_version() return metadata def get_installed(self): # pylint: disable=too-many-branches if not os.path.isdir(self.package_dir): return [] cache_key = "get_installed" if self.memcache_get(cache_key): return self.memcache_get(cache_key) result = [] for name in sorted(os.listdir(self.package_dir)): if name.startswith("_tmp_installing"): # legacy tmp folder continue pkg = None path = os.path.join(self.package_dir, name) if os.path.isdir(path): pkg = PackageItem(path) elif self.is_symlink(path): pkg = self.get_symlinked_package(path) if not pkg: continue if not pkg.metadata: try: spec = self.build_legacy_spec(pkg.path) pkg.metadata = self.build_metadata(pkg.path, spec) except MissingPackageManifestError: pass if not pkg.metadata: continue if self.pkg_type == PackageType.TOOL: try: if not self.is_system_compatible( self.load_manifest(pkg).get("system") ): continue except MissingPackageManifestError: pass result.append(pkg) self.memcache_set(cache_key, result) return result def get_package(self, spec): if isinstance(spec, PackageItem): return spec spec = self.ensure_spec(spec) best = None for pkg in self.get_installed(): if not self.test_pkg_spec(pkg, spec): continue assert isinstance(pkg.metadata.version, semantic_version.Version) if spec.requirements and pkg.metadata.version not in spec.requirements: continue if not best or (pkg.metadata.version > best.metadata.version): best = pkg return best @staticmethod def test_pkg_spec(pkg, spec): # "id" mismatch if spec.id and spec.id != pkg.metadata.spec.id: return False # external "URL" mismatch if spec.external: # local/symlinked folder mismatch check_conds = [ os.path.abspath(spec.uri) == os.path.abspath(pkg.path), spec.uri.startswith("file://") and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[7:]), spec.uri.startswith("symlink://") and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[10:]), ] if any(check_conds): return True if spec.uri != pkg.metadata.spec.uri: return False # "owner" mismatch elif spec.owner and not ci_strings_are_equal( spec.owner, pkg.metadata.spec.owner ): return False # "name" mismatch elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): return False return True def get_pkg_dependencies(self, pkg): return self.load_manifest(pkg).get("dependencies") @staticmethod def dependency_to_spec(dependency): return PackageSpec( owner=dependency.get("owner"), name=dependency.get("name"), requirements=dependency.get("version"), ) def call_pkg_script(self, pkg, event): manifest = None try: manifest = self.load_manifest(pkg) except MissingPackageManifestError: pass scripts = (manifest or {}).get("scripts") if not scripts or not isinstance(scripts, dict): return cmd = scripts.get(event) if not cmd: return shell = False if not isinstance(cmd, list): # issue #5366: workaround when command passed as string without spaces if " " in cmd: shell = True cmd = [cmd] os.environ["PIO_PYTHON_EXE"] = get_pythonexe_path() with fs.cd(pkg.path): if os.path.isfile(cmd[0]) and cmd[0].endswith(".py"): cmd = [os.environ["PIO_PYTHON_EXE"]] + cmd subprocess.run( " ".join(cmd) if shell else cmd, cwd=pkg.path, shell=shell, env=os.environ, check=True, ) ================================================ FILE: platformio/package/manager/core.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from platformio import exception from platformio.dependencies import get_core_dependencies from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec def get_installed_core_packages(): result = [] pm = ToolPackageManager() for name, requirements in get_core_dependencies().items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if pkg: result.append(pkg) return result def get_core_package_dir(name, spec=None, auto_install=True): if name not in get_core_dependencies(): raise exception.PlatformioException("Please upgrade PlatformIO Core") pm = ToolPackageManager() spec = spec or PackageSpec( owner="platformio", name=name, requirements=get_core_dependencies()[name] ) pkg = pm.get_package(spec) if pkg: return pkg.path if not auto_install: return None assert pm.install(spec) remove_unnecessary_core_packages() return pm.get_package(spec).path def update_core_packages(): pm = ToolPackageManager() for name, requirements in get_core_dependencies().items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) try: pm.update(spec, spec) except UnknownPackageError: pass remove_unnecessary_core_packages() return True def remove_unnecessary_core_packages(dry_run=False): candidates = [] pm = ToolPackageManager() best_pkg_versions = {} for name, requirements in get_core_dependencies().items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if not pkg: continue # pylint: disable=no-member best_pkg_versions[pkg.metadata.name] = pkg.metadata.version for pkg in pm.get_installed(): skip_conds = [ os.path.isfile(os.path.join(pkg.path, ".piokeep")), pkg.metadata.spec.owner != "platformio", pkg.metadata.name not in best_pkg_versions, pkg.metadata.name in best_pkg_versions and pkg.metadata.version == best_pkg_versions[pkg.metadata.name], ] if not any(skip_conds): candidates.append(pkg) if dry_run: return candidates for pkg in candidates: pm.uninstall(pkg) return candidates ================================================ FILE: platformio/package/manager/library.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os from platformio import util from platformio.package.exception import MissingPackageManifestError from platformio.package.manager.base import BasePackageManager from platformio.package.meta import PackageSpec, PackageType from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None, **kwargs): super().__init__( PackageType.LIBRARY, package_dir or ProjectConfig.get_instance().get("platformio", "globallib_dir"), **kwargs ) @property def manifest_names(self): return PackageType.get_manifest_map()[PackageType.LIBRARY] def find_pkg_root(self, path, spec): try: return super().find_pkg_root(path, spec) except MissingPackageManifestError: pass assert isinstance(spec, PackageSpec) root_dir = self.find_library_root(path) # automatically generate library manifest with open( os.path.join(root_dir, "library.json"), mode="w", encoding="utf8" ) as fp: json.dump( dict( name=spec.name, version=self.generate_rand_version(), ), fp, indent=2, ) return root_dir @staticmethod def find_library_root(path): root_dir_signs = set(["include", "Include", "inc", "Inc", "src", "Src"]) root_file_signs = set( [ "conanfile.py", # Conan-based library "CMakeLists.txt", # CMake-based library ] ) for root, dirs, files in os.walk(path): if not files and len(dirs) == 1: continue if set(root_dir_signs) & set(dirs): return root if set(root_file_signs) & set(files): return root for fname in files: if fname.endswith((".c", ".cpp", ".h", ".hpp", ".S")): return root return path def install_dependency(self, dependency): spec = self.dependency_to_spec(dependency) # skip built-in dependencies not_builtin_conds = [spec.external, spec.owner] if not any(not_builtin_conds): not_builtin_conds.append(not self.is_builtin_lib(spec.name)) if any(not_builtin_conds): return super().install_dependency(dependency) return None @staticmethod @util.memoized(expire="60s") def get_builtin_libs(storage_names=None): # pylint: disable=import-outside-toplevel from platformio.package.manager.platform import PlatformPackageManager items = [] storage_names = storage_names or [] pm = PlatformPackageManager() for pkg in pm.get_installed(): p = PlatformFactory.new(pkg) for storage in p.get_lib_storages(): if storage_names and storage["name"] not in storage_names: continue lm = LibraryPackageManager(storage["path"]) items.append( { "name": storage["name"], "path": storage["path"], "items": lm.legacy_get_installed(), } ) return items @classmethod def is_builtin_lib(cls, name): for storage in cls.get_builtin_libs(): for lib in storage["items"]: if lib.get("name") == name: return True return False ================================================ FILE: platformio/package/manager/platform.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from platformio import util from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager from platformio.package.manager.core import get_installed_core_packages from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageType from platformio.platform.exception import IncompatiblePlatform, UnknownBoard from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): self.config = ProjectConfig.get_instance() super().__init__( PackageType.PLATFORM, package_dir or self.config.get("platformio", "platforms_dir"), ) @property def manifest_names(self): return PackageType.get_manifest_map()[PackageType.PLATFORM] def install( # pylint: disable=arguments-differ,too-many-arguments,too-many-positional-arguments self, spec, skip_dependencies=False, force=False, project_env=None, project_targets=None, ): already_installed = self.get_package(spec) pkg = super().install(spec, force=force, skip_dependencies=True) try: p = PlatformFactory.new(pkg) # set logging level for underlying tool manager p.pm.set_log_level(self.log.getEffectiveLevel()) p.ensure_engine_compatible() except IncompatiblePlatform as exc: super().uninstall(pkg, skip_dependencies=True) raise exc if project_env: p.configure_project_packages(project_env, project_targets) if not skip_dependencies: p.install_required_packages(force=force) if not already_installed: p.on_installed() return pkg def uninstall( # pylint: disable=arguments-differ self, spec, skip_dependencies=False, project_env=None ): pkg = self.get_package(spec) if not pkg or not pkg.metadata: raise UnknownPackageError(spec) p = PlatformFactory.new(pkg) # set logging level for underlying tool manager p.pm.set_log_level(self.log.getEffectiveLevel()) if project_env: p.configure_project_packages(project_env) if not skip_dependencies: p.uninstall_packages() assert super().uninstall(pkg, skip_dependencies=True) p.on_uninstalled() return pkg def update( # pylint: disable=arguments-differ self, from_spec, to_spec=None, skip_dependencies=False, project_env=None, ): pkg = self.get_package(from_spec) if not pkg or not pkg.metadata: raise UnknownPackageError(from_spec) pkg = super().update( from_spec, to_spec, ) p = PlatformFactory.new(pkg) # set logging level for underlying tool manager p.pm.set_log_level(self.log.getEffectiveLevel()) if project_env: p.configure_project_packages(project_env) if not skip_dependencies: p.update_packages() return pkg @util.memoized(expire="5s") def get_installed_boards(self): boards = [] for pkg in self.get_installed(): p = PlatformFactory.new(pkg) for config in p.get_boards().values(): board = config.get_brief_data() if board not in boards: boards.append(board) return boards def get_registered_boards(self): return self.get_registry_client_instance().fetch_json_data( "get", "/v2/boards", x_cache_valid="1d" ) def get_all_boards(self): boards = self.get_installed_boards() know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] try: for board in self.get_registered_boards(): key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) except (HTTPClientError, InternetConnectionError): pass return sorted(boards, key=lambda b: b["name"]) def board_config(self, id_, platform=None): for manifest in self.get_installed_boards(): if manifest["id"] == id_ and ( not platform or manifest["platform"] == platform ): return manifest for manifest in self.get_registered_boards(): if manifest["id"] == id_ and ( not platform or manifest["platform"] == platform ): return manifest raise UnknownBoard(id_) # # Helpers # def remove_unnecessary_platform_packages(dry_run=False): candidates = [] required = set() core_packages = get_installed_core_packages() for platform in PlatformPackageManager().get_installed(): p = PlatformFactory.new(platform) for pkg in p.get_installed_packages(with_optional_versions=True): required.add(pkg) pm = ToolPackageManager() for pkg in pm.get_installed(): skip_conds = [ pkg.metadata.spec.uri, os.path.isfile(os.path.join(pkg.path, ".piokeep")), pkg in required, pkg in core_packages, ] if not any(skip_conds): candidates.append(pkg) if dry_run: return candidates for pkg in candidates: pm.uninstall(pkg) return candidates ================================================ FILE: platformio/package/manager/tool.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.package.manager.base import BasePackageManager from platformio.package.meta import PackageType from platformio.project.config import ProjectConfig class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): super().__init__( PackageType.TOOL, package_dir or ProjectConfig.get_instance().get("platformio", "packages_dir"), ) @property def manifest_names(self): return PackageType.get_manifest_map()[PackageType.TOOL] ================================================ FILE: platformio/package/manifest/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/package/manifest/parser.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import inspect import io import json import os import re import tarfile from urllib.parse import urlparse from platformio import util from platformio.compat import get_object_members, string_types from platformio.http import fetch_remote_content from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project class ManifestFileType: PLATFORM_JSON = "platform.json" LIBRARY_JSON = "library.json" LIBRARY_PROPERTIES = "library.properties" MODULE_JSON = "module.json" PACKAGE_JSON = "package.json" @classmethod def items(cls): return get_object_members(cls) @classmethod def from_uri(cls, uri): for t in sorted(cls.items().values()): if uri.endswith(t): return t return None @classmethod def from_dir(cls, path): for t in sorted(cls.items().values()): if os.path.isfile(os.path.join(path, t)): return t return None class ManifestParserFactory: @staticmethod def read_manifest_contents(path): last_err = None for encoding in ("utf-8", "latin-1"): try: with io.open(path, encoding=encoding) as fp: return fp.read() except UnicodeDecodeError as exc: last_err = exc raise last_err @classmethod def new_from_file(cls, path, remote_url=False): if not path or not os.path.isfile(path): raise UnknownManifestError("Manifest file does not exist %s" % path) type_from_uri = ManifestFileType.from_uri(path) if not type_from_uri: raise UnknownManifestError("Unknown manifest file type %s" % path) return ManifestParserFactory.new( cls.read_manifest_contents(path), type_from_uri, remote_url ) @classmethod def new_from_dir(cls, path, remote_url=None): assert os.path.isdir(path), "Invalid directory %s" % path type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)): return ManifestParserFactory.new( cls.read_manifest_contents(os.path.join(path, type_from_uri)), type_from_uri, remote_url=remote_url, package_dir=path, ) type_from_dir = ManifestFileType.from_dir(path) if not type_from_dir: raise UnknownManifestError( "Unknown manifest file type in %s directory" % path ) return ManifestParserFactory.new( cls.read_manifest_contents(os.path.join(path, type_from_dir)), type_from_dir, remote_url=remote_url, package_dir=path, ) @staticmethod def new_from_url(remote_url): content = fetch_remote_content(remote_url) return ManifestParserFactory.new( content, ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, remote_url, ) @staticmethod def new_from_archive(path): assert path.endswith("tar.gz") with tarfile.open(path, mode="r:gz") as tf: for t in sorted(ManifestFileType.items().values()): for member in (t, "./" + t): try: return ManifestParserFactory.new( tf.extractfile(member).read().decode(), t ) except KeyError: pass raise UnknownManifestError("Unknown manifest file type in %s archive" % path) @staticmethod def new( # pylint: disable=redefined-builtin contents, type, remote_url=None, package_dir=None ): for _, cls in globals().items(): if ( inspect.isclass(cls) and issubclass(cls, BaseManifestParser) and cls != BaseManifestParser and cls.manifest_type == type ): return cls(contents, remote_url, package_dir) raise UnknownManifestError("Unknown manifest file type %s" % type) class BaseManifestParser: def __init__(self, contents, remote_url=None, package_dir=None): self.remote_url = remote_url self.package_dir = package_dir try: self._data = self.parse(contents) except Exception as exc: raise ManifestParserError("Could not parse manifest -> %s" % exc) from exc self._data = self.normalize_repository(self._data) self._data = self.parse_examples(self._data) # remove None fields for key in list(self._data.keys()): if self._data[key] is None: del self._data[key] def parse(self, contents): raise NotImplementedError def as_dict(self): return self._data @staticmethod def str_to_list(value, sep=",", lowercase=False, unique=False): if isinstance(value, string_types): value = value.split(sep) assert isinstance(value, list) result = [] for item in value: item = item.strip() if not item: continue if lowercase: item = item.lower() if unique and item in result: continue result.append(item) return result @staticmethod def cleanup_author(author): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) if "@" not in author["email"]: author["email"] = None for key in list(author.keys()): if author[key] is None: del author[key] return author @staticmethod def parse_author_name_and_email(raw): if raw == "None" or "://" in raw: return (None, None) name = raw email = None ldel = "<" rdel = ">" if ldel in raw and rdel in raw: name = raw[: raw.index(ldel)] email = raw[raw.index(ldel) + 1 : raw.index(rdel)] if "(" in name: name = name.split("(")[0] return (name.strip(), email.strip() if email else None) @staticmethod def normalize_repository(data): url = (data.get("repository") or {}).get("url") if not url or "://" not in url: return data url_attrs = urlparse(url) if url_attrs.netloc not in ("github.com", "bitbucket.org", "gitlab.com"): return data url = "https://%s%s" % (url_attrs.netloc, url_attrs.path) if url.endswith("/"): url = url[:-1] if not url.endswith(".git"): url += ".git" data["repository"]["url"] = url return data def parse_examples(self, data): examples = data.get("examples") if ( not examples or not isinstance(examples, list) or not all(isinstance(v, dict) for v in examples) ): data["examples"] = None if not data["examples"] and self.package_dir: data["examples"] = self.parse_examples_from_dir(self.package_dir) if "examples" in data and not data["examples"]: del data["examples"] return data @staticmethod def parse_examples_from_dir(package_dir): assert os.path.isdir(package_dir) examples_dir = os.path.join(package_dir, "examples") if not os.path.isdir(examples_dir): examples_dir = os.path.join(package_dir, "Examples") if not os.path.isdir(examples_dir): return None allowed_exts = ( ".c", ".cc", ".cpp", ".h", ".hpp", ".asm", ".ASM", ".s", ".S", ".ino", ".pde", ) result = {} last_pio_project = None for root, _, files in os.walk(examples_dir): # skip hidden files, symlinks, and folders files = [ f for f in files if not f.startswith(".") and not os.path.islink(os.path.join(root, f)) ] if os.path.basename(root).startswith(".") or not files: continue if is_platformio_project(root): last_pio_project = root result[last_pio_project] = dict( name=os.path.relpath(root, examples_dir), base=os.path.relpath(root, package_dir), files=files, ) continue if last_pio_project: if root.startswith(last_pio_project): result[last_pio_project]["files"].extend( [ os.path.relpath(os.path.join(root, f), last_pio_project) for f in files ] ) continue last_pio_project = None matched_files = [f for f in files if f.endswith(allowed_exts)] if not matched_files: continue result[root] = dict( name=( "Examples" if root == examples_dir else os.path.relpath(root, examples_dir) ), base=os.path.relpath(root, package_dir), files=matched_files, ) result = list(result.values()) # normalize example names for item in result: item["name"] = item["name"].replace(os.path.sep, "/") item["name"] = re.sub(r"[^a-z\d\d\-\_/]+", "_", item["name"], flags=re.I) return result or None class LibraryJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.LIBRARY_JSON def parse(self, contents): data = json.loads(contents) data = self._process_renamed_fields(data) # normalize Union[str, list] fields for k in ("keywords", "platforms", "frameworks"): if k in data: data[k] = self.str_to_list( data[k], sep=",", lowercase=True, unique=True ) if "headers" in data: data["headers"] = self.str_to_list(data["headers"], sep=",", unique=True) if "authors" in data: data["authors"] = self._parse_authors(data["authors"]) if "platforms" in data: data["platforms"] = self._fix_platforms(data["platforms"]) or None if "export" in data: data["export"] = self._parse_export(data["export"]) if "dependencies" in data: data["dependencies"] = self._parse_dependencies(data["dependencies"]) return data @staticmethod def _process_renamed_fields(data): if "url" in data: data["homepage"] = data["url"] del data["url"] for key in ("include", "exclude"): if key not in data: continue if "export" not in data: data["export"] = {} data["export"][key] = data[key] del data[key] return data def _parse_authors(self, raw): if not raw: return None # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] return [self.cleanup_author(author) for author in raw] @staticmethod def _fix_platforms(items): assert isinstance(items, list) if "espressif" in items: items[items.index("espressif")] = "espressif8266" return items @staticmethod def _parse_export(raw): if not isinstance(raw, dict): return None result = {} for k in ("include", "exclude"): if not raw.get(k): continue result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]] return result @staticmethod def _parse_dependencies(raw): # compatibility with legacy dependency format if isinstance(raw, dict) and "name" in raw: raw = [raw] if isinstance(raw, dict): result = [] for name, version in raw.items(): if "/" in name: owner, name = name.split("/", 1) result.append(dict(owner=owner, name=name, version=version)) else: result.append(dict(name=name, version=version)) return result if isinstance(raw, list): for i, dependency in enumerate(raw): if isinstance(dependency, dict): for k, v in dependency.items(): if k not in ("platforms", "frameworks", "authors"): continue raw[i][k] = util.items_to_list(v) else: raw[i] = {"name": dependency} return raw raise ManifestParserError( "Invalid dependencies format, should be list or dictionary" ) class ModuleJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.MODULE_JSON def parse(self, contents): data = json.loads(contents) data["frameworks"] = ["mbed"] data["platforms"] = ["*"] data["export"] = {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]} if "author" in data: data["authors"] = self._parse_authors(data.get("author")) del data["author"] if "licenses" in data: data["license"] = self._parse_license(data.get("licenses")) del data["licenses"] if "dependencies" in data: data["dependencies"] = self._parse_dependencies(data["dependencies"]) if "keywords" in data: data["keywords"] = self.str_to_list( data["keywords"], sep=",", lowercase=True, unique=True ) return data def _parse_authors(self, raw): if not raw: return None result = [] for author in raw.split(","): name, email = self.parse_author_name_and_email(author) if not name: continue result.append(self.cleanup_author(dict(name=name, email=email))) return result @staticmethod def _parse_license(raw): if not raw or not isinstance(raw, list): return None return raw[0].get("type") @staticmethod def _parse_dependencies(raw): if isinstance(raw, dict): return [ dict(name=name, version=version, frameworks=["mbed"]) for name, version in raw.items() ] raise ManifestParserError("Invalid dependencies format, should be a dictionary") class LibraryPropertiesManifestParser(BaseManifestParser): manifest_type = ManifestFileType.LIBRARY_PROPERTIES def parse(self, contents): data = self._parse_properties(contents) repository = self._parse_repository(data) homepage = data.get("url") or None if repository and repository["url"] == homepage: homepage = None data.update( dict( frameworks=["arduino"], homepage=homepage, repository=repository or None, description=self._parse_description(data), platforms=self._parse_platforms(data) or None, keywords=self._parse_keywords(data) or None, export=self._parse_export(), ) ) if "includes" in data: data["headers"] = self.str_to_list(data["includes"], sep=",", unique=True) if "author" in data: data["authors"] = self._parse_authors(data) for key in ("author", "maintainer"): if key in data: del data[key] if "depends" in data: data["dependencies"] = self._parse_dependencies(data["depends"]) return data @staticmethod def _parse_properties(contents): data = {} for line in contents.splitlines(): line = line.strip() if not line or "=" not in line: continue # skip comments if line.startswith("#"): continue key, value = line.split("=", 1) if not value.strip(): continue data[key.strip()] = value.strip() return data @staticmethod def _parse_description(properties): lines = [] for k in ("sentence", "paragraph"): if k in properties and properties[k] not in lines: lines.append(properties[k]) if len(lines) == 2: if not lines[0].endswith("."): lines[0] += "." if len(lines[0]) + len(lines[1]) >= 1000: del lines[1] return " ".join(lines) def _parse_keywords(self, properties): return self.str_to_list( re.split( r"[\s/]+", properties.get("category", ""), ), lowercase=True, unique=True, ) def _parse_platforms(self, properties): result = [] platforms_map = { "avr": "atmelavr", "sam": "atmelsam", "samd": "atmelsam", "esp8266": "espressif8266", "esp32": "espressif32", "arc32": "intel_arc32", "stm32": "ststm32", "nrf52": "nordicnrf52", "rp2040": "raspberrypi", } for arch in properties.get("architectures", "").split(","): if "particle-" in arch: raise ManifestParserError("Particle is not supported yet") arch = arch.strip() if not arch: continue if arch == "*": return ["*"] if arch in platforms_map: result.append(platforms_map[arch]) return self.str_to_list(result, lowercase=True, unique=True) def _parse_authors(self, properties): if "author" not in properties: return None authors = [] for author in properties["author"].split(","): name, email = self.parse_author_name_and_email(author) if not name: continue authors.append(self.cleanup_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: continue found = False for item in authors: if item.get("name", "").lower() != name.lower(): continue found = True item["maintainer"] = True # pylint: disable=unsupported-membership-test if not item.get("email") and email and "@" in email: item["email"] = email if not found: authors.append( self.cleanup_author(dict(name=name, email=email, maintainer=True)) ) return authors def _parse_repository(self, properties): if self.remote_url: url_attrs = urlparse(self.remote_url) repo_path_tokens = url_attrs.path[1:].split("/")[:-1] if "github" in url_attrs.netloc: return dict( type="git", url="https://github.com/" + "/".join(repo_path_tokens[:2]), ) if "raw" in repo_path_tokens: return dict( type="git", url="https://%s/%s" % ( url_attrs.netloc, "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]), ), ) if properties.get("url", "").startswith("https://github.com"): return dict(type="git", url=properties["url"]) return None def _parse_export(self): include = None if self.remote_url: url_attrs = urlparse(self.remote_url) repo_path_tokens = url_attrs.path[1:].split("/")[:-1] if "github" in url_attrs.netloc: include = "/".join(repo_path_tokens[3:]) or None elif "raw" in repo_path_tokens: include = ( "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :]) or None ) if include: return dict(include=[include]) return None @staticmethod def _parse_dependencies(raw): result = [] for item in raw.split(","): item = item.strip() if not item: continue if item.endswith(")") and "(" in item: name, version = item.split("(") result.append( dict( name=name.strip(), version=version[:-1].strip(), frameworks=["arduino"], ) ) else: result.append(dict(name=item, frameworks=["arduino"])) return result class PlatformJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.PLATFORM_JSON def parse(self, contents): data = json.loads(contents) if "keywords" in data: data["keywords"] = self.str_to_list( data["keywords"], sep=",", lowercase=True, unique=True ) if "frameworks" in data: data["frameworks"] = ( self.str_to_list( list(data["frameworks"].keys()), lowercase=True, unique=True ) if isinstance(data["frameworks"], dict) else None ) if "packages" in data: data["dependencies"] = self._parse_dependencies(data["packages"]) return data @staticmethod def _parse_dependencies(raw): result = [] for name, opts in raw.items(): item = {"name": name} for k in ("owner", "version"): if k in opts: item[k] = opts[k] result.append(item) return result class PackageJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.PACKAGE_JSON def parse(self, contents): data = json.loads(contents) if "keywords" in data: data["keywords"] = self.str_to_list( data["keywords"], sep=",", lowercase=True, unique=True ) data = self._parse_system(data) data = self._parse_homepage(data) data = self._parse_repository(data) return data def _parse_system(self, data): if "system" not in data: return data if data["system"] in ("*", ["*"], "all"): del data["system"] return data data["system"] = self.str_to_list(data["system"], lowercase=True, unique=True) return data @staticmethod def _parse_homepage(data): if "url" in data: data["homepage"] = data["url"] del data["url"] return data @staticmethod def _parse_repository(data): if isinstance(data.get("repository", {}), dict): return data data["repository"] = dict(type="git", url=str(data["repository"])) if data["repository"]["url"].startswith(("github:", "gitlab:", "bitbucket:")): data["repository"]["url"] = "https://{0}.com/{1}".format( *(data["repository"]["url"].split(":", 1)) ) return data ================================================ FILE: platformio/package/manifest/schema.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-ancestors import json import re import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates from platformio.http import fetch_remote_content from platformio.package.exception import ManifestValidationError from platformio.util import memoized class BaseSchema(Schema): class Meta: unknown = marshmallow.EXCLUDE # pylint: disable=no-member def load_manifest(self, data): return self.load(data) def handle_error(self, error, data, **_): # pylint: disable=arguments-differ raise ManifestValidationError( error.messages, data, error.valid_data if hasattr(error, "valid_data") else error.data, ) class StrictSchema(BaseSchema): def handle_error(self, error, data, **_): # pylint: disable=arguments-differ # skip broken records if self.many: error.valid_data = [ item for idx, item in enumerate(data) if idx not in error.messages ] else: error.valid_data = None raise error class StrictListField(fields.List): def _deserialize( # pylint: disable=arguments-differ self, value, attr, data, **kwargs ): try: return super()._deserialize(value, attr, data, **kwargs) except ValidationError as exc: if exc.data: exc.data = [item for item in exc.data if item is not None] raise exc class AuthorSchema(StrictSchema): name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(dump_default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) class RepositorySchema(StrictSchema): type = fields.Str( required=True, validate=validate.OneOf( ["git", "hg", "svn"], error="Invalid repository type, please use one of [git, hg, svn]", ), ) url = fields.Str(required=True, validate=validate.Length(min=1, max=255)) branch = fields.Str(validate=validate.Length(min=1, max=50)) class DependencySchema(StrictSchema): owner = fields.Str(validate=validate.Length(min=1, max=100)) name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) version = fields.Str(validate=validate.Length(min=1, max=100)) authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50))) platforms = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) frameworks = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) class ExportSchema(BaseSchema): include = StrictListField(fields.Str) exclude = StrictListField(fields.Str) class ExampleSchema(StrictSchema): name = fields.Str( required=True, validate=[ validate.Length(min=1, max=255), validate.Regexp( r"^[a-zA-Z\d\-\_/\. ]+$", error="Only [a-zA-Z0-9-_/. ] chars are allowed", ), ], ) base = fields.Str(required=True) files = StrictListField(fields.Str, required=True) # Fields class ScriptField(fields.Field): def _deserialize(self, value, attr, data, **kwargs): if isinstance(value, (str, list)): return value raise ValidationError( "Script value must be a command (string) or list of arguments" ) # Scheme class ManifestSchema(BaseSchema): # Required fields name = fields.Str( required=True, validate=[ validate.Length(min=1, max=100), validate.Regexp( r"^[^:;/,@\<\>]+$", error="The next chars [:;/,@<>] are not allowed" ), ], ) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) # Optional fields authors = fields.Nested(AuthorSchema, many=True) description = fields.Str(validate=validate.Length(min=1, max=1000)) homepage = fields.Url(validate=validate.Length(min=1, max=255)) license = fields.Str(validate=validate.Length(min=1, max=255)) repository = fields.Nested(RepositorySchema) dependencies = fields.Nested(DependencySchema, many=True) scripts = fields.Dict( keys=fields.Str(validate=validate.OneOf(["postinstall", "preuninstall"])), values=ScriptField(), ) # library.json export = fields.Nested(ExportSchema) examples = fields.Nested(ExampleSchema, many=True) downloadUrl = fields.Url(validate=validate.Length(min=1, max=255)) keywords = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^[a-z\d\-_\+\. ]+$", error="Only [a-z0-9+_-. ] chars are allowed" ), ] ) ) platforms = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) frameworks = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) headers = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=255), ] ) ) # platform.json specific title = fields.Str(validate=validate.Length(min=1, max=100)) # package.json specific system = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed" ), ] ) ) @validates("version") def validate_version(self, value): try: value = str(value) assert "." in value # check leading zeros try: semantic_version.Version(value) except ValueError as exc: if "Invalid leading zero" in str(exc): raise exc semantic_version.Version.coerce(value) except (AssertionError, ValueError) as exc: raise ValidationError( "Invalid semantic versioning format, see https://semver.org/" ) from exc @validates("license") def validate_license(self, value): try: spdx = self.load_spdx_licenses() except requests.exceptions.RequestException as exc: raise ValidationError( "Could not load SPDX licenses for validation" ) from exc known_ids = set(item.get("licenseId") for item in spdx.get("licenses", [])) if value in known_ids: return True # parse license expression # https://spdx.github.io/spdx-spec/SPDX-license-expressions/ package_ids = [ item.strip() for item in re.sub(r"(\s+(?:OR|AND|WITH)\s+|[\(\)])", " ", value).split(" ") if item.strip() ] if known_ids >= set(package_ids): return True raise ValidationError( "Invalid SPDX license identifier. See valid identifiers at " "https://spdx.org/licenses/" ) @staticmethod @memoized(expire="1h") def load_spdx_licenses(): version = "3.28.0" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" f"v{version}/json/licenses.json" ) return json.loads(fetch_remote_content(spdx_data_url)) ================================================ FILE: platformio/package/meta.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import re import tarfile from binascii import crc32 from urllib.parse import urlparse import semantic_version from platformio import fs from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType from platformio.package.version import SemanticVersionError, cast_version_to_semver from platformio.util import items_in_list class PackageType: LIBRARY = "library" PLATFORM = "platform" TOOL = "tool" @classmethod def items(cls): return get_object_members(cls) @classmethod def get_manifest_map(cls): return { cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), cls.LIBRARY: ( ManifestFileType.LIBRARY_JSON, ManifestFileType.LIBRARY_PROPERTIES, ManifestFileType.MODULE_JSON, ), cls.TOOL: (ManifestFileType.PACKAGE_JSON,), } @classmethod def from_archive(cls, path): assert path.endswith("tar.gz") manifest_map = cls.get_manifest_map() with tarfile.open(path, mode="r:gz") as tf: for t in sorted(cls.items().values()): for manifest in manifest_map[t]: try: if tf.getmember(manifest): return t except KeyError: pass return None class PackageCompatibility: KNOWN_QUALIFIERS = ( "owner", "name", "version", "platforms", "frameworks", "authors", ) @classmethod def from_dependency(cls, dependency): assert isinstance(dependency, dict) qualifiers = { key: value for key, value in dependency.items() if key in cls.KNOWN_QUALIFIERS } return PackageCompatibility(**qualifiers) def __init__(self, **kwargs): self.qualifiers = {} for key, value in kwargs.items(): if key not in self.KNOWN_QUALIFIERS: raise ValueError( "Unknown package compatibility qualifier -> `%s`" % key ) self.qualifiers[key] = value def __repr__(self): return "PackageCompatibility <%s>" % self.qualifiers def to_search_qualifiers(self, fields=None): result = {} for name, value in self.qualifiers.items(): if not fields or name in fields: result[name] = value return result def is_compatible(self, other): assert isinstance(other, PackageCompatibility) for key, current_value in self.qualifiers.items(): other_value = other.qualifiers.get(key) if not current_value or not other_value: continue if any(isinstance(v, list) for v in (current_value, other_value)): if not items_in_list(current_value, other_value): return False continue if key == "version": if not self._compare_versions(current_value, other_value): return False continue if current_value != other_value: return False return True def _compare_versions(self, current, other): if current == other: return True try: version = ( other if isinstance(other, semantic_version.Version) else cast_version_to_semver(other) ) return version in semantic_version.SimpleSpec(current) except ValueError: pass return False class PackageOutdatedResult: UPDATE_INCREMENT_MAJOR = "major" UPDATE_INCREMENT_MINOR = "minor" UPDATE_INCREMENT_PATCH = "patch" def __init__(self, current, latest=None, wanted=None, detached=False): self.current = current self.latest = latest self.wanted = wanted self.detached = detached def __repr__(self): return ( "PackageOutdatedResult ".format( current=self.current, latest=self.latest, wanted=self.wanted, detached=self.detached, ) ) def __setattr__(self, name, value): if ( value and name in ("current", "latest", "wanted") and not isinstance(value, semantic_version.Version) ): value = cast_version_to_semver(str(value)) return super().__setattr__(name, value) @property def update_increment_type(self): if not self.current or not self.latest: return None patch_conds = [ self.current.major == self.latest.major, self.current.minor == self.latest.minor, ] if all(patch_conds): return self.UPDATE_INCREMENT_PATCH minor_conds = [ self.current.major == self.latest.major, self.current.major > 0, ] if all(minor_conds): return self.UPDATE_INCREMENT_MINOR return self.UPDATE_INCREMENT_MAJOR def is_outdated(self, allow_incompatible=False): if self.detached or not self.latest or self.current == self.latest: return False if allow_incompatible: return self.current != self.latest if self.wanted: return self.current != self.wanted return True class PackageSpec: # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments,too-many-positional-arguments self, raw=None, owner=None, id=None, name=None, requirements=None, uri=None ): self._requirements = None self.owner = owner self.id = id self.name = name self.uri = uri self.raw = raw if requirements: try: self.requirements = requirements except SemanticVersionError as exc: if not self.name or self.uri or self.raw: raise exc self.raw = "%s=%s" % (self.name, requirements) self._name_is_custom = False self._parse(self.raw) def __eq__(self, other): return all( [ self.owner == other.owner, self.id == other.id, self.name == other.name, self.requirements == other.requirements, self.uri == other.uri, ] ) def __hash__(self): return crc32( hashlib_encode_data( "%s-%s-%s-%s-%s" % (self.owner, self.id, self.name, self.requirements, self.uri) ) ) def __repr__(self): return ( "PackageSpec ".format(**self.as_dict()) ) @property def external(self): return bool(self.uri) @property def symlink(self): return self.uri and self.uri.startswith("symlink://") @property def requirements(self): return self._requirements @requirements.setter def requirements(self, value): if not value: self._requirements = None return try: self._requirements = ( value if isinstance(value, semantic_version.SimpleSpec) else semantic_version.SimpleSpec(str(value)) ) except ValueError as exc: raise SemanticVersionError(exc) from exc def humanize(self): result = "" if self.uri: result = self.uri elif self.name: if self.owner: result = self.owner + "/" result += self.name elif self.id: result = "id:%d" % self.id if self.requirements: result += " @ " + str(self.requirements) return result def has_custom_name(self): return self._name_is_custom def as_dict(self): return dict( owner=self.owner, id=self.id, name=self.name, requirements=str(self.requirements) if self.requirements else None, uri=self.uri, ) def as_dependency(self): if self.uri: return self.raw or self.uri result = "" if self.name: result = "%s/%s" % (self.owner, self.name) if self.owner else self.name elif self.id: result = str(self.id) assert result if self.requirements: result = "%s@%s" % (result, self.requirements) return result def _parse(self, raw): if raw is None: return if not isinstance(raw, string_types): raw = str(raw) raw = raw.strip() parsers = ( self._parse_local_file, self._parse_requirements, self._parse_custom_name, self._parse_id, self._parse_owner, self._parse_uri, ) for parser in parsers: if raw is None: break raw = parser(raw) # if name is not custom, parse it from URI if not self.name and self.uri: self.name = self._parse_name_from_uri(self.uri) elif raw: # the leftover is a package name self.name = raw @staticmethod def _parse_local_file(raw): if raw.startswith(("file://", "symlink://")) or not any( c in raw for c in ("/", "\\") ): return raw if os.path.exists(raw): return "file://%s" % raw return raw def _parse_requirements(self, raw): if "@" not in raw or raw.startswith(("file://", "symlink://")): return raw tokens = raw.rsplit("@", 1) if any(s in tokens[1] for s in (":", "/")): return raw self.requirements = tokens[1].strip() return tokens[0].strip() def _parse_custom_name(self, raw): if "=" not in raw or raw.startswith("id="): return raw tokens = raw.split("=", 1) if "/" in tokens[0]: return raw self.name = tokens[0].strip() self._name_is_custom = True return tokens[1].strip() def _parse_id(self, raw): if raw.isdigit(): self.id = int(raw) return None if raw.startswith("id="): return self._parse_id(raw[3:]) return raw def _parse_owner(self, raw): if raw.count("/") != 1 or "@" in raw: return raw tokens = raw.split("/", 1) self.owner = tokens[0].strip() self.name = tokens[1].strip() return None def _parse_uri(self, raw): if not any(s in raw for s in ("@", ":", "/")): return raw self.uri = raw.strip() parts = urlparse(self.uri) # if local file or valid URI with scheme vcs+protocol:// if ( parts.scheme in ("file", "symlink://") or "+" in parts.scheme or self.uri.startswith("git+") ): return None # parse VCS git_conditions = [ parts.path.endswith(".git"), # Handle GitHub URL (https://github.com/user/package) parts.netloc in ("github.com", "gitlab.com", "bitbucket.com") and not parts.path.endswith((".zip", ".tar.gz", ".tar.xz")), ] hg_conditions = [ # Handle Developer Mbed URL # (https://developer.mbed.org/users/user/code/package/) # (https://os.mbed.com/users/user/code/package/) parts.netloc in ("mbed.com", "os.mbed.com", "developer.mbed.org") ] if any(git_conditions): self.uri = "git+" + self.uri elif any(hg_conditions): self.uri = "hg+" + self.uri return None @staticmethod def _parse_name_from_uri(uri): if uri.endswith("/"): uri = uri[:-1] stop_chars = ["#", "?"] if uri.startswith(("file://", "symlink://")): stop_chars.append("@") # detached path for c in stop_chars: if c in uri: uri = uri[: uri.index(c)] # parse real repository name from Github parts = urlparse(uri) if parts.netloc == "github.com" and parts.path.count("/") > 2: return parts.path.split("/")[2] name = os.path.basename(uri) if "." in name: return name.split(".", 1)[0].strip() return name class PackageMetadata: def __init__( # pylint: disable=redefined-builtin self, type, name, version, spec=None ): # assert type in PackageType.items().values() if spec: assert isinstance(spec, PackageSpec) self.type = type self.name = name self._version = None self.version = version self.spec = spec def __repr__(self): return ( "PackageMetadata # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import re import shutil import tarfile import tempfile from platformio import fs from platformio.compat import IS_WINDOWS from platformio.package.exception import PackageException, UserSideException from platformio.package.manifest.parser import ( LibraryPropertiesManifestParser, ManifestFileType, ManifestParserFactory, ) from platformio.package.manifest.schema import ManifestSchema from platformio.package.meta import PackageItem from platformio.package.unpack import FileUnpacker class PackagePacker: INCLUDE_DEFAULT = list(ManifestFileType.items().values()) + [ "README", "README.md", "README.rst", "LICENSE", ] EXCLUDE_DEFAULT = [ # PlatformIO internal files PackageItem.METAFILE_NAME, ".pio/", "**/.pio/", # Hidden files "._*", "__*", ".DS_Store", ".vscode", "**/.vscode/", ".cache", "**/.cache", "**/__pycache__", "**/*.pyc", # VCS ".git/", ".hg/", ".svn/", ] EXCLUDE_EXTRA = [ # Tests "test", "tests", # Docs "doc", "docs", "mkdocs", "doxygen", "*.doxyfile", "html", "media", "**/*.[pP][dD][fF]", "**/*.[dD][oO][cC]", "**/*.[dD][oO][cC][xX]", "**/*.[pP][pP][tT]", "**/*.[pP][pP][tT][xX]", "**/*.[xX][lL][sS]", "**/*.[xX][lL][sS][xX]", "**/*.[dD][oO][xX]", "**/*.[hH][tT][mM]", "**/*.[hH][tT][mM][lL]", "**/*.[tT][eE][xX]", "**/*.[jJ][sS]", "**/*.[cC][sS][sS]", # Binary files "**/*.[jJ][pP][gG]", "**/*.[jJ][pP][eE][gG]", "**/*.[pP][nN][gG]", "**/*.[gG][iI][fF]", "**/*.[sS][vV][gG]", "**/*.[zZ][iI][pP]", "**/*.[gG][zZ]", "**/*.3[gG][pP]", "**/*.[mM][oO][vV]", "**/*.[mM][pP][34]", "**/*.[pP][sS][dD]", "**/*.[wW][aA][wW]", "**/*.sqlite", ] EXCLUDE_LIBRARY_EXTRA = [ "assets", "extra", "extras", "resources", "**/build/", "**/*.flat", "**/*.[jJ][aA][rR]", "**/*.[eE][xX][eE]", "**/*.[bB][iI][nN]", "**/*.[hH][eE][xX]", "**/*.[dD][bB]", "**/*.[dD][aA][tT]", "**/*.[dD][lL][lL]", ] def __init__(self, package, manifest_uri=None): self.package = package self.manifest_uri = manifest_uri self.manifest_parser = None @staticmethod def get_archive_name(name, version, system=None): return re.sub( r"[^\da-zA-Z\-\._\+]+", "", "{name}{system}-{version}.tar.gz".format( name=name, system=("-" + system) if system else "", version=version, ), ) @staticmethod def load_gitignore_filters(path): result = [] with open(path, encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if not line or line.startswith(("#")): continue if line.startswith("!"): result.append(f"+<{line[1:]}>") else: result.append(f"-<{line}>") return result def pack(self, dst=None): tmp_dir = tempfile.mkdtemp() try: src = self.package # if zip/tar.gz -> unpack to tmp dir if not os.path.isdir(src): if IS_WINDOWS: raise UserSideException( "Packaging from an archive does not work on Windows OS. Please " "extract data from `%s` manually and pack a folder instead" % src ) with FileUnpacker(src) as fu: assert fu.unpack(tmp_dir, silent=True) src = tmp_dir src = self.find_source_root(src) self.manifest_parser = ManifestParserFactory.new_from_dir(src) manifest = ManifestSchema().load_manifest(self.manifest_parser.as_dict()) filename = self.get_archive_name( manifest["name"], manifest["version"], manifest["system"][0] if "system" in manifest else None, ) if not dst: dst = os.path.join(os.getcwd(), filename) elif os.path.isdir(dst): dst = os.path.join(dst, filename) return self.create_tarball(src, dst, manifest) finally: shutil.rmtree(tmp_dir) def find_source_root(self, src): if self.manifest_uri: mp = ( ManifestParserFactory.new_from_file(self.manifest_uri[5:]) if self.manifest_uri.startswith("file:") else ManifestParserFactory.new_from_url(self.manifest_uri) ) manifest = ManifestSchema().load_manifest(mp.as_dict()) include = manifest.get("export", {}).get("include", []) if len(include) == 1: if not os.path.isdir(os.path.join(src, include[0])): raise PackageException( "Non existing `include` directory `%s` in a package" % include[0] ) return os.path.join(src, include[0]) for root, _, __ in os.walk(src): if ManifestFileType.from_dir(root): return root return src def create_tarball(self, src, dst, manifest): include = manifest.get("export", {}).get("include") exclude = manifest.get("export", {}).get("exclude") # remap root if ( include and len(include) == 1 and os.path.isdir(os.path.join(src, include[0])) ): src = os.path.join(src, include[0]) with open( os.path.join(src, "library.json"), mode="w", encoding="utf8" ) as fp: manifest_updated = manifest.copy() del manifest_updated["export"]["include"] json.dump(manifest_updated, fp, indent=2, ensure_ascii=False) include = None src_filters = self.compute_src_filters(src, include, exclude) with tarfile.open(dst, "w:gz") as tar: for f in fs.match_src_files(src, src_filters, followlinks=False): tar.add(os.path.join(src, f), f) return dst def compute_src_filters(self, src, include, exclude): exclude_extra = self.EXCLUDE_EXTRA[:] # extend with library extra filters if any( os.path.isfile(os.path.join(src, name)) for name in ( ManifestFileType.LIBRARY_JSON, ManifestFileType.LIBRARY_PROPERTIES, ManifestFileType.MODULE_JSON, ) ): exclude_extra.extend(self.EXCLUDE_LIBRARY_EXTRA) result = ["+<%s>" % p for p in include or ["*", ".*"]] result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT] # exclude items declared in manifest result += ["-<%s>" % p for p in exclude or []] # apply extra excludes if no custom "export" field in manifest if (not include and not exclude) or isinstance( self.manifest_parser, LibraryPropertiesManifestParser ): result += ["-<%s>" % p for p in exclude_extra] if os.path.exists(os.path.join(src, ".gitignore")): result += self.load_gitignore_filters(os.path.join(src, ".gitignore")) # always include manifests and relevant files result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT] return result ================================================ FILE: platformio/package/unpack.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys from tarfile import open as tarfile_open from time import mktime from zipfile import ZipFile import click from platformio import fs from platformio.compat import is_terminal from platformio.package.exception import PackageException class ExtractArchiveItemError(PackageException): MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " "tool or check this solution -> https://bit.ly/faq-package-manager" ) class BaseArchiver: def __init__(self, arhfileobj): self._afo = arhfileobj def get_items(self): raise NotImplementedError() def get_item_filename(self, item): raise NotImplementedError() def is_link(self, item): raise NotImplementedError() def extract_item(self, item, dest_dir): self._afo.extract(item, dest_dir) self.after_extract(item, dest_dir) def after_extract(self, item, dest_dir): pass def close(self): self._afo.close() class TARArchiver(BaseArchiver): def __init__(self, archpath): super().__init__(tarfile_open(archpath)) # pylint: disable=consider-using-with def get_items(self): return self._afo.getmembers() def get_item_filename(self, item): return item.name @staticmethod def is_link(item): # pylint: disable=arguments-differ return item.islnk() or item.issym() @staticmethod def resolve_path(path): return os.path.realpath(os.path.abspath(path)) def is_bad_path(self, path, base): return not self.resolve_path(os.path.join(base, path)).startswith(base) def is_bad_link(self, item, base): return not self.resolve_path( os.path.join(os.path.join(base, os.path.dirname(item.name)), item.linkname) ).startswith(base) def extract_item(self, item, dest_dir): if sys.version_info >= (3, 12): self._afo.extract(item, dest_dir, filter="data") return self.after_extract(item, dest_dir) # apply custom security logic dest_dir = self.resolve_path(dest_dir) bad_conds = [ self.is_bad_path(item.name, dest_dir), self.is_link(item) and self.is_bad_link(item, dest_dir), ] if any(bad_conds): return click.secho( "Blocked insecure item `%s` from TAR archive" % item.name, fg="red", err=True, ) return super().extract_item(item, dest_dir) class ZIPArchiver(BaseArchiver): def __init__(self, archpath): super().__init__(ZipFile(archpath)) # pylint: disable=consider-using-with @staticmethod def preserve_permissions(item, dest_dir): attrs = item.external_attr >> 16 if attrs: os.chmod(os.path.join(dest_dir, item.filename), attrs) @staticmethod def preserve_mtime(item, dest_dir): fs.change_filemtime( os.path.join(dest_dir, item.filename), mktime(tuple(item.date_time) + tuple([0, 0, 0])), ) @staticmethod def is_link(_): # pylint: disable=arguments-differ return False def get_items(self): return self._afo.infolist() def get_item_filename(self, item): return item.filename def after_extract(self, item, dest_dir): self.preserve_permissions(item, dest_dir) self.preserve_mtime(item, dest_dir) class FileUnpacker: def __init__(self, path): self.path = path self._archiver = None def __enter__(self): self._archiver = self.new_archiver(self.path) return self def __exit__(self, *args): if self._archiver: self._archiver.close() @staticmethod def new_archiver(path): magic_map = { b"\x1f\x8b\x08": TARArchiver, b"\x42\x5a\x68": TARArchiver, b"\xfd\x37\x7a\x58\x5a\x00": TARArchiver, b"\x50\x4b\x03\x04": ZIPArchiver, } magic_len = max(len(k) for k in magic_map) with open(path, "rb") as fp: data = fp.read(magic_len) for magic, archiver in magic_map.items(): if data.startswith(magic): return archiver(path) raise PackageException("Unknown archive type '%s'" % path) def unpack( self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False ): # pylint: disable=too-many-branches assert self._archiver label = "Unpacking" items = self._archiver.get_items() if not dest_dir: dest_dir = os.getcwd() if not with_progress or silent: if not silent: click.echo(f"{label}...") for item in items: self._archiver.extract_item(item, dest_dir) elif not is_terminal(): click.echo(f"{label} 0%", nl=False) print_percent_step = 10 printed_percents = 0 unpacked_nums = 0 for item in items: self._archiver.extract_item(item, dest_dir) unpacked_nums += 1 if (unpacked_nums / len(items) * 100) >= ( printed_percents + print_percent_step ): printed_percents += print_percent_step click.echo(f" {printed_percents}%", nl=False) click.echo("") else: with click.progressbar( items, label=label, update_min_steps=min(50, len(items) / 100), # every 50 files or less ) as pb: for item in pb: self._archiver.extract_item(item, dest_dir) if not check_unpacked: return True # check on disk for item in self._archiver.get_items(): filename = self._archiver.get_item_filename(item) item_path = os.path.join(dest_dir, filename) try: if not self._archiver.is_link(item) and not os.path.exists(item_path): raise ExtractArchiveItemError(filename, dest_dir) except NotImplementedError: pass return True ================================================ FILE: platformio/package/vcsclient.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import subprocess from urllib.parse import urlparse from platformio import proc from platformio.exception import UserSideException class VCSBaseException(UserSideException): pass class VCSClientFactory: @staticmethod def new(src_dir, remote_url, silent=False): result = urlparse(remote_url) type_ = result.scheme tag = None if not type_ and remote_url.startswith("git+"): type_ = "git" remote_url = remote_url[4:] elif "+" in result.scheme: type_, _ = result.scheme.split("+", 1) remote_url = remote_url[len(type_) + 1 :] if "#" in remote_url: remote_url, tag = remote_url.rsplit("#", 1) if not type_: raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) try: obj = globals()["%sClient" % type_.capitalize()]( src_dir, remote_url, tag, silent ) assert isinstance(obj, VCSClientBase) return obj except (KeyError, AssertionError) as exc: raise VCSBaseException( "VCS: Unknown repository type %s" % remote_url ) from exc class VCSClientBase: command = None def __init__(self, src_dir, remote_url=None, tag=None, silent=False): self.src_dir = src_dir self.remote_url = remote_url self.tag = tag self.silent = silent self.check_client() def check_client(self): try: assert self.command if self.silent: self.get_cmd_output(["--version"]) else: assert self.run_cmd(["--version"]) except (AssertionError, OSError) as exc: raise VCSBaseException( "VCS: `%s` client is not installed in your system" % self.command ) from exc return True @property def storage_dir(self): return os.path.join(self.src_dir, "." + self.command) def export(self): raise NotImplementedError def update(self): raise NotImplementedError @property def can_be_updated(self): return not self.tag def get_current_revision(self): raise NotImplementedError def get_latest_revision(self): return None if self.can_be_updated else self.get_current_revision() def run_cmd(self, args, **kwargs): args = [self.command] + args if "cwd" not in kwargs: kwargs["cwd"] = self.src_dir if "env" not in kwargs: kwargs["env"] = os.environ try: subprocess.check_call(args, **kwargs) return True except subprocess.CalledProcessError as exc: raise VCSBaseException( "VCS: Could not process command %s" % exc.cmd ) from exc def get_cmd_output(self, args, **kwargs): args = [self.command] + args if "cwd" not in kwargs: kwargs["cwd"] = self.src_dir result = proc.exec_command(args, **kwargs) if result["returncode"] == 0: return result["out"].strip() raise VCSBaseException( "VCS: Could not receive an output from `%s` command (%s)" % (args, result) ) class GitClient(VCSClientBase): command = "git" _configured = False def __init__(self, *args, **kwargs): self.configure() super().__init__(*args, **kwargs) @classmethod def configure(cls): if cls._configured: return True cls._configured = True try: result = proc.exec_command([cls.command, "--exec-path"]) if result["returncode"] != 0: return False path = result["out"].strip() if path: proc.append_env_path("PATH", path) return True except (subprocess.CalledProcessError, FileNotFoundError): pass return False def check_client(self): try: return VCSClientBase.check_client(self) except UserSideException as exc: raise UserSideException( "Please install Git client from https://git-scm.com/downloads" ) from exc def get_branches(self): output = self.get_cmd_output(["branch"]) output = output.replace("*", "") # fix active branch return [b.strip() for b in output.split("\n")] def get_current_branch(self): output = self.get_cmd_output(["branch"]) for line in output.split("\n"): line = line.strip() if line.startswith("*"): branch = line[1:].strip() if branch != "(no branch)": return branch return None def get_tags(self): output = self.get_cmd_output(["tag", "-l"]) return [t.strip() for t in output.split("\n")] @staticmethod def is_commit_id(text): return text and re.match(r"[0-9a-f]{7,}$", text) is not None @property def can_be_updated(self): return not self.tag or not self.is_commit_id(self.tag) def export(self): is_commit = self.is_commit_id(self.tag) args = ["clone", "--recursive"] if not self.tag or not is_commit: args += ["--depth", "1"] if self.tag: args += ["--branch", self.tag] args += [self.remote_url, self.src_dir] assert self.run_cmd(args, cwd=os.getcwd()) if is_commit: assert self.run_cmd(["reset", "--hard", self.tag]) return self.run_cmd( ["submodule", "update", "--init", "--recursive", "--force"] ) return True def update(self): args = ["pull", "--recurse-submodules"] return self.run_cmd(args) def get_current_revision(self): return self.get_cmd_output(["rev-parse", "--short", "HEAD"]) def get_latest_revision(self): if not self.can_be_updated: return self.get_current_revision() branch = self.get_current_branch() if not branch: return None branch_ref = f"refs/heads/{branch}" result = self.get_cmd_output(["ls-remote", self.remote_url, branch_ref]) if not result: return None for line in result.split("\n"): sha, ref = line.strip().split("\t") if ref == branch_ref: return sha[:7] return None class HgClient(VCSClientBase): command = "hg" def export(self): args = ["clone"] if self.tag: args.extend(["--updaterev", self.tag]) args.extend([self.remote_url, self.src_dir]) return self.run_cmd(args) def update(self): args = ["pull", "--update"] return self.run_cmd(args) def get_current_revision(self): return self.get_cmd_output(["identify", "--id"]) def get_latest_revision(self): if not self.can_be_updated: return self.get_latest_revision() return self.get_cmd_output(["identify", "--id", self.remote_url]) class SvnClient(VCSClientBase): command = "svn" def export(self): args = ["checkout"] if self.tag: args.extend(["--revision", self.tag]) args.extend([self.remote_url, self.src_dir]) return self.run_cmd(args) def update(self): args = ["update"] return self.run_cmd(args) def get_current_revision(self): output = self.get_cmd_output( ["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"] ) for line in output.split("\n"): line = line.strip() if line.startswith("Revision:"): return line.split(":", 1)[1].strip() raise VCSBaseException("Could not detect current SVN revision") ================================================ FILE: platformio/package/version.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re import semantic_version from platformio.exception import UserSideException class SemanticVersionError(UserSideException): pass def cast_version_to_semver(value, force=True, raise_exception=False): assert value try: return semantic_version.Version(value) except ValueError: pass if force: try: return semantic_version.Version.coerce(value) except ValueError: pass if raise_exception: raise SemanticVersionError("Invalid SemVer version %s" % value) # parse commit hash if re.match(r"^[\da-f]+$", value, flags=re.I): return semantic_version.Version("0.0.0+sha." + value) return semantic_version.Version("0.0.0+" + value) def pepver_to_semver(pepver): return cast_version_to_semver( re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, count=1) ) def get_original_version(version): if version.count(".") != 2: return None _, raw = version.split(".")[:2] if int(raw) <= 99: return None if int(raw) <= 9999: return "%s.%s" % (raw[:-2], int(raw[-2:])) return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) ================================================ FILE: platformio/platform/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/platform/_packages.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.package.meta import PackageSpec class PlatformPackagesMixin: def get_package_spec(self, name, version=None): return PackageSpec( owner=self.packages[name].get("owner"), name=name, requirements=version or self.packages[name].get("version"), ) def get_package(self, name, spec=None): if not name: return None return self.pm.get_package(spec or self.get_package_spec(name)) def get_package_dir(self, name): pkg = self.get_package(name) return pkg.path if pkg else None def get_package_version(self, name): pkg = self.get_package(name) return str(pkg.metadata.version) if pkg else None def get_installed_packages(self, with_optional=True, with_optional_versions=False): result = [] for name, options in dict(sorted(self.packages.items())).items(): if not with_optional and options.get("optional"): continue versions = [options.get("version")] if with_optional_versions: versions.extend(options.get("optionalVersions", [])) for version in versions: if not version: continue pkg = self.get_package(name, self.get_package_spec(name, version)) if pkg: result.append(pkg) return result def dump_used_packages(self): result = [] for name, options in self.packages.items(): if options.get("optional"): continue pkg = self.get_package(name) if not pkg or not pkg.metadata: continue item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)} if pkg.metadata.spec.external: item["src_url"] = pkg.metadata.spec.uri result.append(item) return result def install_package(self, name, spec=None, force=False): return self.pm.install(spec or self.get_package_spec(name), force=force) def install_required_packages(self, force=False): for name, options in self.packages.items(): if options.get("optional"): continue self.install_package(name, force=force) def uninstall_packages(self): for pkg in self.get_installed_packages(): self.pm.uninstall(pkg) def update_packages(self): for pkg in self.get_installed_packages(): self.pm.update(pkg, to_spec=self.get_package_spec(pkg.metadata.name)) def are_outdated_packages(self): for pkg in self.get_installed_packages(): if self.pm.outdated( pkg, self.get_package_spec(pkg.metadata.name) ).is_outdated(allow_incompatible=False): return True return False ================================================ FILE: platformio/platform/_run.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import base64 import json import os import re import sys from urllib.parse import quote import click from platformio import app, fs, proc, telemetry from platformio.compat import hashlib_encode_data from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound from platformio.run.helpers import KNOWN_CLEAN_TARGETS, KNOWN_FULLCLEAN_TARGETS class PlatformRunMixin: LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) @staticmethod def encode_scons_arg(value): if isinstance(value, (list, tuple, dict)): value = json.dumps(value) return base64.urlsafe_b64encode(hashlib_encode_data(value)).decode() @staticmethod def decode_scons_arg(data): value = base64.urlsafe_b64decode(data).decode() if value.startswith(("[", "{")): value = json.loads(value) return value def run( # pylint: disable=too-many-arguments,too-many-positional-arguments self, variables, targets, silent, verbose, jobs ): assert isinstance(variables, dict) assert isinstance(targets, list) self.ensure_engine_compatible() self.silent = silent self.verbose = verbose or app.get_setting("force_verbose") if "build_script" not in variables: variables["build_script"] = self.get_build_script() if not os.path.isfile(variables["build_script"]): raise BuildScriptNotFound(variables["build_script"]) telemetry.log_platform_run(self, self.config, variables["pioenv"], targets) result = self._run_scons(variables, targets, jobs) assert "returncode" in result return result def _run_scons(self, variables, targets, jobs): scons_dir = get_core_package_dir("tool-scons") args = [ proc.get_pythonexe_path(), os.path.join(scons_dir, "scons.py"), "-Q", "--warn=no-no-parallel-support", "--jobs", str(jobs), "--sconstruct", os.path.join(fs.get_source_dir(), "builder", "main.py"), ] args.append("PIOVERBOSE=%d" % int(self.verbose)) # pylint: disable=protected-access args.append("ISATTY=%d" % int(click._compat.isatty(sys.stdout))) # encode and append variables for key, value in variables.items(): args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) if set(KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS) & set(targets): args.append("--clean") args.append( "FULLCLEAN=%d" % (1 if set(KNOWN_FULLCLEAN_TARGETS) & set(targets) else 0) ) elif targets: args.extend(targets) # force SCons output to Unicode os.environ["PYTHONIOENCODING"] = "utf-8" if targets and "menuconfig" in targets: return proc.exec_command( args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin ) if click._compat.isatty(sys.stdout): def _write_and_flush(stream, data): try: stream.write(data) stream.flush() except IOError: pass return proc.exec_command( args, stdout=proc.BuildAsyncPipe( line_callback=self._on_stdout_line, data_callback=lambda data: ( None if self.silent else _write_and_flush(sys.stdout, data) ), ), stderr=proc.BuildAsyncPipe( line_callback=self._on_stderr_line, data_callback=lambda data: _write_and_flush(sys.stderr, data), ), ) return proc.exec_command( args, stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), ) def _on_stdout_line(self, line): if "`buildprog' is up to date." in line: return self._echo_line(line, level=1) def _on_stderr_line(self, line): is_error = self.LINE_ERROR_RE.search(line) is not None self._echo_line(line, level=3 if is_error else 2) a_pos = line.find("fatal error:") b_pos = line.rfind(": No such file or directory") if a_pos == -1 or b_pos == -1: return self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) def _echo_line(self, line, level): if line.startswith("scons: "): line = line[7:] assert 1 <= level <= 3 if self.silent and (level < 2 or not line): return fg = (None, "yellow", "red")[level - 1] if level == 1 and "is up to date" in line: fg = "green" click.secho(line, fg=fg, err=level > 1, nl=False) @staticmethod def _echo_missed_dependency(filename): if "/" in filename or not filename.endswith((".h", ".hpp")): return banner = """ {dots} * Looking for {filename_styled} dependency? Check our library registry! * * CLI > platformio lib search "header:{filename}" * Web > {link} * {dots} """.format( filename=filename, filename_styled=click.style(filename, fg="cyan"), link=click.style( "https://registry.platformio.org/search?q=header:%s" % quote(filename, safe=""), fg="blue", ), dots="*" * (56 + len(filename)), ) click.echo(banner, err=True) ================================================ FILE: platformio/platform/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import semantic_version from platformio import __version__, fs from platformio.package.manager.tool import ToolPackageManager from platformio.package.version import pepver_to_semver from platformio.platform._packages import PlatformPackagesMixin from platformio.platform._run import PlatformRunMixin from platformio.platform.board import PlatformBoardConfig from platformio.platform.exception import IncompatiblePlatform, UnknownBoard from platformio.project.config import ProjectConfig class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods PlatformPackagesMixin, PlatformRunMixin ): CORE_SEMVER = pepver_to_semver(__version__) _BOARDS_CACHE = {} def __init__(self, manifest_path): self.manifest_path = manifest_path self.project_env = None # set by factory.from_env(env) self.silent = False self.verbose = False self._manifest = fs.load_json(manifest_path) self._BOARDS_CACHE = {} self._custom_packages = None self.config = ProjectConfig.get_instance() self.pm = ToolPackageManager(self.config.get("platformio", "packages_dir")) @property def name(self): return self._manifest["name"] @property def title(self): return self._manifest["title"] @property def description(self): return self._manifest["description"] @property def version(self): return self._manifest["version"] @property def homepage(self): return self._manifest.get("homepage") @property def repository_url(self): return self._manifest.get("repository", {}).get("url") @property def license(self): return self._manifest.get("license") @property def frameworks(self): return self._manifest.get("frameworks") @property def engines(self): return self._manifest.get("engines") @property def manifest(self): return self._manifest @property def packages(self): packages = self._manifest.get("packages", {}) for item in self._custom_packages or []: name = item version = "*" if "@" in item: name, version = item.split("@", 1) spec = self.pm.ensure_spec(name) options = {"version": version.strip(), "optional": False} if spec.owner: options["owner"] = spec.owner if spec.name not in packages: packages[spec.name] = {} packages[spec.name].update(**options) return packages def ensure_engine_compatible(self): if not self.engines or "platformio" not in self.engines: return True core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) if self.CORE_SEMVER in core_spec: return True # PIO Core 6 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 if any( semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4, 5) ): return True raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) def get_dir(self): return os.path.dirname(self.manifest_path) def get_build_script(self): main_script = os.path.join(self.get_dir(), "builder", "main.py") if os.path.isfile(main_script): return main_script raise NotImplementedError() def is_embedded(self): for opts in self.packages.values(): if opts.get("type") == "uploader": return True return False def get_boards(self, id_=None): def _append_board(board_id, manifest_path): config = PlatformBoardConfig(manifest_path) if "platform" in config and config.get("platform") != self.name: return if "platforms" in config and self.name not in config.get("platforms"): return config.manifest["platform"] = self.name self._BOARDS_CACHE[board_id] = config bdirs = [ self.config.get("platformio", "boards_dir"), os.path.join(self.config.get("platformio", "core_dir"), "boards"), os.path.join(self.get_dir(), "boards"), ] if id_ is None: for boards_dir in bdirs: if not os.path.isdir(boards_dir): continue for item in sorted(os.listdir(boards_dir)): _id = item[:-5] if not item.endswith(".json") or _id in self._BOARDS_CACHE: continue _append_board(_id, os.path.join(boards_dir, item)) else: if id_ not in self._BOARDS_CACHE: for boards_dir in bdirs: if not os.path.isdir(boards_dir): continue manifest_path = os.path.join(boards_dir, "%s.json" % id_) if os.path.isfile(manifest_path): _append_board(id_, manifest_path) break if id_ not in self._BOARDS_CACHE: raise UnknownBoard(id_) return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE def board_config(self, id_): assert id_ return self.get_boards(id_) def get_package_type(self, name): return self.packages[name].get("type") def configure_project_packages(self, env, targets=None): options = self.config.items(env=env, as_dict=True) if "framework" in options: # support PIO Core 3.0 dev/platforms options["pioframework"] = options["framework"] # override user custom packages self._custom_packages = options.get("platform_packages") self.configure_default_packages(options, targets or []) def configure_default_packages(self, options, targets): # enable used frameworks for framework in options.get("framework", []): if not self.frameworks: continue framework = framework.lower().strip() if not framework or framework not in self.frameworks: continue _pkg_name = self.frameworks[framework].get("package") if _pkg_name: self.packages[_pkg_name]["optional"] = False # enable upload tools for upload targets if any(["upload" in t for t in targets] + ["program" in targets]): for name, opts in self.packages.items(): if opts.get("type") == "uploader": self.packages[name]["optional"] = False # skip all packages in "nobuild" mode # allow only upload tools and frameworks elif "nobuild" in targets and opts.get("type") != "framework": self.packages[name]["optional"] = True def configure_debug_session(self, debug_config): raise NotImplementedError def generate_sample_code(self, project_config, environment): raise NotImplementedError def on_installed(self): pass def on_uninstalled(self): pass def get_lib_storages(self): storages = {} for opts in (self.frameworks or {}).values(): if "package" not in opts: continue pkg = self.get_package(opts["package"]) if not pkg or not os.path.isdir(os.path.join(pkg.path, "libraries")): continue libs_dir = os.path.join(pkg.path, "libraries") storages[libs_dir] = opts["package"] libcores_dir = os.path.join(libs_dir, "__cores__") if not os.path.isdir(libcores_dir): continue for item in os.listdir(libcores_dir): libcore_dir = os.path.join(libcores_dir, item) if not os.path.isdir(libcore_dir): continue storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) return [dict(name=name, path=path) for path, name in storages.items()] ================================================ FILE: platformio/platform/board.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from platformio import fs, util from platformio.compat import MISSING from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError from platformio.exception import InvalidJSONFile, UserSideException from platformio.platform.exception import InvalidBoardManifest class PlatformBoardConfig: def __init__(self, manifest_path): self._id = os.path.basename(manifest_path)[:-5] assert os.path.isfile(manifest_path) self.manifest_path = manifest_path try: self._manifest = fs.load_json(manifest_path) except InvalidJSONFile as exc: raise InvalidBoardManifest(manifest_path) from exc if not set(["name", "url", "vendor"]) <= set(self._manifest): raise UserSideException( "Please specify name, url and vendor fields for " + manifest_path ) def get(self, path, default=MISSING): try: value = self._manifest for k in path.split("."): value = value[k] return value except KeyError: if default != MISSING: return default raise KeyError("Invalid board option '%s'" % path) def update(self, path, value): newdict = None for key in path.split(".")[::-1]: if newdict is None: newdict = {key: value} else: newdict = {key: newdict} util.merge_dicts(self._manifest, newdict) def __contains__(self, key): try: self.get(key) return True except KeyError: return False @property def id(self): return self._id @property def id_(self): return self.id @property def manifest(self): return self._manifest def get_brief_data(self): result = { "id": self.id, "name": self._manifest["name"], "platform": self._manifest.get("platform"), "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), "fcpu": int( "".join( [ c for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) if c.isdigit() ] ) ), "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), "rom": self._manifest.get("upload", {}).get("maximum_size", 0), "frameworks": self._manifest.get("frameworks"), "vendor": self._manifest["vendor"], "url": self._manifest["url"], } if self._manifest.get("connectivity"): result["connectivity"] = self._manifest.get("connectivity") debug = self.get_debug_data() if debug: result["debug"] = debug return result def get_debug_data(self): if not self._manifest.get("debug", {}).get("tools"): return None tools = {} for name, options in self._manifest["debug"]["tools"].items(): tools[name] = {} for key, value in options.items(): if key in ("default", "onboard") and value: tools[name][key] = value return {"tools": tools} def get_debug_tool_name(self, custom=None): debug_tools = self._manifest.get("debug", {}).get("tools") tool_name = custom if tool_name == "custom": return tool_name if not debug_tools: raise DebugSupportError(self._manifest["name"]) if tool_name: if tool_name in debug_tools: return tool_name raise DebugInvalidOptionsError( "Unknown debug tool `%s`. Please use one of `%s` or `custom`" % (tool_name, ", ".join(sorted(list(debug_tools)))) ) # automatically select best tool data = {"default": [], "onboard": [], "external": []} for key, value in debug_tools.items(): if value.get("default"): data["default"].append(key) elif value.get("onboard"): data["onboard"].append(key) data["external"].append(key) for key, value in data.items(): if not value: continue return sorted(value)[0] assert any(item for item in data) ================================================ FILE: platformio/platform/exception.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.exception import UserSideException class PlatformException(UserSideException): pass class UnknownPlatform(PlatformException): MESSAGE = "Unknown development platform '{0}'" class IncompatiblePlatform(PlatformException): MESSAGE = ( "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " "depends on PlatformIO Core {2}.\n" ) class UnknownBoard(PlatformException): MESSAGE = "Unknown board ID '{0}'" class InvalidBoardManifest(PlatformException): MESSAGE = "Invalid board JSON manifest '{0}'" class UnknownFramework(PlatformException): MESSAGE = "Unknown framework '{0}'" class BuildScriptNotFound(PlatformException): MESSAGE = "Invalid path '{0}' to build script" ================================================ FILE: platformio/platform/factory.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import sys from platformio import fs from platformio.compat import load_python_module from platformio.package.meta import PackageItem from platformio.platform import base from platformio.platform.exception import UnknownPlatform from platformio.project.config import ProjectConfig from platformio.project.exception import UndefinedEnvPlatformError class PlatformFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) return "%sPlatform" % name.lower().capitalize() @staticmethod def load_platform_module(name, path): # backward compatibility with the legacy dev-platforms sys.modules["platformio.managers.platform"] = base try: return load_python_module("platformio.platform.%s" % name, path) except ImportError as exc: raise UnknownPlatform(name) from exc @classmethod def new(cls, pkg_or_spec, autoinstall=False) -> base.PlatformBase: # pylint: disable=import-outside-toplevel from platformio.package.manager.platform import PlatformPackageManager platform_dir = None platform_name = None if isinstance(pkg_or_spec, PackageItem): platform_dir = pkg_or_spec.path platform_name = pkg_or_spec.metadata.name elif isinstance(pkg_or_spec, (str, bytes)) and os.path.isdir(pkg_or_spec): platform_dir = pkg_or_spec else: pkg = PlatformPackageManager().get_package(pkg_or_spec) if pkg: platform_dir = pkg.path platform_name = pkg.metadata.name if not platform_dir or not os.path.isfile( os.path.join(platform_dir, "platform.json") ): if autoinstall: return cls.new( PlatformPackageManager().install( pkg_or_spec, skip_dependencies=True ) ) raise UnknownPlatform(pkg_or_spec) if not platform_name: platform_name = fs.load_json(os.path.join(platform_dir, "platform.json"))[ "name" ] platform_cls = None if os.path.isfile(os.path.join(platform_dir, "platform.py")): platform_cls = getattr( cls.load_platform_module( platform_name, os.path.join(platform_dir, "platform.py") ), cls.get_clsname(platform_name), ) else: platform_cls = type( str(cls.get_clsname(platform_name)), (base.PlatformBase,), {} ) _instance = platform_cls(os.path.join(platform_dir, "platform.json")) assert isinstance(_instance, base.PlatformBase) return _instance @classmethod def from_env(cls, env, targets=None, autoinstall=False): config = ProjectConfig.get_instance() spec = config.get(f"env:{env}", "platform", None) if not spec: raise UndefinedEnvPlatformError(env) p = cls.new(spec, autoinstall=autoinstall) p.project_env = env p.configure_project_packages(env, targets) return p ================================================ FILE: platformio/proc.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import subprocess import sys from contextlib import contextmanager from threading import Thread from platformio import exception from platformio.compat import ( IS_WINDOWS, get_filesystem_encoding, get_locale_encoding, string_types, ) class AsyncPipeBase: def __init__(self): self._fd_read, self._fd_write = os.pipe() self._pipe_reader = os.fdopen( self._fd_read, encoding="utf-8", errors="backslashreplace" ) self._buffer = "" self._thread = Thread(target=self.run) self._thread.start() def get_buffer(self): return self._buffer def fileno(self): return self._fd_write def run(self): try: self.do_reading() except (KeyboardInterrupt, SystemExit, IOError): self.close() def do_reading(self): raise NotImplementedError() def close(self): self._buffer = "" os.close(self._fd_write) self._thread.join() class BuildAsyncPipe(AsyncPipeBase): def __init__(self, line_callback, data_callback): self.line_callback = line_callback self.data_callback = data_callback super().__init__() def do_reading(self): line = "" print_immediately = False for char in iter(lambda: self._pipe_reader.read(1), ""): # self._buffer += char if line and char.strip() and line[-3:] == (char * 3): print_immediately = True if print_immediately: # leftover bytes if line: self.data_callback(line) line = "" self.data_callback(char) if char == "\n": print_immediately = False else: line += char if char != "\n": continue self.line_callback(line) line = "" self._pipe_reader.close() class LineBufferedAsyncPipe(AsyncPipeBase): def __init__(self, line_callback): self.line_callback = line_callback super().__init__() def do_reading(self): for line in iter(self._pipe_reader.readline, ""): self._buffer += line self.line_callback(line) self._pipe_reader.close() def exec_command(*args, **kwargs): result = {"out": None, "err": None, "returncode": None} default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) default.update(kwargs) kwargs = default with subprocess.Popen(*args, **kwargs) as p: try: result["out"], result["err"] = p.communicate() result["returncode"] = p.returncode except KeyboardInterrupt as exc: raise exception.AbortedByUser() from exc finally: for s in ("stdout", "stderr"): if isinstance(kwargs[s], AsyncPipeBase): kwargs[s].close() # pylint: disable=no-member for s in ("stdout", "stderr"): if isinstance(kwargs[s], AsyncPipeBase): result[s[3:]] = kwargs[s].get_buffer() # pylint: disable=no-member for key, value in result.items(): if isinstance(value, bytes): try: result[key] = value.decode( get_locale_encoding() or get_filesystem_encoding() ) except UnicodeDecodeError: result[key] = value.decode("latin-1") if value and isinstance(value, string_types): result[key] = value.strip() return result @contextmanager def capture_std_streams(stdout, stderr=None): _stdout = sys.stdout _stderr = sys.stderr sys.stdout = stdout sys.stderr = stderr or stdout yield sys.stdout = _stdout sys.stderr = _stderr def is_ci(): return os.getenv("CI", "").lower() == "true" def is_container(): if os.path.exists("/.dockerenv"): return True if not os.path.isfile("/proc/1/cgroup"): return False with open("/proc/1/cgroup", encoding="utf8") as fp: return ":/docker/" in fp.read() def get_pythonexe_path(): return os.environ.get("PYTHONEXEPATH", os.path.normpath(sys.executable)) def copy_pythonpath_to_osenv(): _PYTHONPATH = [] if "PYTHONPATH" in os.environ: _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) for p in os.sys.path: conditions = [p not in _PYTHONPATH] if not IS_WINDOWS: conditions.append( os.path.isdir(os.path.join(p, "click")) or os.path.isdir(os.path.join(p, "platformio")) ) if all(conditions): _PYTHONPATH.append(p) os.environ["PYTHONPATH"] = os.pathsep.join(_PYTHONPATH) def where_is_program(program, envpath=None): env = os.environ.copy() if envpath: env["PATH"] = envpath # look up in $PATH for bin_dir in env.get("PATH", "").split(os.pathsep): if os.path.isfile(os.path.join(bin_dir, program)): return os.path.join(bin_dir, program) if IS_WINDOWS and os.path.isfile(os.path.join(bin_dir, "%s.exe" % program)): return os.path.join(bin_dir, "%s.exe" % program) # try OS's built-in commands try: result = exec_command(["where" if IS_WINDOWS else "which", program], env=env) if result["returncode"] == 0 and os.path.isfile(result["out"].strip()): return result["out"].strip() except OSError: pass return program def append_env_path(name, value): cur_value = os.environ.get(name) or "" if cur_value and value in cur_value.split(os.pathsep): return cur_value os.environ[name] = os.pathsep.join([cur_value, value]) return os.environ[name] def force_exit(code=0): os._exit(code) # pylint: disable=protected-access ================================================ FILE: platformio/project/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/project/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.project.commands.config import project_config_cmd from platformio.project.commands.init import project_init_cmd from platformio.project.commands.metadata import project_metadata_cmd @click.group( "project", commands=[ project_config_cmd, project_init_cmd, project_metadata_cmd, ], short_help="Project Manager", ) def cli(): pass ================================================ FILE: platformio/project/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/project/commands/config.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import click from tabulate import tabulate from platformio import fs from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.project.helpers import is_platformio_project @click.command("config", short_help="Show computed configuration") @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("--lint", is_flag=True) @click.option("--json-output", is_flag=True) def project_config_cmd(project_dir, lint, json_output): if not is_platformio_project(project_dir): raise NotPlatformIOProjectError(project_dir) with fs.cd(project_dir): if lint: return lint_configuration(json_output) return print_configuration(json_output) def print_configuration(json_output=False): config = ProjectConfig.get_instance() if json_output: return click.echo(config.to_json()) click.echo( "Computed project configuration for %s" % click.style(os.getcwd(), fg="cyan") ) for section, options in config.as_tuple(): click.secho(section, fg="cyan") click.echo("-" * len(section)) click.echo( tabulate( [ (name, "=", "\n".join(value) if isinstance(value, list) else value) for name, value in options ], tablefmt="plain", ) ) click.echo() return None def lint_configuration(json_output=False): result = ProjectConfig.lint() errors = result["errors"] warnings = result["warnings"] if json_output: return click.echo(result) if not errors and not warnings: return click.secho( 'The "platformio.ini" configuration file is free from linting errors.', fg="green", ) if errors: click.echo( tabulate( [ ( click.style(error["type"], fg="red"), error["message"], ( error.get("source", "") + (f":{error.get('lineno')}") if "lineno" in error else "" ), ) for error in errors ], tablefmt="plain", ) ) if warnings: click.echo( tabulate( [ (click.style("Warning", fg="yellow"), warning) for warning in warnings ], tablefmt="plain", ) ) return None ================================================ FILE: platformio/project/commands/init.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=line-too-long,too-many-arguments,too-many-locals import json import os import click from platformio import fs from platformio.package.commands.install import install_project_dependencies from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import UndefinedEnvPlatformError from platformio.project.helpers import is_platformio_project from platformio.project.integration.generator import ProjectGenerator from platformio.project.options import ProjectOptions def validate_boards(ctx, param, value): # pylint: disable=unused-argument pm = PlatformPackageManager() for id_ in value: try: pm.board_config(id_) except UnknownBoard as exc: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " "command" % id_ ) from exc return value @click.command("init", short_help="Initialize a project or update existing") @click.option( "--project-dir", "-d", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-b", "--board", "boards", multiple=True, metavar="ID", callback=validate_boards ) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) @click.option("-e", "--environment", help="Update existing environment") @click.option( "-O", "--project-option", "project_options", multiple=True, help="A `name=value` pair", ) @click.option("--sample-code", is_flag=True) @click.option("--no-install-dependencies", is_flag=True) @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) def project_init_cmd( # pylint: disable=too-many-positional-arguments project_dir, boards, ide, environment, project_options, sample_code, no_install_dependencies, env_prefix, silent, ): project_dir = os.path.abspath(project_dir) is_new_project = not is_platformio_project(project_dir) if is_new_project: if not silent: print_header(project_dir) init_base_project(project_dir) with fs.cd(project_dir): if environment: update_project_env(environment, project_options) elif boards: update_board_envs(project_dir, boards, project_options, env_prefix) generator = None config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) if ide: config.validate() # init generator and pick the best env if user didn't specify generator = ProjectGenerator(config, environment, ide, boards) if not environment: environment = generator.env_name # resolve project dependencies if not no_install_dependencies and (environment or boards): install_project_dependencies( options=dict( project_dir=project_dir, environments=[environment] if environment else [], silent=silent, ) ) if environment and sample_code: init_sample_code(config, environment) if generator: if not silent: click.echo( "Updating metadata for the %s IDE..." % click.style(ide, fg="cyan") ) generator.generate() if is_new_project: init_cvs_ignore() if not silent: print_footer(is_new_project) def print_header(project_dir): click.echo("The following files/directories have been created in ", nl=False) try: click.secho(project_dir, fg="cyan") except UnicodeEncodeError: click.secho(json.dumps(project_dir), fg="cyan") click.echo("%s - Put project header files here" % click.style("include", fg="cyan")) click.echo( "%s - Put project specific (private) libraries here" % click.style("lib", fg="cyan") ) click.echo("%s - Put project source files here" % click.style("src", fg="cyan")) click.echo( "%s - Project Configuration File" % click.style("platformio.ini", fg="cyan") ) def print_footer(is_new_project): action = "initialized" if is_new_project else "updated" return click.secho( f"Project has been successfully {action}!", fg="green", ) def init_base_project(project_dir): with fs.cd(project_dir): config = ProjectConfig() config.save() dir_to_readme = [ (config.get("platformio", "src_dir"), None), (config.get("platformio", "include_dir"), init_include_readme), (config.get("platformio", "lib_dir"), init_lib_readme), (config.get("platformio", "test_dir"), init_test_readme), ] for path, cb in dir_to_readme: if os.path.isdir(path): continue os.makedirs(path) if cb: cb(path) def init_include_readme(include_dir): with open(os.path.join(include_dir, "README"), mode="w", encoding="utf8") as fp: fp.write( """ This directory is intended for project header files. A header file is a file containing C declarations and macro definitions to be shared between several project source files. You request the use of a header file in your project source file (C, C++, etc) located in `src` folder by including it, with the C preprocessing directive `#include'. ```src/main.c #include "header.h" int main (void) { ... } ``` Including a header file produces the same results as copying the header file into each source file that needs it. Such copying would be time-consuming and error-prone. With a header file, the related declarations appear in only one place. If they need to be changed, they can be changed in one place, and programs that include the header file will automatically use the new version when next recompiled. The header file eliminates the labor of finding and changing all the copies as well as the risk that a failure to find one copy will result in inconsistencies within a program. In C, the convention is to give header files names that end with `.h'. Read more about using header files in official GCC documentation: * Include Syntax * Include Operation * Once-Only Headers * Computed Includes https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html """, ) def init_lib_readme(lib_dir): with open(os.path.join(lib_dir, "README"), mode="w", encoding="utf8") as fp: fp.write( """ This directory is intended for project specific (private) libraries. PlatformIO will compile them to static libraries and link into the executable file. The source code of each library should be placed in a separate directory ("lib/your_library_name/[Code]"). For example, see the structure of the following example libraries `Foo` and `Bar`: |--lib | | | |--Bar | | |--docs | | |--examples | | |--src | | |- Bar.c | | |- Bar.h | | |- library.json (optional. for custom build options, etc) https://docs.platformio.org/page/librarymanager/config.html | | | |--Foo | | |- Foo.c | | |- Foo.h | | | |- README --> THIS FILE | |- platformio.ini |--src |- main.c Example contents of `src/main.c` using Foo and Bar: ``` #include #include int main (void) { ... } ``` The PlatformIO Library Dependency Finder will find automatically dependent libraries by scanning project source files. More information about PlatformIO Library Dependency Finder - https://docs.platformio.org/page/librarymanager/ldf.html """, ) def init_test_readme(test_dir): with open(os.path.join(test_dir, "README"), mode="w", encoding="utf8") as fp: fp.write( """ This directory is intended for PlatformIO Test Runner and project tests. Unit Testing is a software testing method by which individual units of source code, sets of one or more MCU program modules together with associated control data, usage procedures, and operating procedures, are tested to determine whether they are fit for use. Unit testing finds problems early in the development cycle. More information about PlatformIO Unit Testing: - https://docs.platformio.org/en/latest/advanced/unit-testing/index.html """, ) def init_cvs_ignore(): conf_path = ".gitignore" if os.path.isfile(conf_path): return with open(conf_path, mode="w", encoding="utf8") as fp: fp.write(".pio\n") def update_board_envs(project_dir, boards, extra_project_options, env_prefix): config = ProjectConfig( os.path.join(project_dir, "platformio.ini"), parse_extra=False ) used_boards = [] for section in config.sections(): cond = [section.startswith("env:"), config.has_option(section, "board")] if all(cond): used_boards.append(config.get(section, "board")) pm = PlatformPackageManager() modified = False for id_ in boards: board_config = pm.board_config(id_) if id_ in used_boards: continue used_boards.append(id_) modified = True envopts = {"platform": board_config["platform"], "board": id_} # find default framework for board frameworks = board_config.get("frameworks") if frameworks: envopts["framework"] = frameworks[0] for item in extra_project_options: if "=" not in item: continue _name, _value = item.split("=", 1) envopts[_name.strip()] = _value.strip() section = "env:%s%s" % (env_prefix, id_) config.add_section(section) for option, value in envopts.items(): config.set(section, option, value) if modified: config.save() def update_project_env(environment, extra_project_options=None): if not extra_project_options: return env_section = "env:%s" % environment option_to_sections = {"platformio": [], env_section: []} for item in extra_project_options: assert "=" in item name, value = item.split("=", 1) name = name.strip() destination = env_section for option in ProjectOptions.values(): if option.scope in option_to_sections and option.name == name: destination = option.scope break option_to_sections[destination].append((name, value.strip())) config = ProjectConfig( "platformio.ini", parse_extra=False, expand_interpolations=False ) for section, options in option_to_sections.items(): if not options: continue if not config.has_section(section): config.add_section(section) for name, value in options: config.set(section, name, value) config.save() def init_sample_code(config, environment): try: p = PlatformFactory.from_env(environment) return p.generate_sample_code(config, environment) except (NotImplementedError, UndefinedEnvPlatformError): pass framework = config.get(f"env:{environment}", "framework", None) if framework != ["arduino"]: return None main_content = """ #include // put function declarations here: int myFunction(int, int); void setup() { // put your setup code here, to run once: int result = myFunction(2, 3); } void loop() { // put your main code here, to run repeatedly: } // put function definitions here: int myFunction(int x, int y) { return x + y; } """ is_cpp_project = p.name not in ["intel_mcs51", "ststm8"] src_dir = config.get("platformio", "src_dir") main_path = os.path.join(src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")) if os.path.isfile(main_path): return None if not os.path.isdir(src_dir): os.makedirs(src_dir) with open(main_path, mode="w", encoding="utf8") as fp: fp.write(main_content.strip()) return True ================================================ FILE: platformio/project/commands/metadata.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import click from tabulate import tabulate from platformio import fs from platformio.package.commands.install import install_project_dependencies from platformio.project.config import ProjectConfig from platformio.project.helpers import load_build_metadata @click.command( "metadata", short_help="Dump metadata intended for IDE extensions/plugins" ) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("--json-output", is_flag=True) @click.option("--json-output-path", type=click.Path()) def project_metadata_cmd(project_dir, environments, json_output, json_output_path): project_dir = os.path.abspath(project_dir) with fs.cd(project_dir): config = ProjectConfig.get_instance() config.validate(environments) environments = list(environments or config.envs()) build_metadata = load_build_metadata(project_dir, environments) if not json_output: install_project_dependencies( options=dict( project_dir=project_dir, environments=environments, ) ) click.echo() if json_output or json_output_path: if json_output_path: if os.path.isdir(json_output_path): json_output_path = os.path.join(json_output_path, "metadata.json") with open(json_output_path, mode="w", encoding="utf8") as fp: json.dump(build_metadata, fp) click.secho(f"Saved metadata to the {json_output_path}", fg="green") if json_output: click.echo(json.dumps(build_metadata)) return for envname, metadata in build_metadata.items(): click.echo("Environment: " + click.style(envname, fg="cyan", bold=True)) click.echo("=" * (13 + len(envname))) click.echo( tabulate( [ (click.style(name, bold=True), "=", json.dumps(value, indent=2)) for name, value in metadata.items() ], tablefmt="plain", ) ) click.echo() return ================================================ FILE: platformio/project/config.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import configparser import glob import hashlib import json import os import re import time import click from platformio import fs from platformio.compat import MISSING, hashlib_encode_data, string_types from platformio.project import exception from platformio.project.options import ProjectOptions CONFIG_HEADER = """ ; PlatformIO Project Configuration File ; ; Build options: build flags, source filter ; Upload options: custom upload port, speed and extra flags ; Library options: dependencies, extra library storages ; Advanced options: extra scripting ; ; Please visit documentation for the other options and examples ; https://docs.platformio.org/page/projectconf.html """ class ProjectConfigBase: ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I) INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{(?:([^\.\}\()]+)\.)?([^\}]+)\}") BUILTIN_VARS = { "PROJECT_DIR": lambda: os.getcwd(), # pylint: disable=unnecessary-lambda "PROJECT_HASH": lambda: "%s-%s" % ( os.path.basename(os.getcwd()), hashlib.sha1(hashlib_encode_data(os.getcwd())).hexdigest()[:10], ), "UNIX_TIME": lambda: str(int(time.time())), } CUSTOM_OPTION_PREFIXES = ("custom_", "board_") expand_interpolations = True warnings = [] _parser = None _parsed = [] @staticmethod def parse_multi_values(items): result = [] if not items: return result if not isinstance(items, (list, tuple)): items = items.split("\n" if "\n" in items else ", ") for item in items: item = item.strip() # comment if not item or item.startswith((";", "#")): continue if ";" in item: item = ProjectConfigBase.INLINE_COMMENT_RE.sub("", item).strip() result.append(item) return result @staticmethod def get_default_path(): from platformio import app # pylint: disable=import-outside-toplevel return app.get_session_var("custom_project_conf") or os.path.join( os.getcwd(), "platformio.ini" ) def __init__(self, path=None, parse_extra=True, expand_interpolations=True): path = self.get_default_path() if path is None else path self.path = path self.expand_interpolations = expand_interpolations self.warnings = [] self._parsed = [] self._parser = configparser.ConfigParser(inline_comment_prefixes=("#", ";")) if path and os.path.isfile(path): self.read(path, parse_extra) self._maintain_renamed_options() def __getattr__(self, name): return getattr(self._parser, name) def read(self, path, parse_extra=True): if path in self._parsed: return self._parsed.append(path) try: self._parser.read(path, "utf-8") except configparser.Error as exc: raise exception.InvalidProjectConfError(path, str(exc)) from exc if not parse_extra: return # load extra configs for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): pattern = fs.expanduser(pattern) for item in glob.glob(pattern, recursive=True): self.read(item) def _maintain_renamed_options(self): renamed_options = {} for option in ProjectOptions.values(): if option.oldnames: renamed_options.update({name: option.name for name in option.oldnames}) for section in self._parser.sections(): scope = self.get_section_scope(section) if scope not in ("platformio", "env"): continue for option in self._parser.options(section): if option in renamed_options: self.warnings.append( "`%s` configuration option in section [%s] is " "deprecated and will be removed in the next release! " "Please use `%s` instead" % (option, section, renamed_options[option]) ) # # rename on-the-fly # self._parser.set( # section, # renamed_options[option], # self._parser.get(section, option), # ) # self._parser.remove_option(section, option) continue # unknown unknown_conditions = [ ("%s.%s" % (scope, option)) not in ProjectOptions, scope != "env" or not option.startswith(self.CUSTOM_OPTION_PREFIXES), ] if all(unknown_conditions): self.warnings.append( "Ignore unknown configuration option `%s` " "in section [%s]" % (option, section) ) return True @staticmethod def get_section_scope(section): assert section return section.split(":", 1)[0] if ":" in section else section def walk_options(self, root_section): extends_queue = ( ["env", root_section] if root_section.startswith("env:") else [root_section] ) extends_done = [] while extends_queue: section = extends_queue.pop() extends_done.append(section) if not self._parser.has_section(section): continue for option in self._parser.options(section): yield (section, option) if self._parser.has_option(section, "extends"): extends_queue.extend( self.parse_multi_values(self._parser.get(section, "extends")) ) def options(self, section=None, env=None): result = [] assert section or env if not section: section = "env:" + env if not self.expand_interpolations: return self._parser.options(section) for _, option in self.walk_options(section): if option not in result: result.append(option) # handle system environment variables scope = self.get_section_scope(section) for option_meta in ProjectOptions.values(): if option_meta.scope != scope or option_meta.name in result: continue if option_meta.sysenvvar and option_meta.sysenvvar in os.environ: result.append(option_meta.name) return result def has_option(self, section, option): if self._parser.has_option(section, option): return True return option in self.options(section) def items(self, section=None, env=None, as_dict=False): assert section or env if not section: section = "env:" + env if as_dict: return { option: self.get(section, option) for option in self.options(section) } return [(option, self.get(section, option)) for option in self.options(section)] def set(self, section, option, value): if value is None: value = "" if isinstance(value, (list, tuple)): value = "\n".join(value) elif isinstance(value, bool): value = "yes" if value else "no" elif isinstance(value, (int, float)): value = str(value) # start multi-line value from a new line if "\n" in value and not value.startswith("\n"): value = "\n" + value self._parser.set(section, option, value) def resolve_renamed_option(self, section, old_name): scope = self.get_section_scope(section) if scope not in ("platformio", "env"): return None for option_meta in ProjectOptions.values(): if ( option_meta.oldnames and option_meta.scope == scope and old_name in option_meta.oldnames ): return option_meta.name return None def find_option_meta(self, section, option): scope = self.get_section_scope(section) if scope not in ("platformio", "env"): return None option_meta = ProjectOptions.get("%s.%s" % (scope, option)) if option_meta: return option_meta for option_meta in ProjectOptions.values(): if option_meta.scope == scope and option in (option_meta.oldnames or []): return option_meta return None def _traverse_for_value(self, section, option, option_meta=None): for _section, _option in self.walk_options(section): if _option == option or ( option_meta and ( option_meta.name == _option or _option in (option_meta.oldnames or []) ) ): return self._parser.get(_section, _option) return MISSING def getraw( self, section, option, default=MISSING ): # pylint: disable=too-many-branches if not self.expand_interpolations: return self._parser.get(section, option) option_meta = self.find_option_meta(section, option) value = self._traverse_for_value(section, option, option_meta) if not option_meta: if value == MISSING: value = ( default if default != MISSING else self._parser.get(section, option) ) return self._expand_interpolations(section, option, value) if option_meta.sysenvvar: envvar_value = os.getenv(option_meta.sysenvvar) if not envvar_value and option_meta.oldnames: for oldoption in option_meta.oldnames: envvar_value = os.getenv("PLATFORMIO_" + oldoption.upper()) if envvar_value: break if envvar_value and option_meta.multiple: if value == MISSING: value = "" value += ("\n" if value else "") + envvar_value elif envvar_value: value = envvar_value if value == MISSING: value = default if default != MISSING else option_meta.default if callable(value): value = value() if value == MISSING: return None return self._expand_interpolations(section, option, value) def _expand_interpolations(self, section, option, value): if not value or not isinstance(value, string_types) or not "$" in value: return value # legacy support for variables delclared without "${}" legacy_vars = ["PROJECT_HASH"] stop = False while not stop: stop = True for name in legacy_vars: x = value.find(f"${name}") if x < 0 or value[x - 1] == "$": continue value = "%s${%s}%s" % (value[:x], name, value[x + len(name) + 1 :]) stop = False warn_msg = ( "Invalid variable declaration. Please use " f"`${{{name}}}` instead of `${name}`" ) if warn_msg not in self.warnings: self.warnings.append(warn_msg) if not all(["${" in value, "}" in value]): return value return self.VARTPL_RE.sub( lambda match: self._re_interpolation_handler(section, option, match), value ) def _re_interpolation_handler(self, parent_section, parent_option, match): section, option = match.group(1), match.group(2) # handle built-in variables if section is None: if option in self.BUILTIN_VARS: return self.BUILTIN_VARS[option]() # SCons variables return f"${{{option}}}" # handle system environment variables if section == "sysenv": return os.getenv(option) # handle ${this.*} if section == "this": section = parent_section if option == "__env__": if not parent_section.startswith("env:"): raise exception.ProjectOptionValueError( f"`${{this.__env__}}` is called from the `{parent_section}` " "section that is not valid PlatformIO environment. Please " f"check `{parent_option}` option in the `{section}` section" ) return parent_section[4:] # handle nested calls try: value = self.get(section, option) except RecursionError as exc: raise exception.ProjectOptionValueError( f"Infinite recursion has been detected for `{option}` " f"option in the `{section}` section" ) from exc if isinstance(value, list): return "\n".join(value) return str(value) def get(self, section, option, default=MISSING): value = None try: value = self.getraw(section, option, default) except configparser.Error as exc: raise exception.InvalidProjectConfError(self.path, str(exc)) option_meta = self.find_option_meta(section, option) if not option_meta: return value if option_meta.validate: value = option_meta.validate(value) if option_meta.multiple: value = self.parse_multi_values(value or []) try: return self.cast_to(value, option_meta.type) except click.BadParameter as exc: if not self.expand_interpolations: return value raise exception.ProjectOptionValueError( "%s for `%s` option in the `%s` section (%s)" % (exc.format_message(), option, section, option_meta.description) ) @staticmethod def cast_to(value, to_type): items = value if not isinstance(value, (list, tuple)): items = [value] items = [ to_type(item) if isinstance(to_type, click.ParamType) else item for item in items ] return items if isinstance(value, (list, tuple)) else items[0] def envs(self): return [s[4:] for s in self._parser.sections() if s.startswith("env:")] def default_envs(self): return self.get("platformio", "default_envs", []) def get_default_env(self): default_envs = self.default_envs() if default_envs: return default_envs[0] envs = self.envs() return envs[0] if envs else None def validate(self, envs=None, silent=False): if not os.path.isfile(self.path): raise exception.NotPlatformIOProjectError(os.path.dirname(self.path)) known_envs = set(self.envs()) # check envs if not known_envs: raise exception.ProjectEnvsNotAvailableError() unknown_envs = set(list(envs or []) + self.default_envs()) - known_envs if unknown_envs: raise exception.UnknownEnvNamesError( ", ".join(unknown_envs), ", ".join(known_envs) ) for env in known_envs: # check envs names if not self.ENVNAME_RE.match(env): raise exception.InvalidEnvNameError(env) # check simultaneous use of `monitor_raw` and `monitor_filters` if self.get(f"env:{env}", "monitor_raw", False) and self.get( f"env:{env}", "monitor_filters", None ): self.warnings.append( "The `monitor_raw` and `monitor_filters` options cannot be " f"used simultaneously for the `{env}` environment in the " "`platformio.ini` file. The `monitor_filters` option will " "be disabled to avoid conflicts." ) if not silent: for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") return True class ProjectConfigLintMixin: @classmethod def lint(cls, path=None): errors = [] warnings = [] try: config = cls.get_instance(path) config.validate(silent=True) warnings = config.warnings # in case "as_tuple" fails config.as_tuple() warnings = config.warnings except Exception as exc: # pylint: disable=broad-exception-caught if exc.__cause__ is not None: exc = exc.__cause__ item = {"type": exc.__class__.__name__, "message": str(exc)} for attr in ("lineno", "source"): if hasattr(exc, attr): item[attr] = getattr(exc, attr) if item["type"] == "ParsingError" and hasattr(exc, "errors"): for lineno, line in getattr(exc, "errors"): errors.append( { "type": item["type"], "message": f"Parsing error: {line}", "lineno": lineno, "source": item["source"], } ) else: errors.append(item) return {"errors": errors, "warnings": warnings} class ProjectConfigDirsMixin: def get_optional_dir(self, name): """ Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0 PIO Home 3.0 Project Inspection depends on it """ return self.get("platformio", f"{name}_dir") class ProjectConfig(ProjectConfigBase, ProjectConfigLintMixin, ProjectConfigDirsMixin): _instances = {} @staticmethod def get_instance(path=None): path = ProjectConfig.get_default_path() if path is None else path mtime = os.path.getmtime(path) if os.path.isfile(path) else 0 instance = ProjectConfig._instances.get(path) if instance and instance["mtime"] != mtime: instance = None if not instance: instance = {"mtime": mtime, "config": ProjectConfig(path)} ProjectConfig._instances[path] = instance return instance["config"] def __repr__(self): return "" % (self.path or "in-memory") def as_tuple(self): return [(s, self.items(s)) for s in self.sections()] def to_json(self): return json.dumps(self.as_tuple()) def update(self, data, clear=False): assert isinstance(data, list) if clear: self._parser = configparser.ConfigParser() for section, options in data: if not self._parser.has_section(section): self._parser.add_section(section) for option, value in options: self.set(section, option, value) def save(self, path=None): path = path or self.path if path in self._instances: del self._instances[path] with open(path or self.path, mode="w+", encoding="utf8") as fp: fp.write(CONFIG_HEADER.strip() + "\n\n") self._parser.write(fp) fp.seek(0) contents = fp.read() fp.seek(0) fp.truncate() fp.write(contents.strip() + "\n") return True ================================================ FILE: platformio/project/exception.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.exception import PlatformioException, UserSideException class ProjectError(PlatformioException): pass class NotPlatformIOProjectError(ProjectError, UserSideException): MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " "please use `platformio project init` command" ) class InvalidProjectConfError(ProjectError, UserSideException): MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" class UndefinedEnvPlatformError(ProjectError, UserSideException): MESSAGE = "Please specify platform for '{0}' environment" class ProjectEnvsNotAvailableError(ProjectError, UserSideException): MESSAGE = "Please setup environments in `platformio.ini` file" class UnknownEnvNamesError(ProjectError, UserSideException): MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" class InvalidEnvNameError(ProjectError, UserSideException): MESSAGE = ( "Invalid environment name '{0}'. The name can contain " "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)" ) class ProjectOptionValueError(ProjectError, UserSideException): pass ================================================ FILE: platformio/project/helpers.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import subprocess from hashlib import sha1 from click.testing import CliRunner from platformio import __version__, exception, fs from platformio.compat import IS_MACOS, IS_WINDOWS, hashlib_encode_data from platformio.project.config import ProjectConfig def get_project_dir(): return os.getcwd() def is_platformio_project(project_dir=None): if not project_dir: project_dir = get_project_dir() return os.path.isfile(os.path.join(project_dir, "platformio.ini")) def find_project_dir_above(path): if os.path.isfile(path): path = os.path.dirname(path) if is_platformio_project(path): return path if os.path.isdir(os.path.dirname(path)): return find_project_dir_above(os.path.dirname(path)) return None def get_project_watch_lib_dirs(): """Used by platformio-node-helpers.project.observer.fetchLibDirs""" config = ProjectConfig.get_instance() result = [ config.get("platformio", "globallib_dir"), config.get("platformio", "lib_dir"), ] libdeps_dir = config.get("platformio", "libdeps_dir") if not os.path.isdir(libdeps_dir): return result for d in os.listdir(libdeps_dir): if os.path.isdir(os.path.join(libdeps_dir, d)): result.append(os.path.join(libdeps_dir, d)) return result get_project_all_lib_dirs = get_project_watch_lib_dirs def get_project_cache_dir(): """Deprecated, use ProjectConfig.get("platformio", "cache_dir") instead""" return ProjectConfig.get_instance().get("platformio", "cache_dir") def get_default_projects_dir(): docs_dir = os.path.join(fs.expanduser("~"), "Documents") try: assert IS_WINDOWS import ctypes.wintypes # pylint: disable=import-outside-toplevel buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) docs_dir = buf.value except: # pylint: disable=bare-except if not IS_MACOS: try: docs_dir = ( subprocess.check_output(["xdg-user-dir", "DOCUMENTS"]) .decode("utf-8") .strip() ) except FileNotFoundError: # command not found pass return os.path.join(docs_dir, "PlatformIO", "Projects") def compute_project_checksum(config): # rebuild when PIO Core version changes checksum = sha1(hashlib_encode_data(__version__)) # configuration file state config_data = config.to_json() if IS_WINDOWS: # issue #4600: fix drive letter config_data = re.sub( r"([A-Z]):\\", lambda match: "%s:\\" % match.group(1).lower(), config_data, flags=re.I, ) checksum.update(hashlib_encode_data(config_data)) # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") for d in ( config.get("platformio", "include_dir"), config.get("platformio", "src_dir"), config.get("platformio", "lib_dir"), ): if not os.path.isdir(d): continue chunks = [] for root, _, files in os.walk(d): for f in files: path = os.path.join(root, f) if path.endswith(check_suffixes): chunks.append(path) if not chunks: continue chunks_to_str = ",".join(sorted(chunks)) if IS_WINDOWS: # case insensitive OS chunks_to_str = chunks_to_str.lower() checksum.update(hashlib_encode_data(chunks_to_str)) return checksum.hexdigest() def load_build_metadata(project_dir, env_or_envs, cache=False, build_type=None): assert env_or_envs env_names = env_or_envs if not isinstance(env_names, list): env_names = [env_names] with fs.cd(project_dir): result = _get_cached_build_metadata(env_names) if cache else {} # incompatible build-type data for env_name in list(result.keys()): if build_type is None: build_type = ProjectConfig.get_instance().get( f"env:{env_name}", "build_type" ) if result[env_name].get("build_type", "") != build_type: del result[env_name] missed_env_names = set(env_names) - set(result.keys()) if missed_env_names: result.update( _load_build_metadata(project_dir, missed_env_names, build_type) ) if not isinstance(env_or_envs, list) and env_or_envs in result: return result[env_or_envs] return result or None # Backward compatibility with dev-platforms load_project_ide_data = load_build_metadata def _load_build_metadata(project_dir, env_names, build_type=None): # pylint: disable=import-outside-toplevel from platformio import app from platformio.run.cli import cli as cmd_run args = ["--project-dir", project_dir, "--target", "__idedata"] if build_type == "debug": args.extend(["--target", "__debug"]) # if build_type == "test": # args.extend(["--target", "__test"]) for name in env_names: args.extend(["-e", name]) app.set_session_var("pause_telemetry", True) result = CliRunner().invoke(cmd_run, args) app.set_session_var("pause_telemetry", False) if result.exit_code != 0 and not isinstance( result.exception, exception.ReturnErrorCode ): raise result.exception if '"includes":' not in result.output: raise exception.UserSideException(result.output) return _get_cached_build_metadata(env_names) def _get_cached_build_metadata(env_names): build_dir = ProjectConfig.get_instance().get("platformio", "build_dir") result = {} for env_name in env_names: if not os.path.isfile(os.path.join(build_dir, env_name, "idedata.json")): continue result[env_name] = fs.load_json( os.path.join(build_dir, env_name, "idedata.json") ) return result ================================================ FILE: platformio/project/integration/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/project/integration/generator.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import bottle from platformio import fs, util from platformio.debug.helpers import get_default_debug_env from platformio.proc import where_is_program from platformio.project.helpers import load_build_metadata class ProjectGenerator: def __init__(self, config, env_name, ide, boards=None): self.config = config self.project_dir = os.path.dirname(config.path) self.forced_env_name = env_name self.env_name = str(env_name or self.get_best_envname(boards)) self.ide = str(ide) def get_best_envname(self, boards=None): envname = None default_envs = self.config.default_envs() if default_envs: envname = default_envs[0] if not boards: return envname for env in self.config.envs(): if not boards: return env if not envname: envname = env items = self.config.items(env=env, as_dict=True) if "board" in items and items.get("board") in boards: return env return envname @staticmethod def get_ide_tpls_dir(): return os.path.join(os.path.dirname(__file__), "tpls") @classmethod def get_supported_ides(cls): tpls_dir = cls.get_ide_tpls_dir() return sorted( [ name for name in os.listdir(tpls_dir) if os.path.isdir(os.path.join(tpls_dir, name)) ] ) @staticmethod def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True): ignore_scopes = ignore_scopes or [] result = [] for scope, includes in includes_map.items(): if scope in ignore_scopes: continue for include in includes: if to_unix_path: include = fs.to_unix_path(include) if include not in result: result.append(include) return result def _load_tplvars(self): tpl_vars = { "config": self.config, "systype": util.get_systype(), "project_name": self.config.get( "platformio", "name", os.path.basename(self.project_dir) ), "project_dir": self.project_dir, "forced_env_name": self.forced_env_name, "default_debug_env_name": get_default_debug_env(self.config), "env_name": self.env_name, "user_home_dir": os.path.abspath(fs.expanduser("~")), "platformio_path": ( sys.argv[0] if os.path.isfile(sys.argv[0]) else where_is_program("platformio") ), "env_path": os.getenv("PATH"), "env_pathsep": os.pathsep, } # default env configuration tpl_vars.update(self.config.items(env=self.env_name, as_dict=True)) # build data tpl_vars.update(load_build_metadata(self.project_dir, self.env_name) or {}) with fs.cd(self.project_dir): tpl_vars.update( { "src_files": self.get_src_files(), "project_src_dir": self.config.get("platformio", "src_dir"), "project_lib_dir": self.config.get("platformio", "lib_dir"), "project_test_dir": self.config.get("platformio", "test_dir"), "project_libdeps_dir": os.path.join( self.config.get("platformio", "libdeps_dir"), self.env_name ), } ) for key, value in tpl_vars.items(): if key.endswith(("_path", "_dir")): tpl_vars[key] = fs.to_unix_path(value) for key in ("src_files", "libsource_dirs"): if key not in tpl_vars: continue tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]] tpl_vars["to_unix_path"] = fs.to_unix_path tpl_vars["filter_includes"] = self.filter_includes return tpl_vars def get_src_files(self): result = [] with fs.cd(self.project_dir): for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append( os.path.relpath(os.path.join(os.path.abspath(root), f)) ) return result def get_tpls(self): tpls = [] ide_tpls_dir = os.path.join(self.get_ide_tpls_dir(), self.ide) for root, _, files in os.walk(ide_tpls_dir): for f in files: if not f.endswith(".tpl"): continue _relpath = root.replace(ide_tpls_dir, "") if _relpath.startswith(os.sep): _relpath = _relpath[1:] tpls.append((_relpath, os.path.join(root, f))) return tpls def generate(self): tpl_vars = self._load_tplvars() for tpl_relpath, tpl_path in self.get_tpls(): dst_dir = self.project_dir if tpl_relpath: dst_dir = os.path.join(self.project_dir, tpl_relpath) if not os.path.isdir(dst_dir): os.makedirs(dst_dir) file_name = os.path.basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path, tpl_vars) self._merge_contents(os.path.join(dst_dir, file_name), contents) @staticmethod def _render_tpl(tpl_path, tpl_vars): with open(tpl_path, "r", encoding="utf8") as fp: return bottle.template(fp.read(), **tpl_vars) @staticmethod def _merge_contents(dst_path, contents): if os.path.basename(dst_path) == ".gitignore" and os.path.isfile(dst_path): return with open(dst_path, "w", encoding="utf8") as fp: fp.write(contents) ================================================ FILE: platformio/project/integration/tpls/clion/.gitignore.tpl ================================================ .pio ================================================ FILE: platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl ================================================ ================================================ FILE: platformio/project/integration/tpls/eclipse/.cproject.tpl ================================================ platformio -f -c eclipse run --target program true false false platformio -f -c eclipse run --target uploadfs true false false platformio -f -c eclipse run true false false platformio -f -c eclipse run --verbose true false false platformio -f -c eclipse run --target upload true false false platformio -f -c eclipse run --target upload --verbose true false false platformio -f -c eclipse run --target clean true false false platformio -f -c eclipse test true false false platformio -f -c eclipse remote run --target upload true false false platformio -f -c eclipse init --ide eclipse true false false platformio -f -c eclipse device list true false false platformio -f -c eclipse lib update true false false platformio -f -c eclipse update true false false platformio -f -c eclipse upgrade true false false ================================================ FILE: platformio/project/integration/tpls/eclipse/.project.tpl ================================================ {{project_name}} org.eclipse.cdt.managedbuilder.core.genmakebuilder clean,full,incremental, org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder full,incremental, org.eclipse.cdt.core.cnature org.eclipse.cdt.core.ccnature org.eclipse.cdt.managedbuilder.core.managedBuildNature org.eclipse.cdt.managedbuilder.core.ScannerConfigNature ================================================ FILE: platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl ================================================ ================================================ FILE: platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl ================================================ % cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")] % cxx_std = cxx_stds[-1] if cxx_stds else "" % % if cxx_path.startswith(user_home_dir): % if "windows" in systype: % cxx_path = "${USERPROFILE}" + cxx_path.replace(user_home_dir, "") % else: % cxx_path = "${HOME}" + cxx_path.replace(user_home_dir, "") % end % end % ================================================ FILE: platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl ================================================ eclipse.preferences.version=1 environment/project/0.910961921/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} environment/project/0.910961921/PATH/operation=replace environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} environment/project/0.910961921/append=true environment/project/0.910961921/appendContributed=true environment/project/0.910961921.1363900502/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} environment/project/0.910961921.1363900502/PATH/operation=replace environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} environment/project/0.910961921.1363900502/append=true environment/project/0.910961921.1363900502/appendContributed=true ================================================ FILE: platformio/project/integration/tpls/emacs/.ccls.tpl ================================================ % from platformio.compat import shlex_join % {{ cc_path }} {{"%c"}} {{ shlex_join(cc_flags) }} {{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} % end % for define in defines: -D{{ !define }} % end ================================================ FILE: platformio/project/integration/tpls/emacs/.gitignore.tpl ================================================ .pio .clang_complete .ccls ================================================ FILE: platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl ================================================ platformio.ini nbproject/private/launcher.properties ^(nbproject|.pio)$ . default false false true true . "{{platformio_path}}" -f -c netbeans run "{{platformio_path}}" -f -c netbeans run --target clean % cleaned_includes = filter_includes(includes) src % for include in cleaned_includes: {{include}} % end % for define in defines: {{define}} % end src % for include in cleaned_includes: {{include}} % end % for define in defines: {{define}} % end . ================================================ FILE: platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl ================================================ platformio.ini localhost {{4 if "darwin" in systype else 2 if "linux" in systype else 3}} . ${AUTO_FOLDER} ${AUTO_FOLDER} ${MAKE} ${ITEM_NAME}.o ${AUTO_COMPILE} ${AUTO_COMPILE} gdb "${OUTPUT_PATH}" {{platformio_path}} -f -c netbeans run --target upload {{platformio_path}} -f -c netbeans run --target upload . false 0 0 ================================================ FILE: platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl ================================================ # Launchers File syntax: # # [Must-have property line] # launcher1.runCommand= # [Optional extra properties] # launcher1.displayName= # launcher1.buildCommand= # launcher1.runDir= # launcher1.symbolFiles= # launcher1.env.= # (If this value is quoted with ` it is handled as a native command which execution result will become the value) # [Common launcher properties] # common.runDir= # (This value is overwritten by a launcher specific runDir value if the latter exists) # common.env.= # (Environment variables from common launcher are merged with launcher specific variables) # common.symbolFiles= # (This value is overwritten by a launcher specific symbolFiles value if the latter exists) # # In runDir, symbolFiles and env fields you can use these macros: # ${PROJECT_DIR} - project directory absolute path # ${OUTPUT_PATH} - linker output path (relative to project directory path) # ${OUTPUT_BASENAME}- linker output filename # ${TESTDIR} - test files directory (relative to project directory path) # ${OBJECTDIR} - object files directory (relative to project directory path) # ${CND_DISTDIR} - distribution directory (relative to project directory path) # ${CND_BUILDDIR} - build directory (relative to project directory path) # ${CND_PLATFORM} - platform name # ${CND_CONF} - configuration name # ${CND_DLIB_EXT} - dynamic library extension # # All the project launchers must be listed in the file! # # launcher1.runCommand=... # launcher2.runCommand=... # ... # common.runDir=... # common.env.KEY=VALUE # launcher1.runCommand= ================================================ FILE: platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl ================================================ true 0 0 ================================================ FILE: platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl ================================================ org.netbeans.modules.cnd.makeproject {{project_name}} UTF-8 . Default 0 false ================================================ FILE: platformio/project/integration/tpls/qtcreator/.gitignore.tpl ================================================ .pio .qtc_clangd ================================================ FILE: platformio/project/integration/tpls/qtcreator/Makefile.tpl ================================================ all: platformio -c qtcreator run # regenerate project files to reflect platformio.ini changes project-update: @echo "This will overwrite project metadata files. Are you sure? [y/N] " \ && read ans && [ $${ans:-'N'} = 'y' ] platformio project init --ide qtcreator # forward any other target (clean, build, etc.) to pio run {{'%'}}: platformio -c qtcreator run --target $* ================================================ FILE: platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl ================================================ % from platformio.compat import shlex_join % {{shlex_join(cc_flags).replace('-mlongcalls', '-mlong-calls')}} ================================================ FILE: platformio/project/integration/tpls/qtcreator/platformio.config.tpl ================================================ % for define in defines: % tokens = define.split("=", 1) % if len(tokens) > 1: #define {{tokens[0].strip()}} {{!tokens[1].strip()}} % else: #define {{define}} % end % end ================================================ FILE: platformio/project/integration/tpls/qtcreator/platformio.creator.tpl ================================================ [General] ================================================ FILE: platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl ================================================ % from platformio.compat import shlex_join % {{shlex_join(cxx_flags).replace('-mlongcalls', '-mlong-calls')}} ================================================ FILE: platformio/project/integration/tpls/qtcreator/platformio.files.tpl ================================================ Makefile platformio.ini .gitignore % for file in src_files: {{file}} % end ================================================ FILE: platformio/project/integration/tpls/qtcreator/platformio.includes.tpl ================================================ ./ % for include in filter_includes(includes): {{include}} % end ================================================ FILE: platformio/project/integration/tpls/sublimetext/.ccls.tpl ================================================ % from platformio.compat import shlex_join % {{ cc_path }} {{"%c"}} {{ shlex_join(cc_flags) }} {{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} % end % for define in defines: -D{{ !define }} % end ================================================ FILE: platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl ================================================ { "build_systems": [ { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "run" ], "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", "name": "PlatformIO", "variants": [ { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "run" ], "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", "name": "Build" }, { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "run", "--target", "upload" ], "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", "name": "Upload" }, { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "run", "--target", "clean" ], "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", "name": "Clean" }, { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "test" ], "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", "name": "Test" }, { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "run", "--target", "uploadfs" ], "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", "name": "Upload SPIFFS image" }, { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "update" ], "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", "name": "Update platforms and libraries" }, { "cmd": [ "{{ platformio_path }}", "-c", "sublimetext", "upgrade" ], "name": "Upgrade PlatformIO Core" } ], "working_dir": "${project_path:${folder}}", "selector": "source.c, source.c++" } ], "folders": [ { "path": "." } ], "settings": { "sublimegdb_workingdir": "{{project_dir}}", "sublimegdb_exec_cmd": "", "sublimegdb_commandline": "{{ platformio_path }} -f -c sublimetext debug --interface=gdb --interpreter=mi -x .pioinit" } } ================================================ FILE: platformio/project/integration/tpls/vim/.ccls.tpl ================================================ % from platformio.compat import shlex_join % {{ cc_path }} {{"%c"}} {{ shlex_join(cc_flags) }} {{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} % end % for define in defines: -D{{ !define }} % end ================================================ FILE: platformio/project/integration/tpls/vim/.gitignore.tpl ================================================ .pio .clang_complete .gcc-flags.json .ccls ================================================ FILE: platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl ================================================ {4FC737F1-C7A5-4376-A066-2A32D752A2FF} cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx;ino;pde {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hh;hpp;hxx;hm;inl;inc;xsd % for file in src_files: % if any(file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")): Header Files % else: Source Files %end % end ================================================ FILE: platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl ================================================ {{env_path}} Debug Win32 Release Win32 {0FA9C3A8-452B-41EF-A418-9102B170F49F} MakeFileProj Makefile true v120 Makefile false v120 platformio -f -c visualstudio run platformio -f -c visualstudio run --target clean {{!";".join(defines)}} % cleaned_includes = filter_includes(includes) {{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in cleaned_includes])}} platformio run platformio run --target clean {{!";".join(defines)}} {{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in cleaned_includes])}} % for file in src_files: % if any(file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")): Header Files % else: Source Files %end % end ================================================ FILE: platformio/project/integration/tpls/vscode/.gitignore.tpl ================================================ .pio .vscode/.browse.c_cpp.db* .vscode/c_cpp_properties.json .vscode/launch.json .vscode/ipch ================================================ FILE: platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl ================================================ % import os % import platform % % systype = platform.system().lower() % % cpp_standards_remap = { % "c++0x": "c++11", % "c++1y": "c++14", % "c++1z": "c++17", % "c++2a": "c++20", % "c++2b": "c++23", % "gnu++0x": "gnu++11", % "gnu++1y": "gnu++14", % "gnu++1z": "gnu++17", % "gnu++2a": "gnu++20", % "gnu++2b": "gnu++23" % } % % def _escape(text): % return to_unix_path(text).replace('"', '\\"') % end % % def filter_args(args, allowed, ignore=None): % if not allowed: % return [] % end % % ignore = ignore or [] % result = [] % i = 0 % length = len(args) % while(i < length): % if any(args[i].startswith(f) for f in allowed) and not any( % args[i].startswith(f) for f in ignore): % result.append(args[i]) % if i + 1 < length and not args[i + 1].startswith("-"): % i += 1 % result.append(args[i]) % end % end % i += 1 % end % return result % end % % def _find_abs_path(inc, inc_paths): % for path in inc_paths: % if os.path.isfile(os.path.join(path, inc)): % return os.path.join(path, inc) % end % end % return inc % end % % def _find_forced_includes(flags, inc_paths): % result = [] % include_args = ("-include", "-imacros") % for f in filter_args(flags, include_args): % for arg in include_args: % inc = "" % if f.startswith(arg) and f.split(arg)[1].strip(): % inc = f.split(arg)[1].strip() % elif not f.startswith("-"): % inc = f % end % if inc: % result.append(_find_abs_path(inc, inc_paths)) % break % end % end % end % return result % end % % cleaned_includes = filter_includes(includes, ["toolchain"]) % % cc_stds = [arg[5:] for arg in cc_flags if arg.startswith("-std=")] % cxx_stds = [arg[5:] for arg in cxx_flags if arg.startswith("-std=")] % forced_includes = _find_forced_includes( % filter_args(cc_flags, ["-include", "-imacros"]), cleaned_includes) % // // !!! WARNING !!! AUTO-GENERATED FILE! // PLEASE DO NOT MODIFY IT AND USE "platformio.ini": // https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags // { "configurations": [ { "name": "PlatformIO", "includePath": [ % for include in cleaned_includes: "{{ include }}", % end "" ], "browse": { "limitSymbolsToIncludedHeaders": true, "path": [ % for include in cleaned_includes: "{{ include }}", % end "" ] }, "defines": [ % for define in defines: "{{! _escape(define) }}", % end "" ], % if cc_stds: "cStandard": "{{ cc_stds[-1] }}", % end % if cxx_stds: "cppStandard": "{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}", % end % if forced_includes: "forcedInclude": [ % for include in forced_includes: "{{ include }}", % end "" ], % end "compilerPath": "{{ cc_path }}", "compilerArgs": [ % for flag in [ % f for f in filter_args(cc_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) % ]: "{{ flag }}", % end "" ] } ], "version": 4 } ================================================ FILE: platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl ================================================ % import json % import os % import re % % recommendations = set(["platformio.platformio-ide"]) % unwantedRecommendations = set(["ms-vscode.cpptools-extension-pack"]) % previous_json = os.path.join(project_dir, ".vscode", "extensions.json") % if os.path.isfile(previous_json): % fp = open(previous_json) % contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() % fp.close() % if contents: % try: % data = json.loads(contents) % recommendations |= set(data.get("recommendations", [])) % unwantedRecommendations |= set(data.get("unwantedRecommendations", [])) % except ValueError: % pass % end % end % end { // See http://go.microsoft.com/fwlink/?LinkId=827846 // for the documentation about the extensions.json format "recommendations": [ % for i, item in enumerate(sorted(recommendations)): "{{ item }}"{{ ("," if (i + 1) < len(recommendations) else "") }} % end ], "unwantedRecommendations": [ % for i, item in enumerate(sorted(unwantedRecommendations)): "{{ item }}"{{ ("," if (i + 1) < len(unwantedRecommendations) else "") }} % end ] } ================================================ FILE: platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl ================================================ % import json % import os % % def _escape(text): % return text.replace('"', '\"') % end % % def _escape_path(path): % return path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"') % end % % def get_pio_configurations(): % predebug = { % "type": "platformio-debug", % "request": "launch", % "name": "PIO Debug (skip Pre-Debug)", % "executable": _escape_path(prog_path), % "projectEnvName": env_name if forced_env_name else default_debug_env_name, % "toolchainBinDir": _escape_path(os.path.dirname(cc_path)), % "internalConsoleOptions": "openOnSessionStart", % } % % if svd_path: % predebug["svdPath"] = _escape_path(svd_path) % end % debug = predebug.copy() % debug["name"] = "PIO Debug" % debug["preLaunchTask"] = { % "type": "PlatformIO", % "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and forced_env_name else "Pre-Debug", % } % noloading = predebug.copy() % noloading["name"] = "PIO Debug (without uploading)" % noloading["loadMode"] = "manual" % return [debug, predebug, noloading] % end % % def _remove_comments(lines): % data = "" % for line in lines: % line = line.strip() % if not line.startswith("//"): % data += line % end % end % return data % end % % def _contains_custom_configurations(launch_config): % pio_config_names = [ % c["name"] % for c in get_pio_configurations() % ] % return any( % c.get("type", "") != "platformio-debug" % or c.get("name", "") in pio_config_names % for c in launch_config.get("configurations", []) % ) % end % % def _remove_pio_configurations(launch_config): % if "configurations" not in launch_config: % return launch_config % end % % pio_config_names = [ % c["name"] % for c in get_pio_configurations() % ] % external_configurations = [ % c % for c in launch_config["configurations"] % if c.get("type", "") != "platformio-debug" or c.get("name", "") not in pio_config_names % ] % % launch_config["configurations"] = external_configurations % return launch_config % end % % def get_launch_configuration(): % launch_config = {"version": "0.2.0", "configurations": []} % launch_file = os.path.join(project_dir, ".vscode", "launch.json") % if os.path.isfile(launch_file): % with open(launch_file, "r", encoding="utf8") as fp: % launch_data = _remove_comments(fp.readlines()) % try: % prev_config = json.loads(launch_data) % if _contains_custom_configurations(prev_config): % launch_config = _remove_pio_configurations(prev_config) % end % except: % pass % end % end % end % launch_config["configurations"].extend(get_pio_configurations()) % return launch_config % end % // AUTOMATICALLY GENERATED FILE. PLEASE DO NOT MODIFY IT MANUALLY // // PlatformIO Debugging Solution // // Documentation: https://docs.platformio.org/en/latest/plus/debugging.html // Configuration: https://docs.platformio.org/en/latest/projectconf/sections/env/options/debug/index.html {{ json.dumps(get_launch_configuration(), indent=4, ensure_ascii=False) }} ================================================ FILE: platformio/project/options.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=redefined-builtin, too-many-arguments import os from collections import OrderedDict import click from platformio import fs from platformio.compat import IS_WINDOWS class ConfigOption: # pylint: disable=too-many-instance-attributes,too-many-positional-arguments def __init__( self, scope, group, name, description, type=str, multiple=False, sysenvvar=None, buildenvvar=None, oldnames=None, default=None, validate=None, ): self.scope = scope self.group = group self.name = name self.description = description self.type = type self.multiple = multiple self.sysenvvar = sysenvvar self.buildenvvar = buildenvvar self.oldnames = oldnames self.default = default self.validate = validate def as_dict(self): result = dict( scope=self.scope, group=self.group, name=self.name, description=self.description, type="string", multiple=self.multiple, sysenvvar=self.sysenvvar, default=self.default() if callable(self.default) else self.default, ) if isinstance(self.type, click.ParamType): result["type"] = self.type.name if isinstance(self.type, (click.IntRange, click.FloatRange)): result["min"] = self.type.min result["max"] = self.type.max if isinstance(self.type, click.Choice): result["choices"] = self.type.choices return result def ConfigPlatformioOption(*args, **kwargs): return ConfigOption("platformio", *args, **kwargs) def ConfigEnvOption(*args, **kwargs): return ConfigOption("env", *args, **kwargs) def validate_dir(path): if not path: return path # if not all values expanded, ignore validation if "${" in path and "}" in path: return path if path.startswith("~"): path = fs.expanduser(path) return os.path.abspath(path) def get_default_core_dir(): path = os.path.join(fs.expanduser("~"), ".platformio") if IS_WINDOWS: win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio" if os.path.isdir(win_core_dir): return win_core_dir return path ProjectOptions = OrderedDict( [ ("%s.%s" % (option.scope, option.name), option) for option in [ # # [platformio] # ConfigPlatformioOption( group="generic", name="name", description="A project name", default=lambda: os.path.basename(os.getcwd()), ), ConfigPlatformioOption( group="generic", name="description", description="Describe a project with a short information", ), ConfigPlatformioOption( group="generic", name="default_envs", description=( "Configure a list with environments which PlatformIO should " "process by default" ), oldnames=["env_default"], multiple=True, sysenvvar="PLATFORMIO_DEFAULT_ENVS", ), ConfigPlatformioOption( group="generic", name="extra_configs", description=( "Extend main configuration with the extra configuration files" ), multiple=True, ), # Dirs ConfigPlatformioOption( group="directory", name="core_dir", description=( "PlatformIO Core location where it keeps installed development " "platforms, packages, global libraries, " "and other internal information" ), oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", default=get_default_core_dir, validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="globallib_dir", description=( "A library folder/storage where PlatformIO Library Dependency " "Finder (LDF) looks for global libraries" ), sysenvvar="PLATFORMIO_GLOBALLIB_DIR", default=os.path.join("${platformio.core_dir}", "lib"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="platforms_dir", description=( "A location where PlatformIO Core keeps installed development " "platforms" ), sysenvvar="PLATFORMIO_PLATFORMS_DIR", default=os.path.join("${platformio.core_dir}", "platforms"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="packages_dir", description=( "A location where PlatformIO Core keeps installed packages" ), sysenvvar="PLATFORMIO_PACKAGES_DIR", default=os.path.join("${platformio.core_dir}", "packages"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="cache_dir", description=( "A location where PlatformIO Core stores caching information " "(requests to PlatformIO Registry, downloaded packages and " "other service information)" ), sysenvvar="PLATFORMIO_CACHE_DIR", default=os.path.join("${platformio.core_dir}", ".cache"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="build_cache_dir", description=( "A location where PlatformIO Core keeps derived files from a " "build system (objects, firmwares, ELFs) and caches them between " "build environments" ), sysenvvar="PLATFORMIO_BUILD_CACHE_DIR", validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="workspace_dir", description=( "A path to a project workspace directory where PlatformIO keeps " "by default compiled objects, static libraries, firmwares, and " "external library dependencies" ), sysenvvar="PLATFORMIO_WORKSPACE_DIR", default=os.path.join("${PROJECT_DIR}", ".pio"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="build_dir", description=( "PlatformIO Build System uses this folder for project environments" " to store compiled object files, static libraries, firmwares, " "and other cached information" ), sysenvvar="PLATFORMIO_BUILD_DIR", default=os.path.join("${platformio.workspace_dir}", "build"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="libdeps_dir", description=( "Internal storage where Library Manager will install project " "dependencies declared via `lib_deps` option" ), sysenvvar="PLATFORMIO_LIBDEPS_DIR", default=os.path.join("${platformio.workspace_dir}", "libdeps"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="include_dir", description=( "A default location for project header files. PlatformIO Build " "System automatically adds this path to CPPPATH scope" ), sysenvvar="PLATFORMIO_INCLUDE_DIR", default=os.path.join("${PROJECT_DIR}", "include"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="src_dir", description=( "A default location where PlatformIO Build System looks for the " "project C/C++ source files" ), sysenvvar="PLATFORMIO_SRC_DIR", default=os.path.join("${PROJECT_DIR}", "src"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="lib_dir", description="A storage for the custom/private project libraries", sysenvvar="PLATFORMIO_LIB_DIR", default=os.path.join("${PROJECT_DIR}", "lib"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="data_dir", description=( "A data directory to store contents which can be uploaded to " "file system (SPIFFS, etc.)" ), sysenvvar="PLATFORMIO_DATA_DIR", default=os.path.join("${PROJECT_DIR}", "data"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="test_dir", description=( "A location where PlatformIO Unit Testing engine looks for " "test source files" ), sysenvvar="PLATFORMIO_TEST_DIR", default=os.path.join("${PROJECT_DIR}", "test"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="boards_dir", description="A storage for custom board manifests", sysenvvar="PLATFORMIO_BOARDS_DIR", default=os.path.join("${PROJECT_DIR}", "boards"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="monitor_dir", description="A storage for custom monitor filters", sysenvvar="PLATFORMIO_MONITOR_DIR", default=os.path.join("${PROJECT_DIR}", "monitor"), validate=validate_dir, ), ConfigPlatformioOption( group="directory", name="shared_dir", description=( "A location which PlatformIO Remote Development service uses to " "synchronize extra files between remote machines" ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("${PROJECT_DIR}", "shared"), validate=validate_dir, ), # # [env] # # Platform ConfigEnvOption( group="platform", name="platform", description="A name or specification of development platform", buildenvvar="PIOPLATFORM", ), ConfigEnvOption( group="platform", name="platform_packages", description="Custom packages and specifications", multiple=True, ), # Board ConfigEnvOption( group="platform", name="board", description="A board ID", buildenvvar="BOARD", ), ConfigEnvOption( group="platform", name="framework", description="A list of project dependent frameworks", multiple=True, buildenvvar="PIOFRAMEWORK", ), ConfigEnvOption( group="platform", name="board_build.mcu", description="A custom board MCU", oldnames=["board_mcu"], buildenvvar="BOARD_MCU", ), ConfigEnvOption( group="platform", name="board_build.f_cpu", description="A custom MCU frequency", oldnames=["board_f_cpu"], buildenvvar="BOARD_F_CPU", ), ConfigEnvOption( group="platform", name="board_build.f_flash", description="A custom flash frequency", oldnames=["board_f_flash"], buildenvvar="BOARD_F_FLASH", ), ConfigEnvOption( group="platform", name="board_build.flash_mode", description="A custom flash mode", oldnames=["board_flash_mode"], buildenvvar="BOARD_FLASH_MODE", ), # Build ConfigEnvOption( group="build", name="build_type", description="Project build configuration", type=click.Choice(["release", "test", "debug"]), default="release", ), ConfigEnvOption( group="build", name="build_flags", description=( "Custom build flags/options for preprocessing, compilation, " "assembly, and linking processes" ), multiple=True, sysenvvar="PLATFORMIO_BUILD_FLAGS", buildenvvar="BUILD_FLAGS", ), ConfigEnvOption( group="build", name="build_src_flags", oldnames=["src_build_flags"], description=( "The same as `build_flags` but configures flags the only for " "project source files in the `src` folder" ), multiple=True, sysenvvar="PLATFORMIO_BUILD_SRC_FLAGS", buildenvvar="SRC_BUILD_FLAGS", ), ConfigEnvOption( group="build", name="build_unflags", description="A list with flags/option which should be removed", multiple=True, sysenvvar="PLATFORMIO_BUILD_UNFLAGS", buildenvvar="BUILD_UNFLAGS", ), ConfigEnvOption( group="build", name="build_src_filter", oldnames=["src_filter"], description=( "Control which source files from the `src` folder should " "be included/excluded from a build process" ), multiple=True, sysenvvar="PLATFORMIO_BUILD_SRC_FILTER", buildenvvar="SRC_FILTER", default="+<*> -<.git/> -<.svn/>", ), ConfigEnvOption( group="build", name="targets", description="A custom list of targets for PlatformIO Build System", multiple=True, ), # Upload ConfigEnvOption( group="upload", name="upload_port", description=( "An upload port which `uploader` tool uses for a firmware flashing" ), sysenvvar="PLATFORMIO_UPLOAD_PORT", buildenvvar="UPLOAD_PORT", ), ConfigEnvOption( group="upload", name="upload_protocol", description="A protocol that `uploader` tool uses to talk to a board", buildenvvar="UPLOAD_PROTOCOL", ), ConfigEnvOption( group="upload", name="upload_speed", description=( "A connection speed (baud rate) which `uploader` tool uses when " "sending firmware to a board" ), type=click.INT, buildenvvar="UPLOAD_SPEED", ), ConfigEnvOption( group="upload", name="upload_flags", description="An extra flags for `uploader` tool", multiple=True, sysenvvar="PLATFORMIO_UPLOAD_FLAGS", buildenvvar="UPLOAD_FLAGS", ), ConfigEnvOption( group="upload", name="upload_resetmethod", description="A custom reset method", buildenvvar="UPLOAD_RESETMETHOD", ), ConfigEnvOption( group="upload", name="upload_command", description=( "A custom upload command which overwrites a default from " "development platform" ), buildenvvar="UPLOADCMD", ), # Monitor ConfigEnvOption( group="monitor", name="monitor_port", description="A port, a number or a device name", ), ConfigEnvOption( group="monitor", name="monitor_speed", description="A monitor speed (baud rate)", type=click.INT, oldnames=["monitor_baud"], default=9600, ), ConfigEnvOption( group="monitor", name="monitor_parity", description="A monitor parity checking", type=click.Choice(["N", "E", "O", "S", "M"]), default="N", ), ConfigEnvOption( group="monitor", name="monitor_filters", description=( "Apply the filters and text transformations to monitor output" ), multiple=True, ), ConfigEnvOption( group="monitor", name="monitor_rts", description="A monitor initial RTS line state", type=click.IntRange(0, 1), ), ConfigEnvOption( group="monitor", name="monitor_dtr", description="A monitor initial DTR line state", type=click.IntRange(0, 1), ), ConfigEnvOption( group="monitor", name="monitor_eol", description="A monitor end of line mode", type=click.Choice(["CR", "LF", "CRLF"]), default="CRLF", ), ConfigEnvOption( group="monitor", name="monitor_raw", description="Disable encodings/transformations of device output", type=click.BOOL, default=False, ), ConfigEnvOption( group="monitor", name="monitor_echo", description="Enable a monitor local echo", type=click.BOOL, default=False, ), ConfigEnvOption( group="monitor", name="monitor_encoding", description="Custom encoding (e.g. hexlify, Latin-1, UTF-8)", default="UTF-8", ), # Library ConfigEnvOption( group="library", name="lib_deps", description=( "A list of project library dependencies which should be installed " "automatically before a build process" ), oldnames=["lib_use", "lib_force", "lib_install"], multiple=True, ), ConfigEnvOption( group="library", name="lib_ignore", description=( "A list of library names which should be ignored by " "Library Dependency Finder (LDF)" ), multiple=True, ), ConfigEnvOption( group="library", name="lib_extra_dirs", description=( "A list of extra directories/storages where Library Dependency " "Finder (LDF) will look for dependencies" ), multiple=True, sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS", ), ConfigEnvOption( group="library", name="lib_ldf_mode", description=( "Control how Library Dependency Finder (LDF) should analyze " "dependencies (`#include` directives)" ), type=click.Choice(["off", "chain", "deep", "chain+", "deep+"]), default="chain", ), ConfigEnvOption( group="library", name="lib_compat_mode", description=( "Configure a strictness (compatibility mode by frameworks, " "development platforms) of Library Dependency Finder (LDF)" ), type=click.Choice(["off", "soft", "strict"]), default="soft", ), ConfigEnvOption( group="library", name="lib_archive", description=( "Create an archive (`*.a`, static library) from the object files " "and link it into a firmware (program)" ), type=click.BOOL, default=True, ), # Check ConfigEnvOption( group="check", name="check_tool", description="A list of check tools used for analysis", type=click.Choice(["cppcheck", "clangtidy", "pvs-studio"]), multiple=True, default=["cppcheck"], ), ConfigEnvOption( group="check", name="check_src_filters", oldnames=["check_patterns"], description=( "Configure a list of target files or directories for checking" ), multiple=True, ), ConfigEnvOption( group="check", name="check_flags", description="An extra flags to be passed to a check tool", multiple=True, ), ConfigEnvOption( group="check", name="check_severity", description="List of defect severity types for analysis", multiple=True, type=click.Choice(["low", "medium", "high"]), default=["low", "medium", "high"], ), ConfigEnvOption( group="check", name="check_skip_packages", description="Skip checking includes from packages directory", type=click.BOOL, default=False, ), # Test ConfigEnvOption( group="test", name="test_framework", description="A unit testing framework", type=click.Choice(["doctest", "googletest", "unity", "custom"]), default="unity", ), ConfigEnvOption( group="test", name="test_filter", description="Process tests where the name matches specified patterns", multiple=True, ), ConfigEnvOption( group="test", name="test_ignore", description="Ignore tests where the name matches specified patterns", multiple=True, ), ConfigEnvOption( group="test", name="test_port", description="A serial port to communicate with a target device", ), ConfigEnvOption( group="test", name="test_speed", description="A connection speed (baud rate) to communicate with " "a target device", type=click.INT, default=115200, ), ConfigEnvOption( group="test", name="test_build_src", oldnames=["test_build_project_src"], description="Build main source code in pair with a test code", type=click.BOOL, default=False, ), ConfigEnvOption( group="test", name="test_testing_command", multiple=True, description=( "A custom testing command that runs test cases " "and returns results to the standard output" ), ), # Debug ConfigEnvOption( group="debug", name="debug_tool", description="A name of debugging tool", ), ConfigEnvOption( group="debug", name="debug_build_flags", description=( "Custom debug flags/options for preprocessing, compilation, " "assembly, and linking processes" ), multiple=True, default=["-Og", "-g2", "-ggdb2"], ), ConfigEnvOption( group="debug", name="debug_init_break", description=( "An initial breakpoint that makes program stop whenever a " "certain point in the program is reached" ), default="tbreak main", ), ConfigEnvOption( group="debug", name="debug_init_cmds", description="Initial commands to be passed to a back-end debugger", multiple=True, ), ConfigEnvOption( group="debug", name="debug_extra_cmds", description="An extra commands to be passed to a back-end debugger", multiple=True, ), ConfigEnvOption( group="debug", name="debug_load_cmds", description=( "A list of commands to be used to load program/firmware " "to a target device" ), oldnames=["debug_load_cmd"], multiple=True, default=["load"], ), ConfigEnvOption( group="debug", name="debug_load_mode", description=( "Allows one to control when PlatformIO should load debugging " "firmware to the end target" ), type=click.Choice(["always", "modified", "manual"]), default="always", ), ConfigEnvOption( group="debug", name="debug_server", description="Allows one to setup a custom debugging server", multiple=True, ), ConfigEnvOption( group="debug", name="debug_port", description=( "A debugging port of a remote target (a serial device or " "network address)" ), ), ConfigEnvOption( group="debug", name="debug_speed", description="A debug adapter speed (JTAG speed)", ), ConfigEnvOption( group="debug", name="debug_svd_path", description=( "A custom path to SVD file which contains information about " "device peripherals" ), type=click.Path(exists=True, file_okay=True, dir_okay=False), ), ConfigEnvOption( group="debug", name="debug_server_ready_pattern", description=( "A pattern to determine when debugging server is ready " "for an incoming connection" ), ), ConfigEnvOption( group="debug", name="debug_test", description=("A name of a unit test to be debugged"), ), # Advanced ConfigEnvOption( group="advanced", name="extends", description=( "Inherit configuration from other sections or build environments" ), multiple=True, ), ConfigEnvOption( group="advanced", name="extra_scripts", description="A list of PRE and POST extra scripts", oldnames=["extra_script"], multiple=True, sysenvvar="PLATFORMIO_EXTRA_SCRIPTS", ), ] ] ) def get_config_options_schema(): return [opt.as_dict() for opt in ProjectOptions.values()] ================================================ FILE: platformio/project/savedeps.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from platformio.compat import ci_strings_are_equal from platformio.package.meta import PackageSpec from platformio.project.config import ProjectConfig from platformio.project.exception import InvalidProjectConfError def pkg_to_save_spec(pkg, user_spec): assert isinstance(user_spec, PackageSpec) if user_spec.external: return user_spec return PackageSpec( owner=pkg.metadata.spec.owner, name=pkg.metadata.spec.name, requirements=user_spec.requirements or ( ("^%s" % pkg.metadata.version) if not pkg.metadata.version.build else pkg.metadata.version ), ) def save_project_dependencies( project_dir, specs, scope, action="add", environments=None ): config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) config.validate(environments) for env in config.envs(): if environments and env not in environments: continue config.expand_interpolations = False candidates = [] try: candidates = _ignore_deps_by_specs(config.get("env:" + env, scope), specs) except InvalidProjectConfError: pass if action == "add": candidates.extend(spec.as_dependency() for spec in specs) if candidates: result = [] for item in candidates: item = item.strip() if item and item not in result: result.append(item) config.set("env:" + env, scope, result) elif config.has_option("env:" + env, scope): config.remove_option("env:" + env, scope) config.save() def _ignore_deps_by_specs(deps, specs): result = [] for dep in deps: ignore_conditions = [] depspec = PackageSpec(dep) if depspec.external: ignore_conditions.append(depspec in specs) else: for spec in specs: if depspec.owner: ignore_conditions.append( ci_strings_are_equal(depspec.owner, spec.owner) and ci_strings_are_equal(depspec.name, spec.name) ) else: ignore_conditions.append( ci_strings_are_equal(depspec.name, spec.name) ) if not any(ignore_conditions): result.append(dep) return result ================================================ FILE: platformio/public.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-import from platformio.device.list.util import list_logical_devices, list_serial_ports from platformio.device.monitor.filters.base import DeviceMonitorFilterBase from platformio.fs import to_unix_path from platformio.platform.base import PlatformBase from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_watch_lib_dirs, load_build_metadata from platformio.project.options import get_config_options_schema from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase from platformio.test.runners.doctest import DoctestTestRunner from platformio.test.runners.googletest import GoogletestTestRunner from platformio.test.runners.unity import UnityTestRunner from platformio.util import get_systype ================================================ FILE: platformio/registry/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/registry/access/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/registry/access/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.registry.access.commands.grant import access_grant_cmd from platformio.registry.access.commands.list import access_list_cmd from platformio.registry.access.commands.private import access_private_cmd from platformio.registry.access.commands.public import access_public_cmd from platformio.registry.access.commands.revoke import access_revoke_cmd @click.group( "access", commands=[ access_grant_cmd, access_list_cmd, access_private_cmd, access_public_cmd, access_revoke_cmd, ], short_help="Manage resource access", ) def cli(): pass ================================================ FILE: platformio/registry/access/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/registry/access/commands/grant.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.registry.access.validate import validate_client, validate_urn from platformio.registry.client import RegistryClient @click.command("grant", short_help="Grant access") @click.argument("level", type=click.Choice(["admin", "maintainer", "guest"])) @click.argument( "client", metavar="[|]", callback=lambda _, __, value: validate_client(value), ) @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_grant_cmd(level, client, urn, urn_type): # pylint: disable=unused-argument reg_client = RegistryClient() reg_client.grant_access_for_resource(urn=urn, client=client, level=level) return click.secho( "Access for resource %s has been granted for %s" % (urn, client), fg="green", ) ================================================ FILE: platformio/registry/access/commands/list.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from tabulate import tabulate from platformio.registry.client import RegistryClient @click.command("list", short_help="List published resources") @click.argument("owner", required=False) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") @click.option("--json-output", is_flag=True) def access_list_cmd(owner, urn_type, json_output): # pylint: disable=unused-argument reg_client = RegistryClient() resources = reg_client.list_resources(owner=owner) if json_output: return click.echo(json.dumps(resources)) if not resources: return click.secho("You do not have any resources.", fg="yellow") for resource in resources: click.echo() click.secho(resource.get("name"), fg="cyan") click.echo("-" * len(resource.get("name"))) table_data = [] table_data.append(("URN:", resource.get("urn"))) table_data.append(("Owner:", resource.get("owner"))) table_data.append( ( "Access:", ( click.style("Private", fg="red") if resource.get("private", False) else "Public" ), ) ) table_data.append( ( "Access level(s):", ", ".join( (level.capitalize() for level in resource.get("access_levels")) ), ) ) click.echo(tabulate(table_data, tablefmt="plain")) return click.echo() ================================================ FILE: platformio/registry/access/commands/private.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.registry.access.validate import validate_urn from platformio.registry.client import RegistryClient @click.command("private", short_help="Make resource private") @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_private_cmd(urn, urn_type): # pylint: disable=unused-argument client = RegistryClient() client.update_resource(urn=urn, private=1) return click.secho( "The resource %s has been successfully updated." % urn, fg="green", ) ================================================ FILE: platformio/registry/access/commands/public.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.registry.access.validate import validate_urn from platformio.registry.client import RegistryClient @click.command("public", short_help="Make resource public") @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_public_cmd(urn, urn_type): # pylint: disable=unused-argument client = RegistryClient() client.update_resource(urn=urn, private=0) return click.secho( "The resource %s has been successfully updated." % urn, fg="green", ) ================================================ FILE: platformio/registry/access/commands/revoke.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.registry.access.validate import validate_client, validate_urn from platformio.registry.client import RegistryClient @click.command("revoke", short_help="Revoke access") @click.argument( "client", metavar="[ORGNAME:TEAMNAME|USERNAME]", callback=lambda _, __, value: validate_client(value), ) @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_revoke_cmd(client, urn, urn_type): # pylint: disable=unused-argument reg_client = RegistryClient() reg_client.revoke_access_from_resource(urn=urn, client=client) return click.secho( "Access for resource %s has been revoked for %s" % (urn, client), fg="green", ) ================================================ FILE: platformio/registry/access/validate.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re import click from platformio.account.validate import validate_orgname_teamname, validate_username def validate_urn(value): value = str(value).strip() if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I): raise click.BadParameter("Invalid URN format.") return value def validate_client(value): if ":" in value: validate_orgname_teamname(value) else: validate_username(value) return value ================================================ FILE: platformio/registry/client.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-arguments from platformio import __registry_mirror_hosts__, fs from platformio.account.client import AccountClient, AccountError from platformio.http import HTTPClient, HTTPClientError class RegistryClient(HTTPClient): def __init__(self): endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__] super().__init__(endpoints) @staticmethod def allowed_private_packages(): private_permissions = set( [ "service.registry.publish-private-tool", "service.registry.publish-private-platform", "service.registry.publish-private-library", ] ) try: info = AccountClient().get_account_info() or {} for item in info.get("packages", []): if set(item.keys()) & private_permissions: return True except AccountError: pass return False def publish_package( # pylint: disable=redefined-builtin, too-many-positional-arguments self, owner, type, archive_path, released_at=None, private=False, notify=True ): with open(archive_path, "rb") as fp: return self.fetch_json_data( "post", "/v3/packages/%s/%s" % (owner, type), params={ "private": 1 if private else 0, "notify": 1 if notify else 0, "released_at": released_at, }, headers={ "Content-Type": "application/octet-stream", "X-PIO-Content-SHA256": fs.calculate_file_hashsum( "sha256", archive_path ), }, data=fp, x_with_authorization=True, ) def unpublish_package( # pylint: disable=redefined-builtin, too-many-positional-arguments self, owner, type, name, version=None, undo=False ): path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: path += "/" + version return self.fetch_json_data( "delete", path, params={"undo": 1 if undo else 0}, x_with_authorization=True ) def update_resource(self, urn, private): return self.fetch_json_data( "put", "/v3/resources/%s" % urn, data={"private": int(private)}, x_with_authorization=True, ) def grant_access_for_resource(self, urn, client, level): return self.fetch_json_data( "put", "/v3/resources/%s/access" % urn, data={"client": client, "level": level}, x_with_authorization=True, ) def revoke_access_from_resource(self, urn, client): return self.fetch_json_data( "delete", "/v3/resources/%s/access" % urn, data={"client": client}, x_with_authorization=True, ) def list_resources(self, owner): return self.fetch_json_data( "get", "/v3/resources", params={"owner": owner} if owner else None, x_cache_valid="1h", x_with_authorization=True, ) def list_packages(self, query=None, qualifiers=None, page=None, sort=None): search_query = [] if qualifiers: valid_qualifiers = ( "authors", "keywords", "frameworks", "platforms", "headers", "ids", "names", "owners", "types", ) assert set(qualifiers.keys()) <= set(valid_qualifiers) for name, values in qualifiers.items(): for value in set( values if isinstance(values, (list, tuple)) else [values] ): search_query.append('%s:"%s"' % (name[:-1], value)) if query: search_query.append(query) params = dict(query=" ".join(search_query)) if page: params["page"] = int(page) if sort: params["sort"] = sort return self.fetch_json_data( "get", "/v3/search", params=params, x_cache_valid="1h", x_with_authorization=self.allowed_private_packages(), ) def get_package( self, typex, owner, name, version=None, extra_path=None ): # pylint: disable=too-many-positional-arguments try: return self.fetch_json_data( "get", "/v3/packages/{owner}/{type}/{name}{extra_path}".format( type=typex, owner=owner.lower(), name=name.lower(), extra_path=extra_path or "", ), params=dict(version=version) if version else None, x_cache_valid="1h", x_with_authorization=self.allowed_private_packages(), ) except HTTPClientError as exc: if exc.response is not None and exc.response.status_code == 404: return None raise exc ================================================ FILE: platformio/registry/mirror.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from urllib.parse import urlparse from platformio import __registry_mirror_hosts__ from platformio.cache import ContentCache from platformio.exception import UserSideException from platformio.http import HTTPClient from platformio.registry.client import RegistryClient class RegistryFileMirrorIterator: HTTP_CLIENT_INSTANCES = {} def __init__(self, download_url): self.download_url = download_url self._url_parts = urlparse(download_url) self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) self._visited_mirrors = [] def __iter__(self): # pylint: disable=non-iterator-returned return self def __next__(self): cache_key = ContentCache.key_from_args( "head", self.download_url, self._visited_mirrors ) with ContentCache("http") as cc: result = cc.get(cache_key) if result is not None: try: headers = json.loads(result) return ( headers["Location"], headers["X-PIO-Content-SHA256"], ) except (ValueError, KeyError): pass http = self.get_http_client() response = http.send_request( "head", self._url_parts.path, allow_redirects=False, params=( dict(bypass=",".join(self._visited_mirrors)) if self._visited_mirrors else None ), x_with_authorization=RegistryClient.allowed_private_packages(), ) if response.status_code == 429: raise UserSideException( "Download limit exceeded. Try again in 24 hours. " "If this persists, contact " ) stop_conditions = [ response.status_code not in (302, 307), not response.headers.get("Location"), not response.headers.get("X-PIO-Mirror"), response.headers.get("X-PIO-Mirror") in self._visited_mirrors, ] if any(stop_conditions): raise StopIteration self._visited_mirrors.append(response.headers.get("X-PIO-Mirror")) cc.set( cache_key, json.dumps( { "Location": response.headers.get("Location"), "X-PIO-Content-SHA256": response.headers.get( "X-PIO-Content-SHA256" ), } ), "1h", ) return ( response.headers.get("Location"), response.headers.get("X-PIO-Content-SHA256"), ) def get_http_client(self): if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: endpoints = [self._mirror] for host in __registry_mirror_hosts__: endpoint = f"https://dl.{host}" if endpoint not in endpoints: endpoints.append(endpoint) RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( endpoints ) return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] ================================================ FILE: platformio/remote/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/remote/ac/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/remote/ac/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from twisted.internet import defer # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error class AsyncCommandBase: MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb def __init__(self, options=None, on_end_callback=None): self.options = options or {} self.on_end_callback = on_end_callback self._buffer = b"" self._return_code = None self._d = None self._paused = False try: self.start() except Exception as exc: raise pb.Error(str(exc)) from exc @property def id(self): return id(self) def pause(self): self._paused = True self.stop() def unpause(self): self._paused = False self.start() def start(self): raise NotImplementedError def stop(self): self.transport.loseConnection() # pylint: disable=no-member def _ac_ended(self): if self.on_end_callback: self.on_end_callback() if not self._d or self._d.called: self._d = None return if self._buffer: self._d.callback(self._buffer) else: self._d.callback(None) def _ac_ondata(self, data): self._buffer += data if len(self._buffer) > self.MAX_BUFFER_SIZE: self._buffer = self._buffer[-1 * self.MAX_BUFFER_SIZE :] if self._paused: return if self._d and not self._d.called: self._d.callback(self._buffer) self._buffer = b"" def ac_read(self): if self._buffer: result = self._buffer self._buffer = b"" return result if self._return_code is None: self._d = defer.Deferred() return self._d return None def ac_write(self, data): self.transport.write(data) # pylint: disable=no-member return len(data) def ac_close(self): self.stop() return self._return_code ================================================ FILE: platformio/remote/ac/process.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from twisted.internet import protocol, reactor # pylint: disable=import-error from platformio.remote.ac.base import AsyncCommandBase class ProcessAsyncCmd(protocol.ProcessProtocol, AsyncCommandBase): def start(self): env = dict(os.environ).copy() env.update({"PLATFORMIO_FORCE_ANSI": "true"}) reactor.spawnProcess( self, self.options["executable"], self.options["args"], env ) def outReceived(self, data): self._ac_ondata(data) def errReceived(self, data): self._ac_ondata(data) def processExited(self, reason): self._return_code = reason.value.exitCode def processEnded(self, reason): if self._return_code is None: self._return_code = reason.value.exitCode self._ac_ended() ================================================ FILE: platformio/remote/ac/psync.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import zlib from io import BytesIO from platformio.remote.ac.base import AsyncCommandBase from platformio.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync class ProjectSyncAsyncCmd(AsyncCommandBase): def __init__(self, *args, **kwargs): self.psync = None self._upstream = None super().__init__(*args, **kwargs) def start(self): project_dir = os.path.join( self.options["agent_working_dir"], "projects", self.options["id"] ) self.psync = ProjectSync(project_dir) for name in self.options["items"]: self.psync.add_item(os.path.join(project_dir, name), name) def stop(self): self.psync = None self._upstream = None self._return_code = PROJECT_SYNC_STAGE.COMPLETED.value def ac_write(self, data): stage = PROJECT_SYNC_STAGE.lookupByValue(data.get("stage")) if stage is PROJECT_SYNC_STAGE.DBINDEX: self.psync.rebuild_dbindex() return zlib.compress(json.dumps(self.psync.get_dbindex()).encode()) if stage is PROJECT_SYNC_STAGE.DELETE: return self.psync.delete_dbindex( json.loads(zlib.decompress(data["dbindex"])) ) if stage is PROJECT_SYNC_STAGE.UPLOAD: if not self._upstream: self._upstream = BytesIO() self._upstream.write(data["chunk"]) if self._upstream.tell() == data["total"]: self.psync.decompress_items(self._upstream) self._upstream = None return PROJECT_SYNC_STAGE.EXTRACTED.value return PROJECT_SYNC_STAGE.UPLOAD.value return None ================================================ FILE: platformio/remote/ac/serial.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from time import sleep from twisted.internet import protocol, reactor # pylint: disable=import-error from twisted.internet.serialport import SerialPort # pylint: disable=import-error from platformio.remote.ac.base import AsyncCommandBase class SerialPortAsyncCmd(protocol.Protocol, AsyncCommandBase): def start(self): SerialPort( self, reactor=reactor, **{ "deviceNameOrPortNumber": self.options["port"], "baudrate": self.options["baud"], "parity": self.options["parity"], "rtscts": 1 if self.options["rtscts"] else 0, "xonxoff": 1 if self.options["xonxoff"] else 0, } ) def connectionMade(self): self.reset_device() if self.options.get("rts", None) is not None: self.transport.setRTS(self.options.get("rts")) if self.options.get("dtr", None) is not None: self.transport.setDTR(self.options.get("dtr")) def reset_device(self): self.transport.flushInput() self.transport.setDTR(False) self.transport.setRTS(False) sleep(0.1) self.transport.setDTR(True) self.transport.setRTS(True) sleep(0.1) def dataReceived(self, data): self._ac_ondata(data) def connectionLost(self, reason): # pylint: disable=unused-argument if self._paused: return self._return_code = 0 self._ac_ended() ================================================ FILE: platformio/remote/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-arguments, import-outside-toplevel # pylint: disable=inconsistent-return-statements import os import subprocess import sys import threading from site import addsitedir from tempfile import mkdtemp from time import sleep import click from platformio import fs, proc from platformio.device.monitor.command import ( apply_project_monitor_options, device_monitor_cmd, get_project_options, ) from platformio.package.manager.core import get_core_package_dir from platformio.project.exception import NotPlatformIOProjectError from platformio.project.options import ProjectOptions from platformio.run.cli import cli as cmd_run from platformio.test.cli import cli as test_cmd @click.group("remote", short_help="Remote Development") @click.option("-a", "--agent", multiple=True) @click.pass_context def cli(ctx, agent): ctx.obj = agent # inject twisted dependencies contrib_dir = get_core_package_dir("contrib-pioremote") if contrib_dir not in sys.path: addsitedir(contrib_dir) sys.path.insert(0, contrib_dir) @cli.group("agent", short_help="Start a new agent or list active") def remote_agent(): pass @remote_agent.command("start", short_help="Start agent") @click.option("-n", "--name") @click.option("-s", "--share", multiple=True, metavar="E-MAIL") @click.option( "-d", "--working-dir", envvar="PLATFORMIO_REMOTE_AGENT_DIR", type=click.Path(file_okay=False, dir_okay=True, writable=True), ) def remote_agent_start(name, share, working_dir): from platformio.remote.client.agent_service import RemoteAgentService RemoteAgentService(name, share, working_dir).connect() @remote_agent.command("list", short_help="List active agents") def remote_agent_list(): from platformio.remote.client.agent_list import AgentListClient AgentListClient().connect() @cli.command("update", short_help="Update installed Platforms, Packages and Libraries") @click.option( "-c", "--only-check", is_flag=True, help="DEPRECATED. Please use `--dry-run` instead", ) @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) @click.pass_obj def remote_update(agents, only_check, dry_run): from platformio.remote.client.update_core import UpdateCoreClient UpdateCoreClient("update", agents, dict(only_check=only_check or dry_run)).connect() @cli.command("run", short_help="Process project environments remotely") @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option("--disable-auto-clean", is_flag=True) @click.option("-r", "--force-remote", is_flag=True) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.pass_obj @click.pass_context def remote_run( # pylint: disable=too-many-positional-arguments ctx, agents, environment, target, upload_port, project_dir, disable_auto_clean, force_remote, silent, verbose, ): from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( "run", agents, dict( environment=environment, target=target, upload_port=upload_port, project_dir=project_dir, disable_auto_clean=disable_auto_clean, force_remote=force_remote, silent=silent, verbose=verbose, ), ) if force_remote: return cr.connect() click.secho("Building project locally", bold=True) local_targets = [] if "clean" in target: local_targets = ["clean"] elif set(["buildfs", "uploadfs", "uploadfsota"]) & set(target): local_targets = ["buildfs"] else: local_targets = ["checkprogsize", "buildprog"] ctx.invoke( cmd_run, environment=environment, target=local_targets, project_dir=project_dir, # disable_auto_clean=True, silent=silent, verbose=verbose, ) if any(["upload" in t for t in target] + ["program" in target]): click.secho("Uploading firmware remotely", bold=True) cr.options["target"] += ("nobuild",) cr.options["disable_auto_clean"] = True cr.connect() return True @cli.command("test", short_help="Remote Unit Testing") @click.option("--environment", "-e", multiple=True, metavar="") @click.option( "--filter", "-f", multiple=True, metavar="", help="Filter tests by a pattern", ) @click.option( "--ignore", "-i", multiple=True, metavar="", help="Ignore tests by a pattern", ) @click.option("--upload-port") @click.option("--test-port") @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option("-r", "--force-remote", is_flag=True) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--verbose", "-v", is_flag=True) @click.pass_obj @click.pass_context def remote_test( # pylint: disable=redefined-builtin,too-many-positional-arguments ctx, agents, environment, filter, ignore, upload_port, test_port, project_dir, force_remote, without_building, without_uploading, verbose, ): from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( "test", agents, dict( environment=environment, filter=filter, ignore=ignore, upload_port=upload_port, test_port=test_port, project_dir=project_dir, force_remote=force_remote, without_building=without_building, without_uploading=without_uploading, verbose=verbose, ), ) if force_remote: return cr.connect() click.secho("Building project locally", bold=True) ctx.invoke( test_cmd, environment=environment, filter=filter, ignore=ignore, project_dir=project_dir, without_uploading=True, without_testing=True, verbose=verbose, ) click.secho("Testing project remotely", bold=True) cr.options["without_building"] = True cr.connect() return True @cli.group("device", short_help="Monitor remote device or list existing") def remote_device(): pass @remote_device.command("list", short_help="List remote devices") @click.option("--json-output", is_flag=True) @click.pass_obj def device_list(agents, json_output): from platformio.remote.client.device_list import DeviceListClient DeviceListClient(agents, json_output).connect() @remote_device.command("monitor", short_help="Monitor remote device") @click.option("--port", "-p", help="Port, a number or a device name") @click.option( "-b", "--baud", type=ProjectOptions["env.monitor_speed"].type, help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, ) @click.option( "--parity", type=ProjectOptions["env.monitor_parity"].type, help="Set parity [default=%s]" % ProjectOptions["env.monitor_parity"].default, ) @click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") @click.option("--xonxoff", is_flag=True, help="Enable software flow control") @click.option( "--rts", type=ProjectOptions["env.monitor_rts"].type, help="Set initial RTS line state", ) @click.option( "--dtr", type=ProjectOptions["env.monitor_dtr"].type, help="Set initial DTR line state", ) @click.option("--echo", is_flag=True, help="Enable local echo") @click.option( "--encoding", default="UTF-8", show_default=True, help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)", ) @click.option( "-f", "--filter", "filters", multiple=True, help="Apply filters/text transformations", ) @click.option( "--eol", type=ProjectOptions["env.monitor_eol"].type, help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, ) @click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) @click.option( "--exit-char", type=int, default=3, show_default=True, help="ASCII code of special character that is used to exit " "the application [default=3 (Ctrl+C)]", ) @click.option( "--menu-char", type=int, default=20, help="ASCII code of special character that is used to " "control terminal (menu) [default=20 (DEC)]", ) @click.option( "--quiet", is_flag=True, help="Diagnostics: suppress non-error messages", ) @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", "--environment", help="Load configuration from `platformio.ini` and specified environment", ) @click.option( "--sock", type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.pass_obj @click.pass_context def device_monitor(ctx, agents, **kwargs): from platformio.remote.client.device_monitor import DeviceMonitorClient if kwargs["sock"]: return DeviceMonitorClient(agents, **kwargs).connect() project_options = {} try: with fs.cd(kwargs["project_dir"]): project_options = get_project_options(kwargs["environment"]) except NotPlatformIOProjectError: pass kwargs = apply_project_monitor_options(kwargs, project_options) def _tx_target(sock_dir): subcmd_argv = ["remote"] for agent in agents: subcmd_argv.extend(["--agent", agent]) subcmd_argv.extend(["device", "monitor"]) subcmd_argv.extend(project_options_to_monitor_argv(kwargs)) subcmd_argv.extend(["--sock", sock_dir]) subprocess.call([proc.where_is_program("platformio")] + subcmd_argv) sock_dir = mkdtemp(suffix="pio") sock_file = os.path.join(sock_dir, "sock") try: t = threading.Thread(target=_tx_target, args=(sock_dir,)) t.start() while t.is_alive() and not os.path.isfile(sock_file): sleep(0.1) if not t.is_alive(): return with open(sock_file, encoding="utf8") as fp: kwargs["port"] = fp.read() kwargs["no_reconnect"] = True ctx.invoke(device_monitor_cmd, **kwargs) t.join(2) finally: fs.rmtree(sock_dir) return True def project_options_to_monitor_argv(cli_options): result = [] for item in cli_options["filters"] or []: result.extend(["--filter", item]) for k, v in cli_options.items(): if v is None or k == "filters": continue k = "--" + k.replace("_", "-") if isinstance(v, bool): if v: result.append(k) elif isinstance(v, tuple): for i in v: result.extend([k, i]) else: result.extend([k, str(v)]) return result ================================================ FILE: platformio/remote/client/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/remote/client/agent_list.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from datetime import datetime import click from platformio.remote.client.base import RemoteClientBase class AgentListClient(RemoteClientBase): def agent_pool_ready(self): d = self.agentpool.callRemote("list", True) d.addCallback(self._cbResult) d.addErrback(self.cb_global_error) def _cbResult(self, result): for item in result: click.secho(item["name"], fg="cyan") click.echo("-" * len(item["name"])) click.echo("ID: %s" % item["id"]) click.echo( "Started: %s" % datetime.fromtimestamp(item["started"]).strftime("%Y-%m-%d %H:%M:%S") ) click.echo("") self.disconnect() ================================================ FILE: platformio/remote/client/agent_service.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from twisted.logger import LogLevel # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error from platformio import proc from platformio.device.list.util import list_serial_ports from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.remote.ac.process import ProcessAsyncCmd from platformio.remote.ac.psync import ProjectSyncAsyncCmd from platformio.remote.ac.serial import SerialPortAsyncCmd from platformio.remote.client.base import RemoteClientBase class RemoteAgentService(RemoteClientBase): def __init__(self, name, share, working_dir=None): RemoteClientBase.__init__(self) self.log_level = LogLevel.info self.working_dir = working_dir or os.path.join( ProjectConfig.get_instance().get("platformio", "core_dir"), "remote" ) if not os.path.isdir(self.working_dir): os.makedirs(self.working_dir) if name: self.name = str(name)[:50] self.join_options.update( {"agent": True, "share": [s.lower().strip()[:50] for s in share]} ) self._acs = {} def agent_pool_ready(self): pass def cb_disconnected(self, reason): for ac in self._acs.values(): ac.ac_close() RemoteClientBase.cb_disconnected(self, reason) def remote_acread(self, ac_id): self.log.debug("Async Read: {id}", id=ac_id) if ac_id not in self._acs: raise pb.Error("Invalid Async Identifier") return self._acs[ac_id].ac_read() def remote_acwrite(self, ac_id, data): self.log.debug("Async Write: {id}", id=ac_id) if ac_id not in self._acs: raise pb.Error("Invalid Async Identifier") return self._acs[ac_id].ac_write(data) def remote_acclose(self, ac_id): self.log.debug("Async Close: {id}", id=ac_id) if ac_id not in self._acs: raise pb.Error("Invalid Async Identifier") return_code = self._acs[ac_id].ac_close() del self._acs[ac_id] return return_code def remote_cmd(self, cmd, options): self.log.info("Remote command received: {cmd}", cmd=cmd) self.log.debug("Command options: {options!r}", options=options) callback = "_process_cmd_%s" % cmd.replace(".", "_") return getattr(self, callback)(options) def _defer_async_cmd(self, ac, pass_agent_name=True): self._acs[ac.id] = ac if pass_agent_name: return (self.id, ac.id, self.name) return (self.id, ac.id) def _process_cmd_device_list(self, _): return (self.name, list_serial_ports()) def _process_cmd_device_monitor(self, options): if not options["port"]: for item in list_serial_ports(): if "VID:PID" in item["hwid"]: options["port"] = item["port"] break # terminate opened monitors if options["port"]: for ac in list(self._acs.values()): if ( isinstance(ac, SerialPortAsyncCmd) and ac.options["port"] == options["port"] ): self.log.info( "Terminate previously opened monitor at {port}", port=options["port"], ) ac.ac_close() del self._acs[ac.id] if not options["port"]: raise pb.Error("Please specify serial port using `--port` option") self.log.info("Starting serial monitor at {port}", port=options["port"]) return self._defer_async_cmd(SerialPortAsyncCmd(options), pass_agent_name=False) def _process_cmd_psync(self, options): for ac in list(self._acs.values()): if ( isinstance(ac, ProjectSyncAsyncCmd) and ac.options["id"] == options["id"] ): self.log.info("Terminate previous Project Sync process") ac.ac_close() del self._acs[ac.id] options["agent_working_dir"] = self.working_dir return self._defer_async_cmd( ProjectSyncAsyncCmd(options), pass_agent_name=False ) def _process_cmd_run(self, options): return self._process_cmd_run_or_test("run", options) def _process_cmd_test(self, options): return self._process_cmd_run_or_test("test", options) def _process_cmd_run_or_test( # pylint: disable=too-many-locals,too-many-branches,too-many-statements self, command, options ): assert options and "project_id" in options project_dir = os.path.join(self.working_dir, "projects", options["project_id"]) origin_pio_ini = os.path.join(project_dir, "platformio.ini") back_pio_ini = os.path.join(project_dir, "platformio.ini.bak") # remove insecure project options try: conf = ProjectConfig(origin_pio_ini) if os.path.isfile(back_pio_ini): os.remove(back_pio_ini) os.rename(origin_pio_ini, back_pio_ini) # cleanup if conf.has_section("platformio"): for opt in conf.options("platformio"): if opt.endswith("_dir"): conf.remove_option("platformio", opt) else: conf.add_section("platformio") conf.set("platformio", "build_dir", ".pio/build") conf.save(origin_pio_ini) # restore A/M times os.utime( origin_pio_ini, (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)), ) except NotPlatformIOProjectError as exc: raise pb.Error(str(exc)) from exc cmd_args = ["platformio", "--force", command, "-d", project_dir] for env in options.get("environment", []): cmd_args.extend(["-e", env]) for target in options.get("target", []): cmd_args.extend(["-t", target]) for filter_ in options.get("filter", []): cmd_args.extend(["-f", filter_]) for ignore in options.get("ignore", []): cmd_args.extend(["-i", ignore]) if options.get("upload_port", False): cmd_args.extend(["--upload-port", options.get("upload_port")]) if options.get("test_port", False): cmd_args.extend(["--test-port", options.get("test_port")]) if options.get("disable_auto_clean", False): cmd_args.append("--disable-auto-clean") if options.get("without_building", False): cmd_args.append("--without-building") if options.get("without_uploading", False): cmd_args.append("--without-uploading") if options.get("silent", False): cmd_args.append("-s") if options.get("verbose", False): cmd_args.append("-v") paused_acs = [] for ac in self._acs.values(): if not isinstance(ac, SerialPortAsyncCmd): continue self.log.info("Pause active monitor at {port}", port=ac.options["port"]) ac.pause() paused_acs.append(ac) def _cb_on_end(): if os.path.isfile(back_pio_ini): if os.path.isfile(origin_pio_ini): os.remove(origin_pio_ini) os.rename(back_pio_ini, origin_pio_ini) for ac in paused_acs: ac.unpause() self.log.info( "Unpause active monitor at {port}", port=ac.options["port"] ) return self._defer_async_cmd( ProcessAsyncCmd( {"executable": proc.where_is_program("platformio"), "args": cmd_args}, on_end_callback=_cb_on_end, ) ) def _process_cmd_update(self, options): cmd_args = ["platformio", "--force", "update"] if options.get("only_check"): cmd_args.append("--only-check") return self._defer_async_cmd( ProcessAsyncCmd( {"executable": proc.where_is_program("platformio"), "args": cmd_args} ) ) ================================================ FILE: platformio/remote/client/async_base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from twisted.spread import pb # pylint: disable=import-error from platformio.remote.client.base import RemoteClientBase class AsyncClientBase(RemoteClientBase): def __init__(self, command, agents, options): RemoteClientBase.__init__(self) self.command = command self.agents = agents self.options = options self._acs_total = 0 self._acs_ended = 0 def agent_pool_ready(self): pass def cb_async_result(self, result): if self._acs_total == 0: self._acs_total = len(result) for success, value in result: if not success: raise pb.Error(value) self.acread_data(*value) def acread_data(self, agent_id, ac_id, agent_name=None): d = self.agentpool.callRemote("acread", agent_id, ac_id) d.addCallback(self.cb_acread_result, agent_id, ac_id, agent_name) d.addErrback(self.cb_global_error) def cb_acread_result(self, result, agent_id, ac_id, agent_name): if result is None: self.acclose(agent_id, ac_id) else: if self._acs_total > 1 and agent_name: click.echo("[%s] " % agent_name, nl=False) click.echo(result, nl=False) self.acread_data(agent_id, ac_id, agent_name) def acclose(self, agent_id, ac_id): d = self.agentpool.callRemote("acclose", agent_id, ac_id) d.addCallback(self.cb_acclose_result) d.addErrback(self.cb_global_error) def cb_acclose_result(self, exit_code): self._acs_ended += 1 if self._acs_ended != self._acs_total: return self.disconnect(exit_code) ================================================ FILE: platformio/remote/client/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from datetime import datetime from time import time import click from twisted.internet import defer, endpoints, reactor # pylint: disable=import-error from twisted.logger import ILogObserver # pylint: disable=import-error from twisted.logger import Logger # pylint: disable=import-error from twisted.logger import LogLevel # pylint: disable=import-error from twisted.logger import formatEvent # pylint: disable=import-error from twisted.python import failure # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error from zope.interface import provider # pylint: disable=import-error from platformio import __pioremote_endpoint__, __version__, app, exception, maintenance from platformio.remote.factory.client import RemoteClientFactory from platformio.remote.factory.ssl import SSLContextFactory class RemoteClientBase( # pylint: disable=too-many-instance-attributes pb.Referenceable ): PING_DELAY = 60 PING_MAX_FAILURES = 3 DEBUG = False def __init__(self): self.log_level = LogLevel.warn self.log = Logger(namespace="remote", observer=self._log_observer) self.id = app.get_host_id() self.name = app.get_host_name() self.join_options = {"corever": __version__} self.perspective = None self.agentpool = None self._ping_id = 0 self._ping_caller = None self._ping_counter = 0 self._reactor_stopped = False self._exit_code = 0 @provider(ILogObserver) def _log_observer(self, event): if not self.DEBUG and ( event["log_namespace"] != self.log.namespace or self.log_level > event["log_level"] ): return msg = formatEvent(event) click.echo( "%s [%s] %s" % ( datetime.fromtimestamp(event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"), event["log_level"].name, msg, ) ) def connect(self): self.log.info("Name: {name}", name=self.name) self.log.info("Connecting to PlatformIO Remote Development Cloud") # pylint: disable=protected-access proto, options = endpoints._parse(__pioremote_endpoint__) proto = proto[0] factory = RemoteClientFactory() factory.remote_client = self factory.sslContextFactory = None if proto == "ssl": factory.sslContextFactory = SSLContextFactory(options["host"]) reactor.connectSSL( options["host"], int(options["port"]), factory, factory.sslContextFactory, ) elif proto == "tcp": reactor.connectTCP(options["host"], int(options["port"]), factory) else: raise exception.PlatformioException("Unknown PIO Remote Cloud protocol") reactor.run() if self._exit_code != 0: raise exception.ReturnErrorCode(self._exit_code) def cb_client_authorization_failed(self, err): msg = "Bad account credentials" if err.check(pb.Error): msg = err.getErrorMessage() self.log.error(msg) self.disconnect(exit_code=1) def cb_client_authorization_made(self, perspective): self.log.info("Successfully authorized") self.perspective = perspective d = perspective.callRemote("join", self.id, self.name, self.join_options) d.addCallback(self._cb_client_join_made) d.addErrback(self.cb_global_error) def _cb_client_join_made(self, result): code = result[0] if code == 1: self.agentpool = result[1] self.agent_pool_ready() self.restart_ping() elif code == 2: self.remote_service(*result[1:]) def remote_service(self, command, options): if command == "disconnect": self.log.error( "PIO Remote Cloud disconnected: {msg}", msg=options.get("message") ) self.disconnect() def restart_ping(self, reset_counter=True): # stop previous ping callers self.stop_ping(reset_counter) self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping) def _do_ping(self): self._ping_counter += 1 self._ping_id = int(time()) d = self.perspective.callRemote("service", "ping", {"id": self._ping_id}) d.addCallback(self._cb_pong) d.addErrback(self._cb_pong) def stop_ping(self, reset_counter=True): if reset_counter: self._ping_counter = 0 if not self._ping_caller or not self._ping_caller.active(): return self._ping_caller.cancel() self._ping_caller = None def _cb_pong(self, result): if not isinstance(result, failure.Failure) and self._ping_id == result: self.restart_ping() return if self._ping_counter >= self.PING_MAX_FAILURES: self.stop_ping() self.perspective.broker.transport.loseConnection() else: self.restart_ping(reset_counter=False) def agent_pool_ready(self): raise NotImplementedError def disconnect(self, exit_code=None): self.stop_ping() if exit_code is not None: self._exit_code = exit_code if reactor.running and not self._reactor_stopped: self._reactor_stopped = True reactor.stop() def cb_disconnected(self, _): self.stop_ping() self.perspective = None self.agentpool = None def cb_global_error(self, err): if err.check(pb.PBConnectionLost, defer.CancelledError): return msg = err.getErrorMessage() if err.check(pb.DeadReferenceError): msg = "Remote Client has been terminated" elif "PioAgentNotStartedError" in str(err.type): msg = ( "Could not find active agents. Please start it before on " "a remote machine using `pio remote agent start` command.\n" "See http://docs.platformio.org/page/plus/pio-remote.html" ) else: maintenance.on_platformio_exception(Exception(err.type)) click.secho(msg, fg="red", err=True) self.disconnect(exit_code=1) ================================================ FILE: platformio/remote/client/device_list.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from platformio.remote.client.base import RemoteClientBase class DeviceListClient(RemoteClientBase): def __init__(self, agents, json_output): RemoteClientBase.__init__(self) self.agents = agents self.json_output = json_output def agent_pool_ready(self): d = self.agentpool.callRemote("cmd", self.agents, "device.list") d.addCallback(self._cbResult) d.addErrback(self.cb_global_error) def _cbResult(self, result): data = {} for success, value in result: if not success: click.secho(value, fg="red", err=True) continue agent_name, devlist = value data[agent_name] = devlist if self.json_output: click.echo(json.dumps(data)) else: for agent_name, devlist in data.items(): click.echo("Agent %s" % click.style(agent_name, fg="cyan", bold=True)) click.echo("=" * (6 + len(agent_name))) for item in devlist: click.secho(item["port"], fg="cyan") click.echo("-" * len(item["port"])) click.echo("Hardware ID: %s" % item["hwid"]) click.echo("Description: %s" % item["description"]) click.echo("") self.disconnect() ================================================ FILE: platformio/remote/client/device_monitor.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from fnmatch import fnmatch import click from twisted.internet import protocol, reactor, task # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error from platformio.remote.client.base import RemoteClientBase class SMBridgeProtocol(protocol.Protocol): def connectionMade(self): self.factory.add_client(self) def connectionLost(self, reason): # pylint: disable=unused-argument self.factory.remove_client(self) def dataReceived(self, data): self.factory.send_to_server(data) class SMBridgeFactory(protocol.ServerFactory): def __init__(self, cdm): self.cdm = cdm self._clients = [] def buildProtocol(self, addr): # pylint: disable=unused-argument p = SMBridgeProtocol() p.factory = self # pylint: disable=attribute-defined-outside-init return p def add_client(self, client): self.cdm.log.debug("SMBridge: Client connected") self._clients.append(client) self.cdm.acread_data() def remove_client(self, client): self.cdm.log.debug("SMBridge: Client disconnected") self._clients.remove(client) if not self._clients: self.cdm.client_terminal_stopped() def has_clients(self): return len(self._clients) def send_to_clients(self, data): if not self._clients: return None for client in self._clients: client.transport.write(data) return len(data) def send_to_server(self, data): self.cdm.acwrite_data(data) class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes RemoteClientBase ): MAX_BUFFER_SIZE = 1024 * 1024 def __init__(self, agents, **kwargs): RemoteClientBase.__init__(self) self.agents = agents self.cmd_options = kwargs self._bridge_factory = SMBridgeFactory(self) self._agent_id = None self._ac_id = None self._d_acread = None self._d_acwrite = None self._acwrite_buffer = b"" def agent_pool_ready(self): d = task.deferLater( reactor, 1, self.agentpool.callRemote, "cmd", self.agents, "device.list" ) d.addCallback(self._cb_device_list) d.addErrback(self.cb_global_error) def _cb_device_list(self, result): devices = [] hwid_devindexes = [] for success, value in result: if not success: click.secho(value, fg="red", err=True) continue agent_name, ports = value for item in ports: if "VID:PID" in item["hwid"]: hwid_devindexes.append(len(devices)) devices.append((agent_name, item)) if len(result) == 1 and self.cmd_options["port"]: if set(["*", "?", "[", "]"]) & set(self.cmd_options["port"]): for agent, item in devices: if fnmatch(item["port"], self.cmd_options["port"]): return self.start_remote_monitor(agent, item["port"]) return self.start_remote_monitor(result[0][1][0], self.cmd_options["port"]) device = None if len(hwid_devindexes) == 1: device = devices[hwid_devindexes[0]] else: click.echo("Available ports:") for i, device in enumerate(devices): click.echo( "{index}. {host}{port} \t{description}".format( index=i + 1, host=device[0] + ":" if len(result) > 1 else "", port=device[1]["port"], description=( device[1]["description"] if device[1]["description"] != "n/a" else "" ), ) ) device_index = click.prompt( "Please choose a port (number in the list above)", type=click.Choice([str(i + 1) for i, _ in enumerate(devices)]), ) device = devices[int(device_index) - 1] self.start_remote_monitor(device[0], device[1]["port"]) return None def start_remote_monitor(self, agent, port): options = {"port": port} for key in ("baud", "parity", "rtscts", "xonxoff", "rts", "dtr"): options[key] = self.cmd_options[key] click.echo( "Starting Serial Monitor on {host}:{port}".format( host=agent, port=options["port"] ) ) d = self.agentpool.callRemote("cmd", [agent], "device.monitor", options) d.addCallback(self.cb_async_result) d.addErrback(self.cb_global_error) def cb_async_result(self, result): if len(result) != 1: raise pb.Error("Invalid response from Remote Cloud") success, value = result[0] if not success: raise pb.Error(value) reconnected = self._agent_id is not None self._agent_id, self._ac_id = value if reconnected: self.acread_data(force=True) self.acwrite_data("", force=True) return # start bridge port = reactor.listenTCP(0, self._bridge_factory) address = port.getHost() self.log.debug("Serial Bridge is started on {address!r}", address=address) if "sock" in self.cmd_options: with open( os.path.join(self.cmd_options["sock"], "sock"), mode="w", encoding="utf8", ) as fp: fp.write("socket://localhost:%d" % address.port) def client_terminal_stopped(self): try: d = self.agentpool.callRemote("acclose", self._agent_id, self._ac_id) d.addCallback(lambda r: self.disconnect()) d.addErrback(self.cb_global_error) except (AttributeError, pb.DeadReferenceError): self.disconnect(exit_code=1) def acread_data(self, force=False): if force and self._d_acread: self._d_acread.cancel() self._d_acread = None if ( self._d_acread and not self._d_acread.called ) or not self._bridge_factory.has_clients(): return try: self._d_acread = self.agentpool.callRemote( "acread", self._agent_id, self._ac_id ) self._d_acread.addCallback(self.cb_acread_result) self._d_acread.addErrback(self.cb_global_error) except (AttributeError, pb.DeadReferenceError): self.disconnect(exit_code=1) def cb_acread_result(self, result): if result is None: self.disconnect(exit_code=1) else: self._bridge_factory.send_to_clients(result) self.acread_data() def acwrite_data(self, data, force=False): if force and self._d_acwrite: self._d_acwrite.cancel() self._d_acwrite = None self._acwrite_buffer += data if len(self._acwrite_buffer) > self.MAX_BUFFER_SIZE: self._acwrite_buffer = self._acwrite_buffer[-1 * self.MAX_BUFFER_SIZE :] if (self._d_acwrite and not self._d_acwrite.called) or not self._acwrite_buffer: return data = self._acwrite_buffer self._acwrite_buffer = b"" try: d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data) d.addCallback(self.cb_acwrite_result) d.addErrback(self.cb_global_error) except (AttributeError, pb.DeadReferenceError): self.disconnect(exit_code=1) def cb_acwrite_result(self, result): assert result > 0 if self._acwrite_buffer: self.acwrite_data(b"") ================================================ FILE: platformio/remote/client/run_or_test.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import json import os import zlib from io import BytesIO from twisted.spread import pb # pylint: disable=import-error from platformio import fs from platformio.compat import hashlib_encode_data from platformio.project.config import ProjectConfig from platformio.remote.client.async_base import AsyncClientBase from platformio.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync class RunOrTestClient(AsyncClientBase): MAX_ARCHIVE_SIZE = 50 * 1024 * 1024 # 50Mb UPLOAD_CHUNK_SIZE = 256 * 1024 # 256Kb PSYNC_SRC_EXTS = [ "c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM", "h", "hpp", "ipp", "ino", "pde", "json", "properties", ] PSYNC_SKIP_DIRS = (".git", ".svn", ".hg", "example", "examples", "test", "tests") def __init__(self, *args, **kwargs): AsyncClientBase.__init__(self, *args, **kwargs) self.project_id = self.generate_project_id(self.options["project_dir"]) self.psync = ProjectSync(self.options["project_dir"]) def generate_project_id(self, path): h = hashlib.sha1(hashlib_encode_data(self.id)) h.update(hashlib_encode_data(path)) return "%s-%s" % (os.path.basename(path), h.hexdigest()) def add_project_items(self, psync): with fs.cd(self.options["project_dir"]): cfg = ProjectConfig.get_instance( os.path.join(self.options["project_dir"], "platformio.ini") ) psync.add_item(cfg.path, "platformio.ini") psync.add_item(cfg.get("platformio", "shared_dir"), "shared") psync.add_item(cfg.get("platformio", "boards_dir"), "boards") if self.options["force_remote"]: self._add_project_source_items(cfg, psync) else: self._add_project_binary_items(cfg, psync) if self.command == "test": psync.add_item(cfg.get("platformio", "test_dir"), "test") def _add_project_source_items(self, cfg, psync): psync.add_item(cfg.get("platformio", "lib_dir"), "lib") psync.add_item( cfg.get("platformio", "include_dir"), "include", cb_filter=self._cb_tarfile_filter, ) psync.add_item( cfg.get("platformio", "src_dir"), "src", cb_filter=self._cb_tarfile_filter ) if set(["buildfs", "uploadfs", "uploadfsota"]) & set( self.options.get("target", []) ): psync.add_item(cfg.get("platformio", "data_dir"), "data") @staticmethod def _add_project_binary_items(cfg, psync): build_dir = cfg.get("platformio", "build_dir") for env_name in os.listdir(build_dir): env_dir = os.path.join(build_dir, env_name) if not os.path.isdir(env_dir): continue for fname in os.listdir(env_dir): bin_file = os.path.join(env_dir, fname) bin_exts = (".elf", ".bin", ".hex", ".eep", "program") if os.path.isfile(bin_file) and fname.endswith(bin_exts): psync.add_item( bin_file, os.path.join(".pio", "build", env_name, fname) ) def _cb_tarfile_filter(self, path): if ( os.path.isdir(path) and os.path.basename(path).lower() in self.PSYNC_SKIP_DIRS ): return None if os.path.isfile(path) and not self.is_file_with_exts( path, self.PSYNC_SRC_EXTS ): return None return path @staticmethod def is_file_with_exts(path, exts): if path.endswith(tuple(".%s" % e for e in exts)): return True return False def agent_pool_ready(self): self.psync_init() def psync_init(self): self.add_project_items(self.psync) d = self.agentpool.callRemote( "cmd", self.agents, "psync", dict(id=self.project_id, items=[i[1] for i in self.psync.get_items()]), ) d.addCallback(self.cb_psync_init_result) d.addErrback(self.cb_global_error) # build db index while wait for result from agent self.psync.rebuild_dbindex() def cb_psync_init_result(self, result): self._acs_total = len(result) for success, value in result: if not success: raise pb.Error(value) agent_id, ac_id = value try: d = self.agentpool.callRemote( "acwrite", agent_id, ac_id, dict(stage=PROJECT_SYNC_STAGE.DBINDEX.value), ) d.addCallback(self.cb_psync_dbindex_result, agent_id, ac_id) d.addErrback(self.cb_global_error) except (AttributeError, pb.DeadReferenceError): self.disconnect(exit_code=1) def cb_psync_dbindex_result(self, result, agent_id, ac_id): result = set(json.loads(zlib.decompress(result))) dbindex = set(self.psync.get_dbindex()) delete = list(result - dbindex) delta = list(dbindex - result) self.log.debug( "PSync: stats, total={total}, delete={delete}, delta={delta}", total=len(dbindex), delete=len(delete), delta=len(delta), ) if not delete and not delta: return self.psync_finalize(agent_id, ac_id) if not delete: return self.psync_upload(agent_id, ac_id, delta) try: d = self.agentpool.callRemote( "acwrite", agent_id, ac_id, dict( stage=PROJECT_SYNC_STAGE.DELETE.value, dbindex=zlib.compress(json.dumps(delete).encode()), ), ) d.addCallback(self.cb_psync_delete_result, agent_id, ac_id, delta) d.addErrback(self.cb_global_error) except (AttributeError, pb.DeadReferenceError): self.disconnect(exit_code=1) return None def cb_psync_delete_result(self, result, agent_id, ac_id, dbindex): assert result self.psync_upload(agent_id, ac_id, dbindex) def psync_upload(self, agent_id, ac_id, dbindex): assert dbindex fileobj = BytesIO() compressed = self.psync.compress_items(fileobj, dbindex, self.MAX_ARCHIVE_SIZE) fileobj.seek(0) self.log.debug( "PSync: upload project, size={size}", size=len(fileobj.getvalue()) ) self.psync_upload_chunk( agent_id, ac_id, list(set(dbindex) - set(compressed)), fileobj ) def psync_upload_chunk(self, agent_id, ac_id, dbindex, fileobj): offset = fileobj.tell() total = fileobj.seek(0, os.SEEK_END) # unwind fileobj.seek(offset) chunk = fileobj.read(self.UPLOAD_CHUNK_SIZE) assert chunk try: d = self.agentpool.callRemote( "acwrite", agent_id, ac_id, dict( stage=PROJECT_SYNC_STAGE.UPLOAD.value, chunk=chunk, length=len(chunk), total=total, ), ) d.addCallback( self.cb_psync_upload_chunk_result, agent_id, ac_id, dbindex, fileobj ) d.addErrback(self.cb_global_error) except (AttributeError, pb.DeadReferenceError): self.disconnect(exit_code=1) def cb_psync_upload_chunk_result( # pylint: disable=too-many-arguments,too-many-positional-arguments self, result, agent_id, ac_id, dbindex, fileobj ): result = PROJECT_SYNC_STAGE.lookupByValue(result) self.log.debug("PSync: upload chunk result {r}", r=str(result)) assert result & (PROJECT_SYNC_STAGE.UPLOAD | PROJECT_SYNC_STAGE.EXTRACTED) if result is PROJECT_SYNC_STAGE.EXTRACTED: if dbindex: self.psync_upload(agent_id, ac_id, dbindex) else: self.psync_finalize(agent_id, ac_id) else: self.psync_upload_chunk(agent_id, ac_id, dbindex, fileobj) def psync_finalize(self, agent_id, ac_id): try: d = self.agentpool.callRemote("acclose", agent_id, ac_id) d.addCallback(self.cb_psync_completed_result, agent_id) d.addErrback(self.cb_global_error) except (AttributeError, pb.DeadReferenceError): self.disconnect(exit_code=1) def cb_psync_completed_result(self, result, agent_id): assert PROJECT_SYNC_STAGE.lookupByValue(result) options = self.options.copy() del options["project_dir"] options["project_id"] = self.project_id d = self.agentpool.callRemote("cmd", [agent_id], self.command, options) d.addCallback(self.cb_async_result) d.addErrback(self.cb_global_error) ================================================ FILE: platformio/remote/client/update_core.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.remote.client.async_base import AsyncClientBase class UpdateCoreClient(AsyncClientBase): def agent_pool_ready(self): d = self.agentpool.callRemote("cmd", self.agents, self.command, self.options) d.addCallback(self.cb_async_result) d.addErrback(self.cb_global_error) ================================================ FILE: platformio/remote/factory/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/remote/factory/client.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from twisted.cred import credentials # pylint: disable=import-error from twisted.internet import defer, protocol, reactor # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error from platformio.account.client import AccountClient from platformio.app import get_host_id class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory): def clientConnectionMade(self, broker): if self.sslContextFactory and not self.sslContextFactory.certificate_verified: self.remote_client.log.error( "A remote cloud could not prove that its security certificate is " "from {host}. This may cause a misconfiguration or an attacker " "intercepting your connection.", host=self.sslContextFactory.host, ) return self.remote_client.disconnect() pb.PBClientFactory.clientConnectionMade(self, broker) protocol.ReconnectingClientFactory.resetDelay(self) self.remote_client.log.info("Successfully connected") self.remote_client.log.info("Authenticating") auth_token = None try: auth_token = AccountClient().fetch_authentication_token() except Exception as exc: # pylint:disable=broad-except d = defer.Deferred() d.addErrback(self.clientAuthorizationFailed) d.errback(pb.Error(exc)) return d d = self.login( credentials.UsernamePassword( auth_token.encode(), get_host_id().encode(), ), client=self.remote_client, ) d.addCallback(self.remote_client.cb_client_authorization_made) d.addErrback(self.clientAuthorizationFailed) return d def clientAuthorizationFailed(self, err): AccountClient.delete_local_session() self.remote_client.cb_client_authorization_failed(err) def clientConnectionFailed(self, connector, reason): self.remote_client.log.warn( "Could not connect to PIO Remote Cloud. Reconnecting..." ) self.remote_client.cb_disconnected(reason) protocol.ReconnectingClientFactory.clientConnectionFailed( self, connector, reason ) def clientConnectionLost( # pylint: disable=arguments-differ self, connector, unused_reason ): if not reactor.running: self.remote_client.log.info("Successfully disconnected") return self.remote_client.log.warn( "Connection is lost to PIO Remote Cloud. Reconnecting" ) pb.PBClientFactory.clientConnectionLost( self, connector, unused_reason, reconnecting=1 ) self.remote_client.cb_disconnected(unused_reason) protocol.ReconnectingClientFactory.clientConnectionLost( self, connector, unused_reason ) ================================================ FILE: platformio/remote/factory/ssl.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import certifi from OpenSSL import SSL # pylint: disable=import-error from twisted.internet import ssl # pylint: disable=import-error class SSLContextFactory(ssl.ClientContextFactory): def __init__(self, host): self.host = host self.certificate_verified = False def getContext(self): ctx = super().getContext() ctx.set_verify( SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname ) ctx.load_verify_locations(certifi.where()) return ctx def verifyHostname( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments self, connection, x509, errno, depth, status ): cn = x509.get_subject().commonName if cn.startswith("*"): cn = cn[1:] if self.host.endswith(cn): self.certificate_verified = True return status ================================================ FILE: platformio/remote/projectsync.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import tarfile from binascii import crc32 from os.path import getmtime, getsize, isdir, isfile, join try: from twisted.python import constants # pylint: disable=import-error except ImportError: # https://docs.twisted.org/en/twisted-16.5.0/core/howto/constants.html import constantly as constants # pylint: disable=import-error from platformio.compat import hashlib_encode_data class PROJECT_SYNC_STAGE(constants.Flags): INIT = constants.FlagConstant() DBINDEX = constants.FlagConstant() DELETE = constants.FlagConstant() UPLOAD = constants.FlagConstant() EXTRACTED = constants.FlagConstant() COMPLETED = constants.FlagConstant() class ProjectSync: def __init__(self, path): self.path = path if not isdir(self.path): os.makedirs(self.path) self.items = [] self._db = {} def add_item(self, path, relpath, cb_filter=None): self.items.append((path, relpath, cb_filter)) def get_items(self): return self.items def rebuild_dbindex(self): self._db = {} for path, relpath, cb_filter in self.items: if cb_filter and not cb_filter(path): continue self._insert_to_db(path, relpath) if not isdir(path): continue for root, _, files in os.walk(path, followlinks=True): for name in files: self._insert_to_db( join(root, name), join(relpath, root[len(path) + 1 :], name) ) def _insert_to_db(self, path, relpath): if not isfile(path): return index_hash = "%s-%s-%s" % (relpath, getmtime(path), getsize(path)) index = crc32(hashlib_encode_data(index_hash)) self._db[index] = (path, relpath) def get_dbindex(self): return list(self._db.keys()) def delete_dbindex(self, dbindex): for index in dbindex: if index not in self._db: continue path = self._db[index][0] if isfile(path): os.remove(path) del self._db[index] self.delete_empty_folders() return True def delete_empty_folders(self): deleted = False for item in self.items: if not isdir(item[0]): continue for root, dirs, files in os.walk(item[0]): if not dirs and not files and root != item[0]: deleted = True os.rmdir(root) if deleted: return self.delete_empty_folders() return True def compress_items(self, fileobj, dbindex, max_size): compressed = [] total_size = 0 tar_opts = dict(fileobj=fileobj, mode="w:gz", bufsize=0, dereference=True) with tarfile.open(**tar_opts) as tgz: for index in dbindex: compressed.append(index) if index not in self._db: continue path, relpath = self._db[index] tgz.add(path, relpath) total_size += getsize(path) if total_size > max_size: break return compressed def decompress_items(self, fileobj): fileobj.seek(0) with tarfile.open(fileobj=fileobj, mode="r:gz") as tgz: tgz.extractall(self.path) return True ================================================ FILE: platformio/run/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/run/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import operator import os import shutil from multiprocessing import cpu_count from time import time import click from tabulate import tabulate from platformio import app, exception, fs, util from platformio.device.monitor.command import device_monitor_cmd from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import find_project_dir_above, load_build_metadata from platformio.run.helpers import clean_build_dir from platformio.run.processor import EnvironmentProcessor from platformio.test.runners.base import CTX_META_TEST_IS_RUNNING # pylint: disable=too-many-arguments,too-many-locals,too-many-branches try: SYSTEM_CPU_COUNT = cpu_count() except NotImplementedError: SYSTEM_CPU_COUNT = 1 DEFAULT_JOB_NUMS = int(os.getenv("PLATFORMIO_RUN_JOBS", SYSTEM_CPU_COUNT)) @click.command("run", short_help="Run project targets (build, upload, clean, etc.)") @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") @click.option("--monitor-port") @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option( "-j", "--jobs", type=int, default=DEFAULT_JOB_NUMS, help=( "Allow N jobs at once. " "Default is a number of CPUs in a system (N=%d)" % DEFAULT_JOB_NUMS ), ) @click.option( "-a", "--program-arg", "program_args", multiple=True, help="A program argument (multiple are allowed)", ) @click.option("--disable-auto-clean", is_flag=True) @click.option("--list-targets", is_flag=True) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.pass_context def cli( # pylint: disable=too-many-positional-arguments ctx, environment, target, upload_port, monitor_port, project_dir, project_conf, jobs, program_args, disable_auto_clean, list_targets, silent, verbose, ): app.set_session_var("custom_project_conf", project_conf) # find project directory on upper level if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) targets = list(target) if target else [] del target only_monitor = targets == ["monitor"] is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta command_failed = False with fs.cd(project_dir): config = ProjectConfig.get_instance(project_conf) config.validate(environment) if list_targets: return print_target_list(list(environment) or config.envs()) # clean obsolete build dir if not only_monitor and not disable_auto_clean: build_dir = config.get("platformio", "build_dir") try: clean_build_dir(build_dir, config) except ProjectError as exc: raise exc except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " "it manually to avoid build issues" % build_dir, fg="yellow", ) default_envs = config.default_envs() results = [] for env in config.envs(): skipenv = any( [ environment and env not in environment, not environment and default_envs and env not in default_envs, ] ) if skipenv: results.append({"env": env}) continue # print empty line between multi environment project if not silent and any(r.get("succeeded") is not None for r in results): click.echo() results.append( process_env( ctx, env, config, targets, upload_port, monitor_port, jobs, program_args, is_test_running, silent, verbose, ) ) command_failed = any(r.get("succeeded") is False for r in results) if ( not is_test_running and not only_monitor and (command_failed or not silent) and len(results) > 1 ): print_processing_summary(results, verbose) # Reset custom project config app.set_session_var("custom_project_conf", None) if command_failed: raise exception.ReturnErrorCode(1) return True def process_env( # pylint: disable=too-many-positional-arguments ctx, name, config, targets, upload_port, monitor_port, jobs, program_args, is_test_running, silent, verbose, ): if not is_test_running and not silent: print_processing_header(name, config, verbose) targets = targets or config.get(f"env:{name}", "targets", []) only_monitor = targets == ["monitor"] result = {"env": name, "duration": time(), "succeeded": True} if not only_monitor: result["succeeded"] = EnvironmentProcessor( ctx, name, config, [t for t in targets if t != "monitor"], upload_port, jobs, program_args, silent, verbose, ).process() if result["succeeded"] and "monitor" in targets and "nobuild" not in targets: ctx.invoke( device_monitor_cmd, port=monitor_port, environment=name, ) result["duration"] = time() - result["duration"] # print footer on error or when is not unit testing if ( not is_test_running and not only_monitor and (not silent or not result["succeeded"]) ): print_processing_footer(result) return result def print_processing_header(env, config, verbose=False): env_dump = [] for k, v in config.items(env=env): if verbose or k in ("platform", "framework", "board"): env_dump.append("%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) click.echo( "Processing %s (%s)" % (click.style(env, fg="cyan", bold=True), "; ".join(env_dump)) ) terminal_width = shutil.get_terminal_size().columns click.secho("-" * terminal_width, bold=True) def print_processing_footer(result): is_failed = not result.get("succeeded") util.print_labeled_bar( "[%s] Took %.2f seconds" % ( ( click.style("FAILED", fg="red", bold=True) if is_failed else click.style("SUCCESS", fg="green", bold=True) ), result["duration"], ), is_error=is_failed, ) def print_processing_summary(results, verbose=False): tabular_data = [] succeeded_nums = 0 failed_nums = 0 duration = 0 for result in results: duration += result.get("duration", 0) if result.get("succeeded") is False: failed_nums += 1 status_str = click.style("FAILED", fg="red") elif result.get("succeeded") is None: if not verbose: continue status_str = "IGNORED" else: succeeded_nums += 1 status_str = click.style("SUCCESS", fg="green") tabular_data.append( ( click.style(result["env"], fg="cyan"), status_str, util.humanize_duration_time(result.get("duration")), ) ) click.echo() click.echo( tabulate( tabular_data, headers=[ click.style(s, bold=True) for s in ("Environment", "Status", "Duration") ], ), err=failed_nums, ) util.print_labeled_bar( "%s%d succeeded in %s" % ( "%d failed, " % failed_nums if failed_nums else "", succeeded_nums, util.humanize_duration_time(duration), ), is_error=failed_nums, fg="red" if failed_nums else "green", ) def print_target_list(envs): tabular_data = [] for env, data in load_build_metadata(os.getcwd(), envs).items(): tabular_data.extend( sorted( [ ( click.style(env, fg="cyan"), t["group"], click.style(t.get("name"), fg="yellow"), t["title"], t.get("description"), ) for t in data.get("targets", []) ], key=operator.itemgetter(1, 2), ) ) tabular_data.append((None, None, None, None, None)) click.echo( tabulate( tabular_data, headers=[ click.style(s, bold=True) for s in ("Environment", "Group", "Name", "Title", "Description") ], ), ) ================================================ FILE: platformio/run/helpers.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from os import makedirs from os.path import isdir, isfile, join from platformio import fs from platformio.project.helpers import compute_project_checksum, get_project_dir KNOWN_CLEAN_TARGETS = ("clean",) KNOWN_FULLCLEAN_TARGETS = ("cleanall", "fullclean") KNOWN_ALLCLEAN_TARGETS = KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS def clean_build_dir(build_dir, config): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") if isdir(legacy_build_dir) and legacy_build_dir != build_dir: fs.rmtree(legacy_build_dir) checksum_file = join(build_dir, "project.checksum") checksum = compute_project_checksum(config) if isdir(build_dir): # check project structure if isfile(checksum_file): with open(checksum_file, encoding="utf8") as fp: if fp.read() == checksum: return fs.rmtree(build_dir) makedirs(build_dir) with open(checksum_file, mode="w", encoding="utf8") as fp: fp.write(checksum) ================================================ FILE: platformio/run/processor.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.package.commands.install import install_project_env_dependencies from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError from platformio.run.helpers import KNOWN_ALLCLEAN_TARGETS from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes class EnvironmentProcessor: def __init__( # pylint: disable=too-many-arguments,too-many-positional-arguments self, cmd_ctx, name, config, targets, upload_port, jobs, program_args, silent, verbose, ): self.cmd_ctx = cmd_ctx self.name = name self.config = config self.targets = targets self.upload_port = upload_port self.jobs = jobs self.program_args = program_args self.silent = silent self.verbose = verbose self.options = config.items(env=name, as_dict=True) def get_build_variables(self): variables = dict( pioenv=self.name, project_config=self.config.path, program_args=self.program_args, ) if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta: variables["piotest_running_name"] = self.cmd_ctx.meta[ CTX_META_TEST_RUNNING_NAME ] if self.upload_port: # override upload port with a custom from CLI variables["upload_port"] = self.upload_port return variables def process(self): if "platform" not in self.options: raise UndefinedEnvPlatformError(self.name) build_vars = self.get_build_variables() is_clean = set(KNOWN_ALLCLEAN_TARGETS) & set(self.targets) build_targets = [t for t in self.targets if t not in KNOWN_ALLCLEAN_TARGETS] # pre-clean if is_clean: result = PlatformFactory.from_env( self.name, targets=self.targets, autoinstall=True ).run(build_vars, self.targets, self.silent, self.verbose, self.jobs) if not build_targets: return result["returncode"] == 0 install_project_env_dependencies( self.name, { "project_targets": self.targets, "piotest_running_name": build_vars.get("piotest_running_name"), }, ) result = PlatformFactory.from_env( self.name, targets=build_targets, autoinstall=True ).run(build_vars, build_targets, self.silent, self.verbose, self.jobs) return result["returncode"] == 0 ================================================ FILE: platformio/system/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/system/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.system.commands.completion import system_completion_cmd from platformio.system.commands.info import system_info_cmd from platformio.system.commands.prune import system_prune_cmd @click.group( "system", commands=[ system_completion_cmd, system_info_cmd, system_prune_cmd, ], short_help="Miscellaneous system commands", ) def cli(): pass ================================================ FILE: platformio/system/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/system/commands/completion.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.system.completion import ( ShellType, get_completion_install_path, install_completion_code, uninstall_completion_code, ) @click.group("completion", short_help="Shell completion support") def system_completion_cmd(): pass @system_completion_cmd.command( "install", short_help="Install shell completion files/code" ) @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) def system_completion_install(shell, path): shell = ShellType(shell) path = path or get_completion_install_path(shell) install_completion_code(shell, path) click.echo( "PlatformIO CLI completion has been installed for %s shell to %s \n" "Please restart a current shell session." % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue")) ) @system_completion_cmd.command( "uninstall", short_help="Uninstall shell completion files/code" ) @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) def system_completion_uninstall(shell, path): shell = ShellType(shell) path = path or get_completion_install_path(shell) uninstall_completion_code(shell, path) click.echo( "PlatformIO CLI completion has been uninstalled for %s shell from %s \n" "Please restart a current shell session." % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue")) ) ================================================ FILE: platformio/system/commands/info.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import platform import sys import click from tabulate import tabulate from platformio import __version__, compat, proc, util from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig @click.command("info", short_help="Display system-wide information") @click.option("--json-output", is_flag=True) def system_info_cmd(json_output): project_config = ProjectConfig() data = {} data["core_version"] = {"title": "PlatformIO Core", "value": __version__} data["python_version"] = { "title": "Python", "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)), } data["system"] = {"title": "System Type", "value": util.get_systype()} data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)} data["filesystem_encoding"] = { "title": "File System Encoding", "value": compat.get_filesystem_encoding(), } data["locale_encoding"] = { "title": "Locale Encoding", "value": compat.get_locale_encoding(), } data["core_dir"] = { "title": "PlatformIO Core Directory", "value": project_config.get("platformio", "core_dir"), } data["platformio_exe"] = { "title": "PlatformIO Core Executable", "value": proc.where_is_program( "platformio.exe" if compat.IS_WINDOWS else "platformio" ), } data["python_exe"] = { "title": "Python Executable", "value": proc.get_pythonexe_path(), } data["global_lib_nums"] = { "title": "Global Libraries", "value": len(LibraryPackageManager().get_installed()), } data["dev_platform_nums"] = { "title": "Development Platforms", "value": len(PlatformPackageManager().get_installed()), } data["package_tool_nums"] = { "title": "Tools & Toolchains", "value": len( ToolPackageManager( project_config.get("platformio", "packages_dir") ).get_installed() ), } click.echo( json.dumps(data) if json_output else tabulate([(item["title"], item["value"]) for item in data.values()]) ) ================================================ FILE: platformio/system/commands/prune.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio import fs from platformio.system.prune import ( prune_cached_data, prune_core_packages, prune_platform_packages, ) @click.command("prune", short_help="Remove unused data") @click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") @click.option( "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed" ) @click.option("--cache", is_flag=True, help="Prune only cached data") @click.option( "--core-packages", is_flag=True, help="Prune only unnecessary core packages" ) @click.option( "--platform-packages", is_flag=True, help="Prune only unnecessary development platform packages", ) def system_prune_cmd(force, dry_run, cache, core_packages, platform_packages): if dry_run: click.secho( "Dry run mode (do not prune, only show data that will be removed)", fg="yellow", ) click.echo() reclaimed_cache = 0 reclaimed_core_packages = 0 reclaimed_platform_packages = 0 prune_all = not any([cache, core_packages, platform_packages]) if cache or prune_all: reclaimed_cache = prune_cached_data(force, dry_run) click.echo() if core_packages or prune_all: reclaimed_core_packages = prune_core_packages(force, dry_run) click.echo() if platform_packages or prune_all: reclaimed_platform_packages = prune_platform_packages(force, dry_run) click.echo() click.secho( "Total reclaimed space: %s" % fs.humanize_file_size( reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages ), fg="green", ) ================================================ FILE: platformio/system/completion.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import subprocess from enum import Enum import click from platformio.compat import IS_MACOS class ShellType(Enum): FISH = "fish" ZSH = "zsh" BASH = "bash" def get_bash_version(): output = subprocess.run( ["bash", "--version"], check=True, stdout=subprocess.PIPE ).stdout.decode() match = re.search(r"version\s+(\d+)\.(\d+)", output, re.IGNORECASE) if match: return (int(match.group(1)), int(match.group(2))) return (0, 0) def get_completion_install_path(shell): home_dir = os.path.expanduser("~") prog_name = click.get_current_context().find_root().info_name if shell == ShellType.FISH: return os.path.join( home_dir, ".config", "fish", "completions", "%s.fish" % prog_name ) if shell == ShellType.ZSH: return os.path.join(home_dir, ".zshrc") if shell == ShellType.BASH: return os.path.join(home_dir, ".bash_completion") raise click.ClickException("%s is not supported." % shell) def get_completion_code(shell): if shell == ShellType.FISH: return "eval (env _PIO_COMPLETE=fish_source pio)" if shell == ShellType.ZSH: code = "autoload -Uz compinit\ncompinit\n" if IS_MACOS else "" return code + 'eval "$(_PIO_COMPLETE=zsh_source pio)"' if shell == ShellType.BASH: return 'eval "$(_PIO_COMPLETE=bash_source pio)"' raise click.ClickException("%s is not supported." % shell) def is_completion_code_installed(shell, path): if shell == ShellType.FISH or not os.path.exists(path): return False with open(path, encoding="utf8") as fp: return get_completion_code(shell) in fp.read() def install_completion_code(shell, path): if shell == ShellType.BASH and get_bash_version() < (4, 4): raise click.ClickException("The minimal supported Bash version is 4.4") if is_completion_code_installed(shell, path): return None append = shell != ShellType.FISH with open(path, mode="a" if append else "w", encoding="utf8") as fp: if append: fp.write("\n\n# Begin: PlatformIO Core completion support\n") fp.write(get_completion_code(shell)) if append: fp.write("\n# End: PlatformIO Core completion support\n\n") return True def uninstall_completion_code(shell, path): if not os.path.exists(path): return True if shell == ShellType.FISH: os.remove(path) return True with open(path, "r+", encoding="utf8") as fp: contents = fp.read() fp.seek(0) fp.truncate() fp.write(contents.replace(get_completion_code(shell), "")) return True ================================================ FILE: platformio/system/prune.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from operator import itemgetter import click from tabulate import tabulate from platformio import fs from platformio.package.manager.core import remove_unnecessary_core_packages from platformio.package.manager.platform import remove_unnecessary_platform_packages from platformio.project.helpers import get_project_cache_dir def prune_cached_data(force=False, dry_run=False, silent=False): reclaimed_space = 0 if not silent: click.secho("Prune cached data:", bold=True) click.echo(" - cached API requests") click.echo(" - cached package downloads") click.echo(" - temporary data") cache_dir = get_project_cache_dir() if os.path.isdir(cache_dir): reclaimed_space += fs.calculate_folder_size(cache_dir) if not dry_run: if not force: click.confirm("Do you want to continue?", abort=True) fs.rmtree(cache_dir) if not silent: click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) return reclaimed_space def prune_core_packages(force=False, dry_run=False, silent=False): if not silent: click.secho("Prune unnecessary core packages:", bold=True) return _prune_packages(force, dry_run, silent, remove_unnecessary_core_packages) def prune_platform_packages(force=False, dry_run=False, silent=False): if not silent: click.secho("Prune unnecessary development platform packages:", bold=True) return _prune_packages(force, dry_run, silent, remove_unnecessary_platform_packages) def _prune_packages(force, dry_run, silent, handler): if not silent: click.echo("Calculating...") items = [ ( pkg, fs.calculate_folder_size(pkg.path), ) for pkg in handler(dry_run=True) ] items = sorted(items, key=itemgetter(1), reverse=True) reclaimed_space = sum(item[1] for item in items) if items and not silent: click.echo( tabulate( [ ( pkg.metadata.spec.humanize(), str(pkg.metadata.version), fs.humanize_file_size(size), ) for (pkg, size) in items ], headers=["Package", "Version", "Size"], ) ) if not dry_run: if not force: click.confirm("Do you want to continue?", abort=True) handler(dry_run=False) if not silent: click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) return reclaimed_space def calculate_unnecessary_system_data(): return ( prune_cached_data(force=True, dry_run=True, silent=True) + prune_core_packages(force=True, dry_run=True, silent=True) + prune_platform_packages(force=True, dry_run=True, silent=True) ) ================================================ FILE: platformio/telemetry.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import atexit import os import queue import re import sys import threading import time import traceback from collections import deque import requests from platformio import __title__, __version__, app, exception, fs, util from platformio.cli import PlatformioCLI from platformio.debug.config.base import DebugConfigBase from platformio.http import HTTPSession from platformio.proc import is_ci KEEP_MAX_REPORTS = 100 SEND_MAX_EVENTS = 25 class MeasurementProtocol: def __init__(self, events=None): self.client_id = app.get_cid() self._events = events or [] self._user_properties = {} self.set_user_property("systype", util.get_systype()) created_at = app.get_state_item("created_at", None) if created_at: self.set_user_property("created_at", int(created_at)) @staticmethod def event_to_dict(name, params, timestamp=None): event = {"name": name, "params": params} if timestamp is not None: event["timestamp"] = timestamp return event def set_user_property(self, name, value): self._user_properties[name] = value def add_event(self, name, params): self._events.append(self.event_to_dict(name, params)) def to_payload(self): return { "client_id": self.client_id, "user_properties": self._user_properties, "events": self._events, } @util.singleton class TelemetryLogger: def __init__(self): self._events = deque() self._sender_thread = None self._sender_queue = queue.Queue() self._sender_terminated = False self._http_session = HTTPSession() self._http_offline = False def close(self): self._http_session.close() def log_event(self, name, params, timestamp=None, instant_sending=False): if not app.get_setting("enable_telemetry") or app.get_session_var( "pause_telemetry" ): return None timestamp = timestamp or int(time.time()) self._events.append( MeasurementProtocol.event_to_dict(name, params, timestamp=timestamp) ) if self._http_offline: # if network is off-line return False if instant_sending: self.send() return True def send(self): if not self._events or self._sender_terminated: return if not self._sender_thread: self._sender_thread = threading.Thread( target=self._sender_worker, daemon=True ) self._sender_thread.start() while self._events: events = [] try: while len(events) < SEND_MAX_EVENTS: events.append(self._events.popleft()) except IndexError: pass self._sender_queue.put(events) def _sender_worker(self): while True: if self._sender_terminated: return try: events = self._sender_queue.get() if not self._commit_events(events): self._events.extend(events) self._sender_queue.task_done() except (queue.Empty, ValueError): pass def _commit_events(self, events): if self._http_offline: return False mp = MeasurementProtocol(events) payload = mp.to_payload() # print("_commit_payload", payload) try: r = self._http_session.post( "https://collector.platformio.org/collect", json=payload, timeout=(2, 5), # connect, read ) r.raise_for_status() return True except requests.exceptions.HTTPError as exc: # skip Bad Request if exc.response.status_code >= 400 and exc.response.status_code < 500: return True except: # pylint: disable=bare-except pass self._http_offline = True return False def terminate_sender(self): self._sender_terminated = True def is_sending(self): return self._sender_queue.unfinished_tasks def get_unsent_events(self): result = list(self._events) try: while True: result.extend(self._sender_queue.get_nowait()) except queue.Empty: pass return result def log_event(name, params, instant_sending=False): TelemetryLogger().log_event(name, params, instant_sending=instant_sending) def on_cmd_start(cmd_ctx): process_postponed_logs() log_command(cmd_ctx) def on_exit(): TelemetryLogger().send() def log_command(ctx): params = { "path_args": PlatformioCLI.reveal_cmd_path_args(ctx), } if is_ci(): params["ci_actor"] = resolve_ci_actor() or "Unknown" log_event("cmd_run", params) def resolve_ci_actor(): known_cis = ( "GITHUB_ACTIONS", "TRAVIS", "APPVEYOR", "GITLAB_CI", "CIRCLECI", "SHIPPABLE", "DRONE", ) for name in known_cis: if os.getenv(name, "false").lower() == "true": return name return None def dump_project_env_params(config, env, platform): non_sensitive_data = [ "platform", "framework", "board", "upload_protocol", "check_tool", "debug_tool", "test_framework", ] section = f"env:{env}" params = { option: config.get(section, option) for option in non_sensitive_data if config.has_option(section, option) } params["pid"] = app.get_project_id(os.path.dirname(config.path)) params["platform_name"] = platform.name params["platform_version"] = platform.version return params def log_platform_run(platform, project_config, project_env, targets=None): params = dump_project_env_params(project_config, project_env, platform) if targets: params["targets"] = targets log_event("platform_run", params, instant_sending=True) def log_exception(exc): skip_conditions = [ isinstance(exc, cls) for cls in ( IOError, exception.ReturnErrorCode, exception.UserSideException, ) ] skip_conditions.append(not isinstance(exc, Exception)) if any(skip_conditions): return is_fatal = any( [ not isinstance(exc, exception.PlatformioException), "Error" in exc.__class__.__name__, ] ) def _strip_module_path(match): module_path = match.group(1).replace(fs.get_source_dir() + os.sep, "") sp_folder_name = "site-packages" sp_pos = module_path.find(sp_folder_name) if sp_pos != -1: module_path = module_path[sp_pos + len(sp_folder_name) + 1 :] module_path = fs.to_unix_path(module_path) return f'File "{module_path}",' trace = re.sub( r'File "([^"]+)",', _strip_module_path, traceback.format_exc(), flags=re.MULTILINE, ) params = { "name": exc.__class__.__name__, "description": str(exc), "traceback": trace, "cmd_args": sys.argv[1:], "is_fatal": is_fatal, } log_event("exception", params) def log_debug_started(debug_config: DebugConfigBase): log_event( "debug_started", dump_project_env_params( debug_config.project_config, debug_config.env_name, debug_config.platform ), ) def log_debug_exception(exc, debug_config: DebugConfigBase): # cleanup sensitive information, such as paths description = fs.to_unix_path(str(exc)) description = re.sub( r'(^|\s+|")(?:[a-z]\:)?((/[^"/]+)+)(\s+|"|$)', lambda m: " %s " % os.path.join(*m.group(2).split("/")[-2:]), description, re.I | re.M, ) params = { "name": exc.__class__.__name__, "description": description.strip(), } params.update( dump_project_env_params( debug_config.project_config, debug_config.env_name, debug_config.platform ) ) log_event("debug_exception", params) @atexit.register def _finalize(): timeout = 1000 # msec elapsed = 0 telemetry = TelemetryLogger() telemetry.terminate_sender() try: while elapsed < timeout: if not telemetry.is_sending(): break time.sleep(0.2) elapsed += 200 except KeyboardInterrupt: pass postpone_events(telemetry.get_unsent_events()) telemetry.close() def load_postponed_events(): state_path = app.resolve_state_path( "cache_dir", "telemetry.json", ensure_dir_exists=False ) if not os.path.isfile(state_path): return [] with app.State(state_path) as state: return state.get("events", []) def save_postponed_events(events): state_path = app.resolve_state_path("cache_dir", "telemetry.json") if not events: try: if os.path.isfile(state_path): os.remove(state_path) except: # pylint: disable=bare-except pass return None with app.State(state_path, lock=True) as state: state["events"] = events state.modified = True return True def postpone_events(events): if not events: return None postponed_events = load_postponed_events() or [] timestamp = int(time.time()) for event in events: if "timestamp" not in event: event["timestamp"] = timestamp postponed_events.append(event) save_postponed_events(postponed_events[KEEP_MAX_REPORTS * -1 :]) return True def process_postponed_logs(): events = load_postponed_events() if not events: return None save_postponed_events([]) # clean telemetry = TelemetryLogger() for event in events: if set(["name", "params", "timestamp"]) <= set(event.keys()): telemetry.log_event( event["name"], event["params"], timestamp=event["timestamp"], instant_sending=False, ) telemetry.send() return True ================================================ FILE: platformio/test/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/test/cli.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import subprocess import click from platformio import app, exception, fs, util from platformio.project.config import ProjectConfig from platformio.test.helpers import list_test_suites from platformio.test.reports.base import TestReportFactory from platformio.test.result import TestResult, TestStatus from platformio.test.runners.base import TestRunnerOptions from platformio.test.runners.factory import TestRunnerFactory @click.command("test", short_help="Unit Testing") @click.option("--environment", "-e", multiple=True) @click.option( "--filter", "-f", multiple=True, metavar="PATTERN", help="Filter tests by a pattern", ) @click.option( "--ignore", "-i", multiple=True, metavar="PATTERN", help="Ignore tests by a pattern", ) @click.option("--upload-port") @click.option("--test-port") @click.option( "-d", "--project-dir", default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--without-testing", is_flag=True) @click.option("--no-reset", is_flag=True) @click.option( "--monitor-rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state for Serial Monitor", ) @click.option( "--monitor-dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state for Serial Monitor", ) @click.option( "-a", "--program-arg", "program_args", multiple=True, help="A program argument (multiple are allowed)", ) @click.option("--list-tests", is_flag=True) @click.option("--json-output", is_flag=True) @click.option("--json-output-path", type=click.Path()) @click.option("--junit-output-path", type=click.Path()) @click.option( "--verbose", "-v", count=True, help="Increase verbosity level, maximum is 3 levels (-vvv), see docs for details", ) @click.pass_context def cli( # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals,redefined-builtin ctx, environment, ignore, filter, upload_port, test_port, project_dir, project_conf, without_building, without_uploading, without_testing, no_reset, monitor_rts, monitor_dtr, program_args, list_tests, json_output, json_output_path, junit_output_path, verbose, ): app.set_session_var("custom_project_conf", project_conf) with fs.cd(project_dir): project_config = ProjectConfig.get_instance(project_conf) project_config.validate(envs=environment) test_result = TestResult(project_dir) test_suites = list_test_suites( project_config, environments=environment, filters=filter, ignores=ignore ) test_names = sorted(set(s.test_name for s in test_suites)) if not verbose: click.echo("Verbosity level can be increased via `-v, -vv, or -vvv` option") click.secho("Collected %d tests" % len(test_names), bold=True, nl=not verbose) if verbose: click.echo(" (%s)" % ", ".join(test_names)) for test_suite in test_suites: test_result.add_suite(test_suite) if list_tests or test_suite.is_finished(): # skipped by user continue runner = TestRunnerFactory.new( test_suite, project_config, TestRunnerOptions( verbose=verbose, without_building=without_building, without_uploading=without_uploading, without_testing=without_testing, upload_port=upload_port, test_port=test_port, no_reset=no_reset, monitor_rts=monitor_rts, monitor_dtr=monitor_dtr, program_args=program_args, ), ) click.echo() print_suite_header(test_suite) runner.start(ctx) print_suite_footer(test_suite) stdout_report = TestReportFactory.new("stdout", test_result) stdout_report.generate(verbose=verbose or list_tests) for output_format, output_path in [ ("json", subprocess.STDOUT if json_output else None), ("json", json_output_path), ("junit", junit_output_path), ]: if not output_path: continue custom_report = TestReportFactory.new(output_format, test_result) custom_report.generate(output_path=output_path, verbose=True) # Reset custom project config app.set_session_var("custom_project_conf", None) if test_result.is_errored or test_result.get_status_nums(TestStatus.FAILED): raise exception.ReturnErrorCode(1) def print_suite_header(test_suite): click.echo( "Processing %s in %s environment" % ( click.style(test_suite.test_name, fg="yellow", bold=True), click.style(test_suite.env_name, fg="cyan", bold=True), ) ) terminal_width = shutil.get_terminal_size().columns click.secho("-" * terminal_width, bold=True) def print_suite_footer(test_suite): is_error = test_suite.status in (TestStatus.FAILED, TestStatus.ERRORED) util.print_labeled_bar( "%s [%s] Took %.2f seconds" % ( click.style( "%s:%s" % (test_suite.env_name, test_suite.test_name), bold=True ), ( click.style(test_suite.status.name, fg="red", bold=True) if is_error else click.style("PASSED", fg="green", bold=True) ), test_suite.duration, ), is_error=is_error, sep="-", ) ================================================ FILE: platformio/test/exception.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.exception import PlatformioException, UserSideException class UnitTestError(PlatformioException): pass class TestDirNotExistsError(UnitTestError, UserSideException): MESSAGE = ( "A test folder '{0}' does not exist.\nPlease create 'test' " "directory in the project root and put a test suite.\n" "More details about Unit " "Testing: https://docs.platformio.org/en/latest/advanced/" "unit-testing/index.html" ) class UnitTestSuiteError(UnitTestError): pass ================================================ FILE: platformio/test/helpers.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from fnmatch import fnmatch from platformio.test.exception import TestDirNotExistsError from platformio.test.result import TestSuite def list_test_names(project_config): test_dir = project_config.get("platformio", "test_dir") if not os.path.isdir(test_dir): raise TestDirNotExistsError(test_dir) names = [] for root, _, __ in os.walk(test_dir, followlinks=True): if not os.path.basename(root).startswith("test_"): continue names.append(os.path.relpath(root, test_dir).replace("\\", "/")) if not names: names = ["*"] return names def list_test_suites(project_config, environments, filters, ignores): result = [] test_dir = project_config.get("platformio", "test_dir") default_envs = project_config.default_envs() test_names = list_test_names(project_config) for env_name in project_config.envs(): for test_name in test_names: # filter and ignore patterns patterns = dict(filter=list(filters), ignore=list(ignores)) for key, value in patterns.items(): if value: # overridden from CLI continue patterns[key].extend( # pylint: disable=unnecessary-dict-index-lookup project_config.get(f"env:{env_name}", f"test_{key}", []) ) skip_conditions = [ environments and env_name not in environments, not environments and default_envs and env_name not in default_envs, test_name != "*" and patterns["filter"] and not any(fnmatch(test_name, p) for p in patterns["filter"]), test_name != "*" and any(fnmatch(test_name, p) for p in patterns["ignore"]), ] result.append( TestSuite( env_name, test_name, finished=any(skip_conditions), test_dir=os.path.abspath( test_dir if test_name == "*" else os.path.join(test_dir, test_name) ), ) ) return result ================================================ FILE: platformio/test/reports/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/test/reports/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import importlib from platformio.test.result import TestResult class TestReportBase: def __init__(self, test_result): self.test_result = test_result def generate(self, output_path, verbose): raise NotImplementedError() class TestReportFactory: @staticmethod def new(format, test_result) -> TestReportBase: # pylint: disable=redefined-builtin assert isinstance(test_result, TestResult) mod = importlib.import_module(f"platformio.test.reports.{format}") report_cls = getattr(mod, "%sTestReport" % format.lower().capitalize()) return report_cls(test_result) ================================================ FILE: platformio/test/reports/json.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import json import os import subprocess import click from platformio.test.reports.base import TestReportBase from platformio.test.result import TestStatus class JsonTestReport(TestReportBase): def generate(self, output_path, verbose=False): if output_path == subprocess.STDOUT: return click.echo("\n\n" + json.dumps(self.to_json())) if os.path.isdir(output_path): output_path = os.path.join( output_path, "pio-test-report-%s-%s.json" % ( os.path.basename(self.test_result.project_dir), datetime.datetime.now().strftime("%Y%m%d%H%M%S"), ), ) with open(output_path, mode="w", encoding="utf8") as fp: json.dump(self.to_json(), fp) if verbose: click.secho(f"Saved JSON report to the {output_path}", fg="green") return True def to_json(self): result = dict( version="1.0", project_dir=self.test_result.project_dir, duration=self.test_result.duration, testcase_nums=self.test_result.case_nums, error_nums=self.test_result.get_status_nums(TestStatus.ERRORED), failure_nums=self.test_result.get_status_nums(TestStatus.FAILED), skipped_nums=self.test_result.get_status_nums(TestStatus.SKIPPED), test_suites=[], ) for test_suite in self.test_result.suites: result["test_suites"].append(self.test_suite_to_json(test_suite)) return result def test_suite_to_json(self, test_suite): result = dict( env_name=test_suite.env_name, test_name=test_suite.test_name, test_dir=test_suite.test_dir, status=test_suite.status.name, duration=test_suite.duration, timestamp=( datetime.datetime.fromtimestamp(test_suite.timestamp).strftime( "%Y-%m-%dT%H:%M:%S" ) if test_suite.timestamp else None ), testcase_nums=len(test_suite.cases), error_nums=test_suite.get_status_nums(TestStatus.ERRORED), failure_nums=test_suite.get_status_nums(TestStatus.FAILED), skipped_nums=test_suite.get_status_nums(TestStatus.SKIPPED), test_cases=[], ) for test_case in test_suite.cases: result["test_cases"].append(self.test_case_to_json(test_case)) return result @staticmethod def test_case_to_json(test_case): result = dict( name=test_case.name, status=test_case.status.name, message=test_case.message, stdout=test_case.stdout, duration=test_case.duration, exception=None, source=None, ) if test_case.exception: result["exception"] = "%s: %s" % ( test_case.exception.__class__.__name__, test_case.exception, ) if test_case.source: result["source"] = dict( file=test_case.source.filename, line=test_case.source.line ) return result ================================================ FILE: platformio/test/reports/junit.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import os import xml.etree.ElementTree as ET import click from platformio import __version__ from platformio.test.reports.base import TestReportBase from platformio.test.result import TestStatus class JunitTestReport(TestReportBase): def generate(self, output_path, verbose=False): if os.path.isdir(output_path): output_path = os.path.join( output_path, "pio-test-report-%s-%s-junit.xml" % ( os.path.basename(self.test_result.project_dir), datetime.datetime.now().strftime("%Y%m%d%H%M%S"), ), ) with open(output_path, mode="wb") as fp: self.build_xml_tree().write(fp, encoding="utf8") if verbose: click.secho(f"Saved JUnit report to the {output_path}", fg="green") def build_xml_tree(self): root = ET.Element("testsuites") root.set("name", self.test_result.project_dir) root.set("platformio_version", __version__) root.set("tests", str(self.test_result.case_nums)) root.set("errors", str(self.test_result.get_status_nums(TestStatus.ERRORED))) root.set("failures", str(self.test_result.get_status_nums(TestStatus.FAILED))) root.set("time", str(self.test_result.duration)) for suite in self.test_result.suites: root.append(self.build_testsuite_node(suite)) return ET.ElementTree(root) def build_testsuite_node(self, test_suite): element = ET.Element("testsuite") element.set("name", f"{test_suite.env_name}:{test_suite.test_name}") element.set("tests", str(len(test_suite.cases))) element.set("errors", str(test_suite.get_status_nums(TestStatus.ERRORED))) element.set("failures", str(test_suite.get_status_nums(TestStatus.FAILED))) element.set("skipped", str(test_suite.get_status_nums(TestStatus.SKIPPED))) element.set("time", str(test_suite.duration)) if test_suite.timestamp: element.set( "timestamp", datetime.datetime.fromtimestamp(test_suite.timestamp).strftime( "%Y-%m-%dT%H:%M:%S" ), ) for test_case in test_suite.cases: element.append(self.build_testcase_node(test_case)) return element def build_testcase_node(self, test_case): element = ET.Element("testcase") element.set("name", str(test_case.name)) element.set("time", str(test_case.duration)) element.set("status", str(test_case.status.name)) if test_case.source: element.set("file", test_case.source.filename) element.set("line", str(test_case.source.line)) if test_case.status == TestStatus.SKIPPED: element.append(ET.Element("skipped")) elif test_case.status == TestStatus.ERRORED: element.append(self.build_testcase_error_node(test_case)) elif test_case.status == TestStatus.FAILED: element.append(self.build_testcase_failure_node(test_case)) return element @staticmethod def build_testcase_error_node(test_case): element = ET.Element("error") element.set("type", test_case.exception.__class__.__name__) element.set("message", str(test_case.exception)) if test_case.stdout: element.text = test_case.stdout return element @staticmethod def build_testcase_failure_node(test_case): element = ET.Element("failure") if test_case.message: element.set("message", test_case.message) if test_case.stdout: element.text = test_case.stdout return element ================================================ FILE: platformio/test/reports/stdout.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from tabulate import tabulate from platformio import util from platformio.test.reports.base import TestReportBase from platformio.test.result import TestStatus class StdoutTestReport(TestReportBase): def generate(self, verbose=False): # pylint: disable=arguments-differ click.echo() tabular_data = [] failed_nums = self.test_result.get_status_nums(TestStatus.FAILED) skipped_nums = self.test_result.get_status_nums(TestStatus.SKIPPED) is_error = failed_nums > 0 or self.test_result.is_errored for test_suite in self.test_result.suites: if not verbose and test_suite.status == TestStatus.SKIPPED: continue status_str = test_suite.status.name if test_suite.status in (TestStatus.FAILED, TestStatus.ERRORED): status_str = click.style(status_str, fg="red") elif test_suite.status == TestStatus.PASSED: status_str = click.style(status_str, fg="green") tabular_data.append( ( click.style(test_suite.env_name, fg="cyan"), test_suite.test_name, status_str, util.humanize_duration_time(test_suite.duration or None), ) ) if tabular_data: util.print_labeled_bar( "SUMMARY", is_error=is_error, fg="red" if is_error else "green", ) click.echo( tabulate( tabular_data, headers=[ click.style(s, bold=True) for s in ("Environment", "Test", "Status", "Duration") ], ), err=is_error, ) if failed_nums: self.print_failed_test_cases() util.print_labeled_bar( "%d test cases: %s%s%d succeeded in %s" % ( self.test_result.case_nums, ("%d failed, " % failed_nums) if failed_nums else "", ("%d skipped, " % skipped_nums) if skipped_nums else "", self.test_result.get_status_nums(TestStatus.PASSED), util.humanize_duration_time(self.test_result.duration), ), is_error=is_error, fg="red" if is_error else "green", ) def print_failed_test_cases(self): click.echo() for test_suite in self.test_result.suites: if test_suite.status != TestStatus.FAILED: continue util.print_labeled_bar( click.style( f"{test_suite.env_name}:{test_suite.test_name}", bold=True, fg="red" ), is_error=True, sep="_", ) for test_case in test_suite.cases: if test_case.status != TestStatus.FAILED: continue click.echo((test_case.stdout or "").strip()) click.echo() ================================================ FILE: platformio/test/result.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import enum import functools import operator import time import click class TestStatus(enum.Enum): PASSED = enum.auto() FAILED = enum.auto() SKIPPED = enum.auto() WARNED = enum.auto() ERRORED = enum.auto() @classmethod def from_string(cls, value: str): value = value.lower() if value.startswith(("failed", "fail")): return cls.FAILED if value.startswith(("passed", "pass", "success", "ok")): return cls.PASSED if value.startswith(("skipped", "skip", "ignore", "ignored")): return cls.SKIPPED if value.startswith("WARNING"): return cls.WARNED raise ValueError(f"Unknown test status `{value}`") def to_ansi_color(self): if self == TestStatus.FAILED: return "red" if self == TestStatus.PASSED: return "green" return "yellow" class TestCaseSource: def __init__(self, filename, line=None): self.filename = filename self.line = line class TestCase: def __init__( # pylint: disable=too-many-arguments,too-many-positional-arguments self, name, status, message=None, stdout=None, source=None, duration=0, exception=None, ): assert isinstance(status, TestStatus) if status == TestStatus.ERRORED: assert isinstance(exception, Exception) self.name = name.strip() self.status = status self.message = message self.stdout = stdout self.source = source self.duration = duration self.exception = exception def humanize(self): parts = [] if self.source: parts.append("%s:%d: " % (self.source.filename, self.source.line)) parts.append(self.name) if self.message: parts.append(": " + self.message) parts.extend( [ "\t", "[%s]" % click.style(self.status.name, fg=self.status.to_ansi_color()), ] ) return "".join(parts) class TestSuite: def __init__(self, env_name, test_name, finished=False, test_dir=None): self.env_name = env_name self.test_name = test_name self.test_dir = test_dir self.timestamp = 0 self.duration = 0 self._cases = [] self._finished = finished @property def cases(self): return self._cases @property def status(self): for s in (TestStatus.ERRORED, TestStatus.FAILED): if self.get_status_nums(s): return s if self._cases and any(c.status == TestStatus.PASSED for c in self._cases): return TestStatus.PASSED return TestStatus.SKIPPED def get_status_nums(self, status): return len([True for c in self._cases if c.status == status]) def add_case(self, case: TestCase): assert isinstance(case, TestCase) self._cases.append(case) def is_finished(self): return self._finished def on_start(self): self.timestamp = time.time() def on_finish(self): if self.is_finished(): return self._finished = True self.duration = time.time() - self.timestamp class TestResult: def __init__(self, project_dir): self.project_dir = project_dir self._suites = [] @property def suites(self): return self._suites def add_suite(self, suite): assert isinstance(suite, TestSuite) self._suites.append(suite) @property def duration(self): return functools.reduce(operator.add, [s.duration for s in self._suites]) @property def case_nums(self): return functools.reduce(operator.add, [len(s.cases) for s in self._suites]) @property def is_errored(self): return any(s.status == TestStatus.ERRORED for s in self._suites) def get_status_nums(self, status): return functools.reduce( operator.add, [s.get_status_nums(status) for s in self._suites] ) ================================================ FILE: platformio/test/runners/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/test/runners/base.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.exception import ReturnErrorCode from platformio.platform.factory import PlatformFactory from platformio.test.exception import UnitTestSuiteError from platformio.test.result import TestCase, TestStatus from platformio.test.runners.readers.native import NativeTestOutputReader from platformio.test.runners.readers.serial import SerialTestOutputReader CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name" class TestRunnerOptions: # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=too-many-arguments,too-many-positional-arguments self, verbose=0, without_building=False, without_uploading=False, without_testing=False, without_debugging=True, upload_port=None, test_port=None, no_reset=False, monitor_rts=None, monitor_dtr=None, program_args=None, ): self.verbose = verbose self.without_building = without_building self.without_uploading = without_uploading self.without_testing = without_testing self.without_debugging = without_debugging self.upload_port = upload_port self.test_port = test_port self.no_reset = no_reset self.monitor_rts = monitor_rts self.monitor_dtr = monitor_dtr self.program_args = program_args class TestRunnerBase: NAME = None EXTRA_LIB_DEPS = None TESTCASE_PARSE_RE = None def __init__(self, test_suite, project_config, options=None): self.test_suite = test_suite self.options = options self.project_config = project_config self.platform = PlatformFactory.from_env( self.test_suite.env_name, autoinstall=True, ) self.cmd_ctx = None self._testing_output_buffer = "" @property def name(self): return self.__class__.__name__.replace("TestRunner", "").lower() def get_test_speed(self): return int( self.project_config.get(f"env:{self.test_suite.env_name}", "test_speed") ) def get_test_port(self): return self.options.test_port or self.project_config.get( f"env:{self.test_suite.env_name}", "test_port" ) def start(self, cmd_ctx): # setup command context self.cmd_ctx = cmd_ctx self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True if self.test_suite.test_name != "*": self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_suite.test_name self.test_suite.on_start() try: self.setup() for stage in ("building", "uploading", "testing"): getattr(self, f"stage_{stage}")() if self.options.verbose: click.echo() except Exception as exc: # pylint: disable=broad-except click.secho(str(exc), fg="red", err=True) self.test_suite.add_case( TestCase( name=f"{self.test_suite.env_name}:{self.test_suite.test_name}", status=TestStatus.ERRORED, exception=exc, ) ) finally: self.test_suite.on_finish() self.teardown() def setup(self): pass def stage_building(self): if self.options.without_building: return None # run "building" once at the "uploading" stage for the embedded target if not self.options.without_uploading and self.platform.is_embedded(): return None click.secho("Building...", bold=True) targets = ["__test"] if not self.options.without_debugging: targets.append("__debug") if self.platform.is_embedded(): targets.append("checkprogsize") try: return self.run_project_targets(targets) except ReturnErrorCode as exc: raise UnitTestSuiteError( "Building stage has failed, see errors above. " "Use `pio test -vvv` option to enable verbose output." ) from exc def stage_uploading(self): is_embedded = self.platform.is_embedded() if self.options.without_uploading or not is_embedded: return None click.secho( "Building & Uploading..." if is_embedded else "Uploading...", bold=True ) targets = ["upload"] if self.options.without_building: targets.append("nobuild") else: targets.append("__test") if not self.options.without_debugging: targets.append("__debug") try: return self.run_project_targets(targets) except ReturnErrorCode as exc: raise UnitTestSuiteError( "Uploading stage has failed, see errors above. " "Use `pio test -vvv` option to enable verbose output." ) from exc def stage_testing(self): if self.options.without_testing: return None click.secho("Testing...", bold=True) test_port = self.get_test_port() native_conds = [ not self.platform.is_embedded() and (not test_port or "://" not in test_port), self.project_config.get( f"env:{self.test_suite.env_name}", "test_testing_command" ), ] reader = ( NativeTestOutputReader(self) if any(native_conds) else SerialTestOutputReader(self) ) return reader.begin() def teardown(self): pass def run_project_targets(self, targets): # pylint: disable=import-outside-toplevel from platformio.run.cli import cli as run_cmd assert self.cmd_ctx return self.cmd_ctx.invoke( run_cmd, project_conf=self.project_config.path, upload_port=self.options.upload_port, verbose=self.options.verbose > 2, silent=self.options.verbose < 2, environment=[self.test_suite.env_name], disable_auto_clean="nobuild" in targets, target=targets, ) def configure_build_env(self, env): """ Configure SCons build environment Called in "builder/tools/piotest" tool """ return env def on_testing_data_output(self, data): if isinstance(data, bytes): data = data.decode("utf8", "ignore") self._testing_output_buffer += data self._testing_output_buffer = self._testing_output_buffer.replace("\r", "") while "\n" in self._testing_output_buffer: nl_pos = self._testing_output_buffer.index("\n") line = self._testing_output_buffer[: nl_pos + 1] self._testing_output_buffer = self._testing_output_buffer[nl_pos + 1 :] self.on_testing_line_output(line) def on_testing_line_output(self, line): click.echo(line, nl=False) ================================================ FILE: platformio/test/runners/doctest.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import click from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase class DoctestTestCaseParser: def __init__(self): self._tmp_tc = None self._name_tokens = [] def parse(self, line): if self.is_divider(line): return self._on_divider() if not self._tmp_tc or line.strip().startswith("[doctest]"): return None self._tmp_tc.stdout += line line = line.strip() # source if not self._tmp_tc.source and line: self._tmp_tc.source = self.parse_source(line) return None # name if not self._tmp_tc.name: if line: self._name_tokens.append(line) return None self._tmp_tc.name = self.parse_name(self._name_tokens) return None if self._tmp_tc.status != TestStatus.FAILED: self._parse_assert(line) return None @staticmethod def is_divider(line): line = line.strip() return line.startswith("===") and line.endswith("===") def _on_divider(self): test_case = None if self._tmp_tc: test_case = TestCase( name=self._tmp_tc.name.strip(), status=self._tmp_tc.status, message=(self._tmp_tc.message or "").strip() or None, source=self._tmp_tc.source, stdout=self._tmp_tc.stdout.strip(), ) self._tmp_tc = TestCase("", TestStatus.PASSED, stdout="") self._name_tokens = [] return test_case @staticmethod def parse_source(line): if not line.endswith(":"): return None filename, line = line[:-1].rsplit(":", 1) return TestCaseSource(filename, int(line)) @staticmethod def parse_name(tokens): cleaned_tokens = [] for token in tokens: if token.startswith("TEST ") and ":" in token: token = token[token.index(":") + 1 :] cleaned_tokens.append(token.strip()) return "/".join(cleaned_tokens) def _parse_assert(self, line): status_tokens = [ (TestStatus.FAILED, "ERROR"), (TestStatus.FAILED, "FATAL ERROR"), (TestStatus.WARNED, "WARNING"), ] for status, token in status_tokens: index = line.find(": %s:" % token) if index == -1: continue self._tmp_tc.status = status self._tmp_tc.message = line[index + len(token) + 3 :].strip() or None class DoctestTestRunner(TestRunnerBase): EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.12"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._tc_parser = DoctestTestCaseParser() def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) test_case = self._tc_parser.parse(line) if test_case: self.test_suite.add_case(test_case) if not self.options.verbose: click.echo(test_case.humanize()) if "[doctest] Status:" in line: self.test_suite.on_finish() ================================================ FILE: platformio/test/runners/factory.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import importlib import os import re from platformio.compat import load_python_module from platformio.exception import UserSideException from platformio.project.config import ProjectConfig from platformio.test.result import TestSuite from platformio.test.runners.base import TestRunnerBase, TestRunnerOptions class TestRunnerFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) return "%sTestRunner" % name.lower().capitalize() @classmethod def new(cls, test_suite, project_config, options=None) -> TestRunnerBase: assert isinstance(test_suite, TestSuite) assert isinstance(project_config, ProjectConfig) if options: assert isinstance(options, TestRunnerOptions) test_framework = project_config.get( f"env:{test_suite.env_name}", "test_framework" ) module_name = f"platformio.test.runners.{test_framework}" runner_cls = None if test_framework == "custom": test_dir = project_config.get("platformio", "test_dir") custom_runner_path = os.path.join(test_dir, "test_custom_runner.py") test_name = test_suite.test_name if test_suite.test_name != "*" else None while test_name: if os.path.isfile( os.path.join(test_dir, test_name, "test_custom_runner.py") ): custom_runner_path = os.path.join( test_dir, test_name, "test_custom_runner.py" ) break test_name = os.path.dirname(test_name) # parent dir try: mod = load_python_module(module_name, custom_runner_path) except (FileNotFoundError, ImportError) as exc: raise UserSideException( "Could not find custom test runner " f"by this path -> {custom_runner_path}" ) from exc else: mod = importlib.import_module(module_name) runner_cls = getattr(mod, cls.get_clsname(test_framework)) return runner_cls(test_suite, project_config, options) ================================================ FILE: platformio/test/runners/googletest.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import click from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase class GoogletestTestCaseParser: # Examples: # [ RUN ] FooTest.Bar # ... # [ FAILED ] FooTest.Bar (0 ms) STATUS__NAME_RE = r"^\[\s+(?P[A-Z]+)\s+\]\s+(?P[^\(\s]+)" # Examples: # [ RUN ] FooTest.Bar # test/test_gtest/test_main.cpp:26: Failure # Y:\core\examples\unit-testing\googletest\test\test_gtest\test_main.cpp:26: Failure SOURCE_MESSAGE_RE = r"^(?P.+):(?P\d+):(?P.*)$" def __init__(self): self._tmp_tc = None def parse(self, line): if self._tmp_tc: self._tmp_tc.stdout += line return self._parse_test_case(line) def _parse_test_case(self, line): status, name = self._parse_status_and_name(line) if status == "RUN": self._tmp_tc = TestCase(name, TestStatus.PASSED, stdout=line) return None if not status or not self._tmp_tc: return None source, message = self._parse_source_and_message(self._tmp_tc.stdout) test_case = TestCase( name=self._tmp_tc.name, status=TestStatus.from_string(status), message=message, source=source, stdout=self._tmp_tc.stdout.strip(), ) self._tmp_tc = None return test_case def _parse_status_and_name(self, line): result = (None, None) line = line.strip() if not line.startswith("["): return result match = re.search(self.STATUS__NAME_RE, line) if not match: return result return match.group("status"), match.group("name") def _parse_source_and_message(self, stdout): for line in stdout.split("\n"): line = line.strip() if not line: continue match = re.search(self.SOURCE_MESSAGE_RE, line) if not match: continue return ( TestCaseSource( match.group("source_file"), int(match.group("source_line")) ), (match.group("message") or "").strip() or None, ) return (None, None) class GoogletestTestRunner(TestRunnerBase): EXTRA_LIB_DEPS = ["google/googletest@^1.17.0"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._tc_parser = GoogletestTestCaseParser() os.environ["GTEST_COLOR"] = "no" # disable ANSI symbols def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) test_case = self._tc_parser.parse(line) if test_case: self.test_suite.add_case(test_case) if not self.options.verbose: click.echo(test_case.humanize()) if "Global test environment tear-down" in line: self.test_suite.on_finish() ================================================ FILE: platformio/test/runners/readers/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: platformio/test/runners/readers/native.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import asyncio import os import signal import subprocess import time from platformio.compat import ( IS_WINDOWS, aio_get_running_loop, get_filesystem_encoding, get_locale_encoding, ) from platformio.project.helpers import load_build_metadata from platformio.test.exception import UnitTestError EXITING_TIMEOUT = 5 # seconds class ProgramProcessProtocol(asyncio.SubprocessProtocol): def __init__(self, test_runner, exit_future): self.test_runner = test_runner self.exit_future = exit_future self._exit_timer = None def pipe_data_received(self, _, data): try: data = data.decode(get_locale_encoding() or get_filesystem_encoding()) except UnicodeDecodeError: data = data.decode("latin-1") self.test_runner.on_testing_data_output(data) if self.test_runner.test_suite.is_finished(): self._exit_timer = aio_get_running_loop().call_later( EXITING_TIMEOUT, self._stop_testing ) def process_exited(self): self._stop_testing() def _stop_testing(self): if not self.exit_future.done(): self.exit_future.set_result(True) if self._exit_timer: self._exit_timer.cancel() class NativeTestOutputReader: def __init__(self, test_runner): self.test_runner = test_runner self.aio_loop = ( asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.new_event_loop() ) asyncio.set_event_loop(self.aio_loop) def get_testing_command(self): custom_testing_command = self.test_runner.project_config.get( f"env:{self.test_runner.test_suite.env_name}", "test_testing_command" ) if custom_testing_command: return custom_testing_command build_dir = self.test_runner.project_config.get("platformio", "build_dir") cmd = [ os.path.join( build_dir, self.test_runner.test_suite.env_name, "program.exe" if IS_WINDOWS else "program", ) ] # if user changed PROGNAME if not os.path.exists(cmd[0]): build_data = load_build_metadata( os.getcwd(), self.test_runner.test_suite.env_name, cache=True ) if build_data: cmd[0] = build_data["prog_path"] if self.test_runner.options.program_args: cmd.extend(self.test_runner.options.program_args) return cmd async def gather_results(self): exit_future = asyncio.Future(loop=self.aio_loop) transport, _ = await self.aio_loop.subprocess_exec( lambda: ProgramProcessProtocol(self.test_runner, exit_future), *self.get_testing_command(), stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) await exit_future last_return_code = transport.get_returncode() transport.close() # wait until subprocess will be killed start = time.time() while ( start > (time.time() - EXITING_TIMEOUT) and transport.get_returncode() is None ): await asyncio.sleep(0.5) if last_return_code: self.raise_for_status(last_return_code) @staticmethod def raise_for_status(return_code): try: sig = signal.Signals(abs(return_code)) try: signal_description = signal.strsignal(sig) except AttributeError: signal_description = "" raise UnitTestError( f"Program received signal {sig.name} ({signal_description})" ) except ValueError as exc: raise UnitTestError("Program errored with %d code" % return_code) from exc def begin(self): try: self.aio_loop.run_until_complete(self.gather_results()) finally: self.aio_loop.run_until_complete(self.aio_loop.shutdown_asyncgens()) self.aio_loop.close() ================================================ FILE: platformio/test/runners/readers/serial.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from time import sleep import click import serial from platformio.device.finder import SerialPortFinder from platformio.exception import UserSideException class SerialTestOutputReader: SERIAL_TIMEOUT = 600 def __init__(self, test_runner): self.test_runner = test_runner def begin(self): click.echo( "If you don't see any output for the first 10 secs, " "please reset board (press reset button)" ) click.echo() try: ser = serial.serial_for_url( self.resolve_test_port(), do_not_open=True, baudrate=self.test_runner.get_test_speed(), timeout=self.SERIAL_TIMEOUT, ) ser.rts = self.test_runner.options.monitor_rts ser.dtr = self.test_runner.options.monitor_dtr ser.open() except serial.SerialException as exc: click.secho(str(exc), fg="red", err=True) return if not self.test_runner.options.no_reset: ser.flushInput() ser.setDTR(False) ser.setRTS(False) sleep(0.1) ser.setDTR(True) ser.setRTS(True) sleep(0.1) while not self.test_runner.test_suite.is_finished(): self.test_runner.on_testing_data_output(ser.read(ser.in_waiting or 1)) ser.close() def resolve_test_port(self): project_options = self.test_runner.project_config.items( env=self.test_runner.test_suite.env_name, as_dict=True ) port = SerialPortFinder( board_config=self.test_runner.platform.board_config( project_options["board"] ), upload_protocol=project_options.get("upload_protocol"), ensure_ready=True, verbose=self.test_runner.options.verbose, ).find(initial_port=self.test_runner.get_test_port()) if port: return port raise UserSideException( "Please specify `test_port` for environment or use " "global `--test-port` option." ) ================================================ FILE: platformio/test/runners/unity.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import string from pathlib import Path import click from platformio.test.exception import UnitTestSuiteError from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase from platformio.util import strip_ansi_codes class UnityTestRunner(TestRunnerBase): EXTRA_LIB_DEPS = ["throwtheswitch/Unity@^2.6.1"] # Examples: # test/test_foo.cpp:44:test_function_foo:FAIL: Expected 32 Was 33 # test/group/test_foo/test_main.cpp:5:test::dummy:FAIL: Expression Evaluated To FALSE TESTCASE_PARSE_RE = re.compile( r"(?P[^:]+):(?P\d+):(?P[^\s]+):" r"(?PPASS|IGNORE|FAIL)(:\s*(?P.+)$)?" ) UNITY_CONFIG_H = """ #ifndef UNITY_CONFIG_H #define UNITY_CONFIG_H #ifndef NULL #ifndef __cplusplus #define NULL (void*)0 #else #define NULL 0 #endif #endif #ifdef __cplusplus extern "C" { #endif void unityOutputStart(unsigned long); void unityOutputChar(unsigned int); void unityOutputFlush(void); void unityOutputComplete(void); #define UNITY_OUTPUT_START() unityOutputStart((unsigned long) $baudrate) #define UNITY_OUTPUT_CHAR(c) unityOutputChar(c) #define UNITY_OUTPUT_FLUSH() unityOutputFlush() #define UNITY_OUTPUT_COMPLETE() unityOutputComplete() #ifdef __cplusplus } #endif /* extern "C" */ #endif /* UNITY_CONFIG_H */ """ UNITY_CONFIG_C = """ #include #if !defined(UNITY_WEAK_ATTRIBUTE) && !defined(UNITY_WEAK_PRAGMA) # if defined(__GNUC__) || defined(__ghs__) /* __GNUC__ includes clang */ # if !(defined(__WIN32__) && defined(__clang__)) && !defined(__TMS470__) # define UNITY_WEAK_ATTRIBUTE __attribute__((weak)) # endif # endif #endif #ifdef __cplusplus extern "C" { #endif #ifdef UNITY_WEAK_ATTRIBUTE UNITY_WEAK_ATTRIBUTE void setUp(void) { } UNITY_WEAK_ATTRIBUTE void tearDown(void) { } UNITY_WEAK_ATTRIBUTE void suiteSetUp(void) { } UNITY_WEAK_ATTRIBUTE int suiteTearDown(int num_failures) { return num_failures; } #elif defined(UNITY_WEAK_PRAGMA) #pragma weak setUp void setUp(void) { } #pragma weak tearDown void tearDown(void) { } #pragma weak suiteSetUp void suiteSetUp(void) { } #pragma weak suiteTearDown int suiteTearDown(int num_failures) { return num_failures; } #endif #ifdef __cplusplus } #endif /* extern "C" */ $framework_config_code """ UNITY_FRAMEWORK_CONFIG = dict( native=dict( code=""" #include void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } """, language="c", ), arduino=dict( code=""" #include void unityOutputStart(unsigned long baudrate) { Serial.begin(baudrate); } void unityOutputChar(unsigned int c) { Serial.write(c); } void unityOutputFlush(void) { Serial.flush(); } void unityOutputComplete(void) { Serial.end(); } """, language="cpp", ), mbed=dict( code=""" #include #if MBED_MAJOR_VERSION == 6 UnbufferedSerial pc(USBTX, USBRX); #else RawSerial pc(USBTX, USBRX); #endif void unityOutputStart(unsigned long baudrate) { pc.baud(baudrate); } void unityOutputChar(unsigned int c) { #if MBED_MAJOR_VERSION == 6 pc.write(&c, 1); #else pc.putc(c); #endif } void unityOutputFlush(void) { } void unityOutputComplete(void) { } """, language="cpp", ), espidf=dict( code=""" #include void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } """, language="c", ), zephyr=dict( code=""" #include void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { printk("%c", c); } void unityOutputFlush(void) { } void unityOutputComplete(void) { } """, language="c", ), legacy_custom_transport=dict( code=""" #include void unityOutputStart(unsigned long baudrate) { unittest_uart_begin(); } void unityOutputChar(unsigned int c) { unittest_uart_putchar(c); } void unityOutputFlush(void) { unittest_uart_flush(); } void unityOutputComplete(void) { unittest_uart_end(); } """, language="cpp", ), ) def get_unity_framework_config(self): if not self.platform.is_embedded(): return self.UNITY_FRAMEWORK_CONFIG["native"] if ( self.project_config.get( f"env:{self.test_suite.env_name}", "test_transport", None ) == "custom" ): framework = "legacy_custom_transport" else: framework = ( self.project_config.get(f"env:{self.test_suite.env_name}", "framework") or [None] )[0] if framework and framework in self.UNITY_FRAMEWORK_CONFIG: return self.UNITY_FRAMEWORK_CONFIG[framework] raise UnitTestSuiteError( f"Could not find Unity configuration for the `{framework}` framework.\n" "Learn how to create a custom Unity configuration at" "https://docs.platformio.org/en/latest/advanced/" "unit-testing/frameworks/unity.html" ) def configure_build_env(self, env): env.Append(CPPDEFINES=["UNITY_INCLUDE_CONFIG_H"]) if self.custom_unity_config_exists(): return env env.Replace( UNITY_CONFIG_DIR=os.path.join("$BUILD_DIR", "unity_config"), BUILD_UNITY_CONFIG_DIR=os.path.join("$BUILD_DIR", "unity_config_build"), ) env.Prepend(CPPPATH=["$UNITY_CONFIG_DIR"]) self.generate_unity_extras(env.subst("$UNITY_CONFIG_DIR")) env.BuildSources("$BUILD_UNITY_CONFIG_DIR", "$UNITY_CONFIG_DIR") return env def custom_unity_config_exists(self): test_dir = self.project_config.get("platformio", "test_dir") config_fname = "unity_config.h" if os.path.isfile(os.path.join(test_dir, config_fname)): return True test_name = ( self.test_suite.test_name if self.test_suite.test_name != "*" else None ) while test_name: if os.path.isfile(os.path.join(test_dir, test_name, config_fname)): return True test_name = os.path.dirname(test_name) # parent dir return False def generate_unity_extras(self, dst_dir): dst_dir = Path(dst_dir) if not dst_dir.is_dir(): dst_dir.mkdir(parents=True) unity_h = dst_dir / "unity_config.h" if not unity_h.is_file(): unity_h.write_text( string.Template(self.UNITY_CONFIG_H) .substitute(baudrate=self.get_test_speed()) .strip() + "\n", encoding="utf8", ) framework_config = self.get_unity_framework_config() unity_c = dst_dir / ("unity_config.%s" % framework_config.get("language", "c")) if not unity_c.is_file(): unity_c.write_text( string.Template(self.UNITY_CONFIG_C) .substitute(framework_config_code=framework_config["code"]) .strip() + "\n", encoding="utf8", ) def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) line = strip_ansi_codes(line or "").strip() if not line: return test_case = self.parse_test_case(line) if test_case: self.test_suite.add_case(test_case) if not self.options.verbose: click.echo(test_case.humanize()) if all(s in line for s in ("Tests", "Failures", "Ignored")): self.test_suite.on_finish() def parse_test_case(self, line): if not self.TESTCASE_PARSE_RE: raise NotImplementedError() line = line.strip() if not line: return None match = self.TESTCASE_PARSE_RE.search(line) if not match: return None data = match.groupdict() source = None if "source_file" in data: source = TestCaseSource( filename=data["source_file"], line=int(data.get("source_line")) ) return TestCase( name=data.get("name").strip(), status=TestStatus.from_string(data.get("status")), message=(data.get("message") or "").strip() or None, stdout=line, source=source, ) ================================================ FILE: platformio/util.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import functools import math import os import platform import re import shutil import time import click from platformio import __version__ # pylint: disable=unused-import from platformio.device.list.util import list_serial_ports as get_serial_ports from platformio.fs import cd, load_json from platformio.proc import exec_command # pylint: enable=unused-import # also export list_serial_ports as get_serialports to be # backward compatibility with arduinosam versions 3.9.0 to 3.5.0 (and possibly others) get_serialports = get_serial_ports class memoized: def __init__(self, expire=0): expire = str(expire) if expire.isdigit(): expire = "%ss" % int((int(expire) / 1000)) tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} assert expire.endswith(tuple(tdmap)) self.expire = int(tdmap[expire[-1]] * int(expire[:-1])) self.cache = {} def __call__(self, func): @functools.wraps(func) def wrapper(*args, **kwargs): key = str(args) + str(kwargs) if key not in self.cache or ( self.expire > 0 and self.cache[key][0] < time.time() - self.expire ): self.cache[key] = (time.time(), func(*args, **kwargs)) return self.cache[key][1] wrapper.reset = self._reset return wrapper def _reset(self): self.cache.clear() class throttle: def __init__(self, threshold): self.threshold = threshold # milliseconds self.last = 0 def __call__(self, func): @functools.wraps(func) def wrapper(*args, **kwargs): diff = int(round((time.time() - self.last) * 1000)) if diff < self.threshold: time.sleep((self.threshold - diff) * 0.001) self.last = time.time() return func(*args, **kwargs) return wrapper # Retry: Begin class RetryException(Exception): pass class RetryNextException(RetryException): pass class RetryStopException(RetryException): pass class retry: RetryNextException = RetryNextException RetryStopException = RetryStopException def __init__(self, timeout=0, step=0.25): self.timeout = timeout self.step = step def __call__(self, func): @functools.wraps(func) def wrapper(*args, **kwargs): elapsed = 0 while True: try: return func(*args, **kwargs) except self.RetryNextException: pass if elapsed >= self.timeout: raise self.RetryStopException() elapsed += self.step time.sleep(self.step) return wrapper # Retry: End def singleton(cls): """From PEP-318 http://www.python.org/dev/peps/pep-0318/#examples""" _instances = {} def get_instance(*args, **kwargs): if cls not in _instances: _instances[cls] = cls(*args, **kwargs) return _instances[cls] return get_instance def get_systype(): # allow manual override, eg. for # windows on arm64 systems with emulated x86 if "PLATFORMIO_SYSTEM_TYPE" in os.environ: return os.environ.get("PLATFORMIO_SYSTEM_TYPE") system = platform.system().lower() arch = platform.machine().lower() if system == "windows": if not arch: # issue #4353 arch = "x86_" + platform.architecture()[0] if "x86" in arch: arch = "amd64" if "64" in arch else "x86" if arch == "aarch64" and platform.architecture()[0] == "32bit": arch = "armv7l" return "%s_%s" % (system, arch) if arch else system def pioversion_to_intstr(): """Legacy for framework-zephyr/scripts/platformio/platformio-build-pre.py""" vermatch = re.match(r"^([\d\.]+)", __version__) assert vermatch return [int(i) for i in vermatch.group(1).split(".")[:3]] def items_to_list(items): if isinstance(items, list): return items return [i.strip() for i in items.split(",") if i.strip()] def items_in_list(needle, haystack): needle = items_to_list(needle) haystack = items_to_list(haystack) if "*" in needle or "*" in haystack: return True return set(needle) & set(haystack) def parse_datetime(datestr): assert "T" in datestr and "Z" in datestr return datetime.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") def merge_dicts(d1, d2, path=None): if path is None: path = [] for key in d2: if key in d1 and isinstance(d1[key], dict) and isinstance(d2[key], dict): merge_dicts(d1[key], d2[key], path + [str(key)]) else: d1[key] = d2[key] return d1 def print_labeled_bar(label, is_error=False, fg=None, sep="="): terminal_width = shutil.get_terminal_size().columns width = len(click.unstyle(label)) half_line = sep * int((terminal_width - width - 2) / 2) click.secho("%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) def humanize_duration_time(duration): if duration is None: return duration duration = duration * 1000 tokens = [] for multiplier in (3600000, 60000, 1000, 1): fraction = math.floor(duration / multiplier) tokens.append(int(round(duration) if multiplier == 1 else fraction)) duration -= fraction * multiplier return "{:02d}:{:02d}:{:02d}.{:03d}".format(*tokens) def strip_ansi_codes(text): # pylint: disable=protected-access return click._compat.strip_ansi(text) ================================================ FILE: scripts/docspregen.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import functools import os import sys import tempfile from urllib.parse import ParseResult, urlparse, urlunparse sys.path.append("..") import click # noqa: E402 from platformio import fs # noqa: E402 from platformio.package.manager.platform import PlatformPackageManager # noqa: E402 from platformio.platform.factory import PlatformFactory # noqa: E402 RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ SKIP_DEBUG_TOOLS = ["esp-bridge", "esp-builtin", "dfu"] STATIC_FRAMEWORK_DATA = { "arduino": { "title": "Arduino", "description": ( "Arduino Wiring-based Framework allows writing cross-platform software " "to control devices attached to a wide range of Arduino boards to " "create all kinds of creative coding, interactive objects, spaces " "or physical experiences." ), }, "cmsis": { "title": "CMSIS", "description": ( "Vendor-independent hardware abstraction layer for the Cortex-M processor series" ), }, "freertos": { "title": "FreeRTOS", "description": ( "FreeRTOS is a real-time operating system kernel for embedded devices " "that has been ported to 40 microcontroller platforms." ), }, } DOCS_ROOT_DIR = os.path.realpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "docs") ) REGCLIENT = PlatformPackageManager().get_registry_client_instance() def reg_package_url(type_, owner, name): if type_ == "library": type_ = "libraries" else: type_ += "s" return f"https://registry.platformio.org/{type_}/{owner}/{name}" def campaign_url(url, source="platformio.org", medium="docs"): data = urlparse(url) query = data.query if query: query += "&" query += "utm_source=%s&utm_medium=%s" % (source, medium) return urlunparse( ParseResult( data.scheme, data.netloc, data.path, data.params, query, data.fragment ) ) def install_platforms(): print("Installing platforms...") page = 1 pm = PlatformPackageManager() while True: result = REGCLIENT.list_packages(qualifiers=dict(types=["platform"]), page=page) for item in result["items"]: spec = "%s/%s" % (item["owner"]["username"], item["name"]) skip_conds = [ item["owner"]["username"] != "platformio", item["tier"] == "community", ] if all(skip_conds): click.secho("Skip community platform: %s" % spec, fg="yellow") continue pm.install(spec, skip_dependencies=True) page += 1 if not result["items"] or result["page"] * result["limit"] >= result["total"]: break @functools.cache def get_frameworks(): items = {} for platform in PlatformPackageManager().get_installed(): p = PlatformFactory.new(platform) for name, options in (p.frameworks or {}).items(): if name in items: continue if name in STATIC_FRAMEWORK_DATA: items[name] = dict( name=name, title=STATIC_FRAMEWORK_DATA[name]["title"], description=STATIC_FRAMEWORK_DATA[name]["description"], ) continue title = options.get("title") or name.title() description = options.get("description") if "package" in options: regdata = REGCLIENT.get_package( "tool", p.packages[options["package"]].get("owner", "platformio"), options["package"], ) title = regdata["title"] or title description = regdata["description"] items[name] = dict(name=name, title=title, description=description) return sorted(items.values(), key=lambda item: item["name"]) def is_compat_platform_and_framework(platform, framework): p = PlatformFactory.new(platform) return framework in (p.frameworks or {}).keys() def generate_boards_table(boards, skip_columns=None): columns = [ ("Name", ":ref:`board_{platform}_{id}`"), ("Platform", ":ref:`platform_{platform}`"), ("Debug", "{debug}"), ("MCU", "{mcu}"), ("Frequency", "{f_cpu}MHz"), ("Flash", "{rom}"), ("RAM", "{ram}"), ] lines = [] lines.append( """ .. list-table:: :header-rows: 1 """ ) # add header for name, template in columns: if skip_columns and name in skip_columns: continue prefix = " * - " if name == "Name" else " - " lines.append(prefix + name) for data in sorted(boards, key=lambda item: item["name"]): has_onboard_debug = data.get("debug") and any( t.get("onboard") for (_, t) in data["debug"]["tools"].items() ) debug = "No" if has_onboard_debug: debug = "On-board" elif data.get("debug"): debug = "External" variables = dict( id=data["id"], name=data["name"], platform=data["platform"], debug=debug, mcu=data["mcu"].upper(), f_cpu=int(data["fcpu"] / 1000000.0), ram=fs.humanize_file_size(data["ram"]), rom=fs.humanize_file_size(data["rom"]), ) for name, template in columns: if skip_columns and name in skip_columns: continue prefix = " * - " if name == "Name" else " - " lines.append(prefix + template.format(**variables)) if lines: lines.append("") return lines def generate_frameworks_contents(frameworks): if not frameworks: return [] lines = [] lines.append( """ Frameworks ---------- .. list-table:: :header-rows: 1 * - Name - Description""" ) known = set() for framework in get_frameworks(): known.add(framework["name"]) if framework["name"] not in frameworks: continue lines.append( """ * - :ref:`framework_{name}` - {description}""".format( **framework ) ) if set(frameworks) - known: click.secho("Unknown frameworks %s " % (set(frameworks) - known), fg="red") return lines def generate_platforms_contents(platforms): if not platforms: return [] lines = [] lines.append( """ Platforms --------- .. list-table:: :header-rows: 1 * - Name - Description""" ) for name in sorted(platforms): p = PlatformFactory.new(name) lines.append( """ * - :ref:`platform_{name}` - {description}""".format( name=p.name, description=p.description ) ) return lines def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): if not skip_board_columns: skip_board_columns = [] skip_board_columns.append("Debug") lines = [] onboard_debug = [ b for b in boards if b.get("debug") and any(t.get("onboard") for (_, t) in b["debug"]["tools"].items()) ] external_debug = [b for b in boards if b.get("debug") and b not in onboard_debug] if not onboard_debug and not external_debug: return lines lines.append( """ Debugging --------- :ref:`piodebug` - "1-click" solution for debugging with a zero configuration. .. contents:: :local: """ ) if extra_rst: lines.append(".. include:: %s" % extra_rst) lines.append( """ Tools & Debug Probes ~~~~~~~~~~~~~~~~~~~~ Supported debugging tools are listed in "Debug" column. For more detailed information, please scroll table by horizontal. You can switch between debugging :ref:`debugging_tools` using :ref:`projectconf_debug_tool` option in :ref:`projectconf`. .. warning:: You will need to install debug tool drivers depending on your system. Please click on compatible debug tool below for the further instructions. """ ) if onboard_debug: lines.append( """ On-Board Debug Tools ^^^^^^^^^^^^^^^^^^^^ Boards listed below have on-board debug probe and **ARE READY** for debugging! You do not need to use/buy external debug probe. """ ) lines.extend( generate_boards_table(onboard_debug, skip_columns=skip_board_columns) ) if external_debug: lines.append( """ External Debug Tools ^^^^^^^^^^^^^^^^^^^^ Boards listed below are compatible with :ref:`piodebug` but **DEPEND ON** external debug probe. They **ARE NOT READY** for debugging. Please click on board name for the further details. """ ) lines.extend( generate_boards_table(external_debug, skip_columns=skip_board_columns) ) return lines def generate_packages(platform, packages, is_embedded): if not packages: return lines = [] lines.append( """ Packages -------- """ ) lines.append( """.. list-table:: :header-rows: 1 * - Name - Description""" ) for name, options in dict(sorted(packages.items())).items(): if name == "toolchain-gccarmnoneeab": # aceinna typo fix name = name + "i" package = REGCLIENT.get_package( "tool", options.get("owner", "platformio"), name ) lines.append( """ * - `{name} <{url}>`__ - {description}""".format( name=package["name"], url=reg_package_url( "tool", package["owner"]["username"], package["name"] ), description=package["description"], ) ) if is_embedded: lines.append( """ .. warning:: **Linux Users**: * Install "udev" rules :ref:`platformio_udev_rules` * Raspberry Pi users, please read this article `Enable serial port on Raspberry Pi `__. """ ) if platform == "teensy": lines.append( """ **Windows Users:** Teensy programming uses only Windows built-in HID drivers. When Teensy is programmed to act as a USB Serial device, Windows XP, Vista, 7 and 8 require `this serial driver `_ is needed to access the COM port your program uses. No special driver installation is necessary on Windows 10. """ ) else: lines.append( """ **Windows Users:** Please check that you have a correctly installed USB driver from board manufacturer """ ) return "\n".join(lines) def generate_platform(pkg, rst_dir): owner = pkg.metadata.spec.owner name = pkg.metadata.name print("Processing platform: %s" % name) compatible_boards = [ board for board in PlatformPackageManager().get_installed_boards() if name == board["platform"] ] lines = [] lines.append(RST_COPYRIGHT) p = PlatformFactory.new(name) assert p.repository_url.endswith(".git") github_url = p.repository_url[:-4] registry_url = reg_package_url("platform", owner, name) lines.append(".. _platform_%s:" % name) lines.append("") lines.append(p.title) lines.append("=" * len(p.title)) lines.append("") lines.append(":Registry:") lines.append(" `%s <%s>`__" % (registry_url, registry_url)) lines.append(":Configuration:") lines.append(" :ref:`projectconf_env_platform` = ``%s/%s``" % (owner, name)) lines.append("") lines.append(p.description) lines.append( """ For more detailed information please visit `vendor site <%s>`_.""" % campaign_url(p.homepage) ) lines.append( """ .. contents:: Contents :local: :depth: 1 """ ) # # Extra # if os.path.isfile(os.path.join(rst_dir, "%s_extra.rst" % name)): lines.append(".. include:: %s_extra.rst" % p.name) # # Examples # lines.append( """ Examples -------- Examples are listed from `%s development platform repository <%s>`_: """ % (p.title, campaign_url("%s/tree/master/examples" % github_url)) ) examples_dir = os.path.join(p.get_dir(), "examples") if os.path.isdir(examples_dir): for eitem in os.listdir(examples_dir): example_dir = os.path.join(examples_dir, eitem) if not os.path.isdir(example_dir) or not os.listdir(example_dir): continue url = "%s/tree/master/examples/%s" % (github_url, eitem) lines.append("* `%s <%s>`_" % (eitem, campaign_url(url))) # # Debugging # if compatible_boards: lines.extend( generate_debug_contents( compatible_boards, skip_board_columns=["Platform"], extra_rst="%s_debug.rst" % name if os.path.isfile(os.path.join(rst_dir, "%s_debug.rst" % name)) else None, ) ) # # Development version of dev/platform # lines.append( """ Stable and upstream versions ---------------------------- You can switch between `stable releases <{github_url}/releases>`__ of {title} development platform and the latest upstream version using :ref:`projectconf_env_platform` option in :ref:`projectconf` as described below. Stable ~~~~~~ .. code-block:: ini ; Latest stable version, NOT recommended ; Pin the version as shown below [env:latest_stable] platform = {name} {board} ; Specific version [env:custom_stable] platform = {name}@x.y.z {board} Upstream ~~~~~~~~ .. code-block:: ini [env:upstream_develop] platform = {github_url}.git {board}""".format( name=p.name, title=p.title, github_url=github_url, board="board = ...\n" if p.is_embedded() else "", ) ) # # Packages # _packages_content = generate_packages(name, p.packages, p.is_embedded()) if _packages_content: lines.append(_packages_content) # # Frameworks # compatible_frameworks = [] for framework in get_frameworks(): if is_compat_platform_and_framework(name, framework["name"]): compatible_frameworks.append(framework["name"]) lines.extend(generate_frameworks_contents(compatible_frameworks)) # # Boards # if compatible_boards: vendors = {} for board in compatible_boards: if board["vendor"] not in vendors: vendors[board["vendor"]] = [] vendors[board["vendor"]].append(board) lines.append( """ Boards ------ .. note:: * You can list pre-configured boards by :ref:`cmd_boards` command * For more detailed ``board`` information please scroll the tables below by horizontally. """ ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) lines.extend(generate_boards_table(boards, skip_columns=["Platform"])) return "\n".join(lines) def update_platform_docs(): platforms_dir = os.path.join(DOCS_ROOT_DIR, "platforms") for pkg in PlatformPackageManager().get_installed(): rst_path = os.path.join(platforms_dir, "%s.rst" % pkg.metadata.name) with open(rst_path, "w") as f: f.write(generate_platform(pkg, platforms_dir)) def generate_framework(type_, framework, rst_dir=None): print("Processing framework: %s" % type_) compatible_platforms = [ pkg for pkg in PlatformPackageManager().get_installed() if is_compat_platform_and_framework(pkg.metadata.name, type_) ] compatible_boards = [ board for board in PlatformPackageManager().get_installed_boards() if type_ in board["frameworks"] ] lines = [] lines.append(RST_COPYRIGHT) lines.append(".. _framework_%s:" % type_) lines.append("") lines.append(framework["title"]) lines.append("=" * len(framework["title"])) lines.append("") lines.append(":Configuration:") lines.append(" :ref:`projectconf_env_framework` = ``%s``" % type_) lines.append("") lines.append(framework["description"]) lines.append( """ .. contents:: Contents :local: :depth: 1""" ) # Extra if os.path.isfile(os.path.join(rst_dir, "%s_extra.rst" % type_)): lines.append(".. include:: %s_extra.rst" % type_) if compatible_platforms: # Platforms lines.extend( generate_platforms_contents( [pkg.metadata.name for pkg in compatible_platforms] ) ) # examples lines.append( """ Examples -------- """ ) for pkg in compatible_platforms: p = PlatformFactory.new(pkg) lines.append( "* `%s for %s <%s>`_" % ( framework["title"], p.title, campaign_url("%s/tree/master/examples" % p.repository_url[:-4]), ) ) # # Debugging # if compatible_boards: lines.extend( generate_debug_contents( compatible_boards, extra_rst="%s_debug.rst" % type_ if os.path.isfile(os.path.join(rst_dir, "%s_debug.rst" % type_)) else None, ) ) # # Boards # if compatible_boards: vendors = {} for board in compatible_boards: if board["vendor"] not in vendors: vendors[board["vendor"]] = [] vendors[board["vendor"]].append(board) lines.append( """ Boards ------ .. note:: * You can list pre-configured boards by :ref:`cmd_boards` command * For more detailed ``board`` information please scroll the tables below by horizontally. """ ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) lines.extend(generate_boards_table(boards)) return "\n".join(lines) def update_framework_docs(): frameworks_dir = os.path.join(DOCS_ROOT_DIR, "frameworks") for framework in get_frameworks(): name = framework["name"] rst_path = os.path.join(frameworks_dir, "%s.rst" % name) with open(rst_path, "w") as f: f.write(generate_framework(name, framework, frameworks_dir)) def update_boards(): print("Updating boards...") lines = [] lines.append(RST_COPYRIGHT) lines.append(".. _boards:") lines.append("") lines.append("Boards") lines.append("======") lines.append( """ Rapid Embedded Development, Continuous and IDE integration in a few steps with PlatformIO thanks to built-in project generator for the most popular embedded boards and IDEs. .. note:: * You can list pre-configured boards by :ref:`cmd_boards` command * For more detailed ``board`` information please scroll tables below by horizontal. """ ) platforms = {} installed_boards = PlatformPackageManager().get_installed_boards() for data in installed_boards: platform = data["platform"] if platform in platforms: platforms[platform].append(data) else: platforms[platform] = [data] for platform, boards in sorted(platforms.items()): p = PlatformFactory.new(platform) lines.append(p.title) lines.append("-" * len(p.title)) lines.append( """ .. toctree:: :maxdepth: 1 """ ) for board in sorted(boards, key=lambda item: item["name"]): lines.append(" %s/%s" % (platform, board["id"])) lines.append("") emboards_rst = os.path.join(DOCS_ROOT_DIR, "boards", "index.rst") with open(emboards_rst, "w") as f: f.write("\n".join(lines)) # individual board page for data in installed_boards: rst_path = os.path.join( DOCS_ROOT_DIR, "boards", data["platform"], "%s.rst" % data["id"] ) if not os.path.isdir(os.path.dirname(rst_path)): os.makedirs(os.path.dirname(rst_path)) update_embedded_board(rst_path, data) def update_embedded_board(rst_path, board): platform = PlatformFactory.new(board["platform"]) board_config = platform.board_config(board["id"]) board_manifest_url = platform.repository_url assert board_manifest_url if board_manifest_url.endswith(".git"): board_manifest_url = board_manifest_url[:-4] board_manifest_url += "/blob/master/boards/%s.json" % board["id"] variables = dict( id=board["id"], name=board["name"], platform=board["platform"], platform_description=platform.description, url=campaign_url(board["url"]), mcu=board_config.get("build", {}).get("mcu", ""), mcu_upper=board["mcu"].upper(), f_cpu=board["fcpu"], f_cpu_mhz=int(int(board["fcpu"]) / 1000000), ram=fs.humanize_file_size(board["ram"]), rom=fs.humanize_file_size(board["rom"]), vendor=board["vendor"], board_manifest_url=board_manifest_url, upload_protocol=board_config.get("upload.protocol", ""), ) lines = [RST_COPYRIGHT] lines.append(".. _board_{platform}_{id}:".format(**variables)) lines.append("") lines.append(board["name"]) lines.append("=" * len(board["name"])) lines.append( """ .. contents:: Hardware -------- Platform :ref:`platform_{platform}`: {platform_description} .. list-table:: * - **Microcontroller** - {mcu_upper} * - **Frequency** - {f_cpu_mhz:d}MHz * - **Flash** - {rom} * - **RAM** - {ram} * - **Vendor** - `{vendor} <{url}>`__ """.format( **variables ) ) # # Configuration # lines.append( """ Configuration ------------- Please use ``{id}`` ID for :ref:`projectconf_env_board` option in :ref:`projectconf`: .. code-block:: ini [env:{id}] platform = {platform} board = {id} You can override default {name} settings per build environment using ``board_***`` option, where ``***`` is a JSON object path from board manifest `{id}.json <{board_manifest_url}>`_. For example, ``board_build.mcu``, ``board_build.f_cpu``, etc. .. code-block:: ini [env:{id}] platform = {platform} board = {id} ; change microcontroller board_build.mcu = {mcu} ; change MCU frequency board_build.f_cpu = {f_cpu}L """.format( **variables ) ) # # Uploading # upload_protocols = board_config.get("upload.protocols", []) if len(upload_protocols) > 1: lines.append( """ Uploading --------- %s supports the following uploading protocols: """ % board["name"] ) for protocol in sorted(upload_protocols): lines.append("* ``%s``" % protocol) lines.append( """ Default protocol is ``%s``""" % variables["upload_protocol"] ) lines.append( """ You can change upload protocol using :ref:`projectconf_upload_protocol` option: .. code-block:: ini [env:{id}] platform = {platform} board = {id} upload_protocol = {upload_protocol} """.format( **variables ) ) # # Debugging # lines.append("Debugging") lines.append("---------") if not board.get("debug"): lines.append( ":ref:`piodebug` currently does not support {name} board.".format( **variables ) ) else: default_debug_tool = board_config.get_debug_tool_name() has_onboard_debug = any( t.get("onboard") for (_, t) in board["debug"]["tools"].items() ) lines.append( """ :ref:`piodebug` - "1-click" solution for debugging with a zero configuration. .. warning:: You will need to install debug tool drivers depending on your system. Please click on compatible debug tool below for the further instructions and configuration information. You can switch between debugging :ref:`debugging_tools` using :ref:`projectconf_debug_tool` option in :ref:`projectconf`. """ ) if has_onboard_debug: lines.append( "{name} has on-board debug probe and **IS READY** for " "debugging. You don't need to use/buy external debug probe.".format( **variables ) ) else: lines.append( "{name} does not have on-board debug probe and **IS NOT " "READY** for debugging. You will need to use/buy one of " "external probe listed below.".format(**variables) ) lines.append( """ .. list-table:: :header-rows: 1 * - Compatible Tools - On-board - Default""" ) for tool_name, tool_data in sorted(board["debug"]["tools"].items()): lines.append( """ * - {tool} - {onboard} - {default}""".format( tool=f"``{tool_name}``" if tool_name in SKIP_DEBUG_TOOLS else f":ref:`debugging_tool_{tool_name}`", onboard="Yes" if tool_data.get("onboard") else "", default="Yes" if tool_name == default_debug_tool else "", ) ) if board["frameworks"]: lines.extend(generate_frameworks_contents(board["frameworks"])) with open(rst_path, "w") as f: f.write("\n".join(lines)) def update_debugging(): tool_to_platforms = {} tool_to_boards = {} vendors = {} platforms = [] frameworks = [] for data in PlatformPackageManager().get_installed_boards(): if not data.get("debug"): continue for tool in data["debug"]["tools"]: tool = str(tool) if tool not in tool_to_platforms: tool_to_platforms[tool] = [] tool_to_platforms[tool].append(data["platform"]) if tool not in tool_to_boards: tool_to_boards[tool] = [] tool_to_boards[tool].append(data["id"]) platforms.append(data["platform"]) frameworks.extend(data["frameworks"]) vendor = data["vendor"] if vendor in vendors: vendors[vendor].append(data) else: vendors[vendor] = [data] platforms = sorted(set(platforms)) frameworks = sorted(set(frameworks)) lines = [".. _debugging_platforms:"] lines.extend(generate_platforms_contents(platforms)) lines.extend(generate_frameworks_contents(frameworks)) # Boards lines.append( """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. """ ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) lines.extend(generate_boards_table(boards)) # save with open( os.path.join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+" ) as fp: content = fp.read() fp.seek(0) fp.truncate() fp.write( content[: content.index(".. _debugging_platforms:")] + "\n".join(lines) ) # Debug tools for tool, platforms in tool_to_platforms.items(): tool_path = os.path.join(DOCS_ROOT_DIR, "plus", "debug-tools", "%s.rst" % tool) if not os.path.isfile(tool_path): if tool in SKIP_DEBUG_TOOLS: click.secho("Skipped debug tool `%s`" % tool, fg="yellow") else: click.secho("Unknown debug tool `%s`" % tool, fg="red") continue platforms = sorted(set(platforms)) lines = [".. begin_platforms"] lines.extend(generate_platforms_contents(platforms)) tool_frameworks = [] for platform in platforms: for framework in frameworks: if is_compat_platform_and_framework(platform, framework): tool_frameworks.append(framework) lines.extend(generate_frameworks_contents(tool_frameworks)) lines.append( """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. """ ) lines.extend( generate_boards_table( [ b for b in PlatformPackageManager().get_installed_boards() if b["id"] in tool_to_boards[tool] ], skip_columns=None, ) ) with open(tool_path, "r+") as fp: content = fp.read() fp.seek(0) fp.truncate() fp.write(content[: content.index(".. begin_platforms")] + "\n".join(lines)) def update_project_examples(): platform_readme_tpl = """ # {title}: development platform for [PlatformIO](https://platformio.org) {description} * [Home](https://platformio.org/platforms/{name}) (home page in PlatformIO Registry) * [Documentation](https://docs.platformio.org/page/platforms/{name}.html) (advanced usage, packages, boards, frameworks, etc.) # Examples {examples} """ framework_readme_tpl = """ # {title}: framework for [PlatformIO](https://platformio.org) {description} * [Home](https://platformio.org/frameworks/{name}) (home page in PlatformIO Registry) * [Documentation](https://docs.platformio.org/page/frameworks/{name}.html) # Examples {examples} """ project_examples_dir = os.path.join(fs.get_source_dir(), "..", "examples") framework_examples_md_lines = {} embedded = [] desktop = [] for pkg in PlatformPackageManager().get_installed(): p = PlatformFactory.new(pkg) github_url = p.repository_url[:-4] # Platform README platform_examples_dir = os.path.join(p.get_dir(), "examples") examples_md_lines = [] if os.path.isdir(platform_examples_dir): for item in sorted(os.listdir(platform_examples_dir)): example_dir = os.path.join(platform_examples_dir, item) if not os.path.isdir(example_dir) or not os.listdir(example_dir): continue url = "%s/tree/master/examples/%s" % (github_url, item) examples_md_lines.append("* [%s](%s)" % (item, url)) readme_dir = os.path.join(project_examples_dir, "platforms", p.name) if not os.path.isdir(readme_dir): os.makedirs(readme_dir) with open(os.path.join(readme_dir, "README.md"), "w") as fp: fp.write( platform_readme_tpl.format( name=p.name, title=p.title, description=p.description, examples="\n".join(examples_md_lines), ) ) # Framework README for framework in get_frameworks(): if not is_compat_platform_and_framework(p.name, framework["name"]): continue if framework["name"] not in framework_examples_md_lines: framework_examples_md_lines[framework["name"]] = [] lines = [] lines.append("- [%s](%s)" % (p.title, github_url)) lines.extend(" %s" % line for line in examples_md_lines) lines.append("") framework_examples_md_lines[framework["name"]].extend(lines) # Root README line = "* [%s](%s)" % (p.title, "%s/tree/master/examples" % github_url) if p.is_embedded(): embedded.append(line) else: desktop.append(line) # Frameworks frameworks = [] for framework in get_frameworks(): if framework["name"] not in framework_examples_md_lines: continue readme_dir = os.path.join(project_examples_dir, "frameworks", framework["name"]) if not os.path.isdir(readme_dir): os.makedirs(readme_dir) with open(os.path.join(readme_dir, "README.md"), "w") as fp: fp.write( framework_readme_tpl.format( name=framework["name"], title=framework["title"], description=framework["description"], examples="\n".join(framework_examples_md_lines[framework["name"]]), ) ) url = campaign_url( "https://docs.platformio.org/en/latest/frameworks/%s.html#examples" % framework["name"], source="github", medium="examples", ) frameworks.append("* [%s](%s)" % (framework["title"], url)) with open(os.path.join(project_examples_dir, "README.md"), "w") as fp: fp.write( """# PlatformIO Project Examples - [Development platforms](#development-platforms): - [Embedded](#embedded) - [Desktop](#desktop) - [Frameworks](#frameworks) ## Development platforms ### Embedded %s ### Desktop %s ## Frameworks %s """ % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks)) ) def main(): with tempfile.TemporaryDirectory() as tmp_dir: print("Core directory: %s" % tmp_dir) os.environ["PLATFORMIO_CORE_DIR"] = tmp_dir install_platforms() update_platform_docs() update_framework_docs() update_boards() update_debugging() update_project_examples() if __name__ == "__main__": sys.exit(main()) ================================================ FILE: scripts/fixsymlink.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from os import chdir, getcwd, readlink, remove, symlink, walk from os.path import exists, islink, join, relpath from sys import exit as sys_exit def fix_symlink(root, fname, brokenlink): print(root, fname, brokenlink) prevcwd = getcwd() chdir(root) remove(fname) symlink(relpath(brokenlink, root), fname) chdir(prevcwd) def main(): for root, dirnames, filenames in walk("."): for f in filenames: path = join(root, f) if not islink(path) or exists(path): continue fix_symlink(root, f, readlink(path)) if __name__ == "__main__": sys_exit(main()) ================================================ FILE: scripts/install_devplatforms.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import subprocess import sys import click @click.command() @click.option("--desktop", is_flag=True, default=False) @click.option( "--names", envvar="PIO_INSTALL_DEVPLATFORM_NAMES", help="Install specified platform (split by comma)", ) @click.option( "--ownernames", envvar="PIO_INSTALL_DEVPLATFORM_OWNERNAMES", help="Filter by ownernames (split by comma)", ) def main(desktop, names, ownernames): platforms = json.loads( subprocess.check_output(["pio", "platform", "search", "--json-output"]).decode() ) names = [n.strip() for n in (names or "").split(",") if n.strip()] ownernames = [n.strip() for n in (ownernames or "").split(",") if n.strip()] for platform in platforms: skip = [ not desktop and platform["forDesktop"], names and platform["name"] not in names, ownernames and platform["ownername"] not in ownernames, ] if any(skip): continue subprocess.check_call( [ "pio", "pkg", "install", "--global", "--skip-dependencies", "--platform", "{ownername}/{name}".format(**platform), ] ) if __name__ == "__main__": sys.exit(main()) ================================================ FILE: setup.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from setuptools import find_packages, setup from platformio import ( __author__, __description__, __email__, __license__, __title__, __url__, __version__, ) from platformio.dependencies import get_pip_dependencies setup( name=__title__, version=__version__, description=__description__, long_description=open("README.rst").read(), author=__author__, author_email=__email__, url=__url__, license=__license__, install_requires=get_pip_dependencies(), python_requires=">=3.6", packages=find_packages(include=["platformio", "platformio.*"]), package_data={ "platformio": [ "assets/system/99-platformio-udev.rules", "project/integration/tpls/*/*.tpl", "project/integration/tpls/*/.*.tpl", # include hidden files "project/integration/tpls/*/.*/*.tpl", # include hidden folders "project/integration/tpls/*/*/*.tpl", # NetBeans "project/integration/tpls/*/*/*/*.tpl", # NetBeans ] }, entry_points={ "console_scripts": [ "platformio = platformio.__main__:main", "pio = platformio.__main__:main", "piodebuggdb = platformio.__main__:debug_gdb_main", ] }, classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Compilers", ], keywords=[ "iot", "embedded", "arduino", "mbed", "esp8266", "esp32", "fpga", "firmware", "continuous-integration", "cloud-ide", "avr", "arm", "ide", "unit-testing", "hardware", "verilog", "microcontroller", "debug", ], ) ================================================ FILE: tests/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: tests/commands/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: tests/commands/pkg/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: tests/commands/pkg/test_exec.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import pytest from platformio.package.commands.exec import package_exec_cmd from platformio.util import strip_ansi_codes def test_pkg_not_installed(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( package_exec_cmd, ["--", "openocd"], ) with pytest.raises( AssertionError, match=("Could not find a package with 'openocd' executable file"), ): validate_cliresult(result) def test_pkg_specified(clirunner, validate_cliresult, isolated_pio_core): # with install result = clirunner.invoke( package_exec_cmd, ["-p", "platformio/tool-openocd", "--", "openocd", "--version"], obj=dict(force_click_stream=True), ) validate_cliresult(result) output = strip_ansi_codes(result.output) assert "Tool Manager: Installing platformio/tool-openocd" in output assert "Open On-Chip Debugger" in output def test_unrecognized_options(clirunner, validate_cliresult, isolated_pio_core): # unrecognized option result = clirunner.invoke( package_exec_cmd, ["--", "openocd", "--test-unrecognized"], obj=dict(force_click_stream=True), ) with pytest.raises( AssertionError, match=(r"openocd: (unrecognized|unknown) option"), ): validate_cliresult(result) ================================================ FILE: tests/commands/pkg/test_install.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import os import pytest from platformio import fs from platformio.dependencies import get_core_dependencies from platformio.package.commands.install import package_install_cmd from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.project.config import ProjectConfig PROJECT_CONFIG_TPL = """ [env] platform = platformio/atmelavr@^3.4.0 lib_deps = milesburton/DallasTemperature@^4.0.4 https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip [env:baremetal] board = uno [env:devkit] framework = arduino board = attiny88 """ def pkgs_to_specs(pkgs): return [ PackageSpec(name=pkg.metadata.name, requirements=pkg.metadata.version) for pkg in pkgs ] def test_global_packages( clirunner, validate_cliresult, func_isolated_pio_core, get_pkg_latest_version, tmp_path, ): # libraries result = clirunner.invoke( package_install_cmd, [ "--global", "-l", "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#3.9.0", "--skip-dependencies", ], ) validate_cliresult(result) assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ PackageSpec("DallasTemperature@3.9.0+sha.964939d") ] # with dependencies result = clirunner.invoke( package_install_cmd, [ "--global", "-l", "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#3.9.0", "-l", "bblanchon/ArduinoJson@^5", ], ) validate_cliresult(result) assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec("DallasTemperature@3.9.0+sha.964939d"), PackageSpec("OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire")), ] # custom storage storage_dir = tmp_path / "custom_lib_storage" storage_dir.mkdir() result = clirunner.invoke( package_install_cmd, [ "--global", "--storage-dir", str(storage_dir), "-l", "bblanchon/ArduinoJson@^5", ], ) validate_cliresult(result) assert pkgs_to_specs(LibraryPackageManager(storage_dir).get_installed()) == [ PackageSpec("ArduinoJson@5.13.4") ] # tools result = clirunner.invoke( package_install_cmd, ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1.5.2"], ) validate_cliresult(result) assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.5.2") ] # platforms result = clirunner.invoke( package_install_cmd, ["--global", "-p", "platformio/atmelavr@^3.4.0", "--skip-dependencies"], ) validate_cliresult(result) assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@3.4.0") ] def test_skip_dependencies( clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "--skip-dependencies"], ) validate_cliresult(result) with fs.cd(str(project_dir)): installed_lib_pkgs = LibraryPackageManager( os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ PackageSpec( "DallasTemperature@%s" % get_pkg_latest_version("milesburton/DallasTemperature") ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), ] assert len(ToolPackageManager().get_installed()) == 1 # SCons def test_baremetal_project( clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "baremetal"], ) validate_cliresult(result) with fs.cd(str(project_dir)): installed_lib_pkgs = LibraryPackageManager( os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "baremetal") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ PackageSpec( "DallasTemperature@%s" % get_pkg_latest_version("milesburton/DallasTemperature") ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("tool-scons@%s" % get_core_dependencies()["tool-scons"][1:]), PackageSpec("toolchain-atmelavr@1.70300.191015"), ] def test_project( clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec( "DallasTemperature@%s" % get_pkg_latest_version("milesburton/DallasTemperature") ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.5.2"), PackageSpec("tool-scons@%s" % get_core_dependencies()["tool-scons"][1:]), PackageSpec("toolchain-atmelavr@1.70300.191015"), ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^4.0.4", "https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip", ] # test "Already up-to-date" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) assert "Already up-to-date" in result.output def test_private_lib_deps( clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" private_lib_dir = project_dir / "lib" / "private" private_lib_dir.mkdir(parents=True) (private_lib_dir / "library.json").write_text(""" { "name": "My Private Lib", "version": "1.0.0", "dependencies": { "bblanchon/ArduinoJson": "^5", "milesburton/DallasTemperature": "^4.0.4" } } """) (project_dir / "platformio.ini").write_text(""" [env:private] platform = native """) with fs.cd(str(project_dir)): config = ProjectConfig() # some deps were added by user manually result = clirunner.invoke( package_install_cmd, [ "-g", "--storage-dir", config.get("platformio", "lib_dir"), "-l", "paulstoffregen/OneWire@^2.3.5", ], ) validate_cliresult(result) # ensure all deps are installed result = clirunner.invoke(package_install_cmd) validate_cliresult(result) installed_private_pkgs = LibraryPackageManager( config.get("platformio", "lib_dir") ).get_installed() assert pkgs_to_specs(installed_private_pkgs) == [ PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), PackageSpec("My Private Lib@1.0.0"), ] installed_env_pkgs = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "private") ).get_installed() assert pkgs_to_specs(installed_env_pkgs) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec( "DallasTemperature@%s" % get_pkg_latest_version("milesburton/DallasTemperature") ), ] def test_remove_project_unused_libdeps( clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "baremetal"], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() storage_dir = os.path.join(config.get("platformio", "libdeps_dir"), "baremetal") lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec( "DallasTemperature@%s" % get_pkg_latest_version("milesburton/DallasTemperature") ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] # add new deps lib_deps = config.get("env:baremetal", "lib_deps") config.set("env:baremetal", "lib_deps", lib_deps + ["bblanchon/ArduinoJson@^5"]) config.save() result = clirunner.invoke( package_install_cmd, ["-e", "baremetal"], ) validate_cliresult(result) lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec( "DallasTemperature@%s" % get_pkg_latest_version("milesburton/DallasTemperature") ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] # manually remove from configuration file config.set("env:baremetal", "lib_deps", ["bblanchon/ArduinoJson@^5"]) config.save() result = clirunner.invoke( package_install_cmd, ["-e", "baremetal"], ) validate_cliresult(result) lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [PackageSpec("ArduinoJson@5.13.4")] def test_unknown_project_dependencies( clirunner, validate_cliresult, isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(""" [env:unknown_platform] platform = unknown_platform [env:unknown_lib_deps] lib_deps = SPI, platformio/unknown_library """) with fs.cd(str(project_dir)): result = clirunner.invoke( package_install_cmd, ["-e", "unknown_platform"], ) with pytest.raises( AssertionError, match=("Could not find the package with 'unknown_platform' requirements"), ): validate_cliresult(result) # unknown libraries result = clirunner.invoke( package_install_cmd, ["-e", "unknown_lib_deps"], ) with pytest.raises( AssertionError, match=( "Could not find the package with 'platformio/unknown_library' requirements" ), ): validate_cliresult(result) def test_custom_project_libraries( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) spec = "bblanchon/ArduinoJson@^5" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-l", spec], ) validate_cliresult(result) with fs.cd(str(project_dir)): # try again result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-l", spec], ) validate_cliresult(result) assert "already installed" in result.output # try again in the silent mode result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-l", spec, "--silent"], ) validate_cliresult(result) assert not result.output.strip() # check folders config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [PackageSpec("ArduinoJson@5.13.4")] # do not expect any platforms/tools assert not os.path.exists(config.get("platformio", "platforms_dir")) assert not os.path.exists(config.get("platformio", "packages_dir")) # check saved deps assert config.get("env:devkit", "lib_deps") == [ "bblanchon/ArduinoJson@^5", ] # install library without saving to config result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-l", "nanopb/Nanopb@^0.4.6", "--no-save"], ) validate_cliresult(result) config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec("Nanopb@0.4.91"), ] assert config.get("env:devkit", "lib_deps") == [ "bblanchon/ArduinoJson@^5", ] # unknown libraries result = clirunner.invoke( package_install_cmd, ["-l", "platformio/unknown_library"] ) with pytest.raises( AssertionError, match=( "Could not find the package with " "'platformio/unknown_library' requirements" ), ): validate_cliresult(result) def test_custom_project_tools( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) spec = "platformio/tool-openocd @ ^2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-t", spec], ) validate_cliresult(result) with fs.cd(str(project_dir)): # try again result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-t", spec], ) validate_cliresult(result) assert "already installed" in result.output # try again in the silent mode result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-t", spec, "--silent"], ) validate_cliresult(result) assert not result.output.strip() config = ProjectConfig() assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("tool-openocd@2.1100.211028") ] assert not LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ).get_installed() # do not expect any platforms assert not os.path.exists(config.get("platformio", "platforms_dir")) # check saved deps assert config.get("env:devkit", "platform_packages") == [ "platformio/tool-openocd@^2", ] # install tool without saving to config result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-t", "platformio/tool-esptoolpy@1.20310.0", "--no-save"], ) validate_cliresult(result) config = ProjectConfig() assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("tool-esptoolpy@1.20310.0"), PackageSpec("tool-openocd@2.1100.211028"), ] assert config.get("env:devkit", "platform_packages") == [ "platformio/tool-openocd@^2", ] # unknown tool result = clirunner.invoke( package_install_cmd, ["-t", "platformio/unknown_tool"] ) with pytest.raises( AssertionError, match=( "Could not find the package with " "'platformio/unknown_tool' requirements" ), ): validate_cliresult(result) def test_custom_project_platforms( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) spec = "atmelavr@^3.4.0" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-p", spec, "--skip-dependencies"], ) validate_cliresult(result) with fs.cd(str(project_dir)): # try again result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-p", spec, "--skip-dependencies"], ) validate_cliresult(result) assert "already installed" in result.output # try again in the silent mode result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-p", spec, "--silent", "--skip-dependencies"], ) validate_cliresult(result) assert not result.output.strip() config = ProjectConfig() assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@3.4.0") ] assert not LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ).get_installed() # do not expect any packages assert not os.path.exists(config.get("platformio", "packages_dir")) # unknown platform result = clirunner.invoke(package_install_cmd, ["-p", "unknown_platform"]) with pytest.raises( AssertionError, match="Could not find the package with 'unknown_platform' requirements", ): validate_cliresult(result) # incompatible board result = clirunner.invoke(package_install_cmd, ["-e", "devkit", "-p", "sifive"]) with pytest.raises( AssertionError, match="Unknown board ID", ): validate_cliresult(result) ================================================ FILE: tests/commands/pkg/test_list.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument from platformio.package.commands.install import package_install_cmd from platformio.package.commands.list import package_list_cmd PROJECT_CONFIG_TPL = """ [env] platform = platformio/atmelavr@^3.4.0 [env:baremetal] board = uno [env:devkit] framework = arduino board = attiny88 lib_deps = milesburton/DallasTemperature@^3.9.1 https://github.com/bblanchon/ArduinoJson.git#v6.19.0 """ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) # test all envs result = clirunner.invoke( package_list_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) assert all(token in result.output for token in ("baremetal", "devkit")) assert result.output.count("Platform atmelavr @ 3.4.0") == 2 assert ( result.output.count( "toolchain-atmelavr @ 1.70300.191015 (required: " "platformio/toolchain-atmelavr @ ~1.70300.0)" ) == 2 ) assert result.output.count("Libraries") == 1 assert ( "ArduinoJson @ 6.19.0+sha.9693fd2 (required: " "git+https://github.com/bblanchon/ArduinoJson.git#v6.19.0)" ) in result.output assert "OneWire @ 2" in result.output # test "baremetal" result = clirunner.invoke( package_list_cmd, ["-d", str(project_dir), "-e", "baremetal"], ) validate_cliresult(result) assert "Platform atmelavr @ 3" in result.output assert "Libraries" not in result.output # filter by "tool" package result = clirunner.invoke( package_list_cmd, ["-d", str(project_dir), "-t", "toolchain-atmelavr@~1.70300.0"], ) assert "framework-arduino" not in result.output assert "Libraries" not in result.output # list only libraries result = clirunner.invoke( package_list_cmd, ["-d", str(project_dir), "--only-libraries"], ) assert "Platform atmelavr" not in result.output # list only libraries for baremetal result = clirunner.invoke( package_list_cmd, ["-d", str(project_dir), "-e", "baremetal", "--only-libraries"], ) assert "No packages" in result.output def test_global_packages(clirunner, validate_cliresult, isolated_pio_core, tmp_path): result = clirunner.invoke(package_list_cmd, ["-g"]) validate_cliresult(result) assert "atmelavr @ 3" in result.output assert "framework-arduino-avr-attiny" in result.output # only tools result = clirunner.invoke(package_list_cmd, ["-g", "--only-tools"]) validate_cliresult(result) assert "toolchain-atmelavr" in result.output assert "Platforms" not in result.output # find tool package result = clirunner.invoke(package_list_cmd, ["-g", "-t", "toolchain-atmelavr"]) validate_cliresult(result) assert "toolchain-atmelavr" in result.output assert "framework-arduino-avr-attiny@" not in result.output # only libraries - no packages result = clirunner.invoke(package_list_cmd, ["-g", "--only-libraries"]) validate_cliresult(result) assert not result.output.strip() # check global libs result = clirunner.invoke( package_install_cmd, ["-g", "-l", "milesburton/DallasTemperature@^3.9.1"] ) validate_cliresult(result) result = clirunner.invoke(package_list_cmd, ["-g", "--only-libraries"]) validate_cliresult(result) assert "DallasTemperature" in result.output assert "OneWire" in result.output # filter by lib result = clirunner.invoke(package_list_cmd, ["-g", "-l", "OneWire"]) validate_cliresult(result) assert "DallasTemperature" in result.output assert "OneWire" in result.output ================================================ FILE: tests/commands/pkg/test_outdated.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import re from platformio.package.commands.install import package_install_cmd from platformio.package.commands.outdated import package_outdated_cmd PROJECT_OUTDATED_CONFIG_TPL = """ [env:devkit] platform = platformio/atmelavr@^2 framework = arduino board = attiny88 lib_deps = milesburton/DallasTemperature@~3.9.0 """ PROJECT_UPDATED_CONFIG_TPL = """ [env:devkit] platform = platformio/atmelavr@<4 framework = arduino board = attiny88 lib_deps = milesburton/DallasTemperature@^3.9.0 """ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) result = clirunner.invoke(package_install_cmd, ["-d", str(project_dir)]) validate_cliresult(result) # overwrite config (project_dir / "platformio.ini").write_text(PROJECT_UPDATED_CONFIG_TPL) result = clirunner.invoke(package_outdated_cmd, ["-d", str(project_dir)]) validate_cliresult(result) # validate output assert "Checking" in result.output assert re.search( r"^atmelavr\s+2\.2\.0\s+3\.\d+\.\d+\s+[456789]\.\d+\.\d+\s+Platform\s+devkit", result.output, re.MULTILINE, ) assert re.search( r"^DallasTemperature\s+3\.\d\.1\s+3\.\d+\.\d+\s+4\.\d+\.\d+\s+Library\s+devkit", result.output, re.MULTILINE, ) ================================================ FILE: tests/commands/pkg/test_search.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.package.commands.search import package_search_cmd def test_empty_query(clirunner, validate_cliresult): result = clirunner.invoke( package_search_cmd, [""], ) validate_cliresult(result) assert all(t in result.output for t in ("Found", "Official", "page 1 of")) def test_pagination(clirunner, validate_cliresult): result = clirunner.invoke( package_search_cmd, ["type:tool"], ) validate_cliresult(result) assert all(t in result.output for t in ("Verified Tool", "page 1 of")) result = clirunner.invoke( package_search_cmd, ["type:tool", "-p", "10"], ) validate_cliresult(result) assert all(t in result.output for t in ("Tool", "page 10 of")) def test_sorting(clirunner, validate_cliresult): result = clirunner.invoke( package_search_cmd, ["OneWire", "-s", "popularity"], ) validate_cliresult(result) assert "paulstoffregen/OneWire" in result.output def test_not_found(clirunner, validate_cliresult): result = clirunner.invoke( package_search_cmd, ["name:unknown-package"], ) validate_cliresult(result) assert "Nothing has been found" in result.output ================================================ FILE: tests/commands/pkg/test_show.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pytest from platformio.exception import UserSideException from platformio.package.commands.show import package_show_cmd def test_spec_name(clirunner, validate_cliresult): # library result = clirunner.invoke( package_show_cmd, ["ArduinoJSON"], ) validate_cliresult(result) assert "bblanchon/ArduinoJson" in result.output assert "Library" in result.output # platform result = clirunner.invoke( package_show_cmd, ["espressif32"], ) validate_cliresult(result) assert "platformio/espressif32" in result.output assert "Platform" in result.output # tool result = clirunner.invoke( package_show_cmd, ["tool-jlink"], ) validate_cliresult(result) assert "platformio/tool-jlink" in result.output assert "tool" in result.output def test_spec_owner(clirunner, validate_cliresult): result = clirunner.invoke( package_show_cmd, ["bblanchon/ArduinoJSON"], ) validate_cliresult(result) assert "bblanchon/ArduinoJson" in result.output assert "Library" in result.output # test broken owner result = clirunner.invoke( package_show_cmd, ["unknown/espressif32"], ) with pytest.raises(UserSideException, match="Could not find"): raise result.exception def test_complete_spec(clirunner, validate_cliresult): result = clirunner.invoke( package_show_cmd, ["bblanchon/ArduinoJSON", "-t", "library"], ) validate_cliresult(result) assert "bblanchon/ArduinoJson" in result.output assert "Library" in result.output # tool result = clirunner.invoke( package_show_cmd, ["platformio/tool-jlink", "-t", "tool"], ) validate_cliresult(result) assert "platformio/tool-jlink" in result.output assert "tool" in result.output def test_name_conflict(clirunner): result = clirunner.invoke( package_show_cmd, ["OneWire", "-t", "library"], ) assert "More than one package" in result.output assert isinstance(result.exception, UserSideException) def test_spec_version(clirunner, validate_cliresult): result = clirunner.invoke( package_show_cmd, ["bblanchon/ArduinoJSON@5.13.4"], ) validate_cliresult(result) assert "bblanchon/ArduinoJson" in result.output assert "Library • 5.13.4" in result.output ================================================ FILE: tests/commands/pkg/test_uninstall.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import os from platformio import fs from platformio.package.commands.install import package_install_cmd from platformio.package.commands.uninstall import package_uninstall_cmd from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig PROJECT_CONFIG_TPL = """ [env] platform = platformio/atmelavr@^3.4.0 lib_deps = milesburton/DallasTemperature@^3.9.1 [env:baremetal] board = uno [env:devkit] framework = arduino board = attiny88 """ def pkgs_to_names(pkgs): return [pkg.metadata.name for pkg in pkgs] def test_global_packages( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): # libraries result = clirunner.invoke( package_install_cmd, [ "--global", "-l", "marvinroger/Homie@^3.0.1", ], ) validate_cliresult(result) assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ "ArduinoJson", "Async TCP", "AsyncMqttClient", "AsyncTCP", "AsyncTCP_RP2040W", "Bounce2", "ESP Async WebServer", "ESPAsyncTCP", "ESPAsyncTCP-esphome", "Homie", ] # uninstall all deps result = clirunner.invoke( package_uninstall_cmd, [ "--global", "-l", "Homie", ], ) validate_cliresult(result) assert not pkgs_to_names(LibraryPackageManager().get_installed()) # skip dependencies validate_cliresult( clirunner.invoke( package_install_cmd, [ "--global", "-l", "marvinroger/Homie@^3.0.1", ], ) ) result = clirunner.invoke( package_uninstall_cmd, ["--global", "-l", "marvinroger/Homie@^3.0.1", "--skip-dependencies"], ) validate_cliresult(result) assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ "ArduinoJson", "Async TCP", "AsyncMqttClient", "AsyncTCP", "AsyncTCP_RP2040W", "Bounce2", "ESP Async WebServer", "ESPAsyncTCP", "ESPAsyncTCP-esphome", ] # remove specific dependency result = clirunner.invoke( package_uninstall_cmd, [ "--global", "-l", "ESP Async WebServer", ], ) validate_cliresult(result) assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "Bounce2", "ESPAsyncTCP", ] # custom storage storage_dir = tmp_path / "custom_lib_storage" storage_dir.mkdir() result = clirunner.invoke( package_install_cmd, [ "--global", "--storage-dir", str(storage_dir), "-l", "marvinroger/Homie@^3.0.1", "--skip-dependencies", ], ) validate_cliresult(result) assert pkgs_to_names(LibraryPackageManager(storage_dir).get_installed()) == [ "Homie" ] result = clirunner.invoke( package_uninstall_cmd, [ "--global", "--storage-dir", str(storage_dir), "-l", "marvinroger/Homie@^3.0.1", ], ) validate_cliresult(result) assert not pkgs_to_names(LibraryPackageManager(storage_dir).get_installed()) # tools result = clirunner.invoke( package_install_cmd, ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1.5.2"], ) validate_cliresult(result) assert pkgs_to_names(ToolPackageManager().get_installed()) == [ "framework-arduino-avr-attiny" ] result = clirunner.invoke( package_uninstall_cmd, ["--global", "-t", "framework-arduino-avr-attiny"], ) validate_cliresult(result) assert not pkgs_to_names(ToolPackageManager().get_installed()) # platforms result = clirunner.invoke( package_install_cmd, ["--global", "-p", "platformio/atmelavr@^3.4.0"], ) validate_cliresult(result) assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"] assert pkgs_to_names(ToolPackageManager().get_installed()) == ["toolchain-atmelavr"] result = clirunner.invoke( package_uninstall_cmd, ["--global", "-p", "platformio/atmelavr@^3.4.0"], ) validate_cliresult(result) assert not pkgs_to_names(PlatformPackageManager().get_installed()) assert not pkgs_to_names(ToolPackageManager().get_installed()) def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_names(lm.get_installed()) == ["DallasTemperature", "OneWire"] assert pkgs_to_names(ToolPackageManager().get_installed()) == [ "framework-arduino-avr-attiny", "tool-scons", "toolchain-atmelavr", ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1" ] # try again result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) assert "Already up-to-date" in result.output # uninstall result = clirunner.invoke( package_uninstall_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert not pkgs_to_names(lm.get_installed()) assert pkgs_to_names(ToolPackageManager().get_installed()) == ["tool-scons"] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1" ] def test_custom_project_libraries( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) spec = "bblanchon/ArduinoJson@^6.19.2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-l", spec], ) validate_cliresult(result) assert "Already up-to-date" not in result.output with fs.cd(str(project_dir)): # check folders config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_names(lm.get_installed()) == ["ArduinoJson"] # do not expect any platforms/tools assert not os.path.exists(config.get("platformio", "platforms_dir")) assert not os.path.exists(config.get("platformio", "packages_dir")) # check saved deps assert config.get("env:devkit", "lib_deps") == [ "bblanchon/ArduinoJson@^6.19.2", ] # uninstall result = clirunner.invoke( package_uninstall_cmd, ["-e", "devkit", "-l", spec], ) validate_cliresult(result) config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert not pkgs_to_names(lm.get_installed()) # do not expect any platforms/tools assert not os.path.exists(config.get("platformio", "platforms_dir")) assert not os.path.exists(config.get("platformio", "packages_dir")) # check saved deps assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1" ] # install library without saving to config result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-l", spec, "--no-save"], ) validate_cliresult(result) config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_names(lm.get_installed()) == ["ArduinoJson"] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1", ] result = clirunner.invoke( package_uninstall_cmd, ["-e", "devkit", "-l", spec, "--no-save"], ) validate_cliresult(result) config = ProjectConfig() assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1", ] # unknown libraries result = clirunner.invoke( package_uninstall_cmd, ["-l", "platformio/unknown_library"] ) assert isinstance(result.exception, UnknownPackageError) def test_custom_project_tools( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) spec = "platformio/tool-openocd@^2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-t", spec], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() assert pkgs_to_names(ToolPackageManager().get_installed()) == ["tool-openocd"] assert not LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ).get_installed() # do not expect any platforms assert not os.path.exists(config.get("platformio", "platforms_dir")) # check saved deps assert config.get("env:devkit", "platform_packages") == [ "platformio/tool-openocd@^2", ] # uninstall result = clirunner.invoke( package_uninstall_cmd, ["-e", "devkit", "-t", spec], ) validate_cliresult(result) assert not pkgs_to_names(ToolPackageManager().get_installed()) # check saved deps assert not ProjectConfig().get("env:devkit", "platform_packages") # install tool without saving to config result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-t", "platformio/tool-esptoolpy@1.20310.0"], ) validate_cliresult(result) assert pkgs_to_names(ToolPackageManager().get_installed()) == [ "tool-esptoolpy", ] assert ProjectConfig().get("env:devkit", "platform_packages") == [ "platformio/tool-esptoolpy@1.20310.0", ] # uninstall result = clirunner.invoke( package_uninstall_cmd, ["-e", "devkit", "-t", "platformio/tool-esptoolpy@^1", "--no-save"], ) validate_cliresult(result) assert not pkgs_to_names(ToolPackageManager().get_installed()) assert ProjectConfig().get("env:devkit", "platform_packages") == [ "platformio/tool-esptoolpy@1.20310.0", ] # unknown tool result = clirunner.invoke( package_uninstall_cmd, ["-t", "platformio/unknown_tool"] ) assert isinstance(result.exception, UnknownPackageError) def test_custom_project_platforms( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) spec = "platformio/atmelavr@^3.4.0" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-p", spec], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"] assert not LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ).get_installed() assert pkgs_to_names(ToolPackageManager().get_installed()) == [ "framework-arduino-avr-attiny", "toolchain-atmelavr", ] # uninstall result = clirunner.invoke( package_uninstall_cmd, ["-e", "devkit", "-p", spec], ) validate_cliresult(result) assert not pkgs_to_names(PlatformPackageManager().get_installed()) assert not pkgs_to_names(ToolPackageManager().get_installed()) # unknown platform result = clirunner.invoke(package_uninstall_cmd, ["-p", "unknown_platform"]) assert isinstance(result.exception, UnknownPackageError) ================================================ FILE: tests/commands/pkg/test_update.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import os from platformio import fs from platformio.dependencies import get_core_dependencies from platformio.package.commands.install import package_install_cmd from platformio.package.commands.update import package_update_cmd from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.project.config import ProjectConfig DALLASTEMPERATURE_LATEST_VERSION = "3.11.0" PROJECT_OUTDATED_CONFIG_TPL = """ [env:devkit] platform = platformio/atmelavr@^2 framework = arduino board = attiny88 lib_deps = milesburton/DallasTemperature@^3.9.1 """ PROJECT_UPDATED_CONFIG_TPL = """ [env:devkit] platform = platformio/atmelavr@<4 framework = arduino board = attiny88 lib_deps = milesburton/DallasTemperature@^3.9.1 """ def pkgs_to_specs(pkgs): return [ PackageSpec(name=pkg.metadata.name, requirements=pkg.metadata.version) for pkg in pkgs ] def test_global_packages( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): # libraries result = clirunner.invoke( package_install_cmd, ["--global", "-l", "bblanchon/ArduinoJson@^5"], ) validate_cliresult(result) assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ PackageSpec("ArduinoJson@5.13.4") ] # update to the latest version result = clirunner.invoke( package_update_cmd, ["--global", "-l", "bblanchon/ArduinoJson"], ) validate_cliresult(result) pkgs = LibraryPackageManager().get_installed() assert len(pkgs) == 1 assert pkgs[0].metadata.version.major > 5 # custom storage storage_dir = tmp_path / "custom_lib_storage" storage_dir.mkdir() result = clirunner.invoke( package_install_cmd, [ "--global", "--storage-dir", str(storage_dir), "-l", "bblanchon/ArduinoJson@^5", ], ) validate_cliresult(result) assert pkgs_to_specs(LibraryPackageManager(storage_dir).get_installed()) == [ PackageSpec("ArduinoJson@5.13.4") ] # update to the latest version result = clirunner.invoke( package_update_cmd, ["--global", "--storage-dir", str(storage_dir), "-l", "bblanchon/ArduinoJson"], ) validate_cliresult(result) pkgs = LibraryPackageManager(storage_dir).get_installed() assert len(pkgs) == 1 assert pkgs[0].metadata.version.major > 5 # tools result = clirunner.invoke( package_install_cmd, ["--global", "-t", "platformio/framework-arduino-avr-attiny@~1.4"], ) validate_cliresult(result) assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.4.1") ] # update to the latest version result = clirunner.invoke( package_update_cmd, ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1"], ) validate_cliresult(result) pkgs = ToolPackageManager().get_installed() assert len(pkgs) == 1 assert pkgs[0].metadata.version.major == 1 assert pkgs[0].metadata.version.minor > 4 # platforms result = clirunner.invoke( package_install_cmd, ["--global", "-p", "platformio/atmelavr@^2", "--skip-dependencies"], ) validate_cliresult(result) assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@2.2.0") ] # update to the latest version result = clirunner.invoke( package_update_cmd, ["--global", "-p", "platformio/atmelavr", "--skip-dependencies"], ) validate_cliresult(result) pkgs = PlatformPackageManager().get_installed() assert len(pkgs) == 1 assert pkgs[0].metadata.version.major > 2 # update unknown package result = clirunner.invoke( package_update_cmd, ["--global", "-l", "platformio/unknown_package_for_update"], ) assert isinstance(result.exception, UnknownPackageError) def test_project( clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir)], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec(f"DallasTemperature@{DALLASTEMPERATURE_LATEST_VERSION}"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@2.2.0") ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.3.2"), PackageSpec("tool-scons@%s" % get_core_dependencies()["tool-scons"][1:]), PackageSpec("toolchain-atmelavr@1.50400.190710"), ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1" ] # update packages (project_dir / "platformio.ini").write_text(PROJECT_UPDATED_CONFIG_TPL) result = clirunner.invoke(package_update_cmd) validate_cliresult(result) config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) pkgs = PlatformPackageManager().get_installed() assert len(pkgs) == 1 assert pkgs[0].metadata.name == "atmelavr" assert pkgs[0].metadata.version.major == 3 assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.11.0"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.3.2"), PackageSpec("tool-scons@%s" % get_core_dependencies()["tool-scons"][1:]), PackageSpec("toolchain-atmelavr@1.70300.191015"), PackageSpec("toolchain-atmelavr@1.50400.190710"), ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1" ] # update again result = clirunner.invoke(package_update_cmd) validate_cliresult(result) assert "Already up-to-date." in result.output # update again in the silent ,pde result = clirunner.invoke(package_update_cmd, ["--silent"]) validate_cliresult(result) assert not result.output def test_custom_project_libraries( clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) spec = "milesburton/DallasTemperature@^3.9.1" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-l", spec], ) validate_cliresult(result) with fs.cd(str(project_dir)): config = ProjectConfig() assert config.get("env:devkit", "lib_deps") == [spec] lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec(f"DallasTemperature@{DALLASTEMPERATURE_LATEST_VERSION}"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] # update package result = clirunner.invoke( package_update_cmd, ["-e", "devkit", "-l", "milesburton/DallasTemperature@^3.9.1"], ) assert ProjectConfig().get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1" ] # try again result = clirunner.invoke( package_update_cmd, ["-e", "devkit", "-l", "milesburton/DallasTemperature@^3.9.1"], ) validate_cliresult(result) assert "Already up-to-date." in result.output # install library without saving to config result = clirunner.invoke( package_update_cmd, ["-e", "devkit", "-l", "milesburton/DallasTemperature@^3", "--no-save"], ) validate_cliresult(result) assert "Already up-to-date." in result.output config = ProjectConfig() lm = LibraryPackageManager( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.11.0"), PackageSpec( "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") ), ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.9.1" ] # unknown libraries result = clirunner.invoke( package_update_cmd, ["-l", "platformio/unknown_library"] ) assert isinstance(result.exception, UnknownPackageError) def test_custom_project_tools( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) spec = "toolchain-atmelavr@~1.50400.0" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-t", spec], ) validate_cliresult(result) with fs.cd(str(project_dir)): assert ProjectConfig().get("env:devkit", "platform_packages") == [ "platformio/toolchain-atmelavr@~1.50400.0" ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("toolchain-atmelavr@1.50400.190710") ] result = clirunner.invoke( package_update_cmd, ["-e", "devkit", "-t", "toolchain-atmelavr@^1"], ) validate_cliresult(result) assert ProjectConfig().get("env:devkit", "platform_packages") == [ "platformio/toolchain-atmelavr@^1" ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("toolchain-atmelavr@1.70300.191015") ] # install without saving to config result = clirunner.invoke( package_update_cmd, ["-e", "devkit", "-t", "toolchain-atmelavr@~1.70300.191015", "--no-save"], ) validate_cliresult(result) assert "Already up-to-date." in result.output assert ProjectConfig().get("env:devkit", "platform_packages") == [ "platformio/toolchain-atmelavr@^1" ] # unknown tool result = clirunner.invoke(package_update_cmd, ["-t", "platformio/unknown_tool"]) assert isinstance(result.exception, UnknownPackageError) def test_custom_project_platforms( clirunner, validate_cliresult, func_isolated_pio_core, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) spec = "atmelavr@^2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-p", spec, "--skip-dependencies"], ) validate_cliresult(result) with fs.cd(str(project_dir)): assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@2.2.0") ] assert ProjectConfig().get("env:devkit", "platform") == "platformio/atmelavr@^2" # update result = clirunner.invoke( package_install_cmd, ["-e", "devkit", "-p", "platformio/atmelavr@^3", "--skip-dependencies"], ) validate_cliresult(result) assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@3.4.0"), PackageSpec("atmelavr@2.2.0"), ] assert ProjectConfig().get("env:devkit", "platform") == "platformio/atmelavr@^2" # unknown platform result = clirunner.invoke(package_install_cmd, ["-p", "unknown_platform"]) assert isinstance(result.exception, UnknownPackageError) ================================================ FILE: tests/commands/test_account_org_team.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=global-statement,unused-argument import json import os import random import pytest import requests from platformio.account.cli import cli as cmd_account from platformio.account.org.cli import cli as cmd_org from platformio.account.team.cli import cli as cmd_team pytestmark = pytest.mark.skipif( not all( os.environ.get(name) for name in ( "TEST_EMAIL_LOGIN", "TEST_EMAIL_PASSWORD", "TEST_EMAIL_IMAP_SERVER", ) ), reason=( "requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD, " "and TEST_EMAIL_IMAP_SERVER environment variables" ), ) USER_NAME = "test-piocore-%s" % str(random.randint(0, 100000)) USER_EMAIL = os.environ.get("TEST_EMAIL_LOGIN", "").replace("@", f"+{USER_NAME}@") USER_PASSWORD = f"Qwerty-{random.randint(0, 100000)}" USER_FIRST_NAME = "FirstName" USER_LAST_NAME = "LastName" ORG_NAME = "testorg-piocore-%s" % str(random.randint(0, 100000)) ORG_DISPLAY_NAME = "Test Org for PIO Core" EXISTING_OWNER = "piolabs" TEAM_NAME = "test-" + str(random.randint(0, 100000)) TEAM_DESCRIPTION = "team for CI test" def verify_account(email_contents): link = ( email_contents.split("Click on the link below to start this process.")[1] .split("This link will expire within 12 hours.")[0] .strip() ) with requests.Session() as session: result = session.get(link).text link = result.split(' 0 assert result.exception assert "You are not authorized!" in str(result.exception) result = clirunner.invoke( cmd_account, ["login", "-u", USER_NAME, "-p", USER_PASSWORD], ) validate_cliresult(result) # def _test_account_destroy_with_linked_resources( # clirunner, validate_cliresult, receive_email, isolated_pio_core, tmpdir_factory # ): # package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" # # tmp_dir = tmpdir_factory.mktemp("package") # fd = FileDownloader(package_url, str(tmp_dir)) # pkg_dir = tmp_dir.mkdir("raw_package") # fd.start(with_progress=False, silent=True) # with FileUnpacker(fd.get_filepath()) as unpacker: # unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) # # result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) # validate_cliresult(result) # try: # result = receive_email(email) # assert "Congrats" in result # assert "was published" in result # except: # pylint:disable=bare-except # pass # # result = clirunner.invoke(cmd_account, ["destroy"], "y") # assert result.exit_code != 0 # assert ( # "We can not destroy the %s account due to 1 linked resources from registry" # % username # ) # # result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) # validate_cliresult(result) def test_org_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_org, ["create", "--email", USER_EMAIL, "--displayname", ORG_DISPLAY_NAME, ORG_NAME], ) validate_cliresult(result) def test_org_list(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) json_result = json.loads(result.output.strip()) assert json_result == [ { "orgname": ORG_NAME, "displayname": ORG_DISPLAY_NAME, "email": USER_EMAIL, "owners": [ { "username": USER_NAME, "firstname": USER_FIRST_NAME, "lastname": USER_LAST_NAME, } ], } ] def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_org, ["add", ORG_NAME, EXISTING_OWNER]) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) assert EXISTING_OWNER in result.output def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_org, ["remove", ORG_NAME, EXISTING_OWNER]) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) assert EXISTING_OWNER not in result.output def test_org_update(clirunner, validate_cliresult, isolated_pio_core): new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) new_display_name = "Test Org for PIO Core" result = clirunner.invoke( cmd_org, [ "update", ORG_NAME, "--orgname", new_orgname, "--displayname", new_display_name, ], ) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) json_result = json.loads(result.output.strip()) assert json_result == [ { "orgname": new_orgname, "displayname": new_display_name, "email": USER_EMAIL, "owners": [ { "username": USER_NAME, "firstname": USER_FIRST_NAME, "lastname": USER_LAST_NAME, } ], } ] result = clirunner.invoke( cmd_org, [ "update", new_orgname, "--orgname", ORG_NAME, "--displayname", ORG_DISPLAY_NAME, ], ) validate_cliresult(result) def test_team_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, [ "create", "%s:%s" % (ORG_NAME, TEAM_NAME), "--description", TEAM_DESCRIPTION, ], ) validate_cliresult(result) def test_team_list(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["list", "%s" % ORG_NAME, "--json-output"], ) validate_cliresult(result) json_result = json.loads(result.output.strip()) for item in json_result: del item["id"] assert json_result == [ {"name": TEAM_NAME, "description": TEAM_DESCRIPTION, "members": []} ] def _test_team_add_member(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["add", "%s:%s" % (ORG_NAME, TEAM_NAME), EXISTING_OWNER], ) validate_cliresult(result) result = clirunner.invoke( cmd_team, ["list", "%s" % ORG_NAME, "--json-output"], ) validate_cliresult(result) assert EXISTING_OWNER in result.output def _test_team_remove(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["remove", "%s:%s" % (ORG_NAME, TEAM_NAME), EXISTING_OWNER], ) validate_cliresult(result) result = clirunner.invoke( cmd_team, ["list", "%s" % ORG_NAME, "--json-output"], ) validate_cliresult(result) assert EXISTING_OWNER not in result.output def _test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_core): new_teamname = "new-" + str(random.randint(0, 100000)) newteam_description = "Updated Description" result = clirunner.invoke( cmd_team, [ "update", "%s:%s" % (ORG_NAME, TEAM_NAME), "--name", new_teamname, "--description", newteam_description, ], ) validate_cliresult(result) result = clirunner.invoke( cmd_team, ["list", "%s" % ORG_NAME, "--json-output"], ) validate_cliresult(result) json_result = json.loads(result.output.strip()) for item in json_result: del item["id"] assert json_result == [ {"name": new_teamname, "description": newteam_description, "members": []} ] result = clirunner.invoke( cmd_team, [ "update", "%s:%s" % (ORG_NAME, new_teamname), "--name", TEAM_NAME, "--description", TEAM_DESCRIPTION, ], ) validate_cliresult(result) def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_core): result = clirunner.invoke( cmd_team, ["destroy", "%s:%s" % (ORG_NAME, TEAM_NAME)], "y" ) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["destroy", ORG_NAME], "y") validate_cliresult(result) result = clirunner.invoke(cmd_account, ["destroy"], "y") validate_cliresult(result) ================================================ FILE: tests/commands/test_boards.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from platformio.commands.boards import cli as cmd_boards from platformio.commands.platform import platform_search as cmd_platform_search def test_board_json_output(clirunner, validate_cliresult): result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) validate_cliresult(result) boards = json.loads(result.output) assert isinstance(boards, list) assert any("mbed" in b["frameworks"] for b in boards) def test_board_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cmd_boards, ["espidf"]) validate_cliresult(result) assert "espressif32" in result.output def test_board_options(clirunner, validate_cliresult): required_opts = set(["fcpu", "frameworks", "id", "mcu", "name", "platform"]) # fetch available platforms result = clirunner.invoke(cmd_platform_search, ["--json-output"]) validate_cliresult(result) search_result = json.loads(result.output) assert isinstance(search_result, list) assert search_result platforms = [item["name"] for item in search_result] result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) validate_cliresult(result) boards = json.loads(result.output) for board in boards: assert required_opts.issubset(set(board)) assert board["platform"] in platforms ================================================ FILE: tests/commands/test_check.py ================================================ # Copyright (c) 2019-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=redefined-outer-name import json import os import sys import pytest from platformio import fs from platformio.check.cli import cli as cmd_check DEFAULT_CONFIG = """ [env:native] platform = native """ TEST_CODE = """ #include void run_defects() { /* Freeing a pointer twice */ int* doubleFreePi = (int*)malloc(sizeof(int)); *doubleFreePi=2; free(doubleFreePi); free(doubleFreePi); /* High */ /* Reading uninitialized memory */ int* uninitializedPi = (int*)malloc(sizeof(int)); *uninitializedPi++; /* High + Medium*/ free(uninitializedPi); /* Delete instead of delete [] */ int* wrongDeletePi = new int[10]; wrongDeletePi++; delete wrongDeletePi; /* High */ /* Index out of bounds */ int arr[10]; for(int i=0; i < 11; i++) { arr[i] = 0; /* High */ } } int main() { int uninitializedVar; /* Low */ run_defects(); } """ PVS_STUDIO_FREE_LICENSE_HEADER = """ // This is an open source non-commercial project. Dear PVS-Studio, please check it. // PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com """ EXPECTED_ERRORS = 5 EXPECTED_WARNINGS = 1 EXPECTED_STYLE = 4 EXPECTED_DEFECTS = EXPECTED_ERRORS + EXPECTED_WARNINGS + EXPECTED_STYLE @pytest.fixture(scope="module") def check_dir(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) return tmpdir def count_defects(output): error, warning, style = 0, 0, 0 for line in output.split("\n"): if "[high:error]" in line: error += 1 elif "[medium:warning]" in line: warning += 1 elif "[low:style]" in line: style += 1 return error, warning, style def test_check_cli_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir)]) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors + warnings + style == EXPECTED_DEFECTS def test_check_json_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--json-output"] ) validate_cliresult(result) output = json.loads(result.stdout.strip()) assert isinstance(output, list) assert len(output[0].get("defects", [])) == EXPECTED_DEFECTS def test_check_tool_defines_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) output = result.output assert "PLATFORMIO=" in output assert "__GNUC__" in output def test_check_tool_complex_defines_handled( clirunner, validate_cliresult, tmpdir_factory ): project_dir = tmpdir_factory.mktemp("project_dir") project_dir.join("platformio.ini").write(DEFAULT_CONFIG + R""" check_tool = cppcheck, clangtidy, pvs-studio build_flags = -DEXTERNAL_INCLUDE_FILE=\"test.h\" "-DDEFINE_WITH_SPACE="Hello World!"" """) src_dir = project_dir.mkdir("src") src_dir.join("test.h").write(""" #ifndef TEST_H #define TEST_H #define ARBITRARY_CONST_VALUE 10 #endif """) src_dir.join("main.c").write(PVS_STUDIO_FREE_LICENSE_HEADER + """ #if !defined(EXTERNAL_INCLUDE_FILE) #error "EXTERNAL_INCLUDE_FILE is not declared!" #else #include EXTERNAL_INCLUDE_FILE #endif int main() { /* Index out of bounds */ int arr[ARBITRARY_CONST_VALUE]; for(int i=0; i < ARBITRARY_CONST_VALUE+1; i++) { arr[i] = 0; /* High */ } return 0; } """) default_result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir)]) validate_cliresult(default_result) def test_check_language_standard_definition_passed(clirunner, tmpdir): config = DEFAULT_CONFIG + "\nbuild_flags = -std=c++17" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) assert "__cplusplus=201703L" in result.output assert "--std=c++17" in result.output def test_check_language_standard_option_is_converted(clirunner, tmpdir): config = DEFAULT_CONFIG + """ build_flags = -std=gnu++1y """ tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) assert "--std=c++14" in result.output def test_check_language_standard_is_prioritized_over_build_flags(clirunner, tmpdir): config = DEFAULT_CONFIG + """ check_flags = --std=c++03 build_flags = -std=c++17 """ tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) assert "--std=c++03" in result.output assert "--std=c++17" not in result.output def test_check_language_standard_for_c_language(clirunner, tmpdir): config = DEFAULT_CONFIG + "\nbuild_flags = -std=c11" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) assert "--std=c11" in result.output assert "__STDC_VERSION__=201112L" in result.output assert "__cplusplus" not in result.output def test_check_severity_threshold(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--severity=high"] ) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) inc_count = 0 for line in result.output.split("\n"): if line.startswith("Includes:"): inc_count = line.count("-I") # at least 1 include path for default mode assert inc_count > 0 def test_check_silent_mode(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--silent"]) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 def test_check_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) errors, warnings, style = count_defects(result.output) assert result.exit_code != 0 assert errors == 0 assert warnings == 0 assert style == 0 def test_check_bad_flag_passed(clirunner, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), '"--flags=--UNKNOWN"'] ) errors, warnings, style = count_defects(result.output) assert result.exit_code != 0 assert errors == 0 assert warnings == 0 assert style == 0 def test_check_success_if_no_errors(clirunner, validate_cliresult, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src").join("main.c").write(""" #include void unused_function(){ int unusedVar = 0; int* iP = &unusedVar; *iP++; } int main() { } """) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert "[PASSED]" in result.output assert errors == 0 assert warnings == 1 assert style == 1 def test_check_individual_flags_passed(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy, pvs-studio" config += """\ncheck_flags = cppcheck: --std=c++11 clangtidy: --fix-errors pvs-studio: --analysis-mode=4 """ tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write( PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) validate_cliresult(result) clang_flags_found = cppcheck_flags_found = pvs_flags_found = False for l in result.output.split("\n"): if "--fix" in l and "clang-tidy" in l and "--std=c++11" not in l: clang_flags_found = True elif "--std=c++11" in l and "cppcheck" in l and "--fix" not in l: cppcheck_flags_found = True elif ( "--analysis-mode=4" in l and "pvs-studio" in l.lower() and "--fix" not in l ): pvs_flags_found = True assert clang_flags_found assert cppcheck_flags_found assert pvs_flags_found def test_check_cppcheck_misra_addon(clirunner, validate_cliresult, tmpdir_factory): check_dir = tmpdir_factory.mktemp("project") check_dir.join("platformio.ini").write(DEFAULT_CONFIG) check_dir.mkdir("src").join("main.c").write(TEST_CODE) check_dir.join("misra.json").write(""" { "script": "addons/misra.py", "args": ["--rule-texts=rules.txt"] } """) check_dir.join("rules.txt").write(""" Appendix A Summary of guidelines Rule 3.1 Required R3.1 text. Rule 4.1 Required R4.1 text. Rule 10.4 Mandatory R10.4 text. Rule 11.5 Advisory R11.5 text. Rule 15.5 Advisory R15.5 text. Rule 15.6 Required R15.6 text. Rule 17.7 Required R17.7 text. Rule 20.1 Advisory R20.1 text. Rule 21.3 Required R21.3 Found MISRA defect Rule 21.4 R21.4 text. """) result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--flags=--addon=misra.json"] ) validate_cliresult(result) assert "R21.3 Found MISRA defect" in result.output assert not os.path.isfile(os.path.join(str(check_dir), "src", "main.cpp.dump")) def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) result_with_flag = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) validate_cliresult(default_result) assert result_with_flag.exit_code != 0 def test_check_fails_on_defects_only_on_specified_level( clirunner, validate_cliresult, tmpdir ): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write(""" #include void unused_function(){ int unusedVar = 0; int* iP = &unusedVar; *iP++; } int main() { } """) high_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) validate_cliresult(high_result) low_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=low"] ) assert low_result.exit_code != 0 def test_check_pvs_studio_free_license(clirunner, tmpdir): config = """ [env:test] platform = teensy board = teensy35 framework = arduino check_tool = pvs-studio """ tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high", "-v"] ) errors, warnings, style = count_defects(result.output) assert result.exit_code != 0 assert errors != 0 assert warnings != 0 assert style == 0 def test_check_pvs_studio_fails_without_license(clirunner, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = pvs-studio" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write(TEST_CODE) default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) verbose_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) assert default_result.exit_code != 0 assert "failed to perform check" in default_result.output.lower() assert verbose_result.exit_code != 0 assert "license was not entered" in verbose_result.output.lower() @pytest.mark.skipif( sys.platform != "win32", reason="For some reason the error message is different on Windows", ) def test_check_pvs_studio_fails_broken_license(clirunner, tmpdir): config = DEFAULT_CONFIG + """ check_tool = pvs-studio check_flags = --lic-file=./pvs-studio.lic """ tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write(TEST_CODE) tmpdir.join("pvs-studio.lic").write(""" TEST TEST-TEST-TEST-TEST """) default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) verbose_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) assert default_result.exit_code != 0 assert "failed to perform check" in default_result.output.lower() assert verbose_result.exit_code != 0 assert "license information is incorrect" in verbose_result.output.lower() @pytest.mark.parametrize("framework", ["arduino", "stm32cube", "zephyr"]) @pytest.mark.parametrize("check_tool", ["cppcheck", "clangtidy", "pvs-studio"]) def test_check_embedded_platform_all_tools( clirunner, validate_cliresult, tmpdir, framework, check_tool ): config = f""" [env:test] platform = ststm32 board = nucleo_f401re framework = {framework} check_tool = {check_tool} """ tmpdir.mkdir("src").join("main.c").write(PVS_STUDIO_FREE_LICENSE_HEADER + """ #include void unused_function(int val){ int unusedVar = 0; int* iP = &unusedVar; *iP++; } int main() { } """) if framework == "zephyr": zephyr_dir = tmpdir.mkdir("zephyr") zephyr_dir.join("prj.conf").write("# nothing here") zephyr_dir.join("CMakeLists.txt").write( """cmake_minimum_required(VERSION 3.16.0) find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) project(hello_world) target_sources(app PRIVATE ../src/main.c)""" ) tmpdir.join("platformio.ini").write(config) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) defects = sum(count_defects(result.output)) assert defects > 0, "Not defects were found!" def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): config = """ [env:test] platform = nordicnrf52 board = nrf52_dk framework = arduino """ tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write(TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--skip-packages", "-v"] ) validate_cliresult(result) project_path = fs.to_unix_path(str(tmpdir)) for line in result.output.split("\n"): if not line.startswith("Includes:"): continue for inc in line.split(" "): if inc.startswith("-I") and project_path not in inc: pytest.fail("Detected an include path from packages: " + inc) def test_check_multiline_error(clirunner, tmpdir_factory): project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(DEFAULT_CONFIG) project_dir.mkdir("include").join("main.h").write(""" #error This is a multiline error message \\ that should be correctly reported \\ in both default and verbose modes. """) project_dir.mkdir("src").join("main.c").write(""" #include #include "main.h" int main() {} """) result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir)]) errors, _, _ = count_defects(result.output) result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir), "-v"]) verbose_errors, _, _ = count_defects(result.output) assert verbose_errors == errors == 1 @pytest.mark.parametrize("check_tool", ["cppcheck", "clangtidy", "pvs-studio"]) def test_check_handles_spaces_in_paths( clirunner, validate_cliresult, tmpdir_factory, check_tool ): package_dir_with_spaces = tmpdir_factory.mktemp("pio pkg dir") project_dir_with_spaces = tmpdir_factory.mktemp("project dir") config = f""" [platformio] ; redirect toolchain and tool packages to a directory with whitespaces packages_dir = {package_dir_with_spaces} [env:test] platform = atmelsam board = adafruit_feather_m0 framework = arduino check_tool = {check_tool} """ project_dir_with_spaces.join("platformio.ini").write(config) project_dir_with_spaces.mkdir("src").join("main.cpp").write( PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE ) result = clirunner.invoke( cmd_check, ["--project-dir", str(project_dir_with_spaces), "-v"] ) validate_cliresult(result) # Make sure toolchain defines were successfully extracted if check_tool != "pvs-studio": # PVS doesn't write defines to stdout assert "__GNUC__" in result.output # # Files filtering functionality # @pytest.mark.parametrize( "src_filter,number_of_checked_files", [ (["+"], 1), (["+"], 1), (["+", "-"], 2), (["-<*> + + +"], 3), ], ids=["Single file", "Glob pattern", "Exclude pattern", "Filter as string"], ) def test_check_src_filter( clirunner, validate_cliresult, tmpdir_factory, src_filter, number_of_checked_files, ): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) src_dir = tmpdir.mkdir("src") src_dir.join("main.cpp").write(TEST_CODE) src_dir.join("app.cpp").write(TEST_CODE) src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) cmd_args = ["--project-dir", str(tmpdir)] + [ "--src-filters=%s" % f for f in src_filter ] result = clirunner.invoke(cmd_check, cmd_args) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors + warnings + style == EXPECTED_DEFECTS * number_of_checked_files def test_check_src_filter_from_config(clirunner, validate_cliresult, tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") config = DEFAULT_CONFIG + """ check_src_filters = + + """ tmpdir.join("platformio.ini").write(config) src_dir = tmpdir.mkdir("src") src_dir.join("main.cpp").write(TEST_CODE) src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors + warnings + style == EXPECTED_DEFECTS * 2 assert "main.cpp" not in result.output def test_check_custom_pattern_absolute_path_legacy( clirunner, validate_cliresult, tmpdir_factory ): project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(DEFAULT_CONFIG) check_dir = tmpdir_factory.mktemp("custom_src_dir") check_dir.join("main.cpp").write(TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] ) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors == EXPECTED_ERRORS assert warnings == EXPECTED_WARNINGS assert style == EXPECTED_STYLE def test_check_custom_pattern_relative_path_legacy( clirunner, validate_cliresult, tmpdir_factory ): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) src_dir = tmpdir.mkdir("src") src_dir.join("main.cpp").write(TEST_CODE) src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--pattern=src/uart", "--pattern=src/spi"], ) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors + warnings + style == EXPECTED_DEFECTS * 2 def test_check_src_filter_from_config_legacy( clirunner, validate_cliresult, tmpdir_factory ): tmpdir = tmpdir_factory.mktemp("project") config = DEFAULT_CONFIG + """ check_patterns = src/spi/*.c* tests/test.cpp """ tmpdir.join("platformio.ini").write(config) src_dir = tmpdir.mkdir("src") src_dir.join("main.cpp").write(TEST_CODE) src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors + warnings + style == EXPECTED_DEFECTS * 2 assert "main.cpp" not in result.output def test_check_src_filter_multiple_envs(clirunner, validate_cliresult, tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") config = """ [env] check_tool = cppcheck check_src_filters = + [env:check_sources] platform = native [env:check_tests] platform = native check_src_filters = + """ tmpdir.join("platformio.ini").write(config) src_dir = tmpdir.mkdir("src") src_dir.join("main.cpp").write(TEST_CODE) src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) tmpdir.mkdir("test").join("test.cpp").write(TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "-e", "check_tests"] ) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert errors + warnings + style == EXPECTED_DEFECTS assert "test.cpp" in result.output assert "main.cpp" not in result.output def test_check_sources_in_project_root(clirunner, validate_cliresult, tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") config = """ [platformio] src_dir = ./ """ + DEFAULT_CONFIG tmpdir.join("platformio.ini").write(config) tmpdir.join("main.cpp").write(TEST_CODE) tmpdir.mkdir("spi").join("uart.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS * 2 def test_check_sources_in_external_dir(clirunner, validate_cliresult, tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") external_src_dir = tmpdir_factory.mktemp("external_src_dir") config = f""" [platformio] src_dir = {external_src_dir} """ + DEFAULT_CONFIG tmpdir.join("platformio.ini").write(config) external_src_dir.join("main.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS ================================================ FILE: tests/commands/test_ci.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci from platformio.package.commands.install import package_install_cmd def test_ci_empty(clirunner): result = clirunner.invoke(cmd_ci) assert result.exit_code != 0 assert "Invalid value: Missing argument 'src'" in result.output def test_ci_boards(clirunner, validate_cliresult): result = clirunner.invoke( cmd_ci, [ join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", "-b", "leonardo", ], ) validate_cliresult(result) def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, [ join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", "--build-dir", build_dir, ], ) validate_cliresult(result) assert not isfile(join(build_dir, "platformio.ini")) def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, [ join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", "--build-dir", build_dir, "--keep-build-dir", ], ) validate_cliresult(result) assert isfile(join(build_dir, "platformio.ini")) # 2nd attempt result = clirunner.invoke( cmd_ci, [ join("examples", "wiring-blink", "src", "main.cpp"), "-b", "metro", "--build-dir", build_dir, "--keep-build-dir", ], ) validate_cliresult(result) assert "board: uno" in result.output assert "board: metro" in result.output def test_ci_keep_build_dir_single_src_dir( clirunner, tmpdir_factory, validate_cliresult ): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) # Run two times to detect possible "AlreadyExists" errors for _ in range(2): result = clirunner.invoke( cmd_ci, [ join("examples", "wiring-blink", "src"), "-b", "uno", "--build-dir", build_dir, "--keep-build-dir", ], ) validate_cliresult(result) def test_ci_keep_build_dir_nested_src_dirs( clirunner, tmpdir_factory, validate_cliresult ): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) # Split default Arduino project in two parts src_dir1 = tmpdir_factory.mktemp("src_1") src_dir1.join("src1.cpp").write(""" #include void setup() {} """) src_dir2 = tmpdir_factory.mktemp("src_2") src_dir2.join("src2.cpp").write(""" #include void loop() {} """) src_dir1 = str(src_dir1) src_dir2 = str(src_dir2) # Run two times to detect possible "AlreadyExists" errors for _ in range(2): result = clirunner.invoke( cmd_ci, [ src_dir1, src_dir2, "-b", "teensy40", "--build-dir", build_dir, "--keep-build-dir", ], ) validate_cliresult(result) def test_ci_project_conf(clirunner, validate_cliresult): project_dir = join("examples", "wiring-blink") result = clirunner.invoke( cmd_ci, [ join(project_dir, "src", "main.cpp"), "--project-conf", join(project_dir, "platformio.ini"), ], ) validate_cliresult(result) assert "uno" in result.output def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( package_install_cmd, ["--global", "--storage-dir", storage_dir, "--library", "1"], ) validate_cliresult(result) result = clirunner.invoke( cmd_ci, [ join( storage_dir, "OneWire", "examples", "DS2408_Switch", "DS2408_Switch.ino", ), "-l", join(storage_dir, "OneWire"), "-b", "uno", ], ) validate_cliresult(result) ================================================ FILE: tests/commands/test_init.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os from platformio import fs from platformio.commands.boards import cli as cmd_boards from platformio.project.commands.init import project_init_cmd from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError def validate_pioproject(pioproject_dir): pioconf_path = os.path.join(pioproject_dir, "platformio.ini") assert os.path.isfile(pioconf_path) and os.path.getsize(pioconf_path) > 0 assert os.path.isdir(os.path.join(pioproject_dir, "src")) and os.path.isdir( os.path.join(pioproject_dir, "lib") ) def test_init_default(clirunner, validate_cliresult): with clirunner.isolated_filesystem(): result = clirunner.invoke(project_init_cmd) validate_cliresult(result) validate_pioproject(os.getcwd()) def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): project_dir = str(tmpdir.join("ext_folder")) os.makedirs(project_dir) with fs.cd(os.path.dirname(project_dir)): result = clirunner.invoke( project_init_cmd, [ "-d", os.path.basename(project_dir), "-b", "uno", "-b", "uno", "--no-install-dependencies", ], ) validate_cliresult(result) validate_pioproject(project_dir) config = ProjectConfig(os.path.join(project_dir, "platformio.ini")) config.validate() assert set(config.sections()) == set(["env:uno"]) def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke(project_init_cmd, ["--ide", "vscode"]) assert result.exit_code != 0 assert isinstance(result.exception, ProjectEnvsNotAvailableError) def test_init_ide_vscode(clirunner, validate_cliresult, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke( project_init_cmd, [ "--ide", "vscode", "-b", "uno", "-b", "teensy31", "--no-install-dependencies", ], ) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert all( tmpdir.join(".vscode").join(f).check() for f in ("c_cpp_properties.json", "launch.json") ) assert ( "framework-arduino-avr" in tmpdir.join(".vscode").join("c_cpp_properties.json").read() ) # switch to NodeMCU result = clirunner.invoke( project_init_cmd, ["--ide", "vscode", "-b", "nodemcuv2", "--no-install-dependencies"], ) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert ( "framework-arduinoespressif8266" in tmpdir.join(".vscode").join("c_cpp_properties.json").read() ) # switch to teensy31 via env name result = clirunner.invoke( project_init_cmd, ["--ide", "vscode", "-e", "teensy31", "--no-install-dependencies"], ) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert ( "framework-arduinoteensy" in tmpdir.join(".vscode").join("c_cpp_properties.json").read() ) # switch to the first board result = clirunner.invoke( project_init_cmd, ["--ide", "vscode", "--no-install-dependencies"] ) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert ( "framework-arduino-avr" in tmpdir.join(".vscode").join("c_cpp_properties.json").read() ) def test_init_ide_eclipse(clirunner, validate_cliresult): with clirunner.isolated_filesystem(): result = clirunner.invoke( project_init_cmd, ["-b", "uno", "--ide", "eclipse", "--no-install-dependencies"], ) validate_cliresult(result) validate_pioproject(os.getcwd()) assert all(os.path.isfile(f) for f in (".cproject", ".project")) def test_init_special_board(clirunner, validate_cliresult): with clirunner.isolated_filesystem(): result = clirunner.invoke(project_init_cmd, ["-b", "uno"]) validate_cliresult(result) validate_pioproject(os.getcwd()) result = clirunner.invoke(cmd_boards, ["Arduino Uno", "--json-output"]) validate_cliresult(result) boards = json.loads(result.output) config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini")) config.validate() expected_result = dict( platform=str(boards[0]["platform"]), board="uno", framework=[str(boards[0]["frameworks"][0])], ) assert config.has_section("env:uno") assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( expected_result.items() ) def test_init_enable_auto_uploading(clirunner, validate_cliresult): with clirunner.isolated_filesystem(): result = clirunner.invoke( project_init_cmd, [ "-b", "uno", "--project-option", "targets=upload", "--no-install-dependencies", ], ) validate_cliresult(result) validate_pioproject(os.getcwd()) config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini")) config.validate() expected_result = dict( targets=["upload"], platform="atmelavr", board="uno", framework=["arduino"] ) assert config.has_section("env:uno") assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( expected_result.items() ) def test_init_custom_framework(clirunner, validate_cliresult): with clirunner.isolated_filesystem(): result = clirunner.invoke( project_init_cmd, [ "-b", "teensy31", "--project-option", "framework=mbed", "--no-install-dependencies", ], ) validate_cliresult(result) validate_pioproject(os.getcwd()) config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini")) config.validate() expected_result = dict(platform="teensy", board="teensy31", framework=["mbed"]) assert config.has_section("env:teensy31") assert sorted(config.items(env="teensy31", as_dict=True).items()) == sorted( expected_result.items() ) def test_init_incorrect_board(clirunner): result = clirunner.invoke(project_init_cmd, ["-b", "missed_board"]) assert result.exit_code == 2 assert "Error: Invalid value for" in result.output assert isinstance(result.exception, SystemExit) ================================================ FILE: tests/commands/test_lib.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import json import os import pytest import semantic_version from platformio.commands.lib import cli as cmd_lib from platformio.package.meta import PackageType from platformio.package.vcsclient import VCSClientFactory from platformio.project.config import ProjectConfig from platformio.registry.client import RegistryClient def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): regclient = RegistryClient() project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(""" [env] lib_deps = ArduinoJson [env:one] board = devkit [env:two] framework = foo lib_deps = CustomLib ArduinoJson @ 6.18.5 """) result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "install", "64", "knolleary/PubSubClient@~2.7"], ) validate_cliresult(result) aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert sorted(config.get("env:one", "lib_deps")) == sorted( [ "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], "knolleary/PubSubClient@~2.7", ] ) assert sorted(config.get("env:two", "lib_deps")) == sorted( [ "CustomLib", "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], "knolleary/PubSubClient@~2.7", ] ) # ensure "build" version without NPM spec result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "-e", "one", "install", "mbed-sam-grove/LinkedList"], ) validate_cliresult(result) ll_pkg_data = regclient.get_package( PackageType.LIBRARY, "mbed-sam-grove", "LinkedList" ) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert sorted(config.get("env:one", "lib_deps")) == sorted( [ "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], "knolleary/PubSubClient@~2.7", "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], ] ) # check external package via Git repo result = clirunner.invoke( cmd_lib, [ "-d", str(project_dir), "-e", "one", "install", "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ], ) validate_cliresult(result) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert len(config.get("env:one", "lib_deps")) == 4 assert config.get("env:one", "lib_deps")[3] == ( "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3" ) # test uninstalling # from all envs result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"] ) validate_cliresult(result) # from "one" env result = clirunner.invoke( cmd_lib, [ "-d", str(project_dir), "-e", "one", "uninstall", "knolleary/PubSubClient@~2.7", ], ) validate_cliresult(result) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert len(config.get("env:one", "lib_deps")) == 2 assert len(config.get("env:two", "lib_deps")) == 2 assert config.get("env:one", "lib_deps") == [ "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ] assert config.get("env:two", "lib_deps") == [ "CustomLib", "knolleary/PubSubClient@~2.7", ] # test list result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) validate_cliresult(result) assert "AsyncMqttClient-esphome @ 0.8.3+sha.f5aa899" in result.stdout result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "list", "--json-output"] ) validate_cliresult(result) data = {} for key, value in json.loads(result.stdout).items(): data[os.path.basename(key)] = value ame_lib = next( item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome" ) ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"]) assert len(data["two"]) == 1 assert data["two"][0]["name"] == "PubSubClient" assert "__pkg_dir" in data["one"][0] assert ( ame_lib["__src_url"] == "git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" ) assert ame_lib["version"] == ("0.8.3+sha.%s" % ame_vcs.get_current_revision()) def test_update(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("test-updates") result = clirunner.invoke( cmd_lib, ["-d", str(storage_dir), "install", "ArduinoJson @ 6.18.5", "Blynk @ ~1.2"], ) validate_cliresult(result) result = clirunner.invoke( cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "--json-output"] ) validate_cliresult(result) outdated = json.loads(result.stdout) assert len(outdated) == 2 # ArduinoJson assert outdated[0]["version"] == "6.18.5" assert outdated[0]["versionWanted"] is None assert semantic_version.Version( outdated[0]["versionLatest"] ) > semantic_version.Version("6.18.5") # Blynk assert outdated[1]["version"] == "1.2.0" assert outdated[1]["versionWanted"] is None assert semantic_version.Version( outdated[1]["versionLatest"] ) > semantic_version.Version("1.2.0") # check with spec result = clirunner.invoke( cmd_lib, [ "-d", str(storage_dir), "update", "--dry-run", "--json-output", "ArduinoJson @ ^6", ], ) validate_cliresult(result) outdated = json.loads(result.stdout) assert outdated[0]["version"] == "6.18.5" assert outdated[0]["versionWanted"] == "6.21.5" assert semantic_version.Version( outdated[0]["versionLatest"] ) > semantic_version.Version("6.16.0") # update with spec result = clirunner.invoke( cmd_lib, ["-d", str(storage_dir), "update", "--silent", "ArduinoJson @ ^6.18.5"] ) validate_cliresult(result) result = clirunner.invoke( cmd_lib, ["-d", str(storage_dir), "list", "--json-output"] ) validate_cliresult(result) items = json.loads(result.stdout) assert len(items) == 2 assert items[0]["version"] == "6.21.5" assert items[1]["version"] == "1.2.0" # Check incompatible result = clirunner.invoke( cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "ArduinoJson @ ^6"] ) with pytest.raises( AssertionError, match="This command is deprecated", ): validate_cliresult(result) result = clirunner.invoke( cmd_lib, ["-d", str(storage_dir), "update", "ArduinoJson @ ^6"] ) validate_cliresult(result) assert "ArduinoJson@6.21.5 is already up-to-date" in result.stdout ================================================ FILE: tests/commands/test_lib_complex.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=line-too-long import json import re from platformio.cli import PlatformioCLI from platformio.commands.lib import cli as cmd_lib from platformio.package.exception import UnknownPackageError from platformio.util import strip_ansi_codes PlatformioCLI.leftover_args = ["--json-output"] # hook for click ARDUINO_JSON_VERSION = "6.21.5" def test_search(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) validate_cliresult(result) match = re.search(r"Found\s+(\d+)\spackages", result.output) assert int(match.group(1)) > 2 result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) validate_cliresult(result) match = re.search(r"Found\s+(\d+)\spackages", result.output) assert int(match.group(1)) > 1 def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ "-g", "install", "64", "ArduinoJson@~6", "547@2.7.3", "AsyncMqttClient@<=0.8.2", "Adafruit PN532@1.3.2", ], ) validate_cliresult(result) # install unknown library result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) assert result.exit_code != 0 assert isinstance(result.exception, UnknownPackageError) items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "ArduinoJson", f"ArduinoJson@{ARDUINO_JSON_VERSION}", "NeoPixelBus", "AsyncMqttClient", "ESPAsyncTCP", "AsyncTCP", "Adafruit PN532", "Adafruit BusIO", ] assert set(items1) == set(items2) def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ "-g", "install", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", "SomeLib=https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.11.0/DallasTemperature-3.11.0.tar.gz", "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", ], ) validate_cliresult(result) # incorrect requirements result = clirunner.invoke( cmd_lib, [ "-g", "install", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", ], ) assert result.exit_code != 0 items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "ArduinoJson", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "SomeLib", "OneWire", "ESP32WebServer", ] assert set(items1) >= set(items2) def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ "-g", "install", "https://github.com/gioblu/PJON.git#3.0", "https://github.com/gioblu/PJON.git#6.2", "https://github.com/bblanchon/ArduinoJson.git", "https://github.com/platformio/platformio-libmirror.git", # "https://developer.mbed.org/users/simon/code/TextLCD/", "https://github.com/knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", ], ) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "platformio-libmirror", "PubSubClient", ] assert set(items1) >= set(items2) def test_install_duplicates( # pylint: disable=unused-argument clirunner, validate_cliresult, without_internet ): # registry result = clirunner.invoke( cmd_lib, [ "-g", "install", "https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.11.0/DallasTemperature-3.11.0.tar.gz", ], ) validate_cliresult(result) assert "is already installed" in result.output # archive result = clirunner.invoke( cmd_lib, [ "-g", "install", "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", ], ) validate_cliresult(result) assert "is already installed" in result.output # repository result = clirunner.invoke( cmd_lib, ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], ) validate_cliresult(result) assert "is already installed" in result.output def test_global_lib_list(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "list"]) validate_cliresult(result) assert all( n in result.output for n in ( "required: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", f"ArduinoJson @ {ARDUINO_JSON_VERSION}", "required: git+https://github.com/gioblu/PJON.git#3.0", "PJON @ 3.0.0+sha.1fb26f", ) ), result.output result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) assert all( n in result.output for n in ( "__pkg_dir", '"__src_url": "git+https://github.com/gioblu/PJON.git#6.2"', f'"version": "{ARDUINO_JSON_VERSION}"', ) ) items1 = [i["name"] for i in json.loads(result.output)] items2 = [ "Adafruit BusIO", "Adafruit PN532", "ArduinoJson", "ArduinoJson", "ArduinoJson", "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "DallasTemperature", "ESP32WebServer", "ESPAsyncTCP", "NeoPixelBus", "OneWire", "PJON", "PJON", "platformio-libmirror", "PubSubClient", ] assert sorted(items1) == sorted(items2) versions1 = [ "{name}@{version}".format(**item) for item in json.loads(result.output) ] versions2 = [ "ArduinoJson@5.8.2", f"ArduinoJson@{ARDUINO_JSON_VERSION}", "AsyncMqttClient@0.8.2", "NeoPixelBus@2.7.3", "PJON@6.2.0+sha.07fe9aa", "PJON@3.0.0+sha.1fb26fd", "PubSubClient@2.6.0+sha.bef5814", "Adafruit PN532@1.3.2", ] assert set(versions1) >= set(versions2) def test_global_lib_update_check(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) validate_cliresult(result) output = json.loads(result.output) assert set( ["Adafruit PN532", "AsyncMqttClient", "AsyncTCP", "ESPAsyncTCP", "NeoPixelBus"] ) == set(lib["name"] for lib in output) def test_global_lib_update(clirunner, validate_cliresult): # update library using package directory result = clirunner.invoke( cmd_lib, ["-g", "update", "NeoPixelBus", "--dry-run", "--json-output"] ) validate_cliresult(result) oudated = json.loads(result.output) assert len(oudated) == 1 assert "__pkg_dir" in oudated[0] result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) validate_cliresult(result) assert "Removing NeoPixelBus @ 2.7.3" in strip_ansi_codes(result.output) # update all libraries result = clirunner.invoke( cmd_lib, ["-g", "update", "adafruit/Adafruit PN532", "marvinroger/AsyncMqttClient"], ) validate_cliresult(result) # update unknown library result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) assert result.exit_code != 0 assert isinstance(result.exception, UnknownPackageError) def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): # uninstall using package directory result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) validate_cliresult(result) items = json.loads(result.output) items = sorted(items, key=lambda item: item["__pkg_dir"]) result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) validate_cliresult(result) assert "Removing %s" % items[0]["name"] in strip_ansi_codes(result.output) # uninstall the rest libraries result = clirunner.invoke( cmd_lib, [ "-g", "uninstall", "OneWire", "https://github.com/bblanchon/ArduinoJson.git", "ArduinoJson@!=5.6.7", "Adafruit PN532", ], ) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "AsyncMqttClient", "platformio-libmirror", "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "ESPAsyncTCP@1.2.0", "AsyncTCP", "ArduinoJson", "ESPAsyncTCP", "ESP32WebServer", "PJON", "NeoPixelBus", "PJON@src-79de467ebe19de18287becff0a1fb42d", "SomeLib", ] assert set(items1) == set(items2) # uninstall unknown library result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) assert result.exit_code != 0 assert isinstance(result.exception, UnknownPackageError) def test_lib_show(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["show", "64"]) validate_cliresult(result) assert all(s in result.output for s in ("ArduinoJson", "Arduino")) result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) validate_cliresult(result) assert "OneWire" in result.output def test_lib_builtin(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["builtin"]) validate_cliresult(result) result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) validate_cliresult(result) def test_lib_stats(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) validate_cliresult(result) assert set( [ "dlweek", "added", "updated", "topkeywords", "dlmonth", "dlday", "lastkeywords", ] ) == set(json.loads(result.output).keys()) ================================================ FILE: tests/commands/test_platform.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import json import os from platformio.commands import platform as cli_platform from platformio.package.exception import UnknownPackageError from platformio.util import strip_ansi_codes def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_search, ["arduino", "--json-output"] ) validate_cliresult(result) search_result = json.loads(result.output) assert isinstance(search_result, list) assert search_result platforms = [item["name"] for item in search_result] assert "atmelsam" in platforms def test_search_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_search, ["arduino"]) validate_cliresult(result) assert "atmelavr" in result.output def test_install_unknown_version(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) assert result.exit_code != 0 assert isinstance(result.exception, UnknownPackageError) def test_install_unknown_from_registry(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) assert result.exit_code != 0 assert isinstance(result.exception, UnknownPackageError) # def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): # result = clirunner.invoke( # cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], # ) # assert result.exit_code != 0 # assert isinstance(result.exception, IncompatiblePlatform) def test_install_core_3_dev_platform(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, ["atmelavr@2.2.0", "--skip-default-package"], ) assert result.exit_code == 0 def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, ["atmelavr@4.2.0", "--skip-default-package", "--with-package", "tool-avrdude"], ) validate_cliresult(result) output = strip_ansi_codes(result.output) assert "atmelavr@4.2.0" in output assert not os.path.isdir(str(isolated_pio_core.join("packages"))) def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, [ "https://github.com/platformio/platform-espressif8266.git", "--skip-default-package", ], ) validate_cliresult(result) assert "espressif8266" in result.output assert not os.path.isdir(str(isolated_pio_core.join("packages"))) def test_list_json_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_list, ["--json-output"]) validate_cliresult(result) list_result = json.loads(result.output) assert isinstance(list_result, list) assert list_result platforms = [item["name"] for item in list_result] assert set(["atmelavr", "espressif8266"]) == set(platforms) def test_list_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_list) validate_cliresult(result) assert all(s in result.output for s in ("atmelavr", "espressif8266")) def test_update_check(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.package_install_cmd, ["--global", "--tool", "platformio/tool-avrdude@~1.60300.0"], ) validate_cliresult(result) result = clirunner.invoke( cli_platform.platform_update, ["--dry-run", "--json-output"] ) validate_cliresult(result) output = json.loads(result.output) assert len(output) == 1 assert output[0]["name"] == "atmelavr" assert len(isolated_pio_core.join("packages").listdir()) == 1 def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update, ["atmelavr"]) validate_cliresult(result) output = strip_ansi_codes(result.output) assert "Removing atmelavr @ 4.2.0" in output assert "Platform Manager: Installing platformio/atmelavr @" in output assert len(isolated_pio_core.join("packages").listdir()) == 2 def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_uninstall, ["atmelavr@2.2.0", "atmelavr", "espressif8266"] ) validate_cliresult(result) assert not isolated_pio_core.join("platforms").listdir() ================================================ FILE: tests/commands/test_run.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from pathlib import Path from platformio.run.cli import cli as cmd_run def test_generic_build(clirunner, validate_cliresult, tmpdir): build_flags = [ ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), ("-Iinclude", "-Iinclude"), ("-include cpppath-include.h", "cpppath-include.h"), ("-Iextra_inc", "-Iextra_inc"), ("-Inon-existing-dir", "non-existing-dir"), ( "-include $PROJECT_DIR/lib/component/component-forced-include.h", "component-forced-include.h", ), ] tmpdir.join("platformio.ini").write(""" [env:native] platform = native extra_scripts = pre:pre_script.py post_script.py lib_ldf_mode = deep+ build_src_flags = -DI_AM_ONLY_SRC_FLAG build_flags = ; -DCOMMENTED_MACRO %s ; inline comment """ % " ".join([f[0] for f in build_flags])) tmpdir.join("pre_script.py").write(""" Import("env") def post_prog_action(source, target, env): print("post_prog_action is called") env.AddPostAction("$PROGPATH", post_prog_action) """) tmpdir.join("post_script.py").write(""" Import("projenv") projenv.Append(CPPDEFINES="POST_SCRIPT_MACRO") """) tmpdir.mkdir("extra_inc").join("foo.h").write(""" #define FOO """) tmpdir.mkdir("src").join("main.cpp").write(""" #include "foo.h" #ifndef FOO #error "FOO" #endif #ifdef I_AM_ONLY_SRC_FLAG #include #else #error "I_AM_ONLY_SRC_FLAG" #endif #if !defined(TEST_INT) || TEST_INT != 13 #error "TEST_INT" #endif #ifndef TEST_STR_SPACE #error "TEST_STR_SPACE" #endif #ifndef I_AM_COMPONENT #error "I_AM_COMPONENT" #endif #ifndef POST_SCRIPT_MACRO #error "POST_SCRIPT_MACRO" #endif #ifndef I_AM_FORCED_COMPONENT_INCLUDE #error "I_AM_FORCED_COMPONENT_INCLUDE" #endif #ifndef I_AM_FORCED_CPPPATH_INCLUDE #error "I_AM_FORCED_CPPPATH_INCLUDE" #endif #ifdef COMMENTED_MACRO #error "COMMENTED_MACRO" #endif int main() { } """) tmpdir.mkdir("include").join("cpppath-include.h").write(""" #define I_AM_FORCED_CPPPATH_INCLUDE """) component_dir = tmpdir.mkdir("lib").mkdir("component") component_dir.join("component.h").write(""" #define I_AM_COMPONENT #ifndef I_AM_ONLY_SRC_FLAG #error "I_AM_ONLY_SRC_FLAG" #endif void dummy(void); """) component_dir.join("component.cpp").write(""" #ifdef I_AM_ONLY_SRC_FLAG #error "I_AM_ONLY_SRC_FLAG" #endif void dummy(void ) {}; """) component_dir.join("component-forced-include.h").write(""" #define I_AM_FORCED_COMPONENT_INCLUDE """) result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) assert "post_prog_action is called" in result.output build_output = result.output[result.output.find("Scanning dependencies...") :] for flag in build_flags: assert flag[1] in build_output, flag def test_build_unflags(clirunner, validate_cliresult, tmpdir): tmpdir.join("platformio.ini").write(""" [env:native] platform = native build_unflags = -DTMP_MACRO_1=45 -DTMP_MACRO_3=13 -DTMP_MACRO_4 -DNON_EXISTING_MACRO -I. -lunknownLib -Os build_flags = -DTMP_MACRO_3=10 extra_scripts = pre:extra.py """) tmpdir.join("extra.py").write(""" Import("env") env.Append(CPPPATH="%s") env.Append(CPPDEFINES="TMP_MACRO_1") env.Append(CPPDEFINES=["TMP_MACRO_2"]) env.Append(CPPDEFINES=[("TMP_MACRO_3", 13)]) env.Append(CPPDEFINES=[("TMP_MACRO_4", 4)]) env.Append(CCFLAGS=["-Os"]) env.Append(LIBS=["unknownLib"]) """ % str(tmpdir)) tmpdir.mkdir("src").join("main.c").write(""" #ifndef TMP_MACRO_1 #error "TMP_MACRO_1 should be defined" #endif #ifndef TMP_MACRO_2 #error "TMP_MACRO_2 should be defined" #endif #if TMP_MACRO_3 != 10 #error "TMP_MACRO_3 should be 10" #endif #ifdef TMP_MACRO_4 #error "TMP_MACRO_4 should not be defined" #endif int main() { } """) result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) build_output = result.output[result.output.find("Scanning dependencies...") :] assert "-DTMP_MACRO1" not in build_output assert "-Os" not in build_output assert str(tmpdir) not in build_output def test_debug_default_build_flags(clirunner, validate_cliresult, tmpdir): tmpdir.join("platformio.ini").write(""" [env:native] platform = native build_type = debug """) tmpdir.mkdir("src").join("main.c").write(""" int main() { } """) result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) build_output = result.output[result.output.find("Scanning dependencies...") :] for line in build_output.split("\n"): if line.startswith("gcc"): assert all(line.count(flag) == 1 for flag in ("-Og", "-g2", "-ggdb2")) assert all( line.count("-%s%d" % (flag, level)) == 0 for flag in ("O", "g", "ggdb") for level in (0, 1, 3) ) assert "-Os" not in line def test_debug_custom_build_flags(clirunner, validate_cliresult, tmpdir): custom_debug_build_flags = ("-O3", "-g3", "-ggdb3") tmpdir.join("platformio.ini").write(""" [env:native] platform = native build_type = debug debug_build_flags = %s """ % " ".join(custom_debug_build_flags)) tmpdir.mkdir("src").join("main.c").write(""" int main() { } """) result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) build_output = result.output[result.output.find("Scanning dependencies...") :] for line in build_output.split("\n"): if line.startswith("gcc"): assert all(line.count(f) == 1 for f in custom_debug_build_flags) assert all( line.count("-%s%d" % (flag, level)) == 0 for flag in ("O", "g", "ggdb") for level in (0, 1, 2) ) assert all("-O%s" % optimization not in line for optimization in ("g", "s")) def test_symlinked_libs(clirunner, validate_cliresult, tmp_path: Path): external_pkg_dir = tmp_path / "External" external_pkg_dir.mkdir() (external_pkg_dir / "External.h").write_text(""" #define EXTERNAL 1 """) (external_pkg_dir / "library.json").write_text(""" { "name": "External", "version": "1.0.0" } """) project_dir = tmp_path / "project" src_dir = project_dir / "src" src_dir.mkdir(parents=True) (src_dir / "main.c").write_text(""" #include # #if !defined(EXTERNAL) #error "EXTERNAL is not defined" #endif int main() { } """) (project_dir / "platformio.ini").write_text(""" [env:native] platform = native lib_deps = symlink://../External """) result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir)]) validate_cliresult(result) def test_stringification(clirunner, validate_cliresult, tmp_path: Path): project_dir = tmp_path / "project" src_dir = project_dir / "src" src_dir.mkdir(parents=True) (src_dir / "main.c").write_text(""" #include int main(void) { printf("MACRO_1=<%s>\\n", MACRO_1); printf("MACRO_2=<%s>\\n", MACRO_2); printf("MACRO_3=<%s>\\n", MACRO_3); printf("MACRO_4=<%s>\\n", MACRO_4); return(0); } """) (project_dir / "platformio.ini").write_text(""" [env:native] platform = native extra_scripts = script.py build_flags = '-DMACRO_1="Hello World!"' '-DMACRO_2="Text is \\\\"Quoted\\\\""' """) (project_dir / "script.py").write_text(""" Import("projenv") projenv.Append(CPPDEFINES=[ ("MACRO_3", projenv.StringifyMacro('Hello "World"! Isn\\'t true?')), ("MACRO_4", projenv.StringifyMacro("Special chars: ',(,),[,],:")) ]) """) result = clirunner.invoke( cmd_run, ["--project-dir", str(project_dir), "-t", "exec"] ) validate_cliresult(result) assert "MACRO_1=" in result.output assert 'MACRO_2=' in result.output assert 'MACRO_3=' in result.output assert "MACRO_4=" in result.output def test_ldf(clirunner, validate_cliresult, tmp_path: Path): project_dir = tmp_path / "project" # libs lib_dir = project_dir / "lib" a_lib_dir = lib_dir / "a" a_lib_dir.mkdir(parents=True) (a_lib_dir / "a.h").write_text(""" #include """) # b b_lib_dir = lib_dir / "b" b_lib_dir.mkdir(parents=True) (b_lib_dir / "some_from_b.h").write_text("") # c c_lib_dir = lib_dir / "c" c_lib_dir.mkdir(parents=True) (c_lib_dir / "parse_c_by_name.h").write_text(""" void some_func(); """) (c_lib_dir / "parse_c_by_name.c").write_text(""" #include #include void some_func() { } """) (c_lib_dir / "some.c").write_text(""" #include """) # d d_lib_dir = lib_dir / "d" d_lib_dir.mkdir(parents=True) (d_lib_dir / "d.h").write_text("") # project src_dir = project_dir / "src" src_dir.mkdir(parents=True) (src_dir / "main.h").write_text(""" #include #include """) (src_dir / "main.c").write_text(""" #include int main() { } """) (project_dir / "platformio.ini").write_text(""" [env:native] platform = native """) result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir)]) validate_cliresult(result) ================================================ FILE: tests/commands/test_settings.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio import app from platformio.commands.settings import cli def test_settings_check(clirunner, validate_cliresult): result = clirunner.invoke(cli, ["get"]) validate_cliresult(result) assert result.output for item in app.DEFAULT_SETTINGS.items(): assert item[0] in result.output ================================================ FILE: tests/commands/test_test.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import sys import xml.etree.ElementTree as ET from pathlib import Path import pytest from platformio import proc from platformio.fs import load_json from platformio.test.cli import cli as pio_test_cmd def test_calculator_example(tmp_path: Path): junit_output_path = tmp_path / "junit.xml" project_dir = tmp_path / "project" shutil.copytree( os.path.join("examples", "unit-testing", "calculator"), str(project_dir) ) result = proc.exec_command( [ "platformio", "test", "-d", str(project_dir), "-e", "uno", "-e", "native", "--junit-output-path", str(junit_output_path), ] ) assert result["returncode"] != 0 # pylint: disable=unsupported-membership-test assert all( s in (result["err"] + result["out"]) for s in ("ERRORED", "PASSED", "FAILED") ), result["out"] # test JUnit output junit_testsuites = ET.parse(junit_output_path).getroot() assert int(junit_testsuites.get("tests")) == 11 assert int(junit_testsuites.get("errors")) == 2 assert int(junit_testsuites.get("failures")) == 1 assert len(junit_testsuites.findall("testsuite")) == 6 junit_errored_testcase = junit_testsuites.find( ".//testcase[@name='uno:test_embedded']" ) assert junit_errored_testcase.get("status") == "ERRORED" assert junit_errored_testcase.find("error").get("type") == "UnitTestSuiteError" junit_failed_testcase = junit_testsuites.find( ".//testsuite[@name='native:test_desktop']" "/testcase[@name='test_calculator_division']" ) assert junit_failed_testcase.get("status") == "FAILED" assert junit_failed_testcase.find("failure").get("message") == "Expected 32 Was 33" def test_list_tests(clirunner, validate_cliresult, tmp_path: Path): json_output_path = tmp_path / "report.json" project_dir = tmp_path / "project" shutil.copytree( os.path.join("examples", "unit-testing", "calculator"), str(project_dir) ) result = clirunner.invoke( pio_test_cmd, [ "-d", str(project_dir), "--list-tests", "--json-output-path", str(json_output_path), ], ) validate_cliresult(result) # test JSON json_report = load_json(str(json_output_path)) assert json_report["testcase_nums"] == 0 assert json_report["failure_nums"] == 0 assert json_report["skipped_nums"] == 0 assert len(json_report["test_suites"]) == 6 def test_group_and_custom_runner(clirunner, tmp_path: Path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(""" [env:native] platform = native test_framework = custom """) test_dir = project_dir / "test" # non-test folder, does not start with "test_" disabled_dir = test_dir / "disabled" disabled_dir.mkdir(parents=True) (disabled_dir / "main.c").write_text(""" #include int main() { printf("Disabled test suite\\n") } """) # root (test_dir / "my_extra.h").write_text(""" #ifndef MY_EXTRA_H #define MY_EXTRA_H #include void my_extra_fun(void); #endif """) (test_dir / "my_extra.c").write_text(""" #include "my_extra.h" void my_extra_fun(void) { printf("Called from my_extra_fun\\n"); } """) # test group test_group = test_dir / "group" test_group.mkdir(parents=True) (test_group / "test_custom_runner.py").write_text(""" import click from platformio.test.runners.unity import UnityTestRunner class CustomTestRunner(UnityTestRunner): def teardown(self): click.echo("CustomTestRunner::TearDown called") """) # test suite test_suite_dir = test_group / "test_nested" test_include_dir = test_suite_dir / "include" test_include_dir.mkdir(parents=True) (test_include_dir / "my_nested.h").write_text(""" #define TEST_ONE 1 """) (test_suite_dir / "main.c").write_text(""" #include #include #include void setUp(){ my_extra_fun(); } void tearDown(void) { // clean stuff up here } void dummy_test_passed(void) { TEST_ASSERT_EQUAL(1, TEST_ONE); } void dummy_test_failed(void) { TEST_ASSERT_LESS_THAN(10, 15); } int main() { UNITY_BEGIN(); RUN_TEST(dummy_test_passed); RUN_TEST(dummy_test_failed); UNITY_END(); } """) result = clirunner.invoke( pio_test_cmd, ["-d", str(project_dir), "-e", "native", "--verbose"], ) assert result.exit_code != 0 assert "2 Tests 1 Failures 0 Ignored" in result.output assert "Called from my_extra_fun" in result.output assert "CustomTestRunner::TearDown called" in result.output assert "Disabled test suite" not in result.output def test_crashed_program(clirunner, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write(""" [env:native] platform = native """) test_dir = project_dir.mkdir("test") test_dir.join("test_main.c").write(""" #include #include void setUp(){ printf("setUp called"); } void tearDown(){ printf("tearDown called"); } void dummy_test(void) { TEST_ASSERT_EQUAL(1, 1); } int main(int argc, char *argv[]) { printf("Address boundary error is %s", argv[-1]); UNITY_BEGIN(); RUN_TEST(dummy_test); UNITY_END(); return 0; } """) result = clirunner.invoke( pio_test_cmd, ["-d", str(project_dir), "-e", "native"], ) assert result.exit_code != 0 assert any( s in result.output for s in ("Program received signal", "Program errored with") ) # @pytest.mark.skipif( # sys.platform != "darwin", reason="runs only on macOS (issue with SimAVR)" # ) # def test_custom_testing_command(clirunner, validate_cliresult, tmp_path: Path): # project_dir = tmp_path / "project" # project_dir.mkdir() # (project_dir / "platformio.ini").write_text( # """ # [env:uno] # platform = atmelavr # framework = arduino # board = uno # platform_packages = # platformio/tool-simavr @ ^1 # test_speed = 9600 # test_testing_command = # ${platformio.packages_dir}/tool-simavr/bin/simavr # -m # atmega328p # -f # 16000000L # ${platformio.build_dir}/${this.__env__}/firmware.elf # """ # ) # test_dir = project_dir / "test" / "test_dummy" # test_dir.mkdir(parents=True) # (test_dir / "test_main.cpp").write_text( # """ # #include # #include # void setUp(void) { # // set stuff up here # } # void tearDown(void) { # // clean stuff up here # } # void dummy_test(void) { # TEST_ASSERT_EQUAL(1, 1); # } # void setup() { # UNITY_BEGIN(); # RUN_TEST(dummy_test); # UNITY_END(); # } # void loop() { # delay(1000); # } # """ # ) # result = clirunner.invoke( # pio_test_cmd, # ["-d", str(project_dir), "--without-uploading"], # ) # validate_cliresult(result) # assert "dummy_test" in result.output def test_unity_setup_teardown(clirunner, validate_cliresult, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write(""" [env:native] platform = native """) test_dir = project_dir.mkdir("test") test_dir.join("test_main.h").write(""" #include #include """) test_dir.join("test_main.c").write(""" #include "test_main.h" void setUp(){ printf("setUp called"); } void tearDown(){ printf("tearDown called"); } void dummy_test(void) { TEST_ASSERT_EQUAL(1, 1); } int main() { UNITY_BEGIN(); RUN_TEST(dummy_test); UNITY_END(); } """) result = clirunner.invoke( pio_test_cmd, ["-d", str(project_dir), "-e", "native"], ) validate_cliresult(result) assert all(f in result.output for f in ("setUp called", "tearDown called")) def test_unity_custom_config(clirunner, validate_cliresult, tmp_path: Path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(""" [env:native] platform = native """) test_dir = project_dir / "test" / "native" / "test_component" test_dir.mkdir(parents=True) (test_dir.parent / "unity_config.h").write_text(""" #include #define CUSTOM_UNITY_CONFIG #define UNITY_OUTPUT_CHAR(c) putchar(c) #define UNITY_OUTPUT_FLUSH() fflush(stdout) """) (test_dir / "test_main.c").write_text(""" #include #include void setUp(){ #ifdef CUSTOM_UNITY_CONFIG printf("Found custom unity_config.h\\n"); #endif } void tearDown(){ } void dummy_test(void) { TEST_ASSERT_EQUAL(1, 1); } int main() { UNITY_BEGIN(); RUN_TEST(dummy_test); UNITY_END(); } """) result = clirunner.invoke( pio_test_cmd, ["-d", str(project_dir), "-e", "native", "--verbose"], ) validate_cliresult(result) assert all(f in result.output for f in ("Found custom unity_config", "dummy_test")) def test_legacy_unity_custom_transport(clirunner, validate_cliresult, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write(""" [env:embedded] platform = ststm32 framework = stm32cube board = nucleo_f401re test_transport = custom """) test_dir = project_dir.mkdir("test") test_dir.join("test_main.c").write(""" #include void setUp(void) { // set stuff up here } void tearDown(void) { // clean stuff up here } void dummy_test(void) { TEST_ASSERT_EQUAL(1, 1); } int main() { UNITY_BEGIN(); RUN_TEST(dummy_test); UNITY_END(); } """) test_dir.join("unittest_transport.h").write(""" #ifdef __cplusplus extern "C" { #endif void unittest_uart_begin(){} void unittest_uart_putchar(char c){} void unittest_uart_flush(){} void unittest_uart_end(){} #ifdef __cplusplus } #endif """) result = clirunner.invoke( pio_test_cmd, [ "-d", str(project_dir), "--without-testing", "--without-uploading", ], ) validate_cliresult(result) @pytest.mark.skipif( sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", reason="skip Github Actions on Windows (MinGW issue)", ) def test_doctest_framework(clirunner, tmp_path: Path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(""" [env:native] platform = native test_framework = doctest """) test_dir = project_dir / "test" / "test_dummy" test_dir.mkdir(parents=True) (test_dir / "test_main.cpp").write_text(""" #define DOCTEST_CONFIG_IMPLEMENT #include TEST_CASE("[math] basic stuff") { CHECK(6 > 5); CHECK(6 > 7); } TEST_CASE("should be skipped " * doctest::skip()) { CHECK(2 > 5); } TEST_CASE("vectors can be sized and resized") { std::vector v(5); REQUIRE(v.size() == 5); REQUIRE(v.capacity() >= 5); SUBCASE("adding to the vector increases it's size") { v.push_back(1); CHECK(v.size() == 6); CHECK(v.capacity() >= 6); } SUBCASE("reserving increases just the capacity") { v.reserve(6); CHECK(v.size() == 5); CHECK(v.capacity() >= 6); } } TEST_CASE("WARN level of asserts don't fail the test case") { WARN(0); WARN_FALSE(1); WARN_EQ(1, 0); } TEST_SUITE("scoped test suite") { TEST_CASE("part of scoped") { FAIL("Error message"); } TEST_CASE("part of scoped 2") { FAIL(""); } } int main(int argc, char **argv) { doctest::Context context; context.setOption("success", true); context.setOption("no-exitcode", true); context.applyCommandLine(argc, argv); return context.run(); } """) junit_output_path = tmp_path / "junit.xml" result = clirunner.invoke( pio_test_cmd, [ "-d", str(project_dir), "--junit-output-path", str(junit_output_path), ], ) assert result.exit_code != 0 # test JUnit output junit_testsuites = ET.parse(junit_output_path).getroot() assert int(junit_testsuites.get("tests")) == 8 assert int(junit_testsuites.get("errors")) == 0 assert int(junit_testsuites.get("failures")) == 3 assert len(junit_testsuites.findall("testsuite")) == 1 junit_failed_testcase = junit_testsuites.find( ".//testcase[@name='scoped test suite/part of scoped']" ) assert junit_failed_testcase.get("status") == "FAILED" assert junit_failed_testcase.find("failure").get("message") == "Error message" assert "TEST SUITE: scoped test suite" in junit_failed_testcase.find("failure").text # test program arguments json_output_path = tmp_path / "report.json" result = clirunner.invoke( pio_test_cmd, [ "-d", str(project_dir), "--json-output-path", str(json_output_path), "-a", "-aa=1", # fail after the 1 error ], ) assert result.exit_code != 0 assert "1 test cases" in result.output # test JSON json_report = load_json(str(json_output_path)) assert json_report["testcase_nums"] == 1 assert json_report["failure_nums"] == 1 @pytest.mark.skipif( sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", reason="skip Github Actions on Windows (MinGW issue)", ) def test_googletest_framework(clirunner, tmp_path: Path): project_dir = tmp_path / "project" shutil.copytree( os.path.join("examples", "unit-testing", "googletest"), str(project_dir) ) junit_output_path = tmp_path / "junit.xml" result = clirunner.invoke( pio_test_cmd, [ "-d", str(project_dir), "-e", "native", "--junit-output-path", str(junit_output_path), ], ) assert result.exit_code != 0 # test JUnit output junit_testsuites = ET.parse(junit_output_path).getroot() assert int(junit_testsuites.get("tests")) == 4 assert int(junit_testsuites.get("errors")) == 0 assert int(junit_testsuites.get("failures")) == 1 assert len(junit_testsuites.findall("testsuite")) == 4 junit_failed_testcase = junit_testsuites.find(".//testcase[@name='FooTest.Bar']") assert junit_failed_testcase.get("status") == "FAILED" assert "test_main.cpp" in junit_failed_testcase.get("file") assert junit_failed_testcase.get("line") == "26" assert junit_failed_testcase.find("failure").get("message") == "Failure" assert "Expected equality" in junit_failed_testcase.find("failure").text # test program arguments json_output_path = tmp_path / "report.json" result = clirunner.invoke( pio_test_cmd, [ "-d", str(project_dir), "-e", "native", "--json-output-path", str(json_output_path), "-a", "--gtest_filter=-FooTest.Bar", ], ) assert result.exit_code == 0 # test JSON json_report = load_json(str(json_output_path)) assert json_report["testcase_nums"] == 3 assert json_report["failure_nums"] == 0 assert json_report["skipped_nums"] == 1 assert len(json_report["test_suites"]) == 4 ================================================ FILE: tests/conftest.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import email import functools import imaplib import os import time import pytest from click.testing import CliRunner from platformio import http from platformio.package.meta import PackageSpec, PackageType from platformio.registry.client import RegistryClient def pytest_configure(config): config.addinivalue_line("markers", "skip_ci: mark a test that will not run in CI") @pytest.fixture(scope="session") def validate_cliresult(): def decorator(result): assert result.exit_code == 0, "{} => {}".format(result.exception, result.output) assert not result.exception, "{} => {}".format(result.exception, result.output) return decorator @pytest.fixture(scope="session") def clirunner(request, tmpdir_factory): cache_dir = tmpdir_factory.mktemp(".cache") backup_env_vars = { "PLATFORMIO_CACHE_DIR": {"new": str(cache_dir)}, "PLATFORMIO_WORKSPACE_DIR": {"new": None}, } for key, item in backup_env_vars.items(): # pylint: disable=unnecessary-dict-index-lookup backup_env_vars[key]["old"] = os.environ.get(key) if item["new"] is not None: os.environ[key] = item["new"] elif key in os.environ: del os.environ[key] def fin(): for key, item in backup_env_vars.items(): if item["old"] is not None: os.environ[key] = item["old"] elif key in os.environ: del os.environ[key] request.addfinalizer(fin) return CliRunner() def _isolated_pio_core(request, tmpdir_factory): core_dir = tmpdir_factory.mktemp(".platformio") os.environ["PLATFORMIO_CORE_DIR"] = str(core_dir) def fin(): if "PLATFORMIO_CORE_DIR" in os.environ: del os.environ["PLATFORMIO_CORE_DIR"] request.addfinalizer(fin) return core_dir @pytest.fixture(scope="module") def isolated_pio_core(request, tmpdir_factory): return _isolated_pio_core(request, tmpdir_factory) @pytest.fixture(scope="function") def func_isolated_pio_core(request, tmpdir_factory): return _isolated_pio_core(request, tmpdir_factory) @pytest.fixture(scope="function") def without_internet(monkeypatch): monkeypatch.setattr(http, "_internet_on", lambda: False) @pytest.fixture def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals def _receive_email(from_who): test_email = os.environ["TEST_EMAIL_LOGIN"] test_password = os.environ["TEST_EMAIL_PASSWORD"] imap_server = os.environ["TEST_EMAIL_IMAP_SERVER"] def get_body(msg): if msg.is_multipart(): return get_body(msg.get_payload(0)) return msg.get_payload(None, True) result = None start_time = time.time() while not result: time.sleep(5) server = imaplib.IMAP4_SSL(imap_server) server.login(test_email, test_password) server.select("INBOX") _, mails = server.search(None, "ALL") for index in mails[0].split(): status, data = server.fetch(index, "(RFC822)") if status != "OK" or not data or not isinstance(data[0], tuple): continue msg = email.message_from_string( data[0][1].decode("ASCII", errors="surrogateescape") ) if from_who not in msg.get("To"): continue if "gmail" in imap_server: server.store(index, "+X-GM-LABELS", "\\Trash") server.store(index, "+FLAGS", "\\Deleted") server.expunge() result = get_body(msg).decode() if time.time() - start_time > 120: break server.close() server.logout() return result return _receive_email @pytest.fixture(scope="session") def get_pkg_latest_version(): @functools.lru_cache() def wrap(spec, pkg_type=None): if not isinstance(spec, PackageSpec): spec = PackageSpec(spec) pkg_type = pkg_type or PackageType.LIBRARY client = RegistryClient() pkg = client.get_package(pkg_type, spec.owner, spec.name) return pkg["version"]["name"] return wrap ================================================ FILE: tests/misc/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: tests/misc/ino2cpp/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: tests/misc/ino2cpp/examples/basic/basic.ino ================================================ #define SQR(a) \ ( a * a ) typedef struct Item item; struct Item { byte foo[50]; int bar; void (*noob)(item*); }; // test callback class Foo { public: Foo(void (*function)()) { #warning "Line number is 16" } bool childFunc() { } }; Foo foo(&fooCallback); // template T Add(T n1, T n2) { return n1 + n2; } void setup() { struct Item item1; myFunction(&item1); } void loop() { } void myFunction(struct Item *item) { } #warning "Line number is 46" void fooCallback(){ } extern "C" { void some_extern(const char *fmt, ...); }; void some_extern(const char *fmt, ...) { } // юнікод ================================================ FILE: tests/misc/ino2cpp/examples/multifiles/bar.ino ================================================ unsigned int barFunc () // my comment { return 0; } ================================================ FILE: tests/misc/ino2cpp/examples/multifiles/foo.pde ================================================ char buf[5]; void setup() { fooFunc(); } void loop() { } char* fooFunc() { return buf; } ================================================ FILE: tests/misc/ino2cpp/examples/strmultilines/main.ino ================================================ const char headerEndP[] PROGMEM = "\ \ \ "; void setup() { } const char javaScriptPinControlP[] PROGMEM = "
\
\ "; #warning "Line 75" void loop() { } ================================================ FILE: tests/misc/ino2cpp/test_ino2cpp.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from os import listdir from os.path import dirname, isdir, join, normpath from platformio.commands.ci import cli as cmd_ci EXAMPLES_DIR = normpath(join(dirname(__file__), "examples")) def pytest_generate_tests(metafunc): if "piotest_dir" not in metafunc.fixturenames: return test_dirs = [] for name in listdir(EXAMPLES_DIR): if isdir(join(EXAMPLES_DIR, name)): test_dirs.append(join(EXAMPLES_DIR, name)) test_dirs.sort() metafunc.parametrize("piotest_dir", test_dirs) def test_example(clirunner, validate_cliresult, piotest_dir): result = clirunner.invoke(cmd_ci, [piotest_dir, "-b", "uno"]) validate_cliresult(result) def test_warning_line(clirunner, validate_cliresult): result = clirunner.invoke(cmd_ci, [join(EXAMPLES_DIR, "basic"), "-b", "uno"]) validate_cliresult(result) assert 'basic.ino:16:14: warning: #warning "Line number is 16"' in result.output assert 'basic.ino:46:2: warning: #warning "Line number is 46"' in result.output result = clirunner.invoke( cmd_ci, [join(EXAMPLES_DIR, "strmultilines"), "-b", "uno"] ) validate_cliresult(result) assert 'main.ino:75:2: warning: #warning "Line 75"' in result.output ================================================ FILE: tests/misc/test_maintenance.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument from time import time from platformio import app, maintenance from platformio.__main__ import cli as cli_pio from platformio.commands import upgrade as cmd_upgrade def test_check_pio_upgrade(clirunner, isolated_pio_core, validate_cliresult): def _patch_pio_version(version): maintenance.__version__ = version cmd_upgrade.VERSION = version.split(".", 3) interval = int(app.get_setting("check_platformio_interval")) * 3600 * 24 last_check = {"platformio_upgrade": time() - interval - 1} origin_version = maintenance.__version__ # check development version _patch_pio_version("3.0.0-a1") app.set_state_item("last_check", last_check) result = clirunner.invoke(cli_pio, ["platform", "list"]) validate_cliresult(result) assert "There is a new version" in result.output assert "Please upgrade" in result.output # check stable version _patch_pio_version("2.11.0") app.set_state_item("last_check", last_check) result = clirunner.invoke(cli_pio, ["platform", "list"]) validate_cliresult(result) assert "There is a new version" in result.output assert "Please upgrade" in result.output # restore original version _patch_pio_version(origin_version) ================================================ FILE: tests/misc/test_misc.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import pytest import requests from platformio import __check_internet_hosts__, http, proc from platformio.registry.client import RegistryClient def test_platformio_cli(): result = proc.exec_command(["pio", "--help"]) assert result["returncode"] == 0 # pylint: disable=unsupported-membership-test assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] def test_ping_internet_ips(): for host in __check_internet_hosts__: requests.get("http://%s" % host, allow_redirects=False, timeout=2) def test_api_internet_offline(without_internet, isolated_pio_core): regclient = RegistryClient() with pytest.raises(http.InternetConnectionError): regclient.fetch_json_data("get", "/v3/search") def test_api_cache(monkeypatch, isolated_pio_core): regclient = RegistryClient() api_kwargs = {"method": "get", "path": "/v3/search", "x_cache_valid": "10s"} result = regclient.fetch_json_data(**api_kwargs) assert result and "total" in result monkeypatch.setattr(http, "_internet_on", lambda: False) assert regclient.fetch_json_data(**api_kwargs) == result ================================================ FILE: tests/package/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: tests/package/test_manager.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unused-argument import logging import os import time from pathlib import Path from random import random import pytest import semantic_version from platformio import fs, util from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, ) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.package.pack import PackagePacker def test_download(isolated_pio_core): url = "https://github.com/platformio/platformio-core/archive/v4.3.4.zip" checksum = "69d59642cb91e64344f2cdc1d3b98c5cd57679b5f6db7accc7707bd4c5d9664a" lm = LibraryPackageManager() lm.set_log_level(logging.ERROR) archive_path = lm.download(url, checksum) assert fs.calculate_file_hashsum("sha256", archive_path) == checksum lm.cleanup_expired_downloads(random()) assert os.path.isfile(archive_path) # test outdated downloads lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) lm.cleanup_expired_downloads(random()) assert not os.path.isfile(archive_path) # check that key is deleted from DB with open(lm.get_download_usagedb_path(), encoding="utf8") as fp: assert os.path.basename(archive_path) not in fp.read() def test_find_pkg_root(isolated_pio_core, tmpdir_factory): # has manifest pkg_dir = tmpdir_factory.mktemp("package-has-manifest") root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() root_dir.join("platform.json").write("") pm = PlatformPackageManager() found_dir = pm.find_pkg_root(str(pkg_dir), spec=None) assert os.path.realpath(str(root_dir)) == os.path.realpath(found_dir) # does not have manifest pkg_dir = tmpdir_factory.mktemp("package-does-not-have-manifest") pkg_dir.join("nested").mkdir().join("folder").mkdir().join("readme.txt").write("") pm = PlatformPackageManager() with pytest.raises(MissingPackageManifestError): pm.find_pkg_root(str(pkg_dir), spec=None) # library package without manifest, should find source root pkg_dir = tmpdir_factory.mktemp("library-package-without-manifest") root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() root_dir.join("src").mkdir().join("main.cpp").write("") root_dir.join("include").mkdir().join("main.h").write("") assert os.path.realpath(str(root_dir)) == os.path.realpath( LibraryPackageManager.find_library_root(str(pkg_dir)) ) # library manager should create "library.json" lm = LibraryPackageManager() spec = PackageSpec("custom-name@1.0.0") pkg_root = lm.find_pkg_root(str(pkg_dir), spec) manifest_path = os.path.join(pkg_root, "library.json") assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root) assert os.path.isfile(manifest_path) manifest = lm.load_manifest(pkg_root) assert manifest["name"] == "custom-name" assert "0.0.0" in str(manifest["version"]) def test_build_legacy_spec(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") pm = PlatformPackageManager(str(storage_dir)) # test src manifest pkg1_dir = storage_dir.join("pkg-1").mkdir() pkg1_dir.join(".pio").mkdir().join(".piopkgmanager.json").write(""" { "name": "StreamSpy-0.0.1.tar", "url": "https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", "requirements": null } """) assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec( name="StreamSpy-0.0.1.tar", uri="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", ) # without src manifest pkg2_dir = storage_dir.join("pkg-2").mkdir() pkg2_dir.join("main.cpp").write("") with pytest.raises(MissingPackageManifestError): pm.build_legacy_spec(str(pkg2_dir)) # with package manifest pkg3_dir = storage_dir.join("pkg-3").mkdir() pkg3_dir.join("platform.json").write('{"name": "pkg3", "version": "1.2.0"}') assert pm.build_legacy_spec(str(pkg3_dir)) == PackageSpec(name="pkg3") def test_build_metadata(isolated_pio_core, tmpdir_factory): pm = PlatformPackageManager() vcs_revision = "a2ebfd7c0f" pkg_dir = tmpdir_factory.mktemp("package") # test package without manifest with pytest.raises(MissingPackageManifestError): pm.load_manifest(str(pkg_dir)) with pytest.raises(MissingPackageManifestError): pm.build_metadata(str(pkg_dir), PackageSpec("MyLib")) # with manifest pkg_dir.join("platform.json").write( '{"name": "Dev-Platform", "version": "1.2.3-alpha.1"}' ) metadata = pm.build_metadata(str(pkg_dir), PackageSpec("owner/platform-name")) assert metadata.name == "Dev-Platform" assert str(metadata.version) == "1.2.3-alpha.1" # with vcs metadata = pm.build_metadata( str(pkg_dir), PackageSpec("owner/platform-name"), vcs_revision ) assert str(metadata.version) == ("1.2.3-alpha.1+sha." + vcs_revision) assert metadata.version.build[1] == vcs_revision def test_install_from_uri(isolated_pio_core, tmpdir_factory): tmp_dir = tmpdir_factory.mktemp("tmp") storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) # install from local directory src_dir = tmp_dir.join("local-lib-dir").mkdir() src_dir.join("main.cpp").write("") spec = PackageSpec("file://%s" % src_dir) pkg = lm.install(spec) assert os.path.isfile(os.path.join(pkg.path, "main.cpp")) manifest = lm.load_manifest(pkg) assert manifest["name"] == "local-lib-dir" assert manifest["version"].startswith("0.0.0+") assert spec == pkg.metadata.spec # install from local archive src_dir = tmp_dir.join("archive-src").mkdir() root_dir = src_dir.mkdir("root") root_dir.mkdir("src").join("main.cpp").write("#include ") root_dir.join("library.json").write( '{"name": "manifest-lib-name", "version": "2.0.0"}' ) tarball_path = PackagePacker(str(src_dir)).pack(str(tmp_dir)) spec = PackageSpec("file://%s" % tarball_path) pkg = lm.install(spec) assert os.path.isfile(os.path.join(pkg.path, "src", "main.cpp")) assert pkg == lm.get_package(spec) assert spec == pkg.metadata.spec # install from registry src_dir = tmp_dir.join("registry-1").mkdir() src_dir.join("library.properties").write(""" name = wifilib version = 5.2.7 """) spec = PackageSpec("company/wifilib @ ^5") pkg = lm.install_from_uri("file://%s" % src_dir, spec) assert str(pkg.metadata.version) == "5.2.7" # check package folder names lm.memcache_reset() assert ["local-lib-dir", "manifest-lib-name", "wifilib"] == [ os.path.basename(pkg.path) for pkg in lm.get_installed() ] def test_install_from_registry(isolated_pio_core, tmpdir_factory): # Libraries lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) lm.set_log_level(logging.ERROR) # library with dependencies lm.install("AsyncMqttClient-esphome @ 0.8.6") assert len(lm.get_installed()) == 3 pkg = lm.get_package("AsyncTCP-esphome") assert pkg.metadata.spec.owner == "esphome" assert not lm.get_package("non-existing-package") # mbed library assert lm.install("wolfSSL") assert len(lm.get_installed()) == 4 # case sensitive author name assert lm.install("DallasTemperature") assert lm.get_package("OneWire").metadata.version.major >= 2 assert len(lm.get_installed()) == 6 # test conflicted names lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage"))) lm.set_log_level(logging.ERROR) lm.install("z3t0/IRremote") lm.install("mbed-yuhki50/IRremote") assert len(lm.get_installed()) == 2 # Tools tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) tm.set_log_level(logging.ERROR) pkg = tm.install("platformio/tool-stlink @ ~1.10400.0") manifest = tm.load_manifest(pkg) assert tm.is_system_compatible(manifest.get("system")) assert util.get_systype() in manifest.get("system", []) # Test unknown with pytest.raises(UnknownPackageError): tm.install("unknown-package-tool @ 9.1.1") with pytest.raises(UnknownPackageError): tm.install("owner/unknown-package-tool") def test_install_lib_depndencies(isolated_pio_core, tmpdir_factory): tmp_dir = tmpdir_factory.mktemp("tmp") src_dir = tmp_dir.join("lib-with-deps").mkdir() root_dir = src_dir.mkdir("root") root_dir.mkdir("src").join("main.cpp").write("#include ") root_dir.join("library.json").write(""" { "name": "lib-with-deps", "version": "2.0.0", "dependencies": [ { "owner": "bblanchon", "name": "ArduinoJson", "version": "^6.16.1" }, { "name": "external-repo", "version": "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#4a0ccc1" } ] } """) lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) lm.set_log_level(logging.ERROR) lm.install("file://%s" % str(src_dir)) installed = lm.get_installed() assert len(installed) == 4 assert set(["external-repo", "ArduinoJson", "lib-with-deps", "OneWire"]) == set( p.metadata.name for p in installed ) def test_install_force(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) lm.set_log_level(logging.ERROR) # install #64 ArduinoJson pkg = lm.install("64 @ ^5") assert pkg.metadata.version.major == 5 # try install the latest without specification pkg = lm.install("64") assert pkg.metadata.version.major == 5 assert len(lm.get_installed()) == 1 # re-install the latest pkg = lm.install(64, force=True) assert len(lm.get_installed()) == 1 assert pkg.metadata.version.major > 5 def test_symlink(tmp_path: Path): external_pkg_dir = tmp_path / "External" external_pkg_dir.mkdir() (external_pkg_dir / "library.json").write_text(""" { "name": "External", "version": "1.0.0" } """) storage_dir = tmp_path / "storage" installed_pkg_dir = storage_dir / "installed" installed_pkg_dir.mkdir(parents=True) (installed_pkg_dir / "library.json").write_text(""" { "name": "Installed", "version": "1.0.0" } """) spec = "CustomExternal=symlink://%s" % str(external_pkg_dir) lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) pkg = lm.install(spec) assert os.path.isfile(str(storage_dir / "CustomExternal.pio-link")) assert pkg.metadata.name == "External" assert pkg.metadata.version.major == 1 assert ["External", "Installed"] == [ pkg.metadata.name for pkg in lm.get_installed() ] pkg = lm.get_package("External") assert Path(pkg.path) == external_pkg_dir assert pkg.metadata.spec.uri.startswith("symlink://") assert lm.get_package(spec).metadata.spec.uri.startswith("symlink://") # try to update lm.update(pkg) # uninstall lm.uninstall("External") assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()] # ensure original package was not removed assert external_pkg_dir.is_dir() # install again, remove from a disk assert lm.install("symlink://%s" % str(external_pkg_dir)) assert os.path.isfile(str(storage_dir / "External.pio-link")) assert ["External", "Installed"] == [ pkg.metadata.name for pkg in lm.get_installed() ] fs.rmtree(str(external_pkg_dir)) lm.memcache_reset() assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()] def test_scripts(isolated_pio_core, tmp_path: Path): pkg_dir = tmp_path / "foo" scripts_dir = pkg_dir / "scripts" scripts_dir.mkdir(parents=True) (scripts_dir / "script.py").write_text(""" import sys from pathlib import Path action = "postinstall" if len(sys.argv) == 1 else sys.argv[1] Path("%s.flag" % action).touch() if action == "preuninstall": Path("../%s.flag" % action).touch() """) (pkg_dir / "library.json").write_text(""" { "name": "foo", "version": "1.0.0", "scripts": { "postinstall": "scripts/script.py", "preuninstall2": ["scripts/script.py", "preuninstall"] } } """) storage_dir = tmp_path / "storage" lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) lm.install("file://%s" % str(pkg_dir)) assert os.path.isfile(os.path.join(lm.get_package("foo").path, "postinstall.flag")) lm.uninstall("foo") (storage_dir / "preuninstall.flag").is_file() def test_install_circular_dependencies(tmp_path: Path): storage_dir = tmp_path / "storage" # Foo pkg_dir = storage_dir / "foo" pkg_dir.mkdir(parents=True) (pkg_dir / "library.json").write_text(""" { "name": "Foo", "version": "1.0.0", "dependencies": { "Bar": "*" } } """) # Bar pkg_dir = storage_dir / "bar" pkg_dir.mkdir(parents=True) (pkg_dir / "library.json").write_text(""" { "name": "Bar", "version": "1.0.0", "dependencies": { "Foo": "*" } } """) lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) assert len(lm.get_installed()) == 2 # root library pkg_dir = tmp_path / "root" pkg_dir.mkdir(parents=True) (pkg_dir / "library.json").write_text(""" { "name": "Root", "version": "1.0.0", "dependencies": { "Foo": "^1.0.0", "Bar": "^1.0.0" } } """) lm.install("file://%s" % str(pkg_dir)) def test_get_installed(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") pm = ToolPackageManager(str(storage_dir)) # VCS package (storage_dir.join("pkg-vcs").mkdir().join(".git").mkdir().join(".piopm").write(""" { "name": "pkg-via-vcs", "spec": { "id": null, "name": "pkg-via-vcs", "owner": null, "requirements": null, "url": "git+https://github.com/username/repo.git" }, "type": "tool", "version": "0.0.0+sha.1ea4d5e" } """)) # package without metadata file ( storage_dir.join("foo@3.4.5") .mkdir() .join("package.json") .write('{"name": "foo", "version": "3.4.5"}') ) # package with metadata file foo_dir = storage_dir.join("foo").mkdir() foo_dir.join("package.json").write('{"name": "foo", "version": "3.6.0"}') foo_dir.join(".piopm").write(""" { "name": "foo", "spec": { "name": "foo", "owner": null, "requirements": "^3" }, "type": "tool", "version": "3.6.0" } """) # test "system" storage_dir.join("pkg-incompatible-system").mkdir().join("package.json").write( '{"name": "check-system", "version": "4.0.0", "system": ["unknown"]}' ) storage_dir.join("pkg-compatible-system").mkdir().join("package.json").write( '{"name": "check-system", "version": "3.0.0", "system": "%s"}' % util.get_systype() ) # invalid package storage_dir.join("invalid-package").mkdir().join("library.json").write( '{"name": "SomeLib", "version": "4.0.0"}' ) installed = pm.get_installed() assert len(installed) == 4 assert set(["pkg-via-vcs", "foo", "check-system"]) == set( p.metadata.name for p in installed ) assert str(pm.get_package("foo").metadata.version) == "3.6.0" assert str(pm.get_package("check-system").metadata.version) == "3.0.0" def test_uninstall(isolated_pio_core, tmpdir_factory): tmp_dir = tmpdir_factory.mktemp("tmp") storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) # foo @ 1.0.0 pkg_dir = tmp_dir.join("foo").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') foo_1_0_0_pkg = lm.install_from_uri("file://%s" % pkg_dir, "foo") # foo @ 1.3.0 pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') lm.install_from_uri("file://%s" % pkg_dir, "foo") # bar pkg_dir = tmp_dir.join("bar").mkdir() pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') bar_pkg = lm.install("file://%s" % pkg_dir) assert len(lm.get_installed()) == 3 assert os.path.isdir(os.path.join(str(storage_dir), "foo")) assert os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) # check detaching assert lm.uninstall("FOO") assert len(lm.get_installed()) == 2 assert os.path.isdir(os.path.join(str(storage_dir), "foo")) assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) # uninstall the rest assert lm.uninstall(foo_1_0_0_pkg.path) assert lm.uninstall(bar_pkg) assert not lm.get_installed() # test uninstall dependencies assert lm.install("AsyncMqttClient-esphome") assert len(lm.get_installed()) == 3 assert lm.uninstall("AsyncMqttClient-esphome", skip_dependencies=True) assert len(lm.get_installed()) == 2 lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) assert lm.install("AsyncMqttClient-esphome") assert lm.uninstall("AsyncMqttClient-esphome") assert not lm.get_installed() def test_registry(isolated_pio_core): lm = LibraryPackageManager() lm.set_log_level(logging.ERROR) # reveal ID assert lm.reveal_registry_package_id(PackageSpec(id=13)) == 13 assert lm.reveal_registry_package_id(PackageSpec(name="OneWire")) == 1 with pytest.raises(UnknownPackageError): lm.reveal_registry_package_id(PackageSpec(name="/non-existing-package/")) # fetch package data assert lm.fetch_registry_package(PackageSpec(id=1))["name"] == "OneWire" assert lm.fetch_registry_package(PackageSpec(name="ArduinoJson"))["id"] == 64 assert ( lm.fetch_registry_package( PackageSpec(id=13, owner="adafruit", name="Renamed library") )["name"] == "Adafruit GFX Library" ) with pytest.raises(UnknownPackageError): lm.fetch_registry_package( PackageSpec(owner="unknown<>owner", name="/non-existing-package/") ) with pytest.raises(UnknownPackageError): lm.fetch_registry_package(PackageSpec(name="/non-existing-package/")) def test_update_with_metadata(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) # test non SemVer in registry pkg = lm.install("adafruit/Adafruit NeoPixel @ <1.9") outdated = lm.outdated(pkg) assert str(outdated.current) == "1.8.7" assert outdated.latest > semantic_version.Version("1.10.0") pkg = lm.install("ArduinoJson @ 6.19.4") # test latest outdated = lm.outdated(pkg) assert str(outdated.current) == "6.19.4" assert outdated.wanted is None assert outdated.latest > outdated.current assert outdated.latest > semantic_version.Version("5.99.99") # test wanted outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~6")) assert str(outdated.current) == "6.19.4" assert str(outdated.wanted) == "6.21.5" assert outdated.latest > semantic_version.Version("6.16.0") # update to the wanted 6.x new_pkg = lm.update("ArduinoJson@^6", PackageSpec("ArduinoJson@^6")) assert str(new_pkg.metadata.version) == "6.21.5" # check that old version is removed assert len(lm.get_installed()) == 2 # update to the latest lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) pkg = lm.update("ArduinoJson") assert pkg.metadata.version == outdated.latest def test_update_without_metadata(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") storage_dir.join("legacy-package").mkdir().join("library.json").write( '{"name": "AsyncMqttClient-esphome", "version": "0.8"}' ) storage_dir.join("legacy-dep").mkdir().join("library.json").write( '{"name": "AsyncTCP-esphome", "version": "1.1.1"}' ) lm = LibraryPackageManager(str(storage_dir)) pkg = lm.get_package("AsyncMqttClient-esphome") outdated = lm.outdated(pkg) assert len(lm.get_installed()) == 2 assert str(pkg.metadata.version) == "0.8.0" assert outdated.latest > semantic_version.Version("0.8.0") # update lm = LibraryPackageManager(str(storage_dir)) lm.set_log_level(logging.ERROR) new_pkg = lm.update(pkg) assert len(lm.get_installed()) == 4 assert new_pkg.metadata.spec.owner == "heman" ================================================ FILE: tests/package/test_manifest.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import tarfile import jsondiff import pytest from platformio.compat import IS_WINDOWS from platformio.package.manifest import parser from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError def test_library_json_parser(): contents = """ { "name": "TestPackage", "keywords": "kw1, KW2, kw3, KW2, kw 4, kw_5, kw-6", "headers": "include1.h, Include2.hpp", "platforms": ["atmelavr", "espressif"], "repository": { "type": "git", "url": "http://github.com/username/repo/" }, "url": "http://old.url.format", "exclude": [".gitignore", "tests"], "include": "mylib", "build": { "flags": ["-DHELLO"] }, "examples": ["examples/*/*.pde"], "dependencies": { "deps1": "1.2.0", "deps2": "https://github.com/username/package.git", "owner/deps3": "^2.1.3" }, "customField": "Custom Value" } """ raw_data = parser.LibraryJsonManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( raw_data, { "name": "TestPackage", "platforms": ["atmelavr", "espressif8266"], "repository": { "type": "git", "url": "https://github.com/username/repo.git", }, "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, "keywords": ["kw1", "kw2", "kw3", "kw 4", "kw_5", "kw-6"], "headers": ["include1.h", "Include2.hpp"], "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, "dependencies": [ {"name": "deps1", "version": "1.2.0"}, {"name": "deps2", "version": "https://github.com/username/package.git"}, {"owner": "owner", "name": "deps3", "version": "^2.1.3"}, ], "customField": "Custom Value", }, ) contents = """ { "keywords": ["sound", "audio", "music", "SD", "card", "playback"], "headers": ["include 1.h", "include Space.hpp"], "frameworks": "arduino", "export": { "exclude": "audio_samples" }, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, {"owner": "owner", "name": "deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } """ raw_data = parser.LibraryJsonManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( raw_data, { "keywords": ["sound", "audio", "music", "sd", "card", "playback"], "headers": ["include 1.h", "include Space.hpp"], "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, { "owner": "owner", "name": "deps2", "version": "1.0.0", "platforms": ["*"], "frameworks": ["arduino", "espidf"], }, { "name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"], }, ], }, ) raw_data = parser.LibraryJsonManifestParser( '{"dependencies": ["dep1", "dep2", "owner/dep3@1.2.3"]}' ).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( raw_data, { "dependencies": [ {"name": "dep1"}, {"name": "dep2"}, {"name": "owner/dep3@1.2.3"}, ], }, ) # broken dependencies with pytest.raises(parser.ManifestParserError): parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) def test_module_json_parser(): contents = """ { "author": "Name Surname ", "description": "This is Yotta library", "homepage": "https://yottabuild.org", "keywords": [ "mbed", "Yotta" ], "licenses": [ { "type": "Apache-2.0", "url": "https://spdx.org/licenses/Apache-2.0" } ], "name": "YottaLibrary", "repository": { "type": "git", "url": "git@github.com:username/repo.git" }, "version": "1.2.3", "dependencies": { "usefulmodule": "^1.2.3", "simplelog": "ARMmbed/simplelog#~0.0.1" }, "customField": "Custom Value" } """ raw_data = parser.ModuleJsonManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( raw_data, { "name": "YottaLibrary", "description": "This is Yotta library", "homepage": "https://yottabuild.org", "keywords": ["mbed", "yotta"], "license": "Apache-2.0", "platforms": ["*"], "frameworks": ["mbed"], "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, "authors": [{"email": "name@surname.com", "name": "Name Surname"}], "version": "1.2.3", "repository": {"type": "git", "url": "git@github.com:username/repo.git"}, "dependencies": [ { "name": "simplelog", "version": "ARMmbed/simplelog#~0.0.1", "frameworks": ["mbed"], }, {"name": "usefulmodule", "version": "^1.2.3", "frameworks": ["mbed"]}, ], "customField": "Custom Value", }, ) def test_library_properties_parser(): # Base contents = """ name=TestPackage version=1.2.3 author=SomeAuthor , Maintainer Author (nickname) maintainer=Maintainer Author (nickname) sentence=This is Arduino library category=Signal Input/Output customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third ignore_empty_field= includes=Arduino.h, Arduino Space.hpp """ raw_data = parser.LibraryPropertiesManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( raw_data, { "name": "TestPackage", "version": "1.2.3", "description": "This is Arduino library", "sentence": "This is Arduino library", "frameworks": ["arduino"], "authors": [ {"name": "SomeAuthor", "email": "info@author.com"}, {"name": "Maintainer Author", "maintainer": True}, ], "category": "Signal Input/Output", "keywords": ["signal", "input", "output"], "headers": ["Arduino.h", "Arduino Space.hpp"], "includes": "Arduino.h, Arduino Space.hpp", "customField": "Custom Value", "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", "dependencies": [ { "name": "First Library", "version": "=2.0.0", "frameworks": ["arduino"], }, { "name": "Second Library", "version": ">=1.2.0", "frameworks": ["arduino"], }, {"name": "Third", "frameworks": ["arduino"]}, ], }, ) # Platforms ALL data = parser.LibraryPropertiesManifestParser( "architectures=*\n" + contents ).as_dict() assert data["platforms"] == ["*"] # Platforms specific data = parser.LibraryPropertiesManifestParser( "architectures=avr, esp32\n" + contents ).as_dict() assert data["platforms"] == ["atmelavr", "espressif32"] # Remote URL data = parser.LibraryPropertiesManifestParser( contents, remote_url=( "https://raw.githubusercontent.com/username/reponame/master/" "libraries/TestPackage/library.properties" ), ).as_dict() assert data["export"] == { "include": ["libraries/TestPackage"], } assert data["repository"] == { "url": "https://github.com/username/reponame.git", "type": "git", } # Home page data = parser.LibraryPropertiesManifestParser( "url=https://github.com/username/reponame.git\n" + contents ).as_dict() assert data["repository"] == { "type": "git", "url": "https://github.com/username/reponame.git", } # Author + Maintainer data = parser.LibraryPropertiesManifestParser(""" author=Rocket Scream Electronics maintainer=Rocket Scream Electronics """).as_dict() assert data["authors"] == [ {"name": "Rocket Scream Electronics", "maintainer": True} ] assert "keywords" not in data def test_library_json_schema(): contents = """ { "name": "ArduinoJson", "keywords": "JSON, rest, http, web", "description": "An elegant and efficient JSON library for embedded systems", "homepage": "https://arduinojson.org", "repository": { "type": "git", "url": "https://github.com/bblanchon/ArduinoJson.git" }, "version": "6.12.0", "authors": { "name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr" }, "downloadUrl": "https://example.com/package.tar.gz", "exclude": [ "fuzzing", "scripts", "test", "third-party" ], "frameworks": "arduino", "platforms": "*", "license": "MIT", "scripts": { "postinstall": "script.py" }, "examples": [ { "name": "JsonConfigFile", "base": "examples/JsonConfigFile", "files": ["JsonConfigFile.ino"] }, { "name": "JsonHttpClient", "base": "examples/JsonHttpClient", "files": ["JsonHttpClient.ino"] } ], "dependencies": [ {"name": "deps1", "version": "1.0.0"}, {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_JSON ).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) data = ManifestSchema().load_manifest(raw_data) assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git" assert data["examples"][1]["base"] == "examples/JsonHttpClient" assert data["examples"][1]["files"] == ["JsonHttpClient.ino"] assert not jsondiff.diff( data, { "name": "ArduinoJson", "keywords": ["json", "rest", "http", "web"], "description": "An elegant and efficient JSON library for embedded systems", "homepage": "https://arduinojson.org", "repository": { "url": "https://github.com/bblanchon/ArduinoJson.git", "type": "git", }, "version": "6.12.0", "authors": [ {"name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr"} ], "downloadUrl": "https://example.com/package.tar.gz", "export": {"exclude": ["fuzzing", "scripts", "test", "third-party"]}, "frameworks": ["arduino"], "platforms": ["*"], "license": "MIT", "scripts": {"postinstall": "script.py"}, "examples": [ { "name": "JsonConfigFile", "base": "examples/JsonConfigFile", "files": ["JsonConfigFile.ino"], }, { "name": "JsonHttpClient", "base": "examples/JsonHttpClient", "files": ["JsonHttpClient.ino"], }, ], "dependencies": [ {"name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino"]}, {"name": "deps1", "version": "1.0.0"}, { "name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"], }, ], }, ) # legacy dependencies format contents = """ { "name": "DallasTemperature", "version": "3.8.0", "dependencies": { "name": "OneWire", "authors": "Paul Stoffregen", "frameworks": "arduino" } } """ raw_data = parser.LibraryJsonManifestParser(contents).as_dict() data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, { "name": "DallasTemperature", "version": "3.8.0", "dependencies": [ { "name": "OneWire", "authors": ["Paul Stoffregen"], "frameworks": ["arduino"], } ], }, ) # test multiple licenses contents = """ { "name": "MultiLicense", "version": "1.0.0", "license": "MIT AND (LGPL-2.1-or-later OR BSD-3-Clause)" } """ raw_data = parser.LibraryJsonManifestParser(contents).as_dict() data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, { "name": "MultiLicense", "version": "1.0.0", "license": "MIT AND (LGPL-2.1-or-later OR BSD-3-Clause)", }, ) def test_library_properties_schema(): contents = """ name=U8glib version=1.19.1 author=oliver maintainer=oliver sentence=A library for monochrome TFTs and OLEDs paragraph=Supported display controller: SSD1306, SSD1309, SSD1322, SSD1325 category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam,samd depends=First Library (=2.0.0), Second Library (>=1.2.0), Third """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES ).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, { "description": ( "A library for monochrome TFTs and OLEDs. Supported display " "controller: SSD1306, SSD1309, SSD1322, SSD1325" ), "repository": { "url": "https://github.com/olikraus/u8glib.git", "type": "git", }, "frameworks": ["arduino"], "platforms": ["atmelavr", "atmelsam"], "version": "1.19.1", "authors": [ {"maintainer": True, "email": "olikraus@gmail.com", "name": "oliver"} ], "keywords": ["display"], "name": "U8glib", "dependencies": [ { "name": "First Library", "version": "=2.0.0", "frameworks": ["arduino"], }, { "name": "Second Library", "version": ">=1.2.0", "frameworks": ["arduino"], }, {"name": "Third", "frameworks": ["arduino"]}, ], }, ) # Broken fields contents = """ name=Mozzi version=1.0.3 author=Lorem Ipsum is simply dummy text of the printing and typesetting industry Lorem Ipsum has been the industry's standard dummy text ever since the 1500s when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries but also the leap into electronic typesetting remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. maintainer=Tim Barrass sentence=Sound synthesis library for Arduino paragraph=With Mozzi, you can construct sounds using familiar synthesis units like oscillators, delays, filters and envelopes. category=Signal Input/Output url=https://sensorium.github.io/Mozzi/ architectures=* dot_a_linkage=false includes=MozziGuts.h """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES, remote_url=( "https://raw.githubusercontent.com/sensorium/Mozzi/" "master/library.properties" ), ).as_dict() errors = None try: ManifestSchema().load_manifest(raw_data) except ManifestValidationError as exc: data = exc.valid_data errors = exc.messages assert errors["authors"] assert not jsondiff.diff( data, { "name": "Mozzi", "version": "1.0.3", "description": ( "Sound synthesis library for Arduino. With Mozzi, you can construct " "sounds using familiar synthesis units like oscillators, delays, " "filters and envelopes." ), "repository": { "url": "https://github.com/sensorium/Mozzi.git", "type": "git", }, "platforms": ["*"], "frameworks": ["arduino"], "headers": ["MozziGuts.h"], "authors": [ { "maintainer": True, "email": "faveflave@gmail.com", "name": "Tim Barrass", } ], "keywords": ["signal", "input", "output"], "homepage": "https://sensorium.github.io/Mozzi/", }, ) def test_platform_json_schema(): contents = """ { "name": "atmelavr", "title": "Atmel AVR", "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.", "keywords": "arduino, atmel, avr, MCU", "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "engines": { "platformio": "<5" }, "repository": { "type": "git", "url": "https://github.com/platformio/platform-atmelavr.git" }, "version": "1.15.0", "frameworks": { "arduino": { "package": "framework-arduinoavr", "script": "builder/frameworks/arduino.py" }, "simba": { "package": "framework-simba", "script": "builder/frameworks/simba.py" } }, "packages": { "toolchain-atmelavr": { "type": "toolchain", "owner": "platformio", "version": "~1.50400.0" }, "framework-arduinoavr": { "type": "framework", "optional": true, "version": "~4.2.0" }, "tool-avrdude": { "type": "uploader", "optional": true, "version": "~1.60300.0" } } } """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PLATFORM_JSON ).as_dict() raw_data["frameworks"] = sorted(raw_data["frameworks"]) raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, { "name": "atmelavr", "title": "Atmel AVR", "description": ( "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of " "performance, power efficiency and design flexibility. Optimized to " "speed time to market-and easily adapt to new ones-they are based " "on the industrys most code-efficient architecture for C and " "assembly programming." ), "keywords": ["arduino", "atmel", "avr", "mcu"], "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "repository": { "url": "https://github.com/platformio/platform-atmelavr.git", "type": "git", }, "frameworks": sorted(["arduino", "simba"]), "version": "1.15.0", "dependencies": [ {"name": "framework-arduinoavr", "version": "~4.2.0"}, {"name": "tool-avrdude", "version": "~1.60300.0"}, { "name": "toolchain-atmelavr", "owner": "platformio", "version": "~1.50400.0", }, ], }, ) def test_package_json_schema(): contents = """ { "name": "tool-scons", "description": "SCons software construction tool", "keywords": "SCons, build", "homepage": "http://www.scons.org", "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l", "LINUX_ARMV7L"], "version": "3.30101.0" } """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PACKAGE_JSON ).as_dict() data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, { "name": "tool-scons", "description": "SCons software construction tool", "keywords": ["scons", "build"], "homepage": "http://www.scons.org", "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], "version": "3.30101.0", }, ) mp = parser.ManifestParserFactory.new( '{"system": "*"}', parser.ManifestFileType.PACKAGE_JSON ) assert "system" not in mp.as_dict() mp = parser.ManifestParserFactory.new( '{"system": "all"}', parser.ManifestFileType.PACKAGE_JSON ) assert "system" not in mp.as_dict() mp = parser.ManifestParserFactory.new( '{"system": "darwin_x86_64"}', parser.ManifestFileType.PACKAGE_JSON ) assert mp.as_dict()["system"] == ["darwin_x86_64"] # shortcut repository syntax (npm-style) contents = """ { "name": "tool-github", "version": "1.2.0", "repository": "github:user/repo" } """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PACKAGE_JSON ).as_dict() data = ManifestSchema().load_manifest(raw_data) assert data["repository"]["url"] == "https://github.com/user/repo.git" def test_parser_from_dir(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") pkg_dir.join("package.json").write('{"name": "package.json"}') pkg_dir.join("library.json").write('{"name": "library.json"}') pkg_dir.join("library.properties").write("name=library.properties") data = parser.ManifestParserFactory.new_from_dir(str(pkg_dir)).as_dict() assert data["name"] == "library.json" data = parser.ManifestParserFactory.new_from_dir( str(pkg_dir), remote_url="http://localhost/library.properties" ).as_dict() assert data["name"] == "library.properties" def test_examples_from_dir(tmpdir_factory): package_dir = tmpdir_factory.mktemp("project") package_dir.join("library.json").write( '{"name": "pkg", "version": "1.0.0", "examples": ["examples/*/*.pde"]}' ) examples_dir = package_dir.mkdir("examples") # PlatformIO project #1 pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello") pio_dir.join(".vimrc").write("") pio_ini = pio_dir.join("platformio.ini") pio_ini.write("") if not IS_WINDOWS: pio_dir.join("platformio.ini.copy").mksymlinkto(pio_ini) pio_dir.mkdir("include").join("main.h").write("") pio_dir.mkdir("src").join("main.cpp").write("") # wiring examples arduino_dir = examples_dir.mkdir("1. General") arduino_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("") arduino_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("") # custom examples demo_dir = examples_dir.mkdir("demo") demo_dir.join("demo.cpp").write("") demo_dir.join("demo.h").write("") demo_dir.join("util.h").write("") # PlatformIO project #2 pio_dir = examples_dir.mkdir("world") pio_dir.join("platformio.ini").write("") pio_dir.join("README").write("") pio_dir.join("extra.py").write("") pio_dir.mkdir("include").join("world.h").write("") pio_dir.mkdir("src").join("world.c").write("") # example files in root examples_dir.join("root.c").write("") examples_dir.join("root.h").write("") # invalid example examples_dir.mkdir("invalid-example").join("hello.json") # Do testing raw_data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() assert isinstance(raw_data["examples"], list) assert len(raw_data["examples"]) == 6 def _to_unix_path(path): return re.sub(r"[\\/]+", "/", path) def _sort_examples(items): for i, _ in enumerate(items): items[i]["base"] = _to_unix_path(items[i]["base"]) items[i]["files"] = [_to_unix_path(f) for f in sorted(items[i]["files"])] return sorted(items, key=lambda item: item["name"]) raw_data["examples"] = _sort_examples(raw_data["examples"]) data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, { "version": "1.0.0", "name": "pkg", "examples": _sort_examples( [ { "name": "PlatformIO/hello", "base": os.path.join("examples", "PlatformIO", "hello"), "files": [ "platformio.ini", os.path.join("include", "main.h"), os.path.join("src", "main.cpp"), ], }, { "name": "1_General/SomeSketchIno", "base": os.path.join("examples", "1. General", "SomeSketchIno"), "files": ["SomeSketchIno.ino"], }, { "name": "1_General/SomeSketchPde", "base": os.path.join("examples", "1. General", "SomeSketchPde"), "files": ["SomeSketchPde.pde"], }, { "name": "demo", "base": os.path.join("examples", "demo"), "files": ["demo.h", "util.h", "demo.cpp"], }, { "name": "world", "base": "examples/world", "files": [ "platformio.ini", os.path.join("include", "world.h"), os.path.join("src", "world.c"), "README", "extra.py", ], }, { "name": "Examples", "base": "examples", "files": ["root.c", "root.h"], }, ] ), }, ) def test_parser_from_archive(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") pkg_dir.join("package.json").write('{"name": "package.json"}') pkg_dir.join("library.json").write('{"name": "library.json"}') pkg_dir.join("library.properties").write("name=library.properties") archive_path = os.path.join(str(pkg_dir), "package.tar.gz") with tarfile.open(archive_path, mode="w|gz") as tf: for item in os.listdir(str(pkg_dir)): tf.add(os.path.join(str(pkg_dir), item), item) data = parser.ManifestParserFactory.new_from_archive(archive_path).as_dict() assert data["name"] == "library.json" def test_broken_schemas(): # missing required field with pytest.raises( ManifestValidationError, match=("Invalid semantic versioning format") ) as exc_info: ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) assert exc_info.value.valid_data == {"name": "MyPackage"} # invalid StrictList with pytest.raises( ManifestValidationError, match=("Invalid manifest fields.+keywords") ) as exc_info: ManifestSchema().load_manifest( dict(name="MyPackage", version="1.0.0", keywords=["kw1", "*^[]"]) ) assert list(exc_info.value.messages.keys()) == ["keywords"] assert exc_info.value.valid_data["keywords"] == ["kw1"] # broken SemVer with pytest.raises( ManifestValidationError, match=("Invalid semantic versioning format") ): ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) # version with leading zeros with pytest.raises( ManifestValidationError, match=("Invalid semantic versioning format") ): ManifestSchema().load_manifest(dict(name="MyPackage", version="01.02.00")) # broken value for Nested with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"): ManifestSchema().load_manifest( dict( name="MyPackage", description="MyDescription", keywords=["a", "b"], authors=["should be dict here"], version="1.2.3", ) ) # invalid package name with pytest.raises(ManifestValidationError, match="are not allowed"): ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) ================================================ FILE: tests/package/test_meta.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import jsondiff import semantic_version from platformio.package.meta import ( PackageCompatibility, PackageMetadata, PackageOutdatedResult, PackageSpec, PackageType, ) def test_outdated_result(): result = PackageOutdatedResult(current="1.2.3", latest="2.0.0") assert result.is_outdated() assert result.is_outdated(allow_incompatible=True) result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.5.4") assert result.is_outdated() assert result.is_outdated(allow_incompatible=True) result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.2.3") assert not result.is_outdated() assert result.is_outdated(allow_incompatible=True) result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", detached=True) assert not result.is_outdated() assert not result.is_outdated(allow_incompatible=True) def test_spec_owner(): assert PackageSpec("alice/foo library") == PackageSpec( owner="alice", name="foo library" ) spec = PackageSpec(" Bob / BarUpper ") assert spec != PackageSpec(owner="BOB", name="BARUPPER") assert spec.owner == "Bob" assert spec.name == "BarUpper" def test_spec_id(): assert PackageSpec(13) == PackageSpec(id=13) assert PackageSpec("20") == PackageSpec(id=20) spec = PackageSpec("id=199") assert spec == PackageSpec(id=199) assert isinstance(spec.id, int) def test_spec_name(): assert PackageSpec("foo") == PackageSpec(name="foo") assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") def test_spec_requirements(): assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") assert PackageSpec( name="foo", requirements=semantic_version.Version("1.2.3") ) == PackageSpec(name="foo", requirements="1.2.3") assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") assert PackageSpec( name="hello", requirements=semantic_version.SimpleSpec("~1.2.3") ) == PackageSpec(name="hello", requirements="~1.2.3") spec = PackageSpec("id=20 @ !=1.2.3,<2.0") assert not spec.external assert isinstance(spec.requirements, semantic_version.SimpleSpec) assert semantic_version.Version("1.3.0-beta.1") in spec.requirements assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") def test_spec_local_urls(tmpdir_factory): assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( uri="file:///tmp/foo.tar.gz", name="foo" ) assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( uri="file:///tmp/bar.zip", name="customName" ) assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( uri="file:///tmp/some-lib/", name="some-lib" ) assert PackageSpec("symlink:///tmp/soft-link/") == PackageSpec( uri="symlink:///tmp/soft-link/", name="soft-link" ) # detached package assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec( uri="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib" ) # detached folder without scheme pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir() assert PackageSpec(str(pkg_dir)) == PackageSpec( name="detached", uri="file://%s" % pkg_dir ) def test_spec_external_urls(): assert PackageSpec( "https://github.com/platformio/platformio-core/archive/develop.zip" ) == PackageSpec( uri="https://github.com/platformio/platformio-core/archive/develop.zip", name="platformio-core", ) assert PackageSpec( "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" " @ !=2" ) == PackageSpec( uri="https://github.com/platformio/platformio-core/archive/" "develop.zip?param=value", name="platformio-core", requirements="!=2", ) spec = PackageSpec( "Custom-Name=" "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" ) assert spec.external assert spec.has_custom_name() assert spec.name == "Custom-Name" assert spec == PackageSpec( uri="https://github.com/platformio/platformio-core/archive/develop.tar.gz", name="Custom-Name", requirements="4.4.0", ) def test_spec_vcs_urls(): assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec( name="platformio-core", uri="git+https://github.com/platformio/platformio-core" ) assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec( name="reponame", uri="git+https://gitlab.com/username/reponame" ) assert PackageSpec( "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" ) == PackageSpec( name="wolfSSL", uri="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/" ) assert PackageSpec( "https://github.com/platformio/platformio-core.git#master" ) == PackageSpec( name="platformio-core", uri="git+https://github.com/platformio/platformio-core.git#master", ) assert PackageSpec( "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" ) == PackageSpec( name="core", uri="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", requirements="4.4.0", ) assert PackageSpec( "username@github.com:platformio/platformio-core.git" ) == PackageSpec( name="platformio-core", uri="git+username@github.com:platformio/platformio-core.git", ) assert PackageSpec( "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" ) == PackageSpec( name="pkg", uri="git+git@github.com:platformio/platformio-core.git", requirements="^1.2.3,!=5", ) assert PackageSpec( owner="platformio", name="external-repo", requirements="https://github.com/platformio/platformio-core", ) == PackageSpec( owner="platformio", name="external-repo", uri="git+https://github.com/platformio/platformio-core", ) def test_spec_as_dict(): assert not jsondiff.diff( PackageSpec("bob/foo@1.2.3").as_dict(), { "owner": "bob", "id": None, "name": "foo", "requirements": "1.2.3", "uri": None, }, ) assert not jsondiff.diff( PackageSpec( "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" " @ !=2" ).as_dict(), { "owner": None, "id": None, "name": "platformio-core", "requirements": "!=2", "uri": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value", }, ) def test_spec_as_dependency(): assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname" assert PackageSpec(owner="owner", name="pkgname").as_dependency() == "owner/pkgname" assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3" assert ( PackageSpec( "https://github.com/o/r/a/develop.zip?param=value @ !=2" ).as_dependency() == "https://github.com/o/r/a/develop.zip?param=value @ !=2" ) assert ( PackageSpec( "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" ).as_dependency() == "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" ) def test_metadata_as_dict(): metadata = PackageMetadata(PackageType.LIBRARY, "foo", "1.2.3") # test setter metadata.version = "0.1.2+12345" assert metadata.version == semantic_version.Version("0.1.2+12345") assert not jsondiff.diff( metadata.as_dict(), { "type": PackageType.LIBRARY, "name": "foo", "version": "0.1.2+12345", "spec": None, }, ) assert not jsondiff.diff( PackageMetadata( PackageType.TOOL, "toolchain", "2.0.5", PackageSpec("platformio/toolchain@~2.0.0"), ).as_dict(), { "type": PackageType.TOOL, "name": "toolchain", "version": "2.0.5", "spec": { "owner": "platformio", "id": None, "name": "toolchain", "requirements": "~2.0.0", "uri": None, }, }, ) def test_metadata_dump(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") metadata = PackageMetadata( PackageType.TOOL, "toolchain", "2.0.5", PackageSpec("platformio/toolchain@~2.0.0"), ) dst = pkg_dir.join(".piopm") metadata.dump(str(dst)) assert os.path.isfile(str(dst)) contents = dst.read() assert all(s in contents for s in ("null", '"~2.0.0"')) def test_metadata_load(tmpdir_factory): contents = """ { "name": "foo", "spec": { "name": "foo", "owner": "username", "requirements": "!=3.4.5" }, "type": "platform", "version": "0.1.3" } """ pkg_dir = tmpdir_factory.mktemp("package") dst = pkg_dir.join(".piopm") dst.write(contents) metadata = PackageMetadata.load(str(dst)) assert metadata.version == semantic_version.Version("0.1.3") assert metadata == PackageMetadata( PackageType.PLATFORM, "foo", "0.1.3", spec=PackageSpec(owner="username", name="foo", requirements="!=3.4.5"), ) piopm_path = pkg_dir.join(".piopm") metadata = PackageMetadata( PackageType.LIBRARY, "mylib", version="1.2.3", spec=PackageSpec("mylib") ) metadata.dump(str(piopm_path)) restored_metadata = PackageMetadata.load(str(piopm_path)) assert metadata == restored_metadata def test_compatibility(): assert PackageCompatibility().is_compatible(PackageCompatibility()) assert PackageCompatibility().is_compatible( PackageCompatibility(platforms=["espressif32"]) ) assert PackageCompatibility(frameworks=["arduino"]).is_compatible( PackageCompatibility(platforms=["espressif32"]) ) assert PackageCompatibility(platforms="espressif32").is_compatible( PackageCompatibility(platforms=["espressif32"]) ) assert PackageCompatibility( platforms="espressif32", frameworks=["arduino"] ).is_compatible(PackageCompatibility(platforms=None)) assert PackageCompatibility( platforms="espressif32", frameworks=["arduino"] ).is_compatible(PackageCompatibility(platforms=["*"])) assert not PackageCompatibility( platforms="espressif32", frameworks=["arduino"] ).is_compatible(PackageCompatibility(platforms=["atmelavr"])) ================================================ FILE: tests/package/test_pack.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import tarfile import pytest from platformio import fs from platformio.compat import IS_WINDOWS from platformio.package.exception import UnknownManifestError from platformio.package.pack import PackagePacker def test_base(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") pkg_dir.join(".git").mkdir().join("file").write("") pkg_dir.join(".gitignore").write("") pkg_dir.join("._hidden_file").write("") pkg_dir.join("main.cpp").write("#include ") p = PackagePacker(str(pkg_dir)) # test missed manifest with pytest.raises(UnknownManifestError): p.pack() # minimal package pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') pkg_dir.mkdir("include").join("main.h").write("#ifndef") with fs.cd(str(pkg_dir)): p.pack() with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar: assert set(tar.getnames()) == set( [".gitignore", "include/main.h", "library.json", "main.cpp"] ) def test_filters(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") src_dir = pkg_dir.mkdir("src") src_dir.join("main.cpp").write("#include ") src_dir.mkdir("util").join("helpers.cpp").write("void") pkg_dir.mkdir("include").join("main.h").write("#ifndef") test_dir = pkg_dir.mkdir("tests") test_dir.join("test_1.h").write("") test_dir.join("test_2.h").write("") # test include with remap of root pkg_dir.join("library.json").write( json.dumps(dict(name="bar", version="1.2.3", export={"include": "src"})) ) p = PackagePacker(str(pkg_dir)) with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: assert set(tar.getnames()) == set( ["util/helpers.cpp", "main.cpp", "library.json"] ) os.unlink(str(src_dir.join("library.json"))) # test include "src" and "include" pkg_dir.join("library.json").write( json.dumps( dict(name="bar", version="1.2.3", export={"include": ["src", "include"]}) ) ) p = PackagePacker(str(pkg_dir)) with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: assert set(tar.getnames()) == set( ["include/main.h", "library.json", "src/main.cpp", "src/util/helpers.cpp"] ) # test include & exclude pkg_dir.join("library.json").write( json.dumps( dict( name="bar", version="1.2.3", export={"include": ["src", "include"], "exclude": ["*/*.h"]}, ) ) ) p = PackagePacker(str(pkg_dir)) with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: assert set(tar.getnames()) == set( ["library.json", "src/main.cpp", "src/util/helpers.cpp"] ) def test_gitgnore_filters(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") pkg_dir.join(".git").mkdir().join("file").write("") pkg_dir.join(".gitignore").write(""" # comment gi_file gi_folder gi_folder_* **/main_nested.h gi_keep_file !gi_keep_file LICENSE """) pkg_dir.join("LICENSE").write("") pkg_dir.join("gi_keep_file").write("") pkg_dir.join("gi_file").write("") pkg_dir.mkdir("gi_folder").join("main.h").write("#ifndef") pkg_dir.mkdir("gi_folder_name").join("main.h").write("#ifndef") pkg_dir.mkdir("gi_nested_folder").mkdir("a").mkdir("b").join("main_nested.h").write( "#ifndef" ) pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') p = PackagePacker(str(pkg_dir)) with fs.cd(str(pkg_dir)): p.pack() with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar: assert set(tar.getnames()) == set( ["library.json", "LICENSE", ".gitignore", "gi_keep_file"] ) def test_symlinks(tmpdir_factory): # Windows does not support symbolic links if IS_WINDOWS: return pkg_dir = tmpdir_factory.mktemp("package") src_dir = pkg_dir.mkdir("src") src_dir.join("main.cpp").write("#include ") pkg_dir.mkdir("include").join("main.h").write("#ifndef") src_dir.join("main.h").mksymlinkto(os.path.join("..", "include", "main.h")) pkg_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') tarball = pkg_dir.join("bar.tar.gz") with tarfile.open(str(tarball), "w:gz") as tar: for item in pkg_dir.listdir(): tar.add(str(item), str(item.relto(pkg_dir))) p = PackagePacker(str(tarball)) assert p.pack(str(pkg_dir)).endswith("bar-2.0.0.tar.gz") with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar: assert set(tar.getnames()) == set( ["include/main.h", "library.json", "src/main.cpp", "src/main.h"] ) m = tar.getmember("src/main.h") assert m.issym() def test_source_root(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") root_dir = pkg_dir.mkdir("root") src_dir = root_dir.mkdir("src") src_dir.join("main.cpp").write("#include ") root_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') p = PackagePacker(str(pkg_dir)) with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: assert set(tar.getnames()) == set(["library.json", "src/main.cpp"]) def test_manifest_uri(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") root_dir = pkg_dir.mkdir("root") src_dir = root_dir.mkdir("src") src_dir.join("main.cpp").write("#include ") root_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') bar_dir = root_dir.mkdir("library").mkdir("bar") bar_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') bar_dir.mkdir("include").join("bar.h").write("") manifest_path = pkg_dir.join("remote_library.json") manifest_path.write( '{"name": "bar", "version": "3.0.0", "export": {"include": "root/library/bar"}}' ) p = PackagePacker(str(pkg_dir), manifest_uri="file:%s" % manifest_path) p.pack(str(pkg_dir)) with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar: assert set(tar.getnames()) == set(["library.json", "include/bar.h"]) ================================================ FILE: tests/project/__init__.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: tests/project/test_config.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=redefined-outer-name import configparser import os import sys from pathlib import Path import pytest from platformio import fs from platformio.project.config import ProjectConfig from platformio.project.exception import ( InvalidEnvNameError, InvalidProjectConfError, UnknownEnvNamesError, ) BASE_CONFIG = """ [platformio] env_default = base, extra_2 src_dir = ${custom.src_dir} extra_configs = extra_envs.ini extra_debug.ini # global options per [env:*] [env] monitor_speed = 9600 ; inline comment custom_monitor_speed = 115200 lib_deps = Lib1 ; inline comment in multi-line value Lib2 lib_ignore = ${custom.lib_ignore} custom_builtin_option = ${env.build_type} [strict_ldf] lib_ldf_mode = chain+ lib_compat_mode = strict [monitor_custom] monitor_speed = ${env.custom_monitor_speed} [strict_settings] extends = strict_ldf, monitor_custom build_flags = -D RELEASE [custom] src_dir = source debug_flags = -D RELEASE lib_flags = -lc -lm extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS} lib_ignore = LibIgnoreCustom [env:base] build_flags = ${custom.debug_flags} ${custom.extra_flags} lib_compat_mode = ${strict_ldf.lib_compat_mode} targets = [env:test_extends] extends = strict_settings [env:inject_base_env] debug_build_flags = ${env.debug_build_flags} -D CUSTOM_DEBUG_FLAG """ EXTRA_ENVS_CONFIG = """ [env:extra_1] build_flags = -fdata-sections -Wl,--gc-sections ${custom.lib_flags} ${custom.debug_flags} -D SERIAL_BAUD_RATE=${this.monitor_speed} lib_install = 574 [env:extra_2] build_flags = ${custom.debug_flags} ${custom.extra_flags} lib_ignore = ${env.lib_ignore}, Lib3 upload_port = /dev/extra_2/port debug_server = ${custom.debug_server} """ EXTRA_DEBUG_CONFIG = """ # Override original "custom.debug_flags" [custom] debug_flags = -D DEBUG=1 debug_server = ${platformio.packages_dir}/tool-openocd/openocd --help src_filter = -<*> + + [env:extra_2] build_flags = -Og src_filter = ${custom.src_filter} + """ DEFAULT_CORE_DIR = os.path.join(fs.expanduser("~"), ".platformio") @pytest.fixture(scope="module") def config(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(BASE_CONFIG) tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) with tmpdir.as_cwd(): return ProjectConfig(tmpdir.join("platformio.ini").strpath) def test_empty_config(): config = ProjectConfig("/non/existing/platformio.ini") # unknown section with pytest.raises(InvalidProjectConfError): config.get("unknown_section", "unknown_option") assert config.sections() == [] assert config.get("section", "option", 13) == 13 def test_warnings(config): config.validate(["extra_2", "base"], silent=True) assert len(config.warnings) == 3 assert "lib_install" in config.warnings[1] with pytest.raises(UnknownEnvNamesError): config.validate(["non-existing-env"]) def test_defaults(config): assert config.get("platformio", "core_dir") == os.path.join( os.path.expanduser("~"), ".platformio" ) assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] assert config.get("env:extra_2", "lib_compat_mode") == "soft" assert config.get("env:extra_2", "build_type") == "release" assert config.get("env:extra_2", "build_type", None) is None assert config.get("env:extra_2", "lib_archive", "no") is False config.expand_interpolations = False with pytest.raises( InvalidProjectConfError, match="No option 'lib_deps' in section: 'strict_ldf'" ): assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] config.expand_interpolations = True def test_sections(config): with pytest.raises(configparser.NoSectionError): config.getraw("unknown_section", "unknown_option") assert config.sections() == [ "platformio", "env", "strict_ldf", "monitor_custom", "strict_settings", "custom", "env:base", "env:test_extends", "env:inject_base_env", "env:extra_1", "env:extra_2", ] def test_envs(config): assert config.envs() == [ "base", "test_extends", "inject_base_env", "extra_1", "extra_2", ] assert config.default_envs() == ["base", "extra_2"] assert config.get_default_env() == "base" def test_options(config): assert config.options(env="base") == [ "build_flags", "lib_compat_mode", "targets", "monitor_speed", "custom_monitor_speed", "lib_deps", "lib_ignore", "custom_builtin_option", ] assert config.options(env="test_extends") == [ "extends", "build_flags", "monitor_speed", "lib_ldf_mode", "lib_compat_mode", "custom_monitor_speed", "lib_deps", "lib_ignore", "custom_builtin_option", ] def test_has_option(config): assert config.has_option("env:base", "monitor_speed") assert not config.has_option("custom", "monitor_speed") assert config.has_option("env:extra_1", "lib_install") assert config.has_option("env:test_extends", "lib_compat_mode") assert config.has_option("env:extra_2", "src_filter") def test_sysenv_options(config): assert config.getraw("custom", "extra_flags") == "" assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] assert config.get("env:base", "upload_port") is None assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" os.environ["PLATFORMIO_BUILD_FLAGS"] = "-DSYSENVDEPS1 -DSYSENVDEPS2" os.environ["PLATFORMIO_BUILD_UNFLAGS"] = "-DREMOVE_MACRO" os.environ["PLATFORMIO_UPLOAD_PORT"] = "/dev/sysenv/port" os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib" assert config.get("custom", "extra_flags") == "-L /usr/local/lib" assert config.get("env:base", "build_flags") == [ "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2", ] assert config.get("env:base", "upload_port") == "/dev/sysenv/port" assert config.get("env:extra_2", "upload_port") == "/dev/sysenv/port" assert config.get("env:base", "build_unflags") == ["-DREMOVE_MACRO"] # env var as option assert config.options(env="test_extends") == [ "extends", "build_flags", "monitor_speed", "lib_ldf_mode", "lib_compat_mode", "custom_monitor_speed", "lib_deps", "lib_ignore", "custom_builtin_option", "build_unflags", "upload_port", ] # sysenv dirs custom_core_dir = os.path.join(os.getcwd(), "custom-core") custom_src_dir = os.path.join(os.getcwd(), "custom-src") custom_build_dir = os.path.join(os.getcwd(), "custom-build") os.environ["PLATFORMIO_HOME_DIR"] = custom_core_dir os.environ["PLATFORMIO_SRC_DIR"] = custom_src_dir os.environ["PLATFORMIO_BUILD_DIR"] = custom_build_dir assert os.path.realpath(config.get("platformio", "core_dir")) == os.path.realpath( custom_core_dir ) assert os.path.realpath(config.get("platformio", "src_dir")) == os.path.realpath( custom_src_dir ) assert os.path.realpath(config.get("platformio", "build_dir")) == os.path.realpath( custom_build_dir ) # cleanup system environment variables del os.environ["PLATFORMIO_BUILD_FLAGS"] del os.environ["PLATFORMIO_BUILD_UNFLAGS"] del os.environ["PLATFORMIO_UPLOAD_PORT"] del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] del os.environ["PLATFORMIO_HOME_DIR"] del os.environ["PLATFORMIO_SRC_DIR"] del os.environ["PLATFORMIO_BUILD_DIR"] def test_getraw_value(config): # unknown option with pytest.raises(configparser.NoOptionError): config.getraw("custom", "unknown_option") # unknown option even if exists in [env] with pytest.raises(configparser.NoOptionError): config.getraw("platformio", "monitor_speed") # default assert config.getraw("unknown", "option", "default") == "default" assert config.getraw("env:base", "custom_builtin_option") == "release" # known assert config.getraw("env:base", "targets") == "" assert config.getraw("env:extra_1", "lib_deps") == "574" assert config.getraw("env:extra_1", "build_flags") == ( "\n-fdata-sections\n-Wl,--gc-sections\n" "-lc -lm\n-D DEBUG=1\n-D SERIAL_BAUD_RATE=9600" ) # extended assert config.getraw("env:test_extends", "lib_ldf_mode") == "chain+" assert config.getraw("env", "monitor_speed") == "9600" assert config.getraw("env:test_extends", "monitor_speed") == "115200" # dir options packages_dir = os.path.join(DEFAULT_CORE_DIR, "packages") assert config.get("platformio", "packages_dir") == packages_dir assert ( config.getraw("custom", "debug_server") == f"\n{packages_dir}/tool-openocd/openocd\n--help" ) # renamed option assert config.getraw("env:extra_1", "lib_install") == "574" assert config.getraw("env:extra_1", "lib_deps") == "574" assert config.getraw("env:base", "debug_load_cmd") == ["load"] def test_get_value(config): assert config.get("custom", "debug_flags") == "-D DEBUG=1" assert config.get("env:extra_1", "build_flags") == [ "-fdata-sections", "-Wl,--gc-sections", "-lc -lm", "-D DEBUG=1", "-D SERIAL_BAUD_RATE=9600", ] assert config.get("env:extra_2", "build_flags") == ["-Og"] assert config.get("env:extra_2", "monitor_speed") == 9600 assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] # get default value from ConfigOption assert config.get("env:inject_base_env", "debug_build_flags") == [ "-Og", "-g2", "-ggdb2", "-D CUSTOM_DEBUG_FLAG", ] # dir options assert config.get("platformio", "packages_dir") == os.path.join( DEFAULT_CORE_DIR, "packages" ) assert config.get("env:extra_2", "debug_server") == [ os.path.join(DEFAULT_CORE_DIR, "packages/tool-openocd/openocd"), "--help", ] # test relative dir assert config.get("platformio", "src_dir") == os.path.abspath( os.path.join(os.getcwd(), "source") ) # renamed option assert config.get("env:extra_1", "lib_install") == ["574"] assert config.get("env:extra_1", "lib_deps") == ["574"] assert config.get("env:base", "debug_load_cmd") == ["load"] def test_items(config): assert config.items("custom") == [ ("src_dir", "source"), ("debug_flags", "-D DEBUG=1"), ("lib_flags", "-lc -lm"), ("extra_flags", ""), ("lib_ignore", "LibIgnoreCustom"), ( "debug_server", "\n%s/tool-openocd/openocd\n--help" % os.path.join(DEFAULT_CORE_DIR, "packages"), ), ("src_filter", "-<*>\n+\n+"), ] assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), ("lib_compat_mode", "strict"), ("targets", []), ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), ("custom_builtin_option", "release"), ] assert config.items(env="extra_1") == [ ( "build_flags", [ "-fdata-sections", "-Wl,--gc-sections", "-lc -lm", "-D DEBUG=1", "-D SERIAL_BAUD_RATE=9600", ], ), ("lib_install", ["574"]), ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["574"]), ("lib_ignore", ["LibIgnoreCustom"]), ("custom_builtin_option", "release"), ] assert config.items(env="extra_2") == [ ("build_flags", ["-Og"]), ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), ("upload_port", "/dev/extra_2/port"), ( "debug_server", [ "%s/tool-openocd/openocd" % os.path.join(DEFAULT_CORE_DIR, "packages"), "--help", ], ), ("src_filter", ["-<*>", "+", "+ +"]), ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("custom_builtin_option", "release"), ] assert config.items(env="test_extends") == [ ("extends", ["strict_settings"]), ("build_flags", ["-D RELEASE"]), ("monitor_speed", 115200), ("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict"), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), ("custom_builtin_option", "release"), ] def test_update_and_save(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(""" [platformio] extra_configs = a.ini, b.ini [env:myenv] board = myboard """) config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config.envs() == ["myenv"] assert config.as_tuple()[0][1][0][1] == ["a.ini", "b.ini"] config.update( [ ["platformio", [("extra_configs", ["extra.ini"])]], ["env:myenv", [("framework", ["espidf", "arduino"])]], ["check_types", [("float_option", 13.99), ("bool_option", True)]], ] ) assert config.get("platformio", "extra_configs") == ["extra.ini"] config.remove_section("platformio") assert config.as_tuple() == [ ("env:myenv", [("board", "myboard"), ("framework", ["espidf", "arduino"])]), ("check_types", [("float_option", "13.99"), ("bool_option", "yes")]), ] config.save() contents = tmpdir.join("platformio.ini").read() assert contents[-4:] == "yes\n" lines = [ line.strip() for line in contents.split("\n") if line.strip() and not line.startswith((";", "#")) ] assert lines == [ "[env:myenv]", "board = myboard", "framework =", "espidf", "arduino", "[check_types]", "float_option = 13.99", "bool_option = yes", ] def test_update_and_clear(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(""" [platformio] extra_configs = a.ini, b.ini [env:myenv] board = myboard """) config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config.sections() == ["platformio", "env:myenv"] config.update([["mysection", [("opt1", "value1"), ("opt2", "value2")]]], clear=True) assert config.as_tuple() == [ ("mysection", [("opt1", "value1"), ("opt2", "value2")]) ] def test_dump(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(BASE_CONFIG) tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) config = ProjectConfig( tmpdir.join("platformio.ini").strpath, parse_extra=False, expand_interpolations=False, ) assert config.as_tuple() == [ ( "platformio", [ ("env_default", ["base", "extra_2"]), ("src_dir", "${custom.src_dir}"), ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), ], ), ( "env", [ ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["${custom.lib_ignore}"]), ("custom_builtin_option", "${env.build_type}"), ], ), ("strict_ldf", [("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict")]), ("monitor_custom", [("monitor_speed", "${env.custom_monitor_speed}")]), ( "strict_settings", [("extends", "strict_ldf, monitor_custom"), ("build_flags", "-D RELEASE")], ), ( "custom", [ ("src_dir", "source"), ("debug_flags", "-D RELEASE"), ("lib_flags", "-lc -lm"), ("extra_flags", "${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}"), ("lib_ignore", "LibIgnoreCustom"), ], ), ( "env:base", [ ("build_flags", ["${custom.debug_flags} ${custom.extra_flags}"]), ("lib_compat_mode", "${strict_ldf.lib_compat_mode}"), ("targets", []), ], ), ("env:test_extends", [("extends", ["strict_settings"])]), ( "env:inject_base_env", [ ( "debug_build_flags", ["${env.debug_build_flags}", "-D CUSTOM_DEBUG_FLAG"], ) ], ), ] @pytest.mark.skipif(sys.platform != "win32", reason="runs only on windows") def test_win_core_root_dir(tmpdir_factory): try: win_core_root_dir = os.path.splitdrive(fs.expanduser("~"))[0] + "\\.platformio" remove_dir_at_exit = False if not os.path.isdir(win_core_root_dir): remove_dir_at_exit = True os.makedirs(win_core_root_dir) # Default config config = ProjectConfig() assert config.get("platformio", "core_dir") == win_core_root_dir assert config.get("platformio", "packages_dir") == os.path.join( win_core_root_dir, "packages" ) # Override in config tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(""" [platformio] core_dir = ~/.pio """) config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config.get("platformio", "core_dir") != win_core_root_dir assert config.get("platformio", "core_dir") == os.path.realpath( fs.expanduser("~/.pio") ) if remove_dir_at_exit: fs.rmtree(win_core_root_dir) except PermissionError: pass def test_this(tmp_path: Path): project_conf = tmp_path / "platformio.ini" project_conf.write_text(""" [common] board = uno [env:myenv] extends = common build_flags = -D${this.__env__} custom_option = ${this.board} """) config = ProjectConfig(str(project_conf)) assert config.get("env:myenv", "custom_option") == "uno" assert config.get("env:myenv", "build_flags") == ["-Dmyenv"] def test_project_name(tmp_path: Path): project_dir = tmp_path / "my-project-name" project_dir.mkdir() project_conf = project_dir / "platformio.ini" project_conf.write_text(""" [env:myenv] """) with fs.cd(str(project_dir)): config = ProjectConfig(str(project_conf)) assert config.get("platformio", "name") == "my-project-name" # custom name project_conf.write_text(""" [platformio] name = custom-project-name """) config = ProjectConfig(str(project_conf)) assert config.get("platformio", "name") == "custom-project-name" def test_nested_interpolation(tmp_path: Path): project_conf = tmp_path / "platformio.ini" project_conf.write_text(""" [platformio] build_dir = /tmp/pio-$PROJECT_HASH data_dir = $PROJECT_DIR/assets [env:myenv] build_flags = -D UTIME=${UNIX_TIME} -I ${PROJECTSRC_DIR}/hal -Wl,-Map,${BUILD_DIR}/${PROGNAME}.map test_testing_command = ${platformio.packages_dir}/tool-simavr/bin/simavr -m atmega328p -f 16000000L ${UPLOAD_PORT and "-p "+UPLOAD_PORT} ${platformio.build_dir}/${this.__env__}/firmware.elf """) config = ProjectConfig(str(project_conf)) assert config.get("platformio", "data_dir").endswith( os.path.join("$PROJECT_DIR", "assets") ) assert config.get("env:myenv", "build_flags")[0][-10:].isdigit() assert config.get("env:myenv", "build_flags")[1] == "-I ${PROJECTSRC_DIR}/hal" assert ( config.get("env:myenv", "build_flags")[2] == "-Wl,-Map,${BUILD_DIR}/${PROGNAME}.map" ) testing_command = config.get("env:myenv", "test_testing_command") assert "$" not in testing_command[0] assert testing_command[5] == '${UPLOAD_PORT and "-p "+UPLOAD_PORT}' def test_extends_order(tmp_path: Path): project_conf = tmp_path / "platformio.ini" project_conf.write_text(""" [a] board = test [b] upload_tool = two [c] upload_tool = three [env:na_ti-ve13] extends = a, b, c """) config = ProjectConfig(str(project_conf)) assert config.get("env:na_ti-ve13", "upload_tool") == "three" def test_invalid_env_names(tmp_path: Path): project_conf = tmp_path / "platformio.ini" project_conf.write_text(""" [env:app:1] """) config = ProjectConfig(str(project_conf)) with pytest.raises(InvalidEnvNameError, match=r".*Invalid environment name 'app:1"): config.validate() def test_linting_errors(tmp_path: Path): project_conf = tmp_path / "platformio.ini" project_conf.write_text(""" [env:app1] lib_use = 1 broken_line """) result = ProjectConfig.lint(str(project_conf)) assert not result["warnings"] assert result["errors"] and len(result["errors"]) == 1 error = result["errors"][0] assert error["type"] == "ParsingError" assert error["lineno"] == 4 def test_linting_warnings(tmp_path: Path): project_conf = tmp_path / "platformio.ini" project_conf.write_text(""" [platformio] build_dir = /tmp/pio-$PROJECT_HASH [env:app1] lib_use = 1 test_testing_command = /usr/bin/flash-tool -p $UPLOAD_PORT -b $UPLOAD_SPEED """) result = ProjectConfig.lint(str(project_conf)) assert not result["errors"] assert result["warnings"] and len(result["warnings"]) == 2 assert "deprecated" in result["warnings"][0] assert "Invalid variable declaration" in result["warnings"][1] ================================================ FILE: tests/project/test_metadata.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from platformio.project.commands.metadata import project_metadata_cmd def test_metadata_dump(clirunner, validate_cliresult, tmpdir): tmpdir.join("platformio.ini").write(""" [env:native] platform = native """) component_dir = tmpdir.mkdir("lib").mkdir("component") component_dir.join("library.json").write(""" { "name": "component", "version": "1.0.0" } """) component_dir.mkdir("include").join("component.h").write(""" #define I_AM_COMPONENT void dummy(void); """) component_dir.mkdir("src").join("component.cpp").write(""" #include void dummy(void ) {}; """) tmpdir.mkdir("src").join("main.c").write(""" #include #ifndef I_AM_COMPONENT #error "I_AM_COMPONENT" #endif int main() { } """) metadata_path = tmpdir.join("metadata.json") result = clirunner.invoke( project_metadata_cmd, [ "--project-dir", str(tmpdir), "-e", "native", "--json-output", "--json-output-path", str(metadata_path), ], ) validate_cliresult(result) with open(str(metadata_path), encoding="utf8") as fp: metadata = json.load(fp)["native"] assert len(metadata["includes"]["build"]) == 3 assert len(metadata["includes"]["compatlib"]) == 2 ================================================ FILE: tests/project/test_savedeps.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from platformio.package.meta import PackageSpec from platformio.project.config import ProjectConfig from platformio.project.savedeps import save_project_dependencies PROJECT_CONFIG_TPL = """ [env] board = uno framework = arduino lib_deps = SPI platform_packages = platformio/tool-jlink@^1.75001.0 [env:bare] [env:release] platform = platformio/espressif32 lib_deps = milesburton/DallasTemperature@^3.8 [env:debug] platform = platformio/espressif32@^3.4.0 lib_deps = ${env.lib_deps} milesburton/DallasTemperature@^3.9.1 bblanchon/ArduinoJson platform_packages = ${env.platform_packages} platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git """ def test_save_libraries(tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) specs = [ PackageSpec("milesburton/DallasTemperature@^3.9"), PackageSpec("adafruit/Adafruit GPS Library@^1.6.0"), PackageSpec("https://github.com/nanopb/nanopb.git"), ] # add to the specified environment save_project_dependencies( str(project_dir), specs, scope="lib_deps", action="add", environments=["debug"] ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:debug", "lib_deps") == [ "SPI", "bblanchon/ArduinoJson", "milesburton/DallasTemperature@^3.9", "adafruit/Adafruit GPS Library@^1.6.0", "https://github.com/nanopb/nanopb.git", ] assert config.get("env:bare", "lib_deps") == ["SPI"] assert config.get("env:release", "lib_deps") == [ "milesburton/DallasTemperature@^3.8" ] # add to the the all environments save_project_dependencies(str(project_dir), specs, scope="lib_deps", action="add") config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:debug", "lib_deps") == [ "SPI", "bblanchon/ArduinoJson", "milesburton/DallasTemperature@^3.9", "adafruit/Adafruit GPS Library@^1.6.0", "https://github.com/nanopb/nanopb.git", ] assert config.get("env:bare", "lib_deps") == [ "milesburton/DallasTemperature@^3.9", "adafruit/Adafruit GPS Library@^1.6.0", "https://github.com/nanopb/nanopb.git", ] assert config.get("env:release", "lib_deps") == [ "milesburton/DallasTemperature@^3.9", "adafruit/Adafruit GPS Library@^1.6.0", "https://github.com/nanopb/nanopb.git", ] # remove deps from env save_project_dependencies( str(project_dir), [PackageSpec("milesburton/DallasTemperature")], scope="lib_deps", action="remove", environments=["release"], ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:release", "lib_deps") == [ "adafruit/Adafruit GPS Library@^1.6.0", "https://github.com/nanopb/nanopb.git", ] # invalid requirements save_project_dependencies( str(project_dir), [PackageSpec("adafruit/Adafruit GPS Library@^9.9.9")], scope="lib_deps", action="remove", environments=["release"], ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:release", "lib_deps") == [ "https://github.com/nanopb/nanopb.git", ] # remove deps from all envs save_project_dependencies( str(project_dir), specs, scope="lib_deps", action="remove" ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:debug", "lib_deps") == [ "SPI", "bblanchon/ArduinoJson", ] assert config.get("env:bare", "lib_deps") == ["SPI"] assert config.get("env:release", "lib_deps") == ["SPI"] def test_save_tools(tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) specs = [ PackageSpec("platformio/framework-espidf@^2"), PackageSpec("platformio/tool-esptoolpy"), ] # add to the specified environment save_project_dependencies( str(project_dir), specs, scope="platform_packages", action="add", environments=["debug"], ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:debug", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0", "platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git", "platformio/framework-espidf@^2", "platformio/tool-esptoolpy", ] assert config.get("env:bare", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0" ] assert config.get("env:release", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0" ] # add to the the all environments save_project_dependencies( str(project_dir), specs, scope="platform_packages", action="add" ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:debug", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0", "platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git", "platformio/framework-espidf@^2", "platformio/tool-esptoolpy", ] assert config.get("env:bare", "platform_packages") == [ "platformio/framework-espidf@^2", "platformio/tool-esptoolpy", ] assert config.get("env:release", "platform_packages") == [ "platformio/framework-espidf@^2", "platformio/tool-esptoolpy", ] # remove deps from env save_project_dependencies( str(project_dir), [PackageSpec("platformio/framework-espidf")], scope="platform_packages", action="remove", environments=["release"], ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:release", "platform_packages") == [ "platformio/tool-esptoolpy", ] # invalid requirements save_project_dependencies( str(project_dir), [PackageSpec("platformio/tool-esptoolpy@9.9.9")], scope="platform_packages", action="remove", environments=["release"], ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:release", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0", ] # remove deps from all envs save_project_dependencies( str(project_dir), specs, scope="platform_packages", action="remove" ) config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) assert config.get("env:debug", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0", "platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git", ] assert config.get("env:bare", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0" ] assert config.get("env:release", "platform_packages") == [ "platformio/tool-jlink@^1.75001.0" ] ================================================ FILE: tests/test_examples.py ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import random from glob import glob import pytest from platformio import fs, proc from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError def pytest_generate_tests(metafunc): if "pioproject_dir" not in metafunc.fixturenames: return examples_dirs = [] # repo examples examples_dirs.append( os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "examples")) ) # dev/platforms for pkg in PlatformPackageManager().get_installed(): p = PlatformFactory.new(pkg) examples_dir = os.path.join(p.get_dir(), "examples") if os.path.isdir(examples_dir): examples_dirs.append(examples_dir) project_dirs = [] for examples_dir in examples_dirs: candidates = {} for root, _, files in os.walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue if "mbed-legacy-examples" in root: continue group = os.path.basename(root) if "-" in group: group = group.split("-", 1)[0] if group not in candidates: candidates[group] = [] candidates[group].append(root) project_dirs.extend( [random.choice(examples) for examples in candidates.values() if examples] ) metafunc.parametrize("pioproject_dir", sorted(project_dirs)) def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() # temporary fix for unreleased dev-platforms with broken env name try: config.validate() except ProjectError as exc: pytest.skip(str(exc)) build_dir = config.get("platformio", "build_dir") if os.path.isdir(build_dir): fs.rmtree(build_dir) env_names = config.envs() result = proc.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) if result["returncode"] != 0: pytest.fail(str(result)) assert os.path.isdir(build_dir) # check .elf file for item in os.listdir(build_dir): if not os.path.isdir(item): continue assert os.path.isfile(os.path.join(build_dir, item, "firmware.elf")) # check .hex or .bin files firmwares = [] for ext in ("bin", "hex"): firmwares += glob(os.path.join(build_dir, item, "firmware*.%s" % ext)) if not firmwares: pytest.fail("Missed firmware file") for firmware in firmwares: assert os.path.getsize(firmware) > 0 ================================================ FILE: tox.ini ================================================ # Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. [isort] profile = black known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope [pytest] filterwarnings = error # Bottle ignore:.*'cgi' is deprecated and slated for removal ignore:'protected_args' is deprecated and will be removed in Click 9.0:DeprecationWarning [testenv] passenv = * usedevelop = True deps = black codespell isort jsondiff pylint pytest pytest-xdist commands = {envpython} --version pio system info [testenv:lint] commands = {envpython} --version pylint --rcfile=./.pylintrc ./platformio pylint --rcfile=./.pylintrc ./tests [testenv:testcore] commands = {envpython} --version py.test -v --basetemp={envtmpdir} -k "not skip_ci" tests --ignore tests/test_examples.py [testenv:testexamples] commands = {envpython} scripts/install_devplatforms.py py.test -v --basetemp={envtmpdir} tests/test_examples.py [testenv:docs] deps = sphinx-rtd-theme==3.0.2 sphinxcontrib-googleanalytics sphinx-notfound-page sphinx-copybutton restructuredtext-lint change_dir = docs commands = sphinx-build -b html . _build/html [testenv:docslinkcheck] deps = {[testenv:docs]deps} change_dir = docs commands = sphinx-build -b linkcheck . _build