[ { "path": ".all_crates.sh", "content": "\nALL_CRATES_PATH=\"data linalg core nnef nnef/nnef-resources pulse-opl pulse extra transformers hir tflite tensorflow onnx-opl onnx gpu metal cuda libcli api api/rs api/ffi api/proxy/sys api/proxy cli\"\n" }, { "path": ".change_crate_dep.sh", "content": "#!/bin/bash\n\ncrate=$1\nversion=$2\n\nperl -pi -e \"s/^($crate = {.*version *= *)\\\"([^\\\"]*)\\\"(.*)$/\\$1\\\"=$version\\\"\\$3/\" \\\n `find . -name Cargo.toml \\! -path \"./target/*\" \\! -path \"./issue*\"`\n" }, { "path": ".clang-format", "content": "BasedOnStyle: LLVM\nIndentWidth: 4\nTabWidth: 4\nUseTab: Never\nIndentPPDirectives: BeforeHash\nPPIndentWidth: 4\nColumnLimit: 100\n\n# OneLineFormatOffRegex: '^\\s*#\\s*pragma\\s+unroll\\b'\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n day: \"monday\"\n groups:\n actions:\n patterns:\n - \"*\"\n\n - package-ecosystem: \"cargo\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n day: \"monday\"\n open-pull-requests-limit: 10\n groups:\n rust-dependencies:\n patterns:\n - \"*\"\n\n - package-ecosystem: \"pip\"\n directory: \"/api/py\"\n ignore:\n # Only update them manually since updating them might break compatibility\n - dependency-name: \"numpy\"\n schedule:\n interval: \"weekly\"\n day: \"monday\"\n" }, { "path": ".github/workflows/asan.yml", "content": "name: Sanitized build tests\n\non:\n workflow_dispatch:\n schedule:\n - cron: '0 5 * * MON'\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n\njobs:\n sanitizer-address:\n strategy:\n fail-fast: false\n matrix:\n os: [ ubuntu-latest, macOS-latest ]\n\n runs-on: ${{matrix.os}}\n\n steps:\n - uses: actions/checkout@v6\n - name: Rustup update\n run: rustup update\n - name: Run sanitized tests\n run: .travis/asan.sh\n" }, { "path": ".github/workflows/binaries.yml", "content": "on:\n release:\n types:\n - created\n\nname: Upload Release Binaries\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n\njobs:\n assets:\n name: Upload Release Binaries\n strategy:\n fail-fast: false\n matrix:\n os: [ ubuntu-latest, macOS-latest ]\n arch: [ x86_64, aarch64, armv7 ]\n include:\n - os: ubuntu-latest\n arch: x86_64\n target: x86_64-unknown-linux-musl\n musl: x86_64-linux-musl\n - os: ubuntu-latest\n arch: aarch64\n target: aarch64-unknown-linux-musl\n musl: aarch64-linux-musl\n - os: ubuntu-latest\n arch: armv7\n target: armv7-unknown-linux-musleabihf\n musl: armv7l-linux-musleabihf\n - os: macOS-latest\n arch: x86_64\n target: x86_64-apple-darwin\n - os: macOS-latest\n arch: aarch64\n target: aarch64-apple-darwin\n exclude:\n - os: macOS-latest\n arch: armv7\n\n runs-on: ${{ matrix.os }}\n steps:\n - name: Checkout code\n uses: actions/checkout@v6\n\n - name: Extract version tag\n id: version\n run: echo value=$(echo ${{ github.ref }} | cut -f 3 -d / | sed 's/^v//' ) >> $GITHUB_OUTPUT\n\n - name: Build tract\n run: |\n set -ex\n target=${{matrix.target}}\n version=${{steps.version.outputs.value}}\n name=${target}-${version}\n\n rustup update\n rustup target add ${target}\n\n if [ -n \"${{matrix.musl}}\" ]\n then\n MUSL_TRIPLE=${{matrix.musl}}\n curl -s https://s3.amazonaws.com/tract-ci-builds/toolchains/${MUSL_TRIPLE}-cross.tgz | tar zx\n\n MUSL_BIN=`pwd`/${MUSL_TRIPLE}-cross/bin\n export PATH=$MUSL_BIN:$PATH\n\n export TARGET_CC=$MUSL_BIN/${MUSL_TRIPLE}-gcc\n\n RUST_TRIPLE_ENV=$(echo ${target} | tr 'a-z-' 'A-Z_')\n export CARGO_TARGET_${RUST_TRIPLE_ENV}_CC=$TARGET_CC\n export CARGO_TARGET_${RUST_TRIPLE_ENV}_LINKER=$TARGET_CC\n fi\n\n cargo build --target ${target} --release -p tract-cli\n mkdir tract-$name\n cp target/${target}/release/tract tract-${name}\n tar czf tract-${name}.tgz tract-${name}\n\n - name: Upload asset\n uses: softprops/action-gh-release@v2\n with:\n files: tract-${{matrix.target}}-${{ steps.version.outputs.value }}.tgz\n name: ${{ steps.version.outputs.value }}\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n\n" }, { "path": ".github/workflows/cost_model.yml", "content": "\non:\n workflow_dispatch:\n inputs:\n dataset_id:\n description: 'dataset identifier'\n required: true\n\nname: Generate cost model analysis dataset\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n\njobs:\n build:\n name: Upload cost model tasks\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n target: [ \"aarch64\", \"armv7\" ]\n steps:\n - name: Checkout code\n uses: actions/checkout@v6\n\n - name: Build and upload\n run: ./.travis/cost_model_task_build.sh ${{matrix.target}} ${{github.event.inputs.dataset_id}}\n env:\n AWS_ACCESS_KEY_ID: ${{secrets.TRACT_CI_AWS_ACCESS_KEY_ID}}\n AWS_SECRET_ACCESS_KEY: ${{secrets.TRACT_CI_AWS_SECRET_ACCESS_KEY}}\n AWS_EC2_METADATA_DISABLED: true\n" }, { "path": ".github/workflows/crates.yml", "content": "name: Rust crates\n\non:\n pull_request:\n schedule:\n - cron: '0 3 * * *'\n workflow_dispatch:\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n\njobs:\n prepare-matrix:\n runs-on: ubuntu-latest\n outputs:\n os: ${{steps.set-matrix.outputs.os}}\n rust: ${{steps.set-matrix.outputs.rust}}\n\n steps:\n - id: set-matrix\n env:\n FULL: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}\n run: |\n if [ \"$FULL\" == \"true\" ]\n then\n echo 'os=[\"ubuntu-latest\", \"macos-latest\"]' >> $GITHUB_OUTPUT\n echo 'rust=[\"1.91.0\", \"stable\", \"beta\", \"nightly\"]' >> $GITHUB_OUTPUT\n else\n echo ::notice::Skipping macOS checks on PR and commit. Dispatch workflow manually if needed.\n echo 'os=[\"ubuntu-latest\"]' >> $GITHUB_OUTPUT\n echo 'rust=[\"1.91.0\"]' >> $GITHUB_OUTPUT\n fi\n\n crates:\n name: ${{matrix.os}} / ${{matrix.crate}} / ${{matrix.rust}}\n needs: prepare-matrix\n strategy:\n matrix:\n os: ${{fromJson(needs.prepare-matrix.outputs.os)}}\n rust: ${{fromJson(needs.prepare-matrix.outputs.rust)}}\n crate: [ tract-data, tract-linalg, tract-core, tract-nnef, tract-hir, tract-onnx,\n tract-pulse, tract-onnx-opl, tract-pulse-opl, tract,\n test-unit-core, test-onnx-core, test-nnef-cycle, test-f16,\n ]\n fail-fast: false\n\n runs-on: ${{matrix.os}}\n env:\n RUSTUP_TOOLCHAIN: ${{matrix.rust}}\n\n steps:\n - uses: actions/checkout@v6\n\n - name: Cargo test\n run: cargo test -p ${{matrix.crate}}\n\n cuda:\n runs-on: cuda-lovelace\n needs: prepare-matrix\n strategy:\n matrix:\n rust: ${{fromJson(needs.prepare-matrix.outputs.rust)}}\n fail-fast: false\n env:\n RUSTUP_TOOLCHAIN: ${{matrix.rust}}\n steps:\n - uses: actions/checkout@v6\n\n - name: Cargo test\n run: cargo test -p tract-cuda -p test-cuda\n\n metal:\n runs-on: macOS\n needs: prepare-matrix\n strategy:\n matrix:\n rust: ${{fromJson(needs.prepare-matrix.outputs.rust)}}\n fail-fast: false\n env:\n RUSTUP_TOOLCHAIN: ${{matrix.rust}}\n steps:\n - uses: actions/checkout@v6\n\n - name: Cargo test\n run: cargo test -p tract-metal -p test-metal\n \n pedantic:\n name: fmt, clippy, etc (${{matrix.os}} / ${{matrix.rust}})\n needs: prepare-matrix\n strategy:\n matrix:\n os: ${{fromJson(needs.prepare-matrix.outputs.os)}}\n rust: ${{fromJson(needs.prepare-matrix.outputs.rust)}}\n fail-fast: false\n runs-on: ${{matrix.os}}\n env:\n RUSTUP_TOOLCHAIN: ${{matrix.rust}}\n steps:\n - uses: actions/checkout@v6\n - run: rustup component add clippy && cargo clippy\n - name: fmt\n run: rustup component add rustfmt && cargo fmt --check\n - name: Warnings\n env:\n RUSTFLAGS: -D warnings\n run: cargo check\n\n cargo-deny:\n strategy:\n fail-fast: false\n\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v6\n - name: Install cargo-deny\n run: |\n curl -L https://github.com/EmbarkStudios/cargo-deny/releases/download/$VERSION/cargo-deny-$VERSION-x86_64-unknown-linux-musl.tar.gz \\\n | tar -zx --strip-components=1 \"cargo-deny-$VERSION-x86_64-unknown-linux-musl/cargo-deny\"\n env:\n VERSION: 0.18.9\n - name: Run cargo-deny\n run: .travis/cargo-deny-check.sh\n\n\n" }, { "path": ".github/workflows/cross-platform.yml", "content": "name: Embedded targets\n\non:\n pull_request:\n schedule:\n - cron: '0 5 * * *'\n workflow_dispatch:\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n RUSTUP_TOOLCHAIN: 1.91.0\n\njobs:\n linux:\n strategy:\n fail-fast: false\n matrix:\n platform:\n - raspbian\n - aarch64-unknown-linux-gnu\n - aarch64-unknown-linux-gnu-stretch\n - armv6vfp-unknown-linux-gnueabihf\n - armv7-unknown-linux-gnueabihf\n - armv7-unknown-linux-gnueabihf-stretch\n - aarch64-unknown-linux-musl\n - cortexa53-unknown-linux-musl\n - armv7-unknown-linux-musl\n - aarch64-linux-android\n - armv7-linux-androideabi\n - i686-linux-android\n - x86_64-linux-android\n - x86_64-unknown-linux-gnu-stretch\n - wasm32-unknown-unknown\n - wasm32-wasi\n\n runs-on: ubuntu-latest\n permissions:\n id-token: write\n contents: read\n\n steps:\n - uses: actions/checkout@v6\n\n - name: Get current date\n id: date\n run: echo \"date=$(date +'%Y-%m-%d')\" >> $GITHUB_OUTPUT\n\n - name: Configure AWS Credentials\n continue-on-error: true\n uses: aws-actions/configure-aws-credentials@v6\n with:\n role-to-assume: arn:aws:iam::567805100031:role/github-runner-tract-ci\n aws-region: us-east-2\n\n - uses: actions/cache@v5\n with:\n path: |\n ~/.rustup\n ~/.cargo/registry\n ~/.cargo/git\n # ~/.cache/sccache\n .cached\n target\n key: ${{ runner.os }}-${{matrix.platform}}-${{steps.date.outputs.date}}\n\n - name: Setup wasmtime\n if: ${{ matrix.platform }} == \"wasm32-wasi\"\n uses: bytecodealliance/actions/wasmtime/setup@v1\n\n - name: Cross script\n env:\n PLATFORM: ${{matrix.platform}}\n AWS_EC2_METADATA_DISABLED: true\n run: .travis/cross.sh\n\n apple:\n strategy:\n fail-fast: false\n matrix:\n platform:\n - aarch64-apple-ios\n - aarch64-apple-darwin\n\n runs-on: macos-latest\n permissions:\n id-token: write\n contents: read\n\n steps:\n - uses: actions/checkout@v6\n\n - name: Configure AWS Credentials\n continue-on-error: true\n uses: aws-actions/configure-aws-credentials@v6\n with:\n role-to-assume: arn:aws:iam::567805100031:role/github-runner-tract-ci\n aws-region: us-east-2\n\n - name: Get current date\n id: date\n run: echo \"date=$(date +'%Y-%m-%d')\" >> $GITHUB_OUTPUT\n\n - name: Cross script\n env:\n PLATFORM: ${{matrix.platform}}\n run: .travis/cross.sh\n" }, { "path": ".github/workflows/examples.yml", "content": "name: Examples\n\non:\n schedule:\n - cron: '0 3 * * *'\n workflow_dispatch:\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n RUSTUP_TOOLCHAIN: 1.91.0\n\njobs:\n examples:\n runs-on: ubuntu-latest\n outputs:\n examples: ${{steps.set-matrix.outputs.examples}}\n\n steps:\n - uses: actions/checkout@v6\n - id: set-matrix\n run: |\n echo examples=`find examples -name ci.sh | cut -d/ -f 2 | jq -Rsc '. / \"\\n\" - [\"\"]'` >> \"$GITHUB_OUTPUT\"\n\n example:\n name: ${{ matrix.ex }}\n runs-on: ubuntu-latest\n needs: examples\n strategy:\n fail-fast: false\n matrix:\n ex: ${{fromJSON(needs.examples.outputs.examples)}}\n\n steps:\n - uses: actions/checkout@v6\n\n - name: Configure AWS Credentials\n # if: github.repository == 'sonos/tract'\n continue-on-error: true\n uses: aws-actions/configure-aws-credentials@v6\n with:\n role-to-assume: arn:aws:iam::567805100031:role/github-runner-tract-ci\n aws-region: us-east-2\n\n - name: example tests\n env:\n AWS_EC2_METADATA_DISABLED: true\n timeout-minutes: 30\n run: |\n cd examples/${{matrix.ex}}\n ./ci.sh\n\n build-tract-cli:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - run: cargo build -p tract-cli --profile opt-no-lto\n - uses: actions/upload-artifact@v7\n with:\n name: tract-cli-x86_64\n path: ./target/opt-no-lto/tract\n\n build-tract-cli-macos:\n runs-on: macOS\n steps:\n - uses: actions/checkout@v6\n - run: cargo build -p tract-cli --profile opt-no-lto\n - uses: actions/upload-artifact@v7\n with:\n name: tract-cli-aarch64-apple\n path: ./target/opt-no-lto/tract\n\n gpu-examples:\n runs-on: ubuntu-latest\n outputs:\n examples: ${{steps.set-matrix.outputs.examples}}\n\n steps:\n - uses: actions/checkout@v6\n - id: set-matrix\n run: |\n echo examples=`find examples -name ci-gpu.sh | cut -d/ -f 2 | jq -Rsc '. / \"\\n\" - [\"\"]'` >> \"$GITHUB_OUTPUT\"\n\n gpu-example:\n name: ${{ matrix.ex }} (CUDA)\n runs-on: cuda-lovelace\n needs: [gpu-examples, build-tract-cli]\n strategy:\n fail-fast: false\n matrix:\n ex: ${{fromJSON(needs.gpu-examples.outputs.examples)}}\n\n steps:\n - uses: actions/checkout@v6\n\n - uses: actions/download-artifact@v8\n with:\n name: tract-cli-x86_64\n path: target/opt-no-lto\n\n - run: chmod +x target/opt-no-lto/tract\n\n - name: GPU example tests\n timeout-minutes: 60\n run: |\n cd examples/${{matrix.ex}}\n ./ci-gpu.sh\n\n gpu-example-metal:\n name: ${{ matrix.ex }} (Metal)\n runs-on: macOS\n needs: [gpu-examples, build-tract-cli-macos]\n strategy:\n fail-fast: false\n matrix:\n ex: ${{fromJSON(needs.gpu-examples.outputs.examples)}}\n\n steps:\n - uses: actions/checkout@v6\n\n - uses: actions/download-artifact@v8\n with:\n name: tract-cli-aarch64-apple\n path: target/opt-no-lto\n\n - run: chmod +x target/opt-no-lto/tract\n\n - name: Metal GPU example tests\n timeout-minutes: 60\n run: |\n cd examples/${{matrix.ex}}\n ./ci-gpu.sh\n" }, { "path": ".github/workflows/full.yml", "content": "name: Full test harness\n\non:\n schedule:\n - cron: '0 3 * * *'\n workflow_dispatch:\n inputs:\n pr_number:\n description: \"Optional PR number to test (from fork ok). Leave empty to run on selected branch.\"\n required: false\n type: number\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n\njobs:\n prepare:\n runs-on: ubuntu-latest\n outputs:\n test_ref: ${{ steps.set.outputs.test_ref }}\n steps:\n - id: set\n uses: actions/github-script@v8\n with:\n script: |\n const prInput = context.payload.inputs?.pr_number;\n core.info(`Fetching PR ${prInput}`);\n if (!prInput) {\n // Use the ref the workflow was triggered on (branch/tag/SHA in base repo)\n core.setOutput('test_ref', process.env.GITHUB_SHA);\n return;\n }\n const pr = await github.rest.pulls.get({\n owner: context.repo.owner,\n repo: context.repo.repo,\n pull_number: Number(prInput),\n });\n core.info(pr.data.head.sha);\n core.setOutput('test_ref', pr.data.head.sha);\n\n old-harness:\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n permissions:\n id-token: write\n contents: read\n needs: prepare\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n\n - name: Configure AWS Credentials\n continue-on-error: true\n uses: aws-actions/configure-aws-credentials@v6\n with:\n role-to-assume: arn:aws:iam::567805100031:role/github-runner-tract-ci\n aws-region: us-east-2\n\n - name: Full test\n run: .travis/test-harness.sh\n\n cli-tests:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: Full test \n env:\n AWS_EC2_METADATA_DISABLED: true\n run: .travis/cli-tests.sh\n\n onnx-tests:\n runs-on: ubuntu-latest\n needs: prepare\n strategy:\n matrix:\n opset: [1_4_1, 1_5_0, 1_6_0, 1_7_0, 1_8_1, 1_9_0, 1_10_2, 1_11_0, 1_12_0, 1_13_0, 1_14_1, 1_15_0, 1_16_2, 1_17_0, 1_18_0, 1_19_1]\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: Full test\n run: .travis/onnx-tests.sh ${{ matrix.opset }}\n\n tflite:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: Full test\n run: .travis/tflite.sh\n\n some-tests-with-paranoid-asserts:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n\n - name: With assertions\n run: |\n rustup update\n cargo test --features tract-core/paranoid_assertions -p test-onnx-core -p test-unit-core\n\n without-default-features:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: Without default features\n run: |\n rustup update\n cargo check -p tract-cli --no-default-features $CARGO_EXTRA\n\n complexes:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: With complexes\n run: |\n rustup update\n cargo check -p tract-nnef --features complex $CARGO_EXTRA\n\n check-all-targets:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: Check all targets\n run: |\n ROOT=$(pwd) ./.travis/ci-system-setup.sh\n cargo check --all-targets --workspace --exclude test-metal --exclude tract-metal\n\n C:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: C smoke tests\n run: |\n cd api/c\n cargo install cbindgen\n make\n\n python:\n runs-on: ubuntu-latest\n needs: prepare\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ needs.prepare.outputs.test_ref }}\n fetch-depth: 0\n - name: Setup Python\n uses: actions/setup-python@v6\n with:\n python-version: \"3.13\"\n\n - name: Install uv\n uses: astral-sh/setup-uv@v7\n\n - name: Pytest bindings\n timeout-minutes: 60\n run: |\n cd api/py\n uv venv --python 3.13\n source .venv/bin/activate\n uv pip install -e \".[dev]\"\n pytest .\n" }, { "path": ".github/workflows/large_models.yml", "content": "name: Large models\n\non:\n pull_request:\n schedule:\n - cron: '0 3 * * *'\n workflow_dispatch:\n\nenv:\n LARGE_MODELS: true\n \njobs:\n cli: \n name: Build tract on ${{ matrix.os }}\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n os: [ macos-latest, ubuntu-latest ]\n steps:\n - uses: actions/checkout@v6\n - run: |\n ROOT=. ./.travis/ci-system-setup.sh\n cargo build -p tract-cli --profile opt-no-lto --no-default-features --features transformers\n - run: echo uname=$(uname) >> $GITHUB_ENV\n - uses: actions/upload-artifact@v7\n with:\n name: tract-cli-${{env.uname}}\n path: ./target/opt-no-lto/tract\n\n foundation-llms:\n runs-on: ubuntu-latest\n outputs:\n models: ${{steps.set-matrix.outputs.models}}\n q: ${{steps.set-matrix.outputs.q}}\n\n steps:\n - id: set-matrix\n env:\n FULL: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}\n run: |\n if [ \"$FULL\" = \"true\" ]\n then\n echo 'models=[ \"openelm-270M\", \"llama-3.2-1B-instruct\", \"llama-3.2-3B-instruct\", \"llama-3.1-8B-instruct\", \"qwen3-1.7B\", \"qwen3-8B\" ]' >> $GITHUB_OUTPUT\n echo 'q=[ \"f16f16\", \"f32f32\", \"q40ef16\" ]' >> $GITHUB_OUTPUT\n else\n echo ::notice::Skipping most checks on PR and commit. Dispatch workflow manually if needed.\n echo 'models=[ \"llama-3.2-1B-instruct\" ]' >> $GITHUB_OUTPUT\n echo 'q=[ \"f32f32\", \"q40ef16\" ]' >> $GITHUB_OUTPUT\n fi\n\n foundation-llm:\n name: ${{ matrix.os }} / ${{matrix.rt}} / ${{ matrix.model }} / ${{ matrix.q }}\n needs: [ cli, foundation-llms ]\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n os: [ macOS, cuda-lovelace ]\n model: ${{fromJson(needs.foundation-llms.outputs.models)}}\n q: ${{fromJson(needs.foundation-llms.outputs.q)}}\n rt: [ cpu, gpu ]\n exclude:\n - model: openelm-270M\n q: f32f32\n - model: Llama-3.2-3B-Instruct\n q: f32f32\n - model: Llama-3.2-3B-Instruct\n q: f32f32\n - model: Llama-3.1-8B-Instruct\n q: f32f32\n - model: Qwen3-1.7B\n q: f32f32\n - model: Qwen3-8B\n q: f32f32\n - model: OpenELM-270M\n q: f32f32\n fail-fast: false\n permissions:\n id-token: write\n contents: read\n\n steps:\n - uses: actions/checkout@v6\n - name: Configure AWS Credentials\n continue-on-error: true\n uses: aws-actions/configure-aws-credentials@v6\n with:\n role-to-assume: arn:aws:iam::567805100031:role/github-runner-tract-ci\n aws-region: us-east-2\n - run: echo uname=$(uname) >> $GITHUB_ENV\n - uses: actions/download-artifact@v8\n with:\n name: tract-cli-${{env.uname}}\n path: tract-cli-${{env.uname}}\n\n - name: Download and run\n run: |\n chmod +x tract-cli-${{env.uname}}/tract\n export TRACT_RUN=$GITHUB_WORKSPACE/tract-cli-${{env.uname}}/tract\n if [ \"${{matrix.rt}}\" = \"gpu\" ]\n then\n case $(uname) in\n Darwin) RT=metal;;\n Linux) RT=cuda;;\n esac\n fi\n .travis/test-llm.sh ${{matrix.model}} ${{matrix.q}} $RT\n\n parakeet-tdt-600m-v3:\n name: ${{matrix.os}} / Parakeet TDT 600m v3\n needs: [ cli ]\n strategy:\n matrix:\n os: [ macOS, cuda-lovelace ]\n fail-fast: false\n permissions:\n id-token: write\n contents: read\n runs-on: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v6\n - run: echo uname=$(uname) >> $GITHUB_ENV\n - uses: actions/download-artifact@v8\n with:\n name: tract-cli-${{env.uname}}\n path: tract-cli-${{env.uname}}\n\n - name: Download and run\n run: |\n chmod +x tract-cli-${{env.uname}}/tract\n export TRACT_RUN=$GITHUB_WORKSPACE/tract-cli-${{env.uname}}/tract\n ./harness/parakeet-tdt-600m-v3/ci.sh\n\n" }, { "path": ".github/workflows/pydoc.yml", "content": "name: Python gh-pages doc\n\non:\n pull_request:\n release:\n workflow_dispatch:\n\nenv:\n CARGO_INCREMENTAL: false\n\njobs:\n build_doc:\n name: Build doc\n runs-on: ubuntu-latest\n if: github.repository == 'sonos/tract'\n\n steps:\n - uses: actions/checkout@v6\n\n - name: Install Rust toolchain\n uses: dtolnay/rust-toolchain@stable\n\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: \"3.12\"\n\n - name: Extract version tag\n id: version\n if: github.event_name == 'release' && github.event.action == 'published'\n run: echo value=$(echo ${{ github.ref }} | cut -f 3 -d / | sed 's/^v//' ) >> $GITHUB_OUTPUT\n\n - name: Build doc\n run: |\n set -ex\n cd api/py\n python -m venv pydocs\n source pydocs/bin/activate\n pip install -r requirements-docs.txt\n pip install -e .\n sphinx-build -b html . _build/html\n cp _static/redirect-index.html _build/html/index.html\n\n - name: Deploy to gh-pages\n if: github.event_name != 'pull_request'\n run: |\n set -ex\n git config user.name \"CI bot\"\n git config user.email ci-bot@tract.rs\n\n version=\"${{ steps.version.outputs.value }}\"\n if [ -z \"$version\" ]; then\n version=\"dev\"\n fi\n\n # fetch existing gh-pages into a work directory\n git fetch origin gh-pages --depth=1 || true\n workdir=$(mktemp -d)\n git worktree add \"$workdir\" gh-pages 2>/dev/null || {\n git worktree add --orphan \"$workdir\" gh-pages\n }\n\n # copy new build into the versioned subdirectory\n rm -rf \"$workdir/$version\"\n cp -r api/py/_build/html \"$workdir/$version\"\n\n # regenerate versions.json (mike-compatible format) from directories present\n cd \"$workdir\"\n python3 -c \"\n import json, os, re\n dirs = sorted(\n [d for d in os.listdir('.') if os.path.isdir(d) and d != '.git'],\n key=lambda v: [int(x) if x.isdigit() else x for x in re.split(r'(\\d+)', v)],\n reverse=True,\n )\n versions = [{'version': d, 'title': d, 'aliases': []} for d in dirs]\n with open('versions.json', 'w') as f:\n json.dump(versions, f, indent=2)\n \"\n\n # commit and push\n git add -A\n git commit -m \"Update Python docs ($version)\" || true\n git push origin gh-pages\n\n # clean up worktree\n cd -\n git worktree remove \"$workdir\"\n" }, { "path": ".github/workflows/release.yml", "content": "on:\n push:\n tags:\n - 'v*'\n\nname: Create release\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n\njobs:\n release:\n name: Create release\n runs-on: ubuntu-latest\n steps:\n - name: Extract version tag\n id: version\n run: echo value=$(echo ${{ github.ref }} | cut -f 3 -d / | sed 's/^v//' ) >> $GITHUB_OUTPUT\n\n - uses: actions/checkout@v6\n\n - name: Create Release\n uses: softprops/action-gh-release@v2\n with:\n name: tract ${{ steps.version.outputs.value }}\n env:\n GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN_RELEASE }}\n\n" }, { "path": ".github/workflows/tract-ci-bench.yml", "content": "name: Bench with tract-ci-minion\n\non:\n schedule:\n - cron: '1 * * * *' # every hour at minute 1\n workflow_dispatch:\n\njobs:\n minion:\n strategy:\n fail-fast: false\n matrix:\n os:\n - macOS\n - cuda-lovelace\n\n runs-on: ${{ matrix.os }}\n\n steps:\n - name: Run minion if found\n run: |\n if [ -d $HOME/tract-minion ]\n then\n echo \"Running minion\"\n cd $HOME/tract-minion\n ./tract-ci-minion --once\n else\n echo \"Not running minion\"\n fi\n" }, { "path": ".github/workflows/wheels.yml", "content": "name: Python wheels\n\non:\n schedule:\n - cron: '0 3 * * MON'\n release:\n types: [created]\n workflow_dispatch:\n inputs:\n publish:\n description: force publish to pypi\n type: boolean\n pypi_version_override:\n description: override version id detection\n type: string\n\n\nenv:\n CARGO_INCREMENTAL: false\n PYPI_VERSION_OVERRIDE: ${{ inputs.pypi_version_override }}\n CIBW_ENVIRONMENT_PASS_LINUX: \"PYPI_VERSION_OVERRIDE\"\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n MACOSX_DEPLOYMENT_TARGET: 10.13\n\njobs:\n build_wheels:\n name: Build wheels on ${{ matrix.os }}\n runs-on: ${{ matrix.os}}\n strategy:\n fail-fast: false\n matrix:\n os: [ubuntu-22.04, windows-2022, macos-14]\n\n steps:\n - uses: actions/checkout@v6\n \n - name: Setup | Rust\n uses: dtolnay/rust-toolchain@stable\n\n - uses: actions/setup-python@v6\n with:\n python-version: \"3.13\"\n\n - name: Install uv\n uses: astral-sh/setup-uv@v7\n\n - name: Install rust toolchains\n if: startsWith(matrix.os, 'macOS')\n run: rustup target install x86_64-apple-darwin aarch64-apple-darwin\n\n - name: Build wheels\n uses: nick-fields/retry@v4\n with:\n max_attempts: 1\n timeout_seconds: 54000 # 15 hours :/\n command: uvx cibuildwheel --output-dir wheelhouse api/py\n\n - uses: actions/upload-artifact@v7\n with:\n name: wheels-${{github.run_id}}-${{matrix.os}}\n path: ./wheelhouse/*.whl\n\n make_sdist:\n name: Make SDist\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n\n - name: Install uv\n uses: astral-sh/setup-uv@v7\n\n - name: Build SDist\n run: cd api/py && uv build --sdist\n\n - uses: actions/upload-artifact@v7\n with:\n name: wheels-${{github.run_id}}-src\n path: api/py/dist/*.tar.gz\n\n upload_all:\n needs: [build_wheels, make_sdist]\n runs-on: ubuntu-latest\n if: (github.event_name == 'release' && github.event.action == 'published') || inputs.publish\n\n steps:\n - uses: actions/download-artifact@v8\n with:\n pattern: wheels-${{github.run_id}}-*\n merge-multiple: true\n path: dist\n\n - uses: pypa/gh-action-pypi-publish@v1.13.0\n with:\n user: __token__\n password: ${{ secrets.PYPI }}\n verbose: true\n" }, { "path": ".github/workflows/windows.yml", "content": "name: Windows unit tests\n\non:\n# pull_request:\n workflow_dispatch:\n schedule:\n - cron: '0 3 * * *'\n\n\nenv:\n CARGO_INCREMENTAL: false\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true\n\njobs:\n windows:\n strategy:\n matrix:\n os: [ windows-2022 ]\n toolchain: [ gnu, msvc ]\n fail-fast: false\n\n runs-on: ${{ matrix.os }}\n\n steps:\n - uses: actions/checkout@v6\n - uses: nick-fields/retry@v4\n name: Install Rustup using win.rustup.rs\n with:\n timeout_minutes: 10\n max_attempts: 8\n shell: pwsh\n command: |\n # Disable the download progress bar which can cause perf issues\n $ProgressPreference = \"SilentlyContinue\"\n Invoke-WebRequest https://win.rustup.rs/ -OutFile rustup-init.exe\n .\\rustup-init.exe -y --default-host=x86_64-pc-windows-msvc --profile=minimal\n - uses: nick-fields/retry@v4\n name: Install the target\n with:\n timeout_minutes: 10\n max_attempts: 8\n shell: pwsh\n command: |\n rustup toolchain add stable-x86_64-pc-windows-${{matrix.toolchain}}\n rustup default stable-x86_64-pc-windows-${{matrix.toolchain}}\n - name: Install LLVM and Clang\n uses: KyleMayes/install-llvm-action@v2\n with:\n version: \"11.0\"\n - name: debug\n run: dir \"C:\\\\Program Files\\\\LLVM\"\n - name: debug lib\n run: dir \"C:\\\\Program Files\\\\LLVM\\\\lib\"\n - name: debug bin\n run: dir \"C:\\\\Program Files\\\\LLVM\\\\bin\"\n - name: top level cargo check\n run: cargo check --workspace --exclude test-blas --exclude tract-metal --exclude test-metal --exclude causal_llm\n env:\n LIBCLANG_PATH: \"C:\\\\Program Files\\\\LLVM\\\\bin\"\n - name: data / linalg / core / nnef / onnx / onnx-opl\n run: cargo test -p tract-data -p tract-linalg -p tract-core -p tract-nnef -p tract-onnx -p tract-onnx-opl\n - name: Onnx test suite\n run: |\n cargo test --release -p test-onnx-core -p test-unit-core\n env:\n TRACT_LOG: info\n" }, { "path": ".gitignore", "content": "target\n**/*.rs.bk\n*.rustfmt\n*.back\nCargo.lock\nexamples/data\n.idea\n.cached/**\nflamegraph.svg\nperf.data*\nreadings.*\nmetrics\ntract.out\n.gdb_history\n/issue-*\n/.dinghy.toml\n.cargo\nproptest-regressions\n/tmp\nwheelhouse\ntarget-bisector*\n/nvidia\n" }, { "path": ".travis/README.md", "content": "# Travis & minions test infrastructure\n\n## Principles\n\n* travis is triggered on each commit, it will run `./.travis/native.sh` to\n perform x86_64 builds, plus a series of `./.travis/cross.sh` for as many\n arm boards configurations.\n* `.travis/cross.sh` pushes a `.tgz` to a s3 bucket for each configuration. The\n bundle contains a `entrypoint.sh` script and anything it depends on,\n including the relevant `tract` cli executable. The script is actually names\n `bundle-entrypoint.sh` in the repository.\n* devices are running `minion.sh` and will pick the new bundles from the s3 bucket,\n untar and run the `entrypoint.sh`\n\n## Testing locally\n\n```\ncargo build --release -p tract-cli && cargo bench -p tract-linalg --no-run && .travis/run-bundle.sh `.travis/make_bundle.sh`\n```\n\n## minion setup\n\n```\nMINION=user@hostname.local\nscp .travis/minionrc $MINION:.minionrc\nscp .travis/minion.sh $MINION:\n```\n\nalso setup aws credentials (.aws/credentials)\n\n```\napt install wget curl perl awscli screen vim netcat\n```\n\nOn device: `.minioncrc` set a MINION_ID. At this point, running `./minion.sh`\nshould work.\n\n## crontab\n\n`crontab -e`\n\n```\n*/10 * * * * $HOME/minion.sh\n```\n\n## systemd timers\n\nin /etc/systemd/system/minion.service\n\n```\n[Unit]\nDescription=Travis ci bench minion\n\n[Service]\nUser=root\nType=oneshot\nExecStart=/home/root/minion.sh\n```\n\nin /etc/systemd/system/minion.timer\n\n```\n[Unit]\nDescription=Run minion.service every 5 minutes\n\n[Timer]\nOnCalendar=*:0/5\n\n[Install]\nWantedBy=timers.target\n\n```\n\nthen\n\n```\nsystemctl enable minion.timer\nsystemctl start minion.timer\n```\n\n# Setup file server (http only)\n\n\n```\nsudo apt install nginx awscli vim\n```\n\n* setup aws credentials (.aws/credentials)\n* in $HOME/sync-data.sh:\n\n```\n\n```\n* chmod +x $HOME/sync-data.sh\n* run it: ./sync-data.sh\n* `crontab -e`\n\n```\n*/5 * * * * $HOME/sync-data.sh\n```\n\n\n* `sudo vi /etc/nginx/sites-available/models`\n\n```\nserver {\n root /home/raspbian/models/;\n\n location /models {\n }\n}\n```\n\n* `sudo ln -s /etc/nginx/sites-available/models /etc/nginx/sites-enabled/`\n* `sudo rm /etc/nginx/sites-enabled/default`\n* `sudo /etc/init.d/nginx reload`\n* test : `curl -I http://localhost/hey_snips_v1.pb`\n" }, { "path": ".travis/android-ndk.sh", "content": "#!/bin/sh\n\nset -ex\n\nwhich java || sudo apt install -y default-jdk\n\nANDROID_SDK=$HOME/cached/android-sdk\nif [ ! -d \"$ANDROID_SDK\" ]\nthen\n mkdir -p $ANDROID_SDK\n cd $ANDROID_SDK\n\n # ANDROID_SDK_VERSION=4333796\n # \"https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip\"\n\n curl -s -o android-sdk.zip \\\n https://dl.google.com/android/repository/commandlinetools-linux-8092744_latest.zip\n unzip -q android-sdk.zip\n rm android-sdk.zip\nfi\n\nyes | $ANDROID_SDK/cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_SDK --licenses > /dev/null\n\n$ANDROID_SDK/cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_SDK \\\n \"build-tools;30.0.0\" \"platform-tools\" \"platforms;android-31\" \"tools\" \"ndk-bundle\" \\\n > /dev/null\n" }, { "path": ".travis/asan.sh", "content": "#!/bin/sh\n\nset -ex\n\n# RUSTFLAGS=-Zsanitizer=address cargo +nightly test -Zbuild-std --target $(rustc -vV | sed -n 's|host: ||p')\n\nTARGET=$(rustc -vV | sed -n 's|host: ||p')\n\nrustup toolchain add nightly\nrustup component add rust-src --toolchain nightly-$TARGET\n\nexport RUSTFLAGS=-Zsanitizer=address \nexport RUSTDOCFLAGS=$RUSTFLAGS\nexport RUSTUP_TOOLCHAIN=nightly\nexport RUST_VERSION=nightly\nexport CARGO_EXTRA=\"--target $TARGET\"\n\ncargo -q test -q -p tract-linalg $CARGO_EXTRA\n\n# inventory, asan and macos liner are not playing nice, so we have to stop there \nif [ $(uname) == \"Darwin\" ]\nthen\n exit 0\nfi\n\ncargo -q test -q -p tract-core --features paranoid_assertions $CARGO_EXTRA\n\n./.travis/regular-tests.sh\nif [ -n \"$CI\" ]\nthen\n cargo clean\nfi\n./.travis/onnx-tests.sh\nif [ -n \"$CI\" ]\nthen\n cargo clean\nfi\n./.travis/cli-tests.sh\n\n" }, { "path": ".travis/bundle-entrypoint.sh", "content": "#!/bin/sh\n\nset -ex\n\nstart=$(date +%s)\n\nROOT=`pwd`\n\nif [ -n \"$TRACT_RUN\" ]\nthen\n TRACT=$TRACT_RUN\nelif [ -x tract ]\nthen\n TRACT=\"./tract\"\nelse\n cargo build -p tract-cli -q --release\n TRACT=\"./target/release/tract\"\nfi\n\nCACHEDIR=${CACHEDIR:-$HOME/.cache/tract-ci-minion-models}\ncase $CACHEDIR in\n \"http\"*)\n wget $CACHEDIR/private/private-benches.sh\n PRIVATE=`pwd`/private-benches.sh\n ;;\n *)\n [ -d $CACHEDIR ] || mkdir $CACHEDIR\n PATH=$PATH:/usr/local/bin # for aws command on darwin\n aws s3 sync s3://tract-ci-builds/model $CACHEDIR || echo \"Warning: aws s3 sync failed, continuing with cached models\"\n (cd $CACHEDIR\n [ -d en_libri_real ] || tar zxf en_libri_real.tar.gz\n [ -d en_tdnn_lstm_bn_q7 ] || tar zxf en_tdnn_lstm_bn_q7.tar.gz\n )\n PRIVATE=$CACHEDIR/private/private-benches.sh\n ;;\nesac\n\n\n\ntouch metrics\nif [ -e sizes ]\nthen\n cat sizes >> metrics\nfi\n\nif [ $(uname) = \"Linux\" ]\nthen\n if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor -a `cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor` = \"userspace\" ]\n then\n F=$(printf \"%s\\n\" `cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies` | sort -n | tail -1)\n echo $F > /sys/devices/system/cpu/cpu0/cpufreq/scaling_setspeed\n fi\nfi\n\nnet_bench() {\n net=$1\n pb=$2\n shift 2\n\n $TRACT \"$@\" --machine-friendly -O bench --allow-random-input $BENCH_OPTS > tract.out\n v=`cat tract.out | grep -a real | cut -f 2 -d ' ' | sed 's/\\([0-9]\\{9,9\\}\\)[0-9]*/\\1/'`\n echo net.$net.evaltime.$pb $v >> metrics\n\n $TRACT \"$@\" --readings --readings-heartbeat 1000 --machine-friendly -O bench --allow-random-input $BENCH_OPTS > tract.out\n\n for stage in model_ready before_optimize\n do\n pattern=$(echo $stage | sed 's/[_-]/./g')\n v=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 1 -d ' ')\n echo net.$net.time_to_$stage.$pb $v >> metrics\n v=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 4 -d ' ')\n echo net.$net.rsz_at_$stage.$pb $v >> metrics\n f=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 11 -d ' ')\n a=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 10 -d ' ')\n echo net.$net.active_at_$stage.$pb $(($a-$f)) >> metrics\n done\n}\n\nllm_bench() {\n net=$1\n pb=$2\n shift 2\n\n if $TRACT \"$@\" --llm --machine-friendly -O llm-bench $BENCH_OPTS > tract.out\n then\n cat tract.out\n echo llm.$net.pp512.$pb $(cat tract.out | grep -a PP512 | cut -f 2 -d ' ') >> metrics\n echo llm.$net.tg128.$pb $(cat tract.out | grep -a TG128 | cut -f 2 -d ' ') >> metrics\n fi \n\n if $TRACT \"$@\" --readings --readings-heartbeat 1000 --llm --machine-friendly -O llm-bench $BENCH_OPTS > /dev/null\n then\n for stage in model_ready before_optimize\n do\n pattern=$(echo $stage | sed 's/[_-]/./g')\n v=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 1 -d ' ')\n echo llm.$net.time_to_$stage.$pb $v >> metrics\n v=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 4 -d ' ')\n echo llm.$net.rsz_at_$stage.$pb $v >> metrics\n f=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 11 -d ' ')\n a=$(grep -a $pattern readings.out | sed 's/ */ /g;s/^ *//' | cut -f 10 -d ' ')\n if [ -n \"$a\" -a -n \"$f\" ]\n then\n echo llm.$net.active_at_$stage.$pb $(($a-$f)) >> metrics\n fi\n done\n fi\n}\n\nnet_bench arm_ml_kws_cnn_m pass $CACHEDIR/ARM-ML-KWS-CNN-M.pb -i 49,10,f32 --partial --input-node Mfcc\n\nnet_bench hey_snips_v1 400ms $CACHEDIR/hey_snips_v1.pb -i 80,40,f32\nnet_bench hey_snips_v31 400ms $CACHEDIR/hey_snips_v3.1.pb -i 40,40,f32\n\nnet_bench hey_snips_v4_model17 2sec $CACHEDIR/hey_snips_v4_model17.pb -i 200,20,f32\nnet_bench hey_snips_v4_model17 pulse8 $CACHEDIR/hey_snips_v4_model17.pb -i S,20,f32 --pulse 8\nnet_bench hey_snips_v4_model17_nnef pulse8 --nnef-tract-pulse $CACHEDIR/hey_snips_v4_model17.alpha1.tar\n\nnet_bench mobilenet_v1_1 pass $CACHEDIR/mobilenet_v1_1.0_224_frozen.pb -i 1,224,224,3,f32\nnet_bench mobilenet_v2_1 pass $CACHEDIR/mobilenet_v2_1.4_224_frozen.pb -i 1,224,224,3,f32\n\nnet_bench inceptionv1q pass $CACHEDIR/inceptionv1_quant.nnef.tar.gz --nnef-tract-core\nnet_bench inceptionv3 pass $CACHEDIR/inception_v3_2016_08_28_frozen.pb -i 1,299,299,3,f32\n\n\nnet_bench mdl-en-2019-Q3-librispeech_onnx 2600ms $CACHEDIR/en_libri_real/model.onnx --output-node output -i 264,40\nnet_bench mdl-en-2019-Q3-librispeech_onnx pulse_240ms $CACHEDIR/en_libri_real/model.onnx --output-node output -i S,40 --pulse 24\nnet_bench en_tdnn_lstm_bn_q7 2600ms $CACHEDIR/en_tdnn_lstm_bn_q7/model.onnx --output-node output -i 264,40\nnet_bench en_tdnn_lstm_bn_q7 pulse_240ms $CACHEDIR/en_tdnn_lstm_bn_q7/model.onnx --output-node output -i S,40 --pulse 24\nnet_bench en_tdnn_8M 2600ms $CACHEDIR/mdl-en-2019-12-24-aho-corasick-18h01m33s.onnx --output-node output -i 264,40\nnet_bench en_tdnn_8M pulse_240ms $CACHEDIR/mdl-en-2019-12-24-aho-corasick-18h01m33s.onnx --output-node output -i S,40 --pulse 24\nnet_bench en_tdnn_8M pulse_180ms $CACHEDIR/mdl-en-2019-12-24-aho-corasick-18h01m33s.onnx --output-node output -i S,40 --pulse 18\nnet_bench en_tdnn_8M pulse_120ms $CACHEDIR/mdl-en-2019-12-24-aho-corasick-18h01m33s.onnx --output-node output -i S,40 --pulse 12\nnet_bench en_tdnn_8M_nnef pulse_240ms $CACHEDIR/mdl-en-2019-12-24-aho-corasick-18h01m33s.alpha1.a.tar --nnef-tract-pulse\nnet_bench en_tdnn_15M 2600ms $CACHEDIR/en_tdnn_15M.onnx --output-node output -i 264,40\nnet_bench en_tdnn_15M pulse_240ms $CACHEDIR/en_tdnn_15M.onnx --output-node output -i S,40 --pulse 24\nnet_bench en_tdnn_15M pulse_120ms $CACHEDIR/en_tdnn_15M.onnx --output-node output -i S,40 --pulse 12\nnet_bench en_tdnn_15M_nnef pulse_240ms $CACHEDIR/en_tdnn_15M.alpha1.tar --nnef-tract-pulse\nnet_bench dummy-conmer-12M pulse_120ms $CACHEDIR/dummy-conmer-12M.nnef.tar --nnef-tract-core --pulse 12\n\nnet_bench en_tdnn_pyt_15M pulse_120ms $CACHEDIR/mdl-en-2023-03-27-allen-17h11m50s.nnef.tar --nnef-tract-core --pulse 12\n\nnet_bench speaker_id pulse8 $CACHEDIR/speaker-id-2019-03.onnx -i 1,S,40,f32 --output-node 257 --partial --pulse 8\n\nnet_bench voicecom_fake_quant 2sec $CACHEDIR/snips-voice-commands-cnn-fake-quant.pb -i 200,10,f32\nnet_bench voicecom_float 2sec $CACHEDIR/snips-voice-commands-cnn-float.pb -i 200,10,f32\n\nnet_bench trunet pulse1_f32 $CACHEDIR/trunet_dummy.nnef.tgz --nnef-tract-core --pulse 1\nnet_bench trunet pulse1_f16 $CACHEDIR/trunet_dummy.nnef.tgz --nnef-tract-core -t f32_to_f16 --pulse 1\n\n. $PRIVATE\n\nif [ $(uname) = \"Darwin\" ]\nthen\n LLM_BACKENDS=\"cpu metal\"\nfi\n\nif which nvidia-smi \nthen \n LLM_BACKENDS=\"cpu cuda\"\nfi\n\nif [ -n \"$LLM_BACKENDS\" ]\nthen\n for backend in $LLM_BACKENDS\n do\n case $backend in\n cpu) extra=\"--timeout 180\";;\n metal) extra=\"--metal --timeout 60\"\n BENCH_OPTS=\"--warmup-loops 1\"\n ;;\n cuda) extra=\"--cuda --timeout 60\"\n BENCH_OPTS=\"--warmup-loops 1\"\n ;;\n esac\n llm_bench llama-3_2-1B-q40ef32-516 $backend $CACHEDIR/Llama-3.2-1B-q40ef32.516.nnef.tgz $extra\n llm_bench openelm-270M-q40ef16-516 $backend $CACHEDIR/OpenELM-270M-q40ef16.516.nnef.tgz $extra\n llm_bench llama-3_2-1B-instruct-q40ef16-541 $backend $CACHEDIR/Llama-3.2-1B-Instruct-q40ef16.541.nnef.tgz $extra\n llm_bench openelm-270M-q40ef16-541 $backend $CACHEDIR/OpenELM-270M-q40ef16.541.nnef.tgz $extra\n net_bench parakeet-tdt-600m-v3-f32f32-preprocessor_1s $backend $CACHEDIR/parakeet-tdt-0.6b-v3-f32f32.608.preprocessor.nnef.tgz \\\n -t transformers_detect_all --nnef-tract-transformers --set B=1 --set A=16000 $extra\n net_bench parakeet-tdt-600m-v3-f32f32-encoder_1s $backend $CACHEDIR/parakeet-tdt-0.6b-v3-f32f32.608.encoder.p1.nnef.tgz \\\n -t transformers_detect_all --nnef-tract-transformers --set B=1 --set S=100 $extra\n net_bench parakeet-tdt-600m-v3-f32f32-decoder_pass $backend $CACHEDIR/parakeet-tdt-0.6b-v3-f32f32.608.decoder.nnef.tgz \\\n -t transformers_detect_all --nnef-tract-transformers --set B=1 --set T=1 $extra\n net_bench parakeet-tdt-600m-v3-f32f32-joint_pass $backend $CACHEDIR/parakeet-tdt-0.6b-v3-f32f32.608.joint.nnef.tgz \\\n -t transformers_detect_all --nnef-tract-transformers --set B=1 --set R=1 --set U=1 $extra\n\n if [ \"$backend\" != \"cpu\" ]\n then\n llm_bench llama-3_2-3B-q40ef32-516 $backend $CACHEDIR/Llama-3.2-3B-q40ef32.516.nnef.tgz $extra\n llm_bench llama-3_1-8B-instruct-q40ef16-541 $backend $CACHEDIR/Llama-3.1-8B-Instruct-q40ef16.541.nnef.tgz $extra\n llm_bench llama-3_2-3B-instruct-q40ef16-541 $backend $CACHEDIR/Llama-3.2-3B-Instruct-q40ef16.541.nnef.tgz $extra\n llm_bench qwen3-1_7B-q40ef16-541 $backend $CACHEDIR/Qwen3-1.7B-q40ef16.541.nnef.tgz $extra\n fi\n done\nfi\n\nend=$(date +%s)\n\necho bundle.bench-runtime $(($end - $start)) >> metrics\n\n" }, { "path": ".travis/cache_file.sh", "content": "#!/bin/sh\n\nset -e\n\nif [ -z \"$CACHEDIR\" ]\nthen\n CACHEDIR=`dirname $0`/../.cached\nfi\n\nmkdir -p $CACHEDIR\ncd $CACHEDIR\nfor file in $@\ndo\n mkdir -p $(dirname $file)\n if [ ! -e $file ]\n then\n wget --no-verbose https://s3.amazonaws.com/tract-ci-builds/tests/$file -O $file.tmp \\\n || aws s3 cp s3://tract-ci-builds/tests/$file $file.tmp\n mv $file.tmp $file\n fi\ndone\n\nexit 0\n" }, { "path": ".travis/cargo-deny-check.sh", "content": "#!/bin/sh\n\nif [ -e cargo-deny ]\nthen\n CARGO_DENY=`pwd`/cargo-deny\nelse\n CARGO_DENY=\"cargo deny\"\nfi\n\n(cd api/rs ; $CARGO_DENY check)\n" }, { "path": ".travis/ci-system-setup.sh", "content": "#!/bin/sh\nset -e\n\n[ -d $ROOT/.travis ] || exit 1 \"\\$ROOT not set correctly '$ROOT'\"\n\nif [ -z \"$RUSTUP_TOOLCHAIN\" ]\nthen\n export RUSTUP_TOOLCHAIN=1.91.0\nfi\n\nexport RUSTUP_TOOLCHAIN\nPATH=$PATH:$HOME/.cargo/bin\n\nif [ -n \"$CI\" -a ! -e /tmp/ci-setup-done ]\nthen\n if [ `uname` = \"Darwin\" ]\n then\n sysctl -n machdep.cpu.brand_string\n python3 --version\n brew install coreutils numpy python-setuptools jshon\n PATH=\"/opt/homebrew/opt/coreutils/libexec/gnubin:$PATH\"\n export PYTHON_BIN_PATH=python3\n else\n if [ \"$RUNNER_ENVIRONMENT\" != \"self-hosted\" ]\n then\n if [ `whoami` != \"root\" ]\n then\n SUDO=sudo\n fi\n $SUDO apt-get update\n # $SUDO apt-get upgrade -y\n $SUDO apt-get install -y llvm python3 python3-numpy jshon wget curl build-essential sudo jshon clang \n if ! which aws\n then\n curl -fsSL \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o awscliv2.zip\n $SUDO apt-get install -y unzip\n unzip -q awscliv2.zip\n $SUDO ./aws/install\n aws --version\n fi\n fi\n fi\n\n which rustup || curl https://sh.rustup.rs -sSf | sh -s -- -y\n rustup update\n rustup toolchain add $RUSTUP_TOOLCHAIN\n [ -n \"$GITHUB_PATH\" ] && echo $HOME/.cargo/bin >> $GITHUB_PATH\n\n touch /tmp/ci-setup-done\nfi\n\nS3=https://s3.amazonaws.com/tract-ci-builds/tests\n\nif [ -n \"$LARGE_MODELS\" ]\nthen\n export CACHE_FILE=$ROOT/.travis/cache_file.sh\n export MODELS=$HOME/.cache/models\n export CACHEDIR=$MODELS\n mkdir -p $MODELS\nelif [ -n \"$CI\" ]\nthen\n MODELS=$S3\n CACHE_FILE=true\nelse \n CACHE_FILE=$ROOT/.travis/cache_file.sh\n MODELS=${MODELS:-$ROOT/.cached}\n mkdir -p $MODELS\nfi\n\nif [ -z \"$TRACT_RUN\" ]\nthen\n TRACT_RUN=\"cargo run -p tract-cli $CARGO_EXTRA --profile opt-no-lto --no-default-features --features transformers,pulse --\"\n export TRACT_RUN\nfi\n\nTRACT_RUNTIMES=\"-O\"\nif [ \"$(uname)\" = \"Darwin\" ] && (system_profiler SPDisplaysDataType | grep -i \"Metal\")\nthen \n TRACT_RUNTIMES=\"$TRACT_RUNTIMES --metal\"\nfi\n\nif which nvidia-smi\nthen\n TRACT_RUNTIMES=\"$TRACT_RUNTIMES --cuda\"\nfi\n\necho $TRACT_RUNTIMES\n" }, { "path": ".travis/cli-tests.sh", "content": "#!/bin/sh\n\nWHITE='\\033[1;37m'\nNC='\\033[0m' # No Color\n\nset -e\n\nROOT=$(dirname $(dirname $(realpath $0)))\n. $ROOT/.travis/ci-system-setup.sh\n\necho\necho $WHITE • build tract $NC\necho\n\nTRACT_RUN=$(cargo build --message-format json -p tract-cli $CARGO_EXTRA --profile opt-no-lto | jq -r 'select(.target.name == \"tract\" and .executable).executable')\necho TRACT_RUN=$TRACT_RUN\nexport TRACT_RUN\n\necho\necho $WHITE • harness/nnef-test-cases $NC\necho\n\nfor t in `find harness/nnef-test-cases -name runme.sh`\ndo\n echo $WHITE$t$NC\n $t\ndone\n\necho\necho $WHITE • onnx/test_cases $NC\necho\n\n# ( cd onnx/test_cases ; CACHEDIR=$MODELS ./run_all.sh )\n\necho\necho $WHITE • full models command line test cases $NC\necho\n\necho $WHITE image $NC\n\n$CACHE_FILE squeezenet.onnx\n$TRACT_RUN $MODELS/squeezenet.onnx -O \\\n run -q \\\n --allow-random-input \\\n --assert-output-fact 1,1000,1,1,f32\n\n$CACHE_FILE inception_v3_2016_08_28_frozen.pb\n$TRACT_RUN \\\n $MODELS/inception_v3_2016_08_28_frozen.pb \\\n -i 1,299,299,3,f32 -O \\\n run -q \\\n --allow-random-input \\\n --assert-output-fact 1,1001,f32\n\n$TRACT_RUN \\\n $MODELS/inception_v3_2016_08_28_frozen.pb \\\n -i 1,299,299,3,f32 -O \\\n run -q \\\n --allow-random-input \\\n --assert-output-fact 1,1001,f32\n\n$CACHE_FILE mobilenet_v1_1.0_224_frozen.pb\n$TRACT_RUN $MODELS/mobilenet_v1_1.0_224_frozen.pb \\\n -O -i 1,224,224,3,f32 \\\n run -q \\\n --allow-random-input \\\n --assert-output-fact 1,1001,f32\n\n$CACHE_FILE mobilenet_v2_1.4_224_frozen.pb\n$TRACT_RUN $MODELS/mobilenet_v2_1.4_224_frozen.pb \\\n -O -i 1,224,224,3,f32 \\\n run -q \\\n --allow-random-input \\\n --assert-output-fact 1,1001,f32\n\n$CACHE_FILE inceptionv1_quant.nnef.tar.gz inceptionv1_quant.io.npz\n$TRACT_RUN $MODELS/inceptionv1_quant.nnef.tar.gz \\\n --nnef-tract-core \\\n --input-facts-from-bundle $MODELS/inceptionv1_quant.io.npz -O \\\n run \\\n --input-from-bundle $MODELS/inceptionv1_quant.io.npz \\\n --allow-random-input \\\n --assert-output-bundle $MODELS/inceptionv1_quant.io.npz\n\necho $WHITE audio $NC\n\n$CACHE_FILE ARM-ML-KWS-CNN-M.pb\n$TRACT_RUN $MODELS/ARM-ML-KWS-CNN-M.pb \\\n -O -i 49,10,f32 --partial \\\n --input-node Mfcc \\\n run -q \\\n --allow-random-input\n\n$CACHE_FILE GRU128KeywordSpotter-v2-10epochs.onnx\n$TRACT_RUN $MODELS/GRU128KeywordSpotter-v2-10epochs.onnx \\\n -O run -q \\\n --allow-random-input \\\n --assert-output-fact 1,3,f32\n\n$CACHE_FILE mdl-en-2019-Q3-librispeech.onnx\n$TRACT_RUN $MODELS/mdl-en-2019-Q3-librispeech.onnx \\\n -O -i S,40,f32 --output-node output --pulse 24 \\\n run -q \\\n --allow-random-input\n \n$CACHE_FILE hey_snips_v4_model17.pb\n$TRACT_RUN $MODELS/hey_snips_v4_model17.pb \\\n -i S,20,f32 --pulse 8 dump --cost -q \\\n --assert-cost \"FMA(F32)=2060448,Div(F32)=24576,Buffer(F32)=2920,Params(F32)=222251\"\n\n$TRACT_RUN $MODELS/hey_snips_v4_model17.pb -i S,20,f32 \\\n dump -q \\\n --assert-op-count AddAxis 0\n\n$CACHE_FILE trunet_dummy.nnef.tgz\n$TRACT_RUN --nnef-tract-core $MODELS/trunet_dummy.nnef.tgz dump -q\n\necho $WHITE LLM $NC\n\nTEMP_ELM=$(mktemp -d)\n$CACHE_FILE 2024_06_25_elm_micro_export_with_kv_cache.nnef.tgz\n$TRACT_RUN $MODELS/2024_06_25_elm_micro_export_with_kv_cache.nnef.tgz \\\n --nnef-tract-core \\\n --assert \"S>0\" --assert \"P>0\" --assert \"S+P<2048\" \\\n dump -q --nnef $TEMP_ELM/with-asserts.nnef.tgz\n$TRACT_RUN --nnef-tract-core $TEMP_ELM/with-asserts.nnef.tgz dump -q\nrm -rf $TEMP_ELM\n\nfor t in harness/pre-optimized-graphes/*\ndo\n ( cd $t ; ./runme.sh)\ndone\n\n(\nif aws s3 ls tract-ci-builds/model/private\nthen\n echo\n echo $WHITE • private tests $NC\n echo\n if [ -n \"$CI\" ]\n then\n OUTPUT=/dev/null\n else\n set -x\n OUTPUT=/dev/stdout\n fi\n (\n mkdir -p $CACHEDIR\n cd $CACHEDIR\n aws s3 sync s3://tract-ci-builds/model/private private\n for t in `find private -name t.sh`\n do\n ( cd `dirname $t` ; sh ./t.sh )\n done\n ) 2>&1 > $OUTPUT\n\n echo\n echo $WHITE • benches on full models $NC\n echo\n\n ./.travis/bundle-entrypoint.sh\nfi\n)\n\n" }, { "path": ".travis/cost_model_task_build.sh", "content": "#!/bin/sh\n\nset -ex\n\nARCH=$1\nID=$2\n\ncase $ARCH in\n aarch64)\n MUSL_TRIPLE=aarch64-linux-musl\n RUST_TRIPLE=aarch64-unknown-linux-musl\n PLATFORM=aarch64-unknown-linux-musl\n ;;\n armv7)\n MUSL_TRIPLE=armv7l-linux-musleabihf\n RUST_TRIPLE=armv7-unknown-linux-musleabihf\n PLATFORM=armv7-unknown-linux-musl\n ;;\n *)\n exit \"Can't build with musl for $ARCH\"\n ;;\nesac\n\nrustup update\nrustup target add $RUST_TRIPLE\n\n#curl -s https://musl.cc/${MUSL_TRIPLE}-cross.tgz | tar zx\ncurl -s https://s3.amazonaws.com/tract-ci-builds/toolchains/${MUSL_TRIPLE}-cross.tgz | tar zx\n\nMUSL_BIN=`pwd`/${MUSL_TRIPLE}-cross/bin\nexport PATH=$MUSL_BIN:$PATH\n\nexport TARGET_CC=$MUSL_BIN/${MUSL_TRIPLE}-gcc\n\nRUST_TRIPLE_ENV=$(echo $RUST_TRIPLE | tr 'a-z-' 'A-Z_')\nexport CARGO_TARGET_${RUST_TRIPLE_ENV}_CC=$TARGET_CC\nexport CARGO_TARGET_${RUST_TRIPLE_ENV}_LINKER=$TARGET_CC\n\n( cd linalg/cost_model ; cargo build --target $RUST_TRIPLE --release )\n\nTASK_NAME=cost-model-dataset-$ID\n\nmkdir $TASK_NAME\nmv linalg/cost_model/target/${RUST_TRIPLE}/release/cost_model $TASK_NAME\necho \"export TIMEOUT=$((86400*4))\" > $TASK_NAME/vars\necho \"#!/bin/sh\" > $TASK_NAME/entrypoint.sh\necho \"mkdir product\" >> $TASK_NAME/entrypoint.sh\necho \"./cost_model ds --size 10000 product/$TASK_NAME.txt\" >> $TASK_NAME/entrypoint.sh\n# echo \"./cost_model ds --size 2000 -k 128 -n 16 product/$TASK_NAME-small-k-tiny-n.txt\" >> $TASK_NAME/entrypoint.sh\n# echo \"./cost_model ds --size 5000 -m 1-512 -k 16,64,256 -n 1-20 product/$TASK_NAME-multiple-k-tiny-n.txt\" >> $TASK_NAME/entrypoint.sh\n# echo \"./cost_model ds --size 1000 -m 1-512 -k 256,1024 -n 1-512 product/$TASK_NAME-bigmn\" >> $TASK_NAME/entrypoint.sh\nchmod +x $TASK_NAME/entrypoint.sh\ntar czf $TASK_NAME.tgz $TASK_NAME\n\nif [ -n \"$AWS_ACCESS_KEY_ID\" ]\nthen\n aws s3 cp $TASK_NAME.tgz s3://tract-ci-builds/tasks/$PLATFORM/$TASK_NAME.tgz\nfi\n" }, { "path": ".travis/cross.sh", "content": "#!/bin/sh\n\nset -ex\n\nROOT=$(dirname $(dirname $(realpath $0)))\n. $ROOT/.travis/ci-system-setup.sh\n\nwhich cargo-dinghy || ( mkdir -p /tmp/cargo-dinghy\n if [ `arch` = x86_64 -o `arch` = i386 -o `arch` = arm64 ]\n then\n cd /tmp/cargo-dinghy\n if [ `uname` = \"Darwin\" ]\n then\n NAME=macos\n else\n NAME=linux\n fi\n VERSION=0.8.0\n wget -q https://github.com/snipsco/dinghy/releases/download/$VERSION/cargo-dinghy-$NAME-$VERSION.tgz -O cargo-dinghy.tgz\n tar vzxf cargo-dinghy.tgz --strip-components 1\n mv cargo-dinghy $HOME/.cargo/bin\n else\n cargo install cargo-dinghy\n fi\n)\n\nif [ -z \"$PLATFORM\" -a -n \"$1\" ]\nthen\n PLATFORM=$1\nfi\n\ncase \"$PLATFORM\" in\n \"raspbian\")\n [ -e $HOME/cached/raspitools ] || git clone --depth 1 https://github.com/raspberrypi/tools $HOME/cached/raspitools\n TOOLCHAIN=$HOME/cached/raspitools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf\n export RUSTC_TRIPLE=arm-unknown-linux-gnueabihf\n rustup target add $RUSTC_TRIPLE\n echo \"[platforms.$PLATFORM]\\nrustc_triple='$RUSTC_TRIPLE'\\ntoolchain='$TOOLCHAIN'\" > .dinghy.toml\n cargo dinghy --platform $PLATFORM build --release -p tract-cli -p example-tensorflow-mobilenet-v2 -p tract-ffi\n ;;\n\n \"aarch64-linux-android\"|\"armv7-linux-androideabi\"|\"i686-linux-android\"|\"x86_64-linux-android\")\n case \"$PLATFORM\" in\n \"aarch64-linux-android\")\n ANDROID_CPU=aarch64\n RUSTC_TRIPLE=aarch64-linux-android\n ;;\n \"armv7-linux-androideabi\")\n ANDROID_CPU=armv7\n RUSTC_TRIPLE=armv7-linux-androideabi\n ;;\n \"i686-linux-android\")\n ANDROID_CPU=i686\n RUSTC_TRIPLE=i686-linux-android\n ;;\n \"x86_64-linux-android\")\n ANDROID_CPU=x86_64\n RUSTC_TRIPLE=x86_64-linux-android\n ;;\n esac\n\n export TARGET_AR=ar\n\n if [ -e /usr/local/lib/android/sdk/ndk-bundle ]\n then\n export ANDROID_NDK_HOME=/usr/local/lib/android/sdk/ndk-bundle\n else\n export ANDROID_SDK_HOME=$HOME/cached/android-sdk\n [ -e $ANDROID_SDK_HOME ] || ./.travis/android-ndk.sh\n fi\n\n rustup target add $RUSTC_TRIPLE\n cargo dinghy --platform auto-android-$ANDROID_CPU build -p tract-linalg -p tract-ffi\n ;;\n\n \"aarch64-apple-ios\")\n rustup target add aarch64-apple-ios\n cargo dinghy --platform auto-ios-aarch64 check -p tract-linalg -p tract-ffi\n ;;\n\n \"aarch64-apple-darwin\" | \"x86_64-unknown-linux-gnu\")\n RUSTC_TRIPLE=$PLATFORM\n rustup target add $RUSTC_TRIPLE\n cargo build --target $RUSTC_TRIPLE -p tract-cli --release\n ;;\n\n \"aarch64-unknown-linux-gnu-stretch\" | \"armv7-unknown-linux-gnueabihf-stretch\" | \"x86_64-unknown-linux-gnu-stretch\")\n INNER_PLATFORM=${PLATFORM%-stretch}\n (cd .travis/docker-debian-stretch; docker build --tag debian-stretch .)\n docker run -v `pwd`:/tract -w /tract \\\n -e CI=true \\\n -e SKIP_QEMU_TEST=skip \\\n -e PLATFORM=$INNER_PLATFORM debian-stretch \\\n ./.travis/cross.sh\n sudo chown -R `whoami` .\n export RUSTC_TRIPLE=$INNER_PLATFORM\n ;;\n\n \"aarch64-unknown-linux-gnu\" | \"armv6vfp-unknown-linux-gnueabihf\" | \"armv7-unknown-linux-gnueabihf\" | \\\n \"aarch64-unknown-linux-musl\" | \"armv7-unknown-linux-musl\" | \"cortexa53-unknown-linux-musl\" )\n\n case \"$PLATFORM\" in\n \"aarch64-unknown-linux-gnu\")\n export ARCH=aarch64\n export QEMU_ARCH=aarch64\n export LIBC_ARCH=arm64\n export TRACT_CPU_AARCH64_KIND=a55\n export RUSTC_TRIPLE=$ARCH-unknown-linux-gnu\n export DEBIAN_TRIPLE=$ARCH-linux-gnu\n ;;\n \"armv6vfp-unknown-linux-gnueabihf\")\n export ARCH=armv6vfp\n export LIBC_ARCH=armhf\n export QEMU_ARCH=arm\n export QEMU_OPTS=\"-cpu cortex-a15\"\n export RUSTC_TRIPLE=arm-unknown-linux-gnueabihf\n export DEBIAN_TRIPLE=arm-linux-gnueabihf\n ;;\n \"armv7-unknown-linux-gnueabihf\")\n export ARCH=armv7\n export QEMU_ARCH=arm\n export LIBC_ARCH=armhf\n export QEMU_OPTS=\"-cpu cortex-a15\"\n export RUSTC_TRIPLE=armv7-unknown-linux-gnueabihf\n export DEBIAN_TRIPLE=arm-linux-gnueabihf\n export TARGET_CC=$DEBIAN_TRIPLE-gcc\n export TRACT_CPU_ARM32_NEON=true\n export DINGHY_TEST_ARGS=\"--env TRACT_CPU_ARM32_NEON=true\"\n ;;\n \"aarch64-unknown-linux-musl\")\n export ARCH=aarch64\n export QEMU_ARCH=aarch64\n export LIBC_ARCH=arm64\n export RUSTC_TRIPLE=$ARCH-unknown-linux-musl\n export DEBIAN_TRIPLE=$ARCH-linux-gnu\n export TRACT_CPU_AARCH64_KIND=a55\n export CUSTOM_TC=`pwd`/aarch64-linux-musl-cross\n [ -d \"$CUSTOM_TC\" ] || curl -s https://s3.amazonaws.com/tract-ci-builds/toolchains/aarch64-linux-musl-cross.tgz | tar zx\n ;;\n \"cortexa53-unknown-linux-musl\")\n export ARCH=aarch64\n export QEMU_ARCH=aarch64\n export LIBC_ARCH=arm64\n export RUSTC_TRIPLE=$ARCH-unknown-linux-musl\n export DEBIAN_TRIPLE=$ARCH-linux-gnu\n export TRACT_CPU_AARCH64_KIND=a53\n export QEMU_OPTS=\"-cpu cortex-a53\"\n export CUSTOM_TC=`pwd`/aarch64-linux-musl-cross\n [ -d \"$CUSTOM_TC\" ] || curl -s https://s3.amazonaws.com/tract-ci-builds/toolchains/aarch64-linux-musl-cross.tgz | tar zx\n ;;\n \"armv7-unknown-linux-musl\")\n export ARCH=armv7\n export QEMU_ARCH=arm\n export LIBC_ARCH=armhf\n export RUSTC_TRIPLE=armv7-unknown-linux-musleabihf\n export DEBIAN_TRIPLE=arm-linux-gnueabihf\n export CUSTOM_TC=`pwd`/armv7l-linux-musleabihf-cross\n export TRACT_CPU_ARM32_NEON=true\n export DINGHY_TEST_ARGS=\"--env TRACT_CPU_ARM32_NEON=true\"\n [ -d \"$CUSTOM_TC\" ] || curl -s https://s3.amazonaws.com/tract-ci-builds/toolchains/armv7l-linux-musleabihf-cross.tgz | tar zx\n export TARGET_CFLAGS=\"-mfpu=neon\"\n ;;\n *)\n echo \"unsupported platform $PLATFORM\"\n exit 1\n ;;\n esac\n\n mkdir -p $ROOT/target/$RUSTC_TRIPLE\n echo \"[platforms.$PLATFORM]\\nrustc_triple='$RUSTC_TRIPLE'\" > .dinghy.toml\n if [ -n \"$DEBIAN_TRIPLE\" ]\n then\n PACKAGES=\"$PACKAGES binutils-$DEBIAN_TRIPLE gcc-$DEBIAN_TRIPLE libc6-dev-$LIBC_ARCH-cross\"\n echo \"deb_multiarch='$DEBIAN_TRIPLE'\" >> .dinghy.toml\n fi\n\n if [ -n \"$CUSTOM_TC\" ]\n then\n echo \"toolchain='$CUSTOM_TC'\" >> .dinghy.toml\n fi\n\n echo \"[script_devices.qemu-$PLATFORM]\\nplatform='$PLATFORM'\\npath='$ROOT/target/$RUSTC_TRIPLE/qemu-$PLATFORM'\" >> .dinghy.toml\n echo \"#!/bin/sh\\nexe=\\$1\\nshift\\n/usr/bin/qemu-$QEMU_ARCH $QEMU_OPTS -L /usr/$DEBIAN_TRIPLE/ \\$exe --test-threads 1 \\\"\\$@\\\"\" > $ROOT/target/$RUSTC_TRIPLE/qemu-$PLATFORM\n chmod +x $ROOT/target/$RUSTC_TRIPLE/qemu-$PLATFORM\n\n DINGHY_TEST_ARGS=\"$DINGHY_TEST_ARGS --env PROPTEST_MAX_SHRINK_ITERS=100000000\"\n\n $SUDO apt-get -y install --no-install-recommends qemu-system-arm qemu-user libssl-dev pkg-config $PACKAGES\n rustup target add $RUSTC_TRIPLE\n if [ -z \"$SKIP_QEMU_TEST\" ]\n then\n qemu-$QEMU_ARCH --version\n cargo dinghy --platform $PLATFORM $DINGHY_TEST_ARGS test --profile opt-no-lto -p tract-linalg -- --nocapture\n cargo dinghy --platform $PLATFORM $DINGHY_TEST_ARGS test --profile opt-no-lto -p tract-core\n fi\n\n cargo dinghy --platform $PLATFORM $DINGHY_TEST_ARGS check -p tract-ffi\n # keep lto for these two are they're going to devices.\n cargo dinghy --platform $PLATFORM build --release -p tract-cli -p example-tensorflow-mobilenet-v2\n ;;\n\n wasm32-wasi)\n PLATFORM=wasm32-wasip1\n wasmtime --version\n\n rustup target add $PLATFORM\n cargo check --target $PLATFORM --features getrandom-js -p tract-onnx -p tract-tensorflow\n RUSTFLAGS='-C target-feature=+simd128' CARGO_TARGET_WASM32_WASIP1_RUNNER=wasmtime \\\n cargo test --target=$PLATFORM -p tract-linalg -p tract-core -p test-unit-core\n ;;\n wasm32-*)\n rustup target add $PLATFORM\n cargo check --target $PLATFORM --features getrandom-js -p tract-onnx -p tract-tensorflow\n ;;\n *)\n echo \"Don't know what to do for platform: $PLATFORM\"\n exit 2\n ;;\nesac\n\nif [ -e \"target/$RUSTC_TRIPLE/release/tract\" ]\nthen\n export RUSTC_TRIPLE\n TASK_NAME=`.travis/make_bundle.sh`\n echo bench task: $TASK_NAME \n if [ -n \"$AWS_ACCESS_KEY_ID\" ]\n then\n aws s3 cp $TASK_NAME.tgz s3://tract-ci-builds/tasks/$PLATFORM/$TASK_NAME.tgz\n fi\nfi\n" }, { "path": ".travis/debug-tests.sh", "content": "#!/bin/sh\n\nset -ex\n\nif [ -z \"$CACHEDIR\" ]\nthen\n CACHEDIR=`dirname $0`/../.cached\nfi\n\n# useful as debug_asserts will come into play\ncargo test -p tract-core\ncargo test -p test-onnx-core -p test-nnef-cycle -p test-unit-core\n" }, { "path": ".travis/docker-debian-stretch/Dockerfile", "content": "FROM debian:stretch\nCOPY sources.list /etc/apt/sources.list\n" }, { "path": ".travis/docker-debian-stretch/sources.list", "content": "deb http://archive.debian.org/debian/ stretch contrib main non-free\ndeb http://archive.debian.org/debian stretch-backports main\ndeb http://archive.debian.org/debian-security stretch/updates main\n" }, { "path": ".travis/examples.sh", "content": "#!/bin/sh\n\nWHITE='\\033[1;37m'\nNC='\\033[0m' # No Color\n\nset -e\n\nROOT=$(dirname $(dirname $(realpath $0)))\n. $ROOT/.travis/ci-system-setup.sh\n\nfor t in `find examples -name ci.sh`\ndo\n df -h\n ex=$(dirname $t)\n echo ::group:: $ex\n echo $WHITE $ex $NC\n ( cd $ex ; sh ./ci.sh )\n if [ -n \"$CI\" ]\n then\n cargo clean\n fi\n echo ::endgroup::\ndone\n\n" }, { "path": ".travis/llm-expectations-541", "content": "Qwen--Qwen3-1.7B-f16f16.p0s100.arm64.cpu 0.96\nQwen--Qwen3-1.7B-f16f16.p0s100.arm64.metal 0.96\nQwen--Qwen3-1.7B-f16f16.p0s100.x86_64.cpu 0.99\nQwen--Qwen3-1.7B-f16f16.p0s100.x86_64.cuda 0.99\nQwen--Qwen3-1.7B-f16f16.p50s50.arm64.cpu 0.97\nQwen--Qwen3-1.7B-f16f16.p50s50.arm64.metal 0.97\nQwen--Qwen3-1.7B-f16f16.p50s50.x86_64.cpu 0.99\nQwen--Qwen3-1.7B-f16f16.p50s50.x86_64.cuda 0.99\nQwen--Qwen3-1.7B-f16f16.p99s1.arm64.cpu 0.99\nQwen--Qwen3-1.7B-f16f16.p99s1.arm64.metal 0.99\nQwen--Qwen3-1.7B-f16f16.p99s1.x86_64.cpu 0.99\nQwen--Qwen3-1.7B-f16f16.p99s1.x86_64.cuda 0.99\nQwen--Qwen3-1.7B-q40ef16.p0s100.arm64.cpu 0.92\nQwen--Qwen3-1.7B-q40ef16.p0s100.arm64.metal 0.98\nQwen--Qwen3-1.7B-q40ef16.p0s100.x86_64.cpu 0.99\nQwen--Qwen3-1.7B-q40ef16.p0s100.x86_64.cuda 0.92\nQwen--Qwen3-1.7B-q40ef16.p50s50.arm64.cpu 0.96\nQwen--Qwen3-1.7B-q40ef16.p50s50.arm64.metal 0.99\nQwen--Qwen3-1.7B-q40ef16.p50s50.x86_64.cpu 0.99\nQwen--Qwen3-1.7B-q40ef16.p50s50.x86_64.cuda 0.98\nQwen--Qwen3-1.7B-q40ef16.p99s1.arm64.cpu 0.97\nQwen--Qwen3-1.7B-q40ef16.p99s1.arm64.metal 0.99\nQwen--Qwen3-1.7B-q40ef16.p99s1.x86_64.cpu 0.99\nQwen--Qwen3-1.7B-q40ef16.p99s1.x86_64.cuda 0.96\nQwen--Qwen3-8B-f16f16.p0s100.arm64.cpu 0.94\nQwen--Qwen3-8B-f16f16.p0s100.arm64.metal 0.95\nQwen--Qwen3-8B-f16f16.p0s100.x86_64.cpu 0.99\nQwen--Qwen3-8B-f16f16.p0s100.x86_64.cuda 0.99\nQwen--Qwen3-8B-f16f16.p50s50.arm64.cpu 0.94\nQwen--Qwen3-8B-f16f16.p50s50.arm64.metal 0.95\nQwen--Qwen3-8B-f16f16.p50s50.x86_64.cpu 0.99\nQwen--Qwen3-8B-f16f16.p50s50.x86_64.cuda 0.99\nQwen--Qwen3-8B-f16f16.p99s1.arm64.cpu 0.96\nQwen--Qwen3-8B-f16f16.p99s1.arm64.metal 0.99\nQwen--Qwen3-8B-f16f16.p99s1.x86_64.cpu 0.99\nQwen--Qwen3-8B-f16f16.p99s1.x86_64.cuda 0.99\nQwen--Qwen3-8B-q40ef16.p0s100.arm64.cpu 0.86\nQwen--Qwen3-8B-q40ef16.p0s100.arm64.metal 0.97\nQwen--Qwen3-8B-q40ef16.p0s100.x86_64.cpu 0.99\nQwen--Qwen3-8B-q40ef16.p0s100.x86_64.cuda 0.96\nQwen--Qwen3-8B-q40ef16.p50s50.arm64.cpu 0.98\nQwen--Qwen3-8B-q40ef16.p50s50.arm64.metal 0.99\nQwen--Qwen3-8B-q40ef16.p50s50.x86_64.cpu 0.99\nQwen--Qwen3-8B-q40ef16.p50s50.x86_64.cuda 0.99\nQwen--Qwen3-8B-q40ef16.p99s1.arm64.cpu 0.96\nQwen--Qwen3-8B-q40ef16.p99s1.arm64.metal 0.98\nQwen--Qwen3-8B-q40ef16.p99s1.x86_64.cpu 0.99\nQwen--Qwen3-8B-q40ef16.p99s1.x86_64.cuda 0.96\napple--OpenELM-270M-f16f16.p0s100.arm64.cpu 0.98\napple--OpenELM-270M-f16f16.p0s100.arm64.metal 0.99\napple--OpenELM-270M-f16f16.p0s100.x86_64.cpu 0.99\napple--OpenELM-270M-f16f16.p0s100.x86_64.cuda 0.98\napple--OpenELM-270M-f16f16.p50s50.arm64.cpu 0.92\napple--OpenELM-270M-f16f16.p50s50.arm64.metal 0.92\napple--OpenELM-270M-f16f16.p50s50.x86_64.cpu 0.99\napple--OpenELM-270M-f16f16.p50s50.x86_64.cuda 0.99\napple--OpenELM-270M-f16f16.p99s1.arm64.cpu 0.97\napple--OpenELM-270M-f16f16.p99s1.arm64.metal 0.99\napple--OpenELM-270M-f16f16.p99s1.x86_64.cpu 0.99\napple--OpenELM-270M-f16f16.p99s1.x86_64.cuda 0.99\napple--OpenELM-270M-q40ef16.p0s100.arm64.cpu 0.99\napple--OpenELM-270M-q40ef16.p0s100.arm64.metal 0.99\napple--OpenELM-270M-q40ef16.p0s100.x86_64.cpu 0.99\napple--OpenELM-270M-q40ef16.p0s100.x86_64.cuda 0.95\napple--OpenELM-270M-q40ef16.p50s50.arm64.cpu 0.97\napple--OpenELM-270M-q40ef16.p50s50.arm64.metal 0.95\napple--OpenELM-270M-q40ef16.p50s50.x86_64.cpu 0.99\napple--OpenELM-270M-q40ef16.p50s50.x86_64.cuda 0.94\napple--OpenELM-270M-q40ef16.p99s1.arm64.cpu 0.99\napple--OpenELM-270M-q40ef16.p99s1.arm64.metal 0.99\napple--OpenELM-270M-q40ef16.p99s1.x86_64.cpu 0.99\napple--OpenELM-270M-q40ef16.p99s1.x86_64.cuda 0.89\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p0s100.arm64.cpu 0.96\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p0s100.arm64.metal 0.92\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p0s100.x86_64.cpu 0.99\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p0s100.x86_64.cuda 0.99\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p50s50.arm64.cpu 0.95\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p50s50.arm64.metal 0.95\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p50s50.x86_64.cpu 0.98\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p50s50.x86_64.cuda 0.98\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p99s1.arm64.cpu 0.97\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p99s1.arm64.metal 0.99\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p99s1.x86_64.cpu 0.99\nmeta-llama--Llama-3.1-8B-Instruct-f16f16.p99s1.x86_64.cuda 0.99\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p0s100.arm64.cpu 0.93\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p0s100.arm64.metal 0.99\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p0s100.x86_64.cpu 0.97\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p0s100.x86_64.cuda 0.97\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p50s50.arm64.cpu 0.93\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p50s50.arm64.metal 0.98\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p50s50.x86_64.cpu 0.99\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p50s50.x86_64.cuda 0.99\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p99s1.arm64.cpu 0.97\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p99s1.arm64.metal 0.99\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p99s1.x86_64.cpu 0.99\nmeta-llama--Llama-3.1-8B-Instruct-q40ef16.p99s1.x86_64.cuda 0.97\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p0s100.arm64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p0s100.arm64.metal 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p0s100.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p0s100.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p50s50.arm64.cpu 0.96\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p50s50.arm64.metal 0.96\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p50s50.x86_64.cpu 0.98\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p50s50.x86_64.cuda 0.97\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p99s1.arm64.cpu 0.97\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p99s1.arm64.metal 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p99s1.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f16f16.p99s1.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p0s100.arm64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p0s100.arm64.metal 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p0s100.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p0s100.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p50s50.arm64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p50s50.arm64.metal 0.96\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p50s50.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p50s50.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p99s1.arm64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p99s1.arm64.metal 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p99s1.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-f32f32.p99s1.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p0s100.arm64.cpu 0.97\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p0s100.arm64.metal 0.99\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p0s100.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p0s100.x86_64.cuda 0.98\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p50s50.arm64.cpu 0.86\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p50s50.arm64.metal 0.99\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p50s50.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p50s50.x86_64.cuda 0.94\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p99s1.arm64.cpu 0.98\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p99s1.arm64.metal 0.99\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p99s1.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-1B-Instruct-q40ef16.p99s1.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p0s100.arm64.cpu 0.98\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p0s100.arm64.metal 0.97\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p0s100.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p0s100.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p50s50.arm64.cpu 0.96\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p50s50.arm64.metal 0.98\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p50s50.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p50s50.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p99s1.arm64.cpu 0.96\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p99s1.arm64.metal 0.98\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p99s1.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-3B-Instruct-f16f16.p99s1.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p0s100.arm64.cpu 0.96\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p0s100.arm64.metal 0.99\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p0s100.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p0s100.x86_64.cuda 0.99\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p50s50.arm64.cpu 0.97\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p50s50.arm64.metal 0.98\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p50s50.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p50s50.x86_64.cuda 0.97\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p99s1.arm64.cpu 0.93\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p99s1.arm64.metal 0.99\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p99s1.x86_64.cpu 0.99\nmeta-llama--Llama-3.2-3B-Instruct-q40ef16.p99s1.x86_64.cuda 0.94\n" }, { "path": ".travis/make_bundle.sh", "content": "#!/bin/sh\n\nset -ex\n\nTRAVIS_COMMIT=${GITHUB_SHA:-dummy-commit-id}\nBRANCH=$(echo $GITHUB_HEAD_REF | tr '/' '_')\nBRANCH=${BRANCH:-main}\nPLATFORM=${PLATFORM:-dummy-platform}\n\ndates=`date -u +\"%Y%m%dT%H%M%S %s\"`\ndate_iso=`echo $dates | cut -f 1 -d ' '`\ntimestamp=`echo $dates | cut -f 2 -d ' '`\nTASK_NAME=tract-$date_iso\nmkdir -p $TASK_NAME\necho \"export TASK_NAME=$TASK_NAME\" > $TASK_NAME/vars\necho \"export TRAVIS_COMMIT=$TRAVIS_COMMIT\" >> $TASK_NAME/vars\nTRAVIS_BRANCH_SANE=`echo $BRANCH | tr '/' '_'`\necho \"export TRAVIS_BRANCH_SANE=$TRAVIS_BRANCH_SANE\" >> $TASK_NAME/vars\necho \"export DATE_ISO=$date_iso\" >> $TASK_NAME/vars\necho \"export TIMESTAMP=$timestamp\" >> $TASK_NAME/vars\necho \"export PLATFORM=$PLATFORM\" >> $TASK_NAME/vars\n\nif which gstat > /dev/null\nthen\n STAT=gstat\nelse\n STAT=stat\nfi\n\ntouch sizes\nfor bin in example-tensorflow-mobilenet-v2 tract\ndo\n if [ -e target/$RUSTC_TRIPLE/release/$bin ]\n then\n\n binary_size_cli=$($STAT -c \"%s\" target/$RUSTC_TRIPLE/release/$bin)\n token=$(echo $bin | tr '-' '_')\n if [ \"$bin\" = \"tract\" ]\n then\n token=cli\n fi\n echo binary_size.$token $binary_size_cli >> sizes\n fi\ndone\n\ncp target/$RUSTC_TRIPLE/release/tract $TASK_NAME\ncp sizes $TASK_NAME\ncp .travis/bundle-entrypoint.sh $TASK_NAME/entrypoint.sh\ntar czf $TASK_NAME.tgz $TASK_NAME/\n\necho $TASK_NAME\n" }, { "path": ".travis/minion.sh", "content": "#!/bin/bash\n\nset -ex\n. $HOME/.minionrc\n\nexec 200>$LOCKFILE || exit 1\nflock -n 200 || { echo \"WARN: flock() failed.\" >&2; exit 0; }\n\nmkdir -p $WORKDIR/taskdone/\nfor task in `aws s3 ls $S3PATH_TASKS/$PLATFORM/ | awk '{ print $4; }'`\ndo\n cd $HOME\n task_name=\"${task%.tgz}\"\n if [ -e $WORKDIR/taskdone/$task_name ]\n then\n continue\n fi\n echo considering task $task\n rm -rf $WORKDIR/current\n mkdir -p $WORKDIR/current\n cd $WORKDIR/current\n aws s3 cp s3://$S3PATH_TASKS/$PLATFORM/$task .\n tar zxf $task\n . $task_name/vars\n cd $task_name\n (\n ./entrypoint.sh 2> stderr.log > stdout.log || true\n )\n gzip stderr.log\n gzip stdout.log\n aws s3 cp stderr.log.gz s3://$S3PATH_RESULTS/$MINION_ID/$task_name/stderr.log.gz\n aws s3 cp stdout.log.gz s3://$S3PATH_RESULTS/$MINION_ID/$task_name/stdout.log.gz\n touch $WORKDIR/taskdone/$task_name\n cat metrics | sed \"s/^/$GRAPHITE_PREFIX.$PLATFORM.$MINION_ID.$TRAVIS_BRANCH_SANE./;s/$/ $TIMESTAMP/\" \\\n | tr '-' '_' > graphite\n if nc --version\n then\n\t# GNU\n\texport GRAPHITE_HOST\n\texport GRAPHITE_PORT\n\tcat graphite | while read line\n\tdo\n\t\techo $line | nc -c -w 1 $GRAPHITE_HOST $GRAPHITE_PORT\n\tdone\n else\n # BSD\n\tnc -q 5 $GRAPHITE_HOST $GRAPHITE_PORT < graphite\n fi\n\ndone\n\nsleep 1\n\necho \"DONE\"\n" }, { "path": ".travis/minionrc", "content": "MINION_ID=\nLOCKFILE=/tmp/minion-lock\nPLATFORM=raspbian\nGRAPHITE_HOST=graphite-proxy.snips.net\nGRAPHITE_PORT=2003\nGRAPHITE_PREFIX=tract\n\nS3PATH_TASKS=tract-ci-builds/tasks\nS3PATH_LOGS=tract-ci-builds/logs\nS3PATH_RESULTS=tract-ci-builds/logs\nWORKDIR=$HOME/tract-minion\nCACHEDIR=$WORKDIR/cache\n" }, { "path": ".travis/native.sh", "content": "#!/bin/sh\n\nset -ex\n\nif [ -z \"$RUSTUP_TOOLCHAIN\" ]\nthen\n export RUSTUP_TOOLCHAIN=1.91.0\nfi\n\nrustup update\n\ncargo update\ncargo check --all-targets --workspace --exclude test-tflite --exclude test-metal --exclude tract-metal\n\n./.travis/onnx-tests.sh\n./.travis/regular-tests.sh\n./.travis/test-harness.sh\n\nif [ -n \"$CI\" ]\nthen\n cargo clean\nfi\n\nif [ `uname` = \"Linux\" ]\nthen\n ./.travis/tflite.sh\nfi\n\nif [ -n \"$CI\" ]\nthen\n cargo clean\nfi\nif nvidia-smi > /dev/null 2>&1\nthen\n cargo test -p tract-cuda --lib\n cargo test -p test-cuda\nfi\n\n./.travis/cli-tests.sh\n" }, { "path": ".travis/onnx-tests.sh", "content": "#!/bin/sh\n\nset -ex\n\nROOT=$(dirname $(realpath $0))/..\n. $ROOT/.travis/ci-system-setup.sh\n\nopset=onnx_\"${1:-1_13_0}\"\n\ncargo -q test -p test-unit-core $CARGO_EXTRA -q \ncargo -q test -p test-onnx-core $CARGO_EXTRA -q --no-default-features --features $opset\ncargo -q test -p test-nnef-cycle $CARGO_EXTRA -q --no-default-features\n" }, { "path": ".travis/regular-tests.sh", "content": "#!/bin/sh\n\nset -e\nset -x\n\ncd $(dirname $0)\n\n./test-published-crates.sh\nif [ -n \"$CI\" ]\nthen\n cargo clean\nfi\n./test-rt.sh\nif [ -n \"$CI\" ]\nthen\n cargo clean\nfi\n" }, { "path": ".travis/run-bundle.sh", "content": "#!/bin/sh\n\nset -ex\n\nBUNDLE_NAME=$1\n\ntar zxf $BUNDLE_NAME.tgz\n(\n cd $BUNDLE_NAME\n . ./vars\n ./entrypoint.sh\n)\n# rm -rf \"$BUNDLE_NAME\" \"$BUNDLE_NAME.tgz\"\n" }, { "path": ".travis/run_all.sh", "content": "#!/bin/sh\n\nset -ex\n\n`dirname $0`/native.sh\ncd `dirname $0`/../examples\nfor i in *\ndo\n (cd $i; cargo test --release)\ndone\n" }, { "path": ".travis/setup-sccache.sh", "content": "#!/bin/sh\n\nset -ex\n\nexport SCCACHE_DIR=$HOME/.cache/sccache\nexport SCCACHE_CACHE_SIZE=2G\n\nif [ -n \"$GITHUB_ENV\" ]\nthen\n echo \"SCCACHE_DIR=$HOME/.cache/sccache\" >> $GITHUB_ENV\n echo \"SCCACHE_CACHE_SIZE=2G\" >> $GITHUB_ENV\n echo \"RUSTC_WRAPPER=sccache\" >> $GITHUB_ENV\n echo \"$HOME/.local/bin\" >> $GITHUB_PATH\nfi\n\nLINK=https://github.com/mozilla/sccache/releases/download\nSCCACHE_VERSION=v0.10.0\n\necho $HOME\nif [ `uname` = \"Linux\" ]\nthen\n SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl\nelse\n SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-apple-darwin\nfi\n\nmkdir -p $SCCACHE_DIR\nmkdir -p $HOME/.local/bin\nfor i in 1 2 3 4 5\ndo\n curl -L \"$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz\" | tar xz && break\n sleep 15\ndone\nmv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache\nchmod +x $HOME/.local/bin/sccache\n\n\n\n" }, { "path": ".travis/test-harness.sh", "content": "#!/bin/sh\n\nWHITE='\\033[1;37m'\nNC='\\033[0m' # No Color\n\nif [ -e /proc/cpuinfo ]\nthen\n grep \"^flags\" /proc/cpuinfo | head -1 | \\\n grep --color=always '\\(s\\?sse[0-9_]*\\|fma\\|f16c\\|avx[^ ]*\\)'\nfi\n\nset -x\n\nROOT=$(dirname $0)/..\n. $ROOT/.travis/ci-system-setup.sh\n\nset -e\n\nif [ `arch` = \"x86_64\" -a \"$RUST_VERSION\" = \"stable\" ]\nthen\n ALL_FEATURES=--all-features\nfi\n\nset +x\n\ncargo -q test $CARGO_EXTRA -q -p tract\ncargo -q test $CARGO_EXTRA -q --profile opt-no-lto -p core-proptest-pulse $ALL_FEATURES\ncargo -q test $CARGO_EXTRA -q --profile opt-no-lto -p nnef-inceptionv3 $ALL_FEATURES\ncargo -q test $CARGO_EXTRA -q --profile opt-no-lto -p tf-inceptionv3 $ALL_FEATURES\ncargo -q test $CARGO_EXTRA -q --profile opt-no-lto -p tf-mobilenet-v2 $ALL_FEATURES\ncargo -q test $CARGO_EXTRA -q --profile opt-no-lto -p tfl-mobilenet-v2-q $ALL_FEATURES\n\n" }, { "path": ".travis/test-llm.sh", "content": "#!/bin/bash\n\nset -e\nset -o pipefail\n\nexport LC_ALL=C\n\nROOT=$(dirname $(dirname $(realpath $0)))\n. $ROOT/.travis/ci-system-setup.sh\n\nmodel=$1\nq=$2\ndevice=$3\nif [ -z \"$device\" ]\nthen\n device=cpu\nfi\ngeneration=541\n\nif [ \"$model\" = \"all\" ]\nthen\n for m in \\\n openelm-270M \\\n\tllama-3.2-1B-instruct \\\n\tllama-3.2-3B-instruct \\\n\tllama-3.1-8B-instruct \\\n\tqwen3-1.7B \\\n\tqwen3-8B\n do\n $0 $m $2 $device\n done\n exit 0\nfi\n\nmodel=$(echo $model | tr 'A-Z' 'a-z' | tr -d \"_.-\")\n\nfor m in \\\n apple--OpenELM-270M \\\n meta-llama--Llama-3.2-1B-Instruct \\\n meta-llama--Llama-3.2-3B-Instruct \\\n meta-llama--Llama-3.1-8B-Instruct \\\n Qwen--Qwen3-1.7B \\\n Qwen--Qwen3-8B\ndo\n norm=$(echo $m | tr \"A-Z\" \"a-z\" | tr -d \"_.-\")\n if [[ \"$norm\" == *\"$model\"* ]];\n then\n model_id=$m\n fi\ndone\n\nif [ -z \"$model_id\" ]\nthen\n echo \"No model matched\"\nfi\n\nif [ \"$q\" = \"all\" ]\nthen\n for q in q40ef16 f16f16 f32f32\n do\n $0 $1 $q $device\n done\n exit 0\nfi\n\nid=$model_id-$q\n\nif which gstat > /dev/null\nthen\n STAT=gstat\nelse\n STAT=stat\nfi\n\nnnef=llm/$generation/$id/$id.nnef.tgz\n\n$CACHE_FILE $nnef\n\nif [ -e $MODELS/$nnef ]\nthen\n size=$($STAT -c %s $MODELS/$nnef)\nelse\n size=$(curl -s -I $MODELS/$nnef | grep Content-Length | cut -d \" \" -f 2 | tr -cd 0123456789)\nfi\n\nif which nvidia-smi > /dev/null\nthen\n vram=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | awk '{print $1*1024*1024}')\n if [ $vram -lt $size ]\n then\n echo \"::warning::Skipping this test, not enough VRAM.\"\n exit 0\n fi\nfi\n\n$TRACT_RUN -v --nnef-tract-transformers $MODELS/$nnef -O --readings --assert-maximal-mm-quality-cost 0 $TRACT_EXTRA_ARGS dump -q\nalloc_max=$(cat readings.out | tail -n +2 | awk '{print $10-$11}' | sort -n | tail -1)\nratio=$((alloc_max * 100 / size))\n\necho \" ###########################################\"\necho \" Alloc max to model size ratio: ${ratio}%.\"\necho \" ###########################################\"\n\nlimit=125\n\nif [ $ratio -gt $limit ]\nthen\n echo \"RSZ max is ${ratio}% the size of the unzipped model!\"\n exit 1\nfi\n\nfor t in p0s100 p50s50 p99s1 \ndo\n npz=llm/$generation/$id/$id.$t.io.npz\n $CACHE_FILE $npz\n\n key=$id.$t.$(arch).$device\n expectations=\"$ROOT/.travis/llm-expectations-541\"\n\n echo\n echo \" Key: $key\"\n echo\n\n case $device in\n cuda)\n DEVICE=\"--cuda\"\n GPU_ASSERT=\"--assert-op-only Cuda*,Gpu*,DeviceSync*,Const,Source,Range,Gather\"\n ;;\n metal)\n DEVICE=\"--metal\"\n GPU_ASSERT=\"--assert-op-only Metal*,Gpu*,DeviceSync*,Const,Source,Range,Gather\"\n ;;\n *)\n GPU_ASSERT=\"\"\n ;;\n esac\n\n if [ -n \"$RESET\" ]\n then\n $TRACT_RUN -v $MODELS/$nnef $TRACT_EXTRA_ARGS \\\n --llm --transform unfold-kv-cache -O $DEVICE run --prompt-chunk-size 60 --allow-missing-outputs \\\n --input-from-npz $MODELS/$npz \\\n --assert-output-bundle $MODELS/$npz \\\n --assert-llm-rbo 0.0 \\\n $approx --allow-float-casts $GPU_ASSERT 2>&1 | tee output.txt\n found=$(cat output.txt | perl -MPOSIX=floor -ne 'printf(\"%.2f\\n\", floor($1 * 100) / 100) if /LLM RBO:\\s+([\\d.]+)/')\n ( ( grep -v $key $expectations || true) ; echo $key $found) | sort > $expectations.tmp\n mv $expectations.tmp $expectations\n elif [ -n \"$RELAX\" ]\n then\n prior=$(grep $key $expectations | cut -f 2 -d ' ')\n $TRACT_RUN -v $MODELS/$nnef $TRACT_EXTRA_ARGS \\\n --llm --transform unfold-kv-cache -O $DEVICE run --prompt-chunk-size 60 --allow-missing-outputs \\\n --input-from-npz $MODELS/$npz \\\n --assert-output-bundle $MODELS/$npz \\\n --assert-llm-rbo 0.0 \\\n $approx --allow-float-casts $GPU_ASSERT 2>&1 | tee output.txt\n found=$(cat output.txt | perl -MPOSIX=floor -ne 'printf(\"%.2f\\n\", floor($1 * 100) / 100) if /LLM RBO:\\s+([\\d.]+)/')\n if [ -n \"$prior\" ] && perl -e 'exit($ARGV[0] <= $ARGV[1] ? 1 : 0)' \"$found\" \"$prior\"\n then\n found=$prior\n fi\n ( ( grep -v $key $expectations || true) ; echo $key $found) | sort > $expectations.tmp\n mv $expectations.tmp $expectations\n else # test !\n expectation=$(grep $key $expectations | cut -f 2 -d ' ')\n $TRACT_RUN -v $MODELS/$nnef $TRACT_EXTRA_ARGS \\\n --llm --transform unfold-kv-cache -O $DEVICE run --prompt-chunk-size 60 --allow-missing-outputs \\\n --input-from-npz $MODELS/$npz \\\n --assert-output-bundle $MODELS/$npz \\\n --assert-llm-rbo $expectation \\\n $approx --allow-float-casts $GPU_ASSERT\n fi\n\ndone\n" }, { "path": ".travis/test-published-crates.sh", "content": "#!/bin/sh\n\nWHITE='\\033[1;37m'\nNC='\\033[0m' # No Color\n\nif [ -e /proc/cpuinfo ]\nthen\n grep \"^flags\" /proc/cpuinfo | head -1 | \\\n grep --color=always '\\(s\\?sse[0-9_]*\\|fma\\|f16c\\|avx[^ ]*\\)'\nfi\n\nset -x\n\nROOT=$(dirname $0)/..\n. $ROOT/.travis/ci-system-setup.sh\n\nset -e\n\nif [ `arch` = \"x86_64\" -a \"$RUST_VERSION\" = \"stable\" ]\nthen\n ALL_FEATURES=--all-features\nfi\n\nset +x\ncargo update\n\necho\necho \"$WHITE ### tract ### $NC\"\necho\n\ncargo -q test $CARGO_EXTRA -q -p tract\n\nfor c in data linalg core nnef hir onnx pulse onnx-opl pulse-opl\ndo\n echo\n echo \"$WHITE ### $c ### $NC\"\n echo\n cargo -q test $CARGO_EXTRA -q -p tract-$c\ndone\n\nif [ `uname` = \"Darwin\" -a -z \"$CI\" ]\nthen\n echo\n echo \"$WHITE ### metal ### $NC\"\n echo\n cargo -q test $CARGO_EXTRA -q -p tract-metal\nfi\n\nif command -v nvcc >/dev/null 2>&1 && [ -z \"$CI\" ]\nthen\n echo\n echo \"$WHITE ### cuda ### $NC\"\n echo\n cargo -q test -q -p tract-cuda\nfi\n\n$ROOT/api/proxy/ci.sh\n\n# doc test are not finding libtensorflow.so\nif ! cargo -q test $CARGO_EXTRA -q -p tract-tensorflow --lib $ALL_FEATURES\nthen\n # this crate triggers an incremental bug on nightly.\n cargo clean -p tract-tensorflow\n cargo -q test $CARGO_EXTRA -q -p tract-tensorflow --lib $ALL_FEATURES\nfi\n" }, { "path": ".travis/test-rt.sh", "content": "#!/bin/sh\n\nWHITE='\\033[1;37m'\nNC='\\033[0m' # No Color\n\nif [ -e /proc/cpuinfo ]\nthen\n grep \"^flags\" /proc/cpuinfo | head -1 | \\\n grep --color=always '\\(s\\?sse[0-9_]*\\|fma\\|f16c\\|avx[^ ]*\\)'\nfi\n\nset -x\n\nROOT=$(dirname $0)/..\n. $ROOT/.travis/ci-system-setup.sh\n\nset -e\n\nif [ `arch` = \"x86_64\" -a \"$RUST_VERSION\" = \"stable\" ]\nthen\n ALL_FEATURES=--all-features\nfi\n\nset +x\n\ncd $ROOT\nfor c in test-rt/test*; do\n case \"$c\" in\n test-rt/test-tflite)\n echo \"$WHITE ### $c ### IGNORED $NC\"\n continue\n ;;\n test-rt/test-metal)\n if [ \"$(uname)\" != \"Darwin\" ] || [ -n \"$CI\" ]; then\n echo \"$WHITE ### $c ### IGNORED $NC\"\n continue\n fi\n ;;\n test-rt/test-cuda)\n if ! command -v nvcc >/dev/null; then\n echo \"$WHITE ### $c ### IGNORED $NC\"\n continue\n fi\n ;;\n esac\n\n echo\n echo \"$WHITE ### $c ### $NC\"\n echo\n (cd \"$c\" && cargo test -q $CARGO_EXTRA)\n\n if [ -n \"$CI\" ]; then\n df -h\n cargo clean\n fi\ndone\n\n" }, { "path": ".travis/tf.sh", "content": "#!/bin/sh\n\nset -ex\n\nif [ -z \"$CACHEDIR\" ]\nthen\n CACHEDIR=`dirname $0`/../.cached\nfi\n\n\n(cd tensorflow; cargo test --release --features conform)\n" }, { "path": ".travis/tflite/Dockerfile.tensorflow-aarch64", "content": "# vim: set syntax=Dockerfile:\n\nFROM tensorflow/tensorflow:devel\n\nRUN apt-get update ; apt-get upgrade -y\nRUN apt-get install -y crossbuild-essential-arm64\nCOPY linux_makefile.inc /tensorflow_src/tensorflow/lite/tools/make/targets/linux_makefile.inc\nCOPY disable_nnapi.patch /tensorflow_src\n\nWORKDIR /tensorflow_src\nRUN patch -p1 < disable_nnapi.patch\n" }, { "path": ".travis/tflite/Dockerfile.tensorflow-official-rpi", "content": "# vim: set syntax=Dockerfile:\n\nFROM tensorflow/tensorflow:nightly-devel\n\nRUN apt-get update ; apt-get upgrade -y\nRUN apt-get -y install git crossbuild-essential-armhf\n\nWORKDIR /tensorflow\nRUN ./tensorflow/lite/tools/make/download_dependencies.sh\n\n\n" }, { "path": ".travis/tflite/Dockerfile.tensorflow-rpitools", "content": "# vim: set syntax=Dockerfile:\n\nFROM tensorflow/tensorflow:nightly-devel\n\nRUN apt-get update ; apt-get upgrade -y\nRUN apt-get -yy install git\n\nWORKDIR /tensorflow\nRUN ./tensorflow/lite/tools/make/download_dependencies.sh\n\nRUN git clone https://github.com/raspberrypi/tools /raspitools\nENV PATH=/raspitools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin:$PATH\n\n" }, { "path": ".travis/tflite/build_tflite_aarch64.sh", "content": "#!/bin/sh\n\nset -ex\nmkdir -p result\n\ndocker build -f Dockerfile.tensorflow-aarch64 --tag tensorflow-aarch64 .\ndocker run --rm -it \\\n -v `pwd`/result:/result \\\n tensorflow-aarch64 \\\n sh -c \"\n cd /tensorflow_src ;\n export EXTRA_CXXFLAGS=-flax-vector-conversions \n export DISABLE_NNAPI=true\n ./tensorflow/lite/tools/make/download_dependencies.sh\n make -j 3 -f tensorflow/lite/tools/make/Makefile TARGET=linux TARGET_ARCH=aarch64 ;\n cp /tensorflow_src/tensorflow/lite/tools/make/gen/linux_aarch64/bin/benchmark_model /result/tflite_benchmark_model_aarch64\n \"\n" }, { "path": ".travis/tflite/build_tflite_raspbian.sh", "content": "#!/bin/sh\n\nset -ex\nmkdir -p result\n\n# build pseudo-rpi official tensorflow, https://www.tensorflow.org/lite/rpi, only works on pi3\n\ndocker build -f Dockerfile.tensorflow-official-rpi --tag tensorflow-official-rpi .\ndocker run --rm \\\n -e CC_PREFIX=arm-linux-gnueabihf- \\\n -v `pwd`/result:/result \\\n tensorflow-official-rpi \\\n sh -c \"\n make -j 3 -f tensorflow/lite/tools/make/Makefile TARGET=rpi TARGET_ARCH=armv7l;\n cp /tensorflow/tensorflow/lite/tools/make/gen/rpi_armv7l/bin/benchmark_model /result/tflite_benchmark_model_official_rpi\n \"\n\n# build with rpi tools (works on rpi0, 1 and 2)\n\ndocker build -f Dockerfile.tensorflow-rpitools --tag tensorflow-rpitools .\ndocker run --rm \\\n -e CC_PREFIX=arm-linux-gnueabihf- \\\n -v `pwd`/result:/result \\\n tensorflow-rpitools \\\n sh -c \"\n make -j 3 -f tensorflow/lite/tools/make/Makefile TARGET=rpi TARGET_ARCH=armv6;\n cp /tensorflow/tensorflow/lite/tools/make/gen/rpi_armv6/bin/benchmark_model /result/tflite_benchmark_model_rpitools\n \"\n" }, { "path": ".travis/tflite/convert_all.sh", "content": "\nrun_in_tf_docker() {\n docker run --rm -v $HOME/.cache:/models -it tensorflow/tensorflow:nightly-devel sh -c \"$@\"\n}\n\n# # inception v3 \n# run_in_tf_docker \"cd /models ; tflite_convert \\\n# --graph_def_file inception_v3_2016_08_28_frozen.pb \\\n# --input_arrays input \\\n# --input_shapes 1,299,299,3 \\\n# --output_arrays InceptionV3/Predictions/Reshape_1 \\\n# --output_format tflite \\\n# --output_file inception_v3_2016_08_28_frozen.tflite\"\n# \n# # arm ml kws\n# run_in_tf_docker \"cd /models ; tflite_convert \\\n# --graph_def_file ARM-ML-KWS-CNN-M.pb \\\n# --input_arrays Mfcc \\\n# --input_shapes 1,49,10 \\\n# --output_arrays labels_softmax \\\n# --output_format tflite \\\n# --output_file ARM-ML-KWS-CNN-M.tflite\"\n\n# hey_snips v1\nrun_in_tf_docker \"cd /models ; tflite_convert \\\n --graph_def_file hey_snips_v1.pb \\\n --input_arrays inputs \\\n --input_shapes 80,40 \\\n --output_arrays logits \\\n --output_format tflite \\\n --output_file hey_snips_v1.tflite\"\n\n# hey_snips v3.1\n# (tflite does not support 1D dil)\n# run_in_tf_docker \"cd /models ; tflite_convert \\\n# --graph_def_file hey_snips_v3.1.pb \\\n# --input_arrays inputs \\\n# --input_shapes 40,40 \\\n# --output_arrays logits \\\n# --output_format tflite \\\n# --output_file hey_snips_v3.1.tflite\"\n# \n# # hey_snips v4 model17, 2seconds\n# (tflite does not support AddN)\n# run_in_tf_docker \"cd /models ; tflite_convert \\\n# --graph_def_file hey_snips_v4_model17.pb \\\n# --input_arrays input_node \\\n# --input_shapes 200,20 \\\n# --output_arrays output_node \\\n# --output_format tflite \\\n# --output_file hey_snips_v4_model17.tflite\"\n" }, { "path": ".travis/tflite/linux_makefile.inc", "content": "# Settings for Linux.\nifeq ($(TARGET), linux)\n CXXFLAGS += \\\n -fPIC \\\n -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \\\n -pthread\n # TODO(petewarden): In the future we may want to add architecture-specific\n # flags like -msse4.2\n\tLIBS := -lstdc++ -lpthread -lm -ldl\n TARGET_ARCH=aarch64\n TARGET_TOOLCHAIN_PREFIX := aarch64-linux-gnu-\nendif\n" }, { "path": ".travis/tflite/run_all.sh", "content": "#!/bin/sh\n\n./benchmark_model --graph=inception_v3_2016_08_28_frozen.tflite\n\n" }, { "path": ".travis/tflite.sh", "content": "#!/bin/sh\n\nset -ex\n\nROOT=$(dirname $(dirname $(realpath $0)))\n. $ROOT/.travis/ci-system-setup.sh\n\nif [ `uname` = \"Darwin\" ]\nthen\n brew install coreutils\nfi\nif [ -n \"$GITHUB_ACTIONS\" ]\nthen\n pip install numpy\nfi\n\ncargo check -p tract-tflite\ncargo -q test -p test-tflite $CARGO_EXTRA -q \n" }, { "path": ".travis/travis.sh", "content": "#!/bin/sh\n\nset -ex\n\nif [ -z \"$PLATFORM\" ]\nthen\n .travis/native.sh\nelse\n .travis/cross.sh\nfi\n" }, { "path": ".travis.yml", "content": "language: rust\ndist: trusty\ncache:\n # - cargo\n #- directories:\n #- $HOME/cached\naddons:\n apt:\n packages:\n - awscli\n - jshon\n - moreutils\nenv:\n global:\n - RUST_BACKTRACE=1\n - AWS_DEFAULT_REGION=us-east-1\n # - CACHEDIR=$HOME/cached\n - secure: DdQqS4fFDevY/JknS0M6CwW+ooj6ZhKmBk4lFliG3WQgjYietbPMCkiHPEC8Df8U07l54+8G4j+sZJJ4VYQY8WQcuKGWt9/ALjoYHYZ1Dlw0KW0rRJ1BZWLUh8MwpJ5pxHSWfl1a8QqTy/0mI3eJ8iVIDxiZR6b1fGwPYDkNXyqnfOvz31X1aMyoGslNkP7LitObCdBJyzobPlvWafGCQLf7oLbK4a5Ysyc9T607n1B0okco3Te2ztahEOwNoxmHlEFRojM6ZmAmo8LzwkCdFQNjHw+mQ3vScC8gpngi61G9U35luAfweMt30m1UmecVGADeEmwSnJLeAHo5HYKT5n6Q1begxlMGMxezinstTHUX6G8EhEumO/ii3PAscFJ6C+VfciGA7JDS2ICghygKSIqQvVeugNR0glW97lhszLnoXCNY45siknAZVTVqwhgn1ctTQiPWqGVuQp+m9NYIoQAYUpOFNo1mEtEjurHOk96Q0XjRJMfUSLOB5KfPakk/ghEY1ZYDDB9wi134f3Z5hLw1FGj/Uiw9LcnEIcORV2o8fbFrb2IgGsKQbRtdPEJ92q5bdeA00TbSrzoDDRFGbwBg+8ibFMF8O2J3Q54sUu6LmkP7qjtMIT2vB09M7LWQYtW3Vd6ovHwI6v+tNJK2D4cJA0KOSwzpOgIBhEubrZI=\n - secure: su5wLbN7lr59HSKGM8M9pW9VSFMtdTBbQ1oBZFTvw+RNoNew3BBMh8XymbjEv6R+yyGmDAa8Dw6E7HPiBtI5O4q89rHxl7MO5keZINjpXDzTydZ5MQ2juZPIRQpLfpl6AN1meG8I3SoOrTMDgGqfPh2rqEjIS6cBZbo8Re/0KSBuJB781qdT6x80Qfpy87sDu9GTRM8ueXsV+jRw0Yek709m29MSB8pkFAP9OitKHxzF5OFnxVPV2becj53racOe7q9ZE5QWmWnzPZUxflVyrtzDsN0J/C4g0SoEbxLFg1OLbffO1GVJ/Iv6ZeaggzYRvCYwSSANyfqorqSUDT5NPwQlUXjgBPHlOXbCfY2s5hFzQQ3z4R86fyzfdfBQ56uXTXkB4CWpn/JieUoviuDO0/YNaI0KU6hOrOn68BZrBSqwHcxwhtcP3cw/EXfR1aiok7OoDPAWnR4f3Lu/+fmkW+VUEg5Ufh/GgjZt7XwsNBfo+pmvO3mO/5okxa2/HbOwcoTpELAzPMKyh1xn5gjrk5bWcZofeGtFMoXN4+G8+1qlQ/sLp3144QHyRf00n4qlhA6xZwplpBWN12haXyKRx67lPTzE5QuT1BRoyCdRvbjQiOdo61xGvoOK9J8PL9C06xtnKQ+6iDnjFaWielASoENvcNL+DCKqiecpUb5hoR4=\n - secure: HZHubUhHLYh+v5yuyMy5TmfleHqAdcmVZd6hZf7c2sXQLsrcjoLGPxu4jzS9OJt27Sfp6xXVFeeA7SFDMobxe8AFm3+SRCbzvT6mu8/LlsuTsQO3jYQtt8j5OIhtLQ7yfDyOscXwy2I3SgluhVQ0HCIi6ShQ8YRD1vE119Qku2x/sWlKcZQckcl5T/yyig08sXfOM+IfFQPIW1gvMWM3dv6RigiCiy4qjfeQy8v8xbvbRayXeT4Vpfv9CqN79kAQ03r7MSmBBn6i88mGoQEzVDMEZPZq8rMNqn8qyIn8LxjXaCpUz0fTCYJrHSmzKyCE7+k7IEidlkyT6hJpvpXCfNYBSk2fB7SDxlm98ELVgqPBq6vjYoPaqsqs7Cz+pTTQYOCnCKvRDYhccqhAsgjNVKDRIJc0H7cT5sv4TuJxsMp/vYVh7RxoFem2r37ns4pu8XPP9RVsmoAVYzlHu8Fnd8TWY30MXACf3t43ceaPfor0IARrZcjQR1lt7eMJGQDkduJRxzq0cBB1djP8HfZGab/I0cVFEXGeJWDQfaHZ8Pq/M3+bBPLD9XLqKmpoNbW+gqQQl49/w01/EJrp9QhK/Og6ujfkeA1OCziPCUDLAHYvmwaYZYzV9z5VSPfUYwiiJzva92/ywWfhCmz3SpXPbq8cTPoDWzeBUeqcw8iIWVU=\n - secure: vH8bS7RVgaHLGZUeqtViCQYDJfhubMiCMETLPD959pv9sODmSfjOhYtFgZtbn0wZ2fhCQFgKhYKUJdti5Vo9OUlyBiUsfLPilAAaeZu0Y2SQIKpbuNU9kJibuzyj6KZoRhjvsifhO5/mB03W7CpzjSGvJntK62BM0b6CrDtUlHlOgjwd3U1c5brZS9VWfnkh8pohgneB/XYtefTJXHuGjJgf75uw2TO/ZKQmmaKJWPoMVN76cgarRmXS/SoGMLr0+7ArnvIMNW9QRMABrSzUgP0RBvNfndwjiIQDZpIefIz/UVTa5e/pS79CLoQKM9FSWZANf3ZJgz0SzYgMprSe9f3RZGu5i0BLQA0SzdxCjCra5/3/pz+p86/iWGHnBfV6pvH9c2W1OUCCTiohNk7bgUfXxVrxk2RHxhc375MFiCxu6KtPRW8kJoRTSZP+k2itaBPUSevV0cdWrVlRjnTwoCskxMIFQH+vStxQjUXV0/g9SZzwdIR/j1aKIjb6VdQS2WOh0+BKgHy0jy2w4GJHtuObIg0aTcQAtt44aK0T/VeHJ0f1FxfjzPxrcqcgSxvi2E4HgedSCvtOHPWs5YYYGt76yH0G5ZOMOF8xP2CRStlcNB0TtLdpcUvQT2ejK7t4sCOj8Kz81s2cbLCZnJdkFaaBsffV7JtbjexXRwohGxI=\n\nmatrix:\n include:\n - rust: stable\n - rust: stable\n env: PLATFORM=raspbian\n - rust: 1.35.0\n - rust: stable\n script: .travis/tf.sh\n - rust: stable\n script: .travis/debug-tests.sh\n - rust: stable\n os: osx\n osx_image: xcode9.3\n env: PARTIAL_CI=true\n - rust: stable\n env: PLATFORM=aarch64-unknown-linux-gnu\n - rust: stable\n env:\n - PLATFORM=armv6vfp-unknown-linux-gnueabihf\n - TRACT_CPU_EXPECT_ARM32_NEON=false\n - rust: stable\n env:\n - PLATFORM=armv7-unknown-linux-gnueabihf\n - TRACT_CPU_ARM32_NEON=true\n - TRACT_CPU_EXPECT_ARM32_NEON=true\n - rust: stable\n env: PLATFORM=aarch64-linux-android\n - rust: stable\n env: PLATFORM=armv7-linux-androideabi\n - rust: stable\n env: PLATFORM=i686-linux-android\n - rust: stable\n env: PLATFORM=x86_64-linux-android\n - rust: stable\n os: osx\n osx_image: xcode9.3\n env: PLATFORM=aarch64-apple-ios\n - rust: beta\n - rust: nightly\n - rust: stable\n os: windows\n script: cargo test -p tract-linalg\n allow_failures:\n - rust: nightly\n - os: windows\n\nscript: \".travis/travis.sh\"\n" }, { "path": ".vim/coc-settings.json", "content": "{\n \"rust-analyzer.imports.granularity.group\": \"module\"\n}\n" }, { "path": "CHANGELOG.md", "content": "# Unreleased\n\n* [Breaking][MSRV] MSRV bumped to 1.91.0 (for `const TypeId::of`).\n\n# 0.23.0-dev.3 — 2026-03-20\n\n### Breaking changes\n\n- **`Value` renamed to `Tensor`** across the entire public API surface (Rust, C, Python). The deprecated `Value` alias has been removed.\n- **Crate renamed: `tract-rs` → `tract`** — update your `Cargo.toml` dependency accordingly. The CLI binary is now `tract-cli` (previously `tract`).\n- **`into_tract()` renamed to `into_model()`** in all API layers.\n- **`DatumType` variant names shortened** — the `TRACT_DATUM_TYPE_` prefix is dropped (C API).\n- **Deprecated state methods removed**: `init_states()`, `state_initializers`, and the `n_states` parameter are gone from `State` trait and `RunTensors`.\n- **Python**: `concretize_symbols` and `pulse` methods replaced by typed transform classes; `TransformSpec` is now an abstract base class.\n\n### Improvements\n\n- **`UnfoldKeyValueCacheTransform`** — explicit KV-cache I/O mode now available as a first-class transform (CLI: `--transform unfold-kv-cache`).\n- **Structured `NodeFilter`** for `FloatPrecisionTranslator` — replaces raw filter strings.\n- Python docs migrated from mkdocs to **Sphinx** (hosted on GitHub Pages with version switcher).\n- New GPU inference section (CUDA example, `Runtime` usage).\n\n# 0.23.0-dev.2 - 2026-02-18\n* This is a pre-release release. It will be a pretty big one, here are some hilights.\n * New public api: tract-rs should be the main point of entry for any new project. A caveat: it does support most of tract simple uses as is, but some specialized sections like state management and model transforms are not satisfactory yet, so the real 0.23.0 will presumably break these again. The plan is that this facade will be tract public API, and that it will be the only surface under semver rules. Up to there is was essentially everything `pub`, which boils down to mostly \"everything\".\n * GPU: The new API puts forward the Runtime trait. It allows running models on GPU (runtimes \"cuda\" and \"metal\" offser some support, while \"default\" is the CPU runtime). See https://github.com/sonos/tract/blob/main/examples/nemo-parakeet-asr/src/main.rs#L21 for an example. Instead of calling into_runnable(), use Runtime::prepare().\n * Additionally, the internal API compatibility is broken in many places (e.g. RunnableModel always takes the model as an Arc while it was accepting AsRef before). As you're going to need to fix code to upgrade, it is recommended to try and use the new \"tract-rs\" facade (please tell us if the current API coverage is not enough, or awkward to use).\n* [Breaking][MSRV] MSRV bumped to 1.89.0\n* [linalg] Avoid panic in Apple sysctl-based feature probing (AMX detection).\n\n# 0.22.1 - 2026-02-23\n* [backport] Small bug fixes release (Slice decluttering bug)\n\n# 0.22.0 - 2025-08-25\n* [Breaking][MSRV] MSRV bumped to 1.85.0\n* port to edition 2024\n* bump virtuaally each and every dependency\n* (wip, experimental) cuda support for llm\n\n# 0.21.14 - 2026-02-23\n* [backport] Small bug fixes release (Slice decluttering bug)\n\n# 0.21.12 - 2025-04-10\n\n* multithread matmul is feature gated now (\"multithread-mm\" on linal)\n* full hand made arm64 f32-accumulator matmul kit\n* more auditing improvment around einsum and its matmul translations\n* bugfix in matmul translation and gather\n* more test-rt-level coverage of low-level matmuls (metal and cpu)\n* memory arena improvements (metal)\n* q40 for convolution weights\n\n\n# 0.21.11 - 2025-03-19\n\n* [cli] augment audit capabilities for mm implementation choices\n* revisit matmul kernel selection\n* improve gather with compressed inputs\n* revisit slice bubbling up to unlock optimisations\n* fix a bug around flipping substractions\n* support for left q40 input in arm64 f32 accumulating kernels (unlocks q40f32 compression on arm64)\n\n# 0.21.10 - 2025-02-21\n* WIP llm testability (--approx-custom)\n* [metal] ggml-ported kernels\n* WIP einsum-to-matmul testability\n* optimisation around reduce impacting some modern/exotic normalisation layers\n* WIP towards better handling of shared weights (e.g. embeddings duplication)\n\n# 0.21.9 - 2025-01-08\n* [metal] experimental profile\n* [cpu] new versatile (mmm/mmmv) kernels combinations for various architectures\n* [metal] scaled-masked-softmax detector and impl\n\n\n# 0.21.8 - 2024-12-05\n* [linalg, compression] introduce mmm kits\n* [linalg] (wip) rework f16 on non-f16 machines\n* [linalg] element-wise binary operators optimisation\n* [core, compression] gather with compressed weights\n* [metal] new kernels\n* [metal] new memory management\n* [nnef] opt-in deterministic tar encoding\n\n# 0.21.7 - 2024-09-23\n* [metal] (experimental) introduce partial support for Apple Metal\n* [core] Potential internal API breaking changes (operator names, comparison ops refactored)\n* [data] (experimental) Smarter TDim simplification, handling of Min and Max. TDim assertions for simplifications.\n* [data] (experimental) WIP around multiple scenarios (modes) for LLM inference\n* Extra examples\n* [linalg] (experimental) kernels targetting LLM block-quantized tasks (inc. intel 32x1 q40f32)\n\n# 0.21.6 - 2024-07-24\n* [data] Rework tdim and symbols, introduce inequalities assertions, min and max operators\n* [data] Generalize Blob usage in Tensor\n* [linalg] Rework reduce implementation, introduce more generic binary ops support (wip)\n* [linalg] Introduce multithreaded matrix multiplication runner\n* [linalg] Introduce Q4_0 block quantization for weights (wip)\n* [linalg] Introduce AMX f16 kernels, Neon Q40F16 kernel (experimental)\n* [linalg] wasm f32 4x4 kernel \n* [core] Introduce Opaque and OpaqueFact to escape Tensor and TValue formalism\n* [core] generalize/improve float precision translator, with translation filter\n* [core] Introduce garbage collecting in patch application, new compact algo, and rework constant propagation to spare memory\n* [core] Rework packed format and packing metadata\n* [linalg/core] Introduce multiple packing format for matmul kernels\n* [core] Work In Progress refactoring binary, towards more optimized execution strategies\n* [nnef] inequalities assertions extension, q4_0 extension\n* [tflite] plug in tanh and sigmoid\n\n# 0.21.5 - 2024-05-11\n* [TFLite] fixes for fully connected and max pool layers\n* Allow opting out of new memory friendly execution order optimisation\n\n# 0.21.4 - 2024-04-23\n* More memory/cache friendly execution order\n* Several fixes around symbolic dimensions computation (some should help with attention models)\n\n# 0.21.3 - 2024-04-03\n* [AMX] Put AMX for iOS behind a feature gate (\"tract-linalg/apple-amx-ios\").\n\n# 0.21.2 - 2024-03-29 (yanked)\n* [ONNX] Support for external storage of tensors with offset and length\n* [ONNX] Lots of fixes around binary quantized operators (add, mul, etc)\n* [PY] Fix python source distribution\n* [AMX] Activate AMX on iOS\n* [API] Introduce transforms in external api\n* [BLAS] Introduce a simple BLAS transform for Matrix multiplication\n* [F16] Introduce a Reduce that solves many L2 normalization errors in f16\n\nThis version has been yanked to revert systematic activation of AMX on iOS. AMX is a private API and Apple may reject an App that performs AMX instructions.\n\n# 0.21.1 - 2024-02-08\n* [ONNX] Support for external storage of tensors with offset and length\n\n# 0.21.0 - 2024-01-16\n* MSRV is now 1.75.0\n* [internal] ConvUnary and MatmulUnary are replaced by binary, potentially dynamic equivalent\n\n# 0.20.22 - 2023-11-28\n* [ONNX] LayerNormalization support\n\n# 0.20.21 - 2023-10-31\n* [ONNX] ignoring output shapes is now the default\n* \n\n# 0.20.18 - 2023-08-30\n* [intel] fix in AVX512F matrix vector product\n* [tflite] alpha, embryonic support. some convolutional models working.\n* [kaldi] remove abandonned kaldi experimental support\n* [refactoring] Runtime abstraction and runtime-targetting tests\n* [refactoring] Refactoring Python and C API around a possible tract-api. Introducing dylib support.\n* [pytorch compat] fixes around node names starting by / (bug triggered by recent pytorch versions)\n\n0.20.7 to 0.20.17 are misfires\n\n# 0.20.6 - 2023-06-09\n* Bug fixes, fix display of If operator\n\n# 0.20.5 - 2023-05-26\n* Various bugfix around Einsum\n* Einsum now has functions to translate to MatMul and other axes manipulations\n\n# 0.20.0, 0.20.1, 0,20.2, 0.20.3 - 2023-04-25\n* [optim] 32x32 f32 AMX kernel (for Apple Silicon M family)\n* [optim] bunch of AMX512F kernels (square, skinny, vector)\n* [ONNX] introduce Trilu, TopK\n* [NNEF/OPL] submodel loader\n* [ONNX] support alternative layout for LSTM (layout=1, batch becomes first axis)\n* [ONNX] If operators with dynamic condition (very basic optimisations, no nnef support yet).\n\n# 0.19.9 & 0.19.10 - 2023-04-17\n* HardSwiwh ONNX, tract_core_hard_swish in NNEF/OPL\n* introducing tract_core_submodel in NNEF/OPL\n* JSON resource loader in NNEF/OPL\n* Profiling API tweaks\n* `--folded` view for model command line dump (hide Scan loops)\n\n# 0.19.8 - 2023-03-27\n* Various bug fixes\n\n# 0.19.7 & 0.19.6 - 2023-02-24\n* more bug fixes\n* wip on python doc auto-deploy\n\n# 0.19.5 - 2023-02-22\n* 0.19.3 and 0.19.4 are release misfires\n* lots of bugfixes following 0.19 big changes\n* introducing the JSON NNEF resource\n\n# 0.19.2 - 2023-01-30\n* [NNEF/OPL] introduce json resource loader\n* extend Complex number support (under a feature flag)\n\n# 0.19.1 - 2023-01-23\n* [nnef] new identifier syntax is now opt-in for serialization (both accepted at loading)\n* alpha-level C interface. how and how to deploy it (where to put the .h, whether or not to build and ship dylibs)\n* alpha-level python interface. deployed on pypi as \"tract\". At this stage, API is undocumented and may still change significantly.\n\n# 0.19.0 - 2023-01-11\n* [BREAKING] TValue are now used in run() instead of the previous mix of Tensor and Arc\n* internal API breaking changes: no more op_families, libcli split away. State is no longer Send (but can be \"frozen\" to a Send counterpart).\n* Symbols can now be String instead of char. They are not shared globally anymore, but scoped in the Model instead.\n* [pulse] S symbol is no longer magic. The time dimension symbol must be provided at pulsification time.\n* [pulse] In most cases, we can now pulsify without an explicit pulse len (pulse len can be expression).\n* [cli] deprecated \"x\" syntax for shape is removed\n* [nnef/opl] new i\"...\" syntax for escaping identifiers: i\"some arbitrary string\". Allow serialization of any ONNX model with any kind of string as node names.\n* [ONNX] Signal processing operators (DTF, STFT, MelWeightMatrix, BlackmanWindow, HammingWindow, HannWindow)\n* [ONNX] bitwise operations\n* [ONNX] Compatibility target raised to operator set 18\n\n# 0.18.3 - 2022-10-27\n* [NNEF] Introduce a \"resource\" extension for loading values from a separate source (as a config file)\n* Workaround for cpu detection failure on FreeBSD / arm64\n* Various bug fixes\n\n# 0.18.2 - 2022-10-18\n* [pulse] improve convolution (and others) pulsification to avoid some unecessary buffering delay\n* [cli] support multiple streaming inputs and outputs\n* [ONNX] more relaxed Clip operator rules\n\n# 0.18.1 - 2022-10-06\n* prepare NNEF for further tract-opl extension (resource support)\n* more generic matmul\n* optimise some EinSum cases as matmul\n\n# 0.18.0 - 2022-09-21\n* [ONNX Breaking] Several changes to move towards supporting ONNX symbolic dimensions (actual fixes, but they may break stuff that was working more or less by accident). It may be required to erase output shapes explicitely when input shape is overriden on models that were working before.\n* [CLI breaking] ONXN symbolic dimensions has some impact here too. --input-bundle is deprecated, is was overriden and ambiguous. Instead, there is a --input-facts-from-bundle global option, and a --input-from-bundle option in the subcommands run, profile, dump. --allow-random-input is also moved to subcommands. We think all previously supported behaviours are still there. Please open issues if not.\n\n# 0.17.7 - 2022-09-05\n* clippy up all tract code\n* various fixes\n* 0.17.5 and 0.17.6 are misfired\n\n# 0.17.4 - 2022-08-11\n* [cli] global --set (as a somehat cleaner --concretize successor) allow to set a symbol value after decluttering\n* [cli] run --save-outputs output.npz to save execution outputs\n* dozens of fixs and code cleanup (clippy-fication in progress)\n\n# 0.17.3 - 2022-07-25\n* [License] Allowing https://spdx.org/licenses/Unicode-DFS-2016.html (no tldr yet, but pretty similar to BSD-2)\n* [Breaking] CLI --json option reports costs as strings instead of numbers (in order to allow symbolic values).\n* Sigmoid/Tanh f32 reimpl, plus new f16 impl.\n\n# 0.17.1 - 2022-07-11\n* Sanitiser=address in the CI. Fixed a couple of overflow/memleaks. (Nothing looked too awful.)\n* ONNX NonMaxSuppression\n\n# 0.17.0 - 2022-06-13\n* [Breaking] [ONNX-ML] TreeEnsembleClassifier with binary output (single class) now mimicks scikit-learn output layout.\n\n# 0.16.9 - 2022-06-10\n* bump ONNX protobuf file and support external tensors format\n* new \"skinny\" kernels for avx2/fma f32 multiplication (positive impact on low, non 1 batch size for DNN-heavy loads)\n\n# 0.16.7 - 2022-05-16\n* Softmax is now an operator in core, coming with a direct quantized implementation\n* new TypedFact constructor API ( f32::fact(&[1, 4, 12]), f32::fact(shape!(Symbol::from('N'), 12)))\n* fixes and optimisation of re-quantization pipeline\n* fixes around symbols in NNEF/OPL\n\n# 0.16.6 - 2022-05-03\n* Various changes around quantization support (qi32 appearance)\n\n# 0.16.5 - 2022-04-27\n* Intel optimisation are back\n* Range is now more flexible, should unlock some BERT models with symbolic dimensions.\n\n# 0.16.4 - 2022-04-14\n* some optimisations in depthwise convolutions\n* various bugfixes\n* [Breaking] Fixed nnef \"tile\" operator definition (\"repeats\" is plural). As a consequence models using \"tile\" serialized with tract with prior versions can not be loaded anymore (and vice-versa).\n\n# 0.16.3 - 2022-03-30\n* [Breaking] tract-opl models Scan syntax changed a bit. Models exported by <0.16.2 are loadable in >=0.16.2, but not the other way around.\n* Optimisation in deconv\n\n# 0.16.1 - 2022-03-02\n* [Breaking] Minimum Rust Supported Version is now 1.59.0\n* [Breaking] Small API changes in model api: .compact(), .optimize(), .declutter() now take &mut self and work in place.\n* [LICENSE] Only the licensing for dependencies of the top-level library crates (tensorflow, onnx, kaldi, pulse) will now be monitored. The command line tool (tract crate in cli folder) is for developpers (tract developpers or tract integrators), is not meant to be shipped to end-user, and it concentrates most of the license and dependency complexity.\n* [LICENSE] BSD-3-Clause is now accepted in tract.\n* Optimisations around convolutions and deconvolution\n* Optimisation on Cortex-A53, first round of Cortex-A55 optimisation too.\n\n# 0.15.8 - 2021-11-18\n* Fix brand new ArrayFeatureExtractor inference\n\n# 0.15.7 - 2021-11-16\n* ONNX ArrayFeatureExtractor\n* ConvTranspose/deconv optimisation\n\n# 0.15.6 - yanked \n* just a release script failure\n\n# 0.15.5 - 2021-10-26\n* hold half at 1.7.x for compat with rust 1.50\n\n# 0.15.4 - 2021-10-21\n* ConvTranspose/deconv pulse support\n* ONNX SpaceToDepth/DepthToSpace\n\n# 0.15.3 - 2021-07-29\n* optimise i8*u8, u8*i8 and u8*u8 matrix products (and convo)\n\n# 0.15.2 - 2021-07-09\n* bump prost dep\n\n# 0.15.1 - 2021-07-08\n* some optimisations for arm32 (cortex-a7 and a9)\n\n# 0.15.0 - 2021-06-24\n\n* Switched the order of item_type and item_type_vendor in the NNEF tendor format to be consistent with NNEF-tools, and changed the item_type of integers due to an error in the specification. Breaking for tensor files containing integers or strings.\n* Scan output batching optimisation\n* Concat pulsification over a secondary axis\n* new aarch64 16x4 f32 kernel\n\n## 0.14.2 - 2021-05-27\n\n* better handling of errors in ONNX parser\n* fix/workaround some performance regressions bubling from recent ndarray changes\n\n## 0.14.1 - 2021-05-18\n\n* ONNX ConvTranspose, Gather, GatherND, GatherElements, Scatter, ScatterND, ScatterElements support (and NNEF deconv)\n* Fixes around integer serialisation in NNEF\n* workaround subtle breaking changes in ndarray (between 0.15.1 and 0.15.2)\n\n## 0.14.0 - 2021-04-19\n\n* low-level functions in linalg are now version tagged: two versions of tract can now co-exist in the same binary\n* rustc minimal version is now 1.50\n* dependencies version bumps (ndarray, itertools, and others)\n\n## 0.13.2\n\n* fix sigmoid and tanh variability on intel\n\n## 0.13.1\n\n* temporary disable binary unicast add fusing (too many bugs)\n\n## 0.13.0\n\n* Release are now \"in sync\": all tract crate versions on a build *must* be aligned\n* optimisations, with a focus on aarch64\n\n## 0.12.5 - 2021-01-12\n\n* Dependency bumps\n\n## 0.12.4 - 2021-01-06\n\n* 0.12.3 is a misfire\n* hotfixes on 0.12.2 new tree classifier\n* fix X compilation from macos/aarch64 to macos/intel\n\n## 0.12.2 - 2021-01-05\n\n* ONNX-ML: CategoryMapper and TreeEnsembleClassifier (partial, SoftmaxZero and Probits are missing). With NNEF support.\n* cargo-deny enforces licences choices\n\n## 0.12.1 - 2020-12-11\n\n* 0.12.0 is a misfire.\n\n* API BREAKING: TypedFact::dt_shape & friends can not fail anymore, no longer return a result (remove `?`)\n* Breaking: Rust minimal version bumped to 1.42\n\n* Early, basic, correct but slow support for i8 by u8 matrix mult.\n* Support for Apple Silicon, aka M1, aka aarch64 darwin (but not in CI yet)\n* dynamic quantization convolution support\n* release now ships cli musl builds for linux\n* optimizations targetting small Cortex-A (like 7, 8, and 9)\n* command line dump --profile --cost now computes flops\n* ONNX: OneHot op support\n\n## 0.11.2 - 2020-10-26\n\n* ONNX: new op: DynamicQuantizeLinear\n* tract-data crate split from core, containing tensor, dim, and datum types.\n\n## 0.11.1 - 2020-10-20\n\n* switch from error_chain to anyhow\n* simplify trivial gathers to a slice\n* generalize symbolic dimension a bit: support \"2S\" and the like\n* deprecate \"x\" syntax in CLI, please use `,` instead\n\n## 0.11.0\n\n### Breaking \n\n* NNEF: tract-nnef no longer performs gunziping, but expect an uncompressed tar\n stream. We found out is it counter-productive (weights matrices are more or\n less random, they do not compress easily, and decompression is expensive).\n NNEF networks in the wild are .tgz file. Using flate2, decompression is a\n one-liner, but it must be done by the client code now.\n* bumped extended nnef compat version (unchecked at this stage) to \"alpha1\"\n* move pulse operators and translation to their own crate and nnef registry\n* generalize TDim to support an arbitrary number of symbols\n* concretize_stream_dim is superseded by concrentize_dims\n\n### Notable\n\n* new crates, building on tract-opl introduction:\n * *tract-pulse-opl*: pulse runtime (handful of ops, including Delay) is now separated from core\n * *tract-onnx-opl*: onnx runtime (4 ops not belonging in core)\n * *tract-pulse*: pulsification of models (model-translation time)\n * tract-onnx is now limited to onnx model loading and conversion\n\n## 0.10.10 - 2020-08-30\n\n* load a NNEF as a TypedModel using tract_nnef, and from the CLI\n* dump a tract TypedModel to NNEF (with extensions for op not nnef compatbile)\n* not a full coverage of nnef, but enough for most CNN (image categorizers zoo working)\n* 80% of onnx tests are surviving a NNEF dump and reload at this stage\n\n## 0.10.0 - 2020-07-28\n\n### ONNX\n\n* covered operators compatible with Operator Sets 9, 10, 11 (new) and 12 (new)\n\n### API Breaking\n\n* Tensor::l1 method is gone\n\n### Windows\n\n* Support for -gnu targets (non-mvsc).\n\n### Notable\n\n* --cost now gives the number of parameters in the model\n* SimpleState is clonable again (actually useful !)\n\n## 0.9.2 - 2020-06-16\n\n* introduce `TypedModel::method.concretize_stream_dim`\n* various pulsification bugfixes\n\n## 0.9.1 - 2020-06-16\n\n* fix Reshape with TDim\n\n## 0.9.0 - 2020-06-15\n\nStill no shortage of version numbers...\n\n### API Breakage\n\n* NormalizedModel (and friends) are gone. They were only useful as a pre-pulse transformation pre-requisite that the current TypedModel (& co) meets.\n* TypedModel::into_optimized() is gone. InferenceModel::into_optimized() stays as an end-to-end shortcut for simple cases. It does .into_typed()?.declutter()?.optimize()).\n* TypedModel::codegen() is now ::optimize()\n\n## 0.8.0 - 2020-06-13\n\nI wish I had seen these issues yesterday. Anyway, version numbers are cheap.\n\n* Bumping minimum rust to 1.41\n\n## 0.7.0 - 2020-06-12\n\n* CLI refactoring (hopefully stabilizing a bit?)\n * `profile --bench` is now bench\n * profile is now `dump --profile`\n * cost is now `dump --cost`\n * profiling is now done during a full net instead of per op\n * new \"compact\" graph dumper, profile visual hints\n * `dump --cost --profile --json` output profiling and cost information\n * show logical names for ops instead of the Op struct names (not 100% sure it's right)\n * criterion integration\n* WASM support for tract-onnx and tract-tensorflow targets (CI)\n* Convenience methods added to Models to allow model building in fluent style, up to Plan instantiation (SimplePlan now nicknamed RunnableModel). Non breaking.\n* Support for ONNX bidi LSTM (CI), GRU and RNN (untested, consider alpha)\n* Fixes around nets with a non trivial batch size (axis simplification code, matmul op fusion)\n\n## 0.6.3 - 2020-04-25\n\n* Lock ndarray version to dodge rustc/llvm issue (https://github.com/rust-lang/rust/issues/71506)\n\n## 0.6.2 - 2020-04-15\n\n* Use http://gihub.com/kali/readings for instrumentation.\n\n## 0.6.0 - 2020-02-19\n\n### Notable\n\n* New jupyter/keras/tf example\n* ARMv8 tanh / sigmoid optimisation\n\n### API Breaking\n\n* refactor exports and dependencies\n * preferred way to use tract is now to `use tract_tensorflow::prelude::*;`\n * singleton framework is built by `let tensorflow = tensorflow()`. The Framework trait is in the prelude too.\n * the prelude contains a reexport of `tract_core`, and of ndarray as `tract_ndarray`\n * no more need to declare dependency on `tract-core` and/or `tract-linalg` in Cargo.toml\n * same goes for `tract_onnx`\n\n## 0.5.9 - 2020-02-07\n\n### Breaking\n\n* Rustc minimum version is now 1.39\n\n### Onnx\n\n* Support for MatMulInteger, ConvInteger\n* Support for QuantizeLinear DequantizeLinear\n* Basic support for QLinearMatMul, QLinearConv\n\n## 0.5.6 - 2019-10-30\n\n### Tensorflow\n\n* Initial support for GatherV2\n\n### Onnx\n\n* Fix PReLu normalization\n\n## 0.5.5 - 2019-10-25\n\n### Tensorflow\n\n* Initial support for AddV2, Mean, Min, Prod, Sum\n\n## 0.5.4 - 2019-09-30\n\n### Notable\n\n* Make Onnx loader operator set aware, and Slice-10 support.\n* Cost now reports Delay ops buffer size\n* Bump dependencies (protobuf) and fix codegen\n* Windows CI now performs a top-level \"cargo check\"\n\n## 0.5.1 - 2019-09-24\n\n### Bugfix\n\n* remove the no_panic checks, as too fragile (breaking non-lto builds)\n\n## 0.5.0 - 2019-09-20\n\n### Breaking\n\n* Change tensor facts names for consistency: TensorFact is now InferenceFact.\n\n### Notable\n\n* Introduce Windows support, including CI coverage for linalg\n* Switch from Travis to GitHub Actions\n* Internal refactoring around tract-core canonic opset\n* Tract CLI can now compute a FLOP number for networks (\"cost\" subcommand). \n Furthermore the CI asserts its value for a few networks to prevent optimisation regressions.\n* Fix: handling of -1 in ONNX Reshape op\n\n## 0.4.2 - 2019-09-10\n\n* Fix release script after 0.4.1 release disaster.\n\n## 0.4.1 - 2019-09-09 [YANKED]\n\n* Fix for OS where CARGO_CFG_TARGET_FAMILY is undefined\n* Linear Algebra package refactor\n* tract-core canonic operator set introduction\n* significant performance boost (up to 20% on some real-life networks)\n\n## 0.4.0 - 2019-07-30\n\n* Start Kaldi networks support (LSTM, Renorm, Affine, downsample)\n\n## Before...\n\nThis Changelog started way too late. But better late than never.\n" }, { "path": "Cargo.toml", "content": "[workspace]\nresolver = \"2\"\nmembers = [\n \"data\",\n \"linalg\",\n \"core\",\n \"pulse\",\n \"pulse-opl\",\n \"hir\",\n \"nnef\",\n \"nnef/cli\",\n \"nnef/nnef-resources\",\n \"tensorflow\",\n \"tflite\",\n \"onnx-opl\",\n \"onnx\",\n \"libcli\",\n \"cli\",\n \"gpu\",\n \"metal\",\n \"extra\",\n \"transformers\",\n \"cuda\",\n\n \"api\",\n \"api/rs\",\n \"api/ffi\",\n \"api/proxy\",\n \"api/proxy/sys\",\n\n \"examples/face_detection_yolov8onnx_example\",\n \"examples/face_similarity_arcface_onnx\",\n \"examples/tensorflow-mobilenet-v2\",\n \"examples/tflite-mobilenet-v3\",\n \"examples/keras-tract-tf2\",\n \"examples/nemo-parakeet-asr\",\n \"examples/nemo-nemotron-asr\",\n \"examples/nnef-dump-mobilenet-v2\",\n \"examples/nnef-mobilenet-v2\",\n \"examples/nnef-mobilenet-v2-api\",\n \"examples/onnx-mobilenet-v2\",\n \"examples/pytorch-albert-v2\",\n \"examples/pytorch-resnet\",\n \"examples/causal_llm\",\n \"examples/stable-diffusion\",\n \"examples/stable-diffusion-3\",\n \"examples/stable-diffusion-xl\",\n\n \"harness/core-proptest-pulse\",\n \"harness/nnef-inceptionv3\",\n \"harness/tf-inceptionv3\",\n \"harness/tf-mobilenet-v2\",\n \"harness/tfl-mobilenet-v2-q\",\n\n \"test-rt/infra\",\n \"test-rt/suite-unit\",\n \"test-rt/suite-onnx\",\n \"test-rt/test-f16\",\n \"test-rt/test-blas\",\n \"test-rt/test-metal\",\n \"test-rt/test-cuda\",\n \"test-rt/test-unit-core\",\n \"test-rt/test-onnx-core\",\n \"test-rt/test-nnef-cycle\",\n \"test-rt/test-tflite\"\n]\n\n# same, without metal, test-metal, cuda, test-cuda and test-tflite which are probelematic on specific targets\ndefault-members = [\n \"data\",\n \"linalg\",\n \"core\",\n \"pulse\",\n \"pulse-opl\",\n \"hir\",\n \"nnef\",\n \"nnef/cli\",\n \"nnef/nnef-resources\",\n \"tensorflow\",\n \"tflite\",\n \"onnx-opl\",\n \"onnx\",\n \"libcli\",\n \"cli\",\n \"extra\",\n \"transformers\",\n\n \"api\",\n \"api/rs\",\n \"api/ffi\",\n\n \"examples/face_detection_yolov8onnx_example\",\n \"examples/face_similarity_arcface_onnx\",\n \"examples/tensorflow-mobilenet-v2\",\n \"examples/tflite-mobilenet-v3\",\n \"examples/keras-tract-tf2\",\n \"examples/nnef-dump-mobilenet-v2\",\n \"examples/nnef-mobilenet-v2\",\n \"examples/onnx-mobilenet-v2\",\n \"examples/pytorch-albert-v2\",\n \"examples/pytorch-resnet\",\n\n \"harness/core-proptest-pulse\",\n \"harness/nnef-inceptionv3\",\n \"harness/tf-inceptionv3\",\n \"harness/tf-mobilenet-v2\",\n \"harness/tfl-mobilenet-v2-q\",\n\n \"test-rt/infra\",\n \"test-rt/suite-unit\",\n \"test-rt/suite-onnx\",\n \"test-rt/test-f16\",\n \"test-rt/test-blas\",\n \"test-rt/test-unit-core\",\n \"test-rt/test-onnx-core\",\n \"test-rt/test-nnef-cycle\",\n]\n\n[workspace.package]\nrust-version = \"1.91\"\n\n[workspace.dependencies]\naccelerate-src = \"0.3\"\nanstyle = \"1.0.2\"\nanstyle-parse = \"1.0.0\"\nanstyle-query = \"1.0.0\"\nanyhow = \"1.0.43\"\nanymap3 = \"1.0\"\napprox = \"0.5\"\natty = \"0.2.14\"\nbit-set = \"0.10.0\"\nboow = \"0.1.3\"\nbox_drawing = \"0.1.2\"\nbyteorder = \"1.4.3\"\nbytes = \"1.0.1\"\ncc = \"1.0.69\"\nclap = { version = \"4\", features = [ \"cargo\", \"derive\" ] }\ncolorous = \"1.0.5\"\ncore_affinity = \"0.8.0\"\ncriterion = \"0.8\"\ncudarc = { version = \"0.19\", features = [\"dynamic-loading\", \"f16\", \"cudnn\"] }\nderive-new = \"0.7\"\ndinghy-test = \"0.8\"\ndirs = \"6.0.0\"\ndowncast-rs = \"2.0\"\ndyn-clone = \"1.0.4\"\ndyn-eq = \"0.1\"\ndyn-hash = \"1.0\"\nenv_logger = \"0.11\"\nerased-serde = \"0.4\"\nflatbuffers = \"25.12.19\"\nflate2 = \"1.0.20\"\nfloat-ord = \"0.3.2\"\nfs-err = \"3\"\nfs2 = \"0.4.3\"\ngetrandom = \"0.4\"\nhalf = { version=\">=2.4,<3.0\", features = [ \"std\", \"num-traits\" ] }\nhome = \"=0.5.12\"\nicu_normalizer = \"2.1\"\nicu_normalizer_data = \"2.1\"\nicu_properties = \"2.1\"\nicu_properties_data = \"2.1\"\nidna_adapter = \"1.2.0\"\nimage = \"0.25\"\ninventory = \"0.3.21\"\nitertools = \"0.14\"\nlazy_static = \"1.5.0\"\nlibc = \"0.2.164\"\nlibloading = \"0.9\"\nlibm = \"0.2.11\"\nliquid = \"0.26.8\"\nliquid-core = \"0.26.8\"\nliquid-derive = \"0.26.8\"\nlitemap = \"0.8\"\nlog = \"0.4.14\"\nmaplit = \"1.0.2\"\nmemmap2 = \"0.9\"\nmetal = { version = \"0.33.0\" }\nndarray = \"0.17\"\nndarray-npy = { version = \"0.10\", features = [ \"compressed_npz\" ] }\nnom = \"8.0.0\"\nnom-language = \"0.1\"\nnu-ansi-term = \"0.50\"\nnum-complex = \"0.4.0\"\nnum-integer = \"0.1.44\"\nnum-traits = \"0.2.14\"\nnum_cpus = \"1\"\nopenblas-src = { version = \"0.10\", features = [\"static\"] }\npastey = \"0.2\"\nproptest = \"1.0.0\"\nprost = \"0.14\"\nprost-types = \"0.14\"\npy_literal = \"0.4.0\"\nrand = \"0.10\"\nrand_distr = \"0.6\"\nrayon = \"1.10\"\nreadings-probe = \"0.1.8\"\nregex = \"1.5.4\"\nron = \"0.12\"\nreqwest = { version = \"0.13\", features = [ \"blocking\", \"rustls-no-provider\" ], default-features = false }\nrustfft = { version = \"6.1\", features = [ \"neon\" ] }\nrustls = { version = \"0.23\", default-features = false, features = [ \"ring\", \"std\", \"tls12\" ] }\nwebpki-roots = \"1\"\nsafetensors = \"0.7\"\nscan_fmt = \"0.2.6\"\nserde = { version = \"1.0.127\", features = [ \"derive\" ] }\nserde_json = \"1.0\"\nsimd-adler32 = { version = \"0.3.7\", features = [\"std\"] }\nsmallvec = \"1.6.1\"\nstring-interner = \"0.19\"\ntar = \"0.4.37\"\ntempfile = \"3.8\"\ntensorflow = \"0.21.0\"\ntflitec = { git = \"https://github.com/kali/tflitec-rs.git\", rev=\"9ceb838\" }\ntime = \"0.3.23\"\ntokenizers = \"0.22\"\nunicode-normalization = \"0.1.19\"\nwalkdir = \"2.3.2\"\nzerofrom = \"0.1.5\"\ntract-api = { version = \"0.23.0-pre\", path = 'api' }\ntract-core = { version = \"0.23.0-pre\", path = 'core' }\ntract-cuda = { version = \"0.23.0-pre\", path = 'cuda' }\ntract-data = { version = \"0.23.0-pre\", path = 'data' }\ntract-extra = { version = \"0.23.0-pre\", path = 'extra' }\ntract-gpu = { version = \"0.23.0-pre\", path = 'gpu' }\ntract-hir = { version = \"0.23.0-pre\", path = 'hir' }\ntract-libcli = { version = \"0.23.0-pre\", path = 'libcli' }\ntract-linalg = { version = \"0.23.0-pre\", path = 'linalg' }\ntract-metal = { version = \"0.23.0-pre\", path = 'metal' }\ntract-nnef-resources = { version = \"0.23.0-pre\", path = 'nnef/nnef-resources' }\ntract-nnef = { version = \"0.23.0-pre\", path = 'nnef' }\ntract-onnx-opl = { version = \"0.23.0-pre\", path = 'onnx-opl' }\ntract-onnx = { version = \"0.23.0-pre\", path = 'onnx' }\ntract-pulse-opl = { version = \"0.23.0-pre\", path = 'pulse-opl' }\ntract-pulse = { version = \"0.23.0-pre\", path = 'pulse' }\ntract-tensorflow = { version = \"0.23.0-pre\", path = 'tensorflow' }\ntract-tflite = { version = \"0.23.0-pre\", path = 'tflite' }\ntract-transformers = { version = \"0.23.0-pre\", path = 'transformers' }\ntract = { version = \"0.23.0-pre\", path = 'api/rs' }\ntract-proxy-sys = { version = \"0.23.0-pre\", path = 'api/proxy/sys' }\ntract-cli = { version = \"0.23.0-pre\", path = 'cli' }\ntract-ffi = { version = \"0.23.0-pre\" }\ntract-proxy = { version = \"0.23.0-pre\" }\n\n\n[profile.opt-no-lto]\ninherits=\"release\"\nlto=false\n\n[profile.release]\n# debug = true\nlto = true\n\n[profile.bench]\ndebug = true\n\n[profile.dev.package.\"*\"]\nopt-level = 2\n\n[profile.dev.build-override]\ndebug = false\n# strip = \"debuginfo\" does not work on android and ios\nincremental = false\n" }, { "path": "LICENSE", "content": "## License\n\nLicensed under either of\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "path": "LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "LICENSE-MIT", "content": "Permission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "![tract-logo](assets/tract-logo/PNG/tract-horizontal-blue.png)\n\n![Rust](https://img.shields.io/badge/rust-%23000000.svg?style=for-the-badge&logo=rust&logoColor=white)\n![rustc >= 1.91.0](https://img.shields.io/badge/rustc-%3E%3D1.91.0-brightgreen)\n![MIT/Apache 2](https://img.shields.io/crates/l/tract)\n[![Native Linux test status](https://github.com/snipsco/tract/workflows/Native%20Linux/badge.svg)](https://github.com/snipsco/tract/actions)\n[![Embedded targets status](https://github.com/snipsco/tract/workflows/Embedded%20targets/badge.svg)](https://github.com/snipsco/tract/actions)\n[![Doc](https://docs.rs/tract-core/badge.svg)](https://docs.rs/tract-core)\n\n[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://pypi.org/project/tract/)\n\n\nSonos' Neural Network inference engine.\n\n_This project used to be called tfdeploy, or Tensorflow-deploy-rust._\n\n## What ?\n\n`tract` is a Neural Network inference toolkit. It can read ONNX or NNEF, optimize them and run them.\n\n## Quick start, examples\n\n* [MobileNet v2 with ONNX](examples/onnx-mobilenet-v2)\n* [BERT example with ONNX](examples/pytorch-albert-v2)\n* [MobileNet v2 with TensorFlow](examples/tensorflow-mobilenet-v2)\n* [From Keras and TensorFlow 2 to tract](examples/keras-tract-tf2)\n* [ResNet with PyTorch](examples/pytorch-resnet)\n\nThere is also [some technical documentation](doc/) and [blog](https://tech-blog.sonos.com/posts/optimising-a-neural-network-for-inference/) posts.\n\n## Tract in the landscape\n\n### ONNX\n\nAs of today, `tract` passes successfully about 85% of ONNX backends\ntests. All \"real life\" integration tests in ONNX test suite are passing: \nbvlc_alexnet, densenet121, inception_v1, inception_v2, resnet50, shufflenet,\nsqueezenet, vgg19, zfnet512.\n\nNotable missing parts are operators dealing with Tensor Sequences and Optional Tensors : tract /really/ wants to flow Tensors and nothing else.\nThis is structural. Changing it would be pretty difficult, and it's unclear whether it can be done without impairing performance or maintainability.\nWe are not convinced these features have shown their interest in the wild yet, so we prefer to leave them aside.\n\nOther dark corners are specific operators like \"Resize\" which fit perfectly in the framework but need a complex internal logic that is far\nfrom our core business. In these cases, we are happy to accept contributions and to help. \n\nThe following operators are implemented and tested.\n\nAbs, Acos, Acosh, Add, And, ArgMax, ArgMin, ArrayFeatureExtractor, Asin, Asinh, Atan, Atanh, AveragePool, BatchNormalization, BitShift, BitwiseAnd, BitwiseNot, BitwiseOr, BitwiseXor, BlackmanWindow, Cast, CastLike, CategoryMapper, Ceil, Clip, Compress, Concat, Constant, ConstantLike, ConstantOfShape, Conv, ConvInteger, ConvTranspose, Cos, Cosh, CumSum, DFT, DepthToSpace, DequantizeLinear, Div, Dropout, DynamicQuantizeLinear, Einsum, Elu, Equal, Erf, Exp, Expand, EyeLike, Flatten, Floor, GRU, Gather, GatherElements, GatherND, Gemm, GlobalAveragePool, GlobalLpPool, GlobalMaxPool, Greater, GreaterOrEqual, HammingWindow, HannWindow, HardSigmoid, Hardmax, Identity, If, InstanceNormalization, IsInf, IsNaN, LRN, LSTM, LeakyRelu, Less, LessOrEqual, Log, LogSoftmax, MatMul, MatMulInteger, Max, MaxPool, Mean, MelWeightMatrix, Min, Mod, Mul, Multinomial, Neg, NonMaxSuppression, NonZero, Not, OneHot, Or, PRelu, Pad, ParametricSoftplus, Pow, QLinearConv, QLinearMatMul, QuantizeLinear, RNN, RandomNormal, RandomNormalLike, RandomUniform, RandomUniformLike, Range, Reciprocal, ReduceL1, ReduceL2, ReduceLogSum, ReduceLogSumExp, ReduceMax, ReduceMean, ReduceMin, ReduceProd, ReduceSum, ReduceSumSquare, Relu, Reshape, Resize, Round, Rsqrt, STFT, ScaledTanh, Scan, Scatter, ScatterElements, ScatterND, Selu, Shape, Shrink, Sigmoid, Sign, Sin, Sinh, Size, Slice, Softmax, Softplus, Softsign, SpaceToDepth, Split, Sqrt, Squeeze, Sub, Sum, Tan, Tanh, ThresholdedRelu, Tile, Transpose, TreeEnsembleClassifier, Unsqueeze, Where, Xor\n\nWe test these operators against from ONNX 1.4.1 (operator set 9), up to ONNX 1.13.0 (operator set 18).\n\nWe are using ONNX test suite, but it does not cover everything.\nWe also deliberately ignore some tests, or restricting their scope depending on what we feel is realistic.\nSometimes these decisions are just wrong, and sometimes they become wrong as time goes by and the fields moves in unexpected directions.\nSo if you are puzzled by an ONNX model that does not work in tract, we are happy to take a look.\n\n### NNEF\n\nLong story short, TensorFlow and ONNX formats are good for designing and\ntraining networks. They need to move fast to follow the research field, tend to\nintegrate new features and operators greedily. They also exhibit a high level\nof expressivity to facilitate network design.\n\nOn the other hand, only a subset of operators and network features actually\nreach production, so systems running production network do not have to deal\nwith so many operators. Furthermore, some information required for training can\nbe stripped from the network before going to production for prediction.\n\nNNEF tries to bridge the gap between training frameworks and inference by\nproposing a format dedicated to production and prediction.\n\nTract supports NNEF:\n\n* tract_nnef can load and execute NNEF networks\n* tract supports most of the NNEF specification, the most notable exception\n being the ROI operators\n* tract introduces tract-OPL, a series of NNEF extensions to support other\n operators (or extend some operators semantics) in order to represent the\n full range of tract-core neural network support: any network understood by\n tract should be serializable to tract-OPL. This is a work in progress.\n* tract command line can translate networks from TensorFlow or ONNX to NNEF/OPL.\n\n### tract-opl version compatibility\n\nA remainder: NNEF is not expressive enough to represent all ONNX. tract-OPL extends\nNNEF using proprietary to support what is missing. Notable extensions are pulse\noperators, recurring operators (as Scan) and symbolic extensions.\n\nThere is no strict check in place here, so... implementation is not bullet proof.\n* NNEF part aims at being very stable. It is strongly constrained with compatibility\nwith NNEF specification.\n* tract-opl is a bit more in flux. Nevertheless we try to maintain the following\ngolden rule:\n\n `models serialized with tract 0.x.y should work with tract 0.x.z where z >= y`\n\n* in practice, breaking changes have been relatively rare so far. Most models are\nforward and retro compatible from when tract has acquired NNEF support.\n\nNotable breakage occurred:\n* 0.16.3 (forward compatible) on Scan operator\n* 0.17.0 for binary decision tree classifier\n\nStarting with `0.17.0`, a model property is injected in tract-opl files (`tract_nnef_ser_version`)\nto tag which version of tract generated the file. As most models will remain compatible,\ntract will not do any version check. It is up to the application developer to do so.\n\nA softer version tag exists as `tract_nnef_format_version`. pre-0.17.0 version set it to\n`alpha1`, post-0.17.0 set it `beta1`. Don't put too much emphasis into the \"alpha-ness\" naming \nof versions here.\n\n### Note: support for TensorFlow 1.x\n\nEven if `tract` is very far from supporting any arbitrary model, it can run\nGoogle Inception v3 and Snips wake word models. Missing operators are relatively \neasy to add. The lack of easy to reuse test suite, and the wide diversity of \noperators in Tensorflow make it difficult to target a full support.\n\nThe following operators are implemented and tested:\n\nAbs, Add, AddN, AddV2, Assign, AvgPool, BatchToSpaceND, BiasAdd, BlockLSTM, Cast, Ceil, ConcatV2, Const, Conv2D, DepthwiseConv2dNative, Div, Enter, Equal, Exit, ExpandDims, FakeQuantWithMinMaxVars, Fill, FloorMod, FusedBatchNorm, GatherNd, GatherV2, Greater, GreaterEqual, Identity, Less, LessEqual, Log, LogicalAnd, LogicalOr, LoopCond, MatMul, Max, MaxPool, Maximum, Mean, Merge, Min, Minimum, Mul, Neg, NoOp, Pack, Pad, Placeholder, Pow, Prod, RandomUniform, RandomUniformInt, Range, RealDiv, Relu, Relu6, Reshape, Rsqrt, Shape, Sigmoid, Slice, Softmax, SpaceToBatchND, Squeeze, StridedSlice, Sub, Sum, Switch, Tanh, Tile, Transpose, VariableV2\n\nAdditionally, the complexity of TensorFlow 2 make it very unlikely that a direct\nsupport will ever exist in tract. But many TensorFlow 2 models can be\nconverted to ONNX and then loaded in tract.\n\n## Example of supported networks\n\nThese models among others, are used to track tract performance evolution as\npart of the Continuous Integration jobs. See [.travis/README.md](readme) and \n[.travis/bundle-entrypoint.sh](.travis/bundle-entrypoint.sh) for more\ninformation.\n\n### Keyword spotting on Arm Cortex-M Microcontrollers\n\nhttps://github.com/ARM-software/ML-KWS-for-MCU\n\nARM demonstrated the capabilities of the Cortex-M family by providing\ntutorials and pre-trained models for keyword spotting. While the exercise\nis ultimately meant for micro-controllers, `tract` can run the intermediate\nTensorFlow models.\n\nFor instance, on a Raspberry Pi Zero, the \"CNN M\" model runs in about 70\nmicro-seconds, and 11 micro-seconds on a Raspberry Pi 3.\n\n### Snips wake word models\n\nhttps://arxiv.org/abs/1811.07684\n\nSnips uses `tract` to run the wake word detectors. While earlier models were\nclass-based and did not require any special treatment, `tract` pulsing\ncapabilities made it possible to run WaveNet models efficiently enough for a\nRaspberry Pi Zero.\n\n### Inception v3\n\n| Device | Family | TensorFlow-lite | tract |\n|---------------------|----------------|-------------------|---------|\n| Raspberry Pi Zero | Armv6 VFP | 113s | 39s |\n| Raspberry Pi 2 | Armv7 NEON | 25s | 7s |\n| Raspberry Pi 3 | aarch32 NEON | 5s | 5s |\n\nNotes:\n\n * while the Raspberry Pi 3 is an Armv8 device, this bench is running\n on Raspbian, an armv6 operating system, crippling the performance\n of both benches\n * there exists other benches on the internet that show better\n performance results for TensorFlow (not -Lite) on the Pi 3.\n They use all four cores of the device. Both TensorFlow-Lite and tract\n here have been made to run on a single-core.\n\n# License\n\nNote: files in the `tensorflow/protos` directory are copied from the\n[TensorFlow](https://github.com/tensorflow/tensorflow) project and are not\ncovered by the following licence statement.\n\nNote: files in the `onnx/protos` directory are copied from the\n[ONNX](https://github.com/onnx/onnx) project and are not\ncovered by the following license statement.\n\n## Apache 2.0/MIT\n\nAll original work licensed under either of\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\nat your option.\n\n## Contribution\n\nUnless you explicitly state otherwise, any Contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "path": "api/.gitignore", "content": "*.nnef.tgz\n*.onnx\n" }, { "path": "api/Cargo.toml", "content": "[package]\nname = \"tract-api\"\nversion = \"0.23.0-pre\"\nlicense = \"MIT OR Apache-2.0\"\nauthors = [\"Mathieu Poumeyrol \"]\ndescription = \"Tiny, no-nonsense, self contained, TensorFlow and ONNX inference\"\nrepository = \"https://github.com/sonos/tract\"\nkeywords = [ \"NeuralNetworks\" ]\ncategories = [ \"science\" ]\nautobenches = false\nedition = \"2024\"\nrust-version.workspace = true\ninclude = [ \"Cargo.toml\", \"src/**/*.rs\", \"LICENSE*\", \"tract.h\" ]\n\n[dependencies]\nanyhow.workspace = true\nboow.workspace = true\nflate2.workspace = true\nhalf.workspace = true\nndarray.workspace = true\nserde.workspace = true\nserde_json.workspace = true\n\n[features]\ncomplex = []\n\n[dev-dependencies]\nlazy_static = \"1.4.0\"\nreqwest.workspace = true\ntempfile.workspace = true\n" }, { "path": "api/LICENSE", "content": "## License\n\nLicensed under either of\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "path": "api/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "api/LICENSE-MIT", "content": "Permission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "api/c/.gitignore", "content": "*.h\n*.so\nmobilenet\nmobilenet_v2_1.0.onnx.nnef.tgz\n\n\n" }, { "path": "api/c/Makefile", "content": "run: mobilenet mobilenet_v2_1.0.onnx.nnef.tgz\n\tLD_LIBRARY_PATH=. ./mobilenet\n\nclean:\n\trm -f mobilenet libtract.so tract.h\n\nmobilenet: tract.h libtract.so mobilenet.c\n\tcc mobilenet.c -o mobilenet -L. -ltract\n\nlibtract.so:\n\tcargo build -p tract-ffi --profile opt-no-lto\n\tcp ../../target/opt-no-lto/libtract.so .\n\ntract.h:\n\tcd ../ffi ; cbindgen -l c > tract.h\n\tcp ../ffi/tract.h .\n\nmobilenet_v2_1.0.onnx.nnef.tgz:\n\twget -q https://s3.amazonaws.com/tract-ci-builds/tests/mobilenet_v2_1.0.onnx.nnef.tgz\n" }, { "path": "api/c/mobilenet.c", "content": "#include \"tract.h\"\n#include \n#include \n#include \n\n#define check(call) \\\n { \\\n TRACT_RESULT result = call; \\\n if (result == TRACT_RESULT_KO) { \\\n fprintf(stderr, \"Error calling tract: %s\", \\\n tract_get_last_error()); \\\n exit(1); \\\n } \\\n }\n\nint main() {\n // Initialize nnef parser\n TractNnef *nnef = NULL;\n check(tract_nnef_create(&nnef));\n assert(nnef);\n\n // Load the model\n TractModel *model = NULL;\n check(tract_nnef_load(nnef, \"mobilenet_v2_1.0.onnx.nnef.tgz\", &model));\n assert(model);\n assert(nnef);\n\n // once the model is build, the framework is not necessary anymore\n check(tract_nnef_destroy(&nnef));\n assert(!nnef);\n\n // Pick a runtime\n TractRuntime *runtime = NULL;\n check(tract_runtime_for_name(\"default\", &runtime));\n assert(runtime);\n\n // Make the model runnable\n TractRunnable *runnable = NULL;\n check(tract_runtime_prepare(runtime, &model, &runnable));\n assert(runnable);\n assert(!model);\n\n float *image = malloc(3 * 224 * 224 * sizeof(float));\n FILE *fd = fopen(\"grace_hopper_3_224_224.f32.raw\", \"rb\");\n assert(fread(image, sizeof(float), 3 * 224 * 224, fd) == 3 * 224 * 224);\n fclose(fd);\n\n TractTensor *input = NULL;\n size_t shape[] = {1, 3, 224, 224};\n check(\n tract_tensor_from_bytes(TRACT_DATUM_TYPE_F32, 4, shape, image, &input));\n free(image);\n\n TractTensor *output = NULL;\n\n // simple stateless run...\n check(tract_runnable_run(runnable, &input, &output));\n\n const float *data = NULL;\n check(tract_tensor_as_bytes(output, NULL, NULL, NULL, (const void **)&data));\n float max = data[0];\n int argmax = 0;\n for (int i = 0; i < 1000; i++) {\n float val = data[i];\n if (val > max) {\n max = val;\n argmax = i;\n }\n }\n printf(\"Max is %f for category %d\\n\", max, argmax);\n check(tract_tensor_destroy(&output));\n\n // or spawn a state to run the model\n TractState *state = NULL;\n check(tract_runnable_spawn_state(runnable, &state));\n assert(state);\n\n // runnable is refcounted by the spawned states, so we can release it now.\n check(tract_runnable_release(&runnable));\n assert(!runnable);\n\n check(tract_state_run(state, &input, &output));\n\n check(tract_tensor_as_bytes(output, NULL, NULL, NULL, (const void **)&data));\n assert(data[argmax] == max);\n check(tract_tensor_destroy(&output));\n\n // done with out state and input\n check(tract_state_destroy(&state));\n check(tract_tensor_destroy(&input));\n}\n" }, { "path": "api/ffi/Cargo.toml", "content": "[package]\nname = \"tract-ffi\"\nversion = \"0.23.0-pre\"\nlicense = \"MIT OR Apache-2.0\"\nauthors = [\"Mathieu Poumeyrol \"]\ndescription = \"Tiny, no-nonsense, self contained, neural network inference\"\nrepository = \"https://github.com/snipsco/tract\"\nkeywords = [ \"TensorFlow\", \"NeuralNetworks\" ]\ncategories = [ \"science\" ]\nautobenches = false\nedition = \"2024\"\ninclude = [ \"Cargo.toml\", \"src/**/*.rs\", \"LICENSE*\" ]\n\n[lib]\nname = \"tract\"\ncrate-type = [\"cdylib\"]\n\n[badges]\nmaintenance = { status = \"actively-developed\" }\n\n[dependencies]\nanyhow.workspace = true\nflate2.workspace = true\nserde.workspace = true\nserde_json.workspace = true\ntract-api.workspace = true\ntract-rs = { version = \"0.23.0-pre\", path = \"../rs\", package = \"tract\" }\n" }, { "path": "api/ffi/cbindgen.toml", "content": "language = \"C\"\n\nafter_includes = \"\"\"\n typedef enum DatumType {\n TRACT_DATUM_TYPE_BOOL = 1,\n TRACT_DATUM_TYPE_U8 = 17,\n TRACT_DATUM_TYPE_U16 = 18,\n TRACT_DATUM_TYPE_U32 = 20,\n TRACT_DATUM_TYPE_U64 = 24,\n TRACT_DATUM_TYPE_I8 = 33,\n TRACT_DATUM_TYPE_I16 = 34,\n TRACT_DATUM_TYPE_I32 = 36,\n TRACT_DATUM_TYPE_I64 = 40,\n TRACT_DATUM_TYPE_F16 = 50,\n TRACT_DATUM_TYPE_F32 = 52,\n TRACT_DATUM_TYPE_F64 = 56,\n TRACT_DATUM_TYPE_COMPLEX_I16 = 66,\n TRACT_DATUM_TYPE_COMPLEX_I32 = 68,\n TRACT_DATUM_TYPE_COMPLEX_I64 = 72,\n TRACT_DATUM_TYPE_COMPLEX_F16 = 82,\n TRACT_DATUM_TYPE_COMPLEX_F32 = 84,\n TRACT_DATUM_TYPE_COMPLEX_F64 = 88,\n } DatumType;\n\"\"\"\n" }, { "path": "api/ffi/src/lib.rs", "content": "#![allow(clippy::missing_safety_doc)]\n\nuse anyhow::{Context, Result};\nuse std::cell::RefCell;\nuse std::ffi::{CStr, CString, c_char, c_void};\nuse tract_api::{\n AsFact, DatumType, DimInterface, FactInterface, InferenceModelInterface, ModelInterface,\n NnefInterface, OnnxInterface, RunnableInterface, RuntimeInterface, StateInterface,\n TensorInterface,\n};\nuse tract_rs::{State, Tensor};\n\n/// Used as a return type of functions that can encounter errors.\n/// If the function encountered an error, you can retrieve it using the `tract_get_last_error`\n/// function\n#[repr(C)]\n#[allow(non_camel_case_types)]\n#[derive(Debug, PartialEq, Eq)]\npub enum TRACT_RESULT {\n /// The function returned successfully\n TRACT_RESULT_OK = 0,\n /// The function returned an error\n TRACT_RESULT_KO = 1,\n}\n\nthread_local! {\n pub(crate) static LAST_ERROR: RefCell> = const { RefCell::new(None) };\n}\n\nfn wrap anyhow::Result<()>>(func: F) -> TRACT_RESULT {\n match func() {\n Ok(_) => TRACT_RESULT::TRACT_RESULT_OK,\n Err(e) => {\n let msg = format!(\"{e:?}\");\n if std::env::var(\"TRACT_ERROR_STDERR\").is_ok() {\n eprintln!(\"{msg}\");\n }\n LAST_ERROR.with(|p| {\n *p.borrow_mut() = Some(CString::new(msg).unwrap_or_else(|_| {\n CString::new(\"tract error message contains 0, can't convert to CString\")\n .unwrap()\n }))\n });\n TRACT_RESULT::TRACT_RESULT_KO\n }\n }\n}\n\n/// Retrieve the last error that happened in this thread. A function encountered an error if\n/// its return type is of type `TRACT_RESULT` and it returned `TRACT_RESULT_KO`.\n///\n/// # Return value\n/// It returns a pointer to a null-terminated UTF-8 string that will contain the error description.\n/// Rust side keeps ownership of the buffer. It will be valid as long as no other tract calls is\n/// performed by the thread.\n/// If no error occured, null is returned.\n#[unsafe(no_mangle)]\npub extern \"C\" fn tract_get_last_error() -> *const std::ffi::c_char {\n LAST_ERROR.with(|msg| msg.borrow().as_ref().map(|s| s.as_ptr()).unwrap_or(std::ptr::null()))\n}\n\n/// Returns a pointer to a static buffer containing a null-terminated version string.\n///\n/// The returned pointer must not be freed.\n#[unsafe(no_mangle)]\npub extern \"C\" fn tract_version() -> *const std::ffi::c_char {\n unsafe {\n CStr::from_bytes_with_nul_unchecked(concat!(env!(\"CARGO_PKG_VERSION\"), \"\\0\").as_bytes())\n .as_ptr()\n }\n}\n\n/// Frees a string allocated by libtract.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_free_cstring(ptr: *mut std::ffi::c_char) {\n unsafe {\n if !ptr.is_null() {\n let _ = CString::from_raw(ptr);\n }\n }\n}\n\nmacro_rules! check_not_null {\n ($($ptr:expr),*) => {\n $(\n if $ptr.is_null() {\n anyhow::bail!(concat!(\"Unexpected null pointer \", stringify!($ptr)));\n }\n )*\n }\n}\n\nmacro_rules! release {\n ($ptr:expr) => {\n wrap(|| unsafe {\n check_not_null!($ptr, *$ptr);\n let _ = Box::from_raw(*$ptr);\n *$ptr = std::ptr::null_mut();\n Ok(())\n })\n };\n}\n\n// NNEF\npub struct TractNnef(tract_rs::Nnef);\n\n/// Creates an instance of an NNEF framework and parser that can be used to load and dump NNEF models.\n///\n/// The returned object should be destroyed with `tract_nnef_destroy` once the model\n/// has been loaded.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_create(nnef: *mut *mut TractNnef) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n *nnef = Box::into_raw(Box::new(TractNnef(tract_rs::nnef()?)));\n Ok(())\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_enable_tract_core(nnef: *mut TractNnef) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n (*nnef).0.enable_tract_core()\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_enable_tract_extra(nnef: *mut TractNnef) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n (*nnef).0.enable_tract_extra()\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_enable_tract_transformers(\n nnef: *mut TractNnef,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n (*nnef).0.enable_tract_transformers()\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_enable_onnx(nnef: *mut TractNnef) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n (*nnef).0.enable_onnx()\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_enable_pulse(nnef: *mut TractNnef) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n (*nnef).0.enable_pulse()\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_enable_extended_identifier_syntax(\n nnef: *mut TractNnef,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n (*nnef).0.enable_extended_identifier_syntax()\n })\n}\n\n/// Destroy the NNEF parser. It is safe to detroy the NNEF parser once the model had been loaded.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_destroy(nnef: *mut *mut TractNnef) -> TRACT_RESULT {\n release!(nnef)\n}\n\n/// Parse and load an NNEF model as a tract TypedModel.\n///\n/// `path` is a null-terminated utf-8 string pointer. It can be an archive (tar or tar.gz file) or a\n/// directory.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_load(\n nnef: *const TractNnef,\n path: *const c_char,\n model: *mut *mut TractModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef, model, path);\n *model = std::ptr::null_mut();\n let path = CStr::from_ptr(path).to_str()?;\n let m = Box::new(TractModel(\n (*nnef).0.load(path).with_context(|| format!(\"opening file {path:?}\"))?,\n ));\n *model = Box::into_raw(m);\n Ok(())\n })\n}\n\n/// Parse and load an NNEF buffer as a tract TypedModel.\n///\n/// `data` is a buffer pointer\n/// `len` ise the buffer len\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_load_buffer(\n nnef: *const TractNnef,\n data: *const c_void,\n len: usize,\n model: *mut *mut TractModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef, model, data);\n *model = std::ptr::null_mut();\n let slice = std::slice::from_raw_parts(data as *const u8, len);\n let m = Box::new(TractModel((*nnef).0.load_buffer(slice)?));\n *model = Box::into_raw(m);\n Ok(())\n })\n}\n\n/// Dump a TypedModel as a NNEF tar file.\n///\n/// `path` is a null-terminated utf-8 string pointer to the `.tar` file to be created.\n///\n/// This function creates a plain, non-compressed, archive.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_write_model_to_tar(\n nnef: *const TractNnef,\n path: *const c_char,\n model: *const TractModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef, model, path);\n let path = CStr::from_ptr(path).to_str()?;\n (*nnef).0.write_model_to_tar(path, &(*model).0)?;\n Ok(())\n })\n}\n\n/// Dump a TypedModel as a NNEF .tar.gz file.\n///\n/// `path` is a null-terminated utf-8 string pointer to the `.tar.gz` file to be created.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_write_model_to_tar_gz(\n nnef: *const TractNnef,\n path: *const c_char,\n model: *const TractModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef, model, path);\n let path = CStr::from_ptr(path).to_str()?;\n (*nnef).0.write_model_to_tar_gz(path, &(*model).0)?;\n Ok(())\n })\n}\n\n/// Dump a TypedModel as a NNEF directory.\n///\n/// `path` is a null-terminated utf-8 string pointer to the directory to be created.\n///\n/// This function creates a plain, non-compressed, archive.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_nnef_write_model_to_dir(\n nnef: *const TractNnef,\n path: *const c_char,\n model: *const TractModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef, model, path);\n let path = CStr::from_ptr(path).to_str()?;\n (*nnef).0.write_model_to_dir(path, &(*model).0)?;\n Ok(())\n })\n}\n\n// ONNX\npub struct TractOnnx(tract_rs::Onnx);\n\n/// Creates an instance of an ONNX framework and parser that can be used to load models.\n///\n/// The returned object should be destroyed with `tract_nnef_destroy` once the model\n/// has been loaded.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_onnx_create(onnx: *mut *mut TractOnnx) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(onnx);\n *onnx = Box::into_raw(Box::new(TractOnnx(tract_rs::onnx()?)));\n Ok(())\n })\n}\n\n/// Destroy the NNEF parser. It is safe to detroy the NNEF parser once the model had been loaded.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_onnx_destroy(onnx: *mut *mut TractOnnx) -> TRACT_RESULT {\n release!(onnx)\n}\n\n/// Parse and load an ONNX model as a tract InferenceModel.\n///\n/// `path` is a null-terminated utf-8 string pointer. It must point to a `.onnx` model file.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_onnx_load(\n onnx: *const TractOnnx,\n path: *const c_char,\n model: *mut *mut TractInferenceModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(onnx, path, model);\n *model = std::ptr::null_mut();\n let path = CStr::from_ptr(path).to_str()?;\n let m = Box::new(TractInferenceModel((*onnx).0.load(path)?));\n *model = Box::into_raw(m);\n Ok(())\n })\n}\n\n/// Parse and load an ONNX buffer as a tract InferenceModel.\n///\n/// `data` is a buffer pointer\n/// `len` ise the buffer len\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_onnx_load_buffer(\n onnx: *const TractOnnx,\n data: *const c_void,\n len: usize,\n model: *mut *mut TractInferenceModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(onnx, model, data);\n *model = std::ptr::null_mut();\n let slice = std::slice::from_raw_parts(data as *const u8, len);\n let m = Box::new(TractInferenceModel((*onnx).0.load_buffer(slice)?));\n *model = Box::into_raw(m);\n Ok(())\n })\n}\n\n// INFERENCE MODEL\npub struct TractInferenceModel(tract_rs::InferenceModel);\n\n/// Query an InferenceModel input counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_input_count(\n model: *const TractInferenceModel,\n inputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, inputs);\n let model = &(*model).0;\n *inputs = model.input_count()?;\n Ok(())\n })\n}\n\n/// Query an InferenceModel output counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_output_count(\n model: *const TractInferenceModel,\n outputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, outputs);\n let model = &(*model).0;\n *outputs = model.output_count()?;\n Ok(())\n })\n}\n\n/// Query the name of a model input.\n///\n/// The returned name must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_input_name(\n model: *const TractInferenceModel,\n input: usize,\n name: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, name);\n *name = std::ptr::null_mut();\n let m = &(*model).0;\n *name = CString::new(&*m.input_name(input)?)?.into_raw();\n Ok(())\n })\n}\n\n/// Query the name of a model output.\n///\n/// The returned name must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_output_name(\n model: *const TractInferenceModel,\n output: usize,\n name: *mut *mut i8,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, name);\n *name = std::ptr::null_mut();\n let m = &(*model).0;\n *name = CString::new(&*m.output_name(output)?)?.into_raw() as _;\n Ok(())\n })\n}\n\n/// Query a model input fact.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_input_fact(\n model: *const TractInferenceModel,\n input_id: usize,\n fact: *mut *mut TractInferenceFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, fact);\n *fact = std::ptr::null_mut();\n let f = (*model).0.input_fact(input_id)?;\n *fact = Box::into_raw(Box::new(TractInferenceFact(f)));\n Ok(())\n })\n}\n\n/// Set an input fact of an InferenceModel.\n///\n/// The `fact` argument is only borrowed by this function, it still must be destroyed.\n/// `fact` can be set to NULL to erase the current output fact of the model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_set_input_fact(\n model: *mut TractInferenceModel,\n input_id: usize,\n fact: *const TractInferenceFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model);\n let f = fact.as_ref().map(|f| &f.0).cloned().unwrap_or_default();\n (*model).0.set_input_fact(input_id, f)?;\n Ok(())\n })\n}\n\n/// Query an output fact for an InferenceModel.\n///\n/// The return model must be freed using `tract_inference_fact_destroy`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_output_fact(\n model: *const TractInferenceModel,\n output_id: usize,\n fact: *mut *mut TractInferenceFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, fact);\n *fact = std::ptr::null_mut();\n let f = (*model).0.output_fact(output_id)?;\n *fact = Box::into_raw(Box::new(TractInferenceFact(f)));\n Ok(())\n })\n}\n\n/// Set an output fact of an InferenceModel.\n///\n/// The `fact` argument is only borrowed by this function, it still must be destroyed.\n/// `fact` can be set to NULL to erase the current output fact of the model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_set_output_fact(\n model: *mut TractInferenceModel,\n output_id: usize,\n fact: *const TractInferenceFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model);\n let f = fact.as_ref().map(|f| &f.0).cloned().unwrap_or_default();\n (*model).0.set_output_fact(output_id, f)?;\n Ok(())\n })\n}\n\n/// Analyse an InferencedModel in-place.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_analyse(\n model: *mut TractInferenceModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model);\n (*model).0.analyse()?;\n Ok(())\n })\n}\n\n/// Transform a fully analysed InferenceModel to a TypedModel.\n///\n/// This function takes ownership of the InferenceModel `model` whether it succeeds\n/// or not. `tract_inference_model_destroy` must not be used on `model`.\n///\n/// On the other hand, caller will be owning the newly created typed model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_into_model(\n model: *mut *mut TractInferenceModel,\n typed: *mut *mut TractModel,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, *model, typed);\n *typed = std::ptr::null_mut();\n let m = Box::from_raw(*model);\n *model = std::ptr::null_mut();\n let result = m.0.into_model()?;\n *typed = Box::into_raw(Box::new(TractModel(result))) as _;\n Ok(())\n })\n}\n\n/// Destroy an InferenceModel.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_model_destroy(\n model: *mut *mut TractInferenceModel,\n) -> TRACT_RESULT {\n release!(model)\n}\n// TYPED MODEL\n\npub struct TractModel(tract_rs::Model);\n\n/// Query an InferenceModel input counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_input_count(\n model: *const TractModel,\n inputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, inputs);\n let model = &(*model).0;\n *inputs = model.input_count()?;\n Ok(())\n })\n}\n\n/// Query an InferenceModel output counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_output_count(\n model: *const TractModel,\n outputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, outputs);\n let model = &(*model).0;\n *outputs = model.output_count()?;\n Ok(())\n })\n}\n\n/// Query the name of a model input.\n///\n/// The returned name must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_input_name(\n model: *const TractModel,\n input: usize,\n name: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, name);\n *name = std::ptr::null_mut();\n let m = &(*model).0;\n *name = CString::new(m.input_name(input)?)?.into_raw();\n Ok(())\n })\n}\n\n/// Query the input fact of a model.\n///\n/// Thre returned fact must be freed with tract_fact_destroy.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_input_fact(\n model: *const TractModel,\n input_id: usize,\n fact: *mut *mut TractFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, fact);\n *fact = std::ptr::null_mut();\n let f = (*model).0.input_fact(input_id)?;\n *fact = Box::into_raw(Box::new(TractFact(f)));\n Ok(())\n })\n}\n\n/// Query the name of a model output.\n///\n/// The returned name must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_output_name(\n model: *const TractModel,\n output: usize,\n name: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, name);\n *name = std::ptr::null_mut();\n let m = &(*model).0;\n *name = CString::new(m.output_name(output)?)?.into_raw();\n Ok(())\n })\n}\n\n/// Query the output fact of a model.\n///\n/// Thre returned fact must be freed with tract_fact_destroy.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_output_fact(\n model: *const TractModel,\n input_id: usize,\n fact: *mut *mut TractFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, fact);\n *fact = std::ptr::null_mut();\n let f = (*model).0.output_fact(input_id)?;\n *fact = Box::into_raw(Box::new(TractFact(f)));\n Ok(())\n })\n}\n\n/// Apply a transform to the model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_transform(\n model: *mut TractModel,\n transform: *const i8,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, transform);\n let t = CStr::from_ptr(transform as _)\n .to_str()\n .context(\"failed to parse transform name (not utf8)\")?;\n (*model).0.transform(t)\n })\n}\n\n/// Perform a profile of the model using the provided inputs.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_profile_json(\n model: *mut TractRunnable,\n inputs: *mut *mut TractTensor,\n json: *mut *mut i8,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, json);\n\n let input: Option> = if !inputs.is_null() {\n let input_len = (*model).0.input_count()?;\n Some(\n std::slice::from_raw_parts(inputs, input_len)\n .iter()\n .map(|tv| (**tv).0.clone())\n .collect(),\n )\n } else {\n None\n };\n\n let profile = (*model).0.profile_json(input)?;\n *json = CString::new(profile)?.into_raw() as _;\n Ok(())\n })\n}\n\n/// Convert a TypedModel into a TypedRunnableModel.\n///\n/// This function transfers ownership of the `model` argument to the newly-created `runnable` model.\n///\n/// Runnable are reference counted. When done, it should be released with `tract_runnable_release`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_into_runnable(\n model: *mut *mut TractModel,\n runnable: *mut *mut TractRunnable,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, runnable);\n let m = Box::from_raw(*model).0;\n *model = std::ptr::null_mut();\n *runnable = Box::into_raw(Box::new(TractRunnable(m.into_runnable()?))) as _;\n Ok(())\n })\n}\n\n/// Query the number of properties in a model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_property_count(\n model: *const TractModel,\n count: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, count);\n *count = (*model).0.property_keys()?.len();\n Ok(())\n })\n}\n\n/// Query the properties names of a model.\n///\n/// The \"names\" array should be big enough to fit `tract_model_property_count` string pointers.\n///\n/// Each name will have to be freed using `tract_free_cstring`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_property_names(\n model: *const TractModel,\n names: *mut *mut i8,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, names);\n for (ix, name) in (*model).0.property_keys()?.iter().enumerate() {\n *names.add(ix) = CString::new(&**name)?.into_raw() as _;\n }\n Ok(())\n })\n}\n\n/// Query a property tensor in a model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_property(\n model: *const TractModel,\n name: *const i8,\n tensor: *mut *mut TractTensor,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, name, tensor);\n let name = CStr::from_ptr(name as _)\n .to_str()\n .context(\"failed to parse property name (not utf8)\")?\n .to_owned();\n let v = (*model).0.property(name).context(\"Property not found\")?;\n *tensor = Box::into_raw(Box::new(TractTensor(v)));\n Ok(())\n })\n}\n\n/// Parse a fact specification string into an Fact.\n///\n/// The returned fact must be free with `tract_fact_destroy`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_parse_fact(\n model: *mut TractModel,\n spec: *const c_char,\n fact: *mut *mut TractFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, spec, fact);\n let spec = CStr::from_ptr(spec).to_str()?;\n let f: tract_rs::Fact = spec.as_fact(&(*model).0)?.as_ref().clone();\n *fact = Box::into_raw(Box::new(TractFact(f)));\n Ok(())\n })\n}\n\n/// Destroy a TypedModel.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_model_destroy(model: *mut *mut TractModel) -> TRACT_RESULT {\n release!(model)\n}\n\n// RUNTIME MODEL\npub struct TractRuntime(tract_rs::Runtime);\n\n/// Creates an instance of a tract Runtime that can be used to run model on a specific\n/// hardware / software stack (like a GPU).\n///\n/// The returned object should be released with `tract_runtime_release`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runtime_for_name(\n name: *const c_char,\n nnef: *mut *mut TractRuntime,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(nnef);\n let name = CStr::from_ptr(name).to_str()?;\n *nnef = Box::into_raw(Box::new(TractRuntime(tract_rs::runtime_for_name(name)?)));\n Ok(())\n })\n}\n\n/// Query the name of a Runtime.\n///\n/// The returned name must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runtime_name(\n runtime: *const TractRuntime,\n name: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(runtime, name);\n *name = std::ptr::null_mut();\n let n = (*runtime).0.name()?;\n *name = CString::new(n)?.into_raw();\n Ok(())\n })\n}\n\n/// Convert a Model into a Runnable for this Runtime.\n///\n/// This function transfers ownership of the `model` argument to the newly-created `runnable` model.\n///\n/// Runnable are reference counted. When done, it should be released with `tract_runnable_release`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runtime_prepare(\n runtime: *const TractRuntime,\n model: *mut *mut TractModel,\n runnable: *mut *mut TractRunnable,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(runtime, model, runnable);\n let m = Box::from_raw(*model).0;\n *model = std::ptr::null_mut();\n *runnable = Box::into_raw(Box::new(TractRunnable((*runtime).0.prepare(m)?))) as _;\n Ok(())\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runtime_release(runtime: *mut *mut TractRuntime) -> TRACT_RESULT {\n release!(runtime)\n}\n\n// RUNNABLE MODEL\npub struct TractRunnable(tract_rs::Runnable);\n\n/// Spawn a session state from a runnable model.\n///\n/// This function does not take ownership of the `runnable` object, it can be used again to spawn\n/// other state instances. The runnable object is internally reference counted, it will be\n/// kept alive as long as any associated `State` exists (or as long as the `runnable` is not\n/// explicitely release with `tract_runnable_release`).\n///\n/// `state` is a newly-created object. It should ultimately be detroyed with `tract_state_destroy`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_spawn_state(\n runnable: *mut TractRunnable,\n state: *mut *mut TractState,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(runnable, state);\n *state = std::ptr::null_mut();\n let s = (*runnable).0.spawn_state()?;\n *state = Box::into_raw(Box::new(TractState(s)));\n Ok(())\n })\n}\n\n/// Convenience function to run a stateless model.\n///\n/// `inputs` is a pointer to an pre-existing array of input TractTensor. Its length *must* be equal\n/// to the number of inputs of the models. The function does not take ownership of the input\n/// tensors.\n/// `outputs` is a pointer to a pre-existing array of TractTensor pointers that will be overwritten\n/// with pointers to output tensors. These tensors are under the responsiblity of the caller, it\n/// will have to release them with `tract_tensor_destroy`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_run(\n runnable: *mut TractRunnable,\n inputs: *mut *mut TractTensor,\n outputs: *mut *mut TractTensor,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(runnable);\n let mut s = (*runnable).0.spawn_state()?;\n state_run(&mut s, inputs, outputs)\n })\n}\n\n/// Query a Runnable input counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_input_count(\n model: *const TractRunnable,\n inputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, inputs);\n let model = &(*model).0;\n *inputs = model.input_count()?;\n Ok(())\n })\n}\n\n/// Query an Runnable output counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_output_count(\n model: *const TractRunnable,\n outputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, outputs);\n let model = &(*model).0;\n *outputs = model.output_count()?;\n Ok(())\n })\n}\n\n/// Query the input fact of a runnable model.\n///\n/// Thre returned fact must be freed with tract_fact_destroy.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_input_fact(\n runnable: *const TractRunnable,\n input_id: usize,\n fact: *mut *mut TractFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(runnable, fact);\n *fact = std::ptr::null_mut();\n let f = (*runnable).0.input_fact(input_id)?;\n *fact = Box::into_raw(Box::new(TractFact(f)));\n Ok(())\n })\n}\n\n/// Query the output fact of a runnable model.\n///\n/// Thre returned fact must be freed with tract_fact_destroy.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_output_fact(\n runnable: *const TractRunnable,\n output_id: usize,\n fact: *mut *mut TractFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(runnable, fact);\n *fact = std::ptr::null_mut();\n let f = (*runnable).0.output_fact(output_id)?;\n *fact = Box::into_raw(Box::new(TractFact(f)));\n Ok(())\n })\n}\n\n/// Query the number of properties in a runnable model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_property_count(\n model: *const TractRunnable,\n count: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, count);\n *count = (*model).0.property_keys()?.len();\n Ok(())\n })\n}\n\n/// Query the properties names of a runnable model.\n///\n/// The \"names\" array should be big enough to fit `tract_model_property_count` string pointers.\n///\n/// Each name will have to be freed using `tract_free_cstring`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_property_names(\n model: *const TractRunnable,\n names: *mut *mut i8,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, names);\n for (ix, name) in (*model).0.property_keys()?.iter().enumerate() {\n *names.add(ix) = CString::new(&**name)?.into_raw() as _;\n }\n Ok(())\n })\n}\n\n/// Query a property tensor in a runnable model.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_property(\n model: *const TractRunnable,\n name: *const i8,\n tensor: *mut *mut TractTensor,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, name, tensor);\n let name = CStr::from_ptr(name as _)\n .to_str()\n .context(\"failed to parse property name (not utf8)\")?\n .to_owned();\n let v = (*model).0.property(name).context(\"Property not found\")?;\n *tensor = Box::into_raw(Box::new(TractTensor(v)));\n Ok(())\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_runnable_release(runnable: *mut *mut TractRunnable) -> TRACT_RESULT {\n release!(runnable)\n}\n\n// TENSOR\npub struct TractTensor(tract_rs::Tensor);\n\n/// Create a TractTensor from caller data and metadata.\n///\n/// This call copies the data into tract space. All the pointers only need to be alive for the\n/// duration of the call.\n///\n/// rank is the number of dimensions of the tensor (i.e. the length of the shape vector).\n///\n/// The returned tensor must be destroyed by `tract_tensor_destroy`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_tensor_from_bytes(\n datum_type: DatumType,\n rank: usize,\n shape: *const usize,\n data: *mut c_void,\n tensor: *mut *mut TractTensor,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(tensor);\n *tensor = std::ptr::null_mut();\n let shape = std::slice::from_raw_parts(shape, rank);\n let len = shape.iter().product::();\n let data = std::slice::from_raw_parts(data as *const u8, len * datum_type.size_of());\n let it = Tensor::from_bytes(datum_type, shape, data)?;\n *tensor = Box::into_raw(Box::new(TractTensor(it)));\n Ok(())\n })\n}\n\n/// Write a tensor as a debug string\n///\n/// The returned string must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_tensor_dump(\n tensor: *const TractTensor,\n spec: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(tensor, spec);\n *spec = CString::new(format!(\"{:?}\", (*tensor).0))?.into_raw();\n Ok(())\n })\n}\n\n/// Convert a tensor to a new datum type.\n///\n/// This function will perform a cheap shallow clone if the destination type is\n/// the same as the current type, otherwise it returns a newly allocated Tensor instead.\n///\n/// In both cases, the returned tensor must be destroyed by `tract_tensor_destroy`.\n/// The input tensor is not consumed, it still need to be destroyed.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_tensor_convert_to(\n input: *const TractTensor,\n datum_type: DatumType,\n output: *mut *mut TractTensor,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(input, output);\n *output = std::ptr::null_mut();\n let new = (*input).0.convert_to(datum_type)?;\n *output = Box::into_raw(Box::new(TractTensor(new)));\n Ok(())\n })\n}\n\n/// Destroy a tensor.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_tensor_destroy(tensor: *mut *mut TractTensor) -> TRACT_RESULT {\n release!(tensor)\n}\n\n/// Inspect part of a tensor. Except `tensor`, all argument pointers can be null if only some specific bits\n/// are required.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_tensor_as_bytes(\n tensor: *mut TractTensor,\n datum_type: *mut DatumType,\n rank: *mut usize,\n shape: *mut *const usize,\n data: *mut *const c_void,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(tensor);\n let tensor = &(*tensor).0;\n let bits = tensor.as_bytes()?;\n if !datum_type.is_null() {\n *datum_type = bits.0;\n }\n if !rank.is_null() {\n *rank = bits.1.len();\n }\n if !shape.is_null() {\n *shape = bits.1.as_ptr();\n }\n if !data.is_null() {\n *data = bits.2.as_ptr() as _;\n }\n Ok(())\n })\n}\n\n// STATE\npub struct TractState(tract_rs::State);\n\n/// Run a turn on a model state\n///\n/// `inputs` is a pointer to an pre-existing array of input TractTensor. Its length *must* be equal\n/// to the number of inputs of the models. The function does not take ownership of the input\n/// tensors.\n/// `outputs` is a pointer to a pre-existing array of TractTensor pointers that will be overwritten\n/// with pointers to output tensors. These tensors are under the responsiblity of the caller, it\n/// will have to release them with `tract_tensor_destroy`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_state_run(\n state: *mut TractState,\n inputs: *mut *mut TractTensor,\n outputs: *mut *mut TractTensor,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(state, inputs, outputs);\n state_run(&mut (*state).0, inputs, outputs)\n })\n}\n\n/// Query a State input counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_state_input_count(\n state: *const TractState,\n inputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(state, inputs);\n let state = &(*state).0;\n *inputs = state.input_count()?;\n Ok(())\n })\n}\n\n/// Query an State output counts.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_state_output_count(\n state: *const TractState,\n outputs: *mut usize,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(state, outputs);\n let state = &(*state).0;\n *outputs = state.output_count()?;\n Ok(())\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_state_destroy(state: *mut *mut TractState) -> TRACT_RESULT {\n release!(state)\n}\n\n// FACT\npub struct TractFact(tract_rs::Fact);\n\n/// Gets the rank (aka number of axes/dimensions) of a fact.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_fact_rank(fact: *const TractFact, rank: *mut usize) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(fact, rank);\n *rank = (*fact).0.rank()?;\n Ok(())\n })\n}\n\n/// Extract the datum type of the fact.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_fact_datum_type(\n fact: *const TractFact,\n datum_type: *mut DatumType,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(fact, datum_type);\n *datum_type = (*fact).0.datum_type()?;\n Ok(())\n })\n}\n\n/// Extract the dimension from one dimension of the fact.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_fact_dim(\n fact: *const TractFact,\n axis: usize,\n dim: *mut *mut TractDim,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(fact, dim);\n let d = (*fact).0.dim(axis)?;\n *dim = Box::into_raw(Box::new(TractDim(d)));\n Ok(())\n })\n}\n\n/// Write a fact as its specification string.\n///\n/// The returned string must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_fact_dump(\n fact: *const TractFact,\n spec: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(fact, spec);\n *spec = CString::new(format!(\"{}\", (*fact).0))?.into_raw();\n Ok(())\n })\n}\n\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_fact_destroy(fact: *mut *mut TractFact) -> TRACT_RESULT {\n release!(fact)\n}\n\n// INFERENCE FACT\npub struct TractInferenceFact(tract_rs::InferenceFact);\n\n/// Parse a fact specification string into an InferenceFact.\n///\n/// The returned fact must be free with `tract_inference_fact_destroy`.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_fact_parse(\n model: *mut TractInferenceModel,\n spec: *const c_char,\n fact: *mut *mut TractInferenceFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(model, spec, fact);\n let spec = CStr::from_ptr(spec).to_str()?;\n let f: tract_rs::InferenceFact = spec.as_fact(&(*model).0)?.as_ref().clone();\n *fact = Box::into_raw(Box::new(TractInferenceFact(f)));\n Ok(())\n })\n}\n\n/// Creates an empty inference fact.\n///\n/// The returned fact must be freed by the caller using tract_inference_fact_destroy\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_fact_empty(\n fact: *mut *mut TractInferenceFact,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(fact);\n *fact = Box::into_raw(Box::new(TractInferenceFact(Default::default())));\n Ok(())\n })\n}\n\n/// Write an inference fact as its specification string.\n///\n/// The returned string must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_fact_dump(\n fact: *const TractInferenceFact,\n spec: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(fact, spec);\n *spec = CString::new(format!(\"{}\", (*fact).0))?.into_raw();\n Ok(())\n })\n}\n\n/// Destroy a fact.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_inference_fact_destroy(\n fact: *mut *mut TractInferenceFact,\n) -> TRACT_RESULT {\n release!(fact)\n}\n\n/// Dim\npub struct TractDim(tract_rs::Dim);\n\n/// Substitute symbols by the provided values in the Dim, generating a new one.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_dim_eval(\n dim: *const TractDim,\n nb_symbols: usize,\n symbols: *const *const i8,\n values: *const i64,\n result: *mut *mut TractDim,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(dim, symbols, values, result);\n let mut table = vec![];\n for i in 0..nb_symbols {\n let name = CStr::from_ptr(*symbols.add(i) as _)\n .to_str()\n .with_context(|| {\n format!(\"failed to parse symbol name for {i}th symbol (not utf8)\")\n })?\n .to_owned();\n table.push((name, *values.add(i)));\n }\n let r = (*dim).0.eval(table)?;\n *result = Box::into_raw(Box::new(TractDim(r)));\n Ok(())\n })\n}\n\n/// Try converting a Dim into an actual integer\n///\n/// Will fail if the Dim contains symbols.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_dim_to_int64(fact: *const TractDim, i: *mut i64) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(fact, i);\n *i = (*fact).0.to_int64()?;\n Ok(())\n })\n}\n\n/// Write a dim as its specification string.\n///\n/// The returned string must be freed by the caller using tract_free_cstring.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_dim_dump(\n dim: *const TractDim,\n spec: *mut *mut c_char,\n) -> TRACT_RESULT {\n wrap(|| unsafe {\n check_not_null!(dim, spec);\n *spec = CString::new((*dim).0.to_string())?.into_raw();\n Ok(())\n })\n}\n\n/// Destroy a dim.\n#[unsafe(no_mangle)]\npub unsafe extern \"C\" fn tract_dim_destroy(dim: *mut *mut TractDim) -> TRACT_RESULT {\n release!(dim)\n}\n\n// MISC\n\n// HELPERS\n\nunsafe fn state_run(\n state: &mut State,\n inputs: *mut *mut TractTensor,\n outputs: *mut *mut TractTensor,\n) -> Result<()> {\n unsafe {\n let values: Vec<_> = std::slice::from_raw_parts(inputs, state.input_count()?)\n .iter()\n .map(|tv| (**tv).0.clone())\n .collect();\n let values = state.run(values)?;\n for (i, value) in values.into_iter().enumerate() {\n *(outputs.add(i)) = Box::into_raw(Box::new(TractTensor(value)))\n }\n Ok(())\n }\n}\n" }, { "path": "api/generate-tract-h.sh", "content": "#!/bin/sh\n\nset -ex\n\ncargo install cbindgen\n\ncbindgen ffi > tract.h\ncp tract.h c\nmv tract.h proxy/sys\n" }, { "path": "api/proxy/Cargo.toml", "content": "[package]\nname = \"tract-proxy\"\nversion = \"0.23.0-pre\"\nlicense = \"MIT OR Apache-2.0\"\nauthors = [\"Mathieu Poumeyrol \"]\ndescription = \"Tiny, no-nonsense, self contained, TensorFlow and ONNX inference\"\nrepository = \"https://github.com/sonos/tract\"\nkeywords = [ \"NeuralNetworks\" ]\ncategories = [ \"science\" ]\nautobenches = false\nedition = \"2024\"\nrust-version.workspace = true\ninclude = [ \"Cargo.toml\", \"src/**/*.rs\", \"LICENSE*\" ]\n\n[dependencies]\nanyhow.workspace = true\nboow.workspace = true\nhome.workspace = true\nndarray.workspace = true\ntract-api.workspace = true\ntract-proxy-sys.workspace = true\n\n[dev-dependencies]\nreqwest.workspace = true\nrustls.workspace = true\ntempfile.workspace = true\nserde_json.workspace = true\n" }, { "path": "api/proxy/LICENSE", "content": "## License\n\nLicensed under either of\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "path": "api/proxy/LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "api/proxy/LICENSE-MIT", "content": "Permission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "api/proxy/ci.sh", "content": "#!/bin/sh\n\nROOT=$(dirname $(realpath $0))/../..\n\nset -ex\n\ncargo build --release -p tract-ffi $CARGO_EXTRA\nSO=$(cargo build --message-format=json --release -p tract-ffi $CARGO_EXTRA | grep cdylib | jshon -e filenames -e 0 -u)\nSO_PATH=$(dirname $SO)\nexport TRACT_DYLIB_SEARCH_PATH=$SO_PATH\nexport LD_LIBRARY_PATH=$SO_PATH\n\ncd $(dirname $(realpath $0))\ncargo test $CARGO_EXTRA\n" }, { "path": "api/proxy/src/lib.rs", "content": "use std::ffi::{CStr, CString};\nuse std::path::Path;\nuse std::ptr::{null, null_mut};\n\nuse tract_api::*;\nuse tract_proxy_sys as sys;\n\nuse anyhow::{Context, Result};\nuse ndarray::*;\n\nmacro_rules! check {\n ($expr:expr) => {\n unsafe {\n if $expr == sys::TRACT_RESULT_TRACT_RESULT_KO {\n let buf = CStr::from_ptr(sys::tract_get_last_error());\n Err(anyhow::anyhow!(buf.to_string_lossy().to_string()))\n } else {\n Ok(())\n }\n }\n };\n}\n\nmacro_rules! wrapper {\n ($new_type:ident, $c_type:ident, $dest:ident $(, $typ:ty )*) => {\n #[derive(Debug, Clone)]\n pub struct $new_type(*mut sys::$c_type $(, $typ)*);\n\n impl Drop for $new_type {\n fn drop(&mut self) {\n unsafe {\n sys::$dest(&mut self.0);\n }\n }\n }\n };\n}\n\npub fn nnef() -> Result {\n let mut nnef = null_mut();\n check!(sys::tract_nnef_create(&mut nnef))?;\n Ok(Nnef(nnef))\n}\n\npub fn onnx() -> Result {\n let mut onnx = null_mut();\n check!(sys::tract_onnx_create(&mut onnx))?;\n Ok(Onnx(onnx))\n}\n\npub fn version() -> &'static str {\n unsafe { CStr::from_ptr(sys::tract_version()).to_str().unwrap() }\n}\n\nwrapper!(Nnef, TractNnef, tract_nnef_destroy);\nimpl NnefInterface for Nnef {\n type Model = Model;\n fn load(&self, path: impl AsRef) -> Result {\n let path = path.as_ref();\n let path = CString::new(\n path.to_str().with_context(|| format!(\"Failed to re-encode {path:?} to uff-8\"))?,\n )?;\n let mut model = null_mut();\n check!(sys::tract_nnef_load(self.0, path.as_ptr(), &mut model))?;\n Ok(Model(model))\n }\n\n fn load_buffer(&self, data: &[u8]) -> Result {\n let mut model = null_mut();\n check!(sys::tract_nnef_load_buffer(self.0, data.as_ptr() as _, data.len(), &mut model))?;\n Ok(Model(model))\n }\n\n fn enable_tract_core(&mut self) -> Result<()> {\n check!(sys::tract_nnef_enable_tract_core(self.0))\n }\n\n fn enable_tract_extra(&mut self) -> Result<()> {\n check!(sys::tract_nnef_enable_tract_extra(self.0))\n }\n\n fn enable_tract_transformers(&mut self) -> Result<()> {\n check!(sys::tract_nnef_enable_tract_transformers(self.0))\n }\n\n fn enable_onnx(&mut self) -> Result<()> {\n check!(sys::tract_nnef_enable_onnx(self.0))\n }\n\n fn enable_pulse(&mut self) -> Result<()> {\n check!(sys::tract_nnef_enable_pulse(self.0))\n }\n\n fn enable_extended_identifier_syntax(&mut self) -> Result<()> {\n check!(sys::tract_nnef_enable_extended_identifier_syntax(self.0))\n }\n\n fn write_model_to_dir(&self, path: impl AsRef, model: &Model) -> Result<()> {\n let path = path.as_ref();\n let path = CString::new(\n path.to_str().with_context(|| format!(\"Failed to re-encode {path:?} to uff-8\"))?,\n )?;\n check!(sys::tract_nnef_write_model_to_dir(self.0, path.as_ptr(), model.0))?;\n Ok(())\n }\n\n fn write_model_to_tar(&self, path: impl AsRef, model: &Model) -> Result<()> {\n let path = path.as_ref();\n let path = CString::new(\n path.to_str().with_context(|| format!(\"Failed to re-encode {path:?} to uff-8\"))?,\n )?;\n check!(sys::tract_nnef_write_model_to_tar(self.0, path.as_ptr(), model.0))?;\n Ok(())\n }\n\n fn write_model_to_tar_gz(&self, path: impl AsRef, model: &Model) -> Result<()> {\n let path = path.as_ref();\n let path = CString::new(\n path.to_str().with_context(|| format!(\"Failed to re-encode {path:?} to uff-8\"))?,\n )?;\n check!(sys::tract_nnef_write_model_to_tar_gz(self.0, path.as_ptr(), model.0))?;\n Ok(())\n }\n}\n\n// ONNX\nwrapper!(Onnx, TractOnnx, tract_onnx_destroy);\n\nimpl OnnxInterface for Onnx {\n type InferenceModel = InferenceModel;\n fn load(&self, path: impl AsRef) -> Result {\n let path = path.as_ref();\n let path = CString::new(\n path.to_str().with_context(|| format!(\"Failed to re-encode {path:?} to uff-8\"))?,\n )?;\n let mut model = null_mut();\n check!(sys::tract_onnx_load(self.0, path.as_ptr(), &mut model))?;\n Ok(InferenceModel(model))\n }\n\n fn load_buffer(&self, data: &[u8]) -> Result {\n let mut model = null_mut();\n check!(sys::tract_onnx_load_buffer(self.0, data.as_ptr() as _, data.len(), &mut model))?;\n Ok(InferenceModel(model))\n }\n}\n\n// INFERENCE MODEL\nwrapper!(InferenceModel, TractInferenceModel, tract_inference_model_destroy);\nimpl InferenceModelInterface for InferenceModel {\n type Model = Model;\n type InferenceFact = InferenceFact;\n fn input_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_inference_model_input_count(self.0, &mut count))?;\n Ok(count)\n }\n\n fn output_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_inference_model_output_count(self.0, &mut count))?;\n Ok(count)\n }\n\n fn input_name(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_inference_model_input_name(self.0, id, &mut ptr))?;\n unsafe {\n let ret = CStr::from_ptr(ptr).to_str()?.to_owned();\n sys::tract_free_cstring(ptr);\n Ok(ret)\n }\n }\n\n fn output_name(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_inference_model_output_name(self.0, id, &mut ptr))?;\n unsafe {\n let ret = CStr::from_ptr(ptr).to_str()?.to_owned();\n sys::tract_free_cstring(ptr);\n Ok(ret)\n }\n }\n\n fn input_fact(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_inference_model_input_fact(self.0, id, &mut ptr))?;\n Ok(InferenceFact(ptr))\n }\n\n fn set_input_fact(\n &mut self,\n id: usize,\n fact: impl AsFact,\n ) -> Result<()> {\n let fact = fact.as_fact(self)?;\n check!(sys::tract_inference_model_set_input_fact(self.0, id, fact.0))?;\n Ok(())\n }\n\n fn output_fact(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_inference_model_output_fact(self.0, id, &mut ptr))?;\n Ok(InferenceFact(ptr))\n }\n\n fn set_output_fact(\n &mut self,\n id: usize,\n fact: impl AsFact,\n ) -> Result<()> {\n let fact = fact.as_fact(self)?;\n check!(sys::tract_inference_model_set_output_fact(self.0, id, fact.0))?;\n Ok(())\n }\n\n fn analyse(&mut self) -> Result<()> {\n check!(sys::tract_inference_model_analyse(self.0))?;\n Ok(())\n }\n\n fn into_model(mut self) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_inference_model_into_model(&mut self.0, &mut ptr))?;\n Ok(Model(ptr))\n }\n}\n\n// MODEL\nwrapper!(Model, TractModel, tract_model_destroy);\n\nimpl ModelInterface for Model {\n type Fact = Fact;\n type Tensor = Tensor;\n type Runnable = Runnable;\n fn input_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_model_input_count(self.0, &mut count))?;\n Ok(count)\n }\n\n fn output_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_model_output_count(self.0, &mut count))?;\n Ok(count)\n }\n\n fn input_name(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_model_input_name(self.0, id, &mut ptr))?;\n unsafe {\n let ret = CStr::from_ptr(ptr).to_str()?.to_owned();\n sys::tract_free_cstring(ptr);\n Ok(ret)\n }\n }\n\n fn output_name(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_model_output_name(self.0, id, &mut ptr))?;\n unsafe {\n let ret = CStr::from_ptr(ptr).to_str()?.to_owned();\n sys::tract_free_cstring(ptr);\n Ok(ret)\n }\n }\n\n fn input_fact(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_model_input_fact(self.0, id, &mut ptr))?;\n Ok(Fact(ptr))\n }\n\n fn output_fact(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_model_output_fact(self.0, id, &mut ptr))?;\n Ok(Fact(ptr))\n }\n\n fn into_runnable(self) -> Result {\n let mut model = self;\n let mut runnable = null_mut();\n check!(sys::tract_model_into_runnable(&mut model.0, &mut runnable))?;\n Ok(Runnable(runnable))\n }\n\n fn transform(&mut self, spec: impl Into) -> Result<()> {\n let transform = spec.into().to_transform_string();\n let t = CString::new(transform)?;\n check!(sys::tract_model_transform(self.0, t.as_ptr()))?;\n Ok(())\n }\n\n fn property_keys(&self) -> Result> {\n let mut len = 0;\n check!(sys::tract_model_property_count(self.0, &mut len))?;\n let mut keys = vec![null_mut(); len];\n check!(sys::tract_model_property_names(self.0, keys.as_mut_ptr()))?;\n unsafe {\n keys.into_iter()\n .map(|pc| {\n let s = CStr::from_ptr(pc).to_str()?.to_owned();\n sys::tract_free_cstring(pc);\n Ok(s)\n })\n .collect()\n }\n }\n\n fn property(&self, name: impl AsRef) -> Result {\n let mut v = null_mut();\n let name = CString::new(name.as_ref())?;\n check!(sys::tract_model_property(self.0, name.as_ptr(), &mut v))?;\n Ok(Tensor(v))\n }\n\n fn parse_fact(&self, spec: &str) -> Result {\n let spec = CString::new(spec)?;\n let mut ptr = null_mut();\n check!(sys::tract_model_parse_fact(self.0, spec.as_ptr(), &mut ptr))?;\n Ok(Fact(ptr))\n }\n}\n\n// RUNTIME\nwrapper!(Runtime, TractRuntime, tract_runtime_release);\n\npub fn runtime_for_name(name: &str) -> Result {\n let mut rt = null_mut();\n let name = CString::new(name)?;\n check!(sys::tract_runtime_for_name(name.as_ptr(), &mut rt))?;\n Ok(Runtime(rt))\n}\n\nimpl RuntimeInterface for Runtime {\n type Runnable = Runnable;\n\n type Model = Model;\n\n fn name(&self) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_runtime_name(self.0, &mut ptr))?;\n unsafe {\n let ret = CStr::from_ptr(ptr).to_str()?.to_owned();\n sys::tract_free_cstring(ptr);\n Ok(ret)\n }\n }\n\n fn prepare(&self, model: Self::Model) -> Result {\n let mut model = model;\n let mut runnable = null_mut();\n check!(sys::tract_runtime_prepare(self.0, &mut model.0, &mut runnable))?;\n Ok(Runnable(runnable))\n }\n}\n\n// RUNNABLE\nwrapper!(Runnable, TractRunnable, tract_runnable_release);\nunsafe impl Send for Runnable {}\nunsafe impl Sync for Runnable {}\n\nimpl RunnableInterface for Runnable {\n type Tensor = Tensor;\n type State = State;\n type Fact = Fact;\n\n fn run(&self, inputs: impl IntoInputs) -> Result> {\n StateInterface::run(&mut self.spawn_state()?, inputs.into_inputs()?)\n }\n\n fn spawn_state(&self) -> Result {\n let mut state = null_mut();\n check!(sys::tract_runnable_spawn_state(self.0, &mut state))?;\n Ok(State(state))\n }\n\n fn input_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_runnable_input_count(self.0, &mut count))?;\n Ok(count)\n }\n\n fn output_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_runnable_output_count(self.0, &mut count))?;\n Ok(count)\n }\n\n fn input_fact(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_runnable_input_fact(self.0, id, &mut ptr))?;\n Ok(Fact(ptr))\n }\n\n fn output_fact(&self, id: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_runnable_output_fact(self.0, id, &mut ptr))?;\n Ok(Fact(ptr))\n }\n\n fn property_keys(&self) -> Result> {\n let mut len = 0;\n check!(sys::tract_runnable_property_count(self.0, &mut len))?;\n let mut keys = vec![null_mut(); len];\n check!(sys::tract_runnable_property_names(self.0, keys.as_mut_ptr()))?;\n unsafe {\n keys.into_iter()\n .map(|pc| {\n let s = CStr::from_ptr(pc).to_str()?.to_owned();\n sys::tract_free_cstring(pc);\n Ok(s)\n })\n .collect()\n }\n }\n\n fn property(&self, name: impl AsRef) -> Result {\n let mut v = null_mut();\n let name = CString::new(name.as_ref())?;\n check!(sys::tract_runnable_property(self.0, name.as_ptr(), &mut v))?;\n Ok(Tensor(v))\n }\n\n fn cost_json(&self) -> Result {\n let input: Option> = None;\n self.profile_json(input)\n }\n\n fn profile_json(&self, inputs: Option) -> Result\n where\n I: IntoIterator,\n IV: TryInto,\n IE: Into,\n {\n let inputs = if let Some(inputs) = inputs {\n let inputs = inputs\n .into_iter()\n .map(|i| i.try_into().map_err(|e| e.into()))\n .collect::>>()?;\n anyhow::ensure!(self.input_count()? == inputs.len());\n Some(inputs)\n } else {\n None\n };\n let mut iptrs: Option> =\n inputs.as_ref().map(|is| is.iter().map(|v| v.0).collect());\n let mut json: *mut i8 = null_mut();\n let values = iptrs.as_mut().map(|it| it.as_mut_ptr()).unwrap_or(null_mut());\n\n check!(sys::tract_runnable_profile_json(self.0, values, &mut json))?;\n anyhow::ensure!(!json.is_null());\n unsafe {\n let s = CStr::from_ptr(json).to_owned();\n sys::tract_free_cstring(json);\n Ok(s.to_str()?.to_owned())\n }\n }\n}\n\n// STATE\nwrapper!(State, TractState, tract_state_destroy);\n\nimpl StateInterface for State {\n type Tensor = Tensor;\n type Fact = Fact;\n\n fn run(&mut self, inputs: impl IntoInputs) -> Result> {\n let inputs = inputs.into_inputs()?;\n let mut outputs = vec![null_mut(); self.output_count()?];\n let mut inputs: Vec<_> = inputs.iter().map(|v| v.0).collect();\n check!(sys::tract_state_run(self.0, inputs.as_mut_ptr(), outputs.as_mut_ptr()))?;\n let outputs = outputs.into_iter().map(Tensor).collect();\n Ok(outputs)\n }\n\n fn input_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_state_input_count(self.0, &mut count))?;\n Ok(count)\n }\n\n fn output_count(&self) -> Result {\n let mut count = 0;\n check!(sys::tract_state_output_count(self.0, &mut count))?;\n Ok(count)\n }\n}\n\n// TENSOR\nwrapper!(Tensor, TractTensor, tract_tensor_destroy);\nunsafe impl Send for Tensor {}\nunsafe impl Sync for Tensor {}\n\nimpl TensorInterface for Tensor {\n fn from_bytes(dt: DatumType, shape: &[usize], data: &[u8]) -> Result {\n anyhow::ensure!(data.len() == shape.iter().product::() * dt.size_of());\n let mut value = null_mut();\n check!(sys::tract_tensor_from_bytes(\n dt as _,\n shape.len(),\n shape.as_ptr(),\n data.as_ptr() as _,\n &mut value\n ))?;\n Ok(Tensor(value))\n }\n\n fn as_bytes(&self) -> Result<(DatumType, &[usize], &[u8])> {\n let mut rank = 0;\n let mut dt = sys::DatumType_TRACT_DATUM_TYPE_BOOL as _;\n let mut shape = null();\n let mut data = null();\n check!(sys::tract_tensor_as_bytes(self.0, &mut dt, &mut rank, &mut shape, &mut data))?;\n unsafe {\n let dt: DatumType = std::mem::transmute(dt);\n let shape = std::slice::from_raw_parts(shape, rank);\n let len: usize = shape.iter().product();\n let data = std::slice::from_raw_parts(data as *const u8, len * dt.size_of());\n Ok((dt, shape, data))\n }\n }\n\n fn datum_type(&self) -> Result {\n let mut dt = sys::DatumType_TRACT_DATUM_TYPE_BOOL as _;\n check!(sys::tract_tensor_as_bytes(\n self.0,\n &mut dt,\n std::ptr::null_mut(),\n std::ptr::null_mut(),\n std::ptr::null_mut()\n ))?;\n unsafe {\n let dt: DatumType = std::mem::transmute(dt);\n Ok(dt)\n }\n }\n\n fn convert_to(&self, to: DatumType) -> Result {\n let mut new = null_mut();\n check!(sys::tract_tensor_convert_to(self.0, to as _, &mut new))?;\n Ok(Tensor(new))\n }\n}\n\nimpl PartialEq for Tensor {\n fn eq(&self, other: &Self) -> bool {\n let Ok((me_dt, me_shape, me_data)) = self.as_bytes() else { return false };\n let Ok((other_dt, other_shape, other_data)) = other.as_bytes() else { return false };\n me_dt == other_dt && me_shape == other_shape && me_data == other_data\n }\n}\n\ntensor_from_to_ndarray!();\n\n// FACT\nwrapper!(Fact, TractFact, tract_fact_destroy);\n\nimpl Fact {\n fn new(model: &Model, spec: impl ToString) -> Result {\n let cstr = CString::new(spec.to_string())?;\n let mut fact = null_mut();\n check!(sys::tract_model_parse_fact(model.0, cstr.as_ptr(), &mut fact))?;\n Ok(Fact(fact))\n }\n\n fn dump(&self) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_fact_dump(self.0, &mut ptr))?;\n unsafe {\n let s = CStr::from_ptr(ptr).to_owned();\n sys::tract_free_cstring(ptr);\n Ok(s.to_str()?.to_owned())\n }\n }\n}\n\nimpl FactInterface for Fact {\n type Dim = Dim;\n\n fn datum_type(&self) -> Result {\n let mut dt = 0u32;\n check!(sys::tract_fact_datum_type(self.0, &mut dt as *const u32 as _))?;\n Ok(unsafe { std::mem::transmute::(dt) })\n }\n\n fn rank(&self) -> Result {\n let mut rank = 0;\n check!(sys::tract_fact_rank(self.0, &mut rank))?;\n Ok(rank)\n }\n\n fn dim(&self, axis: usize) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_fact_dim(self.0, axis, &mut ptr))?;\n Ok(Dim(ptr))\n }\n}\n\nimpl std::fmt::Display for Fact {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self.dump() {\n Ok(s) => f.write_str(&s),\n Err(_) => Err(std::fmt::Error),\n }\n }\n}\n\n// INFERENCE FACT\nwrapper!(InferenceFact, TractInferenceFact, tract_inference_fact_destroy);\n\nimpl InferenceFact {\n fn new(model: &InferenceModel, spec: impl ToString) -> Result {\n let cstr = CString::new(spec.to_string())?;\n let mut fact = null_mut();\n check!(sys::tract_inference_fact_parse(model.0, cstr.as_ptr(), &mut fact))?;\n Ok(InferenceFact(fact))\n }\n\n fn dump(&self) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_inference_fact_dump(self.0, &mut ptr))?;\n unsafe {\n let s = CStr::from_ptr(ptr).to_owned();\n sys::tract_free_cstring(ptr);\n Ok(s.to_str()?.to_owned())\n }\n }\n}\n\nimpl InferenceFactInterface for InferenceFact {\n fn empty() -> Result {\n let mut fact = null_mut();\n check!(sys::tract_inference_fact_empty(&mut fact))?;\n Ok(InferenceFact(fact))\n }\n}\n\nimpl Default for InferenceFact {\n fn default() -> Self {\n Self::empty().unwrap()\n }\n}\n\nimpl std::fmt::Display for InferenceFact {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self.dump() {\n Ok(s) => f.write_str(&s),\n Err(_) => Err(std::fmt::Error),\n }\n }\n}\n\nas_inference_fact_impl!(InferenceModel, InferenceFact);\nas_fact_impl!(Model, Fact);\n\n// Dim\nwrapper!(Dim, TractDim, tract_dim_destroy);\n\nimpl Dim {\n fn dump(&self) -> Result {\n let mut ptr = null_mut();\n check!(sys::tract_dim_dump(self.0, &mut ptr))?;\n unsafe {\n let s = CStr::from_ptr(ptr).to_owned();\n sys::tract_free_cstring(ptr);\n Ok(s.to_str()?.to_owned())\n }\n }\n}\n\nimpl DimInterface for Dim {\n fn eval(&self, values: impl IntoIterator, i64)>) -> Result {\n let (names, values): (Vec<_>, Vec<_>) = values.into_iter().unzip();\n let c_strings: Vec =\n names.into_iter().map(|a| Ok(CString::new(a.as_ref())?)).collect::>()?;\n let ptrs: Vec<_> = c_strings.iter().map(|cs| cs.as_ptr()).collect();\n let mut ptr = null_mut();\n check!(sys::tract_dim_eval(self.0, ptrs.len(), ptrs.as_ptr(), values.as_ptr(), &mut ptr))?;\n Ok(Dim(ptr))\n }\n\n fn to_int64(&self) -> Result {\n let mut i = 0;\n check!(sys::tract_dim_to_int64(self.0, &mut i))?;\n Ok(i)\n }\n}\n\nimpl std::fmt::Display for Dim {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self.dump() {\n Ok(s) => f.write_str(&s),\n Err(_) => Err(std::fmt::Error),\n }\n }\n}\n" }, { "path": "api/proxy/sys/Cargo.toml", "content": "[package]\nname = \"tract-proxy-sys\"\nversion = \"0.23.0-pre\"\nlicense = \"MIT OR Apache-2.0\"\nauthors = [\"Mathieu Poumeyrol \"]\ndescription = \"Tiny, no-nonsense, self contained, TensorFlow and ONNX inference\"\nrepository = \"https://github.com/sonos/tract\"\nkeywords = [ \"NeuralNetworks\" ]\ncategories = [ \"science\" ]\nautobenches = false\nedition = \"2024\"\nrust-version.workspace = true\ninclude = [ \"Cargo.toml\", \"src/**/*.rs\", \"LICENSE*\", \"build.rs\", \"tract.h\" ]\n\n[build-dependencies]\nbindgen = \"0.72.1\"\n" }, { "path": "api/proxy/sys/build.rs", "content": "use std::env;\nuse std::path::PathBuf;\n\nfn main() {\n println!(\"cargo:rerun-if-env-changed=TRACT_DYLIB_SEARCH_PATH\");\n println!(\"cargo:rerun-if-changed=tract.h\");\n if let Ok(path) = std::env::var(\"TRACT_DYLIB_SEARCH_PATH\") {\n println!(\"cargo:rustc-link-search={path}\");\n }\n println!(\"cargo:rustc-link-lib=tract\");\n\n let bindings = bindgen::Builder::default()\n .header(\"tract.h\")\n .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))\n .generate()\n .expect(\"Unable to generate bindings\");\n\n let out_path = PathBuf::from(env::var(\"OUT_DIR\").unwrap());\n bindings.write_to_file(out_path.join(\"bindings.rs\")).expect(\"Couldn't write bindings!\");\n}\n" }, { "path": "api/proxy/sys/src/lib.rs", "content": "#![allow(non_upper_case_globals)]\n#![allow(non_camel_case_types)]\n#![allow(non_snake_case)]\n#![allow(improper_ctypes)]\n#![allow(deref_nullptr)]\n#![allow(unsafe_op_in_unsafe_fn)]\n#![allow(clippy::redundant_static_lifetimes)]\n#![allow(clippy::useless_transmute)]\n\ninclude!(concat!(env!(\"OUT_DIR\"), \"/bindings.rs\"));\n" }, { "path": "api/proxy/sys/tract.h", "content": "#include \n#include \n#include \n#include \n typedef enum DatumType {\n TRACT_DATUM_TYPE_BOOL = 1,\n TRACT_DATUM_TYPE_U8 = 17,\n TRACT_DATUM_TYPE_U16 = 18,\n TRACT_DATUM_TYPE_U32 = 20,\n TRACT_DATUM_TYPE_U64 = 24,\n TRACT_DATUM_TYPE_I8 = 33,\n TRACT_DATUM_TYPE_I16 = 34,\n TRACT_DATUM_TYPE_I32 = 36,\n TRACT_DATUM_TYPE_I64 = 40,\n TRACT_DATUM_TYPE_F16 = 50,\n TRACT_DATUM_TYPE_F32 = 52,\n TRACT_DATUM_TYPE_F64 = 56,\n TRACT_DATUM_TYPE_COMPLEX_I16 = 66,\n TRACT_DATUM_TYPE_COMPLEX_I32 = 68,\n TRACT_DATUM_TYPE_COMPLEX_I64 = 72,\n TRACT_DATUM_TYPE_COMPLEX_F16 = 82,\n TRACT_DATUM_TYPE_COMPLEX_F32 = 84,\n TRACT_DATUM_TYPE_COMPLEX_F64 = 88,\n } DatumType;\n\n\n/**\n * Used as a return type of functions that can encounter errors.\n * If the function encountered an error, you can retrieve it using the `tract_get_last_error`\n * function\n */\ntypedef enum TRACT_RESULT {\n /**\n * The function returned successfully\n */\n TRACT_RESULT_OK = 0,\n /**\n * The function returned an error\n */\n TRACT_RESULT_KO = 1,\n} TRACT_RESULT;\n\n/**\n * Dim\n */\ntypedef struct TractDim TractDim;\n\ntypedef struct TractFact TractFact;\n\ntypedef struct TractInferenceFact TractInferenceFact;\n\ntypedef struct TractInferenceModel TractInferenceModel;\n\ntypedef struct TractModel TractModel;\n\ntypedef struct TractNnef TractNnef;\n\ntypedef struct TractOnnx TractOnnx;\n\ntypedef struct TractRunnable TractRunnable;\n\ntypedef struct TractRuntime TractRuntime;\n\ntypedef struct TractState TractState;\n\ntypedef struct TractTensor TractTensor;\n\n/**\n * Retrieve the last error that happened in this thread. A function encountered an error if\n * its return type is of type `TRACT_RESULT` and it returned `TRACT_RESULT_KO`.\n *\n * # Return value\n * It returns a pointer to a null-terminated UTF-8 string that will contain the error description.\n * Rust side keeps ownership of the buffer. It will be valid as long as no other tract calls is\n * performed by the thread.\n * If no error occured, null is returned.\n */\nconst char *tract_get_last_error(void);\n\n/**\n * Returns a pointer to a static buffer containing a null-terminated version string.\n *\n * The returned pointer must not be freed.\n */\nconst char *tract_version(void);\n\n/**\n * Frees a string allocated by libtract.\n */\nvoid tract_free_cstring(char *ptr);\n\n/**\n * Creates an instance of an NNEF framework and parser that can be used to load and dump NNEF models.\n *\n * The returned object should be destroyed with `tract_nnef_destroy` once the model\n * has been loaded.\n */\nenum TRACT_RESULT tract_nnef_create(struct TractNnef **nnef);\n\nenum TRACT_RESULT tract_nnef_enable_tract_core(struct TractNnef *nnef);\n\nenum TRACT_RESULT tract_nnef_enable_tract_extra(struct TractNnef *nnef);\n\nenum TRACT_RESULT tract_nnef_enable_tract_transformers(struct TractNnef *nnef);\n\nenum TRACT_RESULT tract_nnef_enable_onnx(struct TractNnef *nnef);\n\nenum TRACT_RESULT tract_nnef_enable_pulse(struct TractNnef *nnef);\n\nenum TRACT_RESULT tract_nnef_enable_extended_identifier_syntax(struct TractNnef *nnef);\n\n/**\n * Destroy the NNEF parser. It is safe to detroy the NNEF parser once the model had been loaded.\n */\nenum TRACT_RESULT tract_nnef_destroy(struct TractNnef **nnef);\n\n/**\n * Parse and load an NNEF model as a tract TypedModel.\n *\n * `path` is a null-terminated utf-8 string pointer. It can be an archive (tar or tar.gz file) or a\n * directory.\n */\nenum TRACT_RESULT tract_nnef_load(const struct TractNnef *nnef,\n const char *path,\n struct TractModel **model);\n\n/**\n * Parse and load an NNEF buffer as a tract TypedModel.\n *\n * `data` is a buffer pointer\n * `len` ise the buffer len\n */\nenum TRACT_RESULT tract_nnef_load_buffer(const struct TractNnef *nnef,\n const void *data,\n uintptr_t len,\n struct TractModel **model);\n\n/**\n * Dump a TypedModel as a NNEF tar file.\n *\n * `path` is a null-terminated utf-8 string pointer to the `.tar` file to be created.\n *\n * This function creates a plain, non-compressed, archive.\n */\nenum TRACT_RESULT tract_nnef_write_model_to_tar(const struct TractNnef *nnef,\n const char *path,\n const struct TractModel *model);\n\n/**\n * Dump a TypedModel as a NNEF .tar.gz file.\n *\n * `path` is a null-terminated utf-8 string pointer to the `.tar.gz` file to be created.\n */\nenum TRACT_RESULT tract_nnef_write_model_to_tar_gz(const struct TractNnef *nnef,\n const char *path,\n const struct TractModel *model);\n\n/**\n * Dump a TypedModel as a NNEF directory.\n *\n * `path` is a null-terminated utf-8 string pointer to the directory to be created.\n *\n * This function creates a plain, non-compressed, archive.\n */\nenum TRACT_RESULT tract_nnef_write_model_to_dir(const struct TractNnef *nnef,\n const char *path,\n const struct TractModel *model);\n\n/**\n * Creates an instance of an ONNX framework and parser that can be used to load models.\n *\n * The returned object should be destroyed with `tract_nnef_destroy` once the model\n * has been loaded.\n */\nenum TRACT_RESULT tract_onnx_create(struct TractOnnx **onnx);\n\n/**\n * Destroy the NNEF parser. It is safe to detroy the NNEF parser once the model had been loaded.\n */\nenum TRACT_RESULT tract_onnx_destroy(struct TractOnnx **onnx);\n\n/**\n * Parse and load an ONNX model as a tract InferenceModel.\n *\n * `path` is a null-terminated utf-8 string pointer. It must point to a `.onnx` model file.\n */\nenum TRACT_RESULT tract_onnx_load(const struct TractOnnx *onnx,\n const char *path,\n struct TractInferenceModel **model);\n\n/**\n * Parse and load an ONNX buffer as a tract InferenceModel.\n *\n * `data` is a buffer pointer\n * `len` ise the buffer len\n */\nenum TRACT_RESULT tract_onnx_load_buffer(const struct TractOnnx *onnx,\n const void *data,\n uintptr_t len,\n struct TractInferenceModel **model);\n\n/**\n * Query an InferenceModel input counts.\n */\nenum TRACT_RESULT tract_inference_model_input_count(const struct TractInferenceModel *model,\n uintptr_t *inputs);\n\n/**\n * Query an InferenceModel output counts.\n */\nenum TRACT_RESULT tract_inference_model_output_count(const struct TractInferenceModel *model,\n uintptr_t *outputs);\n\n/**\n * Query the name of a model input.\n *\n * The returned name must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_inference_model_input_name(const struct TractInferenceModel *model,\n uintptr_t input,\n char **name);\n\n/**\n * Query the name of a model output.\n *\n * The returned name must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_inference_model_output_name(const struct TractInferenceModel *model,\n uintptr_t output,\n int8_t **name);\n\n/**\n * Query a model input fact.\n */\nenum TRACT_RESULT tract_inference_model_input_fact(const struct TractInferenceModel *model,\n uintptr_t input_id,\n struct TractInferenceFact **fact);\n\n/**\n * Set an input fact of an InferenceModel.\n *\n * The `fact` argument is only borrowed by this function, it still must be destroyed.\n * `fact` can be set to NULL to erase the current output fact of the model.\n */\nenum TRACT_RESULT tract_inference_model_set_input_fact(struct TractInferenceModel *model,\n uintptr_t input_id,\n const struct TractInferenceFact *fact);\n\n/**\n * Query an output fact for an InferenceModel.\n *\n * The return model must be freed using `tract_inference_fact_destroy`.\n */\nenum TRACT_RESULT tract_inference_model_output_fact(const struct TractInferenceModel *model,\n uintptr_t output_id,\n struct TractInferenceFact **fact);\n\n/**\n * Set an output fact of an InferenceModel.\n *\n * The `fact` argument is only borrowed by this function, it still must be destroyed.\n * `fact` can be set to NULL to erase the current output fact of the model.\n */\nenum TRACT_RESULT tract_inference_model_set_output_fact(struct TractInferenceModel *model,\n uintptr_t output_id,\n const struct TractInferenceFact *fact);\n\n/**\n * Analyse an InferencedModel in-place.\n */\nenum TRACT_RESULT tract_inference_model_analyse(struct TractInferenceModel *model);\n\n/**\n * Transform a fully analysed InferenceModel to a TypedModel.\n *\n * This function takes ownership of the InferenceModel `model` whether it succeeds\n * or not. `tract_inference_model_destroy` must not be used on `model`.\n *\n * On the other hand, caller will be owning the newly created typed model.\n */\nenum TRACT_RESULT tract_inference_model_into_model(struct TractInferenceModel **model,\n struct TractModel **typed);\n\n/**\n * Destroy an InferenceModel.\n */\nenum TRACT_RESULT tract_inference_model_destroy(struct TractInferenceModel **model);\n\n/**\n * Query an InferenceModel input counts.\n */\nenum TRACT_RESULT tract_model_input_count(const struct TractModel *model, uintptr_t *inputs);\n\n/**\n * Query an InferenceModel output counts.\n */\nenum TRACT_RESULT tract_model_output_count(const struct TractModel *model, uintptr_t *outputs);\n\n/**\n * Query the name of a model input.\n *\n * The returned name must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_model_input_name(const struct TractModel *model,\n uintptr_t input,\n char **name);\n\n/**\n * Query the input fact of a model.\n *\n * Thre returned fact must be freed with tract_fact_destroy.\n */\nenum TRACT_RESULT tract_model_input_fact(const struct TractModel *model,\n uintptr_t input_id,\n struct TractFact **fact);\n\n/**\n * Query the name of a model output.\n *\n * The returned name must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_model_output_name(const struct TractModel *model,\n uintptr_t output,\n char **name);\n\n/**\n * Query the output fact of a model.\n *\n * Thre returned fact must be freed with tract_fact_destroy.\n */\nenum TRACT_RESULT tract_model_output_fact(const struct TractModel *model,\n uintptr_t input_id,\n struct TractFact **fact);\n\n/**\n * Apply a transform to the model.\n */\nenum TRACT_RESULT tract_model_transform(struct TractModel *model, const int8_t *transform);\n\n/**\n * Perform a profile of the model using the provided inputs.\n */\nenum TRACT_RESULT tract_runnable_profile_json(struct TractRunnable *model,\n struct TractTensor **inputs,\n int8_t **json);\n\n/**\n * Convert a TypedModel into a TypedRunnableModel.\n *\n * This function transfers ownership of the `model` argument to the newly-created `runnable` model.\n *\n * Runnable are reference counted. When done, it should be released with `tract_runnable_release`.\n */\nenum TRACT_RESULT tract_model_into_runnable(struct TractModel **model,\n struct TractRunnable **runnable);\n\n/**\n * Query the number of properties in a model.\n */\nenum TRACT_RESULT tract_model_property_count(const struct TractModel *model, uintptr_t *count);\n\n/**\n * Query the properties names of a model.\n *\n * The \"names\" array should be big enough to fit `tract_model_property_count` string pointers.\n *\n * Each name will have to be freed using `tract_free_cstring`.\n */\nenum TRACT_RESULT tract_model_property_names(const struct TractModel *model, int8_t **names);\n\n/**\n * Query a property tensor in a model.\n */\nenum TRACT_RESULT tract_model_property(const struct TractModel *model,\n const int8_t *name,\n struct TractTensor **tensor);\n\n/**\n * Parse a fact specification string into an Fact.\n *\n * The returned fact must be free with `tract_fact_destroy`.\n */\nenum TRACT_RESULT tract_model_parse_fact(struct TractModel *model,\n const char *spec,\n struct TractFact **fact);\n\n/**\n * Destroy a TypedModel.\n */\nenum TRACT_RESULT tract_model_destroy(struct TractModel **model);\n\n/**\n * Creates an instance of a tract Runtime that can be used to run model on a specific\n * hardware / software stack (like a GPU).\n *\n * The returned object should be released with `tract_runtime_release`.\n */\nenum TRACT_RESULT tract_runtime_for_name(const char *name, struct TractRuntime **nnef);\n\n/**\n * Query the name of a Runtime.\n *\n * The returned name must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_runtime_name(const struct TractRuntime *runtime, char **name);\n\n/**\n * Convert a Model into a Runnable for this Runtime.\n *\n * This function transfers ownership of the `model` argument to the newly-created `runnable` model.\n *\n * Runnable are reference counted. When done, it should be released with `tract_runnable_release`.\n */\nenum TRACT_RESULT tract_runtime_prepare(const struct TractRuntime *runtime,\n struct TractModel **model,\n struct TractRunnable **runnable);\n\nenum TRACT_RESULT tract_runtime_release(struct TractRuntime **runtime);\n\n/**\n * Spawn a session state from a runnable model.\n *\n * This function does not take ownership of the `runnable` object, it can be used again to spawn\n * other state instances. The runnable object is internally reference counted, it will be\n * kept alive as long as any associated `State` exists (or as long as the `runnable` is not\n * explicitely release with `tract_runnable_release`).\n *\n * `state` is a newly-created object. It should ultimately be detroyed with `tract_state_destroy`.\n */\nenum TRACT_RESULT tract_runnable_spawn_state(struct TractRunnable *runnable,\n struct TractState **state);\n\n/**\n * Convenience function to run a stateless model.\n *\n * `inputs` is a pointer to an pre-existing array of input TractTensor. Its length *must* be equal\n * to the number of inputs of the models. The function does not take ownership of the input\n * tensors.\n * `outputs` is a pointer to a pre-existing array of TractTensor pointers that will be overwritten\n * with pointers to output tensors. These tensors are under the responsiblity of the caller, it\n * will have to release them with `tract_tensor_destroy`.\n */\nenum TRACT_RESULT tract_runnable_run(struct TractRunnable *runnable,\n struct TractTensor **inputs,\n struct TractTensor **outputs);\n\n/**\n * Query a Runnable input counts.\n */\nenum TRACT_RESULT tract_runnable_input_count(const struct TractRunnable *model, uintptr_t *inputs);\n\n/**\n * Query an Runnable output counts.\n */\nenum TRACT_RESULT tract_runnable_output_count(const struct TractRunnable *model,\n uintptr_t *outputs);\n\n/**\n * Query the input fact of a runnable model.\n *\n * Thre returned fact must be freed with tract_fact_destroy.\n */\nenum TRACT_RESULT tract_runnable_input_fact(const struct TractRunnable *runnable,\n uintptr_t input_id,\n struct TractFact **fact);\n\n/**\n * Query the output fact of a runnable model.\n *\n * Thre returned fact must be freed with tract_fact_destroy.\n */\nenum TRACT_RESULT tract_runnable_output_fact(const struct TractRunnable *runnable,\n uintptr_t output_id,\n struct TractFact **fact);\n\n/**\n * Query the number of properties in a runnable model.\n */\nenum TRACT_RESULT tract_runnable_property_count(const struct TractRunnable *model,\n uintptr_t *count);\n\n/**\n * Query the properties names of a runnable model.\n *\n * The \"names\" array should be big enough to fit `tract_model_property_count` string pointers.\n *\n * Each name will have to be freed using `tract_free_cstring`.\n */\nenum TRACT_RESULT tract_runnable_property_names(const struct TractRunnable *model, int8_t **names);\n\n/**\n * Query a property tensor in a runnable model.\n */\nenum TRACT_RESULT tract_runnable_property(const struct TractRunnable *model,\n const int8_t *name,\n struct TractTensor **tensor);\n\nenum TRACT_RESULT tract_runnable_release(struct TractRunnable **runnable);\n\n/**\n * Create a TractTensor from caller data and metadata.\n *\n * This call copies the data into tract space. All the pointers only need to be alive for the\n * duration of the call.\n *\n * rank is the number of dimensions of the tensor (i.e. the length of the shape vector).\n *\n * The returned tensor must be destroyed by `tract_tensor_destroy`.\n */\nenum TRACT_RESULT tract_tensor_from_bytes(DatumType datum_type,\n uintptr_t rank,\n const uintptr_t *shape,\n void *data,\n struct TractTensor **tensor);\n\n/**\n * Write a tensor as a debug string\n *\n * The returned string must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_tensor_dump(const struct TractTensor *tensor, char **spec);\n\n/**\n * Convert a tensor to a new datum type.\n *\n * This function will perform a cheap shallow clone if the destination type is\n * the same as the current type, otherwise it returns a newly allocated tensor instead.\n *\n * In both cases, the returned tensor must be destroyed by `tract_tensor_destroy`.\n * The input tensor is not consumed, it still need to be destroyed.\n */\nenum TRACT_RESULT tract_tensor_convert_to(const struct TractTensor *input,\n DatumType datum_type,\n struct TractTensor **output);\n\n/**\n * Destroy a tensor.\n */\nenum TRACT_RESULT tract_tensor_destroy(struct TractTensor **tensor);\n\n/**\n * Inspect part of a tensor. Except `tensor`, all argument pointers can be null if only some specific bits\n * are required.\n */\nenum TRACT_RESULT tract_tensor_as_bytes(struct TractTensor *tensor,\n DatumType *datum_type,\n uintptr_t *rank,\n const uintptr_t **shape,\n const void **data);\n\n/**\n * Run a turn on a model state\n *\n * `inputs` is a pointer to an pre-existing array of input TractTensor. Its length *must* be equal\n * to the number of inputs of the models. The function does not take ownership of the input\n * tensors.\n * `outputs` is a pointer to a pre-existing array of TractTensor pointers that will be overwritten\n * with pointers to output tensors. These tensors are under the responsiblity of the caller, it\n * will have to release them with `tract_tensor_destroy`.\n */\nenum TRACT_RESULT tract_state_run(struct TractState *state,\n struct TractTensor **inputs,\n struct TractTensor **outputs);\n\n/**\n * Query a State input counts.\n */\nenum TRACT_RESULT tract_state_input_count(const struct TractState *state, uintptr_t *inputs);\n\n/**\n * Query an State output counts.\n */\nenum TRACT_RESULT tract_state_output_count(const struct TractState *state, uintptr_t *outputs);\n\nenum TRACT_RESULT tract_state_destroy(struct TractState **state);\n\n/**\n * Gets the rank (aka number of axes/dimensions) of a fact.\n */\nenum TRACT_RESULT tract_fact_rank(const struct TractFact *fact, uintptr_t *rank);\n\n/**\n * Extract the datum type of the fact.\n */\nenum TRACT_RESULT tract_fact_datum_type(const struct TractFact *fact, DatumType *datum_type);\n\n/**\n * Extract the dimension from one dimension of the fact.\n */\nenum TRACT_RESULT tract_fact_dim(const struct TractFact *fact,\n uintptr_t axis,\n struct TractDim **dim);\n\n/**\n * Write a fact as its specification string.\n *\n * The returned string must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_fact_dump(const struct TractFact *fact, char **spec);\n\nenum TRACT_RESULT tract_fact_destroy(struct TractFact **fact);\n\n/**\n * Parse a fact specification string into an InferenceFact.\n *\n * The returned fact must be free with `tract_inference_fact_destroy`.\n */\nenum TRACT_RESULT tract_inference_fact_parse(struct TractInferenceModel *model,\n const char *spec,\n struct TractInferenceFact **fact);\n\n/**\n * Creates an empty inference fact.\n *\n * The returned fact must be freed by the caller using tract_inference_fact_destroy\n */\nenum TRACT_RESULT tract_inference_fact_empty(struct TractInferenceFact **fact);\n\n/**\n * Write an inference fact as its specification string.\n *\n * The returned string must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_inference_fact_dump(const struct TractInferenceFact *fact, char **spec);\n\n/**\n * Destroy a fact.\n */\nenum TRACT_RESULT tract_inference_fact_destroy(struct TractInferenceFact **fact);\n\n/**\n * Substitute symbols by the provided values in the Dim, generating a new one.\n */\nenum TRACT_RESULT tract_dim_eval(const struct TractDim *dim,\n uintptr_t nb_symbols,\n const int8_t *const *symbols,\n const int64_t *values,\n struct TractDim **result);\n\n/**\n * Try converting a Dim into an actual integer\n *\n * Will fail if the Dim contains symbols.\n */\nenum TRACT_RESULT tract_dim_to_int64(const struct TractDim *fact, int64_t *i);\n\n/**\n * Write a dim as its specification string.\n *\n * The returned string must be freed by the caller using tract_free_cstring.\n */\nenum TRACT_RESULT tract_dim_dump(const struct TractDim *dim, char **spec);\n\n/**\n * Destroy a dim.\n */\nenum TRACT_RESULT tract_dim_destroy(struct TractDim **dim);\n" }, { "path": "api/proxy/tests/mobilenet.rs", "content": "use tract_api::*;\nuse tract_proxy::*;\n\ninclude!(\"../../tests/mobilenet/mod.rs\");\n" }, { "path": "api/py/.gitignore", "content": "__pycache__\n*.so\n*.egg-info\n*.onnx\nbuild\nmobilenet_v2_1.0.onnx.nnef.tgz\nrust-workspace\ndist\n\n" }, { "path": "api/py/MANIFEST.in", "content": "graft rust-workspace\ngraft docs\n" }, { "path": "api/py/_static/redirect-index.html", "content": "\n\n\n \n \n\n\n

Redirecting to documentation...

\n\n\n" }, { "path": "api/py/_static/version-switcher.js", "content": "// Version switcher for multi-version gh-pages docs.\n// Reads versions.json (mike-compatible format) from the site root and injects\n// a