[ { "path": ".dockerignore", "content": "# Ignore build artifacts to keep Docker context small\n.stack-work/\n.git/\nout/\ncontainer/static-build/.stack-work/\nmorloc-manager/.stack-work/\ndata/rust/target/\n*.hi\n*.o\n" }, { "path": ".github/workflows/release.yml", "content": "# Release workflow: build all binaries + containers, run tests, publish release.\n#\n# Triggered by pushing a version tag:\n# git tag v0.68.0\n# git push origin v0.68.0\n#\n# Produces for each platform (linux-x86_64, linux-arm64, macos-arm64):\n# - morloc-manager (Rust binary)\n# - libmorloc.so (Rust runtime library; .dylib on macOS)\n# - morloc-nexus (Rust binary)\n#\n# All three are attached to the GitHub Release.\n\nname: Release\n\non:\n push:\n tags: ['v*']\n\nenv:\n REGISTRY: ghcr.io\n IMAGE_BASE: ghcr.io/morloc-project/morloc\n\njobs:\n # ---- Build Rust binaries (libmorloc + morloc-nexus + morloc-manager) per platform ----\n rust-binary:\n strategy:\n fail-fast: false\n matrix:\n include:\n - os: ubuntu-latest\n platform: linux-x86_64\n method: docker\n - os: ubuntu-24.04-arm\n platform: linux-arm64\n method: docker\n - os: macos-latest\n platform: macos-arm64\n method: native\n runs-on: ${{ matrix.os }}\n timeout-minutes: 30\n steps:\n - uses: actions/checkout@v4\n\n # ---- Linux: Docker container build ----\n # libmorloc.so + morloc-nexus: glibc (Ubuntu 20.04)\n # morloc-manager: static (Alpine/musl)\n - name: Build Rust binaries (Linux)\n if: matrix.method == 'docker'\n run: |\n docker build -t morloc-rust-build \\\n -f container/static-build/Dockerfile .\n mkdir -p out\n docker run --rm -v \"$(pwd)/out:/out\" morloc-rust-build\n # Verify morloc-manager is static\n file out/morloc-manager | grep -qE \"static(ally|-pie) linked\"\n\n - name: Rename artifacts (Linux)\n if: matrix.method == 'docker'\n run: |\n mv out/libmorloc.so out/libmorloc-${{ matrix.platform }}.so\n mv out/morloc-nexus out/morloc-nexus-${{ matrix.platform }}\n mv out/morloc-manager out/morloc-manager-${{ matrix.platform }}\n\n # ---- macOS: native cargo build ----\n - name: Setup Rust (macOS)\n if: matrix.method == 'native'\n uses: dtolnay/rust-toolchain@stable\n\n - name: Cache Cargo (macOS)\n if: matrix.method == 'native'\n uses: actions/cache@v4\n with:\n path: |\n ~/.cargo/registry\n ~/.cargo/git\n data/rust/target\n key: cargo-macos-${{ hashFiles('data/rust/Cargo.lock') }}\n restore-keys: cargo-macos-\n\n - name: Build Rust binaries (macOS)\n if: matrix.method == 'native'\n run: |\n cd data/rust\n # Build libmorloc (cdylib produces .dylib on macOS)\n cargo build --release -p morloc-runtime\n # Install .dylib so nexus can link against it\n mkdir -p $HOME/.local/share/morloc/lib\n cp target/release/libmorloc_runtime.dylib $HOME/.local/share/morloc/lib/libmorloc.dylib\n # Build nexus\n cargo build --release -p morloc-nexus\n # Build manager\n cargo build --release -p morloc-manager\n # Collect artifacts\n mkdir -p ../../out\n cp target/release/libmorloc_runtime.dylib ../../out/libmorloc-${{ matrix.platform }}.dylib\n cp target/release/morloc-nexus ../../out/morloc-nexus-${{ matrix.platform }}\n cp target/release/morloc-manager ../../out/morloc-manager-${{ matrix.platform }}\n strip ../../out/morloc-nexus-${{ matrix.platform }} || true\n strip ../../out/morloc-manager-${{ matrix.platform }} || true\n\n - name: Upload Rust artifacts\n uses: actions/upload-artifact@v4\n with:\n name: rust-binaries-${{ matrix.platform }}\n path: out/*\n\n # ---- Run tests using the Rust binaries ----\n test:\n needs: rust-binary\n runs-on: ubuntu-latest\n timeout-minutes: 60\n env:\n DEBIAN_FRONTEND: noninteractive\n steps:\n - uses: actions/checkout@v4\n - uses: haskell-actions/setup@v2\n with:\n ghc-version: '9.6.7'\n enable-stack: true\n stack-version: 'latest'\n - uses: actions/cache@v4\n with:\n path: |\n ~/.stack/snapshots\n ~/.stack/setup-exe-cache\n .stack-work\n key: stack-deps-release-${{ hashFiles('stack.yaml.lock', 'package.yaml') }}\n restore-keys: stack-deps-release-\n - name: Increase shared memory\n run: sudo mount -o remount,size=4G /dev/shm\n - name: Install system dependencies\n run: |\n sudo apt-get update\n sudo apt-get install -y --no-install-recommends \\\n r-base-core python3 python3-dev python3-pip \\\n python3-numpy g++ gcc make libgsl-dev\n python3 -m pip install --break-system-packages --upgrade setuptools pyarrow\n\n - name: Cache R packages\n uses: actions/cache@v4\n id: r-cache\n with:\n path: ~/R/library\n key: r-lib-${{ runner.os }}-${{ runner.arch }}\n\n - name: Install R packages\n if: steps.r-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p ~/R/library\n LIBARROW_MINIMAL=true ARROW_S3=OFF ARROW_GCS=OFF \\\n Rscript -e 'install.packages(\"arrow\", lib=\"~/R/library\", repos = \"https://cloud.r-project.org\")'\n\n - name: Download Rust binaries\n uses: actions/download-artifact@v4\n with:\n name: rust-binaries-linux-x86_64\n path: rust-bin\n\n - name: Prepare Rust binaries\n run: |\n mkdir -p prebuilt\n mv rust-bin/libmorloc-linux-x86_64.so prebuilt/libmorloc.so\n mv rust-bin/morloc-nexus-linux-x86_64 prebuilt/morloc-nexus\n mv rust-bin/morloc-manager-linux-x86_64 prebuilt/morloc-manager\n chmod +x prebuilt/libmorloc.so prebuilt/morloc-nexus prebuilt/morloc-manager\n\n - name: Add morloc to PATH\n run: |\n echo \"$HOME/.local/bin\" >> $GITHUB_PATH\n echo \"$HOME/.local/share/morloc/bin\" >> $GITHUB_PATH\n echo \"R_LIBS_USER=$HOME/R/library\" >> $GITHUB_ENV\n - name: Build morloc\n run: stack install --system-ghc --no-install-ghc --no-run-tests\n - name: Initialize morloc (using pre-built Rust binaries)\n run: |\n MORLOC_RUST_BIN=$(pwd)/prebuilt morloc init -f\n morloc install stdlib\n - name: Run tests\n run: stack test --system-ghc --no-install-ghc morloc:morloc-test\n timeout-minutes: 10\n\n # ---- Build and push container images ----\n containers:\n needs: test\n runs-on: ubuntu-latest\n timeout-minutes: 120\n permissions:\n packages: write\n steps:\n - uses: actions/checkout@v4\n\n - name: Extract version from tag\n id: ver\n run: echo \"version=${GITHUB_REF_NAME#v}\" >> \"$GITHUB_OUTPUT\"\n\n - name: Login to GHCR\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build tiny (from local source)\n run: |\n docker build \\\n -t ${{ env.IMAGE_BASE }}/morloc-tiny:${{ steps.ver.outputs.version }} \\\n -t ${{ env.IMAGE_BASE }}/morloc-tiny:edge \\\n -f container/tiny/Dockerfile .\n\n - name: Build full (uses local tiny)\n run: |\n docker build \\\n --build-arg MORLOC_VERSION=${{ steps.ver.outputs.version }} \\\n -t ${{ env.IMAGE_BASE }}/morloc-full:${{ steps.ver.outputs.version }} \\\n -t ${{ env.IMAGE_BASE }}/morloc-full:edge \\\n container/full/\n\n - name: Push all images\n run: |\n for img in morloc-tiny morloc-full; do\n docker push ${{ env.IMAGE_BASE }}/${img}:${{ steps.ver.outputs.version }}\n docker push ${{ env.IMAGE_BASE }}/${img}:edge\n done\n\n # ---- Create GitHub Release ----\n release:\n if: always() && needs.containers.result == 'success'\n needs: [containers, rust-binary]\n runs-on: ubuntu-latest\n permissions:\n contents: write\n steps:\n - name: Download all artifacts\n uses: actions/download-artifact@v4\n with:\n pattern: 'rust-binaries-*'\n merge-multiple: true\n\n - name: List artifacts\n run: ls -lh\n\n - name: Create GitHub Release\n uses: softprops/action-gh-release@v2\n with:\n files: |\n morloc-manager-*\n libmorloc-*\n morloc-nexus-*\n generate_release_notes: true\n" }, { "path": ".github/workflows/test.yml", "content": "name: Test Morloc\non: [push]\n\njobs:\n linux-test:\n runs-on: ubuntu-latest\n timeout-minutes: 60\n\n env:\n DEBIAN_FRONTEND: noninteractive\n\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Setup Haskell (GHC + Stack)\n uses: haskell-actions/setup@v2\n with:\n ghc-version: '9.6.7'\n enable-stack: true\n stack-version: 'latest'\n\n - name: Setup Rust\n uses: dtolnay/rust-toolchain@stable\n\n - name: Cache Stack dependencies\n uses: actions/cache@v4\n with:\n path: |\n ~/.stack/snapshots\n ~/.stack/setup-exe-cache\n .stack-work\n key: stack-deps-${{ hashFiles('stack.yaml.lock', 'package.yaml') }}\n restore-keys: |\n stack-deps-\n\n - name: Cache Cargo dependencies\n uses: actions/cache@v4\n with:\n path: |\n ~/.cargo/registry\n ~/.cargo/git\n data/rust/target\n key: cargo-deps-${{ hashFiles('data/rust/Cargo.lock') }}\n restore-keys: |\n cargo-deps-\n\n - name: Increase shared memory (needed for morloc SHM pool tests)\n run: sudo mount -o remount,size=4G /dev/shm\n\n - name: Install system dependencies\n run: |\n sudo apt-get update\n sudo apt-get install -y --no-install-recommends \\\n r-base-core \\\n python3 \\\n python3-dev \\\n python3-pip \\\n python3-numpy \\\n g++ \\\n gcc \\\n make \\\n libgsl-dev\n python3 -m pip install --break-system-packages --upgrade setuptools pyarrow\n\n - name: Cache R packages\n uses: actions/cache@v4\n id: r-cache\n with:\n path: ~/R/library\n key: r-lib-${{ runner.os }}-${{ runner.arch }}\n\n - name: Install R packages\n if: steps.r-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p ~/R/library\n LIBARROW_MINIMAL=true ARROW_S3=OFF ARROW_GCS=OFF \\\n Rscript -e 'install.packages(\"arrow\", lib=\"~/R/library\", repos = \"https://cloud.r-project.org\")'\n\n - name: Add morloc bin folders to PATH\n run: |\n echo \"$HOME/.local/bin\" >> $GITHUB_PATH\n echo \"$HOME/.local/share/morloc/bin\" >> $GITHUB_PATH\n echo \"R_LIBS_USER=$HOME/R/library\" >> $GITHUB_ENV\n\n - name: Build Morloc\n run: |\n stack install --system-ghc --no-install-ghc --no-run-tests\n\n - name: Initialize morloc and install modules\n run: |\n MORLOC_RUST_DIR=$(pwd)/data/rust morloc init -f\n morloc install stdlib\n\n - name: Test morloc\n run: |\n stack test --system-ghc --no-install-ghc morloc:morloc-test\n timeout-minutes: 10\n" }, { "path": ".gitignore", "content": "tags\n*.hi\n*.o\n.stack-work/\n.history\nmorloc.cabal\n*.out\nrun/\n.idea/\n*.iml\n# ignore debugging files\n*.aux\n*.hp\n*.prof\n*.ps\n.bash_history\n.local\n.stack\nCLAUDE.md\n.claude/\nclaude-memory/\n.RData\n.Rhistory\npools/\nnexus\nout/\n" }, { "path": "ChangeLog.md", "content": "0.81.0 [2026-94-22]\n-------------------\n * fix type inference for type families\n * extend intrinsic support for language-agnostic code\n * do shell-expansion of flagfile lines in morloc-manager\n * fix loss of pool stderr on crash (maybe?)\n\n0.80.1 [2026-04-21]\n-------------------\n * add `morloc-manager new --include ` src:dest syntax\n * make stderr/stdout mim principles more consistent\n\n0.80.0 [2026-04-20]\n-------------------\n * cleaner morloc-manager\n * fixed --print bug in nexus\n\n0.79.1-5 [2026-04-19]\n * lot's morloc-manager updates\n\n0.79.0 [2026-04-17]\n-------------------\n * add @datafile intrinsic for finding installed files\n * default to including all files when installing\n - parse `.morlocignore` to find non-install files\n - allow `include` in package.yaml for strict file inclusion\n * add `morloc-manage doctor` subcommand for health checks\n * add additional `freeze/unfreeze` validation\n * fix unfreezing bugs\n\n0.78.0 [2026-04-17]\n-------------------\n * clean up CLI usage statements\n * suppress \"null\" outpput in Unit-returning CLIs\n * fix string escape bugs\n * fix where parsing after do-block\n * fix many small morloc-manager issues\n * fix paths in installed morloc programs\n\n\n0.77.1 [2026-04-16]\n-------------------\n * resurrected `morloc-manager log` subcommand\n\n0.77.0 [2026-04-15]\n-------------------\n * many many deployment fixes\n\n0.76.0 [2026-04-13]\n-------------------\n * fix docstring groups\n * add module docstrings\n * fix bug in unfreeze\n\n0.75.0 [2026-04-12]\n-------------------\n * allow effectful final do-notation statements\n * lots of bug fixes\n * make changelog version source of truth\n\n0.74.0 [2026-04-06]\n-------------------\n * unify morloc-manager env, workspace, and version\n * allow docstrings before declarations\n * allow literal leading spaces in docstrings\n * allow escaped `:` syntax\n\n0.73.0 [2026-04-01]\n-------------------\n * fix @load error on missing file\n * fix error reporting on failed `morloc-manager install`\n\n0.72.0 [2026-03-30]\n-------------------\n * fix unicode support in comments and literal strings\n * many fixes to the morloc-manager\n\n0.71.0-2 [2026-03-29]\n-------------------\n * port morloc-manager to rust\n\n0.70.1 to 0.70.6 [2026-03-29]\n-----------------------------\n * fix portability for libmorloc and morloc-nexus on darwin and linux-arm64\n * python flushing face condition\n\n0.70.0 [2026-03-29]\n-------------------\n * port libmorloc and morloc-nexus from C to Rust\n\n0.69.0 [2026-03-27]\n-------------------\n * port morloc-manager from shell to Haskell\n * share it as a static binary\n\n0.68.0 [2026-03-22]\n-------------------\n * remove explicit universal quntifiers\n - before: `id a :: a -> a`\n - now: `id :: a -> a`\n * add `morloc uninstall --all` option\n * do not require repeated `let` terms\n\n0.67.1 [2026-03-17]\n-------------------\n * add missing Nat kind annotation to root modules\n * better test coverage of Nat kinds\n * add `morloc uninstall --all` option\n * add `stdlib` module that simplifies recursive install of everything\n\n0.67.0 [2026-03-17]\n-------------------\n * add Arrow tables with zero-copy between language sharing\n * add tensors and\n * remove explicit qualifiers\n - no more `id a :: a -> a`\n - now `id :: a -> a`\n - this does not require C++ be more explicit in their typing\n * inline when packet data is smaller than 65kb\n - this avoids fragmenting the shared memory with many small single-use values\n\n0.66.0 [2026-03-12]\n-------------------\n * fix propagation of errors from crashed pools\n * fix shm memory alignment (inefficiency on x86, crashes on ARM)\n * add `--sanitize` option to `morloc init` for strict memory\n * replace megaparsec with parsec\n * fix label and remote handling\n\n0.65.0 [2026-03-10]\n-------------------\n * `morloc eval` for running anonymous morloc expressions\n * add %inline pragma to skip manifold gen\n * allow batch syntax for instances\n * add @show and @read intrinsics\n * add namespaces for morloc imports\n\n0.64.0 [2026-03-03]\n-------------------\n * replace thunks with more granular effect system\n * add intrinsics\n * disambiguate local module imports \n * change null to Null\n\n0.63.0 [2026-02-25]\n-------------------\n * add optionals with implicit coercion\n * for `?Str` with `literal: true`, force `default: null`\n * add daemon args to nexus\n\n0.62.0 [2026-02-24]\n-------------------\n * term recursion support\n * conditional guards\n * nexus pretty printing option (-p/--print)\n * print defaults in usage statements\n * more bug fixes\n\n0.61.0 [2026-02-21]\n-------------------\n\nBuild updates\n * make nexus a constant binary (not recreated and recompiled)\n * add install handling and portable scripts\n * add morloc daemon mode accessible through HTTP/TCP and sockets\n * add `morloc install --build` option for installing both executable modules\n * add `morloc uninstall`\n * add `morloc list` with -v option for listing types of all exports\n\nTypesystem updates\n * add let syntax that enforces execution order\n * class constraints (e.g., unique :: Eq a => [a] -> [a])\n * superclasses (e.g., class Semigroup => Monoid a where ...)\n * add effect system for delayed execution\n * add do-syntax for imperative programming with effects\n\nBetter errors and UI\n * cleaner error messages\n * error message localization\n * clean `morloc typecheck` output\n * add CLI subcommand tested grouping\n * Add shell TAB-completion\n * fix haddock for future hackage release\n\nBug fixes\n * cleaned up memory issues in all C code\n * replaced mcparallel in R with forked pool of workers\n * fix bug in `morloc install .`\n\nTesting\n * added stress test for zombies and memory issues\n * added daemon tests\n\nPerformance\n * Split the monolithic (~7000 line) morloc.h file\n * Use a libmorloc.so shared library rather than importing all as header\n * Removed all the zombie swarms that where killing heavy morloc projects\n * Remove mcparallel from R, move most of the pool to C, 4X interop speed\n * Compile and reuse a single nexus (reduce compile costs)\n\nLanguage onboarding\n * Added codegen IR that greatly simplifies new language addition\n * Factor all language-specific material out of main Haskell codebase\n * Move all grammar into template yaml specs\n * Created MessagePack bridge to bypass voidstar, making lang onboard almost\n trivial (at a performance penalty).\n\nOther\n * Transitioned parser from recursive descent to LR1\n\n0.60.0 [2026-02-07]\n-------------------\n * add infix operator support\n\nTesting\n * Generate many new tests with Claude\n * Extensive testing of infix operators and module inheritance\n\nCleanup\n * add formatting with fourmulo\n * add metric and benchmarking\n\nBug fixes\n * Fix several memory issues in morloc.h and lang bindings\n * Fixed indexing bug in Generator.hs\n * Fix missing space in git module cp\n * Make missing folders in install path\n * Fix `morloc install` deletion of mod folder when install has trailing '/' \n * Disallow space between sign and number\n\nPerformance\n * Fixed exponential case hit on eta resolution\n * Fixed quadratic case in subtype algorithm\n\nClaude memory\n * Added claude memory files\n\n0.59.0 [2026-01-23]\n-------------------\n * allow functions in data structures\n * allow source functions to return functions\n * allow application of expressions that evaluate to functions\n * fix bug in C++ bool deserialization\n\n0.58.3 [2026-01-03]\n-------------------\n * fix record docstring inheritance\n\n0.58.2 [2025-12-29]\n-------------------\n * fix minor bug optional versus positional\n\n0.58.1 [2025-12-28]\n-------------------\n * fix minor bug in record unrolling\n\n0.58.0 [2025-12-28]\n-------------------\n * add hex, octal, and binary numeric representations\n * new record syntax\n * generate CLI from docstrings\n * resolve records into groups of arguments\n * allow literal strings without the extra JSON quoting\n * change to Apache-2.0\n\n0.57.1 [2025-11-12]\n-------------------\n * minor bug fixes\n\n0.57.0 [2025-11-11]\n-------------------\n * re-allow underscores in variable names\n * add placeholder syntax / lambda lifting from holes\n * multi-line strings\n * string interpolation\n * getter patterns\n * setter patterns\n * write full Morloc nexus evaluator\n * Fix bug in local installs with \".\"\n\n0.56.0 [2025-10-08]\n-------------------\n\nNew file organization\n * Change file layout to conform to XDG spec; \n replaced `~/.morloc` with `~/.local/share/morloc`\n * Move standard library to `$MORLOC_HOME/src/morloc/plane/default/morloclib`\n * src/ - stores any source code morloc needs, not just modules \n * morloc/ - stores morloc modules\n * plane/ - stores morloc modules that are part of a plane\n * default/ - the current default plane\n * morloclib/ - the org name for the core modules in the default plane\n\nNew `morloc install` subcommand functionality\n * Multiple imports may be chained on one command\n * Source and ref can be included per install, for example:\n * `root` - install a core module\n * `root@hash:abcdef1234` - specific core instance\n * `root@tag:v1.0.0` - specific tag/version\n * `codeberg:weena/calendar@hash:abcdef1234` - 3rd party with source and ref\n * Support for install of local modules\n \n\n0.55.1 [2025-09-29]\n-------------------\n\n * Bug fixes\n\n0.55.0 [2025-09-27]\n-------------------\n\n * Allow general types to be declared and imported/exported\n * Allow undirected dependencies\n * Allow dashes in module names\n * Fix many bugs in native Morloc code handling\n * Fix bug in certain higher order foreign functions\n * Simplify internal unique naming conventions\n * Move all tests to use the new root module (rather than base)\n * Slightly improved error messages\n\nBreaking changes:\n * Move to reverse tree model of dependency handling\n * Allow exactly one type signature for each term and class\n * Require explicit typeclass import/export\n * Fix handling of Python builtin imports\n Now builtins must not be imported from Python source. Instead\n import directly from Python, for example: `source Py (\"len\")`\n * More efficient Haskell Map usage\n\n0.54.2 [2025-08-09]\n-------------------\n\n * Enforce lowercase module name rule\n * Fix handling of executable file name option\n * Fix record handling\n\n0.54.1 [2025-07-26]\n-------------------\n\n * Fix pickle bug in Python multiprocessing\n * Replace asprintf non-standard C function \n * Partially fix interop for C structs\n * Fix bug in unqualified imports\n\n0.54.0 [2025-07-13]\n-------------------\n\n * Add full MessagePack and VoidStar IO to nexus\n * Fixed type pretty printing in usage and typechecking output\n * Add nexus option and usage info\n * Add support for one-line command docstrings\n\n0.53.7 [2025-05-31]\n-------------------\n\n * Fix bug in parameterized type definition concrete type inference\n\n0.53.6 [2025-05-31]\n-------------------\n\n * Improve container instructions in README\n * Fix all README examples\n * Make Dockerfiles more portable\n * Fix bug in implicit typeclass instance inheritance\n\n0.53.5 [2025-05-12]\n-------------------\n\n * Fixed unnecessary copying in C libs\n * Fixed double let-binding in code gen\n\n0.53.4 [2025-05-08]\n-------------------\n\n * Replace C daemon forking with thread pooling (4X speedup)\n * Fix bugs related to 0-length array memory allocation\n\n0.53.3 [2025-05-06]\n-------------------\n\n * Fix handling of empty vectors\n\n0.53.2 [2025-05-06]\n-------------------\n\n * Fix bug in JSON parsing\n\n0.53.1 [2025-05-06]\n-------------------\n\n * Update containers\n * Update github actions\n * Fix bug in type scoping (sort of)\n\n0.53.0 [2025-05-04]\n-------------------\n\nUnify backend under shared C library\n\n * Replace python nexus with C nexus\n * Move packets to little-endian format\n * Move all packet handling, binary protocol specs, socket handling, JSON\n parsing from pool code into the shared morloc.h library\n * Replace R `future` parallelism scheme with builtin `parallel` library\n\nAdd remote worker and resource management support\n\n * Add conventions for specifying caching and evaluation strategy\n * Add xxhash hashing for caching\n * Add remote handling with experimental slurm support\n\nOther\n\n * `morloc` commands now return proper exit codes\n\n0.52.1 [2025-02-16]\n-------------------\n * Add python native bytes and bytearray support\n\n0.52.0 [2025-02-09]\n-------------------\n\nType evaluation and specialization\n * Delayed general alias evaluation\n * Add strict numeric types (`int8`, `uint8`, `int16` etc)\n * Allow concrete alias specialization\n * Add type hints allowing concrete type conservation across foreign calls\n\nBug fixes\n * Fix bug in opening existing shared memory volumes\n\nLanguage support\n * Add C++ support for std template list-like types (`list`, `forward_list`,\n `deque`, `stack`, and `set` (I know, they're a tad different)\n * Add Python support for `numpy` vectors and arrays\n * Allow raw R vectors to be interpreted as strings\n * Nexus no longer creates python objects from returned data, instead writes\n results directly through C library (`morloc.h`)\n\n0.51.1 [2024-12-04]\n-------------------\n\n * Do not catch STDOUT and STDERR\n * Fix NULL return errors\n * Fix container setup\n\n0.51.0 [2024-12-02]\n------------------\n\nShared memory\n\n * Allow processes to communicate through shared memory\n\nSetup\n\n * `morloc init` command will now build the `morloc` ecosystem, writing all\n required headers to `morloc` home and compiling any required shared libraries.\n\nOther\n\n * Fix cases where morloc stalls when errors are transmitted across languages\n * Moved demos to the dedicated example repo\n\n\n0.50.0 [2024-11-08]\n-------------------\n\nBetter installation\n\n * `morloc install --commit 45d8df12` - for github retrieval by commit hash\n * `morloc install --branch dev` - to retrieve latest from specific branch\n * `morloc install --tag 0.1.0` - to retrieve specific tag\n\nBetter containers\n * Use podman rather than docker in Makefile\n * `morloc-tiny:` - everything needed to compile morloc projects\n * `morloc-full:` - an environment for running projects\n\n\n0.49.0 [2024-11-04]\n-------------------\n\nNew backend with better performance\n\n * Mediate inter-process communication with UNIX domain sockets\n * Transmit data with MessagePack rather than JSON\n * Added a benchmarking suite\n * Added `morloc init` command to configure morloc ecosystem\n\n0.48.0 [2024-05-10]\n-------------------\n\nSecond PeerJ submission (after return by editor for technical reasons)\n\n * Made type parameters explicit as type arguments:\n `id a :: a -> a`\n rather than either of\n `id :: a -> a`\n `id :: forall a . a -> a`.\n * Pass-by-reference in generated C++ code\n * Simplify generated C++ code by using function template arguments rather\n than type casting.\n\n\n0.47.2 [2024-04-28]\n-------------------\n\n * made the build static\n * fixed the dockerfile\n * exported the vim-syntax file out to its own repo\n * added a --version option\n * upgraded to LTS 22.18\n\n0.47.2 [2024-04-28]\n-------------------\n\nFirst PeerJ submission\n\n0.47.0 [2024.03.10]\n-------------------\n * Add value checker\n * Raise error when implementations have equal score (no arbitrary choice)\n\n0.46.0 [2024.03.06]\n-------------------\n\n * Fix precedence of '@' operator - it binds tightest\n * Update flu demo\n * Fix github actions\n * Fix Dockerfile\n * Address all compiler warnings\n\n0.45.0 [2024.02.14]\n-------------------\n\n * Allow explicit import of polymorphic terms\n * Fix infinite loop bug when two many arguments are applied to a function\n * Synchronise tests with new core libraries type classes\n\n0.44.0 [2024.02.08]\n-------------------\n\nAdd support for ad hoc polymorphism.\n * Support sources and declarations in classes \n * Support multiple parameters\n * Support overlapping instances\n * Packers are now implemented through the `Packable` typeclass\n\nSome missing features:\n * No support typeclass constraints in the type signatures.\n * No support for parameterized class variables\n * No support for polymorphic recursion (does anyone want that?)\n\n0.43.0 [2024.01.14]\n-------------------\n\nNew features\n * Allow a module to explicitly export packers\n * Show pool pseudocode for `typecheck -r` \n * Add `typecheck dump` subcommand to show expressions and indices\n * Allow nexus inputs to be files\n * Remove concrete type signatures - always infer\n * Make fields in language-specific table decs optional\n Rather than this:\n table (Person a) = Person {name :: Str, info :: a}\n table R (Person a) = \"data.frame\" {name :: Str, info :: a}\n Allow this:\n table (Person a) = Person {name :: Str, info :: a}\n table R (Person a) = \"data.frame\"\n Really, I need to totally redo the table/record/object handling.\n * Remove support for anonymous records in type signatures\n I will re-add this possibly at a future time when I refactor\n\nInfrastructure changes\n * Pass all data between pools as files rather than string arguments\n * Raise an error if any general type cannot be translated to a concrete type\n\nFixes\n * Fix record type inference \n * Fix bug in collecting packers (missed packers required by root manifold)\n * Fix C++ handling of quotes and special characters in JSON strings\n\n0.42.0 [2023.10.11]\n-------------------\n\n * Infer concrete types from inferred general types when necessary and possible\n * More informative error messages \n * Fix template resolution in generated C++ code\n * Fix include name shadowing conflict in generated C++ code\n * Partially fix naming conflict in Python and R pools caused by use of \"f\". My\n solution was name mangling, though we need a more permanent solution to our\n problem.\n * Let user write correct \"list\" R types for lists, tuples, and records\n * Fix bug in code generation of lets\n\n## Internal\n * For `ForeignInterfaceM` constructor of ExprM, store the full call type, not\n just the return type\n * Parameterize `ExprM` with the type type (e.g., `Maybe TypeP` or `TypeM`)\n * Replace `Argument` and `PreArgument` with a parameterized `Arg` type.\n * Fix broken `ForeignInterfaceM` and `PoolCallM` cases in `typeOfExprM`\n * Refactor backend\n\n Testing and documentation:\n * Complete flu demo - it builds now, just need to implement the library code\n for align and other functions.\n\n\n0.41.1 [2023.05.26]\n-------------------\n\n * Print nothing if \"null\" is returned\n * Fix the import of working directory modules\n * Resolve bug in occur check\n * Streamline github actions script\n * Fix `morloc install` path download for core modules\n * Raise type error for untyped concrete term\n * Fix bug in concrete type synthesis\n\n0.41.0 [2023.04.16]\n-------------------\n\nLanguage updates\n * Add check to avoid infinite typechecker recursion\n * Synthesize concrete types from general types when possible\n * Improve exports\n * Move exports to module list (as in Haskell)\n * Add `*` wildcard to export every top-level named term or type\n * Raise an error if a non-existing term is exported from a module\n * Allow concrete (un)packers to vary in number of generic terms\n * Set functions with empty inputs (e.g., `()`) to have empty lists of arguments\n * Replace the `Null` literal term with `()`\n\n\nPackage updates\n * Default to c++17, rather than c++11\n * Restructure library to avoid name conflicts with pre-existing packages\n * Replace Perl nexus with Python nexus and remove Perl dependencies\n\nBetter error messages and logs\n * Resolve \"shit output\" error message (map index to export name)\n * Tidy up the verbose diagnostics\n * Print general and concrete types for typecheck with -r option\n * Add multiple levels of verbosity (quiet by default)\n\nBug fixes\n * Typechecking bug in record access\n * Fix bug allowing undefined things in module export lists\n * Fix cousin module imports\n * Fix unparameterized (un)packer serialization\n * Fix error raised when a module exporting a type is compiled \n * Fix out of order record fields in nexus output\n\n0.40.0 [2023.02.04]\n-------------------\n\n * Infer types of records\n * Fix bug in concrete type inference\n * Fix bugs in foreign higher order function code generation\n * Simplify generator code\n * Add many tests\n\n0.39.0 [2023.01.03]\n-------------------\n\n * Add compose operator\n * Allow eta-reduction\n\n0.38.0 [2022.12.23]\n-------------------\n\n * Choose casing convention\n * camel case for terms (for now, underscore is illegal)\n * pascal case for types\n * Fix sub-module handling\n * Fix import/export of type definitions\n * Better error messages for import/export errors\n * Somewhat formatted `typecheck` subcommand output \n * Add option to typecheck backend (concrete types and language selection)\n\n0.37.0 [2022.12.11]\n-------------------\n\n * Remove extra space printed at the end of R JSON\n * Clarify error message for missing exports\n * Clarify error message for missing concrete signature\n * Fix exponential time bug in parser\n * Allow prime characters in names after first position\n * Allow '.' to separate namespaces in imports/modules\n * Fix infinite loop bug when module name != import name\n\n0.36.0 [2022.02.17]\n-------------------\n\n * Separate reals from integers\n * Remove global haskell extensions from package.yaml\n\n0.36.0 [2022.02.17]\n-------------------\n\n * Separate reals from integers\n * Remove global haskell extensions from package.yaml\n\n0.35.0 [2021.10.24]\n-------------------\n\nWhere scoping and a total architecture refactor\n\n * Fix handling for generic parameterized types\n * Improve whitespace handling\n * Statements are order invariant\n * Thread link from expression to source expression down to generators \n * Typecheck over final abstract syntax trees rather than expressions\n * Separate general and concrete typechecking\n * Pretty instances for all data types\n * More testing\n * Agonized over deep and wide structures\n\n0.34.0 [2021.03.05]\n-------------------\n\n * Add handling for C++ float primitives\n * Let C++ programs be imported through a header and shared library\n * Remove semicolon requirement\n * Add hie.yaml for compatibility with hsl\n * Update dependency set to LTS-17.4\n * Add subparsers to CLI with pcapriotti/optparse-applicative \n * Remove brace requirement for modules and `where` statements\n * Add `-o` option to compiler to specify output executable names\n * Acceptable syntax error messages\n\n0.33.0 [2020.11.03]\n-------------------\n\nFirst hackage release\n\n * Haddock documentation\n * Update README\n * In help statements write universal, not concrete, types\n * Make default containers non-existential (probably a bad decision?)\n\n0.32.0 [2020.11.01]\n-------------------\n\n * Add record/table field access\n * Fix JSON handling in nexus\n * Fix nexus bug necessitated escaping quotations and braces\n * Print general types in nexus help\n * Resolve most GHC warnings\n\n0.31.0 [2020.10.29]\n-------------------\n\n * Fix anonymous records in C++\n * Distinguish 'record', 'object', and 'table'\n * Add object handling\n * Add table handling\n\n0.30.0 [2020.10.23]\n-------------------\n\n * Add `object` keyword for defining record types\n * Add full record serialization handling (C++, py, R)\n\n0.29.0 [2020.10.21]\n-------------------\n\n * Add AST directed (de)serialization framework\n * Add type constructors for parameterized types\n\n0.28.0 [2020.10.12]\n-------------------\n\n * Allow import/export of type aliases\n * Refactor with DAGs all through the parser and typechecker\n\n0.27.0 [2020.10.04]\n-------------------\n\n * Add systematic tests for data serialization\n * Fix bug in C++ serialization\n * Move to serialize to dedicated libraries that require no import\n\n0.26.0 [2020.09.27]\n-------------------\n\nAdd `type` keyword for defining type aliases\n\n0.25.0 [2020.09.26]\n-------------------\n\nNo explicit forall. Instead use Haskell convention of generics being lowercase\nand non-generics being uppercase. \n\n * no more explicit \"forall\"\n * generics are lowercase in type signatures\n * non-generic types are uppercase\n * normal functions are lowercase\n * class constructors are uppercase (though handling for this is not yet implemented)\n\n0.24.0 [2020.09.22]\n-------------------\n\nAllow integration of many instances\n\n0.23.0 [2020.05.14]\n\nBug fixes and code cleanup\n\nBug fixes / tests\n - [x] [x] github issue #7 - new Var=> typechecking rule\n - [x] [x] github issue #9 - rewire container type inference\n - [x] [x] github issue #10\n - [x] [x] github issue #11\n\n\n0.22.0 [2020.04.28]\n-------------------\n\nImplement a schema-directed composable serialization system\n\nMajor changes\n * Fully composable serialization over containers and primitives\n * Improved C++ support of generic functions\n * Record support for R and Python3 (not C++ yet)\n * Refactor generator - replace old grammar system\n * Allow arguments to be passed to general functions\n (e.g., `foo x = [x]`, where no specific language is needed) \n\nMinor changes\n * change default python3 interpreter from \"python\" to \"python3\"\n * add default library and tmp paths to config handler\n * test composable serialization functions in all supported languages\n * allow wrapped comments in R\n\nTesting - grammar directed testing\n * test record handling\n * remove and replace out-of-date golden tests\n * systematic argument handling tests\n * systematic manifold form tests\n * systematic interop testing\n\n0.21.0 [2020.03.31]\n-------------------\n\nMajor - add handling and test for many many corner cases\n * Allow export of data statements\n * Allow export of non-functions\n * Allow functions with containers at the root\n * Allow export of 0-argument functions \n\nMinor\n * proof-of-concept composable serialization functions in C++ (cppbase)\n * add python tests\n * make the test output look pretty (no weird whitespace)\n\n0.20.0 [2020.03.23]\n-------------------\n\n * Add composable default types\n\n0.19.1 [2020.02.22]\n-------------------\n\n * bug fixes\n\n0.19.0 [2020.02.20]\n-------------------\n\nMajor changes\n * Allow currying\n * Add realization optimizations\n * Refactor generator into series of clear transformations\n * Added handling for dealing with ambiguous ASTs\n\nMinor bug fixes and updates\n * Prettier code generation for C++, Python and R\n * More detailed comments in generated code\n * Allow tags on parenthesized types\n * Fix bug in functions with multiple parameters \n * Fix bug preventing loading of package metadata \n\n0.18.1 [2019.11.08]\n-------------------\n\n * Fix travis\n * Use C++11 for C++ builds\n * Make .morloc/config optional\n * Fix bug in parsing unit type: `()`\n\n0.18.0 [2019.11.04]\n-------------------\n\n * Add bidirectional type system\n * Allow parameterized concrete types\n * Allow higher-order functions\n * Allow properties to contain multiple terms \n * Add many tests\n * Add module system\n * Allow non-primitive types in lists, tuples, and records\n * Removed arq and SPARQL dependency (very fast compilation)\n\n0.17.4 [2019.06.29]\n-------------------\n\n * Add C and C++ handling\n * Define Ord intance for MTypeMeta\n * Allow pools to be called as executables\n * Add type handling to generators\n * Remove redundant SPARQL queries (better performance)\n * New RDF list semantics\n * Use strings to represent concrete types (e.g. \"char\\*\")\n * Write pretty-printed diagnostic files to `$MORLOC_HOME/tmp` \n * Handling for multiple concrete type signatures (e.g., definition of\n a function in multiple languages).\n * Handling for multiple abstract type signatures\n * Handling for multiple function declarations\n\n0.17.3 [2019.06.14]\n-------------------\n\n * Partial C support\n - execution of sourced functions\n - no composition\n - no foreign calls\n\n * Partial transition to typed generators\n - bound arguments are still not typed correctly\n\n * Use integer IDs to identify manifolds in pools and the nexus (can to make\n calls between them) instead of long, mangled names.\n\n * Replace string names of languages (e.g., \"python\") with a sum type.\n\n0.17.2 [2019.05.05]\n-------------------\n\n Pycon release\n\n0.17.1 [2019.04.26]\n-------------------\n\n * Fix output serialization in generate code\n * Fix module linking in generated code\n\n0.17.0 [2019.04.16]\n-------------------\n\n * Add morloc home\n * Load modules from `$MORLOCHOME/lib`\n * Create monad stack\n\n0.16.2 [2018.03.05]\n-------------------\n\n * Add Zenodo badge making the project citable\n * Move to `morloc-project/morloc` github repo\n\n0.16.1 [2018.09.24]\n-------------------\n\nMinor release consisting of internal refactoring\n\n * Pruned unnecessary code\n * Pruned unnecessary imports\n * Compliance with stricter compile flags\n\n0.16.0 [2018.09.14]\n-------------------\n\n * Write RDF bools in lowercase (\"true\", rather than \"True\"), as per specs\n * Stricter node typing (replace ad hoc names with elements from an ADT)\n * Add very rudimentary typechecking\n * Remove SPARQL server dependency (now there's a sluggish Jena dependency)\n\n0.15.1 [2018.09.10]\n-------------------\n\n * Add error handling and reporting to pools\n * Add type signature comments to generated pools \n * Richer internal data structures\n\n0.15.0 [2018.09.05]\n-------------------\n\n * Generalize code generators using grammar records\n * Add Python compatibility\n * Replace unit tests with golden tests\n * Use docopt and USAGE template for argument handling\n * Report number of arguments in nexus usage statements\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n" }, { "path": "README.md", "content": "

\n \n \"build\n \n \n \"github\n \n \n \"license:\n\n

\n\n

\n Manual |\n Discord |\n Paper |\n X |\n BlueSky |\n Email\n

\n\n\n\n
\n

Morloc

\n

compose functions across languages under a common type system

\n
\n\n**Why use Morloc?**\n\n * Universal function composition: Import functions from multiple languages and\n compose them together under a unified, strongly-typed functional framework.\n\n * Polyglot without boilerplate: Use the best language for each task with no\n manual bindings or interop code.\n\n * Type-directed CLI generation: Write concrete function signatures once and\n automatically generate elegant command-line interfaces with argument\n parsing, validation, help text, and documentation.\n\n * Composable CLI tools: Morloc CLI programs can be composed by simply importing\n them into a new Morloc module and re-exporting their functions.\n\n * Seamless benchmarking and testing: Swap implementations and run the same\n benchmarks/tests across languages with consistent type signatures and data\n representation.\n\n * Design universal libraries: Build abstract, type-driven libraries and\n populate them with foreign language implementations, enabling rigorous code\n organization and reuse.\n\n * Smarter workflows: Replace brittle application/file-based pipelines with\n faster, more maintainable pipelines made from functions acting on structured\n data.\n\n\nBelow is a simple example, for installation details and more examples, see the\n[Manual](https://morloc-project.github.io/docs).\n\nA Morloc module can import functions from foreign languages, assign them general\ntypes, and compose new functions:\n\n\n```morloc\n-- Morloc code, in \"main.loc\"\nmodule m (vsum)\n\nimport root-py\nimport root-cpp\n\nsource Py from \"foo.py\" (\"pmap\")\npmap a b :: (a -> b) -> [a] -> [b] \n\nsource Cpp from \"foo.hpp\" (\"sum\")\nsum :: [Real] -> Real\n\n--' Input numeric vectors that will be summed in parallel\n--' metavar: VECTORS\ntype Vectors = [[Real]]\n\n--' Sum a list of numeric vectors\n--' return: Final sum of all elements in all vectors\nvsum :: Vectors -> Real\nvsum = sum . pmap sum \n```\n\nThe imported code is natural code with no Morloc-specific dependencies.\n\nBelow is the C++ code that defines `sum` as a function of a standard C++ vector\nof `double`s that returns a `double`:\n\n```C++\n// C++ code, in \"foo.hpp\"\n\n#pragma once\n\n#include \n#include \n\ndouble sum(std::vector xs) {\n return std::accumulate(\n xs.begin(), xs.end(), 0.0);\n}\n```\n\nBelow is Python code that defines a parallel map function:\n\n```python\n# Python code, in \"foo.py\"\n\nimport multiprocessing as mp\n\n# Parallel map function\ndef pmap(f, xs):\n with mp.Pool() as pool:\n results = pool.map(f, xs)\n return results\n```\n\nThis program can be compiled and run as below:\n\n```\n$ menv morloc make main.loc\n\n$ menv ./nexus vsum -h\nUsage: ./nexus vsum VECTORS\n\nSum a list of numeric vectors\n\nPositional arguments:\n VECTORS Input numeric vectors that will be summed in parallel\n type: [[Real]]\n\nReturn: Real\n Final sum of all elements in all vectors\n\n$ menv ./nexus vsum [[1.2],[0,0.1]]\n1.3\n```\n" }, { "path": "bench/Bench.hs", "content": "{-|\nModule : Bench\nDescription : Benchmark suite for morloc compiler\nCopyright : (c) Zebulun Arendsee, 2016-2026\nLicense : Apache-2.0\nMaintainer : z@morloc.io\n\nThis benchmark suite tracks performance of key compiler components:\n- Parser: parsing .loc source files\n- Type checker: type inference and checking\n- Code generator: nexus and pool generation\n\nRun with: stack bench\nRun with options: stack bench --benchmark-arguments '--csv bench-results.csv'\n-}\n\nmodule Main (main) where\n\nimport Test.Tasty.Bench\nimport qualified Data.Text as T\nimport System.FilePath (())\nimport qualified System.Directory as SD\n\nimport Morloc (typecheckFrontend, typecheck)\nimport Morloc.Namespace.Prim (Code(..), Defaultable(..))\nimport Morloc.Namespace.State (Config(..), MorlocMonad, MorlocError)\nimport qualified Morloc.Monad as MM\n\n-- | Helper to read benchmark test files\nreadTestFile :: FilePath -> IO Code\nreadTestFile name = do\n let path = \"bench\" \"test-data\" name\n Code . T.pack <$> readFile path\n\n-- | Create a minimal config for benchmarking\nemptyConfig :: IO Config\nemptyConfig = do\n home <- SD.getHomeDirectory\n return $ Config\n { configHome = home <> \"/.local/share/morloc\"\n , configLibrary = home <> \"/.local/share/src/morloc\"\n , configPlane = \"default\"\n , configPlaneCore = \"morloclib\"\n , configTmpDir = home <> \"/.morloc/tmp\"\n , configBuildConfig = home <> \"/.morloc/.build-config.yaml\"\n , configLangOverrides = mempty\n }\n\n-- | Run a MorlocMonad action for benchmarking\nrunBench :: MorlocMonad a -> IO (Either MorlocError a)\nrunBench action = do\n config <- emptyConfig\n ((result, _), _) <- MM.runMorlocMonad Nothing 0 config defaultValue action\n return result\n\nmain :: IO ()\nmain = defaultMain\n [ bgroup \"Parser\"\n [ bench \"parse-simple\" $ whnfIO $ do\n code <- readTestFile \"simple.loc\"\n runBench (typecheckFrontend Nothing code)\n\n , bench \"parse-interop\" $ whnfIO $ do\n code <- readTestFile \"interop.loc\"\n runBench (typecheckFrontend Nothing code)\n\n , bench \"parse-complex-types\" $ whnfIO $ do\n code <- readTestFile \"complex-types.loc\"\n runBench (typecheckFrontend Nothing code)\n ]\n\n , bgroup \"Type Checker\"\n [ bench \"typecheck-simple\" $ whnfIO $ do\n code <- readTestFile \"simple.loc\"\n runBench (typecheck Nothing code)\n\n , bench \"typecheck-interop\" $ whnfIO $ do\n code <- readTestFile \"interop.loc\"\n runBench (typecheck Nothing code)\n\n , bench \"typecheck-complex-types\" $ whnfIO $ do\n code <- readTestFile \"complex-types.loc\"\n runBench (typecheck Nothing code)\n ]\n\n -- Note: Code generation benchmarks commented out as they require\n -- file system access and module initialization\n -- Uncomment after setting up appropriate test environment\n {-\n , bgroup \"Code Generation\"\n [ bench \"generate-simple\" $ whnfIO $ do\n code <- readTestFile \"simple.loc\"\n runBench (writeProgram Nothing code)\n ]\n -}\n ]\n" }, { "path": "bench/test-data/complex-types.loc", "content": "module complex (processRecords)\n\ntype Record = {id :: Int, name :: Str, values :: [Real]}\n\nprocessRecords :: [Record] -> [Int]\nprocessRecords rs = map (\\r -> r.id) rs\n" }, { "path": "bench/test-data/interop.loc", "content": "module interop (processList)\n\nimport math (sqrt from python)\n\nprocessList :: [Real] -> [Real]\nprocessList xs = map sqrt xs\n" }, { "path": "bench/test-data/simple.loc", "content": "module simple (add, mul)\n\nadd :: Int -> Int -> Int\nadd x y = x + y\n\nmul :: Int -> Int -> Int\nmul x y = x * y\n" }, { "path": "container/Makefile", "content": "# Pushing to the github registry requires a personal token with package\n# permissions. Login is required, for example:\n#\n# $ echo $GITHUB_TOKEN | podman login ghcr.io -u morloc-project --password-stdin\n#\n# I am currently using the \"classic\" token (ghp_*), not the fine-grained\n# token. These need to be refreshed every few months\n#\n# You need to provide the desired morloc version as an environmental variable,\n# so run the make commands like so:\n#\n# $ make MORLOC_VERSION=0.70.0 build-tiny\n\ndefine HEREDOC\n#!/bin/bash\nmkdir -p ~/.morloc\npodman run --rm \\\n -e HOME=$$HOME \\\n -v $$HOME/.morloc:$$HOME/.morloc \\\n -v $$PWD:$$HOME \\\n -w $$HOME ghcr.io/morloc-project/morloc/morloc-tiny:edge \\\n morloc \"$$@\"\nendef\nexport HEREDOC\n\ninstall:\n\t# Pull the exact version to ensure it exists locally\n\tpodman pull ghcr.io/morloc-project/morloc/morloc-tiny:edge\n\t@mkdir -p ${HOME}/bin\n\t@echo \"$$HEREDOC\" > ${HOME}/bin/morloc-edge\n\tchmod 755 ${HOME}/bin/morloc-edge\n\n\n# Build a container that just has the morloc compiler.\n# Context is the repo root (..) so COPY gets the local source tree.\nbuild-tiny:\n\tpodman build --no-cache --force-rm \\\n\t\t-t ghcr.io/morloc-project/morloc/morloc-tiny:$(MORLOC_VERSION) \\\n\t\t-t ghcr.io/morloc-project/morloc/morloc-tiny:edge \\\n\t\t-f tiny/Dockerfile ..\n\n# Build the full interactive development image.\n# Copies the morloc binary from the locally-built morloc-tiny image.\n# Run build-tiny first.\nbuild-full:\n\tpodman build --no-cache --force-rm \\\n\t\t--build-arg MORLOC_VERSION=$(MORLOC_VERSION) \\\n\t\t-t ghcr.io/morloc-project/morloc/morloc-full:$(MORLOC_VERSION) \\\n\t\t-t ghcr.io/morloc-project/morloc/morloc-full:edge \\\n\t\tfull\n\n# Build the required docker image\nbuild-test:\n\tpodman build --no-cache --force-rm -t ghcr.io/morloc-project/morloc/morloc-test test\n\tpodman tag ghcr.io/morloc-project/morloc/morloc-test ghcr.io/morloc-project/morloc/morloc-test:edge\n\nshell:\n\tpodman run --shm-size=4g --rm -it ghcr.io/morloc-project/morloc/morloc-full:edge /bin/bash\n\nshell-tiny:\n\tpodman run --shm-size=4g --rm -it ghcr.io/morloc-project/morloc/morloc-tiny:edge /bin/bash\n\nshell-test:\n\tpodman run --shm-size=4g --rm -it ghcr.io/morloc-project/morloc/morloc-test /bin/bash\n\n# push local containers to the github registry\n# Ensures edge moves to the same digest as $(MORLOC_VERSION)\npush:\n\t# Push tiny version and edge\n\tpodman push ghcr.io/morloc-project/morloc/morloc-tiny:$(MORLOC_VERSION)\n\tpodman push ghcr.io/morloc-project/morloc/morloc-tiny:edge\n\t# Push full version and edge\n\tpodman push ghcr.io/morloc-project/morloc/morloc-full:$(MORLOC_VERSION)\n\tpodman push ghcr.io/morloc-project/morloc/morloc-full:edge\n\t# Push test (no moving tag)\n\tpodman push ghcr.io/morloc-project/morloc/morloc-test:edge\n\n# retrieve the latest morloc builds from the github registry\npull:\n\tpodman pull ghcr.io/morloc-project/morloc/morloc-tiny:$(MORLOC_VERSION)\n\tpodman pull ghcr.io/morloc-project/morloc/morloc-full:$(MORLOC_VERSION)\n\tpodman pull ghcr.io/morloc-project/morloc/morloc-test\n\tpodman pull ghcr.io/morloc-project/morloc/morloc-test:edge\n\n# Convenience: pull moving edge tags\npull-edge:\n\tpodman pull ghcr.io/morloc-project/morloc/morloc-tiny:edge\n\tpodman pull ghcr.io/morloc-project/morloc/morloc-full:edge\n\n# Cleanup of podman images may be done as follows\n# $ podman image prune\n# $ podman container prune\n# $ podman image rm -f $(podman image ls -q)\n" }, { "path": "container/full/Dockerfile", "content": "# Copy the morloc binary from the matching tiny image\nARG MORLOC_VERSION=edge\nFROM ghcr.io/morloc-project/morloc/morloc-tiny:${MORLOC_VERSION} AS morloc-bin\n\nFROM docker.io/library/ubuntu:24.04\n\nLABEL org.opencontainers.image.source=https://github.com/morloc-project/morloc\nLABEL org.opencontainers.image.description=\"A morloc container intended for interactive shell use\"\nLABEL org.opencontainers.image.licenses=Apache-2.0\n\nCOPY --from=morloc-bin /bin/morloc /bin/morloc\nCOPY --from=morloc-bin /opt/morloc-rust-bin /opt/morloc-rust-bin\n\nWORKDIR $HOME\n\nENV DEBIAN_FRONTEND=noninteractive\nENV PATH=\"/root/.local/bin:/root/.local/share/morloc/bin:${PATH}\"\n# Pre-built Rust binaries: morloc init uses these directly (no cargo needed)\nENV MORLOC_RUST_BIN=/opt/morloc-rust-bin\n\nRUN apt-get update\n\n# Set the timezone, this avoids hanging later on\nRUN TZ=Antarctica/Troll apt-get -y install tzdata\n\nRUN apt-get install -y r-base python3 python3-dev python3-pip python3-numpy libgsl-dev git curl\n\nRUN python3 -m pip install --break-system-packages --upgrade setuptools\n\n# Setup the morloc home (uses pre-built Rust binaries, no cargo needed)\nRUN morloc init\n# Copy morloc-nexus to a system path so it remains on PATH even when\n# morloc-manager bind-mounts over ~/.local/bin with a host directory\nRUN cp /root/.local/share/morloc/bin/morloc-nexus /usr/local/bin/morloc-nexus\nRUN echo \"lang_python3 : python3\" >> $HOME/.local/share/morloc/config\n\nRUN morloc install stdlib\n\nRUN apt-get install -y vim\n\n# Copy over custom vimrc\nCOPY assets/vimrc /root/.vimrc\nCOPY assets/README /root/README\n\n# Set up vim highlighting for morloc\nRUN git clone https://github.com/morloc-project/vimmorloc \\\n && mkdir -p ~/.vim/syntax/ \\\n && mkdir -p ~/.vim/ftdetect/ \\\n && cp vimmorloc/loc.vim ~/.vim/syntax/ \\\n && echo 'au BufRead,BufNewFile *.loc set filetype=loc' > ~/.vim/ftdetect/loc.vim \\\n && rm -rf vimmorloc\n\nRUN git clone https://github.com/morloc-project/morloc ~/morloc\n\nCOPY assets/bashrc /etc/bash.bashrc\n\n# Cleanup to reduce image size\nRUN apt-get clean && rm -rf /var/lib/apt/lists/*\n" }, { "path": "container/full/assets/README", "content": "This container has a full morloc installation and language support for R, Python3, and C++.\n\nThe ~/tests folder contains a set of tests from the morloc test suite. These may\nserve as simplistic examples of morloc scripts. Though they are designed for\ntesting the language not pedagogy.\n" }, { "path": "container/full/assets/bashrc", "content": "# basic morloc bashrc\n\n# If not running interactively, don't do anything\ncase $- in\n *i*) ;;\n *) return;;\nesac\n\n# History Configuration\nHISTCONTROL=ignoreboth:erasedups\nHISTSIZE=10000\nHISTFILESIZE=20000\nshopt -s histappend\n\n# Shell Options\nshopt -s checkwinsize\nshopt -s globstar 2>/dev/null\n\n# Colors\nif [ -x /usr/bin/dircolors ]; then\n test -r ~/.dircolors && eval \"$(dircolors -b ~/.dircolors)\" || eval \"$(dircolors -b)\"\nfi\n\n# Prompt\n# Detect if we have color support\nif [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then\n # Define colors\n RESET='\\[\\033[0m\\]'\n BOLD='\\[\\033[1m\\]'\n DIM='\\[\\033[2m\\]'\n \n # Color palette\n BLUE='\\[\\033[38;5;75m\\]'\n GREEN='\\[\\033[38;5;114m\\]'\n YELLOW='\\[\\033[38;5;221m\\]'\n GRAY='\\[\\033[38;5;245m\\]'\n RED='\\[\\033[38;5;204m\\]'\n\n DEVNAME=\"\"\n if [ ! -z \"$MORLOC_ENV_NAME\" ]; then\n DEVNAME=\"${DIM}${GRAY}(${MORLOC_ENV_NAME})${RESET} \"\n fi \n\n MORLOC_VERSION=$(morloc --version)\n if [ $? -eq 0 ]; then\n MORLOC_VERSION=\"-${MORLOC_VERSION}\"\n fi\n \n # Format: (container) morloc $\n PS1=\"${DEVNAME}${YELLOW}morloc${MORLOC_VERSION}${RESET} $ \"\nelse\n # Fallback for no color\n PS1=\"morloc $ \"\nfi\n\n# aliases\nalias ls='ls --color=auto'\nalias ll='ls -lh'\nalias la='ls -lah'\nalias l='ls -CF'\n\n# coloring\nalias grep='grep --color=auto'\nalias fgrep='fgrep --color=auto'\nalias egrep='egrep --color=auto'\n\n# fast navigation\nalias ..='cd ..'\nalias ...='cd ../..'\nalias ....='cd ../../..'\nalias .....='cd ../../../..'\n\n# setup path\n[ -d \"$HOME/.local/bin\" ] && PATH=\"$HOME/.local/bin:$PATH\"\n[ -d \"$HOME/bin\" ] && PATH=\"$HOME/bin:$PATH\"\n\n# completions\nif ! shopt -oq posix; then\n if [ -f /usr/share/bash-completion/bash_completion ]; then\n . /usr/share/bash-completion/bash_completion\n elif [ -f /etc/bash_completion ]; then\n . /etc/bash_completion\n fi\nfi\n" }, { "path": "container/full/assets/vimrc", "content": "\" Jump to the last position when reopening a file\nau BufReadPost * if line(\"'\\\"\") > 1 && line(\"'\\\"\") <= line(\"$\") | exe \"normal! g'\\\"\" | endif\n\n\" Load indentation rules and plugins according to the detected filetype.\nfiletype plugin indent on\nsyntax on\nset showcmd \" Show (partial) command in status line.\nset showmatch \" Show matching brackets.\nset ignorecase \" Do case insensitive matching\nset smartcase \" Do smart case matching\nset incsearch \" Incremental search\nset autowrite \" Automatically save before commands like :next and :make\nset hidden \" Hide buffers when they are abandoned\n\n\" An OK colorscheme\ncolorscheme torte\n" }, { "path": "container/static-build/Dockerfile", "content": "# Portable build of libmorloc.so, morloc-nexus, and morloc-manager.\n#\n# Strategy:\n# - libmorloc.so: shared library built on Ubuntu 20.04 (glibc >= 2.31)\n# - morloc-nexus: dynamically linked to libmorloc.so (glibc >= 2.31)\n# - morloc-manager: fully static binary (Alpine/musl, runs on any Linux)\n#\n# Build:\n# docker build -t morloc-rust-build -f container/static-build/Dockerfile .\n# docker run --rm -v $(pwd)/out:/out morloc-rust-build\n#\n# Output:\n# ./out/libmorloc.so (shared library, glibc >= 2.31)\n# ./out/morloc-nexus (binary, glibc >= 2.31, links libmorloc.so)\n# ./out/morloc-manager (static binary, runs on any Linux including NixOS)\n\n# ===========================================================================\n# Stage 1: Build libmorloc.so + morloc-nexus on Ubuntu (glibc)\n#\n# morloc-nexus dynamically links libmorloc.so at runtime, so both must be\n# built against the same libc. Ubuntu 20.04 gives glibc 2.31 forward compat.\n# ===========================================================================\nFROM docker.io/library/ubuntu:20.04 AS glibc-builder\n\nENV DEBIAN_FRONTEND=noninteractive\n\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n curl ca-certificates gcc g++ make pkg-config \\\n && rm -rf /var/lib/apt/lists/*\n\nRUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable \\\n && /root/.cargo/bin/cargo --version\nENV PATH=\"/root/.cargo/bin:${PATH}\"\n\nWORKDIR /build\n\n# Copy Cargo manifests first for dependency caching\nCOPY data/rust/Cargo.toml data/rust/Cargo.lock ./\nCOPY data/rust/morloc-runtime/Cargo.toml ./morloc-runtime/\nCOPY data/rust/morloc-nexus/Cargo.toml ./morloc-nexus/\nCOPY data/rust/morloc-manager/Cargo.toml ./morloc-manager/\n\n# Create dummy source files for dependency pre-build\nRUN mkdir -p morloc-runtime/src morloc-nexus/src morloc-manager/src \\\n && echo \"fn main() {}\" > morloc-nexus/src/main.rs \\\n && echo \"fn main() {}\" > morloc-manager/src/main.rs \\\n && echo \"\" > morloc-runtime/src/lib.rs\n\nRUN cargo build --release -p morloc-runtime 2>/dev/null || true\n\n# Copy actual source\nCOPY data/rust/ ./\n\n# Force rebuild after copying real source over dummy stubs\nRUN touch morloc-runtime/src/lib.rs morloc-nexus/src/main.rs\n\n# Build libmorloc.so from staticlib via gcc --whole-archive to export ALL\n# symbols (internal Rust runtime state must be visible to language extensions)\nRUN cargo build --release -p morloc-runtime\n\nRUN mkdir -p /root/.local/share/morloc/lib \\\n && gcc -shared -o /root/.local/share/morloc/lib/libmorloc.so \\\n -Wl,--whole-archive target/release/libmorloc_runtime.a -Wl,--no-whole-archive \\\n -lpthread -lrt -ldl -lm\n\n# Build morloc-nexus (dynamically links libmorloc.so)\nRUN cargo build --release -p morloc-nexus\n\nRUN strip /root/.local/share/morloc/lib/libmorloc.so target/release/morloc-nexus\n\n# ===========================================================================\n# Stage 2: Build morloc-manager on Alpine (musl, fully static)\n#\n# morloc-manager has no native dependencies — pure Rust crates only.\n# Building on Alpine produces a fully static musl binary that runs on any\n# Linux, including NixOS and minimal containers.\n# ===========================================================================\nFROM docker.io/library/alpine:3.20 AS musl-builder\n\nRUN apk add --no-cache curl gcc musl-dev\n\nRUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable \\\n && /root/.cargo/bin/cargo --version\nENV PATH=\"/root/.cargo/bin:${PATH}\"\n\nWORKDIR /build\n\n# Copy Cargo manifests for dependency caching\nCOPY data/rust/Cargo.toml data/rust/Cargo.lock ./\nCOPY data/rust/morloc-runtime/Cargo.toml ./morloc-runtime/\nCOPY data/rust/morloc-nexus/Cargo.toml ./morloc-nexus/\nCOPY data/rust/morloc-manager/Cargo.toml ./morloc-manager/\n\nRUN mkdir -p morloc-runtime/src morloc-nexus/src morloc-manager/src \\\n && echo \"fn main() {}\" > morloc-nexus/src/main.rs \\\n && echo \"fn main() {}\" > morloc-manager/src/main.rs \\\n && echo \"\" > morloc-runtime/src/lib.rs\n\nRUN cargo build --release -p morloc-manager 2>/dev/null || true\n\nCOPY data/rust/ ./\n\n# Force rebuild: cargo may skip if it thinks the cached dummy binary is fresh\nRUN touch morloc-manager/src/main.rs \\\n && cargo build --release -p morloc-manager\n\nRUN strip target/release/morloc-manager\n\n# ===========================================================================\n# Stage 3: Verify and output\n# ===========================================================================\nFROM docker.io/library/alpine:3.20\n\nRUN apk add --no-cache file\n\nCOPY --from=glibc-builder /root/.local/share/morloc/lib/libmorloc.so /out-staging/libmorloc.so\nCOPY --from=glibc-builder /build/target/release/morloc-nexus /out-staging/morloc-nexus\nCOPY --from=musl-builder /build/target/release/morloc-manager /out-staging/morloc-manager\n\n# Verify morloc-manager is static (no dynamic linker needed)\nRUN file /out-staging/morloc-manager \\\n && ! ldd /out-staging/morloc-manager 2>&1 | grep -q \"=>\" \\\n && echo \"OK: morloc-manager has no dynamic dependencies\"\n\nCMD [\"sh\", \"-c\", \"cp /out-staging/libmorloc.so /out-staging/morloc-nexus /out-staging/morloc-manager /out/ && echo 'Wrote libmorloc.so, morloc-nexus, morloc-manager' && ls -lh /out/libmorloc.so /out/morloc-nexus /out/morloc-manager && echo && file /out/morloc-manager\"]\n" }, { "path": "container/static-build/build.sh", "content": "#!/bin/sh\n# Build portable libmorloc.so, morloc-nexus, and morloc-manager.\n#\n# Usage:\n# ./container/static-build/build.sh\n#\n# Output:\n# ./out/libmorloc.so (shared library, glibc >= 2.31)\n# ./out/morloc-nexus (binary, glibc >= 2.31, links libmorloc.so)\n# ./out/morloc-manager (static binary, runs on any Linux)\n\nset -e\n\nSCRIPT_DIR=\"$(cd \"$(dirname \"$0\")\" && pwd)\"\nPROJECT_DIR=\"$(cd \"$SCRIPT_DIR/../..\" && pwd)\"\n\n# Detect container engine (prefer podman)\nif command -v podman >/dev/null 2>&1; then\n ENGINE=podman\nelif command -v docker >/dev/null 2>&1; then\n ENGINE=docker\nelse\n echo \"Error: neither podman nor docker found\" >&2\n exit 1\nfi\n\necho \"Building libmorloc.so, morloc-nexus, and morloc-manager with $ENGINE...\"\n\nmkdir -p \"$PROJECT_DIR/out\"\n\n$ENGINE build \\\n -t morloc-rust-build \\\n -f \"$SCRIPT_DIR/Dockerfile\" \\\n \"$PROJECT_DIR\"\n\n$ENGINE run --rm \\\n -v \"$PROJECT_DIR/out:/out\" \\\n morloc-rust-build\n\necho \"\"\necho \"Binaries:\"\nls -lh \"$PROJECT_DIR/out/libmorloc.so\" \"$PROJECT_DIR/out/morloc-nexus\" \"$PROJECT_DIR/out/morloc-manager\"\nfile \"$PROJECT_DIR/out/libmorloc.so\" \"$PROJECT_DIR/out/morloc-nexus\" \"$PROJECT_DIR/out/morloc-manager\"\n" }, { "path": "container/test/Dockerfile", "content": "FROM docker.io/library/ubuntu:24.04\n\nENV DEBIAN_FRONTEND=noninteractive\n\nRUN apt-get update && apt-get install -y \\\n git \\\n gdb \\\n curl \\\n pkg-config \\\n libglib2.0-dev \\\n build-essential \\\n libffi-dev \\\n libgmp-dev \\\n zlib1g-dev\n\n# Install GHCup into /opt/ghcup (accessible to all users)\nENV GHCUP_INSTALL_BASE_PREFIX=/opt\nENV BOOTSTRAP_HASKELL_NONINTERACTIVE=1\nRUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh\n\n# Add ghcup to PATH\nENV PATH=\"/opt/.ghcup/bin:${PATH}\"\n\n# Install Rust toolchain (needed by morloc init to build libmorloc.so)\nRUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y\nENV PATH=\"/root/.cargo/bin:${PATH}\"\n\n# Set the timezone, this avoids hanging later on\nRUN TZ=Antarctica/Troll apt-get -y install tzdata\n\n# hyperfine is needed for shell benchmarks\nRUN apt-get install -y r-base python3 python3-dev python3-pip libgsl-dev git hyperfine\nRUN python3 -m pip install --break-system-packages --upgrade setuptools numpy pyarrow\n\n# Set up R environment\n# stringi is needed for benchmarks\nRUN Rscript -e 'install.packages(\"stringi\", repos = \"https://cloud.r-project.org\")'\n\n# Install R Arrow package for cross-language Arrow IPC support\nENV LIBARROW_MINIMAL=true\nENV ARROW_S3=OFF\nENV ARROW_GCS=OFF\nRUN Rscript -e 'install.packages(\"arrow\", repos = \"https://cloud.r-project.org\")'\n\n# Cleanup to reduce image size\nRUN apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# Create /home/dev with permissive access so any --user UID:GID can write here.\n# morloc-manager bind-mounts .stack and .local into this directory.\nRUN mkdir -p /home/dev && chmod 1777 /home/dev\n\nCOPY assets/bashrc /etc/bash.bashrc\n" }, { "path": "container/test/assets/bashrc", "content": "# basic morloc bashrc\n\n# If not running interactively, don't do anything\ncase $- in\n *i*) ;;\n *) return;;\nesac\n\n# History Configuration\nHISTCONTROL=ignoreboth:erasedups\nHISTSIZE=10000\nHISTFILESIZE=20000\nshopt -s histappend\n\n# Shell Options\nshopt -s checkwinsize\nshopt -s globstar 2>/dev/null\n\n# Colors\nif [ -x /usr/bin/dircolors ]; then\n test -r ~/.dircolors && eval \"$(dircolors -b ~/.dircolors)\" || eval \"$(dircolors -b)\"\nfi\n\n# Prompt\n# Detect if we have color support\nif [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then\n # Define colors\n RESET='\\[\\033[0m\\]'\n BOLD='\\[\\033[1m\\]'\n DIM='\\[\\033[2m\\]'\n \n # Color palette\n BLUE='\\[\\033[38;5;75m\\]'\n GREEN='\\[\\033[38;5;114m\\]'\n YELLOW='\\[\\033[38;5;221m\\]'\n GRAY='\\[\\033[38;5;245m\\]'\n RED='\\[\\033[38;5;204m\\]'\n\n DEVNAME=\"\"\n if [ ! -z \"$MORLOC_ENV_NAME\" ]; then\n DEVNAME=\"${DIM}${GRAY}(${MORLOC_ENV_NAME})${RESET} \"\n fi \n \n # Format: (container) morloc $\n PS1=\"${DEVNAME}${YELLOW}morloc-dev${RESET} $ \"\nelse\n # Fallback for no color\n PS1=\"morloc-dev $ \"\nfi\n\n# aliases\nalias ls='ls --color=auto'\nalias ll='ls -lh'\nalias la='ls -lah'\nalias l='ls -CF'\n\n# coloring\nalias grep='grep --color=auto'\nalias fgrep='fgrep --color=auto'\nalias egrep='egrep --color=auto'\n\n# fast navigation\nalias ..='cd ..'\nalias ...='cd ../..'\nalias ....='cd ../../..'\nalias .....='cd ../../../..'\n\n# setup path\n[ -d \"$HOME/.local/bin\" ] && PATH=\"$HOME/.local/bin:$PATH\"\n[ -d \"$HOME/bin\" ] && PATH=\"$HOME/bin:$PATH\"\n\n# completions\nif ! shopt -oq posix; then\n if [ -f /usr/share/bash-completion/bash_completion ]; then\n . /usr/share/bash-completion/bash_completion\n elif [ -f /etc/bash_completion ]; then\n . /etc/bash_completion\n fi\nfi\n" }, { "path": "container/tiny/Dockerfile", "content": "###############################################################################\n# Stage 1: Build the morloc compiler from local source\nFROM docker.io/library/ubuntu:24.04 AS morloc-build\n\nENV DEBIAN_FRONTEND=noninteractive\n\nRUN apt-get update && apt-get install -y git curl pkg-config libglib2.0-dev\n\n# Install stack\nRUN curl -SL https://get.haskellstack.org/ | sh\n\n# Copy the local source tree (no git clone -- builds exactly this checkout)\nCOPY . /morloc\n\n# Build morloc\nRUN cd /morloc && stack install --no-run-tests\n\n###############################################################################\n# Stage 2: Build Rust binaries (libmorloc.so + morloc-nexus + morloc-manager)\nFROM docker.io/library/ubuntu:24.04 AS rust-build\n\nENV DEBIAN_FRONTEND=noninteractive\n\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n curl ca-certificates gcc g++ make pkg-config \\\n && rm -rf /var/lib/apt/lists/*\n\nRUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable\nENV PATH=\"/root/.cargo/bin:${PATH}\"\n\nCOPY data/rust/ /build/\nWORKDIR /build\n\n# Build libmorloc.so from staticlib\nRUN cargo build --release -p morloc-runtime\nRUN gcc -shared -o libmorloc.so \\\n -Wl,--whole-archive target/release/libmorloc_runtime.a -Wl,--no-whole-archive \\\n -lpthread -lrt -ldl -lm\n\n# Install libmorloc.so where the nexus linker expects it\nRUN mkdir -p /root/.local/share/morloc/lib \\\n && cp libmorloc.so /root/.local/share/morloc/lib/\n\n# Build morloc-nexus and morloc-manager\nRUN cargo build --release -p morloc-nexus\nRUN cargo build --release -p morloc-manager\n\n# Strip all\nRUN strip libmorloc.so target/release/morloc-nexus target/release/morloc-manager\n\n# Collect into a single directory\nRUN mkdir -p /rust-bin \\\n && cp libmorloc.so /rust-bin/ \\\n && cp target/release/morloc-nexus /rust-bin/ \\\n && cp target/release/morloc-manager /rust-bin/\n\n###############################################################################\n# Stage 3: Final minimal image with compiler + pre-built Rust binaries\n\nLABEL org.opencontainers.image.source=https://github.com/morloc-project/morloc\nLABEL org.opencontainers.image.description=\"Morloc executable in a minimal container\"\nLABEL org.opencontainers.image.licenses=Apache-2.0\n\nFROM docker.io/library/ubuntu:24.04\nCOPY --from=morloc-build /root/.local/bin/morloc /bin/morloc\nCOPY --from=rust-build /rust-bin/ /opt/morloc-rust-bin/\n\n# morloc make builds C++ pools, so g++ is needed\nRUN apt-get update && apt-get install -y --no-install-recommends g++ \\\n && apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# Point morloc init at pre-built binaries (no cargo needed at runtime)\nENV MORLOC_RUST_BIN=/opt/morloc-rust-bin\n" }, { "path": "data/lang/c/lang.yaml", "content": "# C language metadata for morloc compiler\nname: c\nextension: c\naliases: []\nis_compiled: true\nrun_command: []\nserial_type: \"\"\ncost: 1\n" }, { "path": "data/lang/cpp/cppmorloc.cpp", "content": "#include \n#include \n#include \n\n#include \"morloc.h\"\n\nabsptr_t cpp_rel2abs(relptr_t ptr){\n char* errmsg = NULL;\n absptr_t absptr = rel2abs(ptr, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return absptr;\n}\n\nrelptr_t abs2rel_cpp(absptr_t ptr){\n char* errmsg = NULL;\n relptr_t relptr = abs2rel(ptr, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return relptr;\n}\n\nbool shfree_cpp(absptr_t ptr){\n char* errmsg = NULL;\n bool success = shfree(ptr, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return success;\n}\n\nSchema* parse_schema_cpp(const char* schema_ptr){\n char* errmsg = NULL;\n Schema* schema = parse_schema(schema_ptr, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return schema;\n}\n\nvoid* shmalloc_cpp(size_t size){\n char* errmsg = NULL;\n void* new_ptr = shmalloc(size, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return new_ptr;\n}\n\nshm_t* shinit_cpp(const char* shm_basename, size_t volume_index, size_t shm_size) {\n char* errmsg = NULL;\n shm_t* new_ptr = shinit(shm_basename, volume_index, shm_size, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return new_ptr;\n}\n\nint pack_with_schema_cpp(const void* mlc, const Schema* schema, char** mpk, size_t* mpk_size){\n char* errmsg = NULL;\n int exitcode = pack_with_schema(mlc, schema, mpk, mpk_size, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return exitcode;\n}\n\nint unpack_with_schema_cpp(const char* mgk, size_t mgk_size, const Schema* schema, void** mlcptr){\n char* errmsg = NULL;\n int exitcode = unpack_with_schema(mgk, mgk_size, schema, mlcptr, &errmsg);\n if(errmsg != NULL){\n std::string msg(errmsg); free(errmsg);\n throw std::runtime_error(msg);\n }\n return exitcode;\n}\n" }, { "path": "data/lang/cpp/cppmorloc.hpp", "content": "#ifndef __CPPMORLOC_HPP__\n#define __CPPMORLOC_HPP__\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"morloc.h\"\n#include \"mlc_tensor.hpp\"\n\n// ============================================================\n// Type traits for container dispatch\n// ============================================================\n\ntemplate struct is_std_vector : std::false_type {};\ntemplate struct is_std_vector> : std::true_type {};\n\ntemplate struct is_std_list : std::false_type {};\ntemplate struct is_std_list> : std::true_type {};\n\ntemplate struct is_std_forward_list : std::false_type {};\ntemplate struct is_std_forward_list> : std::true_type {};\n\ntemplate struct is_std_deque : std::false_type {};\ntemplate struct is_std_deque> : std::true_type {};\n\ntemplate struct is_std_stack : std::false_type {};\ntemplate struct is_std_stack> : std::true_type {};\n\ntemplate struct is_std_queue : std::false_type {};\ntemplate struct is_std_queue> : std::true_type {};\n\ntemplate struct is_std_tuple : std::false_type {};\ntemplate struct is_std_tuple> : std::true_type {};\n\ntemplate struct is_std_pair : std::false_type {};\ntemplate struct is_std_pair> : std::true_type {};\n\ntemplate struct is_std_optional : std::false_type {};\ntemplate struct is_std_optional> : std::true_type {};\n\ntemplate\ninline constexpr bool is_non_vector_container_v =\n is_std_list::value || is_std_forward_list::value ||\n is_std_deque::value || is_std_stack::value ||\n is_std_queue::value;\n\n\n// ============================================================\n// Container-to-vector conversion\n// ============================================================\n\ntemplate\nauto to_vector(const Container& c) {\n using T = typename Container::value_type;\n if constexpr (is_std_stack::value) {\n std::vector v;\n auto copy = c;\n while (!copy.empty()) { v.push_back(copy.top()); copy.pop(); }\n std::reverse(v.begin(), v.end());\n return v;\n } else if constexpr (is_std_queue::value) {\n std::vector v;\n auto copy = c;\n while (!copy.empty()) { v.push_back(copy.front()); copy.pop(); }\n return v;\n } else {\n return std::vector(c.begin(), c.end());\n }\n}\n\n\n// ============================================================\n// C runtime wrappers (implementations in cppmorloc.cpp)\n// ============================================================\n\nabsptr_t cpp_rel2abs(relptr_t ptr);\nrelptr_t abs2rel_cpp(absptr_t ptr);\n\n// Resolve a relative pointer using either base-pointer arithmetic (inline data)\n// or SHM. When base_ptr is non-null, data lives in a contiguous malloc'd blob.\nstatic inline void* resolve_relptr_cpp(relptr_t relptr, const void* base_ptr) {\n if (base_ptr) {\n return (char*)base_ptr + relptr;\n }\n return cpp_rel2abs(relptr);\n}\nbool shfree_cpp(absptr_t ptr);\nSchema* parse_schema_cpp(const char* schema_ptr);\nvoid* shmalloc_cpp(size_t size);\nshm_t* shinit_cpp(const char* shm_basename, size_t volume_index, size_t shm_size);\nint pack_with_schema_cpp(const void* mlc, const Schema* schema, char** mpk, size_t* mpk_size);\nint unpack_with_schema_cpp(const char* mgk, size_t mgk_size, const Schema* schema, void** mlcptr);\n\n\n// ============================================================\n// mpk_pack / mpk_unpack declarations\n// ============================================================\n\ntemplate\nstd::vector mpk_pack(const T& data, const std::string& schema_str);\n\ntemplate\nT mpk_unpack(const std::vector& packed_data, const std::string& schema_str);\n\n\n// ============================================================\n// schema_alignment (C++ mirror of the C function in schema.c)\n// ============================================================\n\ninline size_t schema_alignment_cpp(const Schema* schema) {\n switch (schema->type) {\n case MORLOC_NIL: case MORLOC_BOOL: case MORLOC_SINT8: case MORLOC_UINT8: return 1;\n case MORLOC_SINT16: case MORLOC_UINT16: return 2;\n case MORLOC_SINT32: case MORLOC_UINT32: case MORLOC_FLOAT32: return 4;\n case MORLOC_SINT64: case MORLOC_UINT64: case MORLOC_FLOAT64:\n case MORLOC_STRING: case MORLOC_ARRAY: case MORLOC_TENSOR: return alignof(size_t);\n case MORLOC_TUPLE: case MORLOC_MAP: {\n size_t max_align = 1;\n for (size_t i = 0; i < schema->size; i++) {\n size_t a = schema_alignment_cpp(schema->parameters[i]);\n if (a > max_align) max_align = a;\n }\n return max_align;\n }\n case MORLOC_OPTIONAL: return schema_alignment_cpp(schema->parameters[0]);\n default: return alignof(size_t);\n }\n}\n\n\n// ============================================================\n// get_shm_size\n// ============================================================\n\n// Forward declaration\ntemplate\nsize_t get_shm_size(const Schema* schema, const T& data);\n\nsize_t get_shm_size(const Schema* schema, const std::nullptr_t&) {\n return sizeof(int8_t);\n}\n\n// Primitives\ntemplate\nsize_t get_shm_size(const Schema* schema, const Primitive& data) {\n return schema->width;\n}\n\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::vector& data) {\n size_t total_size = schema->width;\n // worst-case cursor alignment padding for element data\n total_size += schema_alignment_cpp(schema->parameters[0]) - 1;\n switch(schema->parameters[0]->type){\n case MORLOC_NIL:\n case MORLOC_BOOL:\n case MORLOC_SINT8:\n case MORLOC_SINT16:\n case MORLOC_SINT32:\n case MORLOC_SINT64:\n case MORLOC_UINT8:\n case MORLOC_UINT16:\n case MORLOC_UINT32:\n case MORLOC_UINT64:\n case MORLOC_FLOAT32:\n case MORLOC_FLOAT64:\n total_size += data.size() * schema->parameters[0]->width;\n break;\n case MORLOC_STRING:\n case MORLOC_ARRAY:\n case MORLOC_TUPLE:\n case MORLOC_MAP:\n case MORLOC_OPTIONAL:\n for(size_t i = 0; i < data.size(); i++){\n total_size += get_shm_size(schema->parameters[0], data[i]);\n }\n break;\n }\n return total_size;\n}\n\n// Optional: tag byte + aligned inner value\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::optional& data) {\n if (!data.has_value()) {\n return schema->width;\n }\n size_t inner_size = get_shm_size(schema->parameters[0], *data);\n size_t extra = (inner_size > schema->parameters[0]->width) ? inner_size - schema->parameters[0]->width : 0;\n return schema->width + extra;\n}\n\nsize_t get_shm_size(const Schema* schema, const std::string& data) {\n return schema->width + data.size();\n}\n\nsize_t get_shm_size(void* dest, const Schema* schema, const char* data) {\n return schema->width + strlen(data);\n}\n\ntemplate\nsize_t createTupleShmSizeHelper(const Schema* schema, const Tuple& data, std::index_sequence) {\n size_t total_size = schema->width;\n (void)std::initializer_list{(\n [&](){\n size_t elem = get_shm_size(schema->parameters[Is], std::get(data));\n if (elem > schema->parameters[Is]->width) {\n total_size += elem - schema->parameters[Is]->width;\n }\n }(),\n 0\n )...};\n return total_size;\n}\n\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::tuple& data) {\n return createTupleShmSizeHelper(schema, data, std::index_sequence_for{});\n}\n\n// Non-vector containers: convert to vector and delegate\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::list& data) {\n return get_shm_size(schema, to_vector(data));\n}\n\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::forward_list& data) {\n return get_shm_size(schema, to_vector(data));\n}\n\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::deque& data) {\n return get_shm_size(schema, to_vector(data));\n}\n\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::stack& data) {\n return get_shm_size(schema, to_vector(data));\n}\n\ntemplate\nsize_t get_shm_size(const Schema* schema, const std::queue& data) {\n return get_shm_size(schema, to_vector(data));\n}\n\n// Tensor: header + shape array + contiguous data\ntemplate\nsize_t get_shm_size(const Schema* schema, const mlc::Tensor& data) {\n using S = mlc::tensor_storage_t;\n size_t total = sizeof(Tensor);\n // alignment padding for shape array\n total += alignof(int64_t) - 1;\n total += NDim * sizeof(int64_t);\n // alignment padding for element data\n total += schema_alignment_cpp(schema->parameters[0]) - 1;\n total += data.size() * sizeof(S);\n return total;\n}\n\n\n// ============================================================\n// toAnything - top-level (allocating)\n// ============================================================\n\n// Generic top-level: compute size, allocate, serialize\ntemplate\nvoid* toAnything(const Schema* schema, const T& data){\n size_t total_size = get_shm_size(schema, data);\n void* dest = shmalloc_cpp(total_size);\n void* cursor = (void*)((char*)dest + schema->width);\n try {\n return toAnything(dest, &cursor, schema, data);\n } catch (...) {\n shfree_cpp(dest);\n throw;\n }\n}\n\n// Non-vector containers: convert to vector and delegate\ntemplate\nvoid* toAnything(const Schema* schema, const std::stack& data) {\n return toAnything(schema, to_vector(data));\n}\n\ntemplate\nvoid* toAnything(const Schema* schema, const std::forward_list& data) {\n return toAnything(schema, to_vector(data));\n}\n\ntemplate\nvoid* toAnything(const Schema* schema, const std::queue& data) {\n return toAnything(schema, to_vector(data));\n}\n\ntemplate\nvoid* toAnything(const Schema* schema, const std::deque& data) {\n return toAnything(schema, to_vector(data));\n}\n\ntemplate\nvoid* toAnything(const Schema* schema, const std::list& data) {\n return toAnything(schema, to_vector(data));\n}\n\n\n// ============================================================\n// toAnything - cursor-based (recursive)\n// ============================================================\n\n// Forward declaration\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const T& data);\n\n// Write raw binary data as an array\nvoid* binarytoAnything(void* dest, void** cursor, const Schema* schema, const uint8_t* data, size_t size) {\n Array* result = static_cast(dest);\n result->size = size;\n if(size == 0){\n result->data = RELNULL;\n return dest;\n }\n absptr_t data_ptr = static_cast(*cursor);\n result->data = abs2rel_cpp(data_ptr);\n *cursor = static_cast(*cursor) + size * schema->parameters[0]->width;\n memcpy(data_ptr, data, size);\n return dest;\n}\n\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::nullptr_t&) {\n *((int8_t*)dest) = (int8_t)0;\n return dest;\n}\n\n// Primitives\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const Primitive& data) {\n *((Primitive*)dest) = data;\n return dest;\n}\n\n// Vector (primary array implementation)\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::vector& data) {\n Array* result = static_cast(dest);\n result->size = data.size();\n if(data.size() == 0){\n result->data = RELNULL;\n return dest;\n }\n // align cursor for element data placement\n *cursor = reinterpret_cast(ALIGN_UP(reinterpret_cast(*cursor), schema_alignment_cpp(schema->parameters[0])));\n result->data = abs2rel_cpp(static_cast(*cursor));\n *cursor = static_cast(*cursor) + data.size() * schema->parameters[0]->width;\n char* start = (char*)cpp_rel2abs(result->data);\n size_t width = schema->parameters[0]->width;\n for (size_t i = 0; i < data.size(); ++i) {\n toAnything(start + width * i, cursor, schema->parameters[0], data[i]);\n }\n return dest;\n}\n\n// Shared helper for iterable containers (list, forward_list, deque)\ntemplate\nvoid* toAnything_seq(void* dest, void** cursor, const Schema* schema, const Container& data, size_t size) {\n Array* result = static_cast(dest);\n result->size = size;\n if(size == 0){\n result->data = RELNULL;\n return dest;\n }\n // align cursor for element data placement\n *cursor = reinterpret_cast(ALIGN_UP(reinterpret_cast(*cursor), schema_alignment_cpp(schema->parameters[0])));\n result->data = abs2rel_cpp(static_cast(*cursor));\n *cursor = static_cast(*cursor) + size * schema->parameters[0]->width;\n char* start = (char*)cpp_rel2abs(result->data);\n size_t width = schema->parameters[0]->width;\n size_t i = 0;\n for (const auto& item : data) {\n toAnything(start + width * i, cursor, schema->parameters[0], item);\n ++i;\n }\n return dest;\n}\n\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::list& data) {\n return toAnything_seq(dest, cursor, schema, data, data.size());\n}\n\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::forward_list& data) {\n return toAnything_seq(dest, cursor, schema, data, std::distance(data.begin(), data.end()));\n}\n\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::deque& data) {\n return toAnything_seq(dest, cursor, schema, data, data.size());\n}\n\n// Stack and queue: convert to vector and delegate\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::queue& data) {\n return toAnything(dest, cursor, schema, to_vector(data));\n}\n\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::stack& data) {\n return toAnything(dest, cursor, schema, to_vector(data));\n}\n\n// String and C string\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::string& data) {\n return binarytoAnything(dest, cursor, schema, (const uint8_t*)data.c_str(), data.size());\n}\n\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const char* data) {\n return binarytoAnything(dest, cursor, schema, (const uint8_t*)data, strlen(data));\n}\n\n// Tuple\ntemplate\nvoid* createTupleAnythingHelper(void* dest, const Schema* schema, void** cursor, const Tuple& data, std::index_sequence) {\n (void)std::initializer_list{(\n toAnything((char*)dest + schema->offsets[Is], cursor, schema->parameters[Is], std::get(data)),\n 0\n )...};\n return dest;\n}\n\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::tuple& data) {\n return createTupleAnythingHelper(dest, schema, cursor, data, std::index_sequence_for{});\n}\n\n// Pair (reuses tuple helper since std::pair supports std::get)\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::pair& data) {\n return createTupleAnythingHelper(dest, schema, cursor, data, std::index_sequence<0, 1>{});\n}\n\n// Optional\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const std::optional& data) {\n if (!data.has_value()) {\n *((uint8_t*)dest) = 0;\n memset((char*)dest + schema->offsets[0], 0, schema->parameters[0]->width);\n } else {\n *((uint8_t*)dest) = 1;\n toAnything((char*)dest + schema->offsets[0], cursor, schema->parameters[0], *data);\n }\n return dest;\n}\n\n// Tensor: write Tensor header + shape array + contiguous data\ntemplate\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const mlc::Tensor& data) {\n Tensor* result = static_cast(dest);\n result->total_elements = data.size();\n result->device_type = 0;\n result->device_id = 0;\n\n if (data.size() == 0) {\n result->shape = RELNULL;\n result->data = RELNULL;\n return dest;\n }\n\n // Write shape array\n *cursor = reinterpret_cast(ALIGN_UP(reinterpret_cast(*cursor), alignof(int64_t)));\n result->shape = abs2rel_cpp(static_cast(*cursor));\n int64_t* shape_dst = (int64_t*)*cursor;\n for (int i = 0; i < NDim; i++) shape_dst[i] = data.shape()[i];\n *cursor = (char*)*cursor + NDim * sizeof(int64_t);\n\n // Write data buffer (contiguous row-major)\n using S = mlc::tensor_storage_t;\n size_t elem_align = schema_alignment_cpp(schema->parameters[0]);\n *cursor = reinterpret_cast(ALIGN_UP(reinterpret_cast(*cursor), elem_align));\n result->data = abs2rel_cpp(static_cast(*cursor));\n size_t data_bytes = data.size() * sizeof(S);\n memcpy(*cursor, data.data(), data_bytes);\n *cursor = (char*)*cursor + data_bytes;\n\n return dest;\n}\n\n\n// ============================================================\n// fromAnything - single template with if constexpr dispatch\n// ============================================================\n\n// Forward declaration for recursive calls\ntemplate\nT fromAnything(const Schema* schema, const void* data, T* = nullptr, const void* base_ptr = nullptr);\n\n// Tuple helper (needs forward declaration of fromAnything)\ntemplate\nTuple fromTupleAnythingHelper(\n const Schema* schema,\n const void* anything,\n std::index_sequence,\n Tuple* = nullptr,\n const void* base_ptr = nullptr\n) {\n return Tuple(fromAnything(schema->parameters[Is],\n (char*)anything + schema->offsets[Is],\n static_cast*>(nullptr),\n base_ptr)...);\n}\n\ntemplate\nT fromAnything(const Schema* schema, const void* data, T*, const void* base_ptr) {\n if(data == NULL){\n throw std::runtime_error(\"Void error in fromAnything\");\n }\n\n if constexpr (std::is_same_v) {\n // NOTE: do NOT use bool here since its width is often not 1 byte\n return *(uint8_t*)data == 1;\n }\n else if constexpr (std::is_same_v) {\n Array* array = (Array*)data;\n if(array->size > 0){\n return std::string((char*)resolve_relptr_cpp(array->data, base_ptr), array->size);\n }\n return std::string(\"\");\n }\n else if constexpr (is_std_vector::value) {\n using ElemT = typename T::value_type;\n std::vector result;\n Array* array = (Array*)data;\n if(array->size == 0) return result;\n\n // Fast path for primitive arrays\n switch(schema->parameters[0]->type){\n case MORLOC_NIL:\n case MORLOC_BOOL:\n case MORLOC_SINT8:\n case MORLOC_SINT16:\n case MORLOC_SINT32:\n case MORLOC_SINT64:\n case MORLOC_UINT8:\n case MORLOC_UINT16:\n case MORLOC_UINT32:\n case MORLOC_UINT64:\n case MORLOC_FLOAT32:\n case MORLOC_FLOAT64: {\n ElemT* arr_start = (ElemT*)resolve_relptr_cpp(array->data, base_ptr);\n std::vector pv(arr_start, arr_start + array->size);\n return pv;\n }\n }\n\n // Complex element types\n result.reserve(array->size);\n const Schema* elem_schema = schema->parameters[0];\n char* start = (char*)resolve_relptr_cpp(array->data, base_ptr);\n for(size_t i = 0; i < array->size; i++){\n result.push_back(fromAnything(elem_schema, (void*)(start + i * elem_schema->width), static_cast(nullptr), base_ptr));\n }\n return result;\n }\n else if constexpr (is_non_vector_container_v) {\n using ElemT = typename T::value_type;\n Array* array = (Array*)data;\n T result;\n if(array->size == 0) return result;\n\n const Schema* elem_schema = schema->parameters[0];\n char* start = (char*)resolve_relptr_cpp(array->data, base_ptr);\n\n constexpr bool reverse = is_std_stack::value || is_std_forward_list::value;\n\n if constexpr (reverse) {\n for (size_t i = array->size; i > 0; --i) {\n auto elem = fromAnything(elem_schema, (void*)(start + (i-1) * elem_schema->width), static_cast(nullptr), base_ptr);\n if constexpr (is_std_stack::value) result.push(std::move(elem));\n else result.push_front(std::move(elem));\n }\n } else {\n for (size_t i = 0; i < array->size; ++i) {\n auto elem = fromAnything(elem_schema, (void*)(start + i * elem_schema->width), static_cast(nullptr), base_ptr);\n if constexpr (is_std_queue::value) result.push(std::move(elem));\n else result.push_back(std::move(elem));\n }\n }\n return result;\n }\n else if constexpr (is_std_tuple::value) {\n return fromTupleAnythingHelper(\n schema, data,\n std::make_index_sequence>{},\n static_cast(nullptr),\n base_ptr\n );\n }\n else if constexpr (is_std_pair::value) {\n return fromTupleAnythingHelper(\n schema, data,\n std::index_sequence<0, 1>{},\n static_cast(nullptr),\n base_ptr\n );\n }\n else if constexpr (is_std_optional::value) {\n using InnerT = typename T::value_type;\n uint8_t tag = *(const uint8_t*)data;\n if (tag == 0) {\n return std::nullopt;\n }\n return std::optional(fromAnything(schema->parameters[0], (const char*)data + schema->offsets[0], static_cast(nullptr), base_ptr));\n }\n else if constexpr (mlc::is_mlc_tensor_v) {\n using ElemT = mlc::tensor_element_t;\n using StorageT = mlc::tensor_storage_t;\n constexpr int NDim = mlc::tensor_ndim_v;\n const Tensor* tensor = (const Tensor*)data;\n\n if (tensor->total_elements == 0) {\n int64_t zero_shape[NDim] = {};\n return T(zero_shape);\n }\n\n const int64_t* shape = (const int64_t*)resolve_relptr_cpp(tensor->shape, base_ptr);\n StorageT* tdata = (StorageT*)resolve_relptr_cpp(tensor->data, base_ptr);\n return T(tdata, shape, tensor->total_elements);\n }\n else {\n // Primitives (int, double, float, etc.)\n // Record types are handled by generated overloads which are preferred\n // by overload resolution over this template.\n return *(T*)data;\n }\n}\n\n\n// ============================================================\n// mpk_pack / mpk_unpack\n// ============================================================\n\ntemplate\nstd::vector mpk_pack(const T& data, const std::string& schema_str) {\n const char* schema_ptr = schema_str.c_str();\n Schema* schema = parse_schema_cpp(schema_ptr);\n\n void* voidstar = nullptr;\n char* msgpack_data = NULL;\n size_t msg_size = 0;\n\n try {\n voidstar = toAnything(schema, data);\n pack_with_schema_cpp(voidstar, schema, &msgpack_data, &msg_size);\n } catch (...) {\n if (voidstar) shfree_cpp(voidstar);\n free(msgpack_data);\n free_schema(schema);\n throw;\n }\n\n shfree_cpp(voidstar);\n\n std::vector result(msgpack_data, msgpack_data + msg_size);\n free(msgpack_data);\n free_schema(schema);\n\n return result;\n}\n\ntemplate\nT mpk_unpack(const std::vector& packed_data, const std::string& schema_str) {\n const char* schema_ptr = schema_str.c_str();\n Schema* schema = parse_schema_cpp(schema_ptr);\n\n void* voidstar = nullptr;\n int unpack_result = unpack_with_schema_cpp(packed_data.data(), packed_data.size(), schema, &voidstar);\n if (unpack_result != 0) {\n free_schema(schema);\n throw std::runtime_error(\"Unpacking failed\");\n }\n\n T x;\n try {\n x = fromAnything(schema, voidstar, static_cast(nullptr));\n } catch (...) {\n free_schema(schema);\n shfree_cpp(voidstar);\n throw;\n }\n\n free_schema(schema);\n shfree_cpp(voidstar);\n\n return x;\n}\n\n#endif\n" }, { "path": "data/lang/cpp/init.sh", "content": "#!/bin/bash\nset -e\n\nMORLOC_HOME=\"$1\"\nBUILD_DIR=\"$2\"\nSANITIZE_FLAGS=\"$3\"\nINCLUDE_DIR=\"$MORLOC_HOME/include\"\nLIB_DIR=\"$MORLOC_HOME/lib\"\n\n# Install mlccpptypes if not present\nif [ ! -d \"$INCLUDE_DIR/mlccpptypes\" ]; then\n git clone https://github.com/morloclib/mlccpptypes \"$INCLUDE_DIR/mlccpptypes\"\nfi\n\n# Install headers\ncp \"$BUILD_DIR/cppmorloc.hpp\" \"$INCLUDE_DIR/\"\ncp \"$BUILD_DIR/mlc_arrow.hpp\" \"$INCLUDE_DIR/\"\ncp \"$BUILD_DIR/mlc_tensor.hpp\" \"$INCLUDE_DIR/\"\n\n# Install nanoarrow headers\nmkdir -p \"$INCLUDE_DIR/nanoarrow\"\ncp \"$BUILD_DIR/nanoarrow.h\" \"$INCLUDE_DIR/nanoarrow/\"\n\n# Compile nanoarrow.c\ngcc -c -O2 -fPIC $SANITIZE_FLAGS -I\"$INCLUDE_DIR\" -o \"$BUILD_DIR/nanoarrow.o\" \"$BUILD_DIR/nanoarrow.c\"\n\n# Compile cppmorloc.cpp\ng++ -c --std=c++17 -O2 $SANITIZE_FLAGS -I\"$INCLUDE_DIR\" -o \"$BUILD_DIR/cppmorloc.o\" \"$BUILD_DIR/cppmorloc.cpp\"\n\n# Archive into libcppmorloc.a\nar rcs \"$LIB_DIR/libcppmorloc.a\" \"$BUILD_DIR/cppmorloc.o\" \"$BUILD_DIR/nanoarrow.o\"\n\n# Compile precompiled header\ncp \"$BUILD_DIR/morloc_pch.hpp\" \"$INCLUDE_DIR/\"\ng++ --std=c++17 -O2 $SANITIZE_FLAGS -I\"$INCLUDE_DIR\" -x c++-header \"$INCLUDE_DIR/morloc_pch.hpp\" -o \"$INCLUDE_DIR/morloc_pch.hpp.gch\"\n" }, { "path": "data/lang/cpp/lang.yaml", "content": "# C++ language metadata for morloc compiler\nname: cpp\nextension: cpp\naliases: [\"c++\"]\nis_compiled: true\nrun_command: []\nserial_type: \"uint8_t*\"\ncost: 0\n" }, { "path": "data/lang/cpp/mlc_arrow.hpp", "content": "#ifndef MLC_ARROW_HPP\n#define MLC_ARROW_HPP\n\n// mlc_arrow.hpp -- thin RAII wrapper around Arrow C Data Interface structs\n// for use in morloc C++ pools. Holds ArrowSchema + ArrowArray as a single\n// move-only value. The pool template dispatches arrow-hinted schemas to\n// arrow_to_shm / arrow_from_shm (in libmorloc.so) via this type.\n//\n// User code should include to build and read columns.\n\n#include \"morloc.h\"\n#include \n#include \n#include \n\nnamespace mlc {\n\nclass ArrowTable {\npublic:\n // Construct from moved-in C Data Interface structs.\n // Takes ownership of release callbacks.\n ArrowTable(struct ArrowSchema schema, struct ArrowArray array)\n : schema_(schema), array_(array)\n {\n // Zero the source structs so the caller does not double-release\n memset(&schema, 0, sizeof(schema));\n memset(&array, 0, sizeof(array));\n }\n\n ~ArrowTable() {\n if (array_.release) array_.release(&array_);\n if (schema_.release) schema_.release(&schema_);\n }\n\n // Move-only\n ArrowTable(ArrowTable&& other) noexcept\n : schema_(other.schema_), array_(other.array_)\n {\n memset(&other.schema_, 0, sizeof(other.schema_));\n memset(&other.array_, 0, sizeof(other.array_));\n }\n\n ArrowTable& operator=(ArrowTable&& other) noexcept {\n if (this != &other) {\n if (array_.release) array_.release(&array_);\n if (schema_.release) schema_.release(&schema_);\n schema_ = other.schema_;\n array_ = other.array_;\n memset(&other.schema_, 0, sizeof(other.schema_));\n memset(&other.array_, 0, sizeof(other.array_));\n }\n return *this;\n }\n\n ArrowTable(const ArrowTable&) = delete;\n ArrowTable& operator=(const ArrowTable&) = delete;\n\n // Accessors (const -- arrow data is immutable)\n const struct ArrowSchema* schema() const { return &schema_; }\n const struct ArrowArray* array() const { return &array_; }\n int64_t n_columns() const { return schema_.n_children; }\n int64_t n_rows() const { return array_.length; }\n\n // Build from shared memory header (zero-copy import)\n static ArrowTable from_shm(const arrow_shm_header_t* hdr) {\n struct ArrowSchema as;\n struct ArrowArray aa;\n char* err = nullptr;\n arrow_from_shm(hdr, &as, &aa, &err);\n if (err) {\n std::string msg(err);\n free(err);\n throw std::runtime_error(msg);\n }\n return ArrowTable(std::move(as), std::move(aa));\n }\n\n // Move table data to shared memory: copies buffers into a contiguous SHM\n // block, frees the original heap buffers, then repoints this table's\n // internal ArrowSchema/ArrowArray into the SHM block (zero-copy).\n // After this call the table is still usable but backed by SHM.\n // Returns relptr to the SHM block for use in packets.\n relptr_t move_to_shm() {\n // Step 1: copy all column data into contiguous SHM\n char* copy_err = nullptr;\n relptr_t rp = arrow_to_shm(&array_, &schema_, ©_err);\n if (copy_err) {\n std::string msg(copy_err);\n free(copy_err);\n throw std::runtime_error(msg);\n }\n\n // Step 2: release heap-backed structs (frees all original buffers)\n if (array_.release) array_.release(&array_);\n if (schema_.release) schema_.release(&schema_);\n memset(&schema_, 0, sizeof(schema_));\n memset(&array_, 0, sizeof(array_));\n\n // Step 3: resolve SHM pointer and rebuild structs pointing into it\n char* resolve_err = nullptr;\n void* abs = rel2abs(rp, &resolve_err);\n if (resolve_err) {\n std::string msg(resolve_err);\n free(resolve_err);\n throw std::runtime_error(msg);\n }\n\n char* shm_err = nullptr;\n arrow_from_shm((const arrow_shm_header_t*)abs, &schema_, &array_, &shm_err);\n if (shm_err) {\n std::string msg(shm_err);\n free(shm_err);\n throw std::runtime_error(msg);\n }\n\n return rp;\n }\n\nprivate:\n struct ArrowSchema schema_;\n struct ArrowArray array_;\n};\n\n} // namespace mlc\n\n#endif // MLC_ARROW_HPP\n" }, { "path": "data/lang/cpp/mlc_tensor.hpp", "content": "#ifndef MLC_TENSOR_HPP\n#define MLC_TENSOR_HPP\n\n// mlc_tensor.hpp -- dense N-dimensional tensor for morloc C++ pools.\n// Data is always contiguous row-major (C order). The Tensor struct in\n// schema.h defines the voidstar layout; this header provides the C++\n// user-facing type that maps to it.\n\n#include \"morloc.h\"\n#include \n#include \n#include \n#include \n\nnamespace mlc {\n\n// Storage type trait: maps bool to uint8_t so that tensor memory layout\n// matches the voidstar format (MORLOC_BOOL = 1 byte) regardless of\n// sizeof(bool) on the target platform.\ntemplate struct tensor_storage { using type = T; };\ntemplate<> struct tensor_storage { using type = uint8_t; };\ntemplate using tensor_storage_t = typename tensor_storage::type;\n\ntemplate\nclass Tensor {\n using S = tensor_storage_t;\npublic:\n // Construct with given shape, allocating data on the heap\n Tensor(const int64_t (&dims)[NDim]) : owns_data_(true) {\n for (int i = 0; i < NDim; i++) shape_[i] = dims[i];\n total_ = 1;\n for (int i = 0; i < NDim; i++) total_ *= (size_t)shape_[i];\n data_ = new S[total_]();\n }\n\n // Construct from initializer list of dims\n Tensor(std::initializer_list dims) : owns_data_(true) {\n if ((int)dims.size() != NDim) {\n throw std::runtime_error(\"Tensor dimension mismatch\");\n }\n int i = 0;\n for (auto d : dims) shape_[i++] = d;\n total_ = 1;\n for (i = 0; i < NDim; i++) total_ *= (size_t)shape_[i];\n data_ = new S[total_]();\n }\n\n // Construct as a view over existing data (does not own)\n Tensor(S* data, const int64_t* shape, size_t total)\n : data_(data), total_(total), owns_data_(false) {\n for (int i = 0; i < NDim; i++) shape_[i] = shape[i];\n }\n\n ~Tensor() {\n if (owns_data_ && data_) delete[] data_;\n }\n\n // Move-only\n Tensor(Tensor&& other) noexcept\n : data_(other.data_), total_(other.total_), owns_data_(other.owns_data_) {\n for (int i = 0; i < NDim; i++) shape_[i] = other.shape_[i];\n other.data_ = nullptr;\n other.owns_data_ = false;\n }\n\n Tensor& operator=(Tensor&& other) noexcept {\n if (this != &other) {\n if (owns_data_ && data_) delete[] data_;\n data_ = other.data_;\n total_ = other.total_;\n owns_data_ = other.owns_data_;\n for (int i = 0; i < NDim; i++) shape_[i] = other.shape_[i];\n other.data_ = nullptr;\n other.owns_data_ = false;\n }\n return *this;\n }\n\n Tensor(const Tensor&) = delete;\n Tensor& operator=(const Tensor&) = delete;\n\n // Accessors (S* for raw access; S == T for all types except bool)\n const S* data() const { return data_; }\n S* data() { return data_; }\n constexpr int ndim() const { return NDim; }\n const int64_t* shape() const { return shape_; }\n int64_t shape(int d) const { return shape_[d]; }\n size_t size() const { return total_; }\n\n // Linear access (returns S& which is uint8_t& for bool tensors;\n // implicit conversion to/from bool handles the difference)\n const S& operator[](size_t i) const { return data_[i]; }\n S& operator[](size_t i) { return data_[i]; }\n\n // 1D access\n template>\n const S& operator()(int64_t i) const { return data_[i]; }\n template>\n S& operator()(int64_t i) { return data_[i]; }\n\n // 2D access (row-major)\n template>\n const S& operator()(int64_t i, int64_t j) const {\n return data_[i * shape_[1] + j];\n }\n template>\n S& operator()(int64_t i, int64_t j) {\n return data_[i * shape_[1] + j];\n }\n\n // 3D access (row-major)\n template>\n const S& operator()(int64_t i, int64_t j, int64_t k) const {\n return data_[(i * shape_[1] + j) * shape_[2] + k];\n }\n template>\n S& operator()(int64_t i, int64_t j, int64_t k) {\n return data_[(i * shape_[1] + j) * shape_[2] + k];\n }\n\nprivate:\n S* data_ = nullptr;\n int64_t shape_[NDim] = {};\n size_t total_ = 0;\n bool owns_data_ = false;\n};\n\n// Convenience aliases\ntemplate using Tensor1 = Tensor;\ntemplate using Tensor2 = Tensor;\ntemplate using Tensor3 = Tensor;\ntemplate using Tensor4 = Tensor;\ntemplate using Tensor5 = Tensor;\n\n// Type trait for detecting mlc::Tensor\ntemplate struct is_mlc_tensor : std::false_type {};\ntemplate struct is_mlc_tensor> : std::true_type {};\ntemplate\ninline constexpr bool is_mlc_tensor_v = is_mlc_tensor::value;\n\n// Extract element type from Tensor\ntemplate struct tensor_element;\ntemplate struct tensor_element> { using type = T; };\ntemplate using tensor_element_t = typename tensor_element::type;\n\n// Extract ndim from Tensor\ntemplate struct tensor_ndim;\ntemplate struct tensor_ndim>\n { static constexpr int value = N; };\ntemplate\ninline constexpr int tensor_ndim_v = tensor_ndim::value;\n\n} // namespace mlc\n\n#endif // MLC_TENSOR_HPP\n" }, { "path": "data/lang/cpp/morloc_pch.hpp", "content": "// Precompiled header for morloc C++ pools.\n// Compiled once during 'morloc init', reused for every pool compilation.\n\n#ifndef MORLOC_PCH_HPP\n#define MORLOC_PCH_HPP\n\n// STL containers\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// STL algorithms and utilities\n#include \n#include \n#include \n#include \n#include \n#include \n\n// Strings and I/O\n#include \n#include \n#include \n#include \n\n// C standard library\n#include \n#include \n#include \n#include \n#include \n#include \n\n// POSIX headers\n#include \n#include \n#include \n#include \n#include \n\n// Morloc runtime\n#include \"morloc.h\"\n\n#endif\n" }, { "path": "data/lang/cpp/nanoarrow/nanoarrow.c", "content": "// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"nanoarrow/nanoarrow.h\"\n\nconst char* ArrowNanoarrowVersion(void) { return NANOARROW_VERSION; }\n\nint ArrowNanoarrowVersionInt(void) { return NANOARROW_VERSION_INT; }\n\nArrowErrorCode ArrowErrorSet(struct ArrowError* error, const char* fmt, ...) {\n if (error == NULL) {\n return NANOARROW_OK;\n }\n\n memset(error->message, 0, sizeof(error->message));\n\n va_list args;\n va_start(args, fmt);\n int chars_needed = vsnprintf(error->message, sizeof(error->message), fmt, args);\n va_end(args);\n\n if (chars_needed < 0) {\n return EINVAL;\n } else if (((size_t)chars_needed) >= sizeof(error->message)) {\n return ERANGE;\n } else {\n return NANOARROW_OK;\n }\n}\n\nvoid ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type) {\n layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_VALIDITY;\n layout->buffer_data_type[0] = NANOARROW_TYPE_BOOL;\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA;\n layout->buffer_data_type[1] = storage_type;\n layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_NONE;\n layout->buffer_data_type[2] = NANOARROW_TYPE_UNINITIALIZED;\n\n layout->element_size_bits[0] = 1;\n layout->element_size_bits[1] = 0;\n layout->element_size_bits[2] = 0;\n\n layout->child_size_elements = 0;\n\n switch (storage_type) {\n case NANOARROW_TYPE_UNINITIALIZED:\n case NANOARROW_TYPE_NA:\n case NANOARROW_TYPE_RUN_END_ENCODED:\n layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_NONE;\n layout->buffer_data_type[0] = NANOARROW_TYPE_UNINITIALIZED;\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_NONE;\n layout->buffer_data_type[1] = NANOARROW_TYPE_UNINITIALIZED;\n layout->element_size_bits[0] = 0;\n break;\n\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_MAP:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;\n layout->element_size_bits[1] = 32;\n break;\n\n case NANOARROW_TYPE_LARGE_LIST:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;\n layout->element_size_bits[1] = 64;\n break;\n\n case NANOARROW_TYPE_STRUCT:\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_NONE;\n layout->buffer_data_type[1] = NANOARROW_TYPE_UNINITIALIZED;\n break;\n\n case NANOARROW_TYPE_BOOL:\n layout->element_size_bits[1] = 1;\n break;\n\n case NANOARROW_TYPE_UINT8:\n case NANOARROW_TYPE_INT8:\n layout->element_size_bits[1] = 8;\n break;\n\n case NANOARROW_TYPE_UINT16:\n case NANOARROW_TYPE_INT16:\n case NANOARROW_TYPE_HALF_FLOAT:\n layout->element_size_bits[1] = 16;\n break;\n\n case NANOARROW_TYPE_UINT32:\n case NANOARROW_TYPE_INT32:\n case NANOARROW_TYPE_FLOAT:\n case NANOARROW_TYPE_DECIMAL32:\n layout->element_size_bits[1] = 32;\n break;\n case NANOARROW_TYPE_INTERVAL_MONTHS:\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;\n layout->element_size_bits[1] = 32;\n break;\n\n case NANOARROW_TYPE_UINT64:\n case NANOARROW_TYPE_INT64:\n case NANOARROW_TYPE_DOUBLE:\n case NANOARROW_TYPE_INTERVAL_DAY_TIME:\n case NANOARROW_TYPE_DECIMAL64:\n layout->element_size_bits[1] = 64;\n break;\n\n case NANOARROW_TYPE_DECIMAL128:\n case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:\n layout->element_size_bits[1] = 128;\n break;\n\n case NANOARROW_TYPE_DECIMAL256:\n layout->element_size_bits[1] = 256;\n break;\n\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n layout->buffer_data_type[1] = NANOARROW_TYPE_BINARY;\n break;\n\n case NANOARROW_TYPE_DENSE_UNION:\n layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_TYPE_ID;\n layout->buffer_data_type[0] = NANOARROW_TYPE_INT8;\n layout->element_size_bits[0] = 8;\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_UNION_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;\n layout->element_size_bits[1] = 32;\n break;\n\n case NANOARROW_TYPE_SPARSE_UNION:\n layout->buffer_type[0] = NANOARROW_BUFFER_TYPE_TYPE_ID;\n layout->buffer_data_type[0] = NANOARROW_TYPE_INT8;\n layout->element_size_bits[0] = 8;\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_NONE;\n layout->buffer_data_type[1] = NANOARROW_TYPE_UNINITIALIZED;\n break;\n\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_BINARY:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;\n layout->element_size_bits[1] = 32;\n layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_DATA;\n layout->buffer_data_type[2] = storage_type;\n break;\n\n case NANOARROW_TYPE_LARGE_STRING:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;\n layout->element_size_bits[1] = 64;\n layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_DATA;\n layout->buffer_data_type[2] = NANOARROW_TYPE_STRING;\n break;\n case NANOARROW_TYPE_LARGE_BINARY:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;\n layout->element_size_bits[1] = 64;\n layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_DATA;\n layout->buffer_data_type[2] = NANOARROW_TYPE_BINARY;\n break;\n\n case NANOARROW_TYPE_BINARY_VIEW:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA;\n layout->buffer_data_type[1] = NANOARROW_TYPE_BINARY_VIEW;\n layout->element_size_bits[1] = 128;\n break;\n case NANOARROW_TYPE_STRING_VIEW:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_DATA;\n layout->buffer_data_type[1] = NANOARROW_TYPE_STRING_VIEW;\n layout->element_size_bits[1] = 128;\n break;\n\n case NANOARROW_TYPE_LIST_VIEW:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_VIEW_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT32;\n layout->element_size_bits[1] = 32;\n layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_SIZE;\n layout->buffer_data_type[2] = NANOARROW_TYPE_INT32;\n layout->element_size_bits[2] = 32;\n break;\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n layout->buffer_type[1] = NANOARROW_BUFFER_TYPE_VIEW_OFFSET;\n layout->buffer_data_type[1] = NANOARROW_TYPE_INT64;\n layout->element_size_bits[1] = 64;\n layout->buffer_type[2] = NANOARROW_BUFFER_TYPE_SIZE;\n layout->buffer_data_type[2] = NANOARROW_TYPE_INT64;\n layout->element_size_bits[2] = 64;\n break;\n\n default:\n break;\n }\n}\n\nvoid* ArrowMalloc(int64_t size) { return malloc(size); }\n\nvoid* ArrowRealloc(void* ptr, int64_t size) { return realloc(ptr, size); }\n\nvoid ArrowFree(void* ptr) { free(ptr); }\n\nstatic uint8_t* ArrowBufferAllocatorMallocReallocate(\n struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t old_size,\n int64_t new_size) {\n NANOARROW_UNUSED(allocator);\n NANOARROW_UNUSED(old_size);\n return (uint8_t*)ArrowRealloc(ptr, new_size);\n}\n\nstatic void ArrowBufferAllocatorMallocFree(struct ArrowBufferAllocator* allocator,\n uint8_t* ptr, int64_t size) {\n NANOARROW_UNUSED(allocator);\n NANOARROW_UNUSED(size);\n if (ptr != NULL) {\n ArrowFree(ptr);\n }\n}\n\nstatic struct ArrowBufferAllocator ArrowBufferAllocatorMalloc = {\n &ArrowBufferAllocatorMallocReallocate, &ArrowBufferAllocatorMallocFree, NULL};\n\nstruct ArrowBufferAllocator ArrowBufferAllocatorDefault(void) {\n return ArrowBufferAllocatorMalloc;\n}\n\nstatic uint8_t* ArrowBufferDeallocatorReallocate(struct ArrowBufferAllocator* allocator,\n uint8_t* ptr, int64_t old_size,\n int64_t new_size) {\n NANOARROW_UNUSED(new_size);\n\n // Attempting to reallocate a buffer with a custom deallocator is\n // a programming error. In debug mode, crash here.\n#if defined(NANOARROW_DEBUG)\n NANOARROW_PRINT_AND_DIE(ENOMEM,\n \"It is an error to reallocate a buffer whose allocator is \"\n \"ArrowBufferDeallocator()\");\n#endif\n\n // In release mode, ensure the the deallocator is called exactly\n // once using the pointer it was given and return NULL, which\n // will trigger the caller to return ENOMEM.\n allocator->free(allocator, ptr, old_size);\n *allocator = ArrowBufferAllocatorDefault();\n return NULL;\n}\n\nstruct ArrowBufferAllocator ArrowBufferDeallocator(\n void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,\n int64_t size),\n void* private_data) {\n struct ArrowBufferAllocator allocator;\n allocator.reallocate = &ArrowBufferDeallocatorReallocate;\n allocator.free = custom_free;\n allocator.private_data = private_data;\n return allocator;\n}\n\nstatic const int kInt32DecimalDigits = 9;\n\nstatic const uint64_t kUInt32PowersOfTen[] = {\n 1ULL, 10ULL, 100ULL, 1000ULL, 10000ULL,\n 100000ULL, 1000000ULL, 10000000ULL, 100000000ULL, 1000000000ULL};\n\n// Adapted from Arrow C++ to use 32-bit words for better C portability\n// https://github.com/apache/arrow/blob/cd3321b28b0c9703e5d7105d6146c1270bbadd7f/cpp/src/arrow/util/decimal.cc#L524-L544\nstatic void ShiftAndAdd(struct ArrowStringView value, uint32_t* out, int64_t out_size) {\n // We use strtoll for parsing, which needs input that is null-terminated\n char chunk_string[16];\n\n for (int64_t posn = 0; posn < value.size_bytes;) {\n int64_t remaining = value.size_bytes - posn;\n\n int64_t group_size;\n if (remaining > kInt32DecimalDigits) {\n group_size = kInt32DecimalDigits;\n } else {\n group_size = remaining;\n }\n\n const uint64_t multiple = kUInt32PowersOfTen[group_size];\n\n memcpy(chunk_string, value.data + posn, group_size);\n chunk_string[group_size] = '\\0';\n uint32_t chunk = (uint32_t)strtoll(chunk_string, NULL, 10);\n\n for (int64_t i = 0; i < out_size; i++) {\n uint64_t tmp = out[i];\n tmp *= multiple;\n tmp += chunk;\n out[i] = (uint32_t)(tmp & 0xFFFFFFFFULL);\n chunk = (uint32_t)(tmp >> 32);\n }\n posn += group_size;\n }\n}\n\nArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal,\n struct ArrowStringView value) {\n // Check for sign\n int is_negative = value.data[0] == '-';\n int has_sign = is_negative || value.data[0] == '+';\n value.data += has_sign;\n value.size_bytes -= has_sign;\n\n // Check all characters are digits that are not the negative sign\n for (int64_t i = 0; i < value.size_bytes; i++) {\n char c = value.data[i];\n if (c < '0' || c > '9') {\n return EINVAL;\n }\n }\n\n // Skip over leading 0s\n int64_t n_leading_zeroes = 0;\n for (int64_t i = 0; i < value.size_bytes; i++) {\n if (value.data[i] == '0') {\n n_leading_zeroes++;\n } else {\n break;\n }\n }\n\n value.data += n_leading_zeroes;\n value.size_bytes -= n_leading_zeroes;\n\n // Use 32-bit words for portability\n uint32_t words32[8];\n memset(words32, 0, sizeof(words32));\n int n_words32 = decimal->n_words > 0 ? decimal->n_words * 2 : 1;\n NANOARROW_DCHECK(n_words32 <= 8);\n memset(words32, 0, sizeof(words32));\n\n ShiftAndAdd(value, words32, n_words32);\n\n if (_ArrowIsLittleEndian() || n_words32 == 1) {\n memcpy(decimal->words, words32, sizeof(uint32_t) * n_words32);\n } else {\n uint64_t lo;\n uint64_t hi;\n\n for (int i = 0; i < decimal->n_words; i++) {\n lo = (uint64_t)words32[i * 2];\n hi = (uint64_t)words32[i * 2 + 1] << 32;\n decimal->words[decimal->n_words - i - 1] = lo | hi;\n }\n }\n\n if (is_negative) {\n ArrowDecimalNegate(decimal);\n }\n\n return NANOARROW_OK;\n}\n\n// Adapted from Arrow C++ for C\n// https://github.com/apache/arrow/blob/cd3321b28b0c9703e5d7105d6146c1270bbadd7f/cpp/src/arrow/util/decimal.cc#L365\nArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decimal,\n struct ArrowBuffer* buffer) {\n NANOARROW_DCHECK(decimal->n_words == 0 || decimal->n_words == 1 ||\n decimal->n_words == 2 || decimal->n_words == 4);\n\n // For the 32-bit case, just use snprintf()\n if (decimal->n_words == 0) {\n int32_t value;\n memcpy(&value, decimal->words, sizeof(int32_t));\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, 16));\n int n_chars = snprintf((char*)buffer->data + buffer->size_bytes,\n (buffer->capacity_bytes - buffer->size_bytes), \"%d\", value);\n if (n_chars <= 0) {\n return EINVAL;\n }\n\n buffer->size_bytes += n_chars;\n return NANOARROW_OK;\n }\n\n int is_negative = ArrowDecimalSign(decimal) < 0;\n\n uint64_t words_little_endian[4];\n if (decimal->n_words == 0) {\n words_little_endian[0] = 0;\n memcpy(words_little_endian, decimal->words, sizeof(uint32_t));\n } else if (decimal->low_word_index == 0) {\n memcpy(words_little_endian, decimal->words, decimal->n_words * sizeof(uint64_t));\n } else {\n for (int i = 0; i < decimal->n_words; i++) {\n words_little_endian[i] = decimal->words[decimal->n_words - i - 1];\n }\n }\n\n // We've already made a copy, so negate that if needed\n if (is_negative) {\n if (decimal->n_words == 0) {\n uint32_t elem = (uint32_t)words_little_endian[0];\n elem = ~elem + 1;\n words_little_endian[0] = (int32_t)elem;\n } else {\n uint64_t carry = 1;\n for (int i = 0; i < decimal->n_words; i++) {\n uint64_t elem = words_little_endian[i];\n elem = ~elem + carry;\n carry &= (elem == 0);\n words_little_endian[i] = elem;\n }\n }\n }\n\n // Find the most significant word that is non-zero\n int most_significant_elem_idx = -1;\n if (decimal->n_words == 0) {\n if (words_little_endian[0] != 0) {\n most_significant_elem_idx = 0;\n }\n } else {\n for (int i = decimal->n_words - 1; i >= 0; i--) {\n if (words_little_endian[i] != 0) {\n most_significant_elem_idx = i;\n break;\n }\n }\n }\n\n // If they are all zero, the output is just '0'\n if (most_significant_elem_idx == -1) {\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(buffer, '0'));\n return NANOARROW_OK;\n }\n\n // Define segments such that each segment represents 9 digits with the\n // least significant group of 9 digits first. For example, if the input represents\n // 9876543210123456789, then segments will be [123456789, 876543210, 9].\n // We handle at most a signed 256 bit integer, whose maximum value occupies 77\n // characters. Thus, we need at most 9 segments.\n const uint32_t k1e9 = 1000000000U;\n int num_segments = 0;\n uint32_t segments[9];\n memset(segments, 0, sizeof(segments));\n uint64_t* most_significant_elem = words_little_endian + most_significant_elem_idx;\n\n do {\n // Compute remainder = words_little_endian % 1e9 and words_little_endian =\n // words_little_endian / 1e9.\n uint32_t remainder = 0;\n uint64_t* elem = most_significant_elem;\n\n do {\n // Compute dividend = (remainder << 32) | *elem (a virtual 96-bit integer);\n // *elem = dividend / 1e9;\n // remainder = dividend % 1e9.\n uint32_t hi = (uint32_t)(*elem >> 32);\n uint32_t lo = (uint32_t)(*elem & 0xFFFFFFFFULL);\n uint64_t dividend_hi = ((uint64_t)(remainder) << 32) | hi;\n uint64_t quotient_hi = dividend_hi / k1e9;\n remainder = (uint32_t)(dividend_hi % k1e9);\n uint64_t dividend_lo = ((uint64_t)(remainder) << 32) | lo;\n uint64_t quotient_lo = dividend_lo / k1e9;\n remainder = (uint32_t)(dividend_lo % k1e9);\n\n *elem = (quotient_hi << 32) | quotient_lo;\n } while (elem-- != words_little_endian);\n\n segments[num_segments++] = remainder;\n } while (*most_significant_elem != 0 || most_significant_elem-- != words_little_endian);\n\n // We know our output has no more than 9 digits per segment, plus a negative sign,\n // plus any further digits between our output of 9 digits plus enough\n // extra characters to ensure that snprintf() with n = 21 (maximum length of %lu\n // including a the null terminator) is bounded properly.\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, num_segments * 9 + 1 + 21 - 9));\n if (is_negative) {\n buffer->data[buffer->size_bytes++] = '-';\n }\n\n // The most significant segment should have no leading zeroes\n int n_chars = snprintf((char*)buffer->data + buffer->size_bytes, 21, \"%lu\",\n (unsigned long)segments[num_segments - 1]);\n\n // Ensure that an encoding error from snprintf() does not result\n // in an out-of-bounds access.\n if (n_chars < 0) {\n return ERANGE;\n }\n\n buffer->size_bytes += n_chars;\n\n // Subsequent output needs to be left-padded with zeroes such that each segment\n // takes up exactly 9 digits.\n for (int i = num_segments - 2; i >= 0; i--) {\n int n_chars = snprintf((char*)buffer->data + buffer->size_bytes, 21, \"%09lu\",\n (unsigned long)segments[i]);\n buffer->size_bytes += n_chars;\n NANOARROW_DCHECK(buffer->size_bytes <= buffer->capacity_bytes);\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowDecimalAppendStringToBuffer(const struct ArrowDecimal* decimal,\n struct ArrowBuffer* buffer) {\n int64_t buffer_size = buffer->size_bytes;\n NANOARROW_RETURN_NOT_OK(ArrowDecimalAppendDigitsToBuffer(decimal, buffer));\n int64_t digits_size = buffer->size_bytes - buffer_size;\n\n if (decimal->scale <= 0) {\n // e.g., digits are -12345 and scale is -2 -> -1234500\n // Just add zeros to the end\n for (int i = decimal->scale; i < 0; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(buffer, '0'));\n }\n return NANOARROW_OK;\n }\n\n int is_negative = buffer->data[0] == '-';\n int64_t num_digits = digits_size - is_negative;\n if (num_digits <= decimal->scale) {\n // e.g., digits are -12345 and scale is 6 -> -0.012345\n // Insert \"0.\" between the (maybe) negative sign and the digits\n int64_t num_zeros_after_decimal = decimal->scale - num_digits;\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferResize(buffer, buffer->size_bytes + num_zeros_after_decimal + 2, 0));\n\n uint8_t* digits_start = buffer->data + is_negative;\n memmove(digits_start + num_zeros_after_decimal + 2, digits_start, num_digits);\n *digits_start++ = '0';\n *digits_start++ = '.';\n for (int i = 0; i < num_zeros_after_decimal; i++) {\n *digits_start++ = '0';\n }\n\n } else {\n // e.g., digits are -12345 and scale is 4 -> -1.2345\n // Insert a decimal point before scale digits of output\n NANOARROW_RETURN_NOT_OK(ArrowBufferResize(buffer, buffer->size_bytes + 1, 0));\n uint8_t* decimal_point_to_be = buffer->data + buffer->size_bytes - 1 - decimal->scale;\n memmove(decimal_point_to_be + 1, decimal_point_to_be, decimal->scale);\n *decimal_point_to_be = '.';\n }\n\n return NANOARROW_OK;\n}\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"nanoarrow/nanoarrow.h\"\n\nstatic void ArrowSchemaReleaseInternal(struct ArrowSchema* schema) {\n if (schema->format != NULL) ArrowFree((void*)schema->format);\n if (schema->name != NULL) ArrowFree((void*)schema->name);\n if (schema->metadata != NULL) ArrowFree((void*)schema->metadata);\n\n // This object owns the memory for all the children, but those\n // children may have been generated elsewhere and might have\n // their own release() callback.\n if (schema->children != NULL) {\n for (int64_t i = 0; i < schema->n_children; i++) {\n if (schema->children[i] != NULL) {\n if (schema->children[i]->release != NULL) {\n ArrowSchemaRelease(schema->children[i]);\n }\n\n ArrowFree(schema->children[i]);\n }\n }\n\n ArrowFree(schema->children);\n }\n\n // This object owns the memory for the dictionary but it\n // may have been generated somewhere else and have its own\n // release() callback.\n if (schema->dictionary != NULL) {\n if (schema->dictionary->release != NULL) {\n ArrowSchemaRelease(schema->dictionary);\n }\n\n ArrowFree(schema->dictionary);\n }\n\n // private data not currently used\n if (schema->private_data != NULL) {\n ArrowFree(schema->private_data);\n }\n\n schema->release = NULL;\n}\n\nstatic const char* ArrowSchemaFormatTemplate(enum ArrowType type) {\n switch (type) {\n case NANOARROW_TYPE_UNINITIALIZED:\n return NULL;\n case NANOARROW_TYPE_NA:\n return \"n\";\n case NANOARROW_TYPE_BOOL:\n return \"b\";\n\n case NANOARROW_TYPE_UINT8:\n return \"C\";\n case NANOARROW_TYPE_INT8:\n return \"c\";\n case NANOARROW_TYPE_UINT16:\n return \"S\";\n case NANOARROW_TYPE_INT16:\n return \"s\";\n case NANOARROW_TYPE_UINT32:\n return \"I\";\n case NANOARROW_TYPE_INT32:\n return \"i\";\n case NANOARROW_TYPE_UINT64:\n return \"L\";\n case NANOARROW_TYPE_INT64:\n return \"l\";\n\n case NANOARROW_TYPE_HALF_FLOAT:\n return \"e\";\n case NANOARROW_TYPE_FLOAT:\n return \"f\";\n case NANOARROW_TYPE_DOUBLE:\n return \"g\";\n\n case NANOARROW_TYPE_STRING:\n return \"u\";\n case NANOARROW_TYPE_LARGE_STRING:\n return \"U\";\n case NANOARROW_TYPE_STRING_VIEW:\n return \"vu\";\n case NANOARROW_TYPE_BINARY:\n return \"z\";\n case NANOARROW_TYPE_BINARY_VIEW:\n return \"vz\";\n case NANOARROW_TYPE_LARGE_BINARY:\n return \"Z\";\n\n case NANOARROW_TYPE_DATE32:\n return \"tdD\";\n case NANOARROW_TYPE_DATE64:\n return \"tdm\";\n case NANOARROW_TYPE_INTERVAL_MONTHS:\n return \"tiM\";\n case NANOARROW_TYPE_INTERVAL_DAY_TIME:\n return \"tiD\";\n case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:\n return \"tin\";\n\n case NANOARROW_TYPE_LIST:\n return \"+l\";\n case NANOARROW_TYPE_LARGE_LIST:\n return \"+L\";\n case NANOARROW_TYPE_LIST_VIEW:\n return \"+vl\";\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n return \"+vL\";\n case NANOARROW_TYPE_STRUCT:\n return \"+s\";\n case NANOARROW_TYPE_MAP:\n return \"+m\";\n case NANOARROW_TYPE_RUN_END_ENCODED:\n return \"+r\";\n\n default:\n return NULL;\n }\n}\n\nstatic int ArrowSchemaInitChildrenIfNeeded(struct ArrowSchema* schema,\n enum ArrowType type) {\n switch (type) {\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_LARGE_LIST:\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n case NANOARROW_TYPE_LIST_VIEW:\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, 1));\n ArrowSchemaInit(schema->children[0]);\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[0], \"item\"));\n break;\n case NANOARROW_TYPE_MAP:\n NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, 1));\n NANOARROW_RETURN_NOT_OK(\n ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_STRUCT));\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[0], \"entries\"));\n schema->children[0]->flags &= ~ARROW_FLAG_NULLABLE;\n NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema->children[0], 2));\n ArrowSchemaInit(schema->children[0]->children[0]);\n ArrowSchemaInit(schema->children[0]->children[1]);\n NANOARROW_RETURN_NOT_OK(\n ArrowSchemaSetName(schema->children[0]->children[0], \"key\"));\n schema->children[0]->children[0]->flags &= ~ARROW_FLAG_NULLABLE;\n NANOARROW_RETURN_NOT_OK(\n ArrowSchemaSetName(schema->children[0]->children[1], \"value\"));\n break;\n case NANOARROW_TYPE_RUN_END_ENCODED:\n NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, 2));\n ArrowSchemaInit(schema->children[0]);\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[0], \"run_ends\"));\n schema->children[0]->flags &= ~ARROW_FLAG_NULLABLE;\n ArrowSchemaInit(schema->children[1]);\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(schema->children[1], \"values\"));\n default:\n break;\n }\n\n return NANOARROW_OK;\n}\n\nvoid ArrowSchemaInit(struct ArrowSchema* schema) {\n schema->format = NULL;\n schema->name = NULL;\n schema->metadata = NULL;\n schema->flags = ARROW_FLAG_NULLABLE;\n schema->n_children = 0;\n schema->children = NULL;\n schema->dictionary = NULL;\n schema->private_data = NULL;\n schema->release = &ArrowSchemaReleaseInternal;\n}\n\nArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema, enum ArrowType type) {\n // We don't allocate the dictionary because it has to be nullptr\n // for non-dictionary-encoded arrays.\n\n // Set the format to a valid format string for type\n const char* template_format = ArrowSchemaFormatTemplate(type);\n\n // If type isn't recognized and not explicitly unset\n if (template_format == NULL && type != NANOARROW_TYPE_UNINITIALIZED) {\n return EINVAL;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(schema, template_format));\n\n // For types with an umabiguous child structure, allocate children\n return ArrowSchemaInitChildrenIfNeeded(schema, type);\n}\n\nArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_children) {\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema, NANOARROW_TYPE_STRUCT));\n NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, n_children));\n for (int64_t i = 0; i < n_children; i++) {\n ArrowSchemaInit(schema->children[i]);\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowType type) {\n ArrowSchemaInit(schema);\n\n int result = ArrowSchemaSetType(schema, type);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema);\n return result;\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema,\n enum ArrowType type, int32_t fixed_size) {\n if (fixed_size <= 0) {\n return EINVAL;\n }\n\n char buffer[64];\n int n_chars;\n switch (type) {\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n n_chars = snprintf(buffer, sizeof(buffer), \"w:%\" PRId32, fixed_size);\n break;\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n n_chars = snprintf(buffer, sizeof(buffer), \"+w:%\" PRId32, fixed_size);\n break;\n default:\n return EINVAL;\n }\n\n if (((size_t)n_chars) >= sizeof(buffer) || n_chars < 0) {\n return ERANGE;\n }\n\n buffer[n_chars] = '\\0';\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(schema, buffer));\n\n if (type == NANOARROW_TYPE_FIXED_SIZE_LIST) {\n NANOARROW_RETURN_NOT_OK(ArrowSchemaInitChildrenIfNeeded(schema, type));\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema, enum ArrowType type,\n int32_t decimal_precision,\n int32_t decimal_scale) {\n if (decimal_precision <= 0) {\n return EINVAL;\n }\n\n char buffer[64];\n int n_chars;\n switch (type) {\n case NANOARROW_TYPE_DECIMAL32:\n if (decimal_precision > 9) {\n return EINVAL;\n }\n\n n_chars = snprintf(buffer, sizeof(buffer), \"d:%d,%d,32\", decimal_precision,\n decimal_scale);\n break;\n case NANOARROW_TYPE_DECIMAL64:\n if (decimal_precision > 18) {\n return EINVAL;\n }\n\n n_chars = snprintf(buffer, sizeof(buffer), \"d:%d,%d,64\", decimal_precision,\n decimal_scale);\n break;\n case NANOARROW_TYPE_DECIMAL128:\n if (decimal_precision > 38) {\n return EINVAL;\n }\n\n n_chars =\n snprintf(buffer, sizeof(buffer), \"d:%d,%d\", decimal_precision, decimal_scale);\n break;\n case NANOARROW_TYPE_DECIMAL256:\n if (decimal_precision > 76) {\n return EINVAL;\n }\n\n n_chars = snprintf(buffer, sizeof(buffer), \"d:%d,%d,256\", decimal_precision,\n decimal_scale);\n break;\n default:\n return EINVAL;\n }\n\n if (((size_t)n_chars) >= sizeof(buffer) || n_chars < 0) {\n return ERANGE;\n }\n\n buffer[n_chars] = '\\0';\n return ArrowSchemaSetFormat(schema, buffer);\n}\n\nArrowErrorCode ArrowSchemaSetTypeRunEndEncoded(struct ArrowSchema* schema,\n enum ArrowType run_end_type) {\n switch (run_end_type) {\n case NANOARROW_TYPE_INT16:\n case NANOARROW_TYPE_INT32:\n case NANOARROW_TYPE_INT64:\n break;\n default:\n return EINVAL;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(\n schema, ArrowSchemaFormatTemplate(NANOARROW_TYPE_RUN_END_ENCODED)));\n NANOARROW_RETURN_NOT_OK(\n ArrowSchemaInitChildrenIfNeeded(schema, NANOARROW_TYPE_RUN_END_ENCODED));\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema->children[0], run_end_type));\n NANOARROW_RETURN_NOT_OK(\n ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_UNINITIALIZED));\n\n return NANOARROW_OK;\n}\n\nstatic const char* ArrowTimeUnitFormatString(enum ArrowTimeUnit time_unit) {\n switch (time_unit) {\n case NANOARROW_TIME_UNIT_SECOND:\n return \"s\";\n case NANOARROW_TIME_UNIT_MILLI:\n return \"m\";\n case NANOARROW_TIME_UNIT_MICRO:\n return \"u\";\n case NANOARROW_TIME_UNIT_NANO:\n return \"n\";\n default:\n return NULL;\n }\n}\n\nArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema, enum ArrowType type,\n enum ArrowTimeUnit time_unit,\n const char* timezone) {\n const char* time_unit_str = ArrowTimeUnitFormatString(time_unit);\n if (time_unit_str == NULL) {\n return EINVAL;\n }\n\n char buffer[128];\n int n_chars;\n switch (type) {\n case NANOARROW_TYPE_TIME32:\n if (timezone != NULL) {\n return EINVAL;\n }\n\n switch (time_unit) {\n case NANOARROW_TIME_UNIT_MICRO:\n case NANOARROW_TIME_UNIT_NANO:\n return EINVAL;\n default:\n break;\n }\n\n n_chars = snprintf(buffer, sizeof(buffer), \"tt%s\", time_unit_str);\n break;\n case NANOARROW_TYPE_TIME64:\n if (timezone != NULL) {\n return EINVAL;\n }\n\n switch (time_unit) {\n case NANOARROW_TIME_UNIT_SECOND:\n case NANOARROW_TIME_UNIT_MILLI:\n return EINVAL;\n default:\n break;\n }\n\n n_chars = snprintf(buffer, sizeof(buffer), \"tt%s\", time_unit_str);\n break;\n case NANOARROW_TYPE_TIMESTAMP:\n if (timezone == NULL) {\n timezone = \"\";\n }\n n_chars = snprintf(buffer, sizeof(buffer), \"ts%s:%s\", time_unit_str, timezone);\n break;\n case NANOARROW_TYPE_DURATION:\n if (timezone != NULL) {\n return EINVAL;\n }\n n_chars = snprintf(buffer, sizeof(buffer), \"tD%s\", time_unit_str);\n break;\n default:\n return EINVAL;\n }\n\n if (((size_t)n_chars) >= sizeof(buffer) || n_chars < 0) {\n return ERANGE;\n }\n\n buffer[n_chars] = '\\0';\n\n return ArrowSchemaSetFormat(schema, buffer);\n}\n\nArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowType type,\n int64_t n_children) {\n if (n_children < 0 || n_children > 127) {\n return EINVAL;\n }\n\n // Max valid size would be +ud:0,1,...126 = 401 characters + null terminator\n char format_out[512];\n int64_t format_out_size = 512;\n memset(format_out, 0, format_out_size);\n int n_chars;\n char* format_cursor = format_out;\n\n switch (type) {\n case NANOARROW_TYPE_SPARSE_UNION:\n n_chars = snprintf(format_cursor, format_out_size, \"+us:\");\n format_cursor += n_chars;\n format_out_size -= n_chars;\n break;\n case NANOARROW_TYPE_DENSE_UNION:\n n_chars = snprintf(format_cursor, format_out_size, \"+ud:\");\n format_cursor += n_chars;\n format_out_size -= n_chars;\n break;\n default:\n return EINVAL;\n }\n\n // Ensure that an encoding error from snprintf() does not result\n // in an out-of-bounds access.\n if (n_chars < 0) {\n return ERANGE;\n }\n\n if (n_children > 0) {\n n_chars = snprintf(format_cursor, format_out_size, \"0\");\n format_cursor += n_chars;\n format_out_size -= n_chars;\n\n for (int64_t i = 1; i < n_children; i++) {\n n_chars = snprintf(format_cursor, format_out_size, \",%\" PRId64, i);\n format_cursor += n_chars;\n format_out_size -= n_chars;\n }\n }\n\n // Ensure that an encoding error from snprintf() does not result\n // in an out-of-bounds access.\n if (n_chars < 0) {\n return ERANGE;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowSchemaSetFormat(schema, format_out));\n\n NANOARROW_RETURN_NOT_OK(ArrowSchemaAllocateChildren(schema, n_children));\n for (int64_t i = 0; i < n_children; i++) {\n ArrowSchemaInit(schema->children[i]);\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema, const char* format) {\n if (schema->format != NULL) {\n ArrowFree((void*)schema->format);\n }\n\n if (format != NULL) {\n size_t format_size = strlen(format) + 1;\n schema->format = (const char*)ArrowMalloc(format_size);\n if (schema->format == NULL) {\n return ENOMEM;\n }\n\n memcpy((void*)schema->format, format, format_size);\n } else {\n schema->format = NULL;\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema, const char* name) {\n if (schema->name != NULL) {\n ArrowFree((void*)schema->name);\n }\n\n if (name != NULL) {\n size_t name_size = strlen(name) + 1;\n schema->name = (const char*)ArrowMalloc(name_size);\n if (schema->name == NULL) {\n return ENOMEM;\n }\n\n memcpy((void*)schema->name, name, name_size);\n } else {\n schema->name = NULL;\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* metadata) {\n if (schema->metadata != NULL) {\n ArrowFree((void*)schema->metadata);\n }\n\n if (metadata != NULL) {\n size_t metadata_size = ArrowMetadataSizeOf(metadata);\n schema->metadata = (const char*)ArrowMalloc(metadata_size);\n if (schema->metadata == NULL) {\n return ENOMEM;\n }\n\n memcpy((void*)schema->metadata, metadata, metadata_size);\n } else {\n schema->metadata = NULL;\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema,\n int64_t n_children) {\n if (schema->children != NULL) {\n return EEXIST;\n }\n\n if (n_children > 0) {\n schema->children =\n (struct ArrowSchema**)ArrowMalloc(n_children * sizeof(struct ArrowSchema*));\n\n if (schema->children == NULL) {\n return ENOMEM;\n }\n\n schema->n_children = n_children;\n\n memset(schema->children, 0, n_children * sizeof(struct ArrowSchema*));\n\n for (int64_t i = 0; i < n_children; i++) {\n schema->children[i] = (struct ArrowSchema*)ArrowMalloc(sizeof(struct ArrowSchema));\n\n if (schema->children[i] == NULL) {\n return ENOMEM;\n }\n\n schema->children[i]->release = NULL;\n }\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaAllocateDictionary(struct ArrowSchema* schema) {\n if (schema->dictionary != NULL) {\n return EEXIST;\n }\n\n schema->dictionary = (struct ArrowSchema*)ArrowMalloc(sizeof(struct ArrowSchema));\n if (schema->dictionary == NULL) {\n return ENOMEM;\n }\n\n schema->dictionary->release = NULL;\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema,\n struct ArrowSchema* schema_out) {\n ArrowSchemaInit(schema_out);\n\n int result = ArrowSchemaSetFormat(schema_out, schema->format);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema_out);\n return result;\n }\n\n schema_out->flags = schema->flags;\n\n result = ArrowSchemaSetName(schema_out, schema->name);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema_out);\n return result;\n }\n\n result = ArrowSchemaSetMetadata(schema_out, schema->metadata);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema_out);\n return result;\n }\n\n result = ArrowSchemaAllocateChildren(schema_out, schema->n_children);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema_out);\n return result;\n }\n\n for (int64_t i = 0; i < schema->n_children; i++) {\n result = ArrowSchemaDeepCopy(schema->children[i], schema_out->children[i]);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema_out);\n return result;\n }\n }\n\n if (schema->dictionary != NULL) {\n result = ArrowSchemaAllocateDictionary(schema_out);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema_out);\n return result;\n }\n\n result = ArrowSchemaDeepCopy(schema->dictionary, schema_out->dictionary);\n if (result != NANOARROW_OK) {\n ArrowSchemaRelease(schema_out);\n return result;\n }\n }\n\n return NANOARROW_OK;\n}\n\nstatic void ArrowSchemaViewSetPrimitive(struct ArrowSchemaView* schema_view,\n enum ArrowType type) {\n schema_view->type = type;\n schema_view->storage_type = type;\n}\n\nstatic ArrowErrorCode ArrowSchemaViewParse(struct ArrowSchemaView* schema_view,\n const char* format,\n const char** format_end_out,\n struct ArrowError* error) {\n *format_end_out = format;\n\n // needed for decimal parsing\n const char* parse_start;\n char* parse_end;\n\n switch (format[0]) {\n case 'n':\n schema_view->type = NANOARROW_TYPE_NA;\n schema_view->storage_type = NANOARROW_TYPE_NA;\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'b':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_BOOL);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'c':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT8);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'C':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT8);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 's':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT16);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'S':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT16);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'i':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'I':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT32);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'l':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'L':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_UINT64);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'e':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_HALF_FLOAT);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'f':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_FLOAT);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'g':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DOUBLE);\n *format_end_out = format + 1;\n return NANOARROW_OK;\n\n // decimal\n case 'd':\n if (format[1] != ':' || format[2] == '\\0') {\n ArrowErrorSet(error, \"Expected ':precision,scale[,bitwidth]' following 'd'\");\n return EINVAL;\n }\n\n parse_start = format + 2;\n schema_view->decimal_precision = (int32_t)strtol(parse_start, &parse_end, 10);\n if (parse_end == parse_start || parse_end[0] != ',') {\n ArrowErrorSet(error, \"Expected 'precision,scale[,bitwidth]' following 'd:'\");\n return EINVAL;\n }\n\n parse_start = parse_end + 1;\n schema_view->decimal_scale = (int32_t)strtol(parse_start, &parse_end, 10);\n if (parse_end == parse_start) {\n ArrowErrorSet(error, \"Expected 'scale[,bitwidth]' following 'd:precision,'\");\n return EINVAL;\n } else if (parse_end[0] != ',') {\n schema_view->decimal_bitwidth = 128;\n } else {\n parse_start = parse_end + 1;\n schema_view->decimal_bitwidth = (int32_t)strtol(parse_start, &parse_end, 10);\n if (parse_start == parse_end) {\n ArrowErrorSet(error, \"Expected precision following 'd:precision,scale,'\");\n return EINVAL;\n }\n }\n\n *format_end_out = parse_end;\n\n switch (schema_view->decimal_bitwidth) {\n case 32:\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL32);\n return NANOARROW_OK;\n case 64:\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL64);\n return NANOARROW_OK;\n case 128:\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL128);\n return NANOARROW_OK;\n case 256:\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_DECIMAL256);\n return NANOARROW_OK;\n default:\n ArrowErrorSet(error,\n \"Expected decimal bitwidth of 128 or 256 but found %\" PRId32,\n schema_view->decimal_bitwidth);\n return EINVAL;\n }\n\n // validity + data\n case 'w':\n schema_view->type = NANOARROW_TYPE_FIXED_SIZE_BINARY;\n schema_view->storage_type = NANOARROW_TYPE_FIXED_SIZE_BINARY;\n if (format[1] != ':' || format[2] == '\\0') {\n ArrowErrorSet(error, \"Expected ':' following 'w'\");\n return EINVAL;\n }\n\n schema_view->fixed_size = (int32_t)strtol(format + 2, (char**)format_end_out, 10);\n return NANOARROW_OK;\n\n // validity + offset + data\n case 'z':\n schema_view->type = NANOARROW_TYPE_BINARY;\n schema_view->storage_type = NANOARROW_TYPE_BINARY;\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'u':\n schema_view->type = NANOARROW_TYPE_STRING;\n schema_view->storage_type = NANOARROW_TYPE_STRING;\n *format_end_out = format + 1;\n return NANOARROW_OK;\n\n // validity + large_offset + data\n case 'Z':\n schema_view->type = NANOARROW_TYPE_LARGE_BINARY;\n schema_view->storage_type = NANOARROW_TYPE_LARGE_BINARY;\n *format_end_out = format + 1;\n return NANOARROW_OK;\n case 'U':\n schema_view->type = NANOARROW_TYPE_LARGE_STRING;\n schema_view->storage_type = NANOARROW_TYPE_LARGE_STRING;\n *format_end_out = format + 1;\n return NANOARROW_OK;\n\n // nested types\n case '+':\n switch (format[1]) {\n // list has validity + offset or offset\n case 'l':\n schema_view->storage_type = NANOARROW_TYPE_LIST;\n schema_view->type = NANOARROW_TYPE_LIST;\n *format_end_out = format + 2;\n return NANOARROW_OK;\n\n // large list has validity + large_offset or large_offset\n case 'L':\n schema_view->storage_type = NANOARROW_TYPE_LARGE_LIST;\n schema_view->type = NANOARROW_TYPE_LARGE_LIST;\n *format_end_out = format + 2;\n return NANOARROW_OK;\n\n // run end encoded has no buffer at all\n case 'r':\n schema_view->storage_type = NANOARROW_TYPE_RUN_END_ENCODED;\n schema_view->type = NANOARROW_TYPE_RUN_END_ENCODED;\n *format_end_out = format + 2;\n return NANOARROW_OK;\n\n // just validity buffer\n case 'w':\n if (format[2] != ':' || format[3] == '\\0') {\n ArrowErrorSet(error, \"Expected ':' following '+w'\");\n return EINVAL;\n }\n\n schema_view->storage_type = NANOARROW_TYPE_FIXED_SIZE_LIST;\n schema_view->type = NANOARROW_TYPE_FIXED_SIZE_LIST;\n schema_view->fixed_size =\n (int32_t)strtol(format + 3, (char**)format_end_out, 10);\n return NANOARROW_OK;\n case 's':\n schema_view->storage_type = NANOARROW_TYPE_STRUCT;\n schema_view->type = NANOARROW_TYPE_STRUCT;\n *format_end_out = format + 2;\n return NANOARROW_OK;\n case 'm':\n schema_view->storage_type = NANOARROW_TYPE_MAP;\n schema_view->type = NANOARROW_TYPE_MAP;\n *format_end_out = format + 2;\n return NANOARROW_OK;\n\n // unions\n case 'u':\n switch (format[2]) {\n case 'd':\n schema_view->storage_type = NANOARROW_TYPE_DENSE_UNION;\n schema_view->type = NANOARROW_TYPE_DENSE_UNION;\n break;\n case 's':\n schema_view->storage_type = NANOARROW_TYPE_SPARSE_UNION;\n schema_view->type = NANOARROW_TYPE_SPARSE_UNION;\n break;\n default:\n ArrowErrorSet(error,\n \"Expected union format string +us: or \"\n \"+ud: but found '%s'\",\n format);\n return EINVAL;\n }\n\n if (format[3] == ':') {\n schema_view->union_type_ids = format + 4;\n int64_t n_type_ids =\n _ArrowParseUnionTypeIds(schema_view->union_type_ids, NULL);\n if (n_type_ids != schema_view->schema->n_children) {\n ArrowErrorSet(error,\n \"Expected union type_ids parameter to be a comma-separated \"\n \"list of %\" PRId64 \" values between 0 and 127 but found '%s'\",\n schema_view->schema->n_children, schema_view->union_type_ids);\n return EINVAL;\n }\n *format_end_out = format + strlen(format);\n return NANOARROW_OK;\n } else {\n ArrowErrorSet(error,\n \"Expected union format string +us: or +ud: \"\n \"but found '%s'\",\n format);\n return EINVAL;\n }\n\n // views\n case 'v':\n switch (format[2]) {\n case 'l':\n schema_view->storage_type = NANOARROW_TYPE_LIST_VIEW;\n schema_view->type = NANOARROW_TYPE_LIST_VIEW;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'L':\n schema_view->storage_type = NANOARROW_TYPE_LARGE_LIST_VIEW;\n schema_view->type = NANOARROW_TYPE_LARGE_LIST_VIEW;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n default:\n ArrowErrorSet(\n error, \"Expected view format string +vl or +vL but found '%s'\", format);\n return EINVAL;\n }\n default:\n ArrowErrorSet(error, \"Expected nested type format string but found '%s'\",\n format);\n return EINVAL;\n }\n\n // date/time types\n case 't':\n switch (format[1]) {\n // date\n case 'd':\n switch (format[2]) {\n case 'D':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);\n schema_view->type = NANOARROW_TYPE_DATE32;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'm':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_DATE64;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n default:\n ArrowErrorSet(error, \"Expected 'D' or 'm' following 'td' but found '%s'\",\n format + 2);\n return EINVAL;\n }\n\n // time of day\n case 't':\n switch (format[2]) {\n case 's':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);\n schema_view->type = NANOARROW_TYPE_TIME32;\n schema_view->time_unit = NANOARROW_TIME_UNIT_SECOND;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'm':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT32);\n schema_view->type = NANOARROW_TYPE_TIME32;\n schema_view->time_unit = NANOARROW_TIME_UNIT_MILLI;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'u':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_TIME64;\n schema_view->time_unit = NANOARROW_TIME_UNIT_MICRO;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'n':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_TIME64;\n schema_view->time_unit = NANOARROW_TIME_UNIT_NANO;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n default:\n ArrowErrorSet(\n error, \"Expected 's', 'm', 'u', or 'n' following 'tt' but found '%s'\",\n format + 2);\n return EINVAL;\n }\n\n // timestamp\n case 's':\n switch (format[2]) {\n case 's':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_TIMESTAMP;\n schema_view->time_unit = NANOARROW_TIME_UNIT_SECOND;\n break;\n case 'm':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_TIMESTAMP;\n schema_view->time_unit = NANOARROW_TIME_UNIT_MILLI;\n break;\n case 'u':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_TIMESTAMP;\n schema_view->time_unit = NANOARROW_TIME_UNIT_MICRO;\n break;\n case 'n':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_TIMESTAMP;\n schema_view->time_unit = NANOARROW_TIME_UNIT_NANO;\n break;\n default:\n ArrowErrorSet(\n error, \"Expected 's', 'm', 'u', or 'n' following 'ts' but found '%s'\",\n format + 2);\n return EINVAL;\n }\n\n if (format[3] != ':') {\n ArrowErrorSet(error, \"Expected ':' following '%.3s' but found '%s'\", format,\n format + 3);\n return EINVAL;\n }\n\n schema_view->timezone = format + 4;\n *format_end_out = format + strlen(format);\n return NANOARROW_OK;\n\n // duration\n case 'D':\n switch (format[2]) {\n case 's':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_DURATION;\n schema_view->time_unit = NANOARROW_TIME_UNIT_SECOND;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'm':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_DURATION;\n schema_view->time_unit = NANOARROW_TIME_UNIT_MILLI;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'u':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_DURATION;\n schema_view->time_unit = NANOARROW_TIME_UNIT_MICRO;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'n':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INT64);\n schema_view->type = NANOARROW_TYPE_DURATION;\n schema_view->time_unit = NANOARROW_TIME_UNIT_NANO;\n *format_end_out = format + 3;\n return NANOARROW_OK;\n default:\n ArrowErrorSet(error,\n \"Expected 's', 'm', u', or 'n' following 'tD' but found '%s'\",\n format + 2);\n return EINVAL;\n }\n\n // interval\n case 'i':\n switch (format[2]) {\n case 'M':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INTERVAL_MONTHS);\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'D':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_INTERVAL_DAY_TIME);\n *format_end_out = format + 3;\n return NANOARROW_OK;\n case 'n':\n ArrowSchemaViewSetPrimitive(schema_view,\n NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO);\n *format_end_out = format + 3;\n return NANOARROW_OK;\n default:\n ArrowErrorSet(error,\n \"Expected 'M', 'D', or 'n' following 'ti' but found '%s'\",\n format + 2);\n return EINVAL;\n }\n\n default:\n ArrowErrorSet(\n error, \"Expected 'd', 't', 's', 'D', or 'i' following 't' but found '%s'\",\n format + 1);\n return EINVAL;\n }\n\n // view types\n case 'v': {\n switch (format[1]) {\n case 'u':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_STRING_VIEW);\n *format_end_out = format + 2;\n return NANOARROW_OK;\n case 'z':\n ArrowSchemaViewSetPrimitive(schema_view, NANOARROW_TYPE_BINARY_VIEW);\n *format_end_out = format + 2;\n return NANOARROW_OK;\n default:\n ArrowErrorSet(error, \"Expected 'u', or 'z' following 'v' but found '%s'\",\n format + 1);\n return EINVAL;\n }\n }\n\n default:\n ArrowErrorSet(error, \"Unknown format: '%s'\", format);\n return EINVAL;\n }\n}\n\nstatic ArrowErrorCode ArrowSchemaViewValidateNChildren(\n struct ArrowSchemaView* schema_view, int64_t n_children, struct ArrowError* error) {\n if (n_children != -1 && schema_view->schema->n_children != n_children) {\n ArrowErrorSet(\n error, \"Expected schema with %\" PRId64 \" children but found %\" PRId64 \" children\",\n n_children, schema_view->schema->n_children);\n return EINVAL;\n }\n\n // Don't do a full validation of children but do check that they won't\n // segfault if inspected\n struct ArrowSchema* child;\n for (int64_t i = 0; i < schema_view->schema->n_children; i++) {\n child = schema_view->schema->children[i];\n if (child == NULL) {\n ArrowErrorSet(\n error, \"Expected valid schema at schema->children[%\" PRId64 \"] but found NULL\",\n i);\n return EINVAL;\n } else if (child->release == NULL) {\n ArrowErrorSet(error,\n \"Expected valid schema at schema->children[%\" PRId64\n \"] but found a released schema\",\n i);\n return EINVAL;\n }\n }\n\n return NANOARROW_OK;\n}\n\nstatic ArrowErrorCode ArrowSchemaViewValidateUnion(struct ArrowSchemaView* schema_view,\n struct ArrowError* error) {\n return ArrowSchemaViewValidateNChildren(schema_view, -1, error);\n}\n\nstatic ArrowErrorCode ArrowSchemaViewValidateMap(struct ArrowSchemaView* schema_view,\n struct ArrowError* error) {\n NANOARROW_RETURN_NOT_OK(ArrowSchemaViewValidateNChildren(schema_view, 1, error));\n\n if (schema_view->schema->children[0]->n_children != 2) {\n ArrowErrorSet(error,\n \"Expected child of map type to have 2 children but found %\" PRId64,\n schema_view->schema->children[0]->n_children);\n return EINVAL;\n }\n\n if (strcmp(schema_view->schema->children[0]->format, \"+s\") != 0) {\n ArrowErrorSet(error, \"Expected format of child of map type to be '+s' but found '%s'\",\n schema_view->schema->children[0]->format);\n return EINVAL;\n }\n\n if (schema_view->schema->children[0]->flags & ARROW_FLAG_NULLABLE) {\n ArrowErrorSet(error,\n \"Expected child of map type to be non-nullable but was nullable\");\n return EINVAL;\n }\n\n if (schema_view->schema->children[0]->children[0]->flags & ARROW_FLAG_NULLABLE) {\n ArrowErrorSet(error, \"Expected key of map type to be non-nullable but was nullable\");\n return EINVAL;\n }\n\n return NANOARROW_OK;\n}\n\nstatic ArrowErrorCode ArrowSchemaViewValidateDictionary(\n struct ArrowSchemaView* schema_view, struct ArrowError* error) {\n // check for valid index type\n switch (schema_view->storage_type) {\n case NANOARROW_TYPE_UINT8:\n case NANOARROW_TYPE_INT8:\n case NANOARROW_TYPE_UINT16:\n case NANOARROW_TYPE_INT16:\n case NANOARROW_TYPE_UINT32:\n case NANOARROW_TYPE_INT32:\n case NANOARROW_TYPE_UINT64:\n case NANOARROW_TYPE_INT64:\n break;\n default:\n ArrowErrorSet(\n error,\n \"Expected dictionary schema index type to be an integral type but found '%s'\",\n schema_view->schema->format);\n return EINVAL;\n }\n\n struct ArrowSchemaView dictionary_schema_view;\n return ArrowSchemaViewInit(&dictionary_schema_view, schema_view->schema->dictionary,\n error);\n}\n\nstatic ArrowErrorCode ArrowSchemaViewValidate(struct ArrowSchemaView* schema_view,\n enum ArrowType type,\n struct ArrowError* error) {\n switch (type) {\n case NANOARROW_TYPE_NA:\n case NANOARROW_TYPE_BOOL:\n case NANOARROW_TYPE_UINT8:\n case NANOARROW_TYPE_INT8:\n case NANOARROW_TYPE_UINT16:\n case NANOARROW_TYPE_INT16:\n case NANOARROW_TYPE_UINT32:\n case NANOARROW_TYPE_INT32:\n case NANOARROW_TYPE_UINT64:\n case NANOARROW_TYPE_INT64:\n case NANOARROW_TYPE_HALF_FLOAT:\n case NANOARROW_TYPE_FLOAT:\n case NANOARROW_TYPE_DOUBLE:\n case NANOARROW_TYPE_DECIMAL32:\n case NANOARROW_TYPE_DECIMAL64:\n case NANOARROW_TYPE_DECIMAL128:\n case NANOARROW_TYPE_DECIMAL256:\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_LARGE_STRING:\n case NANOARROW_TYPE_BINARY:\n case NANOARROW_TYPE_LARGE_BINARY:\n case NANOARROW_TYPE_DATE32:\n case NANOARROW_TYPE_DATE64:\n case NANOARROW_TYPE_INTERVAL_MONTHS:\n case NANOARROW_TYPE_INTERVAL_DAY_TIME:\n case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:\n case NANOARROW_TYPE_TIMESTAMP:\n case NANOARROW_TYPE_TIME32:\n case NANOARROW_TYPE_TIME64:\n case NANOARROW_TYPE_DURATION:\n case NANOARROW_TYPE_BINARY_VIEW:\n case NANOARROW_TYPE_STRING_VIEW:\n return ArrowSchemaViewValidateNChildren(schema_view, 0, error);\n\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n if (schema_view->fixed_size <= 0) {\n ArrowErrorSet(error, \"Expected size > 0 for fixed size binary but found size %d\",\n schema_view->fixed_size);\n return EINVAL;\n }\n return ArrowSchemaViewValidateNChildren(schema_view, 0, error);\n\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_LIST_VIEW:\n case NANOARROW_TYPE_LARGE_LIST:\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n return ArrowSchemaViewValidateNChildren(schema_view, 1, error);\n\n case NANOARROW_TYPE_RUN_END_ENCODED:\n return ArrowSchemaViewValidateNChildren(schema_view, 2, error);\n\n case NANOARROW_TYPE_STRUCT:\n return ArrowSchemaViewValidateNChildren(schema_view, -1, error);\n\n case NANOARROW_TYPE_SPARSE_UNION:\n case NANOARROW_TYPE_DENSE_UNION:\n return ArrowSchemaViewValidateUnion(schema_view, error);\n\n case NANOARROW_TYPE_MAP:\n return ArrowSchemaViewValidateMap(schema_view, error);\n\n case NANOARROW_TYPE_DICTIONARY:\n return ArrowSchemaViewValidateDictionary(schema_view, error);\n\n default:\n ArrowErrorSet(error, \"Expected a valid enum ArrowType value but found %d\",\n schema_view->type);\n return EINVAL;\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view,\n const struct ArrowSchema* schema,\n struct ArrowError* error) {\n if (schema == NULL) {\n ArrowErrorSet(error, \"Expected non-NULL schema\");\n return EINVAL;\n }\n\n if (schema->release == NULL) {\n ArrowErrorSet(error, \"Expected non-released schema\");\n return EINVAL;\n }\n\n schema_view->schema = schema;\n\n const char* format = schema->format;\n if (format == NULL) {\n ArrowErrorSet(\n error,\n \"Error parsing schema->format: Expected a null-terminated string but found NULL\");\n return EINVAL;\n }\n\n size_t format_len = strlen(format);\n if (format_len == 0) {\n ArrowErrorSet(error, \"Error parsing schema->format: Expected a string with size > 0\");\n return EINVAL;\n }\n\n const char* format_end_out;\n int result = ArrowSchemaViewParse(schema_view, format, &format_end_out, error);\n\n if (result != NANOARROW_OK) {\n if (error != NULL) {\n char child_error[1024];\n memcpy(child_error, ArrowErrorMessage(error), 1024);\n ArrowErrorSet(error, \"Error parsing schema->format: %s\", child_error);\n }\n\n return result;\n }\n\n if ((format + format_len) != format_end_out) {\n ArrowErrorSet(error, \"Error parsing schema->format '%s': parsed %d/%zu characters\",\n format, (int)(format_end_out - format), format_len);\n return EINVAL;\n }\n\n if (schema->dictionary != NULL) {\n schema_view->type = NANOARROW_TYPE_DICTIONARY;\n }\n\n NANOARROW_RETURN_NOT_OK(\n ArrowSchemaViewValidate(schema_view, schema_view->storage_type, error));\n\n if (schema_view->storage_type != schema_view->type) {\n NANOARROW_RETURN_NOT_OK(\n ArrowSchemaViewValidate(schema_view, schema_view->type, error));\n }\n\n int64_t unknown_flags = schema->flags & ~NANOARROW_FLAG_ALL_SUPPORTED;\n if (unknown_flags != 0) {\n ArrowErrorSet(error, \"Unknown ArrowSchema flag\");\n return EINVAL;\n }\n\n if (schema->flags & ARROW_FLAG_DICTIONARY_ORDERED &&\n schema_view->type != NANOARROW_TYPE_DICTIONARY) {\n ArrowErrorSet(error,\n \"ARROW_FLAG_DICTIONARY_ORDERED is only relevant for dictionaries\");\n return EINVAL;\n }\n\n if (schema->flags & ARROW_FLAG_MAP_KEYS_SORTED &&\n schema_view->type != NANOARROW_TYPE_MAP) {\n ArrowErrorSet(error, \"ARROW_FLAG_MAP_KEYS_SORTED is only relevant for a map type\");\n return EINVAL;\n }\n\n ArrowLayoutInit(&schema_view->layout, schema_view->storage_type);\n if (schema_view->storage_type == NANOARROW_TYPE_FIXED_SIZE_BINARY) {\n schema_view->layout.element_size_bits[1] = (int64_t)schema_view->fixed_size * 8;\n } else if (schema_view->storage_type == NANOARROW_TYPE_FIXED_SIZE_LIST) {\n schema_view->layout.child_size_elements = schema_view->fixed_size;\n }\n\n schema_view->extension_name = ArrowCharView(NULL);\n schema_view->extension_metadata = ArrowCharView(NULL);\n NANOARROW_RETURN_NOT_OK(ArrowMetadataGetValue(schema->metadata,\n ArrowCharView(\"ARROW:extension:name\"),\n &schema_view->extension_name));\n NANOARROW_RETURN_NOT_OK(ArrowMetadataGetValue(schema->metadata,\n ArrowCharView(\"ARROW:extension:metadata\"),\n &schema_view->extension_metadata));\n\n return NANOARROW_OK;\n}\n\nstatic int64_t ArrowSchemaTypeToStringInternal(struct ArrowSchemaView* schema_view,\n char* out, int64_t n) {\n const char* type_string = ArrowTypeString(schema_view->type);\n switch (schema_view->type) {\n case NANOARROW_TYPE_DECIMAL32:\n case NANOARROW_TYPE_DECIMAL64:\n case NANOARROW_TYPE_DECIMAL128:\n case NANOARROW_TYPE_DECIMAL256:\n return snprintf(out, n, \"%s(%\" PRId32 \", %\" PRId32 \")\", type_string,\n schema_view->decimal_precision, schema_view->decimal_scale);\n case NANOARROW_TYPE_TIMESTAMP:\n return snprintf(out, n, \"%s('%s', '%s')\", type_string,\n ArrowTimeUnitString(schema_view->time_unit), schema_view->timezone);\n case NANOARROW_TYPE_TIME32:\n case NANOARROW_TYPE_TIME64:\n case NANOARROW_TYPE_DURATION:\n return snprintf(out, n, \"%s('%s')\", type_string,\n ArrowTimeUnitString(schema_view->time_unit));\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n return snprintf(out, n, \"%s(%\" PRId32 \")\", type_string, schema_view->fixed_size);\n case NANOARROW_TYPE_SPARSE_UNION:\n case NANOARROW_TYPE_DENSE_UNION:\n return snprintf(out, n, \"%s([%s])\", type_string, schema_view->union_type_ids);\n default:\n return snprintf(out, n, \"%s\", type_string);\n }\n}\n\n// Helper for bookkeeping to emulate sprintf()-like behaviour spread\n// among multiple sprintf calls.\nstatic inline void ArrowToStringLogChars(char** out, int64_t n_chars_last,\n int64_t* n_remaining, int64_t* n_chars) {\n // In the unlikely snprintf() returning a negative value (encoding error),\n // ensure the result won't cause an out-of-bounds access.\n if (n_chars_last < 0) {\n n_chars_last = 0;\n }\n\n *n_chars += n_chars_last;\n *n_remaining -= n_chars_last;\n\n // n_remaining is never less than 0\n if (*n_remaining < 0) {\n *n_remaining = 0;\n }\n\n // Can't do math on a NULL pointer\n if (*out != NULL) {\n *out += n_chars_last;\n }\n}\n\nint64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t n,\n char recursive) {\n if (schema == NULL) {\n return snprintf(out, n, \"[invalid: pointer is null]\");\n }\n\n if (schema->release == NULL) {\n return snprintf(out, n, \"[invalid: schema is released]\");\n }\n\n struct ArrowSchemaView schema_view;\n struct ArrowError error;\n\n if (ArrowSchemaViewInit(&schema_view, schema, &error) != NANOARROW_OK) {\n return snprintf(out, n, \"[invalid: %s]\", ArrowErrorMessage(&error));\n }\n\n // Extension type and dictionary should include both the top-level type\n // and the storage type.\n int is_extension = schema_view.extension_name.size_bytes > 0;\n int is_dictionary = schema->dictionary != NULL;\n int64_t n_chars = 0;\n int64_t n_chars_last = 0;\n\n // Uncommon but not technically impossible that both are true\n if (is_extension && is_dictionary) {\n n_chars_last = snprintf(\n out, n, \"%.*s{dictionary(%s)<\", (int)schema_view.extension_name.size_bytes,\n schema_view.extension_name.data, ArrowTypeString(schema_view.storage_type));\n } else if (is_extension) {\n n_chars_last = snprintf(out, n, \"%.*s{\", (int)schema_view.extension_name.size_bytes,\n schema_view.extension_name.data);\n } else if (is_dictionary) {\n n_chars_last =\n snprintf(out, n, \"dictionary(%s)<\", ArrowTypeString(schema_view.storage_type));\n }\n\n ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);\n\n if (!is_dictionary) {\n n_chars_last = ArrowSchemaTypeToStringInternal(&schema_view, out, n);\n } else {\n n_chars_last = ArrowSchemaToString(schema->dictionary, out, n, recursive);\n }\n\n ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);\n\n if (recursive && schema->format[0] == '+') {\n n_chars_last = snprintf(out, n, \"<\");\n ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);\n\n for (int64_t i = 0; i < schema->n_children; i++) {\n if (i > 0) {\n n_chars_last = snprintf(out, n, \", \");\n ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);\n }\n\n // ArrowSchemaToStringInternal() will validate the child and print the error,\n // but we need the name first\n if (schema->children[i] != NULL && schema->children[i]->release != NULL &&\n schema->children[i]->name != NULL) {\n n_chars_last = snprintf(out, n, \"%s: \", schema->children[i]->name);\n ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);\n }\n\n n_chars_last = ArrowSchemaToString(schema->children[i], out, n, recursive);\n ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);\n }\n\n n_chars_last = snprintf(out, n, \">\");\n ArrowToStringLogChars(&out, n_chars_last, &n, &n_chars);\n }\n\n if (is_extension && is_dictionary) {\n n_chars += snprintf(out, n, \">}\");\n } else if (is_extension) {\n n_chars += snprintf(out, n, \"}\");\n } else if (is_dictionary) {\n n_chars += snprintf(out, n, \">\");\n }\n\n // Ensure that we always return a positive result\n if (n_chars > 0) {\n return n_chars;\n } else {\n return 0;\n }\n}\n\nArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader,\n const char* metadata) {\n reader->metadata = metadata;\n\n if (reader->metadata == NULL) {\n reader->offset = 0;\n reader->remaining_keys = 0;\n } else {\n memcpy(&reader->remaining_keys, reader->metadata, sizeof(int32_t));\n reader->offset = sizeof(int32_t);\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader,\n struct ArrowStringView* key_out,\n struct ArrowStringView* value_out) {\n if (reader->remaining_keys <= 0) {\n return EINVAL;\n }\n\n int64_t pos = 0;\n\n int32_t key_size;\n memcpy(&key_size, reader->metadata + reader->offset + pos, sizeof(int32_t));\n pos += sizeof(int32_t);\n\n key_out->data = reader->metadata + reader->offset + pos;\n key_out->size_bytes = key_size;\n pos += key_size;\n\n int32_t value_size;\n memcpy(&value_size, reader->metadata + reader->offset + pos, sizeof(int32_t));\n pos += sizeof(int32_t);\n\n value_out->data = reader->metadata + reader->offset + pos;\n value_out->size_bytes = value_size;\n pos += value_size;\n\n reader->offset += pos;\n reader->remaining_keys--;\n return NANOARROW_OK;\n}\n\nint64_t ArrowMetadataSizeOf(const char* metadata) {\n if (metadata == NULL) {\n return 0;\n }\n\n struct ArrowMetadataReader reader;\n struct ArrowStringView key;\n struct ArrowStringView value;\n if (ArrowMetadataReaderInit(&reader, metadata) != NANOARROW_OK) {\n return 0;\n }\n\n int64_t size = sizeof(int32_t);\n while (ArrowMetadataReaderRead(&reader, &key, &value) == NANOARROW_OK) {\n size += sizeof(int32_t) + key.size_bytes + sizeof(int32_t) + value.size_bytes;\n }\n\n return size;\n}\n\nstatic ArrowErrorCode ArrowMetadataGetValueInternal(const char* metadata,\n struct ArrowStringView* key,\n struct ArrowStringView* value_out) {\n struct ArrowMetadataReader reader;\n struct ArrowStringView existing_key;\n struct ArrowStringView existing_value;\n NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, metadata));\n\n while (ArrowMetadataReaderRead(&reader, &existing_key, &existing_value) ==\n NANOARROW_OK) {\n int key_equal = key->size_bytes == existing_key.size_bytes &&\n strncmp(key->data, existing_key.data, existing_key.size_bytes) == 0;\n if (key_equal) {\n value_out->data = existing_value.data;\n value_out->size_bytes = existing_value.size_bytes;\n break;\n }\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowMetadataGetValue(const char* metadata, struct ArrowStringView key,\n struct ArrowStringView* value_out) {\n if (value_out == NULL) {\n return EINVAL;\n }\n\n return ArrowMetadataGetValueInternal(metadata, &key, value_out);\n}\n\nchar ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key) {\n struct ArrowStringView value = ArrowCharView(NULL);\n if (ArrowMetadataGetValue(metadata, key, &value) != NANOARROW_OK) {\n return 0;\n }\n\n return value.data != NULL;\n}\n\nArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer,\n const char* metadata) {\n ArrowBufferInit(buffer);\n return ArrowBufferAppend(buffer, metadata, ArrowMetadataSizeOf(metadata));\n}\n\nstatic ArrowErrorCode ArrowMetadataBuilderAppendInternal(struct ArrowBuffer* buffer,\n struct ArrowStringView* key,\n struct ArrowStringView* value) {\n if (value == NULL) {\n return NANOARROW_OK;\n }\n\n if (buffer->capacity_bytes == 0) {\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(buffer, 0));\n }\n\n if (((size_t)buffer->capacity_bytes) < sizeof(int32_t)) {\n return EINVAL;\n }\n\n int32_t n_keys;\n memcpy(&n_keys, buffer->data, sizeof(int32_t));\n\n int32_t key_size = (int32_t)key->size_bytes;\n int32_t value_size = (int32_t)value->size_bytes;\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(\n buffer, sizeof(int32_t) + key_size + sizeof(int32_t) + value_size));\n\n ArrowBufferAppendUnsafe(buffer, &key_size, sizeof(int32_t));\n ArrowBufferAppendUnsafe(buffer, key->data, key_size);\n ArrowBufferAppendUnsafe(buffer, &value_size, sizeof(int32_t));\n ArrowBufferAppendUnsafe(buffer, value->data, value_size);\n\n n_keys++;\n memcpy(buffer->data, &n_keys, sizeof(int32_t));\n\n return NANOARROW_OK;\n}\n\nstatic ArrowErrorCode ArrowMetadataBuilderSetInternal(struct ArrowBuffer* buffer,\n struct ArrowStringView* key,\n struct ArrowStringView* value) {\n // Inspect the current value to see if we can avoid copying the buffer\n struct ArrowStringView current_value = ArrowCharView(NULL);\n NANOARROW_RETURN_NOT_OK(\n ArrowMetadataGetValueInternal((const char*)buffer->data, key, ¤t_value));\n\n // The key should be removed but no key exists\n if (value == NULL && current_value.data == NULL) {\n return NANOARROW_OK;\n }\n\n // The key/value can be appended because no key exists\n if (value != NULL && current_value.data == NULL) {\n return ArrowMetadataBuilderAppendInternal(buffer, key, value);\n }\n\n struct ArrowMetadataReader reader;\n struct ArrowStringView existing_key;\n struct ArrowStringView existing_value;\n NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, (const char*)buffer->data));\n\n struct ArrowBuffer new_buffer;\n NANOARROW_RETURN_NOT_OK(ArrowMetadataBuilderInit(&new_buffer, NULL));\n\n while (reader.remaining_keys > 0) {\n int result = ArrowMetadataReaderRead(&reader, &existing_key, &existing_value);\n if (result != NANOARROW_OK) {\n ArrowBufferReset(&new_buffer);\n return result;\n }\n\n if (key->size_bytes == existing_key.size_bytes &&\n strncmp((const char*)key->data, (const char*)existing_key.data,\n existing_key.size_bytes) == 0) {\n result = ArrowMetadataBuilderAppendInternal(&new_buffer, key, value);\n value = NULL;\n } else {\n result =\n ArrowMetadataBuilderAppendInternal(&new_buffer, &existing_key, &existing_value);\n }\n\n if (result != NANOARROW_OK) {\n ArrowBufferReset(&new_buffer);\n return result;\n }\n }\n\n ArrowBufferReset(buffer);\n ArrowBufferMove(&new_buffer, buffer);\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer,\n struct ArrowStringView key,\n struct ArrowStringView value) {\n return ArrowMetadataBuilderAppendInternal(buffer, &key, &value);\n}\n\nArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer,\n struct ArrowStringView key,\n struct ArrowStringView value) {\n return ArrowMetadataBuilderSetInternal(buffer, &key, &value);\n}\n\nArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer,\n struct ArrowStringView key) {\n return ArrowMetadataBuilderSetInternal(buffer, &key, NULL);\n}\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"nanoarrow/nanoarrow.h\"\n\nstatic void ArrowArrayReleaseInternal(struct ArrowArray* array) {\n // Release buffers held by this array\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n if (private_data != NULL) {\n ArrowBitmapReset(&private_data->bitmap);\n ArrowBufferReset(&private_data->buffers[0]);\n ArrowBufferReset(&private_data->buffers[1]);\n ArrowFree(private_data->buffer_data);\n for (int32_t i = 0; i < private_data->n_variadic_buffers; ++i) {\n ArrowBufferReset(&private_data->variadic_buffers[i]);\n }\n ArrowFree(private_data->variadic_buffers);\n ArrowFree(private_data);\n }\n\n // This object owns the memory for all the children, but those\n // children may have been generated elsewhere and might have\n // their own release() callback.\n if (array->children != NULL) {\n for (int64_t i = 0; i < array->n_children; i++) {\n if (array->children[i] != NULL) {\n if (array->children[i]->release != NULL) {\n ArrowArrayRelease(array->children[i]);\n }\n\n ArrowFree(array->children[i]);\n }\n }\n\n ArrowFree(array->children);\n }\n\n // This object owns the memory for the dictionary but it\n // may have been generated somewhere else and have its own\n // release() callback.\n if (array->dictionary != NULL) {\n if (array->dictionary->release != NULL) {\n ArrowArrayRelease(array->dictionary);\n }\n\n ArrowFree(array->dictionary);\n }\n\n // Mark released\n array->release = NULL;\n}\n\nstatic int ArrowArrayIsInternal(struct ArrowArray* array) {\n return array->release == &ArrowArrayReleaseInternal;\n}\n\nstatic ArrowErrorCode ArrowArraySetStorageType(struct ArrowArray* array,\n enum ArrowType storage_type) {\n switch (storage_type) {\n case NANOARROW_TYPE_UNINITIALIZED:\n case NANOARROW_TYPE_NA:\n case NANOARROW_TYPE_RUN_END_ENCODED:\n array->n_buffers = 0;\n break;\n\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n case NANOARROW_TYPE_STRUCT:\n case NANOARROW_TYPE_SPARSE_UNION:\n array->n_buffers = 1;\n break;\n\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_LARGE_LIST:\n case NANOARROW_TYPE_MAP:\n case NANOARROW_TYPE_BOOL:\n case NANOARROW_TYPE_UINT8:\n case NANOARROW_TYPE_INT8:\n case NANOARROW_TYPE_UINT16:\n case NANOARROW_TYPE_INT16:\n case NANOARROW_TYPE_UINT32:\n case NANOARROW_TYPE_INT32:\n case NANOARROW_TYPE_UINT64:\n case NANOARROW_TYPE_INT64:\n case NANOARROW_TYPE_HALF_FLOAT:\n case NANOARROW_TYPE_FLOAT:\n case NANOARROW_TYPE_DOUBLE:\n case NANOARROW_TYPE_DECIMAL32:\n case NANOARROW_TYPE_DECIMAL64:\n case NANOARROW_TYPE_DECIMAL128:\n case NANOARROW_TYPE_DECIMAL256:\n case NANOARROW_TYPE_INTERVAL_MONTHS:\n case NANOARROW_TYPE_INTERVAL_DAY_TIME:\n case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n case NANOARROW_TYPE_DENSE_UNION:\n array->n_buffers = 2;\n break;\n case NANOARROW_TYPE_BINARY_VIEW:\n case NANOARROW_TYPE_STRING_VIEW:\n array->n_buffers = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + 1;\n break;\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_LARGE_STRING:\n case NANOARROW_TYPE_BINARY:\n case NANOARROW_TYPE_LARGE_BINARY:\n case NANOARROW_TYPE_LIST_VIEW:\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n array->n_buffers = 3;\n break;\n\n default:\n return EINVAL;\n\n return NANOARROW_OK;\n }\n\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n private_data->storage_type = storage_type;\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array,\n enum ArrowType storage_type) {\n array->length = 0;\n array->null_count = 0;\n array->offset = 0;\n array->n_buffers = 0;\n array->n_children = 0;\n array->buffers = NULL;\n array->children = NULL;\n array->dictionary = NULL;\n array->release = &ArrowArrayReleaseInternal;\n array->private_data = NULL;\n\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)ArrowMalloc(sizeof(struct ArrowArrayPrivateData));\n if (private_data == NULL) {\n array->release = NULL;\n return ENOMEM;\n }\n\n ArrowBitmapInit(&private_data->bitmap);\n ArrowBufferInit(&private_data->buffers[0]);\n ArrowBufferInit(&private_data->buffers[1]);\n private_data->buffer_data =\n (const void**)ArrowMalloc(sizeof(void*) * NANOARROW_MAX_FIXED_BUFFERS);\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; ++i) {\n private_data->buffer_data[i] = NULL;\n }\n private_data->n_variadic_buffers = 0;\n private_data->variadic_buffers = NULL;\n private_data->list_view_offset = 0;\n\n array->private_data = private_data;\n array->buffers = (const void**)(private_data->buffer_data);\n\n // These are not technically \"storage\" in the sense that they do not appear\n // in the ArrowSchemaView's storage_type member; however, allowing them here\n // is helpful to maximize the number of types that can avoid going through\n // ArrowArrayInitFromSchema().\n switch (storage_type) {\n case NANOARROW_TYPE_DURATION:\n case NANOARROW_TYPE_TIMESTAMP:\n case NANOARROW_TYPE_TIME64:\n case NANOARROW_TYPE_DATE64:\n storage_type = NANOARROW_TYPE_INT64;\n break;\n case NANOARROW_TYPE_TIME32:\n case NANOARROW_TYPE_DATE32:\n storage_type = NANOARROW_TYPE_INT32;\n break;\n default:\n break;\n }\n\n int result = ArrowArraySetStorageType(array, storage_type);\n if (result != NANOARROW_OK) {\n ArrowArrayRelease(array);\n return result;\n }\n\n ArrowLayoutInit(&private_data->layout, storage_type);\n // We can only know this not to be true when initializing based on a schema\n // so assume this to be true.\n private_data->union_type_id_is_child_index = 1;\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array,\n const struct ArrowArrayView* array_view,\n struct ArrowError* error) {\n NANOARROW_RETURN_NOT_OK_WITH_ERROR(\n ArrowArrayInitFromType(array, array_view->storage_type), error);\n int result;\n\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n private_data->layout = array_view->layout;\n\n if (array_view->n_children > 0) {\n result = ArrowArrayAllocateChildren(array, array_view->n_children);\n if (result != NANOARROW_OK) {\n ArrowArrayRelease(array);\n return result;\n }\n\n for (int64_t i = 0; i < array_view->n_children; i++) {\n result =\n ArrowArrayInitFromArrayView(array->children[i], array_view->children[i], error);\n if (result != NANOARROW_OK) {\n ArrowArrayRelease(array);\n return result;\n }\n }\n }\n\n if (array_view->dictionary != NULL) {\n result = ArrowArrayAllocateDictionary(array);\n if (result != NANOARROW_OK) {\n ArrowArrayRelease(array);\n return result;\n }\n\n result =\n ArrowArrayInitFromArrayView(array->dictionary, array_view->dictionary, error);\n if (result != NANOARROW_OK) {\n ArrowArrayRelease(array);\n return result;\n }\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array,\n const struct ArrowSchema* schema,\n struct ArrowError* error) {\n struct ArrowArrayView array_view;\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewInitFromSchema(&array_view, schema, error));\n NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromArrayView(array, &array_view, error));\n if (array_view.storage_type == NANOARROW_TYPE_DENSE_UNION ||\n array_view.storage_type == NANOARROW_TYPE_SPARSE_UNION) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n // We can still build arrays if this isn't true; however, the append\n // functions won't work. Instead, we store this value and error only\n // when StartAppending is called.\n private_data->union_type_id_is_child_index =\n _ArrowUnionTypeIdsWillEqualChildIndices(schema->format + 4, schema->n_children);\n }\n\n ArrowArrayViewReset(&array_view);\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_children) {\n if (array->children != NULL) {\n return EINVAL;\n }\n\n if (n_children == 0) {\n return NANOARROW_OK;\n }\n\n array->children =\n (struct ArrowArray**)ArrowMalloc(n_children * sizeof(struct ArrowArray*));\n if (array->children == NULL) {\n return ENOMEM;\n }\n\n memset(array->children, 0, n_children * sizeof(struct ArrowArray*));\n\n for (int64_t i = 0; i < n_children; i++) {\n array->children[i] = (struct ArrowArray*)ArrowMalloc(sizeof(struct ArrowArray));\n if (array->children[i] == NULL) {\n return ENOMEM;\n }\n array->children[i]->release = NULL;\n }\n\n array->n_children = n_children;\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array) {\n if (array->dictionary != NULL) {\n return EINVAL;\n }\n\n array->dictionary = (struct ArrowArray*)ArrowMalloc(sizeof(struct ArrowArray));\n if (array->dictionary == NULL) {\n return ENOMEM;\n }\n\n array->dictionary->release = NULL;\n return NANOARROW_OK;\n}\n\nvoid ArrowArraySetValidityBitmap(struct ArrowArray* array, struct ArrowBitmap* bitmap) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n ArrowBufferMove(&bitmap->buffer, &private_data->bitmap.buffer);\n private_data->bitmap.size_bits = bitmap->size_bits;\n bitmap->size_bits = 0;\n private_data->buffer_data[0] = private_data->bitmap.buffer.data;\n array->null_count = -1;\n}\n\nArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i,\n struct ArrowBuffer* buffer) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n if (i >= array->n_buffers || i < 0) {\n return EINVAL;\n }\n\n // Find the `i`th buffer, release what is currently there, and move the\n // supplied buffer into that slot.\n struct ArrowBuffer* dst = ArrowArrayBuffer(array, i);\n ArrowBufferReset(dst);\n ArrowBufferMove(buffer, dst);\n\n // Flush the pointer into array->buffers. In theory clients should call\n // ArrowArrayFinishBuilding() to flush the pointer values before passing\n // this array elsewhere; however, in early nanoarrow versions this was not\n // needed and some code may depend on this being true.\n private_data->buffer_data[i] = dst->data;\n array->buffers = private_data->buffer_data;\n\n return NANOARROW_OK;\n}\n\nstatic ArrowErrorCode ArrowArrayViewInitFromArray(struct ArrowArrayView* array_view,\n struct ArrowArray* array,\n struct ArrowError* error) {\n if (!ArrowArrayIsInternal(array)) {\n ArrowErrorSet(error,\n \"Can't initialize internal ArrowArrayView from external ArrowArray\");\n return EINVAL;\n }\n\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n ArrowArrayViewInitFromType(array_view, private_data->storage_type);\n array_view->layout = private_data->layout;\n array_view->array = array;\n array_view->length = array->length;\n array_view->offset = array->offset;\n array_view->null_count = array->null_count;\n\n array_view->buffer_views[0].data.as_uint8 = private_data->bitmap.buffer.data;\n array_view->buffer_views[0].size_bytes = private_data->bitmap.buffer.size_bytes;\n array_view->buffer_views[1].data.as_uint8 = private_data->buffers[0].data;\n array_view->buffer_views[1].size_bytes = private_data->buffers[0].size_bytes;\n array_view->buffer_views[2].data.as_uint8 = private_data->buffers[1].data;\n array_view->buffer_views[2].size_bytes = private_data->buffers[1].size_bytes;\n\n int result = ArrowArrayViewAllocateChildren(array_view, array->n_children);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(array_view);\n return result;\n }\n\n for (int64_t i = 0; i < array->n_children; i++) {\n result =\n ArrowArrayViewInitFromArray(array_view->children[i], array->children[i], error);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(array_view);\n return result;\n }\n }\n\n if (array->dictionary != NULL) {\n result = ArrowArrayViewAllocateDictionary(array_view);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(array_view);\n return result;\n }\n\n result =\n ArrowArrayViewInitFromArray(array_view->dictionary, array->dictionary, error);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(array_view);\n return result;\n }\n }\n\n return NANOARROW_OK;\n}\n\nstatic ArrowErrorCode ArrowArrayReserveInternal(struct ArrowArray* array,\n struct ArrowArrayView* array_view) {\n // Loop through buffers and reserve the extra space that we know about\n for (int64_t i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n // Don't reserve on a validity buffer that hasn't been allocated yet\n if (array_view->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_VALIDITY &&\n ArrowArrayBuffer(array, i)->data == NULL) {\n continue;\n }\n\n int64_t additional_size_bytes =\n array_view->buffer_views[i].size_bytes - ArrowArrayBuffer(array, i)->size_bytes;\n\n if (additional_size_bytes > 0) {\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferReserve(ArrowArrayBuffer(array, i), additional_size_bytes));\n }\n }\n\n // Recursively reserve children\n for (int64_t i = 0; i < array->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(\n ArrowArrayReserveInternal(array->children[i], array_view->children[i]));\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayReserve(struct ArrowArray* array,\n int64_t additional_size_elements) {\n struct ArrowArrayView array_view;\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewInitFromArray(&array_view, array, NULL));\n\n // Calculate theoretical buffer sizes (recursively)\n ArrowArrayViewSetLength(&array_view, array->length + additional_size_elements);\n\n // Walk the structure (recursively)\n int result = ArrowArrayReserveInternal(array, &array_view);\n ArrowArrayViewReset(&array_view);\n if (result != NANOARROW_OK) {\n return result;\n }\n\n return NANOARROW_OK;\n}\n\nstatic ArrowErrorCode ArrowArrayFinalizeBuffers(struct ArrowArray* array) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_VALIDITY ||\n private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_NONE) {\n continue;\n }\n\n struct ArrowBuffer* buffer = ArrowArrayBuffer(array, i);\n if (buffer->data == NULL) {\n NANOARROW_RETURN_NOT_OK((ArrowBufferReserve(buffer, 1)));\n }\n }\n\n for (int64_t i = 0; i < array->n_children; i++) {\n if (ArrowArrayIsInternal(array->children[i])) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayFinalizeBuffers(array->children[i]));\n }\n }\n\n if (array->dictionary != NULL && ArrowArrayIsInternal(array->dictionary)) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayFinalizeBuffers(array->dictionary));\n }\n\n return NANOARROW_OK;\n}\n\nstatic ArrowErrorCode ArrowArrayFlushInternalPointers(struct ArrowArray* array) {\n NANOARROW_DCHECK(ArrowArrayIsInternal(array));\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n if (array->n_buffers > NANOARROW_MAX_FIXED_BUFFERS) {\n // If the variadic sizes buffer was not set and there is at least one variadic\n // buffer, populate it now (if there are no variadic buffers there will be exactly\n // three total buffers and we don't need to do anything special here). Notably, this\n // will occur when building a BinaryView/StringView array by element using the\n // appender.\n struct ArrowBuffer* sizes_buffer = ArrowArrayBuffer(array, array->n_buffers - 1);\n if (sizes_buffer->data == NULL && sizes_buffer->size_bytes == 0) {\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferReserve(sizes_buffer, private_data->n_variadic_buffers));\n for (int64_t i = 0; i < private_data->n_variadic_buffers; i++) {\n struct ArrowBuffer* variadic_buffer =\n ArrowArrayBuffer(array, i + NANOARROW_BINARY_VIEW_FIXED_BUFFERS);\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendInt64(sizes_buffer, variadic_buffer->size_bytes));\n }\n }\n }\n\n for (int32_t i = 0; i < array->n_buffers; i++) {\n private_data->buffer_data[i] = ArrowArrayBuffer(array, i)->data;\n }\n\n array->buffers = (const void**)(private_data->buffer_data);\n\n // Flush internal pointers for child/dictionary arrays if we allocated them. Clients\n // building arrays by buffer might have moved arrays from some other source (e.g.,\n // to create a record batch) and calling this function in that case will cause a crash.\n for (int64_t i = 0; i < array->n_children; i++) {\n if (ArrowArrayIsInternal(array->children[i])) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayFlushInternalPointers(array->children[i]));\n }\n }\n\n if (array->dictionary != NULL && ArrowArrayIsInternal(array->dictionary)) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayFlushInternalPointers(array->dictionary));\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array,\n enum ArrowValidationLevel validation_level,\n struct ArrowError* error) {\n // Even if the data buffer is size zero, the pointer value needed to be non-null\n // in some implementations (at least one version of Arrow C++ at the time this\n // was added and C# as later discovered). Only do this fix if we can assume\n // CPU data access.\n if (validation_level >= NANOARROW_VALIDATION_LEVEL_DEFAULT) {\n NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinalizeBuffers(array), error);\n }\n\n // Make sure the value we get with array->buffers[i] is set to the actual\n // pointer (which may have changed from the original due to reallocation)\n NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFlushInternalPointers(array), error);\n\n if (validation_level == NANOARROW_VALIDATION_LEVEL_NONE) {\n return NANOARROW_OK;\n }\n\n // For validation, initialize an ArrowArrayView with our known buffer sizes\n struct ArrowArrayView array_view;\n NANOARROW_RETURN_NOT_OK_WITH_ERROR(\n ArrowArrayViewInitFromArray(&array_view, array, error), error);\n int result = ArrowArrayViewValidate(&array_view, validation_level, error);\n ArrowArrayViewReset(&array_view);\n return result;\n}\n\nArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array,\n struct ArrowError* error) {\n return ArrowArrayFinishBuilding(array, NANOARROW_VALIDATION_LEVEL_DEFAULT, error);\n}\n\nvoid ArrowArrayViewInitFromType(struct ArrowArrayView* array_view,\n enum ArrowType storage_type) {\n memset(array_view, 0, sizeof(struct ArrowArrayView));\n array_view->storage_type = storage_type;\n ArrowLayoutInit(&array_view->layout, storage_type);\n}\n\nArrowErrorCode ArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view,\n int64_t n_children) {\n if (array_view->children != NULL) {\n return EINVAL;\n }\n\n if (n_children == 0) {\n array_view->n_children = 0;\n return NANOARROW_OK;\n }\n\n array_view->children =\n (struct ArrowArrayView**)ArrowMalloc(n_children * sizeof(struct ArrowArrayView*));\n if (array_view->children == NULL) {\n return ENOMEM;\n }\n\n for (int64_t i = 0; i < n_children; i++) {\n array_view->children[i] = NULL;\n }\n\n array_view->n_children = n_children;\n\n for (int64_t i = 0; i < n_children; i++) {\n array_view->children[i] =\n (struct ArrowArrayView*)ArrowMalloc(sizeof(struct ArrowArrayView));\n if (array_view->children[i] == NULL) {\n return ENOMEM;\n }\n ArrowArrayViewInitFromType(array_view->children[i], NANOARROW_TYPE_UNINITIALIZED);\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view) {\n if (array_view->dictionary != NULL) {\n return EINVAL;\n }\n\n array_view->dictionary =\n (struct ArrowArrayView*)ArrowMalloc(sizeof(struct ArrowArrayView));\n if (array_view->dictionary == NULL) {\n return ENOMEM;\n }\n\n ArrowArrayViewInitFromType(array_view->dictionary, NANOARROW_TYPE_UNINITIALIZED);\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view,\n const struct ArrowSchema* schema,\n struct ArrowError* error) {\n struct ArrowSchemaView schema_view;\n int result = ArrowSchemaViewInit(&schema_view, schema, error);\n if (result != NANOARROW_OK) {\n return result;\n }\n\n ArrowArrayViewInitFromType(array_view, schema_view.storage_type);\n array_view->layout = schema_view.layout;\n\n result = ArrowArrayViewAllocateChildren(array_view, schema->n_children);\n if (result != NANOARROW_OK) {\n ArrowErrorSet(error, \"ArrowArrayViewAllocateChildren() failed\");\n ArrowArrayViewReset(array_view);\n return result;\n }\n\n for (int64_t i = 0; i < schema->n_children; i++) {\n result =\n ArrowArrayViewInitFromSchema(array_view->children[i], schema->children[i], error);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(array_view);\n return result;\n }\n }\n\n if (schema->dictionary != NULL) {\n result = ArrowArrayViewAllocateDictionary(array_view);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(array_view);\n return result;\n }\n\n result =\n ArrowArrayViewInitFromSchema(array_view->dictionary, schema->dictionary, error);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(array_view);\n return result;\n }\n }\n\n if (array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION ||\n array_view->storage_type == NANOARROW_TYPE_DENSE_UNION) {\n array_view->union_type_id_map = (int8_t*)ArrowMalloc(256 * sizeof(int8_t));\n if (array_view->union_type_id_map == NULL) {\n return ENOMEM;\n }\n\n memset(array_view->union_type_id_map, -1, 256);\n int32_t n_type_ids = _ArrowParseUnionTypeIds(schema_view.union_type_ids,\n array_view->union_type_id_map + 128);\n for (int8_t child_index = 0; child_index < n_type_ids; child_index++) {\n int8_t type_id = array_view->union_type_id_map[128 + child_index];\n array_view->union_type_id_map[type_id] = child_index;\n }\n }\n\n return NANOARROW_OK;\n}\n\nvoid ArrowArrayViewReset(struct ArrowArrayView* array_view) {\n if (array_view->children != NULL) {\n for (int64_t i = 0; i < array_view->n_children; i++) {\n if (array_view->children[i] != NULL) {\n ArrowArrayViewReset(array_view->children[i]);\n ArrowFree(array_view->children[i]);\n }\n }\n\n ArrowFree(array_view->children);\n }\n\n if (array_view->dictionary != NULL) {\n ArrowArrayViewReset(array_view->dictionary);\n ArrowFree(array_view->dictionary);\n }\n\n if (array_view->union_type_id_map != NULL) {\n ArrowFree(array_view->union_type_id_map);\n }\n\n ArrowArrayViewInitFromType(array_view, NANOARROW_TYPE_UNINITIALIZED);\n}\n\nvoid ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length) {\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n int64_t element_size_bytes = array_view->layout.element_size_bits[i] / 8;\n\n switch (array_view->layout.buffer_type[i]) {\n case NANOARROW_BUFFER_TYPE_VALIDITY:\n array_view->buffer_views[i].size_bytes = _ArrowBytesForBits(length);\n continue;\n case NANOARROW_BUFFER_TYPE_DATA_OFFSET:\n // Probably don't want/need to rely on the producer to have allocated an\n // offsets buffer of length 1 for a zero-size array\n array_view->buffer_views[i].size_bytes =\n (length != 0) * element_size_bytes * (length + 1);\n continue;\n case NANOARROW_BUFFER_TYPE_DATA:\n array_view->buffer_views[i].size_bytes =\n _ArrowRoundUpToMultipleOf8(array_view->layout.element_size_bits[i] * length) /\n 8;\n continue;\n case NANOARROW_BUFFER_TYPE_TYPE_ID:\n case NANOARROW_BUFFER_TYPE_UNION_OFFSET:\n case NANOARROW_BUFFER_TYPE_VIEW_OFFSET:\n case NANOARROW_BUFFER_TYPE_SIZE:\n array_view->buffer_views[i].size_bytes = element_size_bytes * length;\n continue;\n case NANOARROW_BUFFER_TYPE_VARIADIC_DATA:\n case NANOARROW_BUFFER_TYPE_VARIADIC_SIZE:\n case NANOARROW_BUFFER_TYPE_NONE:\n array_view->buffer_views[i].size_bytes = 0;\n continue;\n }\n }\n\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_STRUCT:\n case NANOARROW_TYPE_SPARSE_UNION:\n for (int64_t i = 0; i < array_view->n_children; i++) {\n ArrowArrayViewSetLength(array_view->children[i], length);\n }\n break;\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n if (array_view->n_children >= 1) {\n ArrowArrayViewSetLength(array_view->children[0],\n length * array_view->layout.child_size_elements);\n }\n default:\n break;\n }\n}\n\n// This version recursively extracts information from the array and stores it\n// in the array view, performing any checks that require the original array.\nstatic int ArrowArrayViewSetArrayInternal(struct ArrowArrayView* array_view,\n const struct ArrowArray* array,\n struct ArrowError* error) {\n array_view->array = array;\n array_view->offset = array->offset;\n array_view->length = array->length;\n array_view->null_count = array->null_count;\n array_view->variadic_buffer_sizes = NULL;\n array_view->variadic_buffers = NULL;\n array_view->n_variadic_buffers = 0;\n\n int64_t buffers_required = 0;\n const int nfixed_buf = array_view->storage_type == NANOARROW_TYPE_STRING_VIEW ||\n array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW\n ? NANOARROW_BINARY_VIEW_FIXED_BUFFERS\n : NANOARROW_MAX_FIXED_BUFFERS;\n for (int i = 0; i < nfixed_buf; i++) {\n if (array_view->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_NONE) {\n break;\n }\n\n buffers_required++;\n\n // Set buffer pointer\n array_view->buffer_views[i].data.data = array->buffers[i];\n\n // If non-null, set buffer size to unknown.\n if (array->buffers[i] == NULL) {\n array_view->buffer_views[i].size_bytes = 0;\n } else {\n array_view->buffer_views[i].size_bytes = -1;\n }\n }\n\n if (array_view->storage_type == NANOARROW_TYPE_STRING_VIEW ||\n array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW) {\n const int64_t n_buffers = array->n_buffers;\n const int32_t nfixed_buf = NANOARROW_BINARY_VIEW_FIXED_BUFFERS;\n\n const int32_t nvariadic_buf = (int32_t)(n_buffers - nfixed_buf - 1);\n array_view->n_variadic_buffers = nvariadic_buf;\n buffers_required += nvariadic_buf + 1;\n array_view->variadic_buffers = array->buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS;\n array_view->variadic_buffer_sizes = (int64_t*)array->buffers[n_buffers - 1];\n }\n\n if (buffers_required != array->n_buffers) {\n ArrowErrorSet(error,\n \"Expected array with %\" PRId64 \" buffer(s) but found %\" PRId64\n \" buffer(s)\",\n buffers_required, array->n_buffers);\n return EINVAL;\n }\n\n // Check number of children\n if (array_view->n_children != array->n_children) {\n ArrowErrorSet(error, \"Expected %\" PRId64 \" children but found %\" PRId64 \" children\",\n array_view->n_children, array->n_children);\n return EINVAL;\n }\n\n // Recurse for children\n for (int64_t i = 0; i < array_view->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArrayInternal(array_view->children[i],\n array->children[i], error));\n }\n\n // Check dictionary\n if (array->dictionary == NULL && array_view->dictionary != NULL) {\n ArrowErrorSet(error, \"Expected dictionary but found NULL\");\n return EINVAL;\n }\n\n if (array->dictionary != NULL && array_view->dictionary == NULL) {\n ArrowErrorSet(error, \"Expected NULL dictionary but found dictionary member\");\n return EINVAL;\n }\n\n if (array->dictionary != NULL) {\n NANOARROW_RETURN_NOT_OK(\n ArrowArrayViewSetArrayInternal(array_view->dictionary, array->dictionary, error));\n }\n\n return NANOARROW_OK;\n}\n\nstatic int ArrowArrayViewValidateMinimal(struct ArrowArrayView* array_view,\n struct ArrowError* error) {\n if (array_view->length < 0) {\n ArrowErrorSet(error, \"Expected length >= 0 but found length %\" PRId64,\n array_view->length);\n return EINVAL;\n }\n\n if (array_view->offset < 0) {\n ArrowErrorSet(error, \"Expected offset >= 0 but found offset %\" PRId64,\n array_view->offset);\n return EINVAL;\n }\n\n // Ensure that offset + length fits within an int64 before a possible overflow\n if ((uint64_t)array_view->offset + (uint64_t)array_view->length > (uint64_t)INT64_MAX) {\n ArrowErrorSet(error, \"Offset + length is > INT64_MAX\");\n return EINVAL;\n }\n\n // Calculate buffer sizes that do not require buffer access. If marked as\n // unknown, assign the buffer size; otherwise, validate it.\n int64_t offset_plus_length = array_view->offset + array_view->length;\n\n // Only loop over the first two buffers because the size of the third buffer\n // is always data dependent for all current Arrow types.\n for (int i = 0; i < 2; i++) {\n int64_t element_size_bytes = array_view->layout.element_size_bits[i] / 8;\n // Initialize with a value that will cause an error if accidentally used uninitialized\n // Need to suppress the clang-tidy warning because gcc warns for possible use\n int64_t min_buffer_size_bytes = // NOLINT(clang-analyzer-deadcode.DeadStores)\n array_view->buffer_views[i].size_bytes + 1;\n\n switch (array_view->layout.buffer_type[i]) {\n case NANOARROW_BUFFER_TYPE_VALIDITY:\n if (array_view->null_count == 0 && array_view->buffer_views[i].size_bytes == 0) {\n continue;\n }\n\n min_buffer_size_bytes = _ArrowBytesForBits(offset_plus_length);\n break;\n case NANOARROW_BUFFER_TYPE_SIZE:\n min_buffer_size_bytes = element_size_bytes * offset_plus_length;\n break;\n case NANOARROW_BUFFER_TYPE_DATA_OFFSET:\n // Probably don't want/need to rely on the producer to have allocated an\n // offsets buffer of length 1 for a zero-size array\n min_buffer_size_bytes =\n (offset_plus_length != 0) * element_size_bytes * (offset_plus_length + 1);\n break;\n case NANOARROW_BUFFER_TYPE_VIEW_OFFSET:\n min_buffer_size_bytes =\n (offset_plus_length != 0) * element_size_bytes * offset_plus_length;\n break;\n case NANOARROW_BUFFER_TYPE_DATA:\n min_buffer_size_bytes =\n _ArrowRoundUpToMultipleOf8(array_view->layout.element_size_bits[i] *\n offset_plus_length) /\n 8;\n break;\n case NANOARROW_BUFFER_TYPE_TYPE_ID:\n case NANOARROW_BUFFER_TYPE_UNION_OFFSET:\n min_buffer_size_bytes = element_size_bytes * offset_plus_length;\n break;\n case NANOARROW_BUFFER_TYPE_VARIADIC_DATA:\n case NANOARROW_BUFFER_TYPE_VARIADIC_SIZE:\n case NANOARROW_BUFFER_TYPE_NONE:\n continue;\n }\n\n // Assign or validate buffer size\n if (array_view->buffer_views[i].size_bytes == -1) {\n array_view->buffer_views[i].size_bytes = min_buffer_size_bytes;\n } else if (array_view->buffer_views[i].size_bytes < min_buffer_size_bytes) {\n ArrowErrorSet(error,\n \"Expected %s array buffer %d to have size >= %\" PRId64\n \" bytes but found \"\n \"buffer with %\" PRId64 \" bytes\",\n ArrowTypeString(array_view->storage_type), i, min_buffer_size_bytes,\n array_view->buffer_views[i].size_bytes);\n return EINVAL;\n }\n }\n\n // For list, fixed-size list and map views, we can validate the number of children\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_LARGE_LIST:\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n case NANOARROW_TYPE_MAP:\n case NANOARROW_TYPE_LIST_VIEW:\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n if (array_view->n_children != 1) {\n ArrowErrorSet(error,\n \"Expected 1 child of %s array but found %\" PRId64 \" child arrays\",\n ArrowTypeString(array_view->storage_type), array_view->n_children);\n return EINVAL;\n }\n break;\n case NANOARROW_TYPE_RUN_END_ENCODED:\n if (array_view->n_children != 2) {\n ArrowErrorSet(\n error, \"Expected 2 children for %s array but found %\" PRId64 \" child arrays\",\n ArrowTypeString(array_view->storage_type), array_view->n_children);\n return EINVAL;\n }\n break;\n default:\n break;\n }\n\n // For struct, the sparse union, and the fixed-size list views, we can validate child\n // lengths.\n int64_t child_min_length;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_SPARSE_UNION:\n case NANOARROW_TYPE_STRUCT:\n child_min_length = (array_view->offset + array_view->length);\n for (int64_t i = 0; i < array_view->n_children; i++) {\n if (array_view->children[i]->length < child_min_length) {\n ArrowErrorSet(error,\n \"Expected struct child %\" PRId64 \" to have length >= %\" PRId64\n \" but found child with \"\n \"length %\" PRId64,\n i + 1, child_min_length, array_view->children[i]->length);\n return EINVAL;\n }\n }\n break;\n\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n child_min_length = (array_view->offset + array_view->length) *\n array_view->layout.child_size_elements;\n if (array_view->children[0]->length < child_min_length) {\n ArrowErrorSet(error,\n \"Expected child of fixed_size_list array to have length >= %\" PRId64\n \" but \"\n \"found array with length %\" PRId64,\n child_min_length, array_view->children[0]->length);\n return EINVAL;\n }\n break;\n\n case NANOARROW_TYPE_RUN_END_ENCODED: {\n if (array_view->n_children != 2) {\n ArrowErrorSet(error,\n \"Expected 2 children for run-end encoded array but found %\" PRId64,\n array_view->n_children);\n return EINVAL;\n }\n struct ArrowArrayView* run_ends_view = array_view->children[0];\n struct ArrowArrayView* values_view = array_view->children[1];\n int64_t max_length;\n switch (run_ends_view->storage_type) {\n case NANOARROW_TYPE_INT16:\n max_length = INT16_MAX;\n break;\n case NANOARROW_TYPE_INT32:\n max_length = INT32_MAX;\n break;\n case NANOARROW_TYPE_INT64:\n max_length = INT64_MAX;\n break;\n default:\n ArrowErrorSet(\n error,\n \"Run-end encoded array only supports INT16, INT32 or INT64 run-ends \"\n \"but found run-ends type %s\",\n ArrowTypeString(run_ends_view->storage_type));\n return EINVAL;\n }\n\n // There is already a check above that offset_plus_length < INT64_MAX\n if (offset_plus_length > max_length) {\n ArrowErrorSet(error,\n \"Offset + length of a run-end encoded array must fit in a value\"\n \" of the run end type %s but is %\" PRId64 \" + %\" PRId64,\n ArrowTypeString(run_ends_view->storage_type), array_view->offset,\n array_view->length);\n return EINVAL;\n }\n\n if (run_ends_view->length > values_view->length) {\n ArrowErrorSet(error,\n \"Length of run_ends is greater than the length of values: %\" PRId64\n \" > %\" PRId64,\n run_ends_view->length, values_view->length);\n return EINVAL;\n }\n\n if (run_ends_view->length == 0 && values_view->length != 0) {\n ArrowErrorSet(error,\n \"Run-end encoded array has zero length %\" PRId64\n \", but values array has \"\n \"non-zero length\",\n values_view->length);\n return EINVAL;\n }\n\n if (run_ends_view->null_count != 0) {\n ArrowErrorSet(error, \"Null count must be 0 for run ends array, but is %\" PRId64,\n run_ends_view->null_count);\n return EINVAL;\n }\n break;\n }\n\n default:\n break;\n }\n\n // Recurse for children\n for (int64_t i = 0; i < array_view->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(\n ArrowArrayViewValidateMinimal(array_view->children[i], error));\n }\n\n // Recurse for dictionary\n if (array_view->dictionary != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateMinimal(array_view->dictionary, error));\n }\n\n return NANOARROW_OK;\n}\n\nstatic int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view,\n struct ArrowError* error) {\n // Perform minimal validation. This will validate or assign\n // buffer sizes as long as buffer access is not required.\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateMinimal(array_view, error));\n\n // Calculate buffer sizes or child lengths that require accessing the offsets\n // buffer. Where appropriate, validate that the first offset is >= 0.\n // If a buffer size is marked as unknown, assign it; otherwise, validate it.\n int64_t offset_plus_length = array_view->offset + array_view->length;\n\n int64_t first_offset;\n int64_t last_offset;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_BINARY:\n if (array_view->buffer_views[1].size_bytes != 0) {\n first_offset = array_view->buffer_views[1].data.as_int32[array_view->offset];\n if (first_offset < 0) {\n ArrowErrorSet(error, \"Expected first offset >= 0 but found %\" PRId64,\n first_offset);\n return EINVAL;\n }\n\n last_offset = array_view->buffer_views[1].data.as_int32[offset_plus_length];\n if (last_offset < 0) {\n ArrowErrorSet(error, \"Expected last offset >= 0 but found %\" PRId64,\n last_offset);\n return EINVAL;\n }\n\n // If the data buffer size is unknown, assign it; otherwise, check it\n if (array_view->buffer_views[2].size_bytes == -1) {\n array_view->buffer_views[2].size_bytes = last_offset;\n } else if (array_view->buffer_views[2].size_bytes < last_offset) {\n ArrowErrorSet(error,\n \"Expected %s array buffer 2 to have size >= %\" PRId64\n \" bytes but found \"\n \"buffer with %\" PRId64 \" bytes\",\n ArrowTypeString(array_view->storage_type), last_offset,\n array_view->buffer_views[2].size_bytes);\n return EINVAL;\n }\n } else if (array_view->buffer_views[2].size_bytes == -1) {\n // If the data buffer size is unknown and there are no bytes in the offset buffer,\n // set the data buffer size to 0.\n array_view->buffer_views[2].size_bytes = 0;\n }\n break;\n\n case NANOARROW_TYPE_LARGE_STRING:\n case NANOARROW_TYPE_LARGE_BINARY:\n if (array_view->buffer_views[1].size_bytes != 0) {\n first_offset = array_view->buffer_views[1].data.as_int64[array_view->offset];\n if (first_offset < 0) {\n ArrowErrorSet(error, \"Expected first offset >= 0 but found %\" PRId64,\n first_offset);\n return EINVAL;\n }\n\n last_offset = array_view->buffer_views[1].data.as_int64[offset_plus_length];\n if (last_offset < 0) {\n ArrowErrorSet(error, \"Expected last offset >= 0 but found %\" PRId64,\n last_offset);\n return EINVAL;\n }\n\n // If the data buffer size is unknown, assign it; otherwise, check it\n if (array_view->buffer_views[2].size_bytes == -1) {\n array_view->buffer_views[2].size_bytes = last_offset;\n } else if (array_view->buffer_views[2].size_bytes < last_offset) {\n ArrowErrorSet(error,\n \"Expected %s array buffer 2 to have size >= %\" PRId64\n \" bytes but found \"\n \"buffer with %\" PRId64 \" bytes\",\n ArrowTypeString(array_view->storage_type), last_offset,\n array_view->buffer_views[2].size_bytes);\n return EINVAL;\n }\n } else if (array_view->buffer_views[2].size_bytes == -1) {\n // If the data buffer size is unknown and there are no bytes in the offset\n // buffer, set the data buffer size to 0.\n array_view->buffer_views[2].size_bytes = 0;\n }\n break;\n\n case NANOARROW_TYPE_STRUCT:\n for (int64_t i = 0; i < array_view->n_children; i++) {\n if (array_view->children[i]->length < offset_plus_length) {\n ArrowErrorSet(error,\n \"Expected struct child %\" PRId64 \" to have length >= %\" PRId64\n \" but found child with \"\n \"length %\" PRId64,\n i + 1, offset_plus_length, array_view->children[i]->length);\n return EINVAL;\n }\n }\n break;\n\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_MAP:\n if (array_view->buffer_views[1].size_bytes != 0) {\n first_offset = array_view->buffer_views[1].data.as_int32[array_view->offset];\n if (first_offset < 0) {\n ArrowErrorSet(error, \"Expected first offset >= 0 but found %\" PRId64,\n first_offset);\n return EINVAL;\n }\n\n last_offset = array_view->buffer_views[1].data.as_int32[offset_plus_length];\n if (last_offset < 0) {\n ArrowErrorSet(error, \"Expected last offset >= 0 but found %\" PRId64,\n last_offset);\n return EINVAL;\n }\n\n if (array_view->children[0]->length < last_offset) {\n ArrowErrorSet(error,\n \"Expected child of %s array to have length >= %\" PRId64\n \" but found array with \"\n \"length %\" PRId64,\n ArrowTypeString(array_view->storage_type), last_offset,\n array_view->children[0]->length);\n return EINVAL;\n }\n }\n break;\n\n case NANOARROW_TYPE_LARGE_LIST:\n if (array_view->buffer_views[1].size_bytes != 0) {\n first_offset = array_view->buffer_views[1].data.as_int64[array_view->offset];\n if (first_offset < 0) {\n ArrowErrorSet(error, \"Expected first offset >= 0 but found %\" PRId64,\n first_offset);\n return EINVAL;\n }\n\n last_offset = array_view->buffer_views[1].data.as_int64[offset_plus_length];\n if (last_offset < 0) {\n ArrowErrorSet(error, \"Expected last offset >= 0 but found %\" PRId64,\n last_offset);\n return EINVAL;\n }\n\n if (array_view->children[0]->length < last_offset) {\n ArrowErrorSet(error,\n \"Expected child of %s array to have length >= %\" PRId64\n \" but found array \"\n \"with length %\" PRId64,\n ArrowTypeString(array_view->storage_type), last_offset,\n array_view->children[0]->length);\n return EINVAL;\n }\n }\n break;\n\n case NANOARROW_TYPE_RUN_END_ENCODED: {\n struct ArrowArrayView* run_ends_view = array_view->children[0];\n if (run_ends_view->length == 0) {\n break;\n }\n\n int64_t first_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0);\n if (first_run_end < 1) {\n ArrowErrorSet(\n error,\n \"All run ends must be greater than 0 but the first run end is %\" PRId64,\n first_run_end);\n return EINVAL;\n }\n\n // offset + length < INT64_MAX is checked in ArrowArrayViewValidateMinimal()\n int64_t last_run_end =\n ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1);\n if (last_run_end < offset_plus_length) {\n ArrowErrorSet(error,\n \"Last run end is %\" PRId64 \" but it should be >= (%\" PRId64\n \" + %\" PRId64 \")\",\n last_run_end, array_view->offset, array_view->length);\n return EINVAL;\n }\n break;\n }\n default:\n break;\n }\n\n // Recurse for children\n for (int64_t i = 0; i < array_view->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(\n ArrowArrayViewValidateDefault(array_view->children[i], error));\n }\n\n // Recurse for dictionary\n if (array_view->dictionary != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateDefault(array_view->dictionary, error));\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view,\n const struct ArrowArray* array,\n struct ArrowError* error) {\n // Extract information from the array into the array view\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArrayInternal(array_view, array, error));\n\n // Run default validation. Because we've marked all non-NULL buffers as having unknown\n // size, validation will also update the buffer sizes as it goes.\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateDefault(array_view, error));\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view,\n const struct ArrowArray* array,\n struct ArrowError* error) {\n // Extract information from the array into the array view\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArrayInternal(array_view, array, error));\n\n // Run default validation. Because we've marked all non-NULL buffers as having unknown\n // size, validation will also update the buffer sizes as it goes.\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateMinimal(array_view, error));\n\n return NANOARROW_OK;\n}\n\nstatic int ArrowAssertIncreasingInt32(struct ArrowBufferView view,\n struct ArrowError* error) {\n if (view.size_bytes <= (int64_t)sizeof(int32_t)) {\n return NANOARROW_OK;\n }\n\n for (int64_t i = 1; i < view.size_bytes / (int64_t)sizeof(int32_t); i++) {\n if (view.data.as_int32[i] < view.data.as_int32[i - 1]) {\n ArrowErrorSet(error, \"[%\" PRId64 \"] Expected element size >= 0\", i);\n return EINVAL;\n }\n }\n\n return NANOARROW_OK;\n}\n\nstatic int ArrowAssertIncreasingInt64(struct ArrowBufferView view,\n struct ArrowError* error) {\n if (view.size_bytes <= (int64_t)sizeof(int64_t)) {\n return NANOARROW_OK;\n }\n\n for (int64_t i = 1; i < view.size_bytes / (int64_t)sizeof(int64_t); i++) {\n if (view.data.as_int64[i] < view.data.as_int64[i - 1]) {\n ArrowErrorSet(error, \"[%\" PRId64 \"] Expected element size >= 0\", i);\n return EINVAL;\n }\n }\n\n return NANOARROW_OK;\n}\n\nstatic int ArrowAssertRangeInt8(struct ArrowBufferView view, int8_t min_value,\n int8_t max_value, struct ArrowError* error) {\n for (int64_t i = 0; i < view.size_bytes; i++) {\n if (view.data.as_int8[i] < min_value || view.data.as_int8[i] > max_value) {\n ArrowErrorSet(error,\n \"[%\" PRId64 \"] Expected buffer value between %\" PRId8 \" and %\" PRId8\n \" but found value %\" PRId8,\n i, min_value, max_value, view.data.as_int8[i]);\n return EINVAL;\n }\n }\n\n return NANOARROW_OK;\n}\n\nstatic int ArrowAssertInt8In(struct ArrowBufferView view, const int8_t* values,\n int64_t n_values, struct ArrowError* error) {\n for (int64_t i = 0; i < view.size_bytes; i++) {\n int item_found = 0;\n for (int64_t j = 0; j < n_values; j++) {\n if (view.data.as_int8[i] == values[j]) {\n item_found = 1;\n break;\n }\n }\n\n if (!item_found) {\n ArrowErrorSet(error, \"[%\" PRId64 \"] Unexpected buffer value %\" PRId8, i,\n view.data.as_int8[i]);\n return EINVAL;\n }\n }\n\n return NANOARROW_OK;\n}\n\nstatic int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view,\n struct ArrowError* error) {\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n switch (array_view->layout.buffer_type[i]) {\n // Only validate the portion of the buffer that is strictly required,\n // which includes not validating the offset buffer of a zero-length array.\n case NANOARROW_BUFFER_TYPE_DATA_OFFSET:\n if (array_view->length == 0) {\n continue;\n }\n if (array_view->layout.element_size_bits[i] == 32) {\n struct ArrowBufferView sliced_offsets;\n sliced_offsets.data.as_int32 =\n array_view->buffer_views[i].data.as_int32 + array_view->offset;\n sliced_offsets.size_bytes = (array_view->length + 1) * sizeof(int32_t);\n NANOARROW_RETURN_NOT_OK(ArrowAssertIncreasingInt32(sliced_offsets, error));\n } else {\n struct ArrowBufferView sliced_offsets;\n sliced_offsets.data.as_int64 =\n array_view->buffer_views[i].data.as_int64 + array_view->offset;\n sliced_offsets.size_bytes = (array_view->length + 1) * sizeof(int64_t);\n NANOARROW_RETURN_NOT_OK(ArrowAssertIncreasingInt64(sliced_offsets, error));\n }\n break;\n default:\n break;\n }\n }\n\n if (array_view->storage_type == NANOARROW_TYPE_DENSE_UNION ||\n array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION) {\n struct ArrowBufferView sliced_type_ids;\n sliced_type_ids.size_bytes = array_view->length * sizeof(int8_t);\n if (array_view->length > 0) {\n sliced_type_ids.data.as_int8 =\n array_view->buffer_views[0].data.as_int8 + array_view->offset;\n } else {\n sliced_type_ids.data.as_int8 = NULL;\n }\n\n if (array_view->union_type_id_map == NULL) {\n // If the union_type_id map is NULL (e.g., when using ArrowArrayInitFromType() +\n // ArrowArrayAllocateChildren() + ArrowArrayFinishBuilding()), we don't have enough\n // information to validate this buffer.\n ArrowErrorSet(error,\n \"Insufficient information provided for validation of union array\");\n return EINVAL;\n } else if (_ArrowParsedUnionTypeIdsWillEqualChildIndices(\n array_view->union_type_id_map, array_view->n_children,\n array_view->n_children)) {\n NANOARROW_RETURN_NOT_OK(ArrowAssertRangeInt8(\n sliced_type_ids, 0, (int8_t)(array_view->n_children - 1), error));\n } else {\n NANOARROW_RETURN_NOT_OK(ArrowAssertInt8In(sliced_type_ids,\n array_view->union_type_id_map + 128,\n array_view->n_children, error));\n }\n }\n\n if (array_view->storage_type == NANOARROW_TYPE_DENSE_UNION &&\n array_view->union_type_id_map != NULL) {\n // Check that offsets refer to child elements that actually exist\n for (int64_t i = 0; i < array_view->length; i++) {\n int8_t child_id = ArrowArrayViewUnionChildIndex(array_view, i);\n int64_t offset = ArrowArrayViewUnionChildOffset(array_view, i);\n int64_t child_length = array_view->children[child_id]->length;\n if (offset < 0 || offset > child_length) {\n ArrowErrorSet(error,\n \"[%\" PRId64 \"] Expected union offset for child id %\" PRId8\n \" to be between 0 and %\" PRId64\n \" but \"\n \"found offset value %\" PRId64,\n i, child_id, child_length, offset);\n return EINVAL;\n }\n }\n }\n\n if (array_view->storage_type == NANOARROW_TYPE_RUN_END_ENCODED) {\n struct ArrowArrayView* run_ends_view = array_view->children[0];\n if (run_ends_view->length > 0) {\n int64_t last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0);\n for (int64_t i = 1; i < run_ends_view->length; i++) {\n const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i);\n if (run_end <= last_run_end) {\n ArrowErrorSet(\n error,\n \"Every run end must be strictly greater than the previous run end, \"\n \"but run_ends[%\" PRId64 \" is %\" PRId64 \" and run_ends[%\" PRId64\n \"] is %\" PRId64,\n i, run_end, i - 1, last_run_end);\n return EINVAL;\n }\n last_run_end = run_end;\n }\n }\n }\n\n if (array_view->storage_type == NANOARROW_TYPE_LIST_VIEW ||\n array_view->storage_type == NANOARROW_TYPE_LARGE_LIST_VIEW) {\n int64_t child_len = array_view->children[0]->length;\n\n struct ArrowBufferView offsets, sizes;\n offsets.data.data = array_view->buffer_views[1].data.data;\n sizes.data.data = array_view->buffer_views[2].data.data;\n\n for (int64_t i = array_view->offset; i < array_view->length + array_view->offset;\n i++) {\n int64_t offset, size;\n if (array_view->storage_type == NANOARROW_TYPE_LIST_VIEW) {\n offset = offsets.data.as_int32[i];\n size = sizes.data.as_int32[i];\n } else {\n offset = offsets.data.as_int64[i];\n size = sizes.data.as_int64[i];\n }\n\n if (offset < 0) {\n ArrowErrorSet(error, \"Invalid negative offset %\" PRId64 \" at index %\" PRId64,\n offset, i);\n return EINVAL;\n }\n\n if (size < 0) {\n ArrowErrorSet(error, \"Invalid negative size %\" PRId64 \" at index %\" PRId64, size,\n i);\n return EINVAL;\n }\n\n if ((offset + size) > child_len) {\n ArrowErrorSet(error,\n \"Offset: %\" PRId64 \" + size: %\" PRId64 \" at index: %\" PRId64\n \" exceeds length of child view: %\" PRId64,\n offset, size, i, child_len);\n return EINVAL;\n }\n }\n }\n\n // Recurse for children\n for (int64_t i = 0; i < array_view->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateFull(array_view->children[i], error));\n }\n\n // Dictionary validation not implemented\n if (array_view->dictionary != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateFull(array_view->dictionary, error));\n // TODO: validate the indices\n }\n\n return NANOARROW_OK;\n}\n\nArrowErrorCode ArrowArrayViewValidate(struct ArrowArrayView* array_view,\n enum ArrowValidationLevel validation_level,\n struct ArrowError* error) {\n switch (validation_level) {\n case NANOARROW_VALIDATION_LEVEL_NONE:\n return NANOARROW_OK;\n case NANOARROW_VALIDATION_LEVEL_MINIMAL:\n return ArrowArrayViewValidateMinimal(array_view, error);\n case NANOARROW_VALIDATION_LEVEL_DEFAULT:\n return ArrowArrayViewValidateDefault(array_view, error);\n case NANOARROW_VALIDATION_LEVEL_FULL:\n NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateDefault(array_view, error));\n return ArrowArrayViewValidateFull(array_view, error);\n }\n\n ArrowErrorSet(error, \"validation_level not recognized\");\n return EINVAL;\n}\n\nstruct ArrowComparisonInternalState {\n enum ArrowCompareLevel level;\n int is_equal;\n struct ArrowError* reason;\n};\n\nNANOARROW_CHECK_PRINTF_ATTRIBUTE static void ArrowComparePrependPath(\n struct ArrowError* out, const char* fmt, ...) {\n if (out == NULL) {\n return;\n }\n\n char prefix[128];\n prefix[0] = '\\0';\n va_list args;\n va_start(args, fmt);\n int prefix_len = vsnprintf(prefix, sizeof(prefix), fmt, args);\n va_end(args);\n\n if (prefix_len <= 0) {\n return;\n }\n\n size_t out_len = strlen(out->message);\n size_t out_len_to_move = sizeof(struct ArrowError) - prefix_len - 1;\n if (out_len_to_move > out_len) {\n out_len_to_move = out_len;\n }\n\n memmove(out->message + prefix_len, out->message, out_len_to_move);\n memcpy(out->message, prefix, prefix_len);\n out->message[out_len + prefix_len] = '\\0';\n}\n\n#define SET_NOT_EQUAL_AND_RETURN_IF_IMPL(cond_, state_, reason_) \\\n do { \\\n if (cond_) { \\\n ArrowErrorSet(state_->reason, \": %s\", reason_); \\\n state_->is_equal = 0; \\\n return; \\\n } \\\n } while (0)\n\n#define SET_NOT_EQUAL_AND_RETURN_IF(condition_, state_) \\\n SET_NOT_EQUAL_AND_RETURN_IF_IMPL(condition_, state_, #condition_)\n\nstatic void ArrowArrayViewCompareBuffer(const struct ArrowArrayView* actual,\n const struct ArrowArrayView* expected, int i,\n struct ArrowComparisonInternalState* state) {\n SET_NOT_EQUAL_AND_RETURN_IF(\n actual->buffer_views[i].size_bytes != expected->buffer_views[i].size_bytes, state);\n\n int64_t buffer_size = actual->buffer_views[i].size_bytes;\n if (buffer_size > 0) {\n SET_NOT_EQUAL_AND_RETURN_IF(\n memcmp(actual->buffer_views[i].data.data, expected->buffer_views[i].data.data,\n buffer_size) != 0,\n state);\n }\n}\n\nstatic void ArrowArrayViewCompareIdentical(const struct ArrowArrayView* actual,\n const struct ArrowArrayView* expected,\n struct ArrowComparisonInternalState* state) {\n SET_NOT_EQUAL_AND_RETURN_IF(actual->storage_type != expected->storage_type, state);\n SET_NOT_EQUAL_AND_RETURN_IF(actual->n_children != expected->n_children, state);\n SET_NOT_EQUAL_AND_RETURN_IF(actual->dictionary == NULL && expected->dictionary != NULL,\n state);\n SET_NOT_EQUAL_AND_RETURN_IF(actual->dictionary != NULL && expected->dictionary == NULL,\n state);\n\n SET_NOT_EQUAL_AND_RETURN_IF(actual->length != expected->length, state);\n SET_NOT_EQUAL_AND_RETURN_IF(actual->offset != expected->offset, state);\n SET_NOT_EQUAL_AND_RETURN_IF(actual->null_count != expected->null_count, state);\n\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n ArrowArrayViewCompareBuffer(actual, expected, i, state);\n if (!state->is_equal) {\n ArrowComparePrependPath(state->reason, \".buffers[%d]\", i);\n return;\n }\n }\n\n for (int64_t i = 0; i < actual->n_children; i++) {\n ArrowArrayViewCompareIdentical(actual->children[i], expected->children[i], state);\n if (!state->is_equal) {\n ArrowComparePrependPath(state->reason, \".children[%\" PRId64 \"]\", i);\n return;\n }\n }\n\n if (actual->dictionary != NULL) {\n ArrowArrayViewCompareIdentical(actual->dictionary, expected->dictionary, state);\n if (!state->is_equal) {\n ArrowComparePrependPath(state->reason, \".dictionary\");\n return;\n }\n }\n}\n\n// Top-level entry point to take care of creating, cleaning up, and\n// propagating the ArrowComparisonInternalState to the caller\nArrowErrorCode ArrowArrayViewCompare(const struct ArrowArrayView* actual,\n const struct ArrowArrayView* expected,\n enum ArrowCompareLevel level, int* out,\n struct ArrowError* reason) {\n struct ArrowComparisonInternalState state;\n state.level = level;\n state.is_equal = 1;\n state.reason = reason;\n\n switch (level) {\n case NANOARROW_COMPARE_IDENTICAL:\n ArrowArrayViewCompareIdentical(actual, expected, &state);\n break;\n default:\n return EINVAL;\n }\n\n *out = state.is_equal;\n if (!state.is_equal) {\n ArrowComparePrependPath(state.reason, \"root\");\n }\n\n return NANOARROW_OK;\n}\n\n#undef SET_NOT_EQUAL_AND_RETURN_IF\n#undef SET_NOT_EQUAL_AND_RETURN_IF_IMPL\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#include \n\n#include \"nanoarrow/nanoarrow.h\"\n\nstruct BasicArrayStreamPrivate {\n struct ArrowSchema schema;\n int64_t n_arrays;\n struct ArrowArray* arrays;\n int64_t arrays_i;\n};\n\nstatic int ArrowBasicArrayStreamGetSchema(struct ArrowArrayStream* array_stream,\n struct ArrowSchema* schema) {\n if (array_stream == NULL || array_stream->release == NULL) {\n return EINVAL;\n }\n\n struct BasicArrayStreamPrivate* private_data =\n (struct BasicArrayStreamPrivate*)array_stream->private_data;\n return ArrowSchemaDeepCopy(&private_data->schema, schema);\n}\n\nstatic int ArrowBasicArrayStreamGetNext(struct ArrowArrayStream* array_stream,\n struct ArrowArray* array) {\n if (array_stream == NULL || array_stream->release == NULL) {\n return EINVAL;\n }\n\n struct BasicArrayStreamPrivate* private_data =\n (struct BasicArrayStreamPrivate*)array_stream->private_data;\n\n if (private_data->arrays_i == private_data->n_arrays) {\n array->release = NULL;\n return NANOARROW_OK;\n }\n\n ArrowArrayMove(&private_data->arrays[private_data->arrays_i++], array);\n return NANOARROW_OK;\n}\n\nstatic const char* ArrowBasicArrayStreamGetLastError(\n struct ArrowArrayStream* array_stream) {\n NANOARROW_UNUSED(array_stream);\n return NULL;\n}\n\nstatic void ArrowBasicArrayStreamRelease(struct ArrowArrayStream* array_stream) {\n if (array_stream == NULL || array_stream->release == NULL) {\n return;\n }\n\n struct BasicArrayStreamPrivate* private_data =\n (struct BasicArrayStreamPrivate*)array_stream->private_data;\n\n if (private_data->schema.release != NULL) {\n ArrowSchemaRelease(&private_data->schema);\n }\n\n for (int64_t i = 0; i < private_data->n_arrays; i++) {\n if (private_data->arrays[i].release != NULL) {\n ArrowArrayRelease(&private_data->arrays[i]);\n }\n }\n\n if (private_data->arrays != NULL) {\n ArrowFree(private_data->arrays);\n }\n\n ArrowFree(private_data);\n array_stream->release = NULL;\n}\n\nArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream,\n struct ArrowSchema* schema, int64_t n_arrays) {\n struct BasicArrayStreamPrivate* private_data =\n (struct BasicArrayStreamPrivate*)ArrowMalloc(\n sizeof(struct BasicArrayStreamPrivate));\n if (private_data == NULL) {\n return ENOMEM;\n }\n\n ArrowSchemaMove(schema, &private_data->schema);\n\n private_data->n_arrays = n_arrays;\n private_data->arrays = NULL;\n private_data->arrays_i = 0;\n\n if (n_arrays > 0) {\n private_data->arrays =\n (struct ArrowArray*)ArrowMalloc(n_arrays * sizeof(struct ArrowArray));\n if (private_data->arrays == NULL) {\n ArrowBasicArrayStreamRelease(array_stream);\n ArrowFree(private_data);\n return ENOMEM;\n }\n }\n\n for (int64_t i = 0; i < private_data->n_arrays; i++) {\n private_data->arrays[i].release = NULL;\n }\n\n array_stream->get_schema = &ArrowBasicArrayStreamGetSchema;\n array_stream->get_next = &ArrowBasicArrayStreamGetNext;\n array_stream->get_last_error = ArrowBasicArrayStreamGetLastError;\n array_stream->release = ArrowBasicArrayStreamRelease;\n array_stream->private_data = private_data;\n return NANOARROW_OK;\n}\n\nvoid ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream, int64_t i,\n struct ArrowArray* array) {\n struct BasicArrayStreamPrivate* private_data =\n (struct BasicArrayStreamPrivate*)array_stream->private_data;\n ArrowArrayMove(array, &private_data->arrays[i]);\n}\n\nArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* array_stream,\n struct ArrowError* error) {\n struct BasicArrayStreamPrivate* private_data =\n (struct BasicArrayStreamPrivate*)array_stream->private_data;\n\n struct ArrowArrayView array_view;\n NANOARROW_RETURN_NOT_OK(\n ArrowArrayViewInitFromSchema(&array_view, &private_data->schema, error));\n\n for (int64_t i = 0; i < private_data->n_arrays; i++) {\n if (private_data->arrays[i].release != NULL) {\n int result = ArrowArrayViewSetArray(&array_view, &private_data->arrays[i], error);\n if (result != NANOARROW_OK) {\n ArrowArrayViewReset(&array_view);\n return result;\n }\n }\n }\n\n ArrowArrayViewReset(&array_view);\n return NANOARROW_OK;\n}\n" }, { "path": "data/lang/cpp/nanoarrow/nanoarrow.h", "content": "// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#ifndef NANOARROW_CONFIG_H_INCLUDED\n#define NANOARROW_CONFIG_H_INCLUDED\n\n#define NANOARROW_VERSION_MAJOR 0\n#define NANOARROW_VERSION_MINOR 8\n#define NANOARROW_VERSION_PATCH 0\n#define NANOARROW_VERSION \"0.8.0\"\n\n#define NANOARROW_VERSION_INT \\\n (NANOARROW_VERSION_MAJOR * 10000 + NANOARROW_VERSION_MINOR * 100 + \\\n NANOARROW_VERSION_PATCH)\n\n// #define NANOARROW_NAMESPACE YourNamespaceHere\n\n#if !defined(NANOARROW_CXX_NAMESPACE)\n#define NANOARROW_CXX_NAMESPACE nanoarrow\n#endif\n\n#define NANOARROW_CXX_NAMESPACE_BEGIN namespace NANOARROW_CXX_NAMESPACE {\n#define NANOARROW_CXX_NAMESPACE_END }\n\n#endif\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#ifndef NANOARROW_NANOARROW_TYPES_H_INCLUDED\n#define NANOARROW_NANOARROW_TYPES_H_INCLUDED\n\n#include \n#include \n\n\n\n#if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE)\n#include \n#include \n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// Extra guard for versions of Arrow without the canonical guard\n#ifndef ARROW_FLAG_DICTIONARY_ORDERED\n\n/// \\defgroup nanoarrow-arrow-cdata Arrow C Data interface\n///\n/// The Arrow C Data (https://arrow.apache.org/docs/format/CDataInterface.html)\n/// and Arrow C Stream (https://arrow.apache.org/docs/format/CStreamInterface.html)\n/// interfaces are part of the\n/// Arrow Columnar Format specification\n/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow documentation for\n/// documentation of these structures.\n///\n/// @{\n\n#ifndef ARROW_C_DATA_INTERFACE\n#define ARROW_C_DATA_INTERFACE\n\n#define ARROW_FLAG_DICTIONARY_ORDERED 1\n#define ARROW_FLAG_NULLABLE 2\n#define ARROW_FLAG_MAP_KEYS_SORTED 4\n\nstruct ArrowSchema {\n // Array type description\n const char* format;\n const char* name;\n const char* metadata;\n int64_t flags;\n int64_t n_children;\n struct ArrowSchema** children;\n struct ArrowSchema* dictionary;\n\n // Release callback\n void (*release)(struct ArrowSchema*);\n // Opaque producer-specific data\n void* private_data;\n};\n\nstruct ArrowArray {\n // Array data description\n int64_t length;\n int64_t null_count;\n int64_t offset;\n int64_t n_buffers;\n int64_t n_children;\n const void** buffers;\n struct ArrowArray** children;\n struct ArrowArray* dictionary;\n\n // Release callback\n void (*release)(struct ArrowArray*);\n // Opaque producer-specific data\n void* private_data;\n};\n\n#endif // ARROW_C_DATA_INTERFACE\n\n#ifndef ARROW_C_STREAM_INTERFACE\n#define ARROW_C_STREAM_INTERFACE\n\nstruct ArrowArrayStream {\n // Callback to get the stream type\n // (will be the same for all arrays in the stream).\n //\n // Return value: 0 if successful, an `errno`-compatible error code otherwise.\n //\n // If successful, the ArrowSchema must be released independently from the stream.\n int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);\n\n // Callback to get the next array\n // (if no error and the array is released, the stream has ended)\n //\n // Return value: 0 if successful, an `errno`-compatible error code otherwise.\n //\n // If successful, the ArrowArray must be released independently from the stream.\n int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);\n\n // Callback to get optional detailed error information.\n // This must only be called if the last stream operation failed\n // with a non-0 return code.\n //\n // Return value: pointer to a null-terminated character array describing\n // the last error, or NULL if no description is available.\n //\n // The returned pointer is only valid until the next operation on this stream\n // (including release).\n const char* (*get_last_error)(struct ArrowArrayStream*);\n\n // Release callback: release the stream's own resources.\n // Note that arrays returned by `get_next` must be individually released.\n void (*release)(struct ArrowArrayStream*);\n\n // Opaque producer-specific data\n void* private_data;\n};\n\n#endif // ARROW_C_STREAM_INTERFACE\n#endif // ARROW_FLAG_DICTIONARY_ORDERED\n\n/// @}\n\n// Utility macros\n#define _NANOARROW_CONCAT(x, y) x##y\n#define _NANOARROW_MAKE_NAME(x, y) _NANOARROW_CONCAT(x, y)\n\n#define _NANOARROW_RETURN_NOT_OK_IMPL(NAME, EXPR) \\\n do { \\\n const int NAME = (EXPR); \\\n if (NAME) return NAME; \\\n } while (0)\n\n#define _NANOARROW_CHECK_RANGE(x_, min_, max_) \\\n NANOARROW_RETURN_NOT_OK((x_ >= min_ && x_ <= max_) ? NANOARROW_OK : EINVAL)\n\n#define _NANOARROW_CHECK_UPPER_LIMIT(x_, max_) \\\n NANOARROW_RETURN_NOT_OK((x_ <= max_) ? NANOARROW_OK : EINVAL)\n\n#if defined(NANOARROW_DEBUG)\n#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \\\n do { \\\n const int NAME = (EXPR); \\\n if (NAME) { \\\n ArrowErrorSet((ERROR_PTR_EXPR), \"%s failed with errno %d\\n* %s:%d\", EXPR_STR, \\\n NAME, __FILE__, __LINE__); \\\n return NAME; \\\n } \\\n } while (0)\n#else\n#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \\\n do { \\\n const int NAME = (EXPR); \\\n if (NAME) { \\\n ArrowErrorSet((ERROR_PTR_EXPR), \"%s failed with errno %d\", EXPR_STR, NAME); \\\n return NAME; \\\n } \\\n } while (0)\n#endif\n\n#if defined(NANOARROW_DEBUG)\n// For checking ArrowErrorSet() calls for valid printf format strings/arguments\n// If using mingw's c99-compliant printf, we need a different format-checking attribute\n#if defined(__USE_MINGW_ANSI_STDIO) && defined(__MINGW_PRINTF_FORMAT)\n#define NANOARROW_CHECK_PRINTF_ATTRIBUTE \\\n __attribute__((format(__MINGW_PRINTF_FORMAT, 2, 3)))\n#elif defined(__GNUC__)\n#define NANOARROW_CHECK_PRINTF_ATTRIBUTE __attribute__((format(printf, 2, 3)))\n#else\n#define NANOARROW_CHECK_PRINTF_ATTRIBUTE\n#endif\n\n// For checking calls to functions that return ArrowErrorCode\n#if defined(__GNUC__) && (__GNUC__ >= 4)\n#define NANOARROW_CHECK_RETURN_ATTRIBUTE __attribute__((warn_unused_result))\n#elif defined(_MSC_VER) && (_MSC_VER >= 1700)\n#define NANOARROW_CHECK_RETURN_ATTRIBUTE _Check_return_\n#else\n#define NANOARROW_CHECK_RETURN_ATTRIBUTE\n#endif\n\n#else\n#define NANOARROW_CHECK_RETURN_ATTRIBUTE\n#define NANOARROW_CHECK_PRINTF_ATTRIBUTE\n#endif\n\n#define NANOARROW_UNUSED(x) (void)(x)\n\n/// \\brief Return code for success.\n/// \\ingroup nanoarrow-errors\n#define NANOARROW_OK 0\n\n/// \\brief Represents an errno-compatible error code\n/// \\ingroup nanoarrow-errors\ntypedef int ArrowErrorCode;\n\n#if defined(NANOARROW_DEBUG)\n#define ArrowErrorCode NANOARROW_CHECK_RETURN_ATTRIBUTE ArrowErrorCode\n#endif\n\n/// \\brief Flags supported by ArrowSchemaViewInit()\n/// \\ingroup nanoarrow-schema-view\n#define NANOARROW_FLAG_ALL_SUPPORTED \\\n (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | ARROW_FLAG_MAP_KEYS_SORTED)\n\n/// \\brief Error type containing a UTF-8 encoded message.\n/// \\ingroup nanoarrow-errors\nstruct ArrowError {\n /// \\brief A character buffer with space for an error message.\n char message[1024];\n};\n\n/// \\brief Ensure an ArrowError is null-terminated by zeroing the first character.\n/// \\ingroup nanoarrow-errors\n///\n/// If error is NULL, this function does nothing.\nstatic inline void ArrowErrorInit(struct ArrowError* error) {\n if (error != NULL) {\n error->message[0] = '\\0';\n }\n}\n\n/// \\brief Get the contents of an error\n/// \\ingroup nanoarrow-errors\n///\n/// If error is NULL, returns \"\", or returns the contents of the error message\n/// otherwise.\nstatic inline const char* ArrowErrorMessage(struct ArrowError* error) {\n if (error == NULL) {\n return \"\";\n } else {\n return error->message;\n }\n}\n\n/// \\brief Set the contents of an error from an existing null-terminated string\n/// \\ingroup nanoarrow-errors\n///\n/// If error is NULL, this function does nothing.\nstatic inline void ArrowErrorSetString(struct ArrowError* error, const char* src) {\n if (error == NULL) {\n return;\n }\n\n int64_t src_len = strlen(src);\n if (src_len >= ((int64_t)sizeof(error->message))) {\n memcpy(error->message, src, sizeof(error->message) - 1);\n error->message[sizeof(error->message) - 1] = '\\0';\n } else {\n memcpy(error->message, src, src_len);\n error->message[src_len] = '\\0';\n }\n}\n\n/// \\brief Check the result of an expression and return it if not NANOARROW_OK\n/// \\ingroup nanoarrow-errors\n#define NANOARROW_RETURN_NOT_OK(EXPR) \\\n _NANOARROW_RETURN_NOT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR)\n\n/// \\brief Check the result of an expression and return it if not NANOARROW_OK,\n/// adding an auto-generated message to an ArrowError.\n/// \\ingroup nanoarrow-errors\n///\n/// This macro is used to ensure that functions that accept an ArrowError\n/// as input always set its message when returning an error code (e.g., when calling\n/// a nanoarrow function that does *not* accept ArrowError).\n#define NANOARROW_RETURN_NOT_OK_WITH_ERROR(EXPR, ERROR_EXPR) \\\n _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \\\n _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, ERROR_EXPR, #EXPR)\n\n#if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE)\n#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \\\n do { \\\n fprintf(stderr, \"%s failed with code %d\\n* %s:%d\\n\", EXPR_STR, (int)(VALUE), \\\n __FILE__, (int)__LINE__); \\\n abort(); \\\n } while (0)\n#endif\n\n#if defined(NANOARROW_DEBUG)\n#define _NANOARROW_ASSERT_OK_IMPL(NAME, EXPR, EXPR_STR) \\\n do { \\\n const int NAME = (EXPR); \\\n if (NAME) NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \\\n } while (0)\n\n/// \\brief Assert that an expression's value is NANOARROW_OK\n/// \\ingroup nanoarrow-errors\n///\n/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is true),\n/// print a message to stderr and abort. If nanoarrow was built in release mode,\n/// this statement has no effect. You can customize fatal error behaviour\n/// be defining the NANOARROW_PRINT_AND_DIE macro before including nanoarrow.h\n/// This macro is provided as a convenience for users and is not used internally.\n#define NANOARROW_ASSERT_OK(EXPR) \\\n _NANOARROW_ASSERT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR)\n\n#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \\\n do { \\\n if (!(EXPR)) NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \\\n } while (0)\n\n#define NANOARROW_DCHECK(EXPR) _NANOARROW_DCHECK_IMPL(EXPR, #EXPR)\n#else\n#define NANOARROW_ASSERT_OK(EXPR) (void)(EXPR)\n#define NANOARROW_DCHECK(EXPR)\n#endif\n\nstatic inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst) {\n NANOARROW_DCHECK(src != NULL);\n NANOARROW_DCHECK(dst != NULL);\n\n memcpy(dst, src, sizeof(struct ArrowSchema));\n src->release = NULL;\n}\n\nstatic inline void ArrowSchemaRelease(struct ArrowSchema* schema) {\n NANOARROW_DCHECK(schema != NULL);\n schema->release(schema);\n NANOARROW_DCHECK(schema->release == NULL);\n}\n\nstatic inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst) {\n NANOARROW_DCHECK(src != NULL);\n NANOARROW_DCHECK(dst != NULL);\n\n memcpy(dst, src, sizeof(struct ArrowArray));\n src->release = NULL;\n}\n\nstatic inline void ArrowArrayRelease(struct ArrowArray* array) {\n NANOARROW_DCHECK(array != NULL);\n array->release(array);\n NANOARROW_DCHECK(array->release == NULL);\n}\n\nstatic inline void ArrowArrayStreamMove(struct ArrowArrayStream* src,\n struct ArrowArrayStream* dst) {\n NANOARROW_DCHECK(src != NULL);\n NANOARROW_DCHECK(dst != NULL);\n\n memcpy(dst, src, sizeof(struct ArrowArrayStream));\n src->release = NULL;\n}\n\nstatic inline const char* ArrowArrayStreamGetLastError(\n struct ArrowArrayStream* array_stream) {\n NANOARROW_DCHECK(array_stream != NULL);\n\n const char* value = array_stream->get_last_error(array_stream);\n if (value == NULL) {\n return \"\";\n } else {\n return value;\n }\n}\n\nstatic inline ArrowErrorCode ArrowArrayStreamGetSchema(\n struct ArrowArrayStream* array_stream, struct ArrowSchema* out,\n struct ArrowError* error) {\n NANOARROW_DCHECK(array_stream != NULL);\n\n int result = array_stream->get_schema(array_stream, out);\n if (result != NANOARROW_OK && error != NULL) {\n ArrowErrorSetString(error, ArrowArrayStreamGetLastError(array_stream));\n }\n\n return result;\n}\n\nstatic inline ArrowErrorCode ArrowArrayStreamGetNext(\n struct ArrowArrayStream* array_stream, struct ArrowArray* out,\n struct ArrowError* error) {\n NANOARROW_DCHECK(array_stream != NULL);\n\n int result = array_stream->get_next(array_stream, out);\n if (result != NANOARROW_OK && error != NULL) {\n ArrowErrorSetString(error, ArrowArrayStreamGetLastError(array_stream));\n }\n\n return result;\n}\n\nstatic inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream) {\n NANOARROW_DCHECK(array_stream != NULL);\n array_stream->release(array_stream);\n NANOARROW_DCHECK(array_stream->release == NULL);\n}\n\nstatic char _ArrowIsLittleEndian(void) {\n uint32_t check = 1;\n char first_byte;\n memcpy(&first_byte, &check, sizeof(char));\n return first_byte;\n}\n\n/// \\brief Arrow type enumerator\n/// \\ingroup nanoarrow-utils\n///\n/// These names are intended to map to the corresponding arrow::Type::type\n/// enumerator; however, the numeric values are specifically not equal\n/// (i.e., do not rely on numeric comparison).\nenum ArrowType {\n NANOARROW_TYPE_UNINITIALIZED = 0,\n NANOARROW_TYPE_NA = 1,\n NANOARROW_TYPE_BOOL,\n NANOARROW_TYPE_UINT8,\n NANOARROW_TYPE_INT8,\n NANOARROW_TYPE_UINT16,\n NANOARROW_TYPE_INT16,\n NANOARROW_TYPE_UINT32,\n NANOARROW_TYPE_INT32,\n NANOARROW_TYPE_UINT64,\n NANOARROW_TYPE_INT64,\n NANOARROW_TYPE_HALF_FLOAT,\n NANOARROW_TYPE_FLOAT,\n NANOARROW_TYPE_DOUBLE,\n NANOARROW_TYPE_STRING,\n NANOARROW_TYPE_BINARY,\n NANOARROW_TYPE_FIXED_SIZE_BINARY,\n NANOARROW_TYPE_DATE32,\n NANOARROW_TYPE_DATE64,\n NANOARROW_TYPE_TIMESTAMP,\n NANOARROW_TYPE_TIME32,\n NANOARROW_TYPE_TIME64,\n NANOARROW_TYPE_INTERVAL_MONTHS,\n NANOARROW_TYPE_INTERVAL_DAY_TIME,\n NANOARROW_TYPE_DECIMAL128,\n NANOARROW_TYPE_DECIMAL256,\n NANOARROW_TYPE_LIST,\n NANOARROW_TYPE_STRUCT,\n NANOARROW_TYPE_SPARSE_UNION,\n NANOARROW_TYPE_DENSE_UNION,\n NANOARROW_TYPE_DICTIONARY,\n NANOARROW_TYPE_MAP,\n NANOARROW_TYPE_EXTENSION,\n NANOARROW_TYPE_FIXED_SIZE_LIST,\n NANOARROW_TYPE_DURATION,\n NANOARROW_TYPE_LARGE_STRING,\n NANOARROW_TYPE_LARGE_BINARY,\n NANOARROW_TYPE_LARGE_LIST,\n NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO,\n NANOARROW_TYPE_RUN_END_ENCODED,\n NANOARROW_TYPE_BINARY_VIEW,\n NANOARROW_TYPE_STRING_VIEW,\n NANOARROW_TYPE_DECIMAL32,\n NANOARROW_TYPE_DECIMAL64,\n NANOARROW_TYPE_LIST_VIEW,\n NANOARROW_TYPE_LARGE_LIST_VIEW,\n};\n\n/// \\brief Get a string value of an enum ArrowType value\n/// \\ingroup nanoarrow-utils\n///\n/// Returns NULL for invalid values for type\nstatic inline const char* ArrowTypeString(enum ArrowType type);\n\nstatic inline const char* ArrowTypeString(enum ArrowType type) {\n switch (type) {\n case NANOARROW_TYPE_NA:\n return \"na\";\n case NANOARROW_TYPE_BOOL:\n return \"bool\";\n case NANOARROW_TYPE_UINT8:\n return \"uint8\";\n case NANOARROW_TYPE_INT8:\n return \"int8\";\n case NANOARROW_TYPE_UINT16:\n return \"uint16\";\n case NANOARROW_TYPE_INT16:\n return \"int16\";\n case NANOARROW_TYPE_UINT32:\n return \"uint32\";\n case NANOARROW_TYPE_INT32:\n return \"int32\";\n case NANOARROW_TYPE_UINT64:\n return \"uint64\";\n case NANOARROW_TYPE_INT64:\n return \"int64\";\n case NANOARROW_TYPE_HALF_FLOAT:\n return \"half_float\";\n case NANOARROW_TYPE_FLOAT:\n return \"float\";\n case NANOARROW_TYPE_DOUBLE:\n return \"double\";\n case NANOARROW_TYPE_STRING:\n return \"string\";\n case NANOARROW_TYPE_BINARY:\n return \"binary\";\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n return \"fixed_size_binary\";\n case NANOARROW_TYPE_DATE32:\n return \"date32\";\n case NANOARROW_TYPE_DATE64:\n return \"date64\";\n case NANOARROW_TYPE_TIMESTAMP:\n return \"timestamp\";\n case NANOARROW_TYPE_TIME32:\n return \"time32\";\n case NANOARROW_TYPE_TIME64:\n return \"time64\";\n case NANOARROW_TYPE_INTERVAL_MONTHS:\n return \"interval_months\";\n case NANOARROW_TYPE_INTERVAL_DAY_TIME:\n return \"interval_day_time\";\n case NANOARROW_TYPE_DECIMAL32:\n return \"decimal32\";\n case NANOARROW_TYPE_DECIMAL64:\n return \"decimal64\";\n case NANOARROW_TYPE_DECIMAL128:\n return \"decimal128\";\n case NANOARROW_TYPE_DECIMAL256:\n return \"decimal256\";\n case NANOARROW_TYPE_LIST:\n return \"list\";\n case NANOARROW_TYPE_STRUCT:\n return \"struct\";\n case NANOARROW_TYPE_SPARSE_UNION:\n return \"sparse_union\";\n case NANOARROW_TYPE_DENSE_UNION:\n return \"dense_union\";\n case NANOARROW_TYPE_DICTIONARY:\n return \"dictionary\";\n case NANOARROW_TYPE_MAP:\n return \"map\";\n case NANOARROW_TYPE_EXTENSION:\n return \"extension\";\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n return \"fixed_size_list\";\n case NANOARROW_TYPE_DURATION:\n return \"duration\";\n case NANOARROW_TYPE_LARGE_STRING:\n return \"large_string\";\n case NANOARROW_TYPE_LARGE_BINARY:\n return \"large_binary\";\n case NANOARROW_TYPE_LARGE_LIST:\n return \"large_list\";\n case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:\n return \"interval_month_day_nano\";\n case NANOARROW_TYPE_RUN_END_ENCODED:\n return \"run_end_encoded\";\n case NANOARROW_TYPE_BINARY_VIEW:\n return \"binary_view\";\n case NANOARROW_TYPE_STRING_VIEW:\n return \"string_view\";\n case NANOARROW_TYPE_LIST_VIEW:\n return \"list_view\";\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n return \"large_list_view\";\n default:\n return NULL;\n }\n}\n\n/// \\brief Arrow time unit enumerator\n/// \\ingroup nanoarrow-utils\n///\n/// These names and values map to the corresponding arrow::TimeUnit::type\n/// enumerator.\nenum ArrowTimeUnit {\n NANOARROW_TIME_UNIT_SECOND = 0,\n NANOARROW_TIME_UNIT_MILLI = 1,\n NANOARROW_TIME_UNIT_MICRO = 2,\n NANOARROW_TIME_UNIT_NANO = 3\n};\n\n/// \\brief Validation level enumerator\n/// \\ingroup nanoarrow-array\nenum ArrowValidationLevel {\n /// \\brief Do not validate buffer sizes or content.\n NANOARROW_VALIDATION_LEVEL_NONE = 0,\n\n /// \\brief Validate buffer sizes that depend on array length but do not validate buffer\n /// sizes that depend on buffer data access.\n NANOARROW_VALIDATION_LEVEL_MINIMAL = 1,\n\n /// \\brief Validate all buffer sizes, including those that require buffer data access,\n /// but do not perform any checks that are O(1) along the length of the buffers.\n NANOARROW_VALIDATION_LEVEL_DEFAULT = 2,\n\n /// \\brief Validate all buffer sizes and all buffer content. This is useful in the\n /// context of untrusted input or input that may have been corrupted in transit.\n NANOARROW_VALIDATION_LEVEL_FULL = 3\n};\n\n/// \\brief Comparison level enumerator\n/// \\ingroup nanoarrow-utils\nenum ArrowCompareLevel {\n /// \\brief Consider arrays equal if buffers contain identical content\n /// and have identical offset, null count, and length. Note that this is\n /// a much stricter check than logical equality, which would take into\n /// account potentially different content of null slots, arrays with a\n /// non-zero offset, and other considerations.\n NANOARROW_COMPARE_IDENTICAL,\n};\n\n/// \\brief Get a string value of an enum ArrowTimeUnit value\n/// \\ingroup nanoarrow-utils\n///\n/// Returns NULL for invalid values for time_unit\nstatic inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit);\n\nstatic inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit) {\n switch (time_unit) {\n case NANOARROW_TIME_UNIT_SECOND:\n return \"s\";\n case NANOARROW_TIME_UNIT_MILLI:\n return \"ms\";\n case NANOARROW_TIME_UNIT_MICRO:\n return \"us\";\n case NANOARROW_TIME_UNIT_NANO:\n return \"ns\";\n default:\n return NULL;\n }\n}\n\n/// \\brief Functional types of buffers as described in the Arrow Columnar Specification\n/// \\ingroup nanoarrow-array-view\nenum ArrowBufferType {\n NANOARROW_BUFFER_TYPE_NONE,\n NANOARROW_BUFFER_TYPE_VALIDITY,\n NANOARROW_BUFFER_TYPE_TYPE_ID,\n NANOARROW_BUFFER_TYPE_UNION_OFFSET,\n NANOARROW_BUFFER_TYPE_DATA_OFFSET,\n NANOARROW_BUFFER_TYPE_DATA,\n NANOARROW_BUFFER_TYPE_VARIADIC_DATA,\n NANOARROW_BUFFER_TYPE_VARIADIC_SIZE,\n NANOARROW_BUFFER_TYPE_VIEW_OFFSET,\n NANOARROW_BUFFER_TYPE_SIZE,\n};\n\n/// \\brief The maximum number of fixed buffers in an ArrowArrayView or ArrowLayout\n/// \\ingroup nanoarrow-array-view\n#define NANOARROW_MAX_FIXED_BUFFERS 3\n\n/// \\brief An non-owning view of a string\n/// \\ingroup nanoarrow-utils\nstruct ArrowStringView {\n /// \\brief A pointer to the start of the string\n ///\n /// If size_bytes is 0, this value may be NULL.\n const char* data;\n\n /// \\brief The size of the string in bytes,\n ///\n /// (Not including the null terminator.)\n int64_t size_bytes;\n};\n\n/// \\brief Return a view of a const C string\n/// \\ingroup nanoarrow-utils\nstatic inline struct ArrowStringView ArrowCharView(const char* value);\n\nstatic inline struct ArrowStringView ArrowCharView(const char* value) {\n struct ArrowStringView out;\n\n out.data = value;\n if (value) {\n out.size_bytes = (int64_t)strlen(value);\n } else {\n out.size_bytes = 0;\n }\n\n return out;\n}\n\nunion ArrowBufferViewData {\n const void* data;\n const int8_t* as_int8;\n const uint8_t* as_uint8;\n const int16_t* as_int16;\n const uint16_t* as_uint16;\n const int32_t* as_int32;\n const uint32_t* as_uint32;\n const int64_t* as_int64;\n const uint64_t* as_uint64;\n const double* as_double;\n const float* as_float;\n const char* as_char;\n const union ArrowBinaryView* as_binary_view;\n};\n\n/// \\brief An non-owning view of a buffer\n/// \\ingroup nanoarrow-utils\nstruct ArrowBufferView {\n /// \\brief A pointer to the start of the buffer\n ///\n /// If size_bytes is 0, this value may be NULL.\n union ArrowBufferViewData data;\n\n /// \\brief The size of the buffer in bytes\n int64_t size_bytes;\n};\n\n/// \\brief Array buffer allocation and deallocation\n/// \\ingroup nanoarrow-buffer\n///\n/// Container for allocate, reallocate, and free methods that can be used\n/// to customize allocation and deallocation of buffers when constructing\n/// an ArrowArray.\nstruct ArrowBufferAllocator {\n /// \\brief Reallocate a buffer or return NULL if it cannot be reallocated\n uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,\n int64_t old_size, int64_t new_size);\n\n /// \\brief Deallocate a buffer allocated by this allocator\n void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size);\n\n /// \\brief Opaque data specific to the allocator\n void* private_data;\n};\n\ntypedef void (*ArrowBufferDeallocatorCallback)(struct ArrowBufferAllocator* allocator,\n uint8_t* ptr, int64_t size);\n\n/// \\brief An owning mutable view of a buffer\n/// \\ingroup nanoarrow-buffer\nstruct ArrowBuffer {\n /// \\brief A pointer to the start of the buffer\n ///\n /// If capacity_bytes is 0, this value may be NULL.\n uint8_t* data;\n\n /// \\brief The size of the buffer in bytes\n int64_t size_bytes;\n\n /// \\brief The capacity of the buffer in bytes\n int64_t capacity_bytes;\n\n /// \\brief The allocator that will be used to reallocate and/or free the buffer\n struct ArrowBufferAllocator allocator;\n};\n\n/// \\brief An owning mutable view of a bitmap\n/// \\ingroup nanoarrow-bitmap\nstruct ArrowBitmap {\n /// \\brief An ArrowBuffer to hold the allocated memory\n struct ArrowBuffer buffer;\n\n /// \\brief The number of bits that have been appended to the bitmap\n int64_t size_bits;\n};\n\n/// \\brief A description of an arrangement of buffers\n/// \\ingroup nanoarrow-utils\n///\n/// Contains the minimum amount of information required to\n/// calculate the size of each buffer in an ArrowArray knowing only\n/// the length and offset of the array.\nstruct ArrowLayout {\n /// \\brief The function of each buffer\n enum ArrowBufferType buffer_type[NANOARROW_MAX_FIXED_BUFFERS];\n\n /// \\brief The data type of each buffer\n enum ArrowType buffer_data_type[NANOARROW_MAX_FIXED_BUFFERS];\n\n /// \\brief The size of an element each buffer or 0 if this size is variable or unknown\n int64_t element_size_bits[NANOARROW_MAX_FIXED_BUFFERS];\n\n /// \\brief The number of elements in the child array per element in this array for a\n /// fixed-size list\n int64_t child_size_elements;\n};\n\n/// \\brief A non-owning view of an ArrowArray\n/// \\ingroup nanoarrow-array-view\n///\n/// This data structure provides access to the values contained within\n/// an ArrowArray with fields provided in a more readily-extractible\n/// form. You can re-use an ArrowArrayView for multiple ArrowArrays\n/// with the same storage type, use it to represent a hypothetical\n/// ArrowArray that does not exist yet, or use it to validate the buffers\n/// of a future ArrowArray.\nstruct ArrowArrayView {\n /// \\brief The underlying ArrowArray or NULL if it has not been set or\n /// if the buffers in this ArrowArrayView are not backed by an ArrowArray.\n const struct ArrowArray* array;\n\n /// \\brief The number of elements from the physical start of the buffers.\n int64_t offset;\n\n /// \\brief The number of elements in this view.\n int64_t length;\n\n /// \\brief A cached null count or -1 to indicate that this value is unknown.\n int64_t null_count;\n\n /// \\brief The type used to store values in this array\n ///\n /// This type represents only the minimum required information to\n /// extract values from the array buffers (e.g., for a Date32 array,\n /// this value will be NANOARROW_TYPE_INT32). For dictionary-encoded\n /// arrays, this will be the index type.\n enum ArrowType storage_type;\n\n /// \\brief The buffer types, strides, and sizes of this Array's buffers\n struct ArrowLayout layout;\n\n /// \\brief This Array's buffers as ArrowBufferView objects\n struct ArrowBufferView buffer_views[NANOARROW_MAX_FIXED_BUFFERS];\n\n /// \\brief The number of children of this view\n int64_t n_children;\n\n /// \\brief Pointers to views of this array's children\n struct ArrowArrayView** children;\n\n /// \\brief Pointer to a view of this array's dictionary\n struct ArrowArrayView* dictionary;\n\n /// \\brief Union type id to child index mapping\n ///\n /// If storage_type is a union type, a 256-byte ArrowMalloc()ed buffer\n /// such that child_index == union_type_id_map[type_id] and\n /// type_id == union_type_id_map[128 + child_index]. This value may be\n /// NULL in the case where child_id == type_id.\n int8_t* union_type_id_map;\n\n /// \\brief Number of variadic buffers\n int32_t n_variadic_buffers;\n\n /// \\brief Pointers to variadic buffers of binary/string_view arrays\n const void** variadic_buffers;\n\n /// \\brief Size of each variadic buffer\n int64_t* variadic_buffer_sizes;\n};\n\n// Used as the private data member for ArrowArrays allocated here and accessed\n// internally within inline ArrowArray* helpers.\nstruct ArrowArrayPrivateData {\n // Holder for the validity buffer (or first buffer for union types, which are\n // the only type whose first buffer is not a valdiity buffer)\n struct ArrowBitmap bitmap;\n\n // Holder for additional buffers as required\n struct ArrowBuffer buffers[NANOARROW_MAX_FIXED_BUFFERS - 1];\n\n // The array of pointers to buffers. This must be updated after a sequence\n // of appends to synchronize its values with the actual buffer addresses\n // (which may have been reallocated during that time)\n const void** buffer_data;\n\n // The storage data type, or NANOARROW_TYPE_UNINITIALIZED if unknown\n enum ArrowType storage_type;\n\n // The buffer arrangement for the storage type\n struct ArrowLayout layout;\n\n // Flag to indicate if there are non-sequence union type ids.\n // In the future this could be replaced with a type id<->child mapping\n // to support constructing unions in append mode where type_id != child_index\n int8_t union_type_id_is_child_index;\n\n // Number of variadic buffers for binary view types\n int32_t n_variadic_buffers;\n\n // Variadic buffers for binary view types\n struct ArrowBuffer* variadic_buffers;\n\n // The current offset used to build list views\n int64_t list_view_offset;\n};\n\n/// \\brief A representation of an interval.\n/// \\ingroup nanoarrow-utils\nstruct ArrowInterval {\n /// \\brief The type of interval being used\n enum ArrowType type;\n /// \\brief The number of months represented by the interval\n int32_t months;\n /// \\brief The number of days represented by the interval\n int32_t days;\n /// \\brief The number of ms represented by the interval\n int32_t ms;\n /// \\brief The number of ns represented by the interval\n int64_t ns;\n};\n\n/// \\brief Zero initialize an Interval with a given unit\n/// \\ingroup nanoarrow-utils\nstatic inline void ArrowIntervalInit(struct ArrowInterval* interval,\n enum ArrowType type) {\n memset(interval, 0, sizeof(struct ArrowInterval));\n interval->type = type;\n}\n\n/// \\brief A representation of a fixed-precision decimal number\n/// \\ingroup nanoarrow-utils\n///\n/// This structure should be initialized with ArrowDecimalInit() once and\n/// values set using ArrowDecimalSetInt(), ArrowDecimalSetBytes128(),\n/// or ArrowDecimalSetBytes256().\nstruct ArrowDecimal {\n /// \\brief An array of 64-bit integers of n_words length defined in native-endian order.\n /// For a 32-bit decimal value, index 0 will be a 32-bit integer value.\n uint64_t words[4];\n\n /// \\brief The number of significant digits this decimal number can represent\n int32_t precision;\n\n /// \\brief The number of digits after the decimal point. This can be negative.\n int32_t scale;\n\n /// \\brief The number of 64-bit words in the words array. For the special case of a\n /// 32-bit decimal value, this will be 0.\n int n_words;\n\n /// \\brief Cached value used by the implementation\n int high_word_index;\n\n /// \\brief Cached value used by the implementation\n int low_word_index;\n};\n\n/// \\brief Initialize a decimal with a given set of type parameters\n/// \\ingroup nanoarrow-utils\nstatic inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwidth,\n int32_t precision, int32_t scale) {\n memset(decimal->words, 0, sizeof(decimal->words));\n decimal->precision = precision;\n decimal->scale = scale;\n // n_words will be 0 for bitwidth == 32\n decimal->n_words = (int)(bitwidth / 8 / sizeof(uint64_t));\n\n if (_ArrowIsLittleEndian()) {\n decimal->low_word_index = 0;\n decimal->high_word_index = decimal->n_words > 0 ? decimal->n_words - 1 : 0;\n } else {\n decimal->low_word_index = decimal->n_words > 0 ? decimal->n_words - 1 : 0;\n decimal->high_word_index = 0;\n }\n}\n\n/// \\brief Get a signed integer value of a sufficiently small ArrowDecimal\n///\n/// This does not check if the decimal's precision sufficiently small to fit\n/// within the signed 64-bit integer range (A precision less than or equal\n/// to 18 is sufficiently small).\nstatic inline int64_t ArrowDecimalGetIntUnsafe(const struct ArrowDecimal* decimal) {\n if (decimal->n_words == 0) {\n int32_t value;\n memcpy(&value, decimal->words, sizeof(int32_t));\n return value;\n }\n\n return (int64_t)decimal->words[decimal->low_word_index];\n}\n\n/// \\brief Copy the bytes of this decimal into a sufficiently large buffer\n/// \\ingroup nanoarrow-utils\nstatic inline void ArrowDecimalGetBytes(const struct ArrowDecimal* decimal,\n uint8_t* out) {\n if (decimal->n_words == 0) {\n memcpy(out, decimal->words, sizeof(int32_t));\n } else {\n memcpy(out, decimal->words, decimal->n_words * sizeof(uint64_t));\n }\n}\n\n/// \\brief Returns 1 if the value represented by decimal is >= 0 or -1 otherwise\n/// \\ingroup nanoarrow-utils\nstatic inline int64_t ArrowDecimalSign(const struct ArrowDecimal* decimal) {\n if (decimal->n_words == 0) {\n return ArrowDecimalGetIntUnsafe(decimal) >= 0 ? 1 : -1;\n } else {\n return 1 | ((int64_t)(decimal->words[decimal->high_word_index]) >> 63);\n }\n}\n\n/// \\brief Sets the integer value of this decimal\n/// \\ingroup nanoarrow-utils\nstatic inline void ArrowDecimalSetInt(struct ArrowDecimal* decimal, int64_t value) {\n if (decimal->n_words == 0) {\n int32_t value32 = (int32_t)value;\n memcpy(decimal->words, &value32, sizeof(int32_t));\n return;\n }\n\n if (value < 0) {\n memset(decimal->words, 0xff, decimal->n_words * sizeof(uint64_t));\n } else {\n memset(decimal->words, 0, decimal->n_words * sizeof(uint64_t));\n }\n\n decimal->words[decimal->low_word_index] = value;\n}\n\n/// \\brief Negate the value of this decimal in place\n/// \\ingroup nanoarrow-utils\nstatic inline void ArrowDecimalNegate(struct ArrowDecimal* decimal) {\n if (decimal->n_words == 0) {\n int32_t value;\n memcpy(&value, decimal->words, sizeof(int32_t));\n value = -value;\n memcpy(decimal->words, &value, sizeof(int32_t));\n return;\n }\n\n uint64_t carry = 1;\n\n if (decimal->low_word_index == 0) {\n for (int i = 0; i < decimal->n_words; i++) {\n uint64_t elem = decimal->words[i];\n elem = ~elem + carry;\n carry &= (elem == 0);\n decimal->words[i] = elem;\n }\n } else {\n for (int i = decimal->low_word_index; i >= 0; i--) {\n uint64_t elem = decimal->words[i];\n elem = ~elem + carry;\n carry &= (elem == 0);\n decimal->words[i] = elem;\n }\n }\n}\n\n/// \\brief Copy bytes from a buffer into this decimal\n/// \\ingroup nanoarrow-utils\nstatic inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal,\n const uint8_t* value) {\n if (decimal->n_words == 0) {\n memcpy(decimal->words, value, sizeof(int32_t));\n } else {\n memcpy(decimal->words, value, decimal->n_words * sizeof(uint64_t));\n }\n}\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#ifndef NANOARROW_H_INCLUDED\n#define NANOARROW_H_INCLUDED\n\n#include \n#include \n#include \n\n\n\n// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will set this\n// define in nanoarrow_config.h. If not, you can optionally #define NANOARROW_NAMESPACE\n// MyNamespace here.\n\n// This section remaps the non-prefixed symbols to the prefixed symbols so that\n// code written against this build can be used independent of the value of\n// NANOARROW_NAMESPACE.\n#ifdef NANOARROW_NAMESPACE\n#define NANOARROW_CAT(A, B) A##B\n#define NANOARROW_SYMBOL(A, B) NANOARROW_CAT(A, B)\n\n#define ArrowNanoarrowVersion NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion)\n#define ArrowNanoarrowVersionInt \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersionInt)\n#define ArrowMalloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMalloc)\n#define ArrowRealloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowRealloc)\n#define ArrowFree NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowFree)\n#define ArrowBufferAllocatorDefault \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferAllocatorDefault)\n#define ArrowBufferDeallocator \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferDeallocator)\n#define ArrowErrorSet NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowErrorSet)\n#define ArrowLayoutInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowLayoutInit)\n#define ArrowDecimalSetDigits NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits)\n#define ArrowDecimalAppendDigitsToBuffer \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendDigitsToBuffer)\n#define ArrowDecimalAppendStringToBuffer \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendStringToBuffer)\n#define ArrowSchemaInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInit)\n#define ArrowSchemaInitFromType \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInitFromType)\n#define ArrowSchemaSetType NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType)\n#define ArrowSchemaSetTypeStruct \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeStruct)\n#define ArrowSchemaSetTypeFixedSize \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeFixedSize)\n#define ArrowSchemaSetTypeDecimal \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDecimal)\n#define ArrowSchemaSetTypeRunEndEncoded \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeRunEndEncoded)\n#define ArrowSchemaSetTypeDateTime \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDateTime)\n#define ArrowSchemaSetTypeUnion \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeUnion)\n#define ArrowSchemaDeepCopy NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy)\n#define ArrowSchemaSetFormat NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat)\n#define ArrowSchemaSetName NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName)\n#define ArrowSchemaSetMetadata \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetMetadata)\n#define ArrowSchemaAllocateChildren \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaAllocateChildren)\n#define ArrowSchemaAllocateDictionary \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaAllocateDictionary)\n#define ArrowMetadataReaderInit \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderInit)\n#define ArrowMetadataReaderRead \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderRead)\n#define ArrowMetadataSizeOf NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf)\n#define ArrowMetadataHasKey NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey)\n#define ArrowMetadataGetValue NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue)\n#define ArrowMetadataBuilderInit \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderInit)\n#define ArrowMetadataBuilderAppend \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderAppend)\n#define ArrowMetadataBuilderSet \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderSet)\n#define ArrowMetadataBuilderRemove \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderRemove)\n#define ArrowSchemaViewInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit)\n#define ArrowSchemaToString NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString)\n#define ArrowArrayInitFromType \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromType)\n#define ArrowArrayInitFromSchema \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromSchema)\n#define ArrowArrayInitFromArrayView \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromArrayView)\n#define ArrowArrayInitFromArrayView \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromArrayView)\n#define ArrowArrayAllocateChildren \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateChildren)\n#define ArrowArrayAllocateDictionary \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateDictionary)\n#define ArrowArraySetValidityBitmap \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap)\n#define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer)\n#define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve)\n#define ArrowArrayFinishBuilding \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding)\n#define ArrowArrayFinishBuildingDefault \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuildingDefault)\n#define ArrowArrayViewInitFromType \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewInitFromType)\n#define ArrowArrayViewInitFromSchema \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewInitFromSchema)\n#define ArrowArrayViewAllocateChildren \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewAllocateChildren)\n#define ArrowArrayViewAllocateDictionary \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewAllocateDictionary)\n#define ArrowArrayViewSetLength \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetLength)\n#define ArrowArrayViewSetArray \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArray)\n#define ArrowArrayViewSetArrayMinimal \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArrayMinimal)\n#define ArrowArrayViewValidate \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewValidate)\n#define ArrowArrayViewCompare NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewCompare)\n#define ArrowArrayViewReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset)\n#define ArrowBasicArrayStreamInit \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamInit)\n#define ArrowBasicArrayStreamSetArray \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamSetArray)\n#define ArrowBasicArrayStreamValidate \\\n NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamValidate)\n\n#endif\n\n#if (defined _WIN32 || defined __CYGWIN__) && defined(NANOARROW_BUILD_DLL)\n#if defined(NANOARROW_EXPORT_DLL)\n#define NANOARROW_DLL __declspec(dllexport)\n#else\n#define NANOARROW_DLL __declspec(dllimport)\n#endif // defined(NANOARROW_EXPORT_DLL)\n#elif !defined(NANOARROW_DLL)\n#if defined(__GNUC__) && __GNUC__ >= 4\n#define NANOARROW_DLL __attribute__((visibility(\"default\")))\n#else\n#define NANOARROW_DLL\n#endif // __GNUC__ >= 4\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/// \\defgroup nanoarrow Nanoarrow C library\n///\n/// Except where noted, objects are not thread-safe and clients should\n/// take care to serialize accesses to methods.\n///\n/// Because this library is intended to be vendored, it provides full type\n/// definitions and encourages clients to stack or statically allocate\n/// where convenient.\n\n/// \\defgroup nanoarrow-malloc Memory management\n///\n/// Non-buffer members of a struct ArrowSchema and struct ArrowArray\n/// must be allocated using ArrowMalloc() or ArrowRealloc() and freed\n/// using ArrowFree() for schemas and arrays allocated here. Buffer members\n/// are allocated using an ArrowBufferAllocator.\n///\n/// @{\n\n/// \\brief Allocate like malloc()\nNANOARROW_DLL void* ArrowMalloc(int64_t size);\n\n/// \\brief Reallocate like realloc()\nNANOARROW_DLL void* ArrowRealloc(void* ptr, int64_t size);\n\n/// \\brief Free a pointer allocated using ArrowMalloc() or ArrowRealloc().\nNANOARROW_DLL void ArrowFree(void* ptr);\n\n/// \\brief Return the default allocator\n///\n/// The default allocator uses ArrowMalloc(), ArrowRealloc(), and\n/// ArrowFree().\nNANOARROW_DLL struct ArrowBufferAllocator ArrowBufferAllocatorDefault(void);\n\n/// \\brief Create a custom deallocator\n///\n/// Creates a buffer allocator with only a free method that can be used to\n/// attach a custom deallocator to an ArrowBuffer. This may be used to\n/// avoid copying an existing buffer that was not allocated using the\n/// infrastructure provided here (e.g., by an R or Python object).\nNANOARROW_DLL struct ArrowBufferAllocator ArrowBufferDeallocator(\n ArrowBufferDeallocatorCallback, void* private_data);\n\n/// @}\n\n/// \\brief Move the contents of an src ArrowSchema into dst and set src->release to NULL\n/// \\ingroup nanoarrow-arrow-cdata\nstatic inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst);\n\n/// \\brief Call the release callback of an ArrowSchema\n/// \\ingroup nanoarrow-arrow-cdata\nstatic inline void ArrowSchemaRelease(struct ArrowSchema* schema);\n\n/// \\brief Move the contents of an src ArrowArray into dst and set src->release to NULL\n/// \\ingroup nanoarrow-arrow-cdata\nstatic inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst);\n\n/// \\brief Call the release callback of an ArrowArray\nstatic inline void ArrowArrayRelease(struct ArrowArray* array);\n\n/// \\brief Move the contents of an src ArrowArrayStream into dst and set src->release to\n/// NULL \\ingroup nanoarrow-arrow-cdata\nstatic inline void ArrowArrayStreamMove(struct ArrowArrayStream* src,\n struct ArrowArrayStream* dst);\n\n/// \\brief Call the get_schema callback of an ArrowArrayStream\n/// \\ingroup nanoarrow-arrow-cdata\n///\n/// Unlike the get_schema callback, this wrapper checks the return code\n/// and propagates the error reported by get_last_error into error. This\n/// makes it significantly less verbose to iterate over array streams\n/// using NANOARROW_RETURN_NOT_OK()-style error handling.\nstatic inline ArrowErrorCode ArrowArrayStreamGetSchema(\n struct ArrowArrayStream* array_stream, struct ArrowSchema* out,\n struct ArrowError* error);\n\n/// \\brief Call the get_next callback of an ArrowArrayStream\n/// \\ingroup nanoarrow-arrow-cdata\n///\n/// Unlike the get_next callback, this wrapper checks the return code\n/// and propagates the error reported by get_last_error into error. This\n/// makes it significantly less verbose to iterate over array streams\n/// using NANOARROW_RETURN_NOT_OK()-style error handling.\nstatic inline ArrowErrorCode ArrowArrayStreamGetNext(\n struct ArrowArrayStream* array_stream, struct ArrowArray* out,\n struct ArrowError* error);\n\n/// \\brief Call the get_last_error callback of an ArrowArrayStream\n/// \\ingroup nanoarrow-arrow-cdata\n///\n/// Unlike the get_last_error callback, this function never returns NULL (i.e.,\n/// its result is safe to use in printf-style error formatters). Null values\n/// from the original callback are reported as\n/// \"\".\nstatic inline const char* ArrowArrayStreamGetLastError(\n struct ArrowArrayStream* array_stream);\n\n/// \\brief Call the release callback of an ArrowArrayStream\nstatic inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream);\n\n/// \\defgroup nanoarrow-errors Error handling\n///\n/// Functions generally return an errno-compatible error code; functions that\n/// need to communicate more verbose error information accept a pointer\n/// to an ArrowError. This can be stack or statically allocated. The\n/// content of the message is undefined unless an error code has been\n/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, the\n/// ArrowError pointed to by the argument will be propagated with a\n/// null-terminated error message. It is safe to pass a NULL ArrowError anywhere\n/// in the nanoarrow API.\n///\n/// Except where documented, it is generally not safe to continue after a\n/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK and\n/// NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ clients can use\n/// the helpers provided in the nanoarrow.hpp header to facilitate using C++ idioms\n/// for memory management and error propgagtion.\n///\n/// @{\n\n/// \\brief Set the contents of an error using printf syntax.\n///\n/// If error is NULL, this function does nothing and returns NANOARROW_OK.\nNANOARROW_DLL NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet(struct ArrowError* error,\n const char* fmt, ...);\n\n/// @}\n\n/// \\defgroup nanoarrow-utils Utility data structures\n///\n/// @{\n\n/// \\brief Return a version string in the form \"major.minor.patch\"\nNANOARROW_DLL const char* ArrowNanoarrowVersion(void);\n\n/// \\brief Return an integer that can be used to compare versions sequentially\nNANOARROW_DLL int ArrowNanoarrowVersionInt(void);\n\n/// \\brief Initialize a description of buffer arrangements from a storage type\nNANOARROW_DLL void ArrowLayoutInit(struct ArrowLayout* layout,\n enum ArrowType storage_type);\n\n/// \\brief Create a string view from a null-terminated string\nstatic inline struct ArrowStringView ArrowCharView(const char* value);\n\n/// \\brief Sets the integer value of an ArrowDecimal from a string\nNANOARROW_DLL ArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal,\n struct ArrowStringView value);\n\n/// \\brief Get the integer value of an ArrowDecimal as string\nNANOARROW_DLL ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(\n const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer);\n\n/// \\brief Get the decimal value of an ArrowDecimal as a string\nNANOARROW_DLL ArrowErrorCode ArrowDecimalAppendStringToBuffer(\n const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer);\n\n/// \\brief Get the half float value of a float\nstatic inline uint16_t ArrowFloatToHalfFloat(float value);\n\n/// \\brief Get the float value of a half float\nstatic inline float ArrowHalfFloatToFloat(uint16_t value);\n\n/// \\brief Resolve a chunk index from increasing int64_t offsets\n///\n/// Given a buffer of increasing int64_t offsets that begin with 0 (e.g., offset buffer\n/// of a large type, run ends of a chunked array implementation), resolve a value v\n/// where lo <= v < hi such that offsets[v] <= index < offsets[v + 1].\nstatic inline int64_t ArrowResolveChunk64(int64_t index, const int64_t* offsets,\n int64_t lo, int64_t hi);\n\n/// @}\n\n/// \\defgroup nanoarrow-schema Creating schemas\n///\n/// These functions allocate, copy, and destroy ArrowSchema structures\n///\n/// @{\n\n/// \\brief Initialize an ArrowSchema\n///\n/// Initializes the fields and release callback of schema_out. Caller\n/// is responsible for calling the schema->release callback if\n/// NANOARROW_OK is returned.\nNANOARROW_DLL void ArrowSchemaInit(struct ArrowSchema* schema);\n\n/// \\brief Initialize an ArrowSchema from an ArrowType\n///\n/// A convenience constructor for that calls ArrowSchemaInit() and\n/// ArrowSchemaSetType() for the common case of constructing an\n/// unparameterized type. The caller is responsible for calling the schema->release\n/// callback if NANOARROW_OK is returned.\nNANOARROW_DLL ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema,\n enum ArrowType type);\n\n/// \\brief Get a human-readable summary of a Schema\n///\n/// Writes a summary of an ArrowSchema to out (up to n - 1 characters)\n/// and returns the number of characters required for the output if\n/// n were sufficiently large. If recursive is non-zero, the result will\n/// also include children.\nNANOARROW_DLL int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out,\n int64_t n, char recursive);\n\n/// \\brief Set the format field of a schema from an ArrowType\n///\n/// Initializes the fields and release callback of schema_out. For\n/// NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, and\n/// NANOARROW_TYPE_MAP, the appropriate number of children are\n/// allocated, initialized, and named; however, the caller must\n/// ArrowSchemaSetType() on the preinitialized children. Schema must have been initialized\n/// using ArrowSchemaInit() or ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema,\n enum ArrowType type);\n\n/// \\brief Set the format field and initialize children of a struct schema\n///\n/// The specified number of children are initialized; however, the caller is responsible\n/// for calling ArrowSchemaSetType() and ArrowSchemaSetName() on each child.\n/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema,\n int64_t n_children);\n\n/// \\brief Set the format field of a fixed-size schema\n///\n/// Returns EINVAL for fixed_size <= 0 or for type that is not\n/// NANOARROW_TYPE_FIXED_SIZE_BINARY or NANOARROW_TYPE_FIXED_SIZE_LIST.\n/// For NANOARROW_TYPE_FIXED_SIZE_LIST, the appropriate number of children are\n/// allocated, initialized, and named; however, the caller must\n/// ArrowSchemaSetType() the first child. Schema must have been initialized using\n/// ArrowSchemaInit() or ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema,\n enum ArrowType type,\n int32_t fixed_size);\n\n/// \\brief Set the format field of a decimal schema\n///\n/// Returns EINVAL for scale <= 0 or for type that is not\n/// NANOARROW_TYPE_DECIMAL32, NANOARROW_TYPE_DECIMAL64, NANOARROW_TYPE_DECIMAL128 or\n/// NANOARROW_TYPE_DECIMAL256. Schema must have been initialized using\n/// ArrowSchemaInit() or ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema,\n enum ArrowType type,\n int32_t decimal_precision,\n int32_t decimal_scale);\n\n/// \\brief Set the format field of a run-end encoded schema\n///\n/// Returns EINVAL for run_end_type that is not\n/// NANOARROW_TYPE_INT16, NANOARROW_TYPE_INT32 or NANOARROW_TYPE_INT64.\n/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy().\n/// The caller must call `ArrowSchemaSetTypeXXX(schema->children[1])` to\n/// set the value type. Note that when building arrays using the `ArrowArrayAppendXXX()`\n/// functions, the run-end encoded array's logical length must be updated manually.\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeRunEndEncoded(struct ArrowSchema* schema,\n enum ArrowType run_end_type);\n\n/// \\brief Set the format field of a time, timestamp, or duration schema\n///\n/// Returns EINVAL for type that is not\n/// NANOARROW_TYPE_TIME32, NANOARROW_TYPE_TIME64,\n/// NANOARROW_TYPE_TIMESTAMP, or NANOARROW_TYPE_DURATION. The\n/// timezone parameter must be NULL for a non-timestamp type. Schema must have been\n/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema,\n enum ArrowType type,\n enum ArrowTimeUnit time_unit,\n const char* timezone);\n\n/// \\brief Set the format field of a union schema\n///\n/// Returns EINVAL for a type that is not NANOARROW_TYPE_DENSE_UNION\n/// or NANOARROW_TYPE_SPARSE_UNION. The specified number of children are\n/// allocated, and initialized.\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema,\n enum ArrowType type,\n int64_t n_children);\n\n/// \\brief Make a (recursive) copy of a schema\n///\n/// Allocates and copies fields of schema into schema_out.\nNANOARROW_DLL ArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema,\n struct ArrowSchema* schema_out);\n\n/// \\brief Copy format into schema->format\n///\n/// schema must have been allocated using ArrowSchemaInitFromType() or\n/// ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema,\n const char* format);\n\n/// \\brief Copy name into schema->name\n///\n/// schema must have been allocated using ArrowSchemaInitFromType() or\n/// ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema,\n const char* name);\n\n/// \\brief Copy metadata into schema->metadata\n///\n/// schema must have been allocated using ArrowSchemaInitFromType() or\n/// ArrowSchemaDeepCopy.\nNANOARROW_DLL ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema,\n const char* metadata);\n\n/// \\brief Allocate the schema->children array\n///\n/// Includes the memory for each child struct ArrowSchema.\n/// schema must have been allocated using ArrowSchemaInitFromType() or\n/// ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema,\n int64_t n_children);\n\n/// \\brief Allocate the schema->dictionary member\n///\n/// schema must have been allocated using ArrowSchemaInitFromType() or\n/// ArrowSchemaDeepCopy().\nNANOARROW_DLL ArrowErrorCode ArrowSchemaAllocateDictionary(struct ArrowSchema* schema);\n\n/// @}\n\n/// \\defgroup nanoarrow-metadata Create, read, and modify schema metadata\n///\n/// @{\n\n/// \\brief Reader for key/value pairs in schema metadata\n///\n/// The ArrowMetadataReader does not own any data and is only valid\n/// for the lifetime of the underlying metadata pointer.\nstruct ArrowMetadataReader {\n /// \\brief A metadata string from a schema->metadata field.\n const char* metadata;\n\n /// \\brief The current offset into the metadata string\n int64_t offset;\n\n /// \\brief The number of remaining keys\n int32_t remaining_keys;\n};\n\n/// \\brief Initialize an ArrowMetadataReader\nNANOARROW_DLL ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader,\n const char* metadata);\n\n/// \\brief Read the next key/value pair from an ArrowMetadataReader\nNANOARROW_DLL ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader,\n struct ArrowStringView* key_out,\n struct ArrowStringView* value_out);\n\n/// \\brief The number of bytes in in a key/value metadata string\nNANOARROW_DLL int64_t ArrowMetadataSizeOf(const char* metadata);\n\n/// \\brief Check for a key in schema metadata\nNANOARROW_DLL char ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key);\n\n/// \\brief Extract a value from schema metadata\n///\n/// If key does not exist in metadata, value_out is unmodified\nNANOARROW_DLL ArrowErrorCode ArrowMetadataGetValue(const char* metadata,\n struct ArrowStringView key,\n struct ArrowStringView* value_out);\n\n/// \\brief Initialize a builder for schema metadata from key/value pairs\n///\n/// metadata can be an existing metadata string or NULL to initialize\n/// an empty metadata string.\nNANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer,\n const char* metadata);\n\n/// \\brief Append a key/value pair to a buffer containing serialized metadata\nNANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer,\n struct ArrowStringView key,\n struct ArrowStringView value);\n\n/// \\brief Set a key/value pair to a buffer containing serialized metadata\n///\n/// Ensures that the only entry for key in the metadata is set to value.\n/// This function maintains the existing position of (the first instance of)\n/// key if present in the data.\nNANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer,\n struct ArrowStringView key,\n struct ArrowStringView value);\n\n/// \\brief Remove a key from a buffer containing serialized metadata\nNANOARROW_DLL ArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer,\n struct ArrowStringView key);\n\n/// @}\n\n/// \\defgroup nanoarrow-schema-view Reading schemas\n///\n/// @{\n\n/// \\brief A non-owning view of a parsed ArrowSchema\n///\n/// Contains more readily extractable values than a raw ArrowSchema.\n/// Clients can stack or statically allocate this structure but are\n/// encouraged to use the provided getters to ensure forward\n/// compatibility.\nstruct ArrowSchemaView {\n /// \\brief A pointer to the schema represented by this view\n const struct ArrowSchema* schema;\n\n /// \\brief The data type represented by the schema\n ///\n /// This value may be NANOARROW_TYPE_DICTIONARY if the schema has a\n /// non-null dictionary member; datetime types are valid values.\n /// This value will never be NANOARROW_TYPE_EXTENSION (see\n /// extension_name and/or extension_metadata to check for\n /// an extension type).\n enum ArrowType type;\n\n /// \\brief The storage data type represented by the schema\n ///\n /// This value will never be NANOARROW_TYPE_DICTIONARY, NANOARROW_TYPE_EXTENSION\n /// or any datetime type. This value represents only the type required to\n /// interpret the buffers in the array.\n enum ArrowType storage_type;\n\n /// \\brief The storage layout represented by the schema\n struct ArrowLayout layout;\n\n /// \\brief The extension type name if it exists\n ///\n /// If the ARROW:extension:name key is present in schema.metadata,\n /// extension_name.data will be non-NULL.\n struct ArrowStringView extension_name;\n\n /// \\brief The extension type metadata if it exists\n ///\n /// If the ARROW:extension:metadata key is present in schema.metadata,\n /// extension_metadata.data will be non-NULL.\n struct ArrowStringView extension_metadata;\n\n /// \\brief Format fixed size parameter\n ///\n /// This value is set when parsing a fixed-size binary or fixed-size\n /// list schema; this value is undefined for other types. For a\n /// fixed-size binary schema this value is in bytes; for a fixed-size\n /// list schema this value refers to the number of child elements for\n /// each element of the parent.\n int32_t fixed_size;\n\n /// \\brief Decimal bitwidth\n ///\n /// This value is set when parsing a decimal type schema;\n /// this value is undefined for other types.\n int32_t decimal_bitwidth;\n\n /// \\brief Decimal precision\n ///\n /// This value is set when parsing a decimal type schema;\n /// this value is undefined for other types.\n int32_t decimal_precision;\n\n /// \\brief Decimal scale\n ///\n /// This value is set when parsing a decimal type schema;\n /// this value is undefined for other types.\n int32_t decimal_scale;\n\n /// \\brief Format time unit parameter\n ///\n /// This value is set when parsing a date/time type. The value is\n /// undefined for other types.\n enum ArrowTimeUnit time_unit;\n\n /// \\brief Format timezone parameter\n ///\n /// This value is set when parsing a timestamp type and represents\n /// the timezone format parameter. This value points to\n /// data within the schema and is undefined for other types.\n const char* timezone;\n\n /// \\brief Union type ids parameter\n ///\n /// This value is set when parsing a union type and represents\n /// type ids parameter. This value points to\n /// data within the schema and is undefined for other types.\n const char* union_type_ids;\n};\n\n/// \\brief Initialize an ArrowSchemaView\nNANOARROW_DLL ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view,\n const struct ArrowSchema* schema,\n struct ArrowError* error);\n\n/// @}\n\n/// \\defgroup nanoarrow-buffer Owning, growable buffers\n///\n/// @{\n\n/// \\brief Initialize an ArrowBuffer\n///\n/// Initialize a buffer with a NULL, zero-size buffer using the default\n/// buffer allocator.\nstatic inline void ArrowBufferInit(struct ArrowBuffer* buffer);\n\n/// \\brief Set a newly-initialized buffer's allocator\n///\n/// Returns EINVAL if the buffer has already been allocated.\nstatic inline ArrowErrorCode ArrowBufferSetAllocator(\n struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator);\n\n/// \\brief Reset an ArrowBuffer\n///\n/// Releases the buffer using the allocator's free method if\n/// the buffer's data member is non-null, sets the data member\n/// to NULL, and sets the buffer's size and capacity to 0.\nstatic inline void ArrowBufferReset(struct ArrowBuffer* buffer);\n\n/// \\brief Move an ArrowBuffer\n///\n/// Transfers the buffer data and lifecycle management to another\n/// address and resets buffer.\nstatic inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst);\n\n/// \\brief Grow or shrink a buffer to a given size\n///\n/// When shrinking the size of the buffer, the buffer is only reallocated\n/// if shrink_to_fit is non-zero.\nstatic inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,\n int64_t new_size_bytes,\n char shrink_to_fit);\n\n/// \\brief Ensure a buffer has at least a given additional capacity\n///\n/// Ensures that the buffer has space to append at least\n/// additional_size_bytes, overallocating when required.\nstatic inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,\n int64_t additional_size_bytes);\n\n/// \\brief Write data to buffer and increment the buffer size\n///\n/// This function does not check that buffer has the required capacity\nstatic inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,\n int64_t size_bytes);\n\n/// \\brief Write data to buffer and increment the buffer size\n///\n/// This function writes and ensures that the buffer has the required capacity,\n/// possibly by reallocating the buffer. Like ArrowBufferReserve, this will\n/// overallocate when reallocation is required.\nstatic inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer,\n const void* data, int64_t size_bytes);\n\n/// \\brief Write fill to buffer and increment the buffer size\n///\n/// This function writes the specified number of fill bytes and\n/// ensures that the buffer has the required capacity,\nstatic inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer,\n uint8_t value, int64_t size_bytes);\n\n/// \\brief Write an 8-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer,\n int8_t value);\n\n/// \\brief Write an unsigned 8-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer,\n uint8_t value);\n\n/// \\brief Write a 16-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer,\n int16_t value);\n\n/// \\brief Write an unsigned 16-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer,\n uint16_t value);\n\n/// \\brief Write a 32-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer,\n int32_t value);\n\n/// \\brief Write an unsigned 32-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer,\n uint32_t value);\n\n/// \\brief Write a 64-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer,\n int64_t value);\n\n/// \\brief Write an unsigned 64-bit integer to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer,\n uint64_t value);\n\n/// \\brief Write a double to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer,\n double value);\n\n/// \\brief Write a float to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer,\n float value);\n\n/// \\brief Write an ArrowStringView to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer,\n struct ArrowStringView value);\n\n/// \\brief Write an ArrowBufferView to a buffer\nstatic inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer,\n struct ArrowBufferView value);\n\n/// @}\n\n/// \\defgroup nanoarrow-bitmap Bitmap utilities\n///\n/// @{\n\n/// \\brief Extract a boolean value from a bitmap\nstatic inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i);\n\n/// \\brief Set a boolean value to a bitmap to true\nstatic inline void ArrowBitSet(uint8_t* bits, int64_t i);\n\n/// \\brief Set a boolean value to a bitmap to false\nstatic inline void ArrowBitClear(uint8_t* bits, int64_t i);\n\n/// \\brief Set a boolean value to a bitmap\nstatic inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t value);\n\n/// \\brief Set a boolean value to a range in a bitmap\nstatic inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length,\n uint8_t bits_are_set);\n\n/// \\brief Count true values in a bitmap\nstatic inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t i_from, int64_t i_to);\n\n/// \\brief Extract int8 boolean values from a range in a bitmap\nstatic inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset,\n int64_t length, int8_t* out);\n\n/// \\brief Extract int32 boolean values from a range in a bitmap\nstatic inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset,\n int64_t length, int32_t* out);\n\n/// \\brief Initialize an ArrowBitmap\n///\n/// Initialize the builder's buffer, empty its cache, and reset the size to zero\nstatic inline void ArrowBitmapInit(struct ArrowBitmap* bitmap);\n\n/// \\brief Move an ArrowBitmap\n///\n/// Transfers the underlying buffer data and lifecycle management to another\n/// address and resets the bitmap.\nstatic inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst);\n\n/// \\brief Ensure a bitmap builder has at least a given additional capacity\n///\n/// Ensures that the buffer has space to append at least\n/// additional_size_bits, overallocating when required.\nstatic inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap,\n int64_t additional_size_bits);\n\n/// \\brief Grow or shrink a bitmap to a given size\n///\n/// When shrinking the size of the bitmap, the bitmap is only reallocated\n/// if shrink_to_fit is non-zero.\nstatic inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap,\n int64_t new_size_bits, char shrink_to_fit);\n\n/// \\brief Reserve space for and append zero or more of the same boolean value to a bitmap\nstatic inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap,\n uint8_t bits_are_set, int64_t length);\n\n/// \\brief Append zero or more of the same boolean value to a bitmap\nstatic inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap,\n uint8_t bits_are_set, int64_t length);\n\n/// \\brief Append boolean values encoded as int8_t to a bitmap\n///\n/// The values must all be 0 or 1.\nstatic inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap,\n const int8_t* values, int64_t n_values);\n\n/// \\brief Append boolean values encoded as int32_t to a bitmap\n///\n/// The values must all be 0 or 1.\nstatic inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap,\n const int32_t* values, int64_t n_values);\n\n/// \\brief Reset a bitmap builder\n///\n/// Releases any memory held by buffer, empties the cache, and resets the size to zero\nstatic inline void ArrowBitmapReset(struct ArrowBitmap* bitmap);\n\n/// @}\n\n/// \\defgroup nanoarrow-array Creating arrays\n///\n/// These functions allocate, copy, and destroy ArrowArray structures.\n/// Once an ArrowArray has been initialized via ArrowArrayInitFromType()\n/// or ArrowArrayInitFromSchema(), the caller is responsible for releasing\n/// it using the embedded release callback.\n///\n/// @{\n\n/// \\brief Initialize the fields of an array\n///\n/// Initializes the fields and release callback of array. Caller\n/// is responsible for calling the array->release callback if\n/// NANOARROW_OK is returned.\nNANOARROW_DLL ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array,\n enum ArrowType storage_type);\n\n/// \\brief Initialize the contents of an ArrowArray from an ArrowSchema\n///\n/// Caller is responsible for calling the array->release callback if\n/// NANOARROW_OK is returned.\nNANOARROW_DLL ArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array,\n const struct ArrowSchema* schema,\n struct ArrowError* error);\n\n/// \\brief Initialize the contents of an ArrowArray from an ArrowArrayView\n///\n/// Caller is responsible for calling the array->release callback if\n/// NANOARROW_OK is returned.\nNANOARROW_DLL ArrowErrorCode ArrowArrayInitFromArrayView(\n struct ArrowArray* array, const struct ArrowArrayView* array_view,\n struct ArrowError* error);\n\n/// \\brief Allocate the array->children array\n///\n/// Includes the memory for each child struct ArrowArray,\n/// whose members are marked as released and may be subsequently initialized\n/// with ArrowArrayInitFromType() or moved from an existing ArrowArray.\n/// schema must have been allocated using ArrowArrayInitFromType().\nNANOARROW_DLL ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array,\n int64_t n_children);\n\n/// \\brief Allocate the array->dictionary member\n///\n/// Includes the memory for the struct ArrowArray, whose contents\n/// is marked as released and may be subsequently initialized\n/// with ArrowArrayInitFromType() or moved from an existing ArrowArray.\n/// array must have been allocated using ArrowArrayInitFromType()\nNANOARROW_DLL ArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array);\n\n/// \\brief Set the validity bitmap of an ArrowArray\n///\n/// array must have been allocated using ArrowArrayInitFromType()\nNANOARROW_DLL void ArrowArraySetValidityBitmap(struct ArrowArray* array,\n struct ArrowBitmap* bitmap);\n\n/// \\brief Set a buffer of an ArrowArray\n///\n/// array must have been allocated using ArrowArrayInitFromType()\nNANOARROW_DLL ArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i,\n struct ArrowBuffer* buffer);\n\n/// \\brief Add variadic buffers to a string or binary view array\n///\n/// array must have been allocated using ArrowArrayInitFromType()\nstatic inline ArrowErrorCode ArrowArrayAddVariadicBuffers(struct ArrowArray* array,\n int32_t n_buffers);\n\n/// \\brief Get the validity bitmap of an ArrowArray\n///\n/// array must have been allocated using ArrowArrayInitFromType()\nstatic inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array);\n\n/// \\brief Get a buffer of an ArrowArray\n///\n/// array must have been allocated using ArrowArrayInitFromType()\nstatic inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i);\n\n/// \\brief Start element-wise appending to an ArrowArray\n///\n/// Initializes any values needed to use ArrowArrayAppend*() functions.\n/// All element-wise appenders append by value and return EINVAL if the exact value\n/// cannot be represented by the underlying storage type.\n/// array must have been allocated using ArrowArrayInitFromType()\nstatic inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array);\n\n/// \\brief Reserve space for future appends\n///\n/// For buffer sizes that can be calculated (i.e., not string data buffers or\n/// child array sizes for non-fixed-size arrays), recursively reserve space for\n/// additional elements. This is useful for reducing the number of reallocations\n/// that occur using the item-wise appenders.\nNANOARROW_DLL ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array,\n int64_t additional_size_elements);\n\n/// \\brief Append a null value to an array\nstatic inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n);\n\n/// \\brief Append an empty, non-null value to an array\nstatic inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n);\n\n/// \\brief Append a signed integer value to an array\n///\n/// Returns NANOARROW_OK if value can be exactly represented by\n/// the underlying storage type or EINVAL otherwise (e.g., value\n/// is outside the valid array range).\nstatic inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, int64_t value);\n\n/// \\brief Append an unsigned integer value to an array\n///\n/// Returns NANOARROW_OK if value can be exactly represented by\n/// the underlying storage type or EINVAL otherwise (e.g., value\n/// is outside the valid array range).\nstatic inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array,\n uint64_t value);\n\n/// \\brief Append a double value to an array\n///\n/// Returns NANOARROW_OK if value can be exactly represented by\n/// the underlying storage type or EINVAL otherwise (e.g., value\n/// is outside the valid array range or there is an attempt to append\n/// a non-integer to an array with an integer storage type).\nstatic inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array,\n double value);\n\n/// \\brief Append a string of bytes to an array\n///\n/// Returns NANOARROW_OK if value can be exactly represented by\n/// the underlying storage type, EOVERFLOW if appending value would overflow\n/// the offset type (e.g., if the data buffer would be larger than 2 GB for a\n/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a\n/// binary, string, large binary, large string, or fixed-size binary array, or value is\n/// the wrong size for a fixed-size binary array).\nstatic inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array,\n struct ArrowBufferView value);\n\n/// \\brief Append a string value to an array\n///\n/// Returns NANOARROW_OK if value can be exactly represented by\n/// the underlying storage type, EOVERFLOW if appending value would overflow\n/// the offset type (e.g., if the data buffer would be larger than 2 GB for a\n/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a\n/// string or large string array).\nstatic inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array,\n struct ArrowStringView value);\n\n/// \\brief Append a Interval to an array\n///\n/// Returns NANOARROW_OK if value can be exactly represented by\n/// the underlying storage type or EINVAL otherwise.\nstatic inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array,\n const struct ArrowInterval* value);\n\n/// \\brief Append a decimal value to an array\n///\n/// Returns NANOARROW_OK if array is a decimal array with the appropriate\n/// bitwidth or EINVAL otherwise.\nstatic inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array,\n const struct ArrowDecimal* value);\n\n/// \\brief Finish a nested array element\n///\n/// Appends a non-null element to the array based on the first child's current\n/// length. Returns NANOARROW_OK if the item was successfully added, EOVERFLOW\n/// if the child of a list or map array would exceed INT_MAX elements, or EINVAL\n/// if the underlying storage type is not a struct, list, large list, or fixed-size\n/// list, or if there was an attempt to add a struct or fixed-size list element where the\n/// length of the child array(s) did not match the expected length.\nstatic inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array);\n\n/// \\brief Finish a union array element\n///\n/// Appends an element to the union type ids buffer and increments array->length.\n/// For sparse unions, up to one element is added to non type-id children. Returns\n/// EINVAL if the underlying storage type is not a union, if type_id is not valid,\n/// or if child sizes after appending are inconsistent.\nstatic inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array,\n int8_t type_id);\n\n/// \\brief Shrink buffer capacity to the size required\n///\n/// Also applies shrinking to any child arrays. array must have been allocated using\n/// ArrowArrayInitFromType\nstatic inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array);\n\n/// \\brief Finish building an ArrowArray\n///\n/// Flushes any pointers from internal buffers that may have been reallocated\n/// into array->buffers and checks the actual size of the buffers\n/// against the expected size based on the final length.\n/// array must have been allocated using ArrowArrayInitFromType()\nNANOARROW_DLL ArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array,\n struct ArrowError* error);\n\n/// \\brief Finish building an ArrowArray with explicit validation\n///\n/// Finish building with an explicit validation level. This could perform less validation\n/// (i.e. NANOARROW_VALIDATION_LEVEL_NONE or NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU\n/// buffer data access is not possible or more validation (i.e.,\n/// NANOARROW_VALIDATION_LEVEL_FULL) if buffer content was obtained from an untrusted or\n/// corruptible source.\nNANOARROW_DLL ArrowErrorCode ArrowArrayFinishBuilding(\n struct ArrowArray* array, enum ArrowValidationLevel validation_level,\n struct ArrowError* error);\n\n/// @}\n\n/// \\defgroup nanoarrow-array-view Reading arrays\n///\n/// These functions read and validate the contents ArrowArray structures.\n///\n/// @{\n\n/// \\brief Initialize the contents of an ArrowArrayView\nNANOARROW_DLL void ArrowArrayViewInitFromType(struct ArrowArrayView* array_view,\n enum ArrowType storage_type);\n\n/// \\brief Move an ArrowArrayView\n///\n/// Transfers the ArrowArrayView data and lifecycle management to another\n/// address and resets the contents of src.\nstatic inline void ArrowArrayViewMove(struct ArrowArrayView* src,\n struct ArrowArrayView* dst);\n\n/// \\brief Initialize the contents of an ArrowArrayView from an ArrowSchema\nNANOARROW_DLL ArrowErrorCode\nArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view,\n const struct ArrowSchema* schema, struct ArrowError* error);\n\n/// \\brief Allocate the array_view->children array\n///\n/// Includes the memory for each child struct ArrowArrayView\nNANOARROW_DLL ArrowErrorCode\nArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view, int64_t n_children);\n\n/// \\brief Allocate array_view->dictionary\nNANOARROW_DLL ArrowErrorCode\nArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view);\n\n/// \\brief Set data-independent buffer sizes from length\nNANOARROW_DLL void ArrowArrayViewSetLength(struct ArrowArrayView* array_view,\n int64_t length);\n\n/// \\brief Set buffer sizes and data pointers from an ArrowArray\nNANOARROW_DLL ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view,\n const struct ArrowArray* array,\n struct ArrowError* error);\n\n/// \\brief Set buffer sizes and data pointers from an ArrowArray except for those\n/// that require dereferencing buffer content.\nNANOARROW_DLL ArrowErrorCode\nArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view,\n const struct ArrowArray* array, struct ArrowError* error);\n\n/// \\brief Get the number of buffers\n///\n/// The number of buffers referred to by this ArrowArrayView. In may cases this can also\n/// be calculated from the ArrowLayout member of the ArrowArrayView or ArrowSchemaView;\n/// however, for binary view and string view types, the number of total buffers depends on\n/// the number of variadic buffers.\nstatic inline int64_t ArrowArrayViewGetNumBuffers(struct ArrowArrayView* array_view);\n\n/// \\brief Get a view of a specific buffer from an ArrowArrayView\n///\n/// This is the ArrowArrayView equivalent of ArrowArray::buffers[i] that includes\n/// size information (if known).\nstatic inline struct ArrowBufferView ArrowArrayViewGetBufferView(\n struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get the function of a specific buffer in an ArrowArrayView\n///\n/// In may cases this can also be obtained from the ArrowLayout member of the\n/// ArrowArrayView or ArrowSchemaView; however, for binary view and string view types,\n/// the function of each buffer may be different between two arrays of the same type\n/// depending on the number of variadic buffers.\nstatic inline enum ArrowBufferType ArrowArrayViewGetBufferType(\n struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get the data type of a specific buffer in an ArrowArrayView\n///\n/// In may cases this can also be obtained from the ArrowLayout member of the\n/// ArrowArrayView or ArrowSchemaView; however, for binary view and string view types,\n/// the data type of each buffer may be different between two arrays of the same type\n/// depending on the number of variadic buffers.\nstatic inline enum ArrowType ArrowArrayViewGetBufferDataType(\n struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get the element size (in bits) of a specific buffer in an ArrowArrayView\n///\n/// In may cases this can also be obtained from the ArrowLayout member of the\n/// ArrowArrayView or ArrowSchemaView; however, for binary view and string view types,\n/// the element width of each buffer may be different between two arrays of the same type\n/// depending on the number of variadic buffers.\nstatic inline int64_t ArrowArrayViewGetBufferElementSizeBits(\n struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Performs checks on the content of an ArrowArrayView\n///\n/// If using ArrowArrayViewSetArray() to back array_view with an ArrowArray,\n/// the buffer sizes and some content (fist and last offset) have already\n/// been validated at the \"default\" level. If setting the buffer pointers\n/// and sizes otherwise, you may wish to perform checks at a different level. See\n/// documentation for ArrowValidationLevel for the details of checks performed\n/// at each level.\nNANOARROW_DLL ArrowErrorCode ArrowArrayViewValidate(\n struct ArrowArrayView* array_view, enum ArrowValidationLevel validation_level,\n struct ArrowError* error);\n\n/// \\brief Compare two ArrowArrayView objects for equality\n///\n/// Given two ArrowArrayView instances, place either 0 (not equal) and\n/// 1 (equal) at the address pointed to by out. If the comparison determines\n/// that actual and expected are not equal, a reason will be communicated via\n/// error if error is non-NULL.\n///\n/// Returns NANOARROW_OK if the comparison completed successfully.\nNANOARROW_DLL ArrowErrorCode ArrowArrayViewCompare(const struct ArrowArrayView* actual,\n const struct ArrowArrayView* expected,\n enum ArrowCompareLevel level, int* out,\n struct ArrowError* reason);\n\n/// \\brief Reset the contents of an ArrowArrayView and frees resources\nNANOARROW_DLL void ArrowArrayViewReset(struct ArrowArrayView* array_view);\n\n/// \\brief Check for a null element in an ArrowArrayView\nstatic inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view,\n int64_t i);\n\n/// \\brief Compute null count for an ArrowArrayView\nstatic inline int64_t ArrowArrayViewComputeNullCount(\n const struct ArrowArrayView* array_view);\n\n/// \\brief Get the type id of a union array element\nstatic inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view,\n int64_t i);\n\n/// \\brief Get the child index of a union array element\nstatic inline int8_t ArrowArrayViewUnionChildIndex(\n const struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get the index to use into the relevant union child array\nstatic inline int64_t ArrowArrayViewUnionChildOffset(\n const struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get an element in an ArrowArrayView as an integer\n///\n/// This function does not check for null values, that values are actually integers, or\n/// that values are within a valid range for an int64.\nstatic inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view,\n int64_t i);\n\n/// \\brief Get an element in an ArrowArrayView as an unsigned integer\n///\n/// This function does not check for null values, that values are actually integers, or\n/// that values are within a valid range for a uint64.\nstatic inline uint64_t ArrowArrayViewGetUIntUnsafe(\n const struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get an element in an ArrowArrayView as a double\n///\n/// This function does not check for null values, or\n/// that values are within a valid range for a double.\nstatic inline double ArrowArrayViewGetDoubleUnsafe(\n const struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get an element in an ArrowArrayView as an ArrowStringView\n///\n/// This function does not check for null values.\nstatic inline struct ArrowStringView ArrowArrayViewGetStringUnsafe(\n const struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get an element in an ArrowArrayView as an ArrowBufferView\n///\n/// This function does not check for null values.\nstatic inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe(\n const struct ArrowArrayView* array_view, int64_t i);\n\n/// \\brief Get an element in an ArrowArrayView as an ArrowDecimal\n///\n/// This function does not check for null values. The out parameter must\n/// be initialized with ArrowDecimalInit() with the proper parameters for this\n/// type before calling this for the first time.\nstatic inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view,\n int64_t i, struct ArrowDecimal* out);\n\n/// @}\n\n/// \\defgroup nanoarrow-basic-array-stream Basic ArrowArrayStream implementation\n///\n/// An implementation of an ArrowArrayStream based on a collection of\n/// zero or more previously-existing ArrowArray objects. Users should\n/// initialize and/or validate the contents before transferring the\n/// responsibility of the ArrowArrayStream elsewhere.\n///\n/// @{\n\n/// \\brief Initialize an ArrowArrayStream backed by this implementation\n///\n/// This function moves the ownership of schema to the array_stream. If\n/// this function returns NANOARROW_OK, the caller is responsible for\n/// releasing the ArrowArrayStream.\nNANOARROW_DLL ArrowErrorCode ArrowBasicArrayStreamInit(\n struct ArrowArrayStream* array_stream, struct ArrowSchema* schema, int64_t n_arrays);\n\n/// \\brief Set the ith ArrowArray in this ArrowArrayStream.\n///\n/// array_stream must have been initialized with ArrowBasicArrayStreamInit().\n/// This function moves the ownership of array to the array_stream. i must\n/// be greater than or equal to zero and less than the value of n_arrays passed in\n/// ArrowBasicArrayStreamInit(). Callers are not required to fill all\n/// n_arrays members (i.e., n_arrays is a maximum bound).\nNANOARROW_DLL void ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream,\n int64_t i, struct ArrowArray* array);\n\n/// \\brief Validate the contents of this ArrowArrayStream\n///\n/// array_stream must have been initialized with ArrowBasicArrayStreamInit().\n/// This function uses ArrowArrayStreamInitFromSchema() and ArrowArrayStreamSetArray()\n/// to validate the contents of the arrays.\nNANOARROW_DLL ArrowErrorCode ArrowBasicArrayStreamValidate(\n const struct ArrowArrayStream* array_stream, struct ArrowError* error);\n\n/// @}\n\n// Undefine ArrowErrorCode, which may have been defined to annotate functions that return\n// it to warn for an unused result.\n#if defined(ArrowErrorCode)\n#undef ArrowErrorCode\n#endif\n\n// Inline function definitions\n\n\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#ifndef NANOARROW_BUFFER_INLINE_H_INCLUDED\n#define NANOARROW_BUFFER_INLINE_H_INCLUDED\n\n#include \n#include \n#include \n\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// Modified from Arrow C++ (1eb46f76) cpp/src/arrow/chunk_resolver.h#L133-L162\nstatic inline int64_t ArrowResolveChunk64(int64_t index, const int64_t* offsets,\n int64_t lo, int64_t hi) {\n // Similar to std::upper_bound(), but slightly different as our offsets\n // array always starts with 0.\n int64_t n = hi - lo;\n // First iteration does not need to check for n > 1\n // (lo < hi is guaranteed by the precondition).\n NANOARROW_DCHECK(n > 1);\n do {\n const int64_t m = n >> 1;\n const int64_t mid = lo + m;\n if (index >= offsets[mid]) {\n lo = mid;\n n -= m;\n } else {\n n = m;\n }\n } while (n > 1);\n return lo;\n}\n\nstatic inline int64_t ArrowResolveChunk32(int32_t index, const int32_t* offsets,\n int32_t lo, int32_t hi) {\n // Similar to std::upper_bound(), but slightly different as our offsets\n // array always starts with 0.\n int32_t n = hi - lo;\n // First iteration does not need to check for n > 1\n // (lo < hi is guaranteed by the precondition).\n NANOARROW_DCHECK(n > 1);\n do {\n const int32_t m = n >> 1;\n const int32_t mid = lo + m;\n if (index >= offsets[mid]) {\n lo = mid;\n n -= m;\n } else {\n n = m;\n }\n } while (n > 1);\n return lo;\n}\n\nstatic inline int64_t _ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) {\n int64_t doubled_capacity = current_capacity * 2;\n if (doubled_capacity > new_capacity) {\n return doubled_capacity;\n } else {\n return new_capacity;\n }\n}\n\n// float to half float conversion, adapted from Arrow Go\n// https://github.com/apache/arrow/blob/main/go/arrow/float16/float16.go\nstatic inline uint16_t ArrowFloatToHalfFloat(float value) {\n union {\n float f;\n uint32_t b;\n } u;\n u.f = value;\n\n uint16_t sn = (uint16_t)((u.b >> 31) & 0x1);\n uint16_t exp = (u.b >> 23) & 0xff;\n int16_t res = (int16_t)(exp - 127 + 15);\n uint16_t fc = (uint16_t)(u.b >> 13) & 0x3ff;\n\n if (exp == 0) {\n res = 0;\n } else if (exp == 0xff) {\n res = 0x1f;\n } else if (res > 0x1e) {\n res = 0x1f;\n fc = 0;\n } else if (res < 0x01) {\n res = 0;\n fc = 0;\n }\n\n return (uint16_t)((sn << 15) | (uint16_t)(res << 10) | fc);\n}\n\n// half float to float conversion, adapted from Arrow Go\n// https://github.com/apache/arrow/blob/main/go/arrow/float16/float16.go\nstatic inline float ArrowHalfFloatToFloat(uint16_t value) {\n uint32_t sn = (uint32_t)((value >> 15) & 0x1);\n uint32_t exp = (value >> 10) & 0x1f;\n uint32_t res = exp + 127 - 15;\n uint32_t fc = value & 0x3ff;\n\n if (exp == 0) {\n res = 0;\n } else if (exp == 0x1f) {\n res = 0xff;\n }\n\n union {\n float f;\n uint32_t b;\n } u;\n u.b = (uint32_t)(sn << 31) | (uint32_t)(res << 23) | (uint32_t)(fc << 13);\n return u.f;\n}\n\nstatic inline void ArrowBufferInit(struct ArrowBuffer* buffer) {\n buffer->data = NULL;\n buffer->size_bytes = 0;\n buffer->capacity_bytes = 0;\n buffer->allocator = ArrowBufferAllocatorDefault();\n}\n\nstatic inline ArrowErrorCode ArrowBufferSetAllocator(\n struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator) {\n // This is not a perfect test for \"has a buffer already been allocated\"\n // but is likely to catch most cases.\n if (buffer->data == NULL) {\n buffer->allocator = allocator;\n return NANOARROW_OK;\n } else {\n return EINVAL;\n }\n}\n\nstatic inline void ArrowBufferReset(struct ArrowBuffer* buffer) {\n buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data,\n buffer->capacity_bytes);\n ArrowBufferInit(buffer);\n}\n\nstatic inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst) {\n memcpy(dst, src, sizeof(struct ArrowBuffer));\n src->data = NULL;\n ArrowBufferInit(src);\n}\n\nstatic inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,\n int64_t new_size_bytes,\n char shrink_to_fit) {\n if (new_size_bytes < 0) {\n return EINVAL;\n }\n\n int needs_reallocation = new_size_bytes > buffer->capacity_bytes ||\n (shrink_to_fit && new_size_bytes < buffer->capacity_bytes);\n\n if (needs_reallocation) {\n buffer->data = buffer->allocator.reallocate(&buffer->allocator, buffer->data,\n buffer->capacity_bytes, new_size_bytes);\n\n if (buffer->data == NULL && new_size_bytes > 0) {\n buffer->capacity_bytes = 0;\n buffer->size_bytes = 0;\n return ENOMEM;\n }\n\n buffer->capacity_bytes = new_size_bytes;\n }\n\n buffer->size_bytes = new_size_bytes;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,\n int64_t additional_size_bytes) {\n int64_t min_capacity_bytes = buffer->size_bytes + additional_size_bytes;\n if (min_capacity_bytes <= buffer->capacity_bytes) {\n return NANOARROW_OK;\n }\n\n int64_t new_capacity_bytes =\n _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes);\n buffer->data = buffer->allocator.reallocate(&buffer->allocator, buffer->data,\n buffer->capacity_bytes, new_capacity_bytes);\n\n if (buffer->data == NULL && new_capacity_bytes > 0) {\n buffer->capacity_bytes = 0;\n buffer->size_bytes = 0;\n return ENOMEM;\n }\n\n buffer->capacity_bytes = new_capacity_bytes;\n return NANOARROW_OK;\n}\n\nstatic inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,\n int64_t size_bytes) {\n if (size_bytes > 0) {\n NANOARROW_DCHECK(buffer->data != NULL);\n memcpy(buffer->data + buffer->size_bytes, data, size_bytes);\n buffer->size_bytes += size_bytes;\n }\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer,\n const void* data, int64_t size_bytes) {\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes));\n\n ArrowBufferAppendUnsafe(buffer, data, size_bytes);\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer,\n int8_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(int8_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer,\n uint8_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(uint8_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer,\n int16_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(int16_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer,\n uint16_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(uint16_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer,\n int32_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(int32_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer,\n uint32_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(uint32_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer,\n int64_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(int64_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer,\n uint64_t value) {\n return ArrowBufferAppend(buffer, &value, sizeof(uint64_t));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer,\n double value) {\n return ArrowBufferAppend(buffer, &value, sizeof(double));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer,\n float value) {\n return ArrowBufferAppend(buffer, &value, sizeof(float));\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer,\n struct ArrowStringView value) {\n return ArrowBufferAppend(buffer, value.data, value.size_bytes);\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer,\n struct ArrowBufferView value) {\n return ArrowBufferAppend(buffer, value.data.data, value.size_bytes);\n}\n\nstatic inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer,\n uint8_t value, int64_t size_bytes) {\n if (size_bytes == 0) {\n return NANOARROW_OK;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes));\n\n NANOARROW_DCHECK(buffer->data != NULL); // To help clang-tidy\n memset(buffer->data + buffer->size_bytes, value, size_bytes);\n buffer->size_bytes += size_bytes;\n\n return NANOARROW_OK;\n}\n\nstatic const uint8_t _ArrowkBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};\nstatic const uint8_t _ArrowkFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127};\nstatic const uint8_t _ArrowkPrecedingBitmask[] = {0, 1, 3, 7, 15, 31, 63, 127};\nstatic const uint8_t _ArrowkTrailingBitmask[] = {255, 254, 252, 248, 240, 224, 192, 128};\n\nstatic const uint8_t _ArrowkBytePopcount[] = {\n 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3,\n 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4,\n 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4,\n 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5,\n 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2,\n 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5,\n 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4,\n 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6,\n 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};\n\nstatic inline int64_t _ArrowRoundUpToMultipleOf8(int64_t value) {\n return (value + 7) & ~((int64_t)7);\n}\n\nstatic inline int64_t _ArrowRoundDownToMultipleOf8(int64_t value) {\n return (value / 8) * 8;\n}\n\nstatic inline int64_t _ArrowBytesForBits(int64_t bits) {\n return (bits >> 3) + ((bits & 7) != 0);\n}\n\nstatic inline void _ArrowBitsUnpackInt8(const uint8_t word, int8_t* out) {\n out[0] = (word & 0x1) != 0;\n out[1] = (word & 0x2) != 0;\n out[2] = (word & 0x4) != 0;\n out[3] = (word & 0x8) != 0;\n out[4] = (word & 0x10) != 0;\n out[5] = (word & 0x20) != 0;\n out[6] = (word & 0x40) != 0;\n out[7] = (word & 0x80) != 0;\n}\n\nstatic inline void _ArrowBitsUnpackInt32(const uint8_t word, int32_t* out) {\n out[0] = (word & 0x1) != 0;\n out[1] = (word & 0x2) != 0;\n out[2] = (word & 0x4) != 0;\n out[3] = (word & 0x8) != 0;\n out[4] = (word & 0x10) != 0;\n out[5] = (word & 0x20) != 0;\n out[6] = (word & 0x40) != 0;\n out[7] = (word & 0x80) != 0;\n}\n\nstatic inline void _ArrowBitmapPackInt8(const int8_t* values, uint8_t* out) {\n *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) |\n ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) |\n ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) |\n ((values[7] + 0x7f) & 0x80));\n}\n\nstatic inline void _ArrowBitmapPackInt32(const int32_t* values, uint8_t* out) {\n *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) |\n ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) |\n ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) |\n ((values[7] + 0x7f) & 0x80));\n}\n\nstatic inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i) {\n return (bits[i >> 3] >> (i & 0x07)) & 1;\n}\n\nstatic inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset,\n int64_t length, int8_t* out) {\n if (length == 0) {\n return;\n }\n\n const int64_t i_begin = start_offset;\n const int64_t i_end = start_offset + length;\n const int64_t i_last_valid = i_end - 1;\n\n const int64_t bytes_begin = i_begin / 8;\n const int64_t bytes_last_valid = i_last_valid / 8;\n\n if (bytes_begin == bytes_last_valid) {\n for (int i = 0; i < length; i++) {\n out[i] = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);\n }\n\n return;\n }\n\n // first byte\n for (int i = 0; i < 8 - (i_begin % 8); i++) {\n *out++ = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);\n }\n\n // middle bytes\n for (int64_t i = bytes_begin + 1; i < bytes_last_valid; i++) {\n _ArrowBitsUnpackInt8(bits[i], out);\n out += 8;\n }\n\n // last byte\n const int bits_remaining = (int)(i_end % 8 == 0 ? 8 : i_end % 8);\n for (int i = 0; i < bits_remaining; i++) {\n *out++ = ArrowBitGet(&bits[bytes_last_valid], i);\n }\n}\n\nstatic inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset,\n int64_t length, int32_t* out) {\n if (length == 0) {\n return;\n }\n\n NANOARROW_DCHECK(bits != NULL && out != NULL);\n\n const int64_t i_begin = start_offset;\n const int64_t i_end = start_offset + length;\n const int64_t i_last_valid = i_end - 1;\n\n const int64_t bytes_begin = i_begin / 8;\n const int64_t bytes_last_valid = i_last_valid / 8;\n\n if (bytes_begin == bytes_last_valid) {\n for (int i = 0; i < length; i++) {\n out[i] = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);\n }\n\n return;\n }\n\n // first byte\n for (int i = 0; i < 8 - (i_begin % 8); i++) {\n *out++ = ArrowBitGet(&bits[bytes_begin], i + i_begin % 8);\n }\n\n // middle bytes\n for (int64_t i = bytes_begin + 1; i < bytes_last_valid; i++) {\n _ArrowBitsUnpackInt32(bits[i], out);\n out += 8;\n }\n\n // last byte\n const int bits_remaining = (int)(i_end % 8 == 0 ? 8 : i_end % 8);\n for (int i = 0; i < bits_remaining; i++) {\n *out++ = ArrowBitGet(&bits[bytes_last_valid], i);\n }\n}\n\nstatic inline void ArrowBitSet(uint8_t* bits, int64_t i) {\n bits[i / 8] |= _ArrowkBitmask[i % 8];\n}\n\nstatic inline void ArrowBitClear(uint8_t* bits, int64_t i) {\n bits[i / 8] &= _ArrowkFlippedBitmask[i % 8];\n}\n\nstatic inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t bit_is_set) {\n bits[i / 8] ^= (uint8_t)(((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) &\n _ArrowkBitmask[i % 8]);\n}\n\nstatic inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length,\n uint8_t bits_are_set) {\n if (length == 0) {\n return;\n }\n\n NANOARROW_DCHECK(bits != NULL);\n\n const int64_t i_begin = start_offset;\n const int64_t i_end = start_offset + length;\n const uint8_t fill_byte = (uint8_t)(-bits_are_set);\n\n const int64_t bytes_begin = i_begin / 8;\n const int64_t bytes_end = i_end / 8 + 1;\n\n const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8];\n const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_end % 8];\n\n if (bytes_end == bytes_begin + 1) {\n // set bits within a single byte\n const uint8_t only_byte_mask =\n i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask);\n bits[bytes_begin] &= only_byte_mask;\n bits[bytes_begin] |= (uint8_t)(fill_byte & ~only_byte_mask);\n return;\n }\n\n // set/clear trailing bits of first byte\n bits[bytes_begin] &= first_byte_mask;\n bits[bytes_begin] |= (uint8_t)(fill_byte & ~first_byte_mask);\n\n if (bytes_end - bytes_begin > 2) {\n // set/clear whole bytes\n memset(bits + bytes_begin + 1, fill_byte, (size_t)(bytes_end - bytes_begin - 2));\n }\n\n if (i_end % 8 == 0) {\n return;\n }\n\n // set/clear leading bits of last byte\n bits[bytes_end - 1] &= last_byte_mask;\n bits[bytes_end - 1] |= (uint8_t)(fill_byte & ~last_byte_mask);\n}\n\nstatic inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset,\n int64_t length) {\n if (length == 0) {\n return 0;\n }\n\n NANOARROW_DCHECK(bits != NULL);\n\n const int64_t i_begin = start_offset;\n const int64_t i_end = start_offset + length;\n const int64_t i_last_valid = i_end - 1;\n\n const int64_t bytes_begin = i_begin / 8;\n const int64_t bytes_last_valid = i_last_valid / 8;\n\n if (bytes_begin == bytes_last_valid) {\n // count bits within a single byte\n const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_end % 8];\n const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_begin % 8];\n\n const uint8_t only_byte_mask =\n i_end % 8 == 0 ? last_byte_mask : (uint8_t)(first_byte_mask & last_byte_mask);\n\n const uint8_t byte_masked = bits[bytes_begin] & only_byte_mask;\n return _ArrowkBytePopcount[byte_masked];\n }\n\n const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8];\n const uint8_t last_byte_mask = i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8];\n int64_t count = 0;\n\n // first byte\n count += _ArrowkBytePopcount[bits[bytes_begin] & ~first_byte_mask];\n\n // middle bytes\n for (int64_t i = bytes_begin + 1; i < bytes_last_valid; i++) {\n count += _ArrowkBytePopcount[bits[i]];\n }\n\n // last byte\n count += _ArrowkBytePopcount[bits[bytes_last_valid] & ~last_byte_mask];\n\n return count;\n}\n\nstatic inline void ArrowBitmapInit(struct ArrowBitmap* bitmap) {\n ArrowBufferInit(&bitmap->buffer);\n bitmap->size_bits = 0;\n}\n\nstatic inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst) {\n ArrowBufferMove(&src->buffer, &dst->buffer);\n dst->size_bits = src->size_bits;\n src->size_bits = 0;\n}\n\nstatic inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap,\n int64_t additional_size_bits) {\n int64_t min_capacity_bits = bitmap->size_bits + additional_size_bits;\n int64_t min_capacity_bytes = _ArrowBytesForBits(min_capacity_bits);\n int64_t current_size_bytes = bitmap->buffer.size_bytes;\n int64_t current_capacity_bytes = bitmap->buffer.capacity_bytes;\n\n if (min_capacity_bytes <= current_capacity_bytes) {\n return NANOARROW_OK;\n }\n\n int64_t additional_capacity_bytes = min_capacity_bytes - current_size_bytes;\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(&bitmap->buffer, additional_capacity_bytes));\n\n // Zero out the last byte for deterministic output in the common case\n // of reserving a known remaining size. We should have returned above\n // if there was not at least one additional byte to allocate; however,\n // DCHECK() just to be sure.\n NANOARROW_DCHECK(bitmap->buffer.capacity_bytes > current_capacity_bytes);\n bitmap->buffer.data[bitmap->buffer.capacity_bytes - 1] = 0;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap,\n int64_t new_size_bits,\n char shrink_to_fit) {\n if (new_size_bits < 0) {\n return EINVAL;\n }\n\n int64_t new_size_bytes = _ArrowBytesForBits(new_size_bits);\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferResize(&bitmap->buffer, new_size_bytes, shrink_to_fit));\n\n bitmap->size_bits = new_size_bits;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap,\n uint8_t bits_are_set, int64_t length) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(bitmap, length));\n\n ArrowBitmapAppendUnsafe(bitmap, bits_are_set, length);\n return NANOARROW_OK;\n}\n\nstatic inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap,\n uint8_t bits_are_set, int64_t length) {\n ArrowBitsSetTo(bitmap->buffer.data, bitmap->size_bits, length, bits_are_set);\n bitmap->size_bits += length;\n bitmap->buffer.size_bytes = _ArrowBytesForBits(bitmap->size_bits);\n}\n\nstatic inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap,\n const int8_t* values, int64_t n_values) {\n if (n_values == 0) {\n return;\n }\n\n NANOARROW_DCHECK(bitmap->buffer.data != NULL);\n NANOARROW_DCHECK(values != NULL);\n\n const int8_t* values_cursor = values;\n int64_t n_remaining = n_values;\n int64_t out_i_cursor = bitmap->size_bits;\n uint8_t* out_cursor = bitmap->buffer.data + bitmap->size_bits / 8;\n\n // First byte\n if ((out_i_cursor % 8) != 0) {\n int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor;\n for (int i = 0; i < n_partial_bits; i++) {\n ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]);\n }\n\n out_cursor++;\n values_cursor += n_partial_bits;\n n_remaining -= n_partial_bits;\n }\n\n // Middle bytes\n int64_t n_full_bytes = n_remaining / 8;\n for (int64_t i = 0; i < n_full_bytes; i++) {\n _ArrowBitmapPackInt8(values_cursor, out_cursor);\n values_cursor += 8;\n out_cursor++;\n }\n\n // Last byte\n out_i_cursor += n_full_bytes * 8;\n n_remaining -= n_full_bytes * 8;\n if (n_remaining > 0) {\n // Zero out the last byte\n *out_cursor = 0x00;\n for (int i = 0; i < n_remaining; i++) {\n ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values_cursor[i]);\n }\n out_cursor++;\n }\n\n bitmap->size_bits += n_values;\n bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data;\n}\n\nstatic inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap,\n const int32_t* values, int64_t n_values) {\n if (n_values == 0) {\n return;\n }\n\n NANOARROW_DCHECK(bitmap->buffer.data != NULL);\n NANOARROW_DCHECK(values != NULL);\n\n const int32_t* values_cursor = values;\n int64_t n_remaining = n_values;\n int64_t out_i_cursor = bitmap->size_bits;\n uint8_t* out_cursor = bitmap->buffer.data + bitmap->size_bits / 8;\n\n // First byte\n if ((out_i_cursor % 8) != 0) {\n int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor;\n for (int i = 0; i < n_partial_bits; i++) {\n ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values[i]);\n }\n\n out_cursor++;\n values_cursor += n_partial_bits;\n n_remaining -= n_partial_bits;\n }\n\n // Middle bytes\n int64_t n_full_bytes = n_remaining / 8;\n for (int64_t i = 0; i < n_full_bytes; i++) {\n _ArrowBitmapPackInt32(values_cursor, out_cursor);\n values_cursor += 8;\n out_cursor++;\n }\n\n // Last byte\n out_i_cursor += n_full_bytes * 8;\n n_remaining -= n_full_bytes * 8;\n if (n_remaining > 0) {\n // Zero out the last byte\n *out_cursor = 0x00;\n for (int i = 0; i < n_remaining; i++) {\n ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]);\n }\n out_cursor++;\n }\n\n bitmap->size_bits += n_values;\n bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data;\n}\n\nstatic inline void ArrowBitmapReset(struct ArrowBitmap* bitmap) {\n ArrowBufferReset(&bitmap->buffer);\n bitmap->size_bits = 0;\n}\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership. The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License. You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied. See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n#ifndef NANOARROW_ARRAY_INLINE_H_INCLUDED\n#define NANOARROW_ARRAY_INLINE_H_INCLUDED\n\n#include \n#include \n#include \n#include \n#include \n\n\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nstatic inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n return &private_data->bitmap;\n}\n\nstatic inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n switch (i) {\n case 0:\n return &private_data->bitmap.buffer;\n case 1:\n return private_data->buffers;\n default:\n if (array->n_buffers > 3 && i == (array->n_buffers - 1)) {\n // The variadic buffer sizes buffer if for a BinaryView/String view array\n // is always stored in private_data->buffers[1]; however, from the numbered\n // buffers perspective this is the array->buffers[array->n_buffers - 1].\n return private_data->buffers + 1;\n } else if (array->n_buffers > 3) {\n // If there are one or more variadic buffers, they are stored in\n // private_data->variadic_buffers\n return private_data->variadic_buffers + (i - 2);\n } else {\n // Otherwise, we're just accessing buffer at index 2 (e.g., String/Binary\n // data buffer or variadic sizes buffer for the case where there are no\n // variadic buffers)\n NANOARROW_DCHECK(i == 2);\n return private_data->buffers + i - 1;\n }\n }\n}\n\n// We don't currently support the case of unions where type_id != child_index;\n// however, these functions are used to keep track of where that assumption\n// is made.\nstatic inline int8_t _ArrowArrayUnionChildIndex(struct ArrowArray* array,\n int8_t type_id) {\n NANOARROW_UNUSED(array);\n return type_id;\n}\n\nstatic inline int8_t _ArrowArrayUnionTypeId(struct ArrowArray* array,\n int8_t child_index) {\n NANOARROW_UNUSED(array);\n return child_index;\n}\n\nstatic inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) {\n if (*type_ids == '\\0') {\n return 0;\n }\n\n int32_t i = 0;\n long type_id;\n char* end_ptr;\n do {\n type_id = strtol(type_ids, &end_ptr, 10);\n if (end_ptr == type_ids || type_id < 0 || type_id > 127) {\n return -1;\n }\n\n if (out != NULL) {\n out[i] = (int8_t)type_id;\n }\n\n i++;\n\n type_ids = end_ptr;\n if (*type_ids == '\\0') {\n return i;\n } else if (*type_ids != ',') {\n return -1;\n } else {\n type_ids++;\n }\n } while (1);\n\n return -1;\n}\n\nstatic inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* type_ids,\n int64_t n_type_ids,\n int64_t n_children) {\n if (n_type_ids != n_children) {\n return 0;\n }\n\n for (int8_t i = 0; i < n_type_ids; i++) {\n if (type_ids[i] != i) {\n return 0;\n }\n }\n\n return 1;\n}\n\nstatic inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices(const char* type_id_str,\n int64_t n_children) {\n int8_t type_ids[128];\n int32_t n_type_ids = _ArrowParseUnionTypeIds(type_id_str, type_ids);\n return _ArrowParsedUnionTypeIdsWillEqualChildIndices(type_ids, n_type_ids, n_children);\n}\n\nstatic inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_UNINITIALIZED:\n return EINVAL;\n case NANOARROW_TYPE_SPARSE_UNION:\n case NANOARROW_TYPE_DENSE_UNION:\n // Note that this value could be -1 if the type_ids string was invalid\n if (private_data->union_type_id_is_child_index != 1) {\n return EINVAL;\n } else {\n break;\n }\n default:\n break;\n }\n if (private_data->storage_type == NANOARROW_TYPE_UNINITIALIZED) {\n return EINVAL;\n }\n\n // Initialize any data offset buffer with a single zero\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET &&\n private_data->layout.element_size_bits[i] == 64) {\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0));\n } else if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET &&\n private_data->layout.element_size_bits[i] == 32) {\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0));\n }\n }\n\n // Start building any child arrays or dictionaries\n for (int64_t i = 0; i < array->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array->children[i]));\n }\n\n if (array->dictionary != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array->dictionary));\n }\n\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array) {\n for (int64_t i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n struct ArrowBuffer* buffer = ArrowArrayBuffer(array, i);\n NANOARROW_RETURN_NOT_OK(ArrowBufferResize(buffer, buffer->size_bytes, 1));\n }\n\n for (int64_t i = 0; i < array->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayShrinkToFit(array->children[i]));\n }\n\n if (array->dictionary != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayShrinkToFit(array->dictionary));\n }\n\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array,\n int64_t buffer_i, uint8_t value,\n int64_t n) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n struct ArrowBuffer* buffer = ArrowArrayBuffer(array, buffer_i);\n int64_t bytes_required =\n _ArrowRoundUpToMultipleOf8(private_data->layout.element_size_bits[buffer_i] *\n (array->length + 1)) /\n 8;\n if (bytes_required > buffer->size_bytes) {\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendFill(buffer, 0, bytes_required - buffer->size_bytes));\n }\n\n ArrowBitsSetTo(buffer->data, array->length, n, value);\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* array,\n int64_t n, uint8_t is_valid) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n if (n == 0) {\n return NANOARROW_OK;\n }\n\n // Some type-specific handling\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_NA:\n // (An empty value for a null array *is* a null)\n array->null_count += n;\n array->length += n;\n return NANOARROW_OK;\n\n case NANOARROW_TYPE_DENSE_UNION: {\n // Add one null to the first child and append n references to that child\n int8_t type_id = _ArrowArrayUnionTypeId(array, 0);\n NANOARROW_RETURN_NOT_OK(\n _ArrowArrayAppendEmptyInternal(array->children[0], 1, is_valid));\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n));\n for (int64_t i = 0; i < n; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(\n ArrowArrayBuffer(array, 1), (int32_t)array->children[0]->length - 1));\n }\n // For the purposes of array->null_count, union elements are never considered \"null\"\n // even if some children contain nulls.\n array->length += n;\n return NANOARROW_OK;\n }\n\n case NANOARROW_TYPE_SPARSE_UNION: {\n // Add n nulls to the first child and append n references to that child\n int8_t type_id = _ArrowArrayUnionTypeId(array, 0);\n NANOARROW_RETURN_NOT_OK(\n _ArrowArrayAppendEmptyInternal(array->children[0], n, is_valid));\n for (int64_t i = 1; i < array->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(array->children[i], n));\n }\n\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n));\n // For the purposes of array->null_count, union elements are never considered \"null\"\n // even if some children contain nulls.\n array->length += n;\n return NANOARROW_OK;\n }\n\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(\n array->children[0], n * private_data->layout.child_size_elements));\n break;\n case NANOARROW_TYPE_STRUCT:\n for (int64_t i = 0; i < array->n_children; i++) {\n NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(array->children[i], n));\n }\n break;\n\n default:\n break;\n }\n\n // Append n is_valid bits to the validity bitmap. If we haven't allocated a bitmap yet\n // and we need to append nulls, do it now.\n if (!is_valid && private_data->bitmap.buffer.data == NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, array->length + n));\n ArrowBitmapAppendUnsafe(&private_data->bitmap, 1, array->length);\n ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n);\n } else if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, n));\n ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n);\n }\n\n // Add appropriate buffer fill\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n struct ArrowBuffer* buffer = ArrowArrayBuffer(array, i);\n int64_t size_bytes = private_data->layout.element_size_bits[i] / 8;\n\n switch (private_data->layout.buffer_type[i]) {\n case NANOARROW_BUFFER_TYPE_NONE:\n case NANOARROW_BUFFER_TYPE_VARIADIC_DATA:\n case NANOARROW_BUFFER_TYPE_VARIADIC_SIZE:\n case NANOARROW_BUFFER_TYPE_VALIDITY:\n // These buffer types don't require initialization for empty appends:\n // - NONE: No buffer exists\n // - VARIADIC_*: Handled by child arrays\n // - VALIDITY: Already handled in previous bitmap logic\n break;\n\n case NANOARROW_BUFFER_TYPE_SIZE:\n // Size buffers (e.g., string/array lengths) should be zero-initialized:\n // This ensures empty elements have logical zero-length\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n));\n break;\n\n case NANOARROW_BUFFER_TYPE_DATA_OFFSET:\n // Offset buffers require special handling to maintain continuity.\n // 1. Reserve space for new offset entries\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n));\n\n // 2. Duplicate last offset value for each new (empty) element\n for (int64_t j = 0; j < n; j++) {\n ArrowBufferAppendUnsafe(buffer, buffer->data + size_bytes * (array->length + j),\n size_bytes);\n }\n\n // 3. Skip next buffer (DATA) since it's paired with offsets\n // Rationale: Offset buffers are always followed by data buffers\n // that don't require separate initialization here\n i++;\n break;\n\n case NANOARROW_BUFFER_TYPE_DATA:\n // Fixed-width data buffers require zero-initialization:\n if (private_data->layout.element_size_bits[i] % 8 == 0) {\n // Byte-aligned: use efficient memset-style fill\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n));\n } else {\n // Bit-packed: use special bitwise initialization\n NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, i, 0, n));\n }\n break;\n\n case NANOARROW_BUFFER_TYPE_VIEW_OFFSET:\n // View offset buffers (for string/binary view types) require zero-initialization.\n NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n));\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n));\n break;\n\n case NANOARROW_BUFFER_TYPE_TYPE_ID:\n case NANOARROW_BUFFER_TYPE_UNION_OFFSET:\n // These buffer types should have been handled by the outer type switch and\n // are not expected here, indicating an internal logic error.\n return EINVAL;\n }\n }\n\n array->length += n;\n array->null_count += n * !is_valid;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n) {\n return _ArrowArrayAppendEmptyInternal(array, n, 0);\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n) {\n return _ArrowArrayAppendEmptyInternal(array, n, 1);\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array,\n int64_t value) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_INT64:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(int64_t)));\n break;\n case NANOARROW_TYPE_INT32:\n _NANOARROW_CHECK_RANGE(value, INT32_MIN, INT32_MAX);\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, (int32_t)value));\n break;\n case NANOARROW_TYPE_INT16:\n _NANOARROW_CHECK_RANGE(value, INT16_MIN, INT16_MAX);\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt16(data_buffer, (int16_t)value));\n break;\n case NANOARROW_TYPE_INT8:\n _NANOARROW_CHECK_RANGE(value, INT8_MIN, INT8_MAX);\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(data_buffer, (int8_t)value));\n break;\n case NANOARROW_TYPE_UINT64:\n case NANOARROW_TYPE_UINT32:\n case NANOARROW_TYPE_UINT16:\n case NANOARROW_TYPE_UINT8:\n _NANOARROW_CHECK_RANGE(value, 0, INT64_MAX);\n return ArrowArrayAppendUInt(array, value);\n case NANOARROW_TYPE_DOUBLE:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value));\n break;\n case NANOARROW_TYPE_FLOAT:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value));\n break;\n case NANOARROW_TYPE_HALF_FLOAT:\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendUInt16(data_buffer, ArrowFloatToHalfFloat((float)value)));\n break;\n case NANOARROW_TYPE_BOOL:\n NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1));\n break;\n default:\n return EINVAL;\n }\n\n if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));\n }\n\n array->length++;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array,\n uint64_t value) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_UINT64:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t)));\n break;\n case NANOARROW_TYPE_UINT32:\n _NANOARROW_CHECK_UPPER_LIMIT(value, UINT32_MAX);\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt32(data_buffer, (uint32_t)value));\n break;\n case NANOARROW_TYPE_UINT16:\n _NANOARROW_CHECK_UPPER_LIMIT(value, UINT16_MAX);\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt16(data_buffer, (uint16_t)value));\n break;\n case NANOARROW_TYPE_UINT8:\n _NANOARROW_CHECK_UPPER_LIMIT(value, UINT8_MAX);\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt8(data_buffer, (uint8_t)value));\n break;\n case NANOARROW_TYPE_INT64:\n case NANOARROW_TYPE_INT32:\n case NANOARROW_TYPE_INT16:\n case NANOARROW_TYPE_INT8:\n _NANOARROW_CHECK_UPPER_LIMIT(value, INT64_MAX);\n return ArrowArrayAppendInt(array, value);\n case NANOARROW_TYPE_DOUBLE:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value));\n break;\n case NANOARROW_TYPE_FLOAT:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value));\n break;\n case NANOARROW_TYPE_HALF_FLOAT:\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendUInt16(data_buffer, ArrowFloatToHalfFloat((float)value)));\n break;\n case NANOARROW_TYPE_BOOL:\n NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1));\n break;\n default:\n return EINVAL;\n }\n\n if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));\n }\n\n array->length++;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array,\n double value) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_DOUBLE:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(double)));\n break;\n case NANOARROW_TYPE_FLOAT:\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value));\n break;\n case NANOARROW_TYPE_HALF_FLOAT:\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendUInt16(data_buffer, ArrowFloatToHalfFloat((float)value)));\n break;\n default:\n return EINVAL;\n }\n\n if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));\n }\n\n array->length++;\n return NANOARROW_OK;\n}\n\n// Binary views only have two fixed buffers, but be aware that they must also\n// always have more 1 buffer to store variadic buffer sizes (even if there are none)\n#define NANOARROW_BINARY_VIEW_FIXED_BUFFERS 2\n#define NANOARROW_BINARY_VIEW_INLINE_SIZE 12\n#define NANOARROW_BINARY_VIEW_PREFIX_SIZE 4\n#define NANOARROW_BINARY_VIEW_BLOCK_SIZE (32 << 10) // 32KB\n\n// The Arrow C++ implementation uses anonymous structs as members\n// of the ArrowBinaryView. For Cython support in this library, we define\n// those structs outside of the ArrowBinaryView\nstruct ArrowBinaryViewInlined {\n int32_t size;\n uint8_t data[NANOARROW_BINARY_VIEW_INLINE_SIZE];\n};\n\nstruct ArrowBinaryViewRef {\n int32_t size;\n uint8_t prefix[NANOARROW_BINARY_VIEW_PREFIX_SIZE];\n int32_t buffer_index;\n int32_t offset;\n};\n\nunion ArrowBinaryView {\n struct ArrowBinaryViewInlined inlined;\n struct ArrowBinaryViewRef ref;\n int64_t alignment_dummy;\n};\n\nstatic inline int32_t ArrowArrayVariadicBufferCount(struct ArrowArray* array) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n return private_data->n_variadic_buffers;\n}\n\nstatic inline ArrowErrorCode ArrowArrayAddVariadicBuffers(struct ArrowArray* array,\n int32_t n_buffers) {\n const int32_t n_current_bufs = ArrowArrayVariadicBufferCount(array);\n const int32_t nvariadic_bufs_needed = n_current_bufs + n_buffers;\n\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n private_data->variadic_buffers = (struct ArrowBuffer*)ArrowRealloc(\n private_data->variadic_buffers, sizeof(struct ArrowBuffer) * nvariadic_bufs_needed);\n if (private_data->variadic_buffers == NULL) {\n return ENOMEM;\n }\n\n private_data->n_variadic_buffers = nvariadic_bufs_needed;\n array->n_buffers = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + 1 + nvariadic_bufs_needed;\n\n private_data->buffer_data = (const void**)ArrowRealloc(\n private_data->buffer_data, array->n_buffers * sizeof(void*));\n\n for (int32_t i = n_current_bufs; i < nvariadic_bufs_needed; i++) {\n ArrowBufferInit(&private_data->variadic_buffers[i]);\n private_data->buffer_data[NANOARROW_BINARY_VIEW_FIXED_BUFFERS + i] = NULL;\n }\n\n // Zero out memory for the final buffer (variadic sizes buffer we haven't built yet)\n private_data->buffer_data[NANOARROW_BINARY_VIEW_FIXED_BUFFERS + nvariadic_bufs_needed] =\n NULL;\n\n // Ensure array->buffers points to a valid value\n array->buffers = private_data->buffer_data;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array,\n struct ArrowBufferView value) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n if (private_data->storage_type == NANOARROW_TYPE_STRING_VIEW ||\n private_data->storage_type == NANOARROW_TYPE_BINARY_VIEW) {\n struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);\n union ArrowBinaryView bvt;\n bvt.inlined.size = (int32_t)value.size_bytes;\n\n if (value.size_bytes <= NANOARROW_BINARY_VIEW_INLINE_SIZE) {\n memcpy(bvt.inlined.data, value.data.as_char, value.size_bytes);\n memset(bvt.inlined.data + bvt.inlined.size, 0,\n NANOARROW_BINARY_VIEW_INLINE_SIZE - bvt.inlined.size);\n } else {\n int32_t current_n_vbufs = ArrowArrayVariadicBufferCount(array);\n if (current_n_vbufs == 0 ||\n private_data->variadic_buffers[current_n_vbufs - 1].size_bytes +\n value.size_bytes >\n NANOARROW_BINARY_VIEW_BLOCK_SIZE) {\n const int32_t additional_bufs_needed = 1;\n NANOARROW_RETURN_NOT_OK(\n ArrowArrayAddVariadicBuffers(array, additional_bufs_needed));\n current_n_vbufs += additional_bufs_needed;\n }\n\n const int32_t buf_index = current_n_vbufs - 1;\n struct ArrowBuffer* variadic_buf = &private_data->variadic_buffers[buf_index];\n memcpy(bvt.ref.prefix, value.data.as_char, NANOARROW_BINARY_VIEW_PREFIX_SIZE);\n bvt.ref.buffer_index = (int32_t)buf_index;\n bvt.ref.offset = (int32_t)variadic_buf->size_bytes;\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(variadic_buf, value.data.as_char, value.size_bytes));\n }\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &bvt, sizeof(bvt)));\n } else {\n struct ArrowBuffer* offset_buffer = ArrowArrayBuffer(array, 1);\n struct ArrowBuffer* data_buffer = ArrowArrayBuffer(\n array, 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY));\n int32_t offset;\n int64_t large_offset;\n int64_t fixed_size_bytes = private_data->layout.element_size_bits[1] / 8;\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_BINARY:\n offset = ((int32_t*)offset_buffer->data)[array->length];\n if ((((int64_t)offset) + value.size_bytes) > INT32_MAX) {\n return EOVERFLOW;\n }\n\n offset += (int32_t)value.size_bytes;\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t)));\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes));\n break;\n\n case NANOARROW_TYPE_LARGE_STRING:\n case NANOARROW_TYPE_LARGE_BINARY:\n large_offset = ((int64_t*)offset_buffer->data)[array->length];\n large_offset += value.size_bytes;\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(offset_buffer, &large_offset, sizeof(int64_t)));\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes));\n break;\n\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n if (value.size_bytes != fixed_size_bytes) {\n return EINVAL;\n }\n\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes));\n break;\n default:\n return EINVAL;\n }\n }\n\n if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));\n }\n\n array->length++;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array,\n struct ArrowStringView value) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n struct ArrowBufferView buffer_view;\n buffer_view.data.data = value.data;\n buffer_view.size_bytes = value.size_bytes;\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_LARGE_STRING:\n case NANOARROW_TYPE_STRING_VIEW:\n case NANOARROW_TYPE_BINARY:\n case NANOARROW_TYPE_LARGE_BINARY:\n case NANOARROW_TYPE_BINARY_VIEW:\n return ArrowArrayAppendBytes(array, buffer_view);\n default:\n return EINVAL;\n }\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array,\n const struct ArrowInterval* value) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_INTERVAL_MONTHS: {\n if (value->type != NANOARROW_TYPE_INTERVAL_MONTHS) {\n return EINVAL;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months));\n break;\n }\n case NANOARROW_TYPE_INTERVAL_DAY_TIME: {\n if (value->type != NANOARROW_TYPE_INTERVAL_DAY_TIME) {\n return EINVAL;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days));\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->ms));\n break;\n }\n case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO: {\n if (value->type != NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO) {\n return EINVAL;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months));\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days));\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(data_buffer, value->ns));\n break;\n }\n default:\n return EINVAL;\n }\n\n if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));\n }\n\n array->length++;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array,\n const struct ArrowDecimal* value) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_DECIMAL32:\n if (value->n_words != 0) {\n return EINVAL;\n } else {\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(data_buffer, value->words, sizeof(uint32_t)));\n break;\n }\n case NANOARROW_TYPE_DECIMAL64:\n if (value->n_words != 1) {\n return EINVAL;\n } else {\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(data_buffer, value->words, sizeof(uint64_t)));\n break;\n }\n case NANOARROW_TYPE_DECIMAL128:\n if (value->n_words != 2) {\n return EINVAL;\n } else {\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(data_buffer, value->words, 2 * sizeof(uint64_t)));\n break;\n }\n case NANOARROW_TYPE_DECIMAL256:\n if (value->n_words != 4) {\n return EINVAL;\n } else {\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppend(data_buffer, value->words, 4 * sizeof(uint64_t)));\n break;\n }\n default:\n return EINVAL;\n }\n\n if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));\n }\n\n array->length++;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n int64_t child_length;\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_MAP:\n child_length = array->children[0]->length;\n if (child_length > INT32_MAX) {\n return EOVERFLOW;\n }\n\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), (int32_t)child_length));\n break;\n case NANOARROW_TYPE_LARGE_LIST:\n child_length = array->children[0]->length;\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendInt64(ArrowArrayBuffer(array, 1), child_length));\n break;\n case NANOARROW_TYPE_FIXED_SIZE_LIST:\n child_length = array->children[0]->length;\n if (child_length !=\n ((array->length + 1) * private_data->layout.child_size_elements)) {\n return EINVAL;\n }\n break;\n case NANOARROW_TYPE_LIST_VIEW: {\n child_length = array->children[0]->length;\n if (child_length > INT32_MAX) {\n return EOVERFLOW;\n }\n\n const int32_t last_valid_offset = (int32_t)private_data->list_view_offset;\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), last_valid_offset));\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(\n ArrowArrayBuffer(array, 2), (int32_t)child_length - last_valid_offset));\n private_data->list_view_offset = child_length;\n break;\n }\n case NANOARROW_TYPE_LARGE_LIST_VIEW: {\n child_length = array->children[0]->length;\n const int64_t last_valid_offset = private_data->list_view_offset;\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendInt64(ArrowArrayBuffer(array, 1), last_valid_offset));\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, 2),\n child_length - last_valid_offset));\n private_data->list_view_offset = child_length;\n break;\n }\n\n case NANOARROW_TYPE_STRUCT:\n for (int64_t i = 0; i < array->n_children; i++) {\n child_length = array->children[i]->length;\n if (child_length != (array->length + 1)) {\n return EINVAL;\n }\n }\n break;\n default:\n return EINVAL;\n }\n\n if (private_data->bitmap.buffer.data != NULL) {\n NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1));\n }\n\n array->length++;\n return NANOARROW_OK;\n}\n\nstatic inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array,\n int8_t type_id) {\n struct ArrowArrayPrivateData* private_data =\n (struct ArrowArrayPrivateData*)array->private_data;\n\n int64_t child_index = _ArrowArrayUnionChildIndex(array, type_id);\n if (child_index < 0 || child_index >= array->n_children) {\n return EINVAL;\n }\n\n switch (private_data->storage_type) {\n case NANOARROW_TYPE_DENSE_UNION:\n // Append the target child length to the union offsets buffer\n _NANOARROW_CHECK_RANGE(array->children[child_index]->length, 0, INT32_MAX);\n NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(\n ArrowArrayBuffer(array, 1), (int32_t)array->children[child_index]->length - 1));\n break;\n case NANOARROW_TYPE_SPARSE_UNION:\n // Append one empty to any non-target column that isn't already the right length\n // or abort if appending a null will result in a column with invalid length\n for (int64_t i = 0; i < array->n_children; i++) {\n if (i == child_index || array->children[i]->length == (array->length + 1)) {\n continue;\n }\n\n if (array->children[i]->length != array->length) {\n return EINVAL;\n }\n\n NANOARROW_RETURN_NOT_OK(ArrowArrayAppendEmpty(array->children[i], 1));\n }\n\n break;\n default:\n return EINVAL;\n }\n\n // Write to the type_ids buffer\n NANOARROW_RETURN_NOT_OK(\n ArrowBufferAppendInt8(ArrowArrayBuffer(array, 0), (int8_t)type_id));\n array->length++;\n return NANOARROW_OK;\n}\n\nstatic inline void ArrowArrayViewMove(struct ArrowArrayView* src,\n struct ArrowArrayView* dst) {\n memcpy(dst, src, sizeof(struct ArrowArrayView));\n ArrowArrayViewInitFromType(src, NANOARROW_TYPE_UNINITIALIZED);\n}\n\nstatic inline int64_t ArrowArrayViewGetNumBuffers(struct ArrowArrayView* array_view) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_BINARY_VIEW:\n case NANOARROW_TYPE_STRING_VIEW:\n return NANOARROW_BINARY_VIEW_FIXED_BUFFERS + array_view->n_variadic_buffers + 1;\n default:\n break;\n }\n\n int64_t n_buffers = 0;\n for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {\n if (array_view->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_NONE) {\n break;\n }\n\n n_buffers++;\n }\n\n return n_buffers;\n}\n\nstatic inline struct ArrowBufferView ArrowArrayViewGetBufferView(\n struct ArrowArrayView* array_view, int64_t i) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_BINARY_VIEW:\n case NANOARROW_TYPE_STRING_VIEW:\n if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {\n return array_view->buffer_views[i];\n } else if (i >=\n (array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {\n struct ArrowBufferView view;\n view.data.as_int64 = array_view->variadic_buffer_sizes;\n view.size_bytes = array_view->n_variadic_buffers * sizeof(double);\n return view;\n } else {\n struct ArrowBufferView view;\n view.data.data =\n array_view->variadic_buffers[i - NANOARROW_BINARY_VIEW_FIXED_BUFFERS];\n view.size_bytes =\n array_view->variadic_buffer_sizes[i - NANOARROW_BINARY_VIEW_FIXED_BUFFERS];\n return view;\n }\n default:\n // We need this check to avoid -Warray-bounds from complaining\n if (i >= NANOARROW_MAX_FIXED_BUFFERS) {\n struct ArrowBufferView view;\n view.data.data = NULL;\n view.size_bytes = 0;\n return view;\n } else {\n return array_view->buffer_views[i];\n }\n }\n}\n\nenum ArrowBufferType ArrowArrayViewGetBufferType(struct ArrowArrayView* array_view,\n int64_t i) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_BINARY_VIEW:\n case NANOARROW_TYPE_STRING_VIEW:\n if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {\n return array_view->layout.buffer_type[i];\n } else if (i ==\n (array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {\n return NANOARROW_BUFFER_TYPE_VARIADIC_SIZE;\n } else {\n return NANOARROW_BUFFER_TYPE_VARIADIC_DATA;\n }\n default:\n // We need this check to avoid -Warray-bounds from complaining\n if (i >= NANOARROW_MAX_FIXED_BUFFERS) {\n return NANOARROW_BUFFER_TYPE_NONE;\n } else {\n return array_view->layout.buffer_type[i];\n }\n }\n}\n\nstatic inline enum ArrowType ArrowArrayViewGetBufferDataType(\n struct ArrowArrayView* array_view, int64_t i) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_BINARY_VIEW:\n case NANOARROW_TYPE_STRING_VIEW:\n if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {\n return array_view->layout.buffer_data_type[i];\n } else if (i >=\n (array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {\n return NANOARROW_TYPE_INT64;\n } else if (array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW) {\n return NANOARROW_TYPE_BINARY;\n } else {\n return NANOARROW_TYPE_STRING;\n }\n default:\n // We need this check to avoid -Warray-bounds from complaining\n if (i >= NANOARROW_MAX_FIXED_BUFFERS) {\n return NANOARROW_TYPE_UNINITIALIZED;\n } else {\n return array_view->layout.buffer_data_type[i];\n }\n }\n}\n\nstatic inline int64_t ArrowArrayViewGetBufferElementSizeBits(\n struct ArrowArrayView* array_view, int64_t i) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_BINARY_VIEW:\n case NANOARROW_TYPE_STRING_VIEW:\n if (i < NANOARROW_BINARY_VIEW_FIXED_BUFFERS) {\n return array_view->layout.element_size_bits[i];\n } else if (i >=\n (array_view->n_variadic_buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS)) {\n return sizeof(int64_t) * 8;\n } else {\n return 0;\n }\n default:\n // We need this check to avoid -Warray-bounds from complaining\n if (i >= NANOARROW_MAX_FIXED_BUFFERS) {\n return 0;\n } else {\n return array_view->layout.element_size_bits[i];\n }\n }\n}\n\nstatic inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view,\n int64_t i) {\n const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8;\n i += array_view->offset;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_NA:\n return 0x01;\n case NANOARROW_TYPE_DENSE_UNION:\n case NANOARROW_TYPE_SPARSE_UNION:\n // Unions are \"never null\" in Arrow land\n return 0x00;\n default:\n return validity_buffer != NULL && !ArrowBitGet(validity_buffer, i);\n }\n}\n\nstatic inline int64_t ArrowArrayViewComputeNullCount(\n const struct ArrowArrayView* array_view) {\n if (array_view->length == 0) {\n return 0;\n }\n\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_NA:\n return array_view->length;\n case NANOARROW_TYPE_DENSE_UNION:\n case NANOARROW_TYPE_SPARSE_UNION:\n // Unions are \"never null\" in Arrow land\n return 0;\n default:\n break;\n }\n\n const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8;\n if (validity_buffer == NULL) {\n return 0;\n }\n return array_view->length -\n ArrowBitCountSet(validity_buffer, array_view->offset, array_view->length);\n}\n\nstatic inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view,\n int64_t i) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_DENSE_UNION:\n case NANOARROW_TYPE_SPARSE_UNION:\n return array_view->buffer_views[0].data.as_int8[array_view->offset + i];\n default:\n return -1;\n }\n}\n\nstatic inline int8_t ArrowArrayViewUnionChildIndex(\n const struct ArrowArrayView* array_view, int64_t i) {\n int8_t type_id = ArrowArrayViewUnionTypeId(array_view, i);\n if (array_view->union_type_id_map == NULL) {\n return type_id;\n } else {\n return array_view->union_type_id_map[type_id];\n }\n}\n\nstatic inline int64_t ArrowArrayViewUnionChildOffset(\n const struct ArrowArrayView* array_view, int64_t i) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_DENSE_UNION:\n return array_view->buffer_views[1].data.as_int32[array_view->offset + i];\n case NANOARROW_TYPE_SPARSE_UNION:\n return array_view->offset + i;\n default:\n return -1;\n }\n}\n\nstatic inline int64_t ArrowArrayViewListChildOffset(\n const struct ArrowArrayView* array_view, int64_t i) {\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_LIST:\n case NANOARROW_TYPE_MAP:\n case NANOARROW_TYPE_LIST_VIEW:\n return array_view->buffer_views[1].data.as_int32[i];\n case NANOARROW_TYPE_LARGE_LIST:\n case NANOARROW_TYPE_LARGE_LIST_VIEW:\n return array_view->buffer_views[1].data.as_int64[i];\n default:\n return -1;\n }\n}\n\nstatic struct ArrowBufferView ArrowArrayViewGetBytesFromViewArrayUnsafe(\n const struct ArrowArrayView* array_view, int64_t i) {\n const union ArrowBinaryView* bv = &array_view->buffer_views[1].data.as_binary_view[i];\n struct ArrowBufferView out = {{NULL}, bv->inlined.size};\n if (bv->inlined.size <= NANOARROW_BINARY_VIEW_INLINE_SIZE) {\n out.data.as_uint8 = bv->inlined.data;\n return out;\n }\n\n out.data.data = array_view->variadic_buffers[bv->ref.buffer_index];\n out.data.as_uint8 += bv->ref.offset;\n return out;\n}\n\nstatic inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view,\n int64_t i) {\n const struct ArrowBufferView* data_view = &array_view->buffer_views[1];\n i += array_view->offset;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_INT64:\n return data_view->data.as_int64[i];\n case NANOARROW_TYPE_UINT64:\n return data_view->data.as_uint64[i];\n case NANOARROW_TYPE_INTERVAL_MONTHS:\n case NANOARROW_TYPE_INT32:\n return data_view->data.as_int32[i];\n case NANOARROW_TYPE_UINT32:\n return data_view->data.as_uint32[i];\n case NANOARROW_TYPE_INT16:\n return data_view->data.as_int16[i];\n case NANOARROW_TYPE_UINT16:\n return data_view->data.as_uint16[i];\n case NANOARROW_TYPE_INT8:\n return data_view->data.as_int8[i];\n case NANOARROW_TYPE_UINT8:\n return data_view->data.as_uint8[i];\n case NANOARROW_TYPE_DOUBLE:\n return (int64_t)data_view->data.as_double[i];\n case NANOARROW_TYPE_FLOAT:\n return (int64_t)data_view->data.as_float[i];\n case NANOARROW_TYPE_HALF_FLOAT:\n return (int64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);\n case NANOARROW_TYPE_BOOL:\n return ArrowBitGet(data_view->data.as_uint8, i);\n default:\n return INT64_MAX;\n }\n}\n\nstatic inline uint64_t ArrowArrayViewGetUIntUnsafe(\n const struct ArrowArrayView* array_view, int64_t i) {\n i += array_view->offset;\n const struct ArrowBufferView* data_view = &array_view->buffer_views[1];\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_INT64:\n return data_view->data.as_int64[i];\n case NANOARROW_TYPE_UINT64:\n return data_view->data.as_uint64[i];\n case NANOARROW_TYPE_INTERVAL_MONTHS:\n case NANOARROW_TYPE_INT32:\n return data_view->data.as_int32[i];\n case NANOARROW_TYPE_UINT32:\n return data_view->data.as_uint32[i];\n case NANOARROW_TYPE_INT16:\n return data_view->data.as_int16[i];\n case NANOARROW_TYPE_UINT16:\n return data_view->data.as_uint16[i];\n case NANOARROW_TYPE_INT8:\n return data_view->data.as_int8[i];\n case NANOARROW_TYPE_UINT8:\n return data_view->data.as_uint8[i];\n case NANOARROW_TYPE_DOUBLE:\n return (uint64_t)data_view->data.as_double[i];\n case NANOARROW_TYPE_FLOAT:\n return (uint64_t)data_view->data.as_float[i];\n case NANOARROW_TYPE_HALF_FLOAT:\n return (uint64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);\n case NANOARROW_TYPE_BOOL:\n return ArrowBitGet(data_view->data.as_uint8, i);\n default:\n return UINT64_MAX;\n }\n}\n\nstatic inline double ArrowArrayViewGetDoubleUnsafe(\n const struct ArrowArrayView* array_view, int64_t i) {\n i += array_view->offset;\n const struct ArrowBufferView* data_view = &array_view->buffer_views[1];\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_INT64:\n return (double)data_view->data.as_int64[i];\n case NANOARROW_TYPE_UINT64:\n return (double)data_view->data.as_uint64[i];\n case NANOARROW_TYPE_INT32:\n return data_view->data.as_int32[i];\n case NANOARROW_TYPE_UINT32:\n return data_view->data.as_uint32[i];\n case NANOARROW_TYPE_INT16:\n return data_view->data.as_int16[i];\n case NANOARROW_TYPE_UINT16:\n return data_view->data.as_uint16[i];\n case NANOARROW_TYPE_INT8:\n return data_view->data.as_int8[i];\n case NANOARROW_TYPE_UINT8:\n return data_view->data.as_uint8[i];\n case NANOARROW_TYPE_DOUBLE:\n return data_view->data.as_double[i];\n case NANOARROW_TYPE_FLOAT:\n return data_view->data.as_float[i];\n case NANOARROW_TYPE_HALF_FLOAT:\n return ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);\n case NANOARROW_TYPE_BOOL:\n return ArrowBitGet(data_view->data.as_uint8, i);\n default:\n return DBL_MAX;\n }\n}\n\nstatic inline struct ArrowStringView ArrowArrayViewGetStringUnsafe(\n const struct ArrowArrayView* array_view, int64_t i) {\n i += array_view->offset;\n const struct ArrowBufferView* offsets_view = &array_view->buffer_views[1];\n const char* data_view = array_view->buffer_views[2].data.as_char;\n\n struct ArrowStringView view;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_BINARY:\n view.data = data_view + offsets_view->data.as_int32[i];\n view.size_bytes =\n (int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i];\n break;\n case NANOARROW_TYPE_LARGE_STRING:\n case NANOARROW_TYPE_LARGE_BINARY:\n view.data = data_view + offsets_view->data.as_int64[i];\n view.size_bytes =\n offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i];\n break;\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n view.size_bytes = array_view->layout.element_size_bits[1] / 8;\n view.data = array_view->buffer_views[1].data.as_char + (i * view.size_bytes);\n break;\n case NANOARROW_TYPE_STRING_VIEW:\n case NANOARROW_TYPE_BINARY_VIEW: {\n struct ArrowBufferView buf_view =\n ArrowArrayViewGetBytesFromViewArrayUnsafe(array_view, i);\n view.data = buf_view.data.as_char;\n view.size_bytes = buf_view.size_bytes;\n break;\n }\n default:\n view.data = NULL;\n view.size_bytes = 0;\n break;\n }\n\n return view;\n}\n\nstatic inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe(\n const struct ArrowArrayView* array_view, int64_t i) {\n i += array_view->offset;\n const struct ArrowBufferView* offsets_view = &array_view->buffer_views[1];\n const uint8_t* data_view = array_view->buffer_views[2].data.as_uint8;\n\n struct ArrowBufferView view;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_STRING:\n case NANOARROW_TYPE_BINARY:\n view.size_bytes =\n (int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i];\n view.data.as_uint8 = data_view + offsets_view->data.as_int32[i];\n break;\n case NANOARROW_TYPE_LARGE_STRING:\n case NANOARROW_TYPE_LARGE_BINARY:\n view.size_bytes =\n offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i];\n view.data.as_uint8 = data_view + offsets_view->data.as_int64[i];\n break;\n case NANOARROW_TYPE_FIXED_SIZE_BINARY:\n view.size_bytes = array_view->layout.element_size_bits[1] / 8;\n view.data.as_uint8 =\n array_view->buffer_views[1].data.as_uint8 + (i * view.size_bytes);\n break;\n case NANOARROW_TYPE_STRING_VIEW:\n case NANOARROW_TYPE_BINARY_VIEW:\n view = ArrowArrayViewGetBytesFromViewArrayUnsafe(array_view, i);\n break;\n default:\n view.data.data = NULL;\n view.size_bytes = 0;\n break;\n }\n\n return view;\n}\n\nstatic inline void ArrowArrayViewGetIntervalUnsafe(\n const struct ArrowArrayView* array_view, int64_t i, struct ArrowInterval* out) {\n const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8;\n const int64_t offset = array_view->offset;\n const int64_t index = offset + i;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_INTERVAL_MONTHS: {\n const size_t size = sizeof(int32_t);\n memcpy(&out->months, data_view + index * size, sizeof(int32_t));\n break;\n }\n case NANOARROW_TYPE_INTERVAL_DAY_TIME: {\n const size_t size = sizeof(int32_t) + sizeof(int32_t);\n memcpy(&out->days, data_view + index * size, sizeof(int32_t));\n memcpy(&out->ms, data_view + index * size + 4, sizeof(int32_t));\n break;\n }\n case NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO: {\n const size_t size = sizeof(int32_t) + sizeof(int32_t) + sizeof(int64_t);\n memcpy(&out->months, data_view + index * size, sizeof(int32_t));\n memcpy(&out->days, data_view + index * size + 4, sizeof(int32_t));\n memcpy(&out->ns, data_view + index * size + 8, sizeof(int64_t));\n break;\n }\n default:\n break;\n }\n}\n\nstatic inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view,\n int64_t i, struct ArrowDecimal* out) {\n i += array_view->offset;\n const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8;\n switch (array_view->storage_type) {\n case NANOARROW_TYPE_DECIMAL32:\n ArrowDecimalSetBytes(out, data_view + (i * 4));\n break;\n case NANOARROW_TYPE_DECIMAL64:\n ArrowDecimalSetBytes(out, data_view + (i * 8));\n break;\n case NANOARROW_TYPE_DECIMAL128:\n ArrowDecimalSetBytes(out, data_view + (i * 16));\n break;\n case NANOARROW_TYPE_DECIMAL256:\n ArrowDecimalSetBytes(out, data_view + (i * 32));\n break;\n default:\n memset(out->words, 0, sizeof(out->words));\n break;\n }\n}\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n" }, { "path": "data/lang/cpp/pool.cpp", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// needed for foreign interface\n#include \n#include \n#include \n#include \n\n#include \n#include \n\n\nusing namespace std;\n\nchar* g_tmpdir;\n\nuint8_t* foreign_call(const char* socket_filename, size_t mid, ...) __attribute__((sentinel));\n\n// AUTO include statements start\n// <<>>\n// AUTO include statements end\n\n// Proper linking of cppmorloc requires it be included AFTER the custom modules\n#include \"mlc_arrow.hpp\"\n#include \"cppmorloc.hpp\"\n\n#define PROPAGATE_ERROR(errmsg) \\\n if(errmsg != NULL) { \\\n char errmsg_buffer[MAX_ERRMSG_SIZE] = { 0 }; \\\n snprintf(errmsg_buffer, MAX_ERRMSG_SIZE, \"Error C++ pool (%s:%d in %s):\\n%s\" , __FILE__, __LINE__, __func__, errmsg); \\\n free(errmsg); \\\n throw std::runtime_error(errmsg_buffer); \\\n }\n\n#define PROPAGATE_FAIL_PACKET(errmsg) \\\n if(errmsg != NULL){ \\\n uint8_t* fail_packet_ = make_fail_packet(errmsg); \\\n free(errmsg); \\\n return fail_packet_; \\\n }\n\n\n// AUTO serialization statements start\n// <<>>\n// AUTO serialization statements end\n\n\n\nstd::string interweave_strings(const std::vector& first, const std::vector& second)\n{\n // Validate sizes - errors here indicate a bug in the morloc compiler\n if (first.size() != second.size() + 1) {\n throw std::invalid_argument(\"First list must have exactly 1 more element than second list\");\n }\n\n // Pre-calculate total size to avoid reallocations\n size_t total_size = 0;\n for (const auto& s : first) total_size += s.size();\n for (const auto& s : second) total_size += s.size();\n\n std::string result;\n result.reserve(total_size);\n\n // Interweave the strings\n for (size_t i = 0; i < second.size(); ++i) {\n result += first[i];\n result += second[i];\n }\n result += first.back(); // Append the final element from first list\n\n return result;\n}\n\n// Thread-local list of SHM pointers allocated by _put_value.\n// Freed after foreign_call returns (args consumed) or at next dispatch start\n// (result consumed by caller in the synchronous call that returned it).\nstruct ShmEntry { absptr_t ptr; Schema* schema; };\nthread_local std::vector _shm_tracker;\n\nstatic void _flush_shm_tracker() {\n for (auto& e : _shm_tracker) {\n char* err = NULL;\n // Only do recursive sub-freeing if we have a schema and this is\n // the last reference. NULL schema entries (from foreign_call result\n // tracking) just decrement the refcount.\n block_header_t* blk = (block_header_t*)((char*)e.ptr - sizeof(block_header_t));\n if (e.schema && blk->reference_count <= 1) {\n shfree_by_schema(e.ptr, e.schema, &err);\n if (err) { free(err); err = NULL; }\n }\n shfree(e.ptr, &err);\n if (err) { free(err); }\n }\n _shm_tracker.clear();\n}\n\n// Thread-local schema cache: avoids re-parsing the same schema strings\nSchema* get_cached_schema(const char* schema_str) {\n thread_local std::unordered_map cache;\n auto it = cache.find(schema_str);\n if (it != cache.end()) return it->second;\n Schema* schema = parse_schema_cpp(schema_str);\n cache[schema_str] = schema;\n return schema;\n}\n\n// Transforms a serialized value into a message ready for the socket\ntemplate \nuint8_t* _put_value(const T& value, const std::string& schema_str) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n\n if constexpr (std::is_same_v) {\n // Arrow export: move table data into SHM, build packet.\n // const_cast is safe here: the value is always a temporary from\n // a manifold call, never a truly const object.\n mlc::ArrowTable& tbl = const_cast(value);\n relptr_t relptr = tbl.move_to_shm();\n\n uint8_t* packet = make_arrow_data_packet(relptr, schema);\n if (!packet) { throw std::runtime_error(\"Failed to create arrow data packet\"); }\n\n char* err = nullptr;\n void* shm_ptr = rel2abs(relptr, &err);\n if (err) { free(err); }\n if (shm_ptr) { _shm_tracker.push_back({(absptr_t)shm_ptr, nullptr}); }\n return packet;\n } else {\n // Arrow dispatch: if schema hint is \"arrow\", the C++ type must be mlc::ArrowTable\n if (schema->hint && strcmp(schema->hint, \"arrow\") == 0) {\n throw std::runtime_error(\"Arrow schema but C++ type is not mlc::ArrowTable\");\n }\n\n void* voidstar = nullptr;\n try {\n voidstar = toAnything(schema, value);\n relptr_t relptr = abs2rel_cpp(voidstar);\n\n char* errmsg = nullptr;\n uint8_t* packet = make_data_packet_auto(voidstar, relptr, schema, &errmsg);\n if (errmsg) {\n shfree_cpp(voidstar);\n PROPAGATE_ERROR(errmsg);\n }\n\n const morloc_packet_header_t* hdr = (const morloc_packet_header_t*)packet;\n if (hdr->command.data.source == PACKET_SOURCE_RPTR) {\n // SHM referenced by packet -- track for deferred cleanup\n _shm_tracker.push_back({(absptr_t)voidstar, schema});\n } else {\n // Data inlined in packet -- free SHM immediately\n char* free_err = NULL;\n shfree_by_schema((absptr_t)voidstar, schema, &free_err);\n if (free_err) { free(free_err); free_err = NULL; }\n shfree((absptr_t)voidstar, &free_err);\n if (free_err) { free(free_err); }\n }\n return packet;\n } catch (...) {\n if (voidstar) shfree_cpp(voidstar);\n throw;\n }\n }\n}\n\n\n// Use a key to retrieve a value\ntemplate \nT _get_value(const uint8_t* packet, const std::string& schema_str){\n const morloc_packet_header_t* header = (const morloc_packet_header_t*)packet;\n uint8_t source = header->command.data.source;\n uint8_t format = header->command.data.format;\n\n if constexpr (std::is_same_v) {\n // Arrow import: packet -> arrow_from_shm -> ArrowTable\n Schema* schema = get_cached_schema(schema_str.c_str());\n char* errmsg = nullptr;\n uint8_t* raw = get_morloc_data_packet_value(packet, schema, &errmsg);\n if (errmsg) { PROPAGATE_ERROR(errmsg); }\n\n const arrow_shm_header_t* hdr = (const arrow_shm_header_t*)raw;\n struct ArrowSchema as;\n struct ArrowArray aa;\n char* aerr = nullptr;\n arrow_from_shm(hdr, &as, &aa, &aerr);\n if (aerr) { PROPAGATE_ERROR(aerr); }\n\n char* ierr = nullptr;\n shincref((absptr_t)raw, &ierr);\n if (ierr) { free(ierr); }\n _shm_tracker.push_back({(absptr_t)raw, nullptr});\n\n return mlc::ArrowTable(std::move(as), std::move(aa));\n } else {\n if (format == PACKET_FORMAT_ARROW) {\n throw std::runtime_error(\"Arrow data but C++ type is not mlc::ArrowTable\");\n }\n\n Schema* schema = get_cached_schema(schema_str.c_str());\n\n // Fast path: inline voidstar -- read directly from packet, no SHM needed\n if (source == PACKET_SOURCE_MESG && format == PACKET_FORMAT_VOIDSTAR) {\n const uint8_t* payload = packet + sizeof(morloc_packet_header_t) + header->offset;\n T* dummy = nullptr;\n return fromAnything(schema, (const void*)payload, dummy, (const void*)payload);\n }\n\n // SHM paths (RPTR or MESG+MSGPACK): existing logic\n bool is_rptr = (source == PACKET_SOURCE_RPTR);\n\n char* errmsg = NULL;\n uint8_t* voidstar = get_morloc_data_packet_value(packet, schema, &errmsg);\n if(errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n\n // For RPTR data, increment refcount so the owner's tracker flush\n // won't destroy data we may still need (e.g. forwarded packets).\n if (is_rptr) {\n char* incref_err = NULL;\n shincref((absptr_t)voidstar, &incref_err);\n if (incref_err) { free(incref_err); }\n _shm_tracker.push_back({(absptr_t)voidstar, schema});\n }\n\n T* dummy = nullptr;\n return fromAnything(schema, (void*)voidstar, dummy);\n }\n}\n\n\n// Hash a value, returning a 16-char hex string\ntemplate \nstd::string _mlc_hash(const T& value, const std::string& schema_str) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n void* voidstar = toAnything(schema, value);\n char* errmsg = NULL;\n char* hex = mlc_hash(voidstar, schema, &errmsg);\n shfree_cpp(voidstar);\n if (errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n std::string result(hex);\n free(hex);\n return result;\n}\n\n// Save a value to file in msgpack format\ntemplate \nvoid _mlc_save(const T& value, const std::string& schema_str, const std::string& path) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n void* voidstar = toAnything(schema, value);\n char* errmsg = NULL;\n mlc_save(voidstar, schema, path.c_str(), &errmsg);\n shfree_cpp(voidstar);\n if (errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n}\n\n// Save a value to file in flat voidstar binary format\ntemplate \nvoid _mlc_save_voidstar(const T& value, const std::string& schema_str, const std::string& path) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n void* voidstar = toAnything(schema, value);\n char* errmsg = NULL;\n mlc_save_voidstar(voidstar, schema, path.c_str(), &errmsg);\n shfree_cpp(voidstar);\n if (errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n}\n\n// Save a value to file in JSON format\ntemplate \nvoid _mlc_save_json(const T& value, const std::string& schema_str, const std::string& path) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n void* voidstar = toAnything(schema, value);\n char* errmsg = NULL;\n mlc_save_json(voidstar, schema, path.c_str(), &errmsg);\n shfree_cpp(voidstar);\n if (errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n}\n\n// Serialize a value to a JSON string\ntemplate \nstd::string _mlc_show(const T& value, const std::string& schema_str) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n void* voidstar = toAnything(schema, value);\n char* errmsg = NULL;\n char* json = mlc_show(voidstar, schema, &errmsg);\n shfree_cpp(voidstar);\n if (errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n std::string result(json);\n free(json);\n return result;\n}\n\n// Deserialize a JSON string to a typed value\n// Returns std::nullopt on parse failure\ntemplate \nstd::optional _mlc_read(const std::string& schema_str, const std::string& json_str) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n char* errmsg = NULL;\n void* voidstar = mlc_read(json_str.c_str(), schema, &errmsg);\n if (errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n if (voidstar == NULL) {\n return std::nullopt;\n }\n T* dummy = nullptr;\n T result = fromAnything(schema, voidstar, dummy);\n shfree_cpp(voidstar);\n return result;\n}\n\n// Load a value from file, auto-detecting format\n// Returns std::nullopt if file does not exist\ntemplate \nstd::optional _mlc_load(const std::string& schema_str, const std::string& path) {\n Schema* schema = get_cached_schema(schema_str.c_str());\n char* errmsg = NULL;\n void* voidstar = mlc_load(path.c_str(), schema, &errmsg);\n if (errmsg != NULL) {\n PROPAGATE_ERROR(errmsg)\n }\n if (voidstar == NULL) {\n return std::nullopt;\n }\n T* dummy = nullptr;\n T result = fromAnything(schema, voidstar, dummy);\n shfree_cpp(voidstar);\n return result;\n}\n\nuint8_t* foreign_call(const char* socket_filename, size_t mid, ...) {\n char* errmsg = NULL;\n va_list args;\n size_t nargs = 0;\n\n char socket_path[128];\n snprintf(socket_path, sizeof(socket_path), \"%s/%s\", g_tmpdir, socket_filename);\n\n // Count arguments (must be NULL-terminated)\n va_start(args, mid);\n while (va_arg(args, uint8_t*) != NULL) nargs++;\n va_end(args);\n\n // Allocate and populate args array\n const uint8_t** args_array = (const uint8_t**)malloc((nargs + 1) * sizeof(uint8_t*));\n if (!args_array) throw std::runtime_error(\"malloc failed in foreign_call\");\n\n va_start(args, mid);\n for (size_t i = 0; i < nargs; i++) {\n args_array[i] = va_arg(args, uint8_t*);\n }\n args_array[nargs] = NULL; // Sentinel\n va_end(args);\n\n // Original logic with variadic args converted to array\n uint8_t* packet = make_morloc_local_call_packet((uint32_t)mid, args_array, nargs, &errmsg);\n if (errmsg != NULL) {\n free(args_array);\n PROPAGATE_ERROR(errmsg)\n }\n\n pool_mark_busy();\n uint8_t* result = send_and_receive_over_socket(socket_path, packet, &errmsg);\n pool_mark_idle();\n\n free(packet);\n\n if (errmsg != NULL) {\n free(args_array);\n PROPAGATE_ERROR(errmsg)\n }\n\n // Incref the result's SHM so the callee's tracker flush won't destroy\n // data we may still need (e.g. forwarded result packets).\n {\n const morloc_packet_header_t* res_header = (const morloc_packet_header_t*)result;\n if (res_header->command.data.source == PACKET_SOURCE_RPTR) {\n size_t relptr = *(size_t*)(result + res_header->offset + sizeof(morloc_packet_header_t));\n char* resolve_err = NULL;\n void* res_voidstar = rel2abs(relptr, &resolve_err);\n if (resolve_err) { free(resolve_err); resolve_err = NULL; }\n if (res_voidstar) {\n char* incref_err = NULL;\n shincref((absptr_t)res_voidstar, &incref_err);\n if (incref_err) { free(incref_err); }\n _shm_tracker.push_back({(absptr_t)res_voidstar, nullptr});\n }\n }\n }\n\n free(args_array);\n return result;\n}\n\n\n\n// AUTO signatures statements start\n// <<>>\n// AUTO signatures statements end\n\n\n\n// AUTO manifolds statements start\n// <<>>\n// AUTO manifolds statements end\n\n\n\n// AUTO dispatch start\n// <<>>\n// AUTO dispatch end\n\n\n// Wrappers to adapt compiler-generated dispatch functions to pool_dispatch_fn_t.\n// These catch C++ exceptions so the C pool_main never sees them.\nstatic uint8_t* cpp_local_dispatch(uint32_t mid, const uint8_t** args,\n size_t nargs, void* ctx) {\n (void)nargs; (void)ctx;\n // Free SHM from previous dispatch (result packet consumed by caller)\n _flush_shm_tracker();\n try {\n return local_dispatch(mid, args);\n } catch (const std::exception& e) {\n return make_fail_packet(e.what());\n } catch (...) {\n return make_fail_packet(\"An unknown error occurred\");\n }\n}\n\nstatic uint8_t* cpp_remote_dispatch(uint32_t mid, const uint8_t** args,\n size_t nargs, void* ctx) {\n (void)nargs; (void)ctx;\n try {\n return remote_dispatch(mid, args);\n } catch (const std::exception& e) {\n return make_fail_packet(e.what());\n } catch (...) {\n return make_fail_packet(\"An unknown error occurred\");\n }\n}\n\n\nint main(int argc, char* argv[]) {\n // Line-buffer stderr so diagnostic output is not lost on pool shutdown.\n // stdout is left fully buffered for performance (genome-scale piping)\n // and flushed after each job by pool.c.\n setvbuf(stderr, NULL, _IOLBF, 0);\n\n // Health check: confirm binary links and print version\n if (argc == 2 && std::string(argv[1]) == \"--health\") {\n std::cout << \"{\\\"status\\\":\\\"ok\\\",\\\"version\\\":\\\"__MORLOC_VERSION__\\\"}\" << std::endl;\n return 0;\n }\n\n if (argc != 4) {\n std::cerr << \"Usage: \" << argv[0] << \" \\n\";\n return 1;\n }\n\n g_tmpdir = strdup(argv[2]);\n\n pool_config_t config = {};\n config.local_dispatch = cpp_local_dispatch;\n config.remote_dispatch = cpp_remote_dispatch;\n config.dispatch_ctx = NULL;\n config.concurrency = POOL_THREADS;\n config.initial_workers = 1;\n config.dynamic_scaling = true;\n\n int result = pool_main(argc, argv, &config);\n\n free(g_tmpdir);\n return result;\n}\n" }, { "path": "data/lang/julia/MorlocRuntime.jl", "content": "\"\"\"\n MorlocRuntime\n\nJulia runtime module for morloc. Provides IPC (daemon lifecycle, packet I/O),\nmsgpack-based serialization, and foreign call support.\n\nAll heavy lifting is done by libmorloc via the thin C bridge (libjuliamorloc.so).\nJulia handles only the msgpack <-> native type conversion using MsgPack.jl.\n\"\"\"\nmodule MorlocRuntime\n\nusing MsgPack\n\n# Path to the bridge shared library (set during morloc init)\nconst LIB_PATH = Ref{String}(\"\")\n\nfunction __init__()\n # Look for libjuliamorloc.so relative to this file, or in standard locations\n candidates = [\n joinpath(dirname(@__FILE__), \"libjuliamorloc.so\"),\n joinpath(dirname(@__FILE__), \"..\", \"lib\", \"libjuliamorloc.so\"),\n ]\n # Also check the morloc home lib directory\n morloc_home = get(ENV, \"MORLOC_HOME\", joinpath(homedir(), \".local\", \"share\", \"morloc\"))\n push!(candidates, joinpath(morloc_home, \"lib\", \"libjuliamorloc.so\"))\n\n for path in candidates\n if isfile(path)\n LIB_PATH[] = path\n return\n end\n end\n error(\"Cannot find libjuliamorloc.so. Run `morloc init` first.\")\nend\n\nlib() = LIB_PATH[]\n\n# -- Error handling --\n\nfunction check_error(context::String)\n msg = unsafe_string(ccall((:jlmorloc_last_error, lib()), Cstring, ()))\n if !isempty(msg)\n error(\"$context: $msg\")\n end\nend\n\n# -- Daemon lifecycle --\n\nfunction start_daemon(socket_path::String, tmpdir::String,\n shm_basename::String, shm_size::Integer)\n ptr = ccall((:jlmorloc_start_daemon, lib()), Ptr{Nothing},\n (Cstring, Cstring, Cstring, Csize_t),\n socket_path, tmpdir, shm_basename, UInt(shm_size))\n ptr == C_NULL && check_error(\"start_daemon\")\n return ptr\nend\n\nfunction close_daemon(daemon::Ptr{Nothing})\n ccall((:jlmorloc_close_daemon, lib()), Nothing, (Ptr{Nothing},), daemon)\nend\n\nfunction wait_for_client(daemon::Ptr{Nothing})\n fd = ccall((:jlmorloc_wait_for_client, lib()), Cint, (Ptr{Nothing},), daemon)\n fd < 0 && check_error(\"wait_for_client\")\n return fd\nend\n\n# -- Packet I/O --\n\nfunction stream_from_client(client_fd)\n out_size = Ref{Csize_t}(0)\n ptr = ccall((:jlmorloc_stream_from_client, lib()), Ptr{UInt8},\n (Cint, Ref{Csize_t}), Int32(client_fd), out_size)\n ptr == C_NULL && check_error(\"stream_from_client\")\n return ptr # opaque packet pointer\nend\n\nfunction send_packet_to_foreign_server(client_fd, packet::Ptr{UInt8})\n rc = ccall((:jlmorloc_send_packet, lib()), Cint,\n (Cint, Ptr{UInt8}), Int32(client_fd), packet)\n rc != 0 && check_error(\"send_packet\")\nend\n\nfunction close_socket(fd)\n ccall((:jlmorloc_close_socket, lib()), Nothing, (Cint,), Int32(fd))\nend\n\n# -- Packet classification --\n\nfunction is_ping(packet::Ptr{UInt8})\n ccall((:jlmorloc_is_ping, lib()), Cint, (Ptr{UInt8},), packet) != 0\nend\n\nfunction is_local_call(packet::Ptr{UInt8})\n ccall((:jlmorloc_is_local_call, lib()), Cint, (Ptr{UInt8},), packet) != 0\nend\n\nfunction is_remote_call(packet::Ptr{UInt8})\n ccall((:jlmorloc_is_remote_call, lib()), Cint, (Ptr{UInt8},), packet) != 0\nend\n\nfunction pong(packet::Ptr{UInt8})\n result = ccall((:jlmorloc_pong, lib()), Ptr{UInt8}, (Ptr{UInt8},), packet)\n result == C_NULL && check_error(\"pong\")\n return result\nend\n\n# -- Call packet parsing --\n\n\"\"\"\n read_morloc_call_packet(packet) -> (mid, args)\n\nParse a call packet into a manifold index and a vector of argument packets.\n\"\"\"\nfunction read_morloc_call_packet(packet::Ptr{UInt8})\n out_mid = Ref{UInt32}(0)\n out_nargs = Ref{Csize_t}(0)\n call_ptr = ccall((:jlmorloc_read_call, lib()), Ptr{Nothing},\n (Ptr{UInt8}, Ref{UInt32}, Ref{Csize_t}),\n packet, out_mid, out_nargs)\n call_ptr == C_NULL && check_error(\"read_call\")\n\n mid = Int(out_mid[])\n nargs = Int(out_nargs[])\n args = Vector{Ptr{UInt8}}(undef, nargs)\n for i in 1:nargs\n args[i] = ccall((:jlmorloc_call_arg, lib()), Ptr{UInt8},\n (Ptr{Nothing}, Csize_t), call_ptr, UInt(i - 1))\n end\n\n ccall((:jlmorloc_free_call, lib()), Nothing, (Ptr{Nothing},), call_ptr)\n return (mid, args)\nend\n\n# -- Msgpack bridge: serialize/deserialize --\n\n\"\"\"\nStrip the `` prefix from schema strings like `\"i4\"` -> `\"i4\"`.\n\"\"\"\nfunction strip_schema_prefix(schema_str::String)\n if !isempty(schema_str) && schema_str[1] == '<'\n i = findfirst('>', schema_str)\n if i !== nothing\n return schema_str[i+1:end]\n end\n end\n return schema_str\nend\n\n\"\"\"\n put_value(value, schema_str) -> Ptr{UInt8}\n\nSerialize a Julia value to a morloc data packet via msgpack.\n\"\"\"\nfunction put_value(value, schema_str::String)\n schema = strip_schema_prefix(schema_str)\n mpk = MsgPack.pack(to_msgpack(value, schema))\n pkt = ccall((:jlmorloc_pack, lib()), Ptr{UInt8},\n (Ptr{UInt8}, Csize_t, Cstring),\n mpk, length(mpk), schema)\n pkt == C_NULL && check_error(\"pack\")\n return pkt\nend\n\n\"\"\"\n get_value(packet, schema_str) -> Julia value\n\nDeserialize a morloc data packet to a Julia value via msgpack.\n\"\"\"\nfunction get_value(packet::Ptr{UInt8}, schema_str::String)\n schema = strip_schema_prefix(schema_str)\n out_size = Ref{Csize_t}(0)\n mpk_ptr = ccall((:jlmorloc_unpack, lib()), Ptr{UInt8},\n (Ptr{UInt8}, Cstring, Ref{Csize_t}),\n packet, schema, out_size)\n mpk_ptr == C_NULL && check_error(\"unpack\")\n\n mpk_bytes = unsafe_wrap(Array, mpk_ptr, out_size[]; own=true)\n raw = MsgPack.unpack(mpk_bytes)\n return from_msgpack(raw, schema)\nend\n\n# -- Error packet --\n\nfunction make_fail_packet(msg::String)\n ccall((:jlmorloc_make_fail_packet, lib()), Ptr{UInt8}, (Cstring,), msg)\nend\n\n# -- Foreign call --\n\n\"\"\"\n foreign_call(tmpdir, socket_name, mid, args) -> Ptr{UInt8}\n\nCall another pool (cross-language IPC). args is a vector of packet pointers.\n\"\"\"\nfunction foreign_call(tmpdir::String, socket_name::String,\n mid::Integer, args::Vector{Ptr{UInt8}})\n nargs = length(args)\n result = ccall((:jlmorloc_foreign_call, lib()), Ptr{UInt8},\n (Cstring, Cstring, UInt32, Ptr{Ptr{UInt8}}, Csize_t),\n tmpdir, socket_name, UInt32(mid), args, UInt(nargs))\n result == C_NULL && check_error(\"foreign_call\")\n return result\nend\n\n# -- Type conversion helpers --\n\n# Schema string format:\n# \"b\" = bool, \"i4\" = int32, \"i8\" = int64, \"f8\" = float64, \"s\" = string\n# \"ai4\" = array of int32, \"t(i4f8s)\" = tuple\n# \"m{name:s,age:i4}\" = record\n\n\"\"\"\nConvert a Julia value to a msgpack-friendly representation based on schema.\nMsgPack.jl handles most types natively, but we need to ensure correct types\nfor the schema (e.g., Int32 vs Int64).\n\"\"\"\nfunction to_msgpack(value, schema::String)\n if startswith(schema, \"a\")\n elem_schema = schema[2:end]\n return [to_msgpack(v, elem_schema) for v in value]\n elseif startswith(schema, \"t(\")\n inner = schema[3:end-1]\n schemas = split_tuple_schema(inner)\n return [to_msgpack(value[i], schemas[i]) for i in 1:length(schemas)]\n elseif startswith(schema, \"m{\")\n inner = schema[3:end-1]\n fields = split_record_schema(inner)\n return Dict(k => to_msgpack(value[k], s) for (k, s) in fields)\n else\n return to_msgpack_scalar(value, schema)\n end\nend\n\nfunction to_msgpack_scalar(value, schema::String)\n if schema == \"b\"\n return Bool(value)\n elseif schema == \"i4\"\n return Int32(value)\n elseif schema == \"i8\"\n return Int64(value)\n elseif schema == \"u4\"\n return UInt32(value)\n elseif schema == \"u8\"\n return UInt64(value)\n elseif schema == \"f4\"\n return Float32(value)\n elseif schema == \"f8\"\n return Float64(value)\n elseif schema == \"s\"\n return String(value)\n elseif schema == \"u\"\n return nothing\n else\n return value\n end\nend\n\n\"\"\"\nConvert a raw msgpack value to a Julia type based on schema.\n\"\"\"\nfunction from_msgpack(raw, schema::String)\n if startswith(schema, \"a\")\n elem_schema = schema[2:end]\n return [from_msgpack(v, elem_schema) for v in raw]\n elseif startswith(schema, \"t(\")\n inner = schema[3:end-1]\n schemas = split_tuple_schema(inner)\n return Tuple(from_msgpack(raw[i], schemas[i]) for i in 1:length(schemas))\n elseif startswith(schema, \"m{\")\n inner = schema[3:end-1]\n fields = split_record_schema(inner)\n return Dict(k => from_msgpack(raw[k], s) for (k, s) in fields)\n else\n return from_msgpack_scalar(raw, schema)\n end\nend\n\nfunction from_msgpack_scalar(raw, schema::String)\n if schema == \"b\"\n return Bool(raw)\n elseif schema == \"i4\"\n return Int32(raw)\n elseif schema == \"i8\"\n return Int64(raw)\n elseif schema == \"u4\"\n return UInt32(raw)\n elseif schema == \"u8\"\n return UInt64(raw)\n elseif schema == \"f4\"\n return Float32(raw)\n elseif schema == \"f8\"\n return Float64(raw)\n elseif schema == \"s\"\n return String(raw)\n elseif schema == \"u\"\n return nothing\n else\n return raw\n end\nend\n\n# -- Schema parsing helpers --\n\nfunction split_tuple_schema(inner::String)\n schemas = String[]\n i = 1\n while i <= length(inner)\n s, i = parse_one_schema(inner, i)\n push!(schemas, s)\n end\n return schemas\nend\n\nfunction split_record_schema(inner::String)\n fields = Pair{String,String}[]\n i = 1\n while i <= length(inner)\n # parse field name\n colon = findnext(':', inner, i)\n name = inner[i:colon-1]\n i = colon + 1\n # parse field schema\n s, i = parse_one_schema(inner, i)\n push!(fields, name => s)\n if i <= length(inner) && inner[i] == ','\n i += 1\n end\n end\n return fields\nend\n\nfunction parse_one_schema(s::String, i::Int)\n if s[i] == 'a'\n inner, next_i = parse_one_schema(s, i + 1)\n return \"a\" * inner, next_i\n elseif s[i] == 't'\n # find matching ')'\n depth = 0\n j = i + 1\n while j <= length(s)\n if s[j] == '('; depth += 1; end\n if s[j] == ')'; depth -= 1; if depth == 0; break; end; end\n j += 1\n end\n return s[i:j], j + 1\n elseif s[i] == 'm'\n depth = 0\n j = i + 1\n while j <= length(s)\n if s[j] == '{'; depth += 1; end\n if s[j] == '}'; depth -= 1; if depth == 0; break; end; end\n j += 1\n end\n return s[i:j], j + 1\n elseif s[i] in ('i', 'u', 'f')\n # numeric: i4, i8, u4, u8, f4, f8\n return s[i:i+1], i + 2\n elseif s[i] == 's'\n return \"s\", i + 1\n elseif s[i] == 'b'\n return \"b\", i + 1\n else\n error(\"Unknown schema character: $(s[i]) at position $i in '$s'\")\n end\nend\n\nend # module\n" }, { "path": "data/lang/julia/init.sh", "content": "#!/bin/bash\nset -e\n\nMORLOC_HOME=\"$1\"\nBUILD_DIR=\"$2\"\nSANITIZE_FLAGS=\"$3\"\nINCLUDE_DIR=\"$MORLOC_HOME/include\"\nLIB_DIR=\"$MORLOC_HOME/lib\"\nLANG_DIR=\"$MORLOC_HOME/lang/julia\"\n\nmkdir -p \"$LANG_DIR\"\n\n# Install language descriptor and runtime files\ncp \"$BUILD_DIR/lang.yaml\" \"$LANG_DIR/\"\ncp \"$BUILD_DIR/pool.jl\" \"$LANG_DIR/\"\ncp \"$BUILD_DIR/MorlocRuntime.jl\" \"$LANG_DIR/\"\n\n# Compile juliabridge.c -> libjuliamorloc.so\ngcc -shared -fPIC -O2 $SANITIZE_FLAGS -I\"$INCLUDE_DIR\" -o \"$LIB_DIR/libjuliamorloc.so\" \\\n \"$BUILD_DIR/juliabridge.c\" -L\"$LIB_DIR\" -Wl,-rpath,\"$LIB_DIR\" -lmorloc -lpthread\n" }, { "path": "data/lang/julia/juliabridge.c", "content": "/* juliabridge.c -- Thin C bridge between Julia and libmorloc.\n *\n * Compiled to libjuliamorloc.so, called from Julia via ccall.\n * Wraps libmorloc functions that use opaque structs (language_daemon_t,\n * morloc_call_t, Schema) or the ERRMSG pattern into simple pointer/int\n * interfaces that Julia's FFI can handle directly.\n */\n\n#include \"morloc.h\"\n#include \n#include \n#include \n\n/* Thread-local error message buffer */\nstatic __thread char jl_errbuf[4096];\nstatic __thread char* jl_errmsg = NULL;\n\nstatic void clear_err(void) {\n jl_errmsg = NULL;\n jl_errbuf[0] = '\\0';\n}\n\n/* Get the last error message (returns \"\" if none). */\nconst char* jlmorloc_last_error(void) {\n return jl_errmsg ? jl_errmsg : \"\";\n}\n\n/* -- Daemon lifecycle -- */\n\nvoid* jlmorloc_start_daemon(const char* socket_path, const char* tmpdir,\n const char* shm_basename, size_t shm_size) {\n clear_err();\n language_daemon_t* d = start_daemon(socket_path, tmpdir, shm_basename,\n shm_size, &jl_errmsg);\n if (!d && jl_errmsg) {\n snprintf(jl_errbuf, sizeof(jl_errbuf), \"%s\", jl_errmsg);\n jl_errmsg = jl_errbuf;\n }\n return (void*)d;\n}\n\nvoid jlmorloc_close_daemon(void* daemon) {\n language_daemon_t* d = (language_daemon_t*)daemon;\n close_daemon(&d);\n}\n\nint jlmorloc_wait_for_client(void* daemon) {\n clear_err();\n return wait_for_client((language_daemon_t*)daemon, &jl_errmsg);\n}\n\n/* -- Packet I/O -- */\n\n/* Returns a pointer to the packet bytes. Caller must NOT free this\n * directly -- it lives in shared memory or was allocated by libmorloc. */\nuint8_t* jlmorloc_stream_from_client(int client_fd, size_t* out_size) {\n clear_err();\n uint8_t* pkt = stream_from_client(client_fd, &jl_errmsg);\n if (pkt && out_size) {\n /* Packet size is in the first 4 bytes (little-endian uint32) */\n uint32_t sz;\n memcpy(&sz, pkt, sizeof(sz));\n *out_size = (size_t)sz;\n }\n return pkt;\n}\n\nint jlmorloc_send_packet(int client_fd, uint8_t* packet) {\n clear_err();\n size_t sent = send_packet_to_foreign_server(client_fd, packet, &jl_errmsg);\n return sent > 0 ? 0 : -1;\n}\n\nvoid jlmorloc_close_socket(int fd) {\n close_socket(fd);\n}\n\n/* -- Packet classification -- */\n\nint jlmorloc_is_ping(const uint8_t* packet) {\n clear_err();\n return packet_is_ping(packet, &jl_errmsg) ? 1 : 0;\n}\n\nint jlmorloc_is_local_call(const uint8_t* packet) {\n clear_err();\n return packet_is_local_call(packet, &jl_errmsg) ? 1 : 0;\n}\n\nint jlmorloc_is_remote_call(const uint8_t* packet) {\n clear_err();\n return packet_is_remote_call(packet, &jl_errmsg) ? 1 : 0;\n}\n\nuint8_t* jlmorloc_pong(const uint8_t* packet) {\n clear_err();\n return return_ping(packet, &jl_errmsg);\n}\n\n/* -- Call packet parsing -- */\n\n/* Parse a call packet. Returns the manifold index via out_mid,\n * the number of arguments via out_nargs, and a pointer to the\n * morloc_call_t (which the caller must free via jlmorloc_free_call). */\nvoid* jlmorloc_read_call(const uint8_t* packet, uint32_t* out_mid,\n size_t* out_nargs) {\n clear_err();\n morloc_call_t* call = read_morloc_call_packet(packet, &jl_errmsg);\n if (!call) return NULL;\n *out_mid = call->midx;\n *out_nargs = call->nargs;\n return (void*)call;\n}\n\n/* Get the i-th argument packet from a parsed call. */\nuint8_t* jlmorloc_call_arg(void* call_ptr, size_t i) {\n morloc_call_t* call = (morloc_call_t*)call_ptr;\n if (i >= call->nargs) return NULL;\n return call->args[i];\n}\n\nvoid jlmorloc_free_call(void* call_ptr) {\n if (call_ptr) free_morloc_call((morloc_call_t*)call_ptr);\n}\n\n/* -- Msgpack bridge -- */\n\n/* Convert msgpack bytes + schema string -> morloc data packet.\n * The schema_str is a compact type descriptor like \"i4\", \"ai4\", \"m{x:f8}\". */\nuint8_t* jlmorloc_pack(const char* mpk, size_t mpk_size,\n const char* schema_str) {\n clear_err();\n Schema* schema = parse_schema(schema_str, &jl_errmsg);\n if (!schema) return NULL;\n uint8_t* pkt = make_data_packet_from_mpk(mpk, mpk_size, schema);\n free_schema(schema);\n return pkt;\n}\n\n/* Convert a morloc data packet -> msgpack bytes.\n * Returns a malloc'd buffer; caller must free it. */\nchar* jlmorloc_unpack(const uint8_t* packet, const char* schema_str,\n size_t* out_size) {\n clear_err();\n Schema* schema = parse_schema(schema_str, &jl_errmsg);\n if (!schema) return NULL;\n char* mpk = NULL;\n size_t mpk_size = 0;\n int ok = get_data_packet_as_mpk(packet, schema, &mpk, &mpk_size, &jl_errmsg);\n free_schema(schema);\n if (!ok) return NULL;\n *out_size = mpk_size;\n return mpk;\n}\n\n/* -- Error packet -- */\n\nuint8_t* jlmorloc_make_fail_packet(const char* msg) {\n return make_fail_packet(msg);\n}\n\n/* -- Foreign call (cross-pool IPC) -- */\n\nuint8_t* jlmorloc_foreign_call(const char* tmpdir, const char* socket_name,\n uint32_t mid, uint8_t** arg_packets,\n size_t nargs) {\n clear_err();\n /* Build the call packet */\n uint8_t* call_pkt = make_morloc_local_call_packet(\n mid, (const uint8_t**)arg_packets, nargs, &jl_errmsg);\n if (!call_pkt) return NULL;\n\n /* Build the socket path */\n size_t pathlen = strlen(tmpdir) + 1 + strlen(socket_name) + 1;\n char* socket_path = (char*)malloc(pathlen);\n snprintf(socket_path, pathlen, \"%s/%s\", tmpdir, socket_name);\n\n /* Send and receive */\n uint8_t* result = send_and_receive_over_socket(socket_path, call_pkt,\n &jl_errmsg);\n free(socket_path);\n free(call_pkt);\n return result;\n}\n\n/* -- Shared memory init (needed before daemon start in some cases) -- */\n\nint jlmorloc_shinit(const char* basename, int volume, size_t size) {\n clear_err();\n return shinit(basename, volume, size, &jl_errmsg) ? 0 : -1;\n}\n\nvoid jlmorloc_set_fallback_dir(const char* dir) {\n shm_set_fallback_dir(dir);\n}\n" }, { "path": "data/lang/julia/lang.yaml", "content": "# Julia language descriptor for morloc compiler\n# Metadata fields (read by LangRegistry) + descriptor fields (read by generic translator)\n\n# Identity and metadata\nname: jl\nextension: jl\naliases: [\"julia\"]\nis_compiled: false\nrun_command: [\"julia\"]\nserial_type: \"bytes\"\ncost: 5\n\n# Descriptor identity (kept for compatibility with generic translator)\nldName: julia\nldExtension: jl\n\n# Literals\nldBoolTrue: \"true\"\nldBoolFalse: \"false\"\nldNullLiteral: \"nothing\"\n\n# Constructors\nldListStyle: bracket\nldTupleConstructor: \"\"\nldRecordConstructor: \"Dict\"\nldRecordSeparator: \"=>\"\n\n# Access styles\nldIndexStyle: one_bracket\nldKeyAccess: bracket\nldFieldAccess: dot\n\n# Serialize/deserialize function names (from juliamorloc.jl)\nldSerializeFn: \"MorlocRuntime.put_value\"\nldDeserializeFn: \"MorlocRuntime.get_value\"\nldIntrinsicPrefix: \"MorlocRuntime.\"\n\n# Foreign call\nldForeignCallFn: \"MorlocRuntime.foreign_call\"\nldForeignCallIntSuffix: \"\"\n\n# Import syntax\nldQualifiedImports: false\nldIncludeRelToFile: true\n\n# Template fields\nldAssignOp: \"=\"\nldLambdaTemplate: \"({{args}}) -> {{body}}\"\nldDoBlockExpr: \"(() -> {{expr}})\"\nldDoBlockBlock: \"\"\nldPartialTemplate: \"({{bound_args}}) -> {{fn}}({{all_args}})\"\nldImportTemplate: \"include(\\\"{{path}}\\\")\"\nldSocketPathTemplate: \"joinpath(global_state[\\\"tmpdir\\\"], {{socket}})\"\nldResourcePackTemplate: \"[{{mem}}, {{time}}, {{cpus}}, {{gpus}}]\"\nldReturnTemplate: \"return({{expr}})\"\nldFuncDefHeader: \"function {{name}}({{args}})\"\nldBlockStyle: end_keyword\nldBlockEnd: \"end\"\nldErrorWrapOpen: \"\"\nldErrorWrapClose: []\nldPatternStyle: concat_call\nldConcatFn: \"string\"\nldQuoteTerminator: '\"'\nldQuoteTerminatorEsc: '\\\"'\nldMapStyle: list_comprehension\nldDispatchLocalHeader: \"dispatch = Dict(\"\nldDispatchLocalEntry: \" {{mid}} => {{name}},\"\nldDispatchLocalFooter: \")\"\nldDispatchRemoteHeader: \"remote_dispatch = Dict(\"\nldDispatchRemoteEntry: \" {{mid}} => {{name}}_remote,\"\nldDispatchRemoteFooter: \")\"\n\n# Pool template (loaded from pool.jl at runtime, left empty here)\nldPoolTemplate: \"\"\nldBreakMarker: \"# <<>>\"\nldCommentMarker: \"#\"\n" }, { "path": "data/lang/julia/pool.jl", "content": "# Morloc Julia pool template\n# Single-threaded daemon: accepts one connection at a time.\n\n# Add morloc runtime to load path\nconst MORLOC_HOME = get(ENV, \"MORLOC_HOME\", joinpath(homedir(), \".local\", \"share\", \"morloc\"))\npush!(LOAD_PATH, joinpath(MORLOC_HOME, \"lang\", \"julia\"))\n\n# Global state accessible to manifolds (e.g., tmpdir for foreign calls)\nglobal_state = Dict{String,String}()\n\n# <<>>\n\nusing MorlocRuntime\n\n# <<>>\n\n# <<>>\n\nfunction run_job(client_fd)\n try\n client_data = MorlocRuntime.stream_from_client(client_fd)\n\n if MorlocRuntime.is_local_call(client_data)\n (mid, args) = MorlocRuntime.read_morloc_call_packet(client_data)\n try\n result = dispatch[mid](args...)\n catch e\n result = MorlocRuntime.make_fail_packet(string(e))\n end\n\n elseif MorlocRuntime.is_remote_call(client_data)\n (mid, args) = MorlocRuntime.read_morloc_call_packet(client_data)\n try\n result = remote_dispatch[mid](args...)\n catch e\n result = MorlocRuntime.make_fail_packet(string(e))\n end\n\n elseif MorlocRuntime.is_ping(client_data)\n result = MorlocRuntime.pong(client_data)\n\n else\n error(\"Expected a ping or call type packet\")\n end\n\n MorlocRuntime.send_packet_to_foreign_server(client_fd, result)\n\n catch e\n # Best-effort: wrap the error in a fail packet and send it back so the\n # caller gets a structured error instead of hanging on a closed socket.\n # Includes the full backtrace so context propagates through the stack.\n msg = sprint(showerror, e, catch_backtrace())\n try\n result = MorlocRuntime.make_fail_packet(msg)\n MorlocRuntime.send_packet_to_foreign_server(client_fd, result)\n catch\n # Client may already be gone (timed-out ping, broken pipe); ignore.\n end\n @error \"job failed\" exception=(e, catch_backtrace())\n finally\n MorlocRuntime.close_socket(client_fd)\n end\nend\n\nfunction main()\n socket_path = ARGS[1]\n tmpdir = ARGS[2]\n shm_basename = ARGS[3]\n\n global_state[\"tmpdir\"] = tmpdir\n\n daemon = MorlocRuntime.start_daemon(socket_path, tmpdir, shm_basename, 0xffff)\n\n # Simple signal handling\n running = Ref(true)\n\n @async begin\n try\n while running[]\n sleep(0.01)\n end\n catch\n end\n end\n\n try\n while running[]\n client_fd = MorlocRuntime.wait_for_client(daemon)\n if client_fd > 0\n run_job(client_fd)\n end\n end\n catch e\n if !(e isa InterruptException)\n @error \"Pool error\" exception=(e, catch_backtrace())\n end\n finally\n MorlocRuntime.close_daemon(daemon)\n end\nend\n\nmain()\n" }, { "path": "data/lang/languages.yaml", "content": "# Pairwise language costs for the morloc optimizer\n#\n# Same-language function overhead (intra-language call cost)\nsame_language_costs:\n c: 1\n cpp: 1\n py: 10\n r: 20\n\n# Cost of calling INTO a language from a different language (IPC overhead)\ncross_language_costs:\n c: 1001\n cpp: 1000\n py: 10000\n r: 40000\n\n# Special optimized pairs (from -> to) that bypass normal IPC\noptimized_pairs:\n - from: cpp\n to: c\n cost: 1\n\n# Defaults for unknown/plugin languages\ndefault_same_language: 10\ndefault_cross_language: 10000\n" }, { "path": "data/lang/py/Makefile", "content": "all:\n\tpython3 setup.py build_ext --inplace\n\tcp -fs pymorloc.cpython* pymorloc\n" }, { "path": "data/lang/py/init.sh", "content": "#!/bin/bash\nset -e\n\nexport MORLOC_HOME=\"$1\"\nBUILD_DIR=\"$2\"\nSANITIZE_FLAGS=\"$3\"\nOPT_DIR=\"$MORLOC_HOME/opt\"\n\n# Clean stale build artifacts\nrm -f \"$OPT_DIR\"/pymorloc.cpython* \"$OPT_DIR/pymorloc\"\nrm -rf \"$OPT_DIR/build\"\n\n# Copy files to opt dir\ncp \"$BUILD_DIR/pymorloc.c\" \"$OPT_DIR/\"\ncp \"$BUILD_DIR/setup.py\" \"$OPT_DIR/\"\ncp \"$BUILD_DIR/Makefile\" \"$OPT_DIR/\"\n\n# Build pymorloc extension\nexport CFLAGS=\"$SANITIZE_FLAGS\"\nmake -C \"$OPT_DIR\" -f Makefile\n" }, { "path": "data/lang/py/lang.yaml", "content": "# Python language descriptor for morloc compiler\n# Metadata fields (read by LangRegistry) + descriptor fields (read by generic translator)\n\n# Identity and metadata\nname: py\nextension: py\naliases: [\"python\", \"python3\"]\nis_compiled: false\nrun_command: [\"python3\"]\nserial_type: \"str\"\ncost: 3\npreamble:\n - 'sys.path = [os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), \"..\", \"..\")), os.path.expanduser(\".\"), os.path.expanduser(\"{{opt}}\"), os.path.expanduser(\"{{lib}}\")] + sys.path'\n - \"import importlib\"\n - \"import pymorloc as morloc\"\n\n# Literals\nldBoolTrue: \"True\"\nldBoolFalse: \"False\"\nldNullLiteral: \"None\"\n\n# Constructors\nldListStyle: bracket\nldTupleConstructor: \"\"\nldRecordConstructor: \"OrderedDict\"\nldRecordSeparator: \"=\"\n\n# Access styles\nldIndexStyle: zero_bracket\nldKeyAccess: \"bracket\"\nldFieldAccess: dot\n\n# Serialize/deserialize\nldSerializeFn: \"morloc.put_value\"\nldDeserializeFn: \"morloc.get_value\"\nldIntrinsicPrefix: \"morloc.\"\n\n# Foreign call\nldForeignCallFn: \"morloc.foreign_call\"\nldForeignCallIntSuffix: \"\"\n\n# Remote call\nldRemoteCallFn: \"morloc.remote_call\"\n\n# Record handling\nldDictStyleRecords: true\nldQuoteRecordKeys: false\n\n# Import syntax\nldQualifiedImports: true\nldIncludeRelToFile: false\n\n# Template fields\nldAssignOp: \"=\"\nldLambdaTemplate: \"lambda {{args}}: {{body}}\"\nldDoBlockExpr: \"(lambda: {{expr}})\"\nldDoBlockBlock: \"\"\nldPartialTemplate: \"functools.partial({{fn_with_context}})\"\nldImportTemplate: \"{{namespace}} = importlib.import_module(\\\"{{module_path}}\\\")\"\nldSocketPathTemplate: \"os.path.join(global_state[\\\"tmpdir\\\"], {{socket}})\"\nldResourcePackTemplate: \"struct.pack('iiii', {{mem}}, {{time}}, {{cpus}}, {{gpus}})\"\nldReturnTemplate: \"return({{expr}})\"\nldFuncDefHeader: \"def {{name}}({{args}}):\"\nldBlockStyle: indent\nldBlockEnd: \"\"\nldErrorWrapOpen: \"try:\"\nldErrorWrapClose:\n - \"except Exception as e:\"\n - \" raise RuntimeError(f\\\"Error (pool daemon in {{name}}):\\\\n{e!s}\\\")\"\nldPatternStyle: fstring\nldQuoteTerminator: '\"\"\"'\nldQuoteTerminatorEsc: '\\\"\\\"\\\"'\nldMapStyle: loop_append\nldDispatchLocalHeader: \"dispatch = {\"\nldDispatchLocalEntry: \" {{mid}}: {{name}},\"\nldDispatchLocalFooter: \"}\"\nldDispatchRemoteHeader: \"remote_dispatch = {\"\nldDispatchRemoteEntry: \" {{mid}}: {{name}}_remote,\"\nldDispatchRemoteFooter: \"}\"\n\n# Pool template (loaded from pool.py, left empty here)\nldPoolTemplate: \"\"\nldBreakMarker: \"# <<>>\"\nldCommentMarker: \"#\"\n" }, { "path": "data/lang/py/pool.py", "content": "import signal\nimport sys\nimport select\nimport os # required for setting path to morloc dependencies\nimport time\nimport copy\nimport array\nimport struct\nimport socket as _socket\nfrom collections import OrderedDict\nfrom multiprocessing import Process, Value, RawValue\nimport ctypes\nimport functools\n\n\n# Global variables for clean signal handling\ndaemon = None\nworkers = []\nglobal_state = dict()\n_shutdown_wakeup_fd = -1\n\n# AUTO include sources start\n# <<>>\n# AUTO include sources end\n\n# Dynamic worker spawning: monkey-patch foreign_call to track busy workers.\n# Workers atomically increment busy_count before a foreign_call and decrement\n# after. When busy_count reaches total_workers, a byte is written to a wake-up\n# pipe to tell the main process to spawn a new worker.\n_original_foreign_call = morloc.foreign_call\n_busy_ref = None\n_total_ref = None\n_wakeup_fd = -1\n\ndef _init_worker_tracking(busy, total, wakeup_fd):\n global _busy_ref, _total_ref, _wakeup_fd\n _busy_ref = busy\n _total_ref = total\n _wakeup_fd = wakeup_fd\n morloc.foreign_call = _tracked_foreign_call\n\ndef _tracked_foreign_call(*args):\n prev = _busy_ref.value\n _busy_ref.value = prev + 1\n if prev + 1 >= _total_ref.value and _wakeup_fd >= 0:\n try:\n os.write(_wakeup_fd, b'!')\n except OSError:\n pass\n try:\n return _original_foreign_call(*args)\n finally:\n _busy_ref.value -= 1\n\n# AUTO include manifolds start\n# <<>>\n# AUTO include manifolds end\n\n\n# AUTO include dispatch start\n# <<>>\n# AUTO include dispatch end\n\ndef run_job(client_fd: int) -> None:\n try:\n # Free SHM from previous dispatch result (consumed by caller)\n morloc.flush_shm_tracker()\n client_data = morloc.stream_from_client(client_fd)\n\n if(morloc.is_local_call(client_data)):\n (mid, args) = morloc.read_morloc_call_packet(client_data)\n\n try:\n result = dispatch[mid](*args)\n except Exception as e:\n result = morloc.make_fail_packet(str(e))\n\n elif(morloc.is_remote_call(client_data)):\n (mid, args) = morloc.read_morloc_call_packet(client_data)\n\n try:\n result = remote_dispatch[mid](*args)\n except Exception as e:\n result = morloc.make_fail_packet(str(e))\n\n elif(morloc.is_ping(client_data)):\n result = morloc.pong(client_data)\n\n else:\n raise ValueError(\"Expected a ping or call type packet\")\n\n # Flush stdout BEFORE sending the result back. The nexus prints its\n # own output (the return value) right after receiving this response.\n # Both processes share the same stdout fd, so if we flush after sending,\n # the nexus can print first, causing out-of-order output.\n sys.stdout.flush()\n\n morloc.send_packet_to_foreign_server(client_fd, result)\n\n except Exception as e:\n # Try to send a fail packet back to the caller before giving up.\n # This may fail (e.g., broken pipe from a timed-out ping), which is OK.\n try:\n result = morloc.make_fail_packet(str(e))\n morloc.send_packet_to_foreign_server(client_fd, result)\n except Exception:\n pass\n print(f\"job failed: {e!s}\", file=sys.stderr)\n finally:\n # Safety-net flush for any output from error handling paths\n sys.stdout.flush()\n # close child copy\n morloc.close_socket(client_fd)\n\n\ndef _send_fd(sock, fd):\n \"\"\"Send a file descriptor over a Unix domain socket.\"\"\"\n sock.sendmsg([b'\\x00'],\n [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS,\n array.array('i', [fd]))])\n\ndef _recv_fd(sock):\n \"\"\"Receive a file descriptor from a Unix domain socket.\"\"\"\n msg, ancdata, flags, addr = sock.recvmsg(1, _socket.CMSG_SPACE(4))\n if not msg and not ancdata:\n raise EOFError(\"Connection closed\")\n for cmsg_level, cmsg_type, cmsg_data in ancdata:\n if (cmsg_level == _socket.SOL_SOCKET and\n cmsg_type == _socket.SCM_RIGHTS):\n a = array.array('i')\n a.frombytes(cmsg_data[:4])\n return a[0]\n raise RuntimeError(\"No fd received in ancillary data\")\n\n\nWORKER_IDLE_TIMEOUT = 5.0 # seconds before an idle worker exits\n\ndef worker_process(job_fd, tmpdir, shm_basename, shutdown_flag, busy_count, total_workers, wakeup_w):\n # Reset signal handlers inherited from main. If user code inside run_job\n # calls multiprocessing.Pool (or anything else that forks and later\n # SIGTERMs its own children), those grandchildren would otherwise inherit\n # main's signal_handler and flip the shared shutdown_flag, causing main\n # to SIGKILL this worker mid-response. See the multiprocessing-py-1 bug.\n signal.signal(signal.SIGTERM, signal.SIG_DFL)\n signal.signal(signal.SIGINT, signal.SIG_DFL)\n morloc.set_fallback_dir(tmpdir)\n morloc.shinit(shm_basename, 0, 0xffff)\n _init_worker_tracking(busy_count, total_workers, wakeup_w)\n sock = _socket.fromfd(job_fd, _socket.AF_UNIX, _socket.SOCK_STREAM)\n os.close(job_fd) # sock owns a dup'd copy\n last_activity = time.monotonic()\n try:\n while not shutdown_flag.value:\n rlist, _, _ = select.select([sock.fileno()], [], [], 0.01)\n if shutdown_flag.value:\n break\n if rlist:\n try:\n client_fd = _recv_fd(sock)\n run_job(client_fd)\n last_activity = time.monotonic()\n except (EOFError, OSError):\n break\n elif total_workers.value > 1 and time.monotonic() - last_activity > WORKER_IDLE_TIMEOUT:\n break\n except BaseException as e:\n # Catch-all for errors that escape run_job's own exception handling:\n # MemoryError, KeyboardInterrupt, SystemExit, or bugs in the worker\n # loop itself. Without this, the worker dies silently and the nexus\n # only sees \"failed to read response header\" with no indication of\n # what went wrong in the pool.\n #\n # Race condition: the nexus detects the broken socket and may start\n # its clean_exit tear-down (SIGTERM -> SIGKILL) while this print is\n # still buffered. We flush immediately to maximize the chance the\n # message reaches the terminal before we are killed. stderr is\n # line-buffered (set in __main__), but the flush is a safety net for\n # edge cases (redirected stderr, forked-process buffer state).\n import traceback\n print(f\"morloc pool worker fatal error: {e!s}\", file=sys.stderr)\n traceback.print_exc(file=sys.stderr)\n sys.stderr.flush()\n finally:\n sock.close()\n\n\ndef signal_handler(sig, frame):\n global daemon\n # Ignore further SIGTERM/SIGINT during cleanup. Python processes pending\n # signals between bytecodes, including while another signal handler is\n # running, so a second SIGTERM arriving mid-cleanup would otherwise\n # re-enter this handler and double-free the daemon pointer.\n try:\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n signal.signal(signal.SIGINT, signal.SIG_IGN)\n except Exception:\n pass\n shutdown_flag.value = True\n if _shutdown_wakeup_fd >= 0:\n try:\n os.write(_shutdown_wakeup_fd, b'!')\n except OSError:\n pass\n # Capture the daemon pointer into a local and clear the global BEFORE\n # invoking close_daemon. If a pending signal still slips through and\n # re-enters this handler, it will see daemon=None and skip the free.\n d = daemon\n daemon = None\n if d is not None:\n morloc.close_daemon(d)\n\n\ndef client_listener(job_fd, socket_path, tmpdir, shm_basename, shutdown_flag):\n global daemon\n daemon = morloc.start_daemon(socket_path, tmpdir, shm_basename, 0xffff)\n sock = _socket.fromfd(job_fd, _socket.AF_UNIX, _socket.SOCK_STREAM)\n os.close(job_fd) # sock owns a dup'd copy\n\n while not shutdown_flag.value:\n try:\n client_fd = morloc.wait_for_client(daemon)\n except Exception as e:\n print(f\"In python daemon, failed to connect to client: {e!s}\", file=sys.stderr)\n continue\n\n if client_fd > 0:\n try:\n _send_fd(sock, client_fd)\n except Exception as e:\n print(f\"In python daemon, failed to start worker: {e!s}\", file=sys.stderr)\n finally:\n morloc.close_socket(client_fd)\n sock.close()\n\n\n\nif __name__ == \"__main__\":\n # Line-buffer stderr so diagnostic output is not lost when pool is killed.\n # stdout is left fully buffered for performance (genome-scale piping) and\n # flushed explicitly after each job and during shutdown.\n sys.stderr.reconfigure(line_buffering=True)\n\n shutdown_flag = Value('b', False) # Shared flag\n\n signal.signal(signal.SIGINT, signal_handler)\n signal.signal(signal.SIGTERM, signal_handler)\n\n # Health check: confirm imports loaded and print version\n if len(sys.argv) > 1 and sys.argv[1] == \"--health\":\n sys.stdout.write('{\"status\":\"ok\",\"version\":\"__MORLOC_VERSION__\"}\\n')\n sys.exit(0)\n\n # Process arguments passed from the nexus\n try:\n socket_path = sys.argv[1]\n tmpdir = sys.argv[2]\n shm_basename = sys.argv[3]\n except IndexError:\n print(\"Usage: script.py \")\n sys.exit(1)\n\n global_state[\"tmpdir\"] = tmpdir\n\n # Shared job queue: listener writes fds to write_sock, workers read from read_sock.\n # Only idle workers (blocked in recvmsg) pick up jobs, preventing the round-robin\n # deadlock where a callback gets dispatched to a busy worker.\n read_sock, write_sock = _socket.socketpair(_socket.AF_UNIX, _socket.SOCK_STREAM)\n\n num_workers = 1\n workers = []\n\n # Shared counters for dynamic worker spawning.\n # Workers increment busy_count before foreign_call and decrement after.\n # When all workers are busy, main process spawns a new one.\n busy_count = RawValue(ctypes.c_int, 0)\n total_workers = RawValue(ctypes.c_int, num_workers)\n wakeup_r, wakeup_w = os.pipe()\n os.set_blocking(wakeup_r, False)\n _shutdown_wakeup_fd = wakeup_w\n\n # Keep a dup of the read end so we can spawn new workers later\n spare_read_fd = os.dup(read_sock.fileno())\n\n for i in range(num_workers):\n worker = Process(target=worker_process,\n args=(read_sock.fileno(), tmpdir, shm_basename, shutdown_flag,\n busy_count, total_workers, wakeup_w))\n worker.start()\n workers.append(worker)\n read_sock.close() # main/listener don't need the read end (spare_read_fd kept)\n\n # Start client listener process\n listener_process = Process(\n target=client_listener,\n args=(write_sock.fileno(), socket_path, tmpdir, shm_basename, shutdown_flag)\n )\n listener_process.start()\n write_sock.close() # main doesn't need the write end\n\n # Main loop: monitor wake-up pipe, spawn new workers when all are busy,\n # and reap idle workers that have exited.\n while not shutdown_flag.value:\n rlist, _, _ = select.select([wakeup_r], [], [], 0.01)\n if rlist:\n try:\n os.read(wakeup_r, 4096) # drain pipe\n except OSError:\n pass\n\n # Reap dead workers (idle timeout or error exit)\n alive = []\n for w in workers:\n if w.is_alive():\n alive.append(w)\n else:\n w.join(timeout=0)\n w.close()\n workers = alive\n total_workers.value = max(1, len(workers))\n\n # Spawn a new worker if all are busy (or all have exited)\n if len(workers) == 0 or busy_count.value >= total_workers.value:\n w = Process(target=worker_process,\n args=(spare_read_fd, tmpdir, shm_basename, shutdown_flag,\n busy_count, total_workers, wakeup_w))\n w.start()\n workers.append(w)\n total_workers.value = len(workers)\n\n # Shutdown sequence\n os.close(wakeup_r)\n os.close(wakeup_w)\n os.close(spare_read_fd)\n\n # 1. Stop listener first\n listener_process.terminate()\n listener_process.join(timeout=0.001)\n listener_process.kill()\n listener_process.join() # Final blocking reap\n listener_process.close()\n\n # 2. Terminate workers with escalating force\n for p in workers:\n if p.is_alive():\n p.kill()\n p.join() # Final blocking reap\n p.close()\n\n sys.exit(0)\n" }, { "path": "data/lang/py/pymorloc.c", "content": "#define PY_SSIZE_T_CLEAN\n#include \"morloc.h\"\n#include \"Python.h\"\n#include \n#include \n#include \n#include \n\n// boilerplate for numpy support\n#define PY_ARRAY_UNIQUE_SYMBOL MORLOC_ARRAY_API\n#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\n#include \n\n// SHM tracker for _put_value allocations (deferred cleanup)\n#define SHM_TRACKER_INIT_CAP 16\ntypedef struct {\n absptr_t ptr;\n Schema* schema;\n} shm_entry_t;\nstatic shm_entry_t* shm_tracker = NULL;\nstatic size_t shm_tracker_count = 0;\nstatic size_t shm_tracker_cap = 0;\n\nstatic void shm_tracker_push(absptr_t ptr, Schema* schema) {\n if (shm_tracker_count >= shm_tracker_cap) {\n size_t new_cap = shm_tracker_cap ? shm_tracker_cap * 2 : SHM_TRACKER_INIT_CAP;\n shm_entry_t* new_buf = (shm_entry_t*)realloc(shm_tracker, new_cap * sizeof(shm_entry_t));\n if (!new_buf) return;\n shm_tracker = new_buf;\n shm_tracker_cap = new_cap;\n }\n shm_tracker[shm_tracker_count].ptr = ptr;\n shm_tracker[shm_tracker_count].schema = schema;\n shm_tracker_count++;\n}\n\nstatic void flush_shm_tracker(void) {\n for (size_t i = 0; i < shm_tracker_count; i++) {\n char* err = NULL;\n block_header_t* blk = (block_header_t*)((char*)shm_tracker[i].ptr - sizeof(block_header_t));\n if (shm_tracker[i].schema && blk->reference_count <= 1) {\n shfree_by_schema(shm_tracker[i].ptr, shm_tracker[i].schema, &err);\n if (err) { free(err); err = NULL; }\n }\n shfree(shm_tracker[i].ptr, &err);\n if (err) { free(err); }\n if (shm_tracker[i].schema) {\n free_schema(shm_tracker[i].schema);\n }\n }\n shm_tracker_count = 0;\n}\n\n#define NOTHING\n\n#define MAYFAIL \\\n char* child_errmsg_ = NULL; \\\n\n// Returns a strdup'd string that the caller must free, or NULL.\nchar* get_prior_err(){\n char* prior_err = NULL;\n if (PyErr_Occurred()) {\n // Fetch existing exception\n PyObject *type, *value, *traceback;\n PyErr_Fetch(&type, &value, &traceback);\n\n // Extract error message\n PyObject* str = PyObject_Str(value); // Convert exception to string\n if (str) {\n const char* raw = PyUnicode_AsUTF8(str);\n if (raw) {\n prior_err = strdup(raw);\n }\n Py_DECREF(str);\n }\n Py_XDECREF(type);\n Py_XDECREF(value);\n Py_XDECREF(traceback);\n }\n return prior_err;\n}\n\n\n#define PyTRY(fun, ...) \\\n fun(__VA_ARGS__ __VA_OPT__(,) &child_errmsg_); \\\n if(child_errmsg_ != NULL){ \\\n char* prior_err = get_prior_err(); \\\n if(prior_err == NULL){ \\\n PyErr_Format(PyExc_RuntimeError, \"Error (%s:%d in %s):\\n%s\", __FILE__, __LINE__, __func__, child_errmsg_); \\\n } else { \\\n PyErr_Format(PyExc_RuntimeError, \"%s\\nError (%s:%d in %s):\\n%s\", prior_err, __FILE__, __LINE__, __func__, child_errmsg_); \\\n free(prior_err); \\\n } \\\n goto error; \\\n }\n\n#define PyRAISE(msg, ...) { \\\n char* prior_err_ = get_prior_err(); \\\n if(prior_err_ == NULL){ \\\n PyErr_Format(PyExc_RuntimeError, \"Error (%s:%d in %s):\\n\" msg \"\\n\", __FILE__, __LINE__, __func__, ##__VA_ARGS__); \\\n } else { \\\n PyErr_Format(PyExc_RuntimeError, \"%s\\nError (%s:%d in %s):\\n\" msg \"\\n\", prior_err_, __FILE__, __LINE__, __func__, ##__VA_ARGS__); \\\n free(prior_err_); \\\n } \\\n goto error; \\\n }\n\n#define PyTRACE(cond) \\\n if(cond){ \\\n char* prior_err = get_prior_err(); \\\n if(prior_err != NULL){ \\\n PyErr_Format(PyExc_TypeError, \"Error (%s:%d in %s):\\n%s\", __FILE__, __LINE__, __func__, prior_err); \\\n free(prior_err); \\\n goto error; \\\n } \\\n }\n\nPyObject* numpy_module = NULL;\n\n\n// This function will be called to import numpy if, and only if, a numpy feature\n// is used. This avoids the agonizingly long numpy import time.\nvoid* import_numpy() {\n numpy_module = PyImport_ImportModule(\"numpy\");\n if(numpy_module == NULL){\n PyRAISE(\"NumPy is not available\");\n }\n\n import_array();\n\nerror:\n return NULL;\n}\n\n\n\n// Map morloc schema element type to numpy type number\nstatic int schema_to_npy_type(morloc_serial_type type) {\n switch (type) {\n case MORLOC_BOOL: return NPY_BOOL;\n case MORLOC_SINT8: return NPY_INT8;\n case MORLOC_SINT16: return NPY_INT16;\n case MORLOC_SINT32: return NPY_INT32;\n case MORLOC_SINT64: return NPY_INT64;\n case MORLOC_UINT8: return NPY_UINT8;\n case MORLOC_UINT16: return NPY_UINT16;\n case MORLOC_UINT32: return NPY_UINT32;\n case MORLOC_UINT64: return NPY_UINT64;\n case MORLOC_FLOAT32: return NPY_FLOAT32;\n case MORLOC_FLOAT64: return NPY_FLOAT64;\n default: return -1;\n }\n}\n\nPyObject* fromAnything(const Schema* schema, const void* data, const void* base_ptr){ MAYFAIL\n\n PyObject* obj = NULL;\n switch (schema->type) {\n case MORLOC_NIL:\n Py_RETURN_NONE;\n case MORLOC_BOOL:\n obj = PyBool_FromLong(*(bool*)data);\n break;\n case MORLOC_SINT8:\n obj = PyLong_FromLong(*(int8_t*)data);\n break;\n case MORLOC_SINT16:\n obj = PyLong_FromLong(*(int16_t*)data);\n break;\n case MORLOC_SINT32:\n obj = PyLong_FromLong(*(int32_t*)data);\n break;\n case MORLOC_SINT64:\n obj = PyLong_FromLongLong(*(int64_t*)data);\n break;\n case MORLOC_UINT8:\n obj = PyLong_FromUnsignedLong(*(uint8_t*)data);\n break;\n case MORLOC_UINT16:\n obj = PyLong_FromUnsignedLong(*(uint16_t*)data);\n break;\n case MORLOC_UINT32:\n obj = PyLong_FromUnsignedLong(*(uint32_t*)data);\n break;\n case MORLOC_UINT64:\n obj = PyLong_FromUnsignedLongLong(*(uint64_t*)data);\n break;\n case MORLOC_FLOAT32:\n obj = PyFloat_FromDouble(*(float*)data);\n break;\n case MORLOC_FLOAT64:\n obj = PyFloat_FromDouble(*(double*)data);\n break;\n case MORLOC_STRING: {\n Array* str_array = (Array*)data;\n void* tmp_ptr = NULL;\n\n if (str_array->size != 0) {\n tmp_ptr = PyTRY(resolve_relptr, str_array->data, base_ptr);\n }\n\n if (schema->hint != NULL && strcmp(schema->hint, \"bytes\") == 0) {\n // load binary data as a python bytes object\n if (str_array->size == 0) {\n obj = PyBytes_FromStringAndSize(\"\", 0); // empty bytes object\n } else {\n obj = PyBytes_FromStringAndSize(tmp_ptr, str_array->size);\n }\n if (!obj) {\n PyRAISE(\"Failed to parse data as bytes\");\n }\n } else if (schema->hint != NULL && strcmp(schema->hint, \"bytearray\") == 0) {\n // load binary data as a python bytearray object\n if (str_array->size == 0) {\n obj = PyByteArray_FromStringAndSize(\"\", 0); // empty bytearray object\n } else {\n obj = PyByteArray_FromStringAndSize(tmp_ptr, str_array->size);\n }\n if (!obj) {\n PyRAISE(\"Failed to parse data as bytearray\");\n }\n } else {\n // otherwise, load this as a str type\n if (str_array->size == 0) {\n obj = PyUnicode_New(0, 127); // empty string object\n } else {\n obj = PyUnicode_FromStringAndSize(tmp_ptr, str_array->size);\n }\n if (!obj) {\n PyRAISE(\"Failed to parse data as string\");\n }\n }\n break;\n }\n case MORLOC_ARRAY: {\n Array* array = (Array*)data;\n if (schema->hint != NULL && strcmp(schema->hint, \"numpy.ndarray\") == 0) {\n import_numpy();\n Schema* element_schema = schema->parameters[0];\n npy_intp dims[] = {array->size};\n void* absptr = NULL;\n int nd = 1; // number of dimensions\n int type_num;\n // Determine the NumPy type number based on the element schema\n switch (element_schema->type) {\n case MORLOC_BOOL: type_num = NPY_BOOL; break;\n case MORLOC_SINT8: type_num = NPY_INT8; break;\n case MORLOC_SINT16: type_num = NPY_INT16; break;\n case MORLOC_SINT32: type_num = NPY_INT32; break;\n case MORLOC_SINT64: type_num = NPY_INT64; break;\n case MORLOC_UINT8: type_num = NPY_UINT8; break;\n case MORLOC_UINT16: type_num = NPY_UINT16; break;\n case MORLOC_UINT32: type_num = NPY_UINT32; break;\n case MORLOC_UINT64: type_num = NPY_UINT64; break;\n case MORLOC_FLOAT32: type_num = NPY_FLOAT32; break;\n case MORLOC_FLOAT64: type_num = NPY_FLOAT64; break;\n default:\n PyRAISE(\"Unsupported element type for NumPy array\");\n }\n\n absptr = PyTRY(resolve_relptr, array->data, base_ptr);\n\n // Create the NumPy array\n obj = PyArray_SimpleNewFromData(nd, dims, type_num, absptr);\n\n if(obj == NULL) {\n PyRAISE(\"Failed to parse data\");\n }\n\n // Note that we do not want to give ownership to Python\n // This is shared memory, which means, python should not mutate\n // it.\n\n } else if (schema->hint != NULL && strcmp(schema->hint, \"bytearray\") == 0) {\n // Create a Python bytearray object\n void* absptr = PyTRY(resolve_relptr, array->data, base_ptr);\n obj = PyByteArray_FromStringAndSize((const char*)absptr, array->size);\n if (!obj) {\n PyErr_SetString(PyExc_TypeError, \"Failed to create bytearray\");\n goto error;\n }\n // Note: Similar to the numpy case, we don't want to give ownership to Python.\n // The bytearray is created from a copy of the data, so no additional handling is needed.\n } else if (schema->parameters[0]->type == MORLOC_UINT8) {\n // Create a Python bytes object for UINT8 arrays\n void* tmp_ptr = PyTRY(resolve_relptr, array->data, base_ptr);\n obj = PyBytes_FromStringAndSize((const char*)tmp_ptr, array->size);\n if (obj == NULL) {\n PyRAISE(\"Failed to one bytes\")\n }\n } else if (schema->hint == NULL || (schema->hint != NULL && strcmp(schema->hint, \"list\") == 0)) {\n // For other types, create a standard list\n obj = PyList_New(array->size);\n if(obj == NULL){\n PyRAISE(\"Failed to one string\");\n }\n if(array->size > 0){\n char* start = (char*) PyTRY(resolve_relptr, array->data, base_ptr);\n size_t width = schema->parameters[0]->width;\n Schema* element_schema = schema->parameters[0];\n for (size_t i = 0; i < array->size; i++) {\n PyObject* item = fromAnything(element_schema, start + width * i, base_ptr);\n if (!item || PyList_SetItem(obj, i, item) < 0) {\n Py_XDECREF(item);\n PyRAISE(\"Failed to access element in list\")\n }\n }\n }\n } else {\n PyRAISE(\"Unexpected array hint\");\n }\n break;\n }\n case MORLOC_TUPLE: {\n obj = PyTuple_New(schema->size);\n if(obj == NULL){\n PyRAISE(\"Failed in tuple\");\n }\n for (size_t i = 0; i < schema->size; i++) {\n void* item_ptr = (char*)data + schema->offsets[i];\n PyObject* item = fromAnything(schema->parameters[i], item_ptr, base_ptr);\n if (!item || PyTuple_SetItem(obj, i, item) < 0) {\n Py_XDECREF(item);\n PyRAISE(\"Failed to access tuple element\");\n }\n }\n break;\n }\n case MORLOC_MAP: {\n obj = PyDict_New();\n if(obj == NULL){\n PyRAISE(\"Failed in map\");\n }\n for (size_t i = 0; i < schema->size; i++) {\n void* item_ptr = (char*)data + schema->offsets[i];\n PyObject* value = fromAnything(schema->parameters[i], item_ptr, base_ptr);\n PyObject* key = PyUnicode_FromString(schema->keys[i]);\n if (!value || !key || PyDict_SetItem(obj, key, value) < 0) {\n Py_XDECREF(value);\n Py_XDECREF(key);\n PyRAISE(\"Failed to access map element\");\n }\n Py_DECREF(key);\n Py_DECREF(value);\n }\n break;\n }\n case MORLOC_OPTIONAL: {\n uint8_t tag = *(const uint8_t*)data;\n if (tag == 0) {\n Py_RETURN_NONE;\n }\n obj = fromAnything(schema->parameters[0], (const char*)data + schema->offsets[0], base_ptr);\n if (!obj) {\n PyRAISE(\"Failed to deserialize optional inner value\");\n }\n break;\n }\n case MORLOC_TENSOR: {\n import_numpy();\n const Tensor* tensor = (const Tensor*)data;\n size_t ndim = schema_tensor_ndim(schema);\n\n int type_num = schema_to_npy_type(schema->parameters[0]->type);\n if (type_num < 0) { PyRAISE(\"Unsupported tensor element type\"); }\n\n if (tensor->total_elements == 0) {\n npy_intp zero_dims[1] = {0};\n obj = PyArray_SimpleNew(1, zero_dims, type_num);\n break;\n }\n\n const int64_t* shape = (const int64_t*)resolve_relptr(tensor->shape, base_ptr, NULL);\n const void* tdata = resolve_relptr(tensor->data, base_ptr, NULL);\n\n npy_intp np_dims[5];\n for (size_t i = 0; i < ndim; i++) np_dims[i] = (npy_intp)shape[i];\n\n // Create numpy array as a copy (R/W) from the data\n obj = PyArray_SimpleNewFromData((int)ndim, np_dims, type_num, (void*)tdata);\n if (!obj) { PyRAISE(\"Failed to create numpy array from tensor\"); }\n\n // Make a copy so the array owns its data (SHM may be freed)\n PyObject* owned = PyArray_NewCopy((PyArrayObject*)obj, NPY_CORDER);\n Py_DECREF(obj);\n obj = owned;\n if (!obj) { PyRAISE(\"Failed to copy tensor data\"); }\n break;\n }\n default:\n PyRAISE(\"Unsupported schema type\");\n }\n\n return obj;\n\nerror:\n Py_XDECREF(obj);\n return NULL;\n}\n\n\n#define HANDLE_SINT_TYPE(CTYPE, PYLONG_FUNC, MIN, MAX) \\\n do { \\\n if (!PyLong_Check(obj)) { \\\n PyErr_Format(PyExc_TypeError, \"Expected int for %s, but got %s\", #CTYPE, Py_TYPE(obj)->tp_name); \\\n goto error; \\\n } \\\n long long value = PYLONG_FUNC(obj); \\\n if (value < MIN || value > MAX || PyErr_Occurred()) { \\\n PyErr_Format(PyExc_OverflowError, \"Integer overflow for %s\", #CTYPE); \\\n goto error; \\\n } \\\n *(CTYPE*)dest = (CTYPE)value; \\\n } while(0)\n\n#define HANDLE_UINT_TYPE(CTYPE, PYLONG_FUNC, MAX) \\\n do { \\\n if (!PyLong_Check(obj)) { \\\n PyErr_Format(PyExc_TypeError, \"Expected int for %s, but got %s\", #CTYPE, Py_TYPE(obj)->tp_name); \\\n goto error; \\\n } \\\n unsigned long long value = PYLONG_FUNC(obj); \\\n if (value > MAX || PyErr_Occurred()) { \\\n PyErr_Format(PyExc_OverflowError, \"Integer overflow for %s\", #CTYPE); \\\n goto error; \\\n } \\\n *(CTYPE*)dest = (CTYPE)value; \\\n } while(0)\n\n\n\nssize_t get_shm_size(const Schema* schema, PyObject* obj) {\n switch (schema->type) {\n case MORLOC_NIL:\n case MORLOC_BOOL:\n case MORLOC_SINT8:\n case MORLOC_SINT16:\n case MORLOC_SINT32:\n case MORLOC_SINT64:\n case MORLOC_UINT8:\n case MORLOC_UINT16:\n case MORLOC_UINT32:\n case MORLOC_UINT64:\n case MORLOC_FLOAT32:\n case MORLOC_FLOAT64:\n return schema->width;\n case MORLOC_STRING:\n case MORLOC_ARRAY:\n if (schema->type == MORLOC_STRING && !(PyUnicode_Check(obj) || PyBytes_Check(obj) || PyByteArray_Check(obj) )) {\n PyRAISE(\"Expected str or bytes for MORLOC_STRING, but got %s\", Py_TYPE(obj)->tp_name);\n }\n if (schema->type == MORLOC_ARRAY && !(PyList_Check(obj) || PyBytes_Check(obj) || PyByteArray_Check(obj) || PyObject_HasAttrString(obj, \"__array_interface__\"))) {\n PyRAISE(\"Expected list, bytes, bytearray, or numpy array for MORLOC_ARRAY, but got %s\", Py_TYPE(obj)->tp_name);\n }\n \n {\n ssize_t required_size = 0;\n // worst-case cursor alignment padding for element data\n required_size += (ssize_t)(schema_alignment(schema->parameters[0]) - 1);\n\n if (PyList_Check(obj)) {\n Py_ssize_t list_size = PyList_Size(obj);\n size_t element_width = schema->parameters[0]->width;\n switch(schema->parameters[0]->type){\n case MORLOC_NIL:\n case MORLOC_BOOL:\n case MORLOC_SINT8:\n case MORLOC_SINT16:\n case MORLOC_SINT32:\n case MORLOC_SINT64:\n case MORLOC_UINT8:\n case MORLOC_UINT16:\n case MORLOC_UINT32:\n case MORLOC_UINT64:\n case MORLOC_FLOAT32:\n case MORLOC_FLOAT64:\n required_size += list_size * element_width;\n break;\n case MORLOC_STRING:\n case MORLOC_ARRAY:\n case MORLOC_TUPLE:\n case MORLOC_MAP:\n case MORLOC_OPTIONAL:\n for(size_t i = 0; i < (size_t)list_size; i++){\n required_size += get_shm_size(schema->parameters[0], PyList_GetItem(obj, i));\n }\n break;\n }\n } else if (PyObject_HasAttrString(obj, \"__array_interface__\")) {\n import_numpy();\n PyArrayObject *arr = (PyArrayObject *)obj;\n npy_intp *dims = PyArray_DIMS(arr);\n int ndim = PyArray_NDIM(arr);\n size_t total_elements = 1;\n for (int i = 0; i < ndim; i++) {\n total_elements *= dims[i];\n }\n required_size += total_elements * PyArray_ITEMSIZE(arr);\n } else if (PyBytes_Check(obj)) {\n required_size += (ssize_t)PyBytes_GET_SIZE(obj);\n } else if (PyByteArray_Check(obj)) {\n required_size += (ssize_t)PyByteArray_GET_SIZE(obj);\n } else if (PyUnicode_Check(obj)) {\n PyUnicode_AsUTF8AndSize(obj, &required_size);\n } else {\n PyRAISE(\"Unsupported data type\");\n }\n\n required_size += sizeof(Array);\n return required_size;\n }\n\n case MORLOC_TUPLE:\n if (!PyTuple_Check(obj) && !PyList_Check(obj)) {\n PyRAISE(\"Expected tuple or list for MORLOC_TUPLE, but got %s\", Py_TYPE(obj)->tp_name);\n }\n\n {\n Py_ssize_t size = PyTuple_Check(obj) ? PyTuple_Size(obj) : PyList_Size(obj);\n if ((size_t)size != schema->size) {\n PyRAISE(\"Tuple/List size mismatch\");\n }\n\n size_t required_size = schema->width;\n\n for (Py_ssize_t i = 0; i < size; ++i) {\n PyObject* item = PyTuple_Check(obj) ? PyTuple_GetItem(obj, i) : PyList_GetItem(obj, i);\n ssize_t element_size = get_shm_size(schema->parameters[i], item);\n if(element_size != -1){\n if ((size_t)element_size > schema->parameters[i]->width) {\n required_size += (size_t)element_size - schema->parameters[i]->width;\n }\n } else {\n return -1;\n }\n }\n return (ssize_t)required_size;\n }\n\n case MORLOC_MAP:\n if (!PyDict_Check(obj)) {\n PyRAISE(\"Expected dict for MORLOC_MAP, but got %s\", Py_TYPE(obj)->tp_name);\n }\n\n {\n size_t required_size = schema->width;\n for (size_t i = 0; i < schema->size; ++i) {\n PyObject* key = PyUnicode_FromString(schema->keys[i]);\n PyObject* value = PyDict_GetItem(obj, key);\n Py_DECREF(key);\n if (value) {\n ssize_t element_size = get_shm_size(schema->parameters[i], value);\n if(element_size != -1){\n if ((size_t)element_size > schema->parameters[i]->width) {\n required_size += (size_t)element_size - schema->parameters[i]->width;\n }\n } else {\n return -1;\n }\n }\n }\n return (ssize_t)required_size;\n }\n\n case MORLOC_OPTIONAL:\n if (obj == Py_None) {\n return (ssize_t)schema->width;\n }\n {\n ssize_t inner_size = get_shm_size(schema->parameters[0], obj);\n if (inner_size == -1) return -1;\n ssize_t extra = (inner_size > (ssize_t)schema->parameters[0]->width) ? inner_size - (ssize_t)schema->parameters[0]->width : 0;\n return (ssize_t)schema->width + extra;\n }\n\n case MORLOC_TENSOR:\n {\n import_numpy();\n int type_num = schema_to_npy_type(schema->parameters[0]->type);\n if (type_num < 0) { PyRAISE(\"Unsupported tensor element type\"); }\n PyArrayObject* arr = (PyArrayObject*)PyArray_FROM_OTF(obj, type_num, NPY_ARRAY_C_CONTIGUOUS);\n if (!arr) { PyRAISE(\"Expected numpy array for MORLOC_TENSOR\"); }\n size_t total = (size_t)PyArray_SIZE(arr);\n size_t elem_width = schema->parameters[0]->width;\n ssize_t required = (ssize_t)sizeof(Tensor);\n required += (ssize_t)(_Alignof(int64_t) - 1);\n required += (ssize_t)(schema_tensor_ndim(schema) * sizeof(int64_t));\n required += (ssize_t)(schema_alignment(schema->parameters[0]) - 1);\n required += (ssize_t)(total * elem_width);\n Py_DECREF(arr);\n return required;\n }\n\n default:\n PyRAISE(\"Unsupported schema type\");\n }\n\n PyRAISE(\"Reached the unreachable\");\n\nerror:\n return -1;\n}\n\n\n\nint to_voidstar_r(void* dest, void** cursor, const Schema* schema, PyObject* obj) { MAYFAIL\n switch (schema->type) {\n case MORLOC_NIL:\n if (obj != Py_None) {\n PyRAISE(\"Expected None for MORLOC_NIL, but got %s\", Py_TYPE(obj)->tp_name);\n }\n *((int8_t*)dest) = (int8_t)0;\n break;\n\n case MORLOC_BOOL:\n if (!PyBool_Check(obj)) {\n PyRAISE(\"Expected bool for MORLOC_BOOL, but got %s\", Py_TYPE(obj)->tp_name);\n }\n *((bool*)dest) = (obj == Py_True);\n break;\n\n case MORLOC_SINT8:\n HANDLE_SINT_TYPE(int8_t, PyLong_AsLongLong, INT8_MIN, INT8_MAX);\n break;\n case MORLOC_SINT16:\n HANDLE_SINT_TYPE(int16_t, PyLong_AsLongLong, INT16_MIN, INT16_MAX);\n break;\n case MORLOC_SINT32:\n HANDLE_SINT_TYPE(int32_t, PyLong_AsLongLong, INT32_MIN, INT32_MAX);\n break;\n case MORLOC_SINT64:\n HANDLE_SINT_TYPE(int64_t, PyLong_AsLongLong, INT64_MIN, INT64_MAX);\n break;\n case MORLOC_UINT8:\n HANDLE_UINT_TYPE(uint8_t, PyLong_AsUnsignedLongLong, UINT8_MAX);\n break;\n case MORLOC_UINT16:\n HANDLE_UINT_TYPE(uint16_t, PyLong_AsUnsignedLongLong, UINT16_MAX);\n break;\n case MORLOC_UINT32:\n HANDLE_UINT_TYPE(uint32_t, PyLong_AsUnsignedLongLong, UINT32_MAX);\n break;\n case MORLOC_UINT64:\n HANDLE_UINT_TYPE(uint64_t, PyLong_AsUnsignedLongLong, UINT64_MAX);\n break;\n\n case MORLOC_FLOAT32:\n if (!PyFloat_Check(obj)) {\n PyRAISE(\"Expected float for MORLOC_FLOAT32, but got %s\", Py_TYPE(obj)->tp_name);\n }\n *((float*)dest) = (float)PyFloat_AsDouble(obj);\n break;\n\n case MORLOC_FLOAT64:\n if(PyFloat_Check(obj))\n {\n *((double*)dest) = PyFloat_AsDouble(obj);\n } else if(PyLong_Check(obj)){\n *((double*)dest) = (double)PyLong_AsLongLong(obj);\n } else {\n PyRAISE(\"Expected float or int for MORLOC_FLOAT64, but got %s\", Py_TYPE(obj)->tp_name);\n }\n break;\n\n case MORLOC_STRING:\n case MORLOC_ARRAY:\n if (schema->type == MORLOC_STRING && !(PyUnicode_Check(obj) || PyBytes_Check(obj) || PyByteArray_Check(obj))) {\n PyRAISE(\"Expected str or bytes for MORLOC_STRING, but got %s\", Py_TYPE(obj)->tp_name);\n }\n \n if (schema->type == MORLOC_ARRAY && !(PyList_Check(obj) || PyBytes_Check(obj) || PyByteArray_Check(obj) || PyObject_HasAttrString(obj, \"__array_interface__\"))) { \n PyRAISE(\"Expected list, bytes, bytearray, or numpy array for MORLOC_ARRAY, but got %s\", Py_TYPE(obj)->tp_name);\n }\n \n {\n Py_ssize_t size;\n\n // \"bytes\" type is mutable, so it exposes a non-const pointer \n char* mutable_data = NULL;\n\n // strings type are immutable, so const\n const char* immutable_data = NULL; \n\n if (PyList_Check(obj)) {\n size = PyList_Size(obj);\n } else if (PyBytes_Check(obj)) {\n // This needs non-const data\n PyBytes_AsStringAndSize(obj, &mutable_data, &size);\n } else if (PyByteArray_Check(obj)) {\n mutable_data = PyByteArray_AS_STRING(obj);\n size = PyByteArray_GET_SIZE(obj);\n } else if (schema->type == MORLOC_ARRAY && PyObject_HasAttrString(obj, \"__array_interface__\")) { // check if it is a numpy array\n import_numpy();\n PyArrayObject* arr = (PyArrayObject*)obj;\n size = PyArray_SIZE(arr);\n // This needs const data\n immutable_data = PyArray_DATA(arr); // Get the data pointer\n\n // Verify that the array is contiguous\n if (!PyArray_ISCONTIGUOUS(arr)) {\n PyRAISE(\"NumPy array must be contiguous\");\n }\n } else {\n immutable_data = PyUnicode_AsUTF8AndSize(obj, &size);\n }\n \n Array* result = (Array*)dest;\n result->size = (size_t)size;\n\n if(result->size == 0){\n result->data = RELNULL;\n break;\n }\n\n // align cursor for element data placement\n *cursor = (void*)ALIGN_UP((uintptr_t)*cursor, schema_alignment(schema->parameters[0]));\n\n result->data = PyTRY(abs2rel, *cursor);\n\n if (PyList_Check(obj)) {\n // Fixed size width of each element (variable size data will\n // be written to the cursor location)\n size_t width = schema->parameters[0]->width;\n \n // Move the cursor to the location immediately after the\n // fixed sized elements\n *cursor = (void*)(*(char**)cursor + size * width);\n\n char* start = (char*) PyTRY(rel2abs, result->data);\n Schema* element_schema = schema->parameters[0];\n for (Py_ssize_t i = 0; i < size; i++) {\n PyObject* item = PyList_GetItem(obj, i);\n if (to_voidstar_r(start + width * i, cursor, element_schema, item) != 0) {\n goto error;\n }\n }\n\n } else if (PyBytes_Check(obj) || PyByteArray_Check(obj)){\n absptr_t tmp_ptr = PyTRY(rel2abs, result->data);\n memcpy(tmp_ptr, mutable_data, size);\n // move cursor to the location after the copied data\n *cursor = (void*)(*(char**)cursor + size);\n }\n else{\n size_t width = schema->parameters[0]->width;\n\n absptr_t tmp_ptr = PyTRY(rel2abs, result->data);\n memcpy(tmp_ptr, immutable_data, size * width);\n\n // Move the cursor to the location immediately after the\n // fixed sized elements\n *cursor = (void*)(*(char**)cursor + size * width);\n }\n }\n break;\n\n\n case MORLOC_TUPLE:\n if (!PyTuple_Check(obj) && !PyList_Check(obj)) {\n PyRAISE(\"Expected tuple or list for MORLOC_TUPLE, but got %s\", Py_TYPE(obj)->tp_name);\n }\n\n {\n Py_ssize_t size = PyTuple_Check(obj) ? PyTuple_Size(obj) : PyList_Size(obj);\n if ((size_t)size != schema->size) {\n PyRAISE(\"Tuple/List size mismatch\");\n }\n for (Py_ssize_t i = 0; i < size; ++i) {\n PyObject* item = PyTuple_Check(obj) ? PyTuple_GetItem(obj, i) : PyList_GetItem(obj, i);\n if (to_voidstar_r((char*)dest + schema->offsets[i], cursor, schema->parameters[i], item) != 0) {\n goto error;\n }\n }\n }\n break;\n\n case MORLOC_MAP:\n if (!PyDict_Check(obj)) {\n PyRAISE(\"Expected dict for MORLOC_MAP, but got %s\", Py_TYPE(obj)->tp_name);\n }\n\n {\n for (size_t i = 0; i < schema->size; ++i) {\n PyObject* key = PyUnicode_FromString(schema->keys[i]);\n PyObject* value = PyDict_GetItem(obj, key);\n Py_DECREF(key);\n if (value) {\n if (to_voidstar_r((char*)dest + schema->offsets[i], cursor, schema->parameters[i], value) != 0) {\n goto error;\n }\n }\n }\n }\n break;\n\n case MORLOC_OPTIONAL:\n if (obj == Py_None) {\n *((uint8_t*)dest) = 0;\n memset((char*)dest + schema->offsets[0], 0, schema->parameters[0]->width);\n } else {\n *((uint8_t*)dest) = 1;\n if (to_voidstar_r((char*)dest + schema->offsets[0], cursor, schema->parameters[0], obj) != 0) {\n goto error;\n }\n }\n break;\n\n case MORLOC_TENSOR:\n {\n import_numpy();\n int type_num = schema_to_npy_type(schema->parameters[0]->type);\n if (type_num < 0) { PyRAISE(\"Unsupported tensor element type\"); }\n PyArrayObject* arr = (PyArrayObject*)PyArray_FROM_OTF(obj, type_num, NPY_ARRAY_C_CONTIGUOUS);\n if (!arr) { PyRAISE(\"Expected numpy array for MORLOC_TENSOR\"); }\n\n int ndim = PyArray_NDIM(arr);\n npy_intp* np_shape = PyArray_DIMS(arr);\n size_t total = (size_t)PyArray_SIZE(arr);\n size_t elem_width = schema->parameters[0]->width;\n\n Tensor* tensor = (Tensor*)dest;\n tensor->total_elements = total;\n tensor->device_type = 0;\n tensor->device_id = 0;\n\n if (total == 0) {\n tensor->shape = RELNULL;\n tensor->data = RELNULL;\n Py_DECREF(arr);\n break;\n }\n\n // Write shape array\n *cursor = (void*)ALIGN_UP((uintptr_t)*cursor, _Alignof(int64_t));\n {\n char* rel_err = NULL;\n tensor->shape = abs2rel((absptr_t)*cursor, &rel_err);\n if (rel_err) { free(rel_err); Py_DECREF(arr); PyRAISE(\"abs2rel failed for tensor shape\"); }\n }\n int64_t* shape_dst = (int64_t*)*cursor;\n for (int i = 0; i < ndim; i++) shape_dst[i] = (int64_t)np_shape[i];\n *cursor = (char*)*cursor + ndim * sizeof(int64_t);\n\n // Write data buffer\n size_t elem_align = schema_alignment(schema->parameters[0]);\n *cursor = (void*)ALIGN_UP((uintptr_t)*cursor, elem_align);\n {\n char* rel_err = NULL;\n tensor->data = abs2rel((absptr_t)*cursor, &rel_err);\n if (rel_err) { free(rel_err); Py_DECREF(arr); PyRAISE(\"abs2rel failed for tensor data\"); }\n }\n memcpy(*cursor, PyArray_DATA(arr), total * elem_width);\n *cursor = (char*)*cursor + total * elem_width;\n\n Py_DECREF(arr);\n }\n break;\n\n default:\n PyRAISE(\"Unsupported schema type\");\n }\n\n return 0;\n\nerror:\n return -1;\n}\n\nvoid* to_voidstar(const Schema* schema, PyObject* obj){ MAYFAIL\n void* dest = NULL;\n\n // calculate the required size of the shared memory object\n ssize_t shm_size = get_shm_size(schema, obj);\n if(shm_size == -1){\n PyRAISE(\"Schema does not match object\");\n }\n\n // allocate the required memory as a single block\n dest = PyTRY(shmalloc, (size_t)shm_size);\n\n // set the write location of variable size chunks\n void* cursor = (void*)((char*)dest + schema->width);\n\n // write the data to the block\n int result = to_voidstar_r(dest, &cursor, schema, obj);\n if (result != 0) {\n goto error;\n }\n\n return dest;\n\nerror:\n if (dest != NULL) {\n char* free_errmsg = NULL;\n shfree(dest, &free_errmsg);\n free(free_errmsg);\n }\n return NULL;\n}\n\n\nstatic PyObject* pybinding__wait_for_client(PyObject* self, PyObject* args) { MAYFAIL\n PyObject* daemon_capsule;\n\n if (!PyArg_ParseTuple(args, \"O\", &daemon_capsule)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n language_daemon_t* daemon = (language_daemon_t*)PyCapsule_GetPointer(daemon_capsule, \"language_daemon_t\");\n\n int client_fd = PyTRY(wait_for_client, daemon);\n\n return PyLong_FromLong((long)client_fd);\n\nerror:\n return NULL;\n}\n\nstatic PyObject* pybinding__start_daemon(PyObject* self, PyObject* args) { MAYFAIL\n const char* socket_path;\n const char* tmpdir;\n const char* shm_basename;\n size_t shm_default_size;\n language_daemon_t* daemon = NULL;\n\n if (!PyArg_ParseTuple(args, \"sssk\", &socket_path, &tmpdir, &shm_basename, &shm_default_size)) {\n goto error;\n }\n\n daemon = PyTRY(\n start_daemon,\n socket_path,\n tmpdir,\n shm_basename,\n shm_default_size\n );\n\n return PyCapsule_New(daemon, \"language_daemon_t\", NULL);\n\nerror:\n FREE(daemon)\n return NULL;\n}\n\n\nstatic PyObject* pybinding__close_daemon(PyObject* self, PyObject* args) {\n PyObject* daemon_capsule;\n\n if (!PyArg_ParseTuple(args, \"O\", &daemon_capsule)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n language_daemon_t* daemon = (language_daemon_t*)PyCapsule_GetPointer(daemon_capsule, \"language_daemon_t\");\n\n if(daemon != NULL){\n close_daemon(&daemon);\n }\n\n Py_RETURN_NONE;\n\nerror:\n return NULL;\n}\n\n\nstatic PyObject* pybinding__read_morloc_call_packet(PyObject* self, PyObject* args){ MAYFAIL\n char* packet;\n size_t packet_size;\n morloc_call_t* call_packet = NULL;\n PyObject* py_tuple = NULL;\n PyObject* py_args = NULL;\n PyObject* py_mid = NULL;\n\n if (!PyArg_ParseTuple(args, \"y#\", &packet, &packet_size)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n call_packet = PyTRY(read_morloc_call_packet, (const uint8_t*)packet);\n\n py_tuple = PyTuple_New(2);\n if (!py_tuple) { PyRAISE(\"Allocation failed\"); }\n py_args = PyList_New(call_packet->nargs);\n if (!py_args) { PyRAISE(\"Allocation failed\"); }\n py_mid = PyLong_FromLong((long)call_packet->midx);\n if (!py_mid) { PyRAISE(\"Allocation failed\"); }\n for(size_t i = 0; i < call_packet->nargs; i++){\n size_t arg_packet_size = PyTRY(morloc_packet_size, call_packet->args[i]);\n PyObject* py_arg = PyBytes_FromStringAndSize(\n (char*)call_packet->args[i],\n arg_packet_size\n );\n PyList_SetItem(py_args, i, py_arg);\n }\n\n PyTuple_SetItem(py_tuple, 0, py_mid);\n PyTuple_SetItem(py_tuple, 1, py_args);\n py_mid = NULL; // stolen by PyTuple_SetItem\n py_args = NULL; // stolen by PyTuple_SetItem\n\n free_morloc_call(call_packet);\n\n return py_tuple;\n\nerror:\n if (call_packet) free_morloc_call(call_packet);\n Py_XDECREF(py_mid);\n Py_XDECREF(py_args);\n Py_XDECREF(py_tuple);\n return NULL;\n}\n\nstatic PyObject* pybinding__send_packet_to_foreign_server(PyObject* self, PyObject* args){ MAYFAIL\n int client_fd = 0;\n uint8_t* packet = NULL;\n size_t packet_size = 0;\n\n if (!PyArg_ParseTuple(args, \"iy#\", &client_fd, &packet, &packet_size)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n size_t bytes_sent = PyTRY(send_packet_to_foreign_server, client_fd, packet);\n\n return PyLong_FromSize_t(bytes_sent);\n\nerror:\n return NULL;\n}\n\n\nstatic PyObject* pybinding__stream_from_client(PyObject* self, PyObject* args){ MAYFAIL\n int client_fd = 0;\n uint8_t* packet = NULL;\n\n if (!PyArg_ParseTuple(args, \"i\", &client_fd)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n packet = PyTRY(stream_from_client, client_fd);\n\n size_t packet_size = PyTRY(morloc_packet_size, packet);\n\n PyObject* retval = PyBytes_FromStringAndSize((char*)packet, packet_size);\n\n free(packet);\n\n return retval;\n\nerror:\n FREE(packet)\n return NULL;\n}\n\n\n\nstatic PyObject* pybinding__close_socket(PyObject* self, PyObject* args){\n int socket_id = 0;\n\n if (!PyArg_ParseTuple(args, \"i\", &socket_id)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n close_socket(socket_id);\n\n Py_RETURN_NONE;\n\nerror:\n return NULL;\n}\n\n// Transforms a value into a message ready for the socket\nstatic PyObject* pybinding__put_value(PyObject* self, PyObject* args){ MAYFAIL\n uint8_t* packet = NULL;\n Schema* schema = NULL;\n void* voidstar = NULL;\n size_t packet_size = 0;\n bool tracked = false;\n\n PyObject* obj;\n const char* schema_str;\n\n if (!PyArg_ParseTuple(args, \"Os\", &obj, &schema_str)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n // Arrow dispatch: if schema hint is \"arrow\", use Arrow C Data Interface\n if (schema->hint && strcmp(schema->hint, \"arrow\") == 0) {\n // Export pyarrow object via C Data Interface -> copy to shm -> packet\n struct ArrowSchema arrow_schema;\n struct ArrowArray arrow_array;\n\n // Call obj._export_to_c(arrow_array_ptr, arrow_schema_ptr)\n PyObject* export_result = PyObject_CallMethod(\n obj, \"_export_to_c\",\n \"nn\", (Py_ssize_t)&arrow_array, (Py_ssize_t)&arrow_schema);\n if (!export_result) {\n free_schema(schema);\n PyRAISE(\"Failed to export pyarrow object via C Data Interface\");\n }\n Py_DECREF(export_result);\n\n char* errmsg = NULL;\n relptr_t relptr = arrow_to_shm(&arrow_array, &arrow_schema, &errmsg);\n\n // Release the exported C Data Interface structs\n if (arrow_schema.release) arrow_schema.release(&arrow_schema);\n if (arrow_array.release) arrow_array.release(&arrow_array);\n\n if (errmsg) {\n free_schema(schema);\n PyErr_SetString(PyExc_RuntimeError, errmsg);\n free(errmsg);\n return NULL;\n }\n\n packet = make_arrow_data_packet(relptr, schema);\n if (!packet) {\n free_schema(schema);\n PyRAISE(\"Failed to create arrow data packet\");\n }\n\n // Track shm for cleanup\n char* resolve_err = NULL;\n void* shm_ptr = rel2abs(relptr, &resolve_err);\n if (resolve_err) { free(resolve_err); }\n if (shm_ptr) {\n shm_tracker_push((absptr_t)shm_ptr, NULL);\n tracked = true;\n }\n\n packet_size = PyTRY(morloc_packet_size, packet);\n PyObject* retval = PyBytes_FromStringAndSize((char*)packet, packet_size);\n free(packet);\n free_schema(schema);\n return retval;\n }\n\n voidstar = to_voidstar(schema, obj);\n PyTRACE(voidstar == NULL)\n\n // convert to a relative pointer conserved between language servers\n relptr_t relptr = PyTRY(abs2rel, voidstar);\n\n packet = PyTRY(make_data_packet_auto, voidstar, relptr, schema);\n\n {\n const morloc_packet_header_t* hdr = (const morloc_packet_header_t*)packet;\n if (hdr->command.data.source == PACKET_SOURCE_RPTR) {\n // SHM referenced by packet -- track for deferred cleanup\n shm_tracker_push((absptr_t)voidstar, schema);\n tracked = true;\n } else {\n // Data inlined in packet -- free SHM immediately\n char* free_err = NULL;\n shfree_by_schema((absptr_t)voidstar, schema, &free_err);\n if (free_err) { free(free_err); free_err = NULL; }\n shfree((absptr_t)voidstar, &free_err);\n if (free_err) { free(free_err); }\n voidstar = NULL;\n }\n }\n\n packet_size = PyTRY(morloc_packet_size, packet);\n\n {\n PyObject* retval = PyBytes_FromStringAndSize((char*)packet, packet_size);\n free(packet);\n if (!tracked) {\n free_schema(schema);\n }\n return retval;\n }\n\nerror:\n FREE(packet)\n if (!tracked) {\n if (voidstar && schema) {\n char* free_err = NULL;\n shfree_by_schema((absptr_t)voidstar, schema, &free_err);\n if (free_err) { free(free_err); free_err = NULL; }\n shfree((absptr_t)voidstar, &free_err);\n if (free_err) { free(free_err); }\n }\n free_schema(schema);\n }\n return NULL;\n}\n\n\n// Use a key to retrieve a value\nstatic PyObject* pybinding__get_value(PyObject* self, PyObject* args){ MAYFAIL\n uint8_t* voidstar = NULL;\n Schema* schema = NULL;\n PyObject* obj = NULL;\n bool tracked = false;\n\n const char* packet;\n size_t packet_size;\n const char* schema_str;\n\n if (!PyArg_ParseTuple(args, \"y#s\", &packet, &packet_size, &schema_str)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n const morloc_packet_header_t* header = (const morloc_packet_header_t*)packet;\n uint8_t source = header->command.data.source;\n uint8_t format = header->command.data.format;\n\n schema = PyTRY(parse_schema, schema_str)\n\n // Arrow dispatch: if packet format is Arrow, import via C Data Interface\n if (format == PACKET_FORMAT_ARROW) {\n voidstar = PyTRY(get_morloc_data_packet_value, (uint8_t*)packet, schema);\n\n const arrow_shm_header_t* arrow_hdr = (const arrow_shm_header_t*)voidstar;\n\n struct ArrowSchema arrow_schema;\n struct ArrowArray arrow_array;\n char* arrow_err = NULL;\n arrow_from_shm(arrow_hdr, &arrow_schema, &arrow_array, &arrow_err);\n if (arrow_err) {\n free_schema(schema);\n PyErr_SetString(PyExc_RuntimeError, arrow_err);\n free(arrow_err);\n return NULL;\n }\n\n // Import via pyarrow RecordBatch.from_buffers or _import_from_c\n PyObject* pyarrow_mod = PyImport_ImportModule(\"pyarrow\");\n if (!pyarrow_mod) {\n if (arrow_schema.release) arrow_schema.release(&arrow_schema);\n if (arrow_array.release) arrow_array.release(&arrow_array);\n free_schema(schema);\n PyRAISE(\"pyarrow is required for arrow-typed data\");\n }\n\n PyObject* rb_class = PyObject_GetAttrString(pyarrow_mod, \"RecordBatch\");\n Py_DECREF(pyarrow_mod);\n if (!rb_class) {\n if (arrow_schema.release) arrow_schema.release(&arrow_schema);\n if (arrow_array.release) arrow_array.release(&arrow_array);\n free_schema(schema);\n PyRAISE(\"Failed to get pyarrow.RecordBatch\");\n }\n\n // Use RecordBatch._import_from_c(array_ptr, schema_ptr)\n obj = PyObject_CallMethod(rb_class, \"_import_from_c\",\n \"nn\", (Py_ssize_t)&arrow_array, (Py_ssize_t)&arrow_schema);\n Py_DECREF(rb_class);\n\n // Incref shm so it stays alive while pyarrow references the buffers\n char* incref_err = NULL;\n shincref((absptr_t)voidstar, &incref_err);\n if (incref_err) { free(incref_err); }\n shm_tracker_push((absptr_t)voidstar, NULL);\n\n free_schema(schema);\n if (!obj) return NULL;\n return obj;\n }\n\n // Fast path: inline voidstar -- read directly from packet, no SHM needed\n if (source == PACKET_SOURCE_MESG && format == PACKET_FORMAT_VOIDSTAR) {\n const uint8_t* payload = (const uint8_t*)packet + sizeof(morloc_packet_header_t) + header->offset;\n obj = fromAnything(schema, (const void*)payload, (const void*)payload);\n PyTRACE(obj == NULL)\n free_schema(schema);\n return obj;\n }\n\n // SHM paths (RPTR or MESG+MSGPACK)\n bool is_rptr = (source == PACKET_SOURCE_RPTR);\n\n voidstar = PyTRY(get_morloc_data_packet_value, (uint8_t*)packet, schema);\n\n // For RPTR data, increment refcount so the owner's tracker flush\n // won't destroy data we may still need (e.g. forwarded packets).\n if (is_rptr) {\n char* incref_err = NULL;\n shincref((absptr_t)voidstar, &incref_err);\n if (incref_err) { free(incref_err); }\n // Track for deferred decref (tracker takes schema ownership)\n shm_tracker_push((absptr_t)voidstar, schema);\n tracked = true;\n }\n\n obj = fromAnything(schema, voidstar, NULL);\n PyTRACE(obj == NULL)\n\n if (!tracked) {\n free_schema(schema);\n }\n\n return obj;\n\nerror:\n if (!tracked) {\n free_schema(schema);\n }\n return NULL;\n}\n\n\n// Free tracked SHM allocations from put_value calls.\n// Called at dispatch start to free result SHM from previous dispatch.\nstatic PyObject* pybinding__flush_shm_tracker(PyObject* self, PyObject* args) {\n (void)self; (void)args;\n flush_shm_tracker();\n Py_RETURN_NONE;\n}\n\n\n// Make a foreign call\n//\n// Arguments:\n// 1. socket path\n// 2. midx\n// 3. list of arguments, each is bytestring packet\nstatic PyObject* pybinding__foreign_call(PyObject* self, PyObject* args) { MAYFAIL\n char* socket_path;\n int mid;\n PyObject* py_args;\n const uint8_t** arg_packets = NULL;\n Py_ssize_t nargs;\n Py_ssize_t i;\n uint8_t* packet = NULL;\n uint8_t* result = NULL;\n size_t result_length = 0;\n\n // Parse arguments: string, integer, and sequence\n if (!PyArg_ParseTuple(args, \"siO\", &socket_path, &mid, &py_args)) {\n PyRAISE(\"Failed to parse argument\")\n }\n\n // Verify third argument is a sequence\n if (!PySequence_Check(py_args)) {\n PyRAISE(\"Third argument must be a sequence\");\n }\n\n // Get sequence size and allocate C arrays\n nargs = PySequence_Size(py_args);\n arg_packets = (const uint8_t**)calloc(nargs, sizeof(uint8_t*));\n if (!arg_packets) {\n PyErr_NoMemory();\n goto error;\n }\n\n // Convert Python bytes to C buffers\n for (i = 0; i < nargs; i++) {\n PyObject* item = PySequence_GetItem(py_args, i);\n if (!PyBytes_Check(item)) {\n Py_DECREF(item);\n free(arg_packets);\n arg_packets = NULL;\n PyRAISE(\"All arguments must be bytes objects\");\n }\n arg_packets[i] = (const uint8_t*)PyBytes_AsString(item);\n Py_DECREF(item);\n }\n\n packet = PyTRY(make_morloc_local_call_packet, (uint32_t)mid, arg_packets, (size_t)nargs);\n\n free(arg_packets);\n arg_packets = NULL;\n\n result = PyTRY(send_and_receive_over_socket, socket_path, packet);\n free(packet);\n packet = NULL;\n\n // Incref the result's SHM so the callee's tracker flush won't destroy\n // data we may still need (e.g. forwarded result packets).\n {\n const morloc_packet_header_t* res_header = (const morloc_packet_header_t*)result;\n if (res_header->command.data.source == PACKET_SOURCE_RPTR) {\n size_t relptr = *(size_t*)((uint8_t*)result + res_header->offset + sizeof(morloc_packet_header_t));\n char* resolve_err = NULL;\n void* res_voidstar = rel2abs(relptr, &resolve_err);\n if (resolve_err) { free(resolve_err); resolve_err = NULL; }\n if (res_voidstar) {\n char* incref_err = NULL;\n shincref((absptr_t)res_voidstar, &incref_err);\n if (incref_err) { free(incref_err); }\n shm_tracker_push((absptr_t)res_voidstar, NULL);\n }\n }\n }\n\n result_length = PyTRY(morloc_packet_size, result);\n\n PyObject* retval = PyBytes_FromStringAndSize((char*)result, result_length);\n\n free(result);\n\n return retval;\n\nerror:\n FREE(arg_packets)\n FREE(packet)\n return NULL;\n}\n\n\nstatic PyObject* pybinding__remote_call(PyObject* self, PyObject* args) { MAYFAIL\n int midx;\n char* socket_base;\n char* cache_path;\n PyObject* res_struct; // python struct that is converted to a resource_t struct\n PyObject* arg_packets_obj; // python list of bytes types\n const uint8_t** arg_packets = NULL;\n uint8_t* result = NULL;\n\n if (!PyArg_ParseTuple(args, \"issOO\", &midx, &socket_base, &cache_path, &res_struct, &arg_packets_obj)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n if (!PyBytes_Check(res_struct)) {\n PyRAISE(\"res_struct must be a bytes object from struct.pack()\");\n }\n\n // Ensure the resources struct is the right size\n if (PyBytes_Size(res_struct) != sizeof(resources_t)) {\n PyRAISE(\"Struct size mismatch\");\n }\n\n resources_t* res = (resources_t*)PyBytes_AsString(res_struct);\n PyTRACE(res == NULL)\n\n Py_ssize_t nargs = PyList_Size(arg_packets_obj);\n\n arg_packets = calloc(nargs, sizeof(uint8_t*));\n if (arg_packets == NULL) {\n PyRAISE(\"Memory allocation failed\");\n }\n\n for (Py_ssize_t i = 0; i < nargs; i++) {\n PyObject* packet_obj = PyList_GetItem(arg_packets_obj, i);\n if (!PyBytes_Check(packet_obj)) {\n PyRAISE(\"Packets must be bytes\");\n }\n arg_packets[i] = (uint8_t*)PyBytes_AsString(packet_obj);\n }\n\n result = PyTRY(\n remote_call,\n midx,\n socket_base,\n cache_path,\n res,\n arg_packets,\n (size_t)nargs\n );\n\n free(arg_packets);\n\n if (result == NULL) Py_RETURN_NONE;\n size_t result_length = PyTRY(morloc_packet_size, result);\n PyObject* py_result = PyBytes_FromStringAndSize((char*)result, result_length);\n free(result);\n return py_result;\n\nerror:\n if (result != NULL){\n free(result);\n }\n if (arg_packets != NULL){\n // The elements are handled by Python and should not be freed\n free(arg_packets);\n }\n return NULL;\n}\n\n\nstatic PyObject* pybinding__is_ping(PyObject* self, PyObject* args) { MAYFAIL\n char* packet;\n size_t packet_size;\n\n if (!PyArg_ParseTuple(args, \"y#\", &packet, &packet_size)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n bool is_ping = PyTRY(packet_is_ping, (uint8_t*)packet);\n\n PyObject* obj = PyBool_FromLong((long)is_ping);\n\n return obj;\n\nerror:\n return NULL;\n}\n\n\nstatic PyObject* pybinding__is_local_call(PyObject* self, PyObject* args) { MAYFAIL\n char* packet;\n size_t packet_size;\n\n if (!PyArg_ParseTuple(args, \"y#\", &packet, &packet_size)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n bool is_local_call = PyTRY(packet_is_local_call, (uint8_t*)packet);\n\n PyObject* obj = PyBool_FromLong((long)is_local_call);\n\n return obj;\n\nerror:\n return NULL;\n}\n\nstatic PyObject* pybinding__is_remote_call(PyObject* self, PyObject* args) { MAYFAIL\n char* packet;\n size_t packet_size;\n\n if (!PyArg_ParseTuple(args, \"y#\", &packet, &packet_size)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n bool is_remote_call = PyTRY(packet_is_remote_call, (uint8_t*)packet);\n\n PyObject* obj = PyBool_FromLong((long)is_remote_call);\n\n return obj;\n\nerror:\n return NULL;\n}\n\n\nstatic PyObject* pybinding__pong(PyObject* self, PyObject* args) { MAYFAIL\n char* packet;\n size_t packet_size;\n uint8_t* pong = NULL;\n\n if (!PyArg_ParseTuple(args, \"y#\", &packet, &packet_size)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n pong = PyTRY(return_ping, (uint8_t*)packet);\n\n size_t pong_size = PyTRY(morloc_packet_size, pong);\n\n {\n PyObject* retval = PyBytes_FromStringAndSize((char*)pong, pong_size);\n free(pong);\n return retval;\n }\n\nerror:\n FREE(pong)\n return NULL;\n}\n\nstatic PyObject* pybinding__set_fallback_dir(PyObject* self, PyObject* args) {\n const char* dir;\n if (!PyArg_ParseTuple(args, \"s\", &dir)) {\n return NULL;\n }\n shm_set_fallback_dir(dir);\n Py_RETURN_NONE;\n}\n\nstatic PyObject* pybinding__shinit(PyObject* self, PyObject* args) { MAYFAIL\n shm_t* shm = NULL;\n \n const char* shm_basename;\n size_t volume_index;\n size_t shm_default_size;\n\n if (!PyArg_ParseTuple(args, \"skk\", &shm_basename, &volume_index, &shm_default_size)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n shm = PyTRY(\n shinit,\n shm_basename,\n volume_index,\n shm_default_size\n );\n\n return PyCapsule_New(shm, \"shm_t\", NULL);\n\nerror:\n FREE(shm)\n return NULL;\n}\n\n\nstatic PyObject* pybinding__make_fail_packetg(PyObject* self, PyObject* args) { MAYFAIL\n const char* packet_errmsg;\n uint8_t* packet = NULL;\n\n if (!PyArg_ParseTuple(args, \"s\", &packet_errmsg)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n packet = make_fail_packet(packet_errmsg);\n\n size_t packet_size = PyTRY(morloc_packet_size, packet);\n\n {\n PyObject* retval = PyBytes_FromStringAndSize((char*)packet, packet_size);\n free(packet);\n return retval;\n }\n\nerror:\n FREE(packet)\n return NULL;\n}\n\nstatic PyObject* pybinding__mlc_hash(PyObject* self, PyObject* args) { MAYFAIL\n PyObject* obj;\n const char* schema_str;\n Schema* schema = NULL;\n void* voidstar = NULL;\n char* hex = NULL;\n\n if (!PyArg_ParseTuple(args, \"Os\", &obj, &schema_str)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n voidstar = to_voidstar(schema, obj);\n PyTRACE(voidstar == NULL)\n\n hex = PyTRY(mlc_hash, voidstar, schema);\n\n {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n\n {\n PyObject* retval = PyUnicode_FromString(hex);\n free(hex);\n return retval;\n }\n\nerror:\n if (voidstar) {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n FREE(hex)\n return NULL;\n}\n\nstatic PyObject* pybinding__mlc_save(PyObject* self, PyObject* args) { MAYFAIL\n PyObject* obj;\n const char* schema_str;\n const char* path;\n Schema* schema = NULL;\n void* voidstar = NULL;\n\n if (!PyArg_ParseTuple(args, \"Oss\", &obj, &schema_str, &path)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n voidstar = to_voidstar(schema, obj);\n PyTRACE(voidstar == NULL)\n\n PyTRY(mlc_save, voidstar, schema, path);\n\n {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n Py_RETURN_NONE;\n\nerror:\n if (voidstar) {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n return NULL;\n}\n\nstatic PyObject* pybinding__mlc_save_voidstar(PyObject* self, PyObject* args) { MAYFAIL\n PyObject* obj;\n const char* schema_str;\n const char* path;\n Schema* schema = NULL;\n void* voidstar = NULL;\n\n if (!PyArg_ParseTuple(args, \"Oss\", &obj, &schema_str, &path)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n voidstar = to_voidstar(schema, obj);\n PyTRACE(voidstar == NULL)\n\n PyTRY(mlc_save_voidstar, voidstar, schema, path);\n\n {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n Py_RETURN_NONE;\n\nerror:\n if (voidstar) {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n return NULL;\n}\n\nstatic PyObject* pybinding__mlc_save_json(PyObject* self, PyObject* args) { MAYFAIL\n PyObject* obj;\n const char* schema_str;\n const char* path;\n Schema* schema = NULL;\n void* voidstar = NULL;\n\n if (!PyArg_ParseTuple(args, \"Oss\", &obj, &schema_str, &path)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n voidstar = to_voidstar(schema, obj);\n PyTRACE(voidstar == NULL)\n\n PyTRY(mlc_save_json, voidstar, schema, path);\n\n {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n Py_RETURN_NONE;\n\nerror:\n if (voidstar) {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n return NULL;\n}\n\nstatic PyObject* pybinding__mlc_show(PyObject* self, PyObject* args) { MAYFAIL\n PyObject* obj;\n const char* schema_str;\n Schema* schema = NULL;\n void* voidstar = NULL;\n char* json = NULL;\n\n if (!PyArg_ParseTuple(args, \"Os\", &obj, &schema_str)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n voidstar = to_voidstar(schema, obj);\n PyTRACE(voidstar == NULL)\n\n json = PyTRY(mlc_show, voidstar, schema);\n\n {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n\n {\n PyObject* retval = PyUnicode_FromString(json);\n free(json);\n return retval;\n }\n\nerror:\n if (voidstar) {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n FREE(json)\n return NULL;\n}\n\nstatic PyObject* pybinding__mlc_read(PyObject* self, PyObject* args) { MAYFAIL\n const char* schema_str;\n const char* json_str;\n Schema* schema = NULL;\n void* voidstar = NULL;\n\n if (!PyArg_ParseTuple(args, \"ss\", &schema_str, &json_str)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n {\n char* errmsg = NULL;\n voidstar = mlc_read(json_str, schema, &errmsg);\n if (errmsg != NULL) {\n free(errmsg);\n }\n }\n\n if (voidstar == NULL) {\n free_schema(schema);\n Py_RETURN_NONE;\n }\n\n {\n PyObject* obj = fromAnything(schema, voidstar, NULL);\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n free_schema(schema);\n PyTRACE(obj == NULL)\n return obj;\n }\n\nerror:\n if (voidstar) {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n return NULL;\n}\n\nstatic PyObject* pybinding__mlc_load(PyObject* self, PyObject* args) { MAYFAIL\n const char* schema_str;\n const char* path;\n Schema* schema = NULL;\n void* voidstar = NULL;\n\n if (!PyArg_ParseTuple(args, \"ss\", &schema_str, &path)) {\n PyRAISE(\"Failed to parse arguments\");\n }\n\n schema = PyTRY(parse_schema, schema_str);\n\n voidstar = PyTRY(mlc_load, path, schema);\n\n if (voidstar == NULL) {\n free_schema(schema);\n Py_RETURN_NONE;\n }\n\n {\n PyObject* obj = fromAnything(schema, voidstar, NULL);\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n free_schema(schema);\n PyTRACE(obj == NULL)\n return obj;\n }\n\nerror:\n if (voidstar) {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n return NULL;\n}\n\nstatic PyMethodDef Methods[] = {\n {\"set_fallback_dir\", pybinding__set_fallback_dir, METH_VARARGS, \"Set fallback directory for file-backed shared memory\"},\n {\"shinit\", pybinding__shinit, METH_VARARGS, \"Open the shared memory pool\"},\n {\"start_daemon\", pybinding__start_daemon, METH_VARARGS, \"Initialize the shared memory and socket for the python daemon\"},\n {\"close_daemon\", pybinding__close_daemon, METH_VARARGS, \"Banish the daemon back to the abyss from whence it came\"},\n {\"wait_for_client\", pybinding__wait_for_client, METH_VARARGS, \"Listen over a pipe until a client packet arrives\"},\n {\"read_morloc_call_packet\", pybinding__read_morloc_call_packet, METH_VARARGS, \"Parse a morloc call packet\"},\n {\"send_packet_to_foreign_server\", pybinding__send_packet_to_foreign_server, METH_VARARGS, \"Send data to a foreign server\"},\n {\"stream_from_client\", pybinding__stream_from_client, METH_VARARGS, \"Stream data from the client\"},\n {\"close_socket\", pybinding__close_socket, METH_VARARGS, \"Close the socket\"},\n {\"flush_shm_tracker\", pybinding__flush_shm_tracker, METH_NOARGS, \"Free tracked SHM allocations from put_value calls\"},\n {\"foreign_call\", pybinding__foreign_call, METH_VARARGS, \"Send a call packet to a foreign pool\"},\n {\"get_value\", pybinding__get_value, METH_VARARGS, \"Convert a packet to a Python value\"},\n {\"put_value\", pybinding__put_value, METH_VARARGS, \"Convert a Python value to a packet\"},\n {\"is_ping\", pybinding__is_ping, METH_VARARGS, \"Packet is a ping\"},\n {\"is_local_call\", pybinding__is_local_call, METH_VARARGS, \"Packet is a local call\"},\n {\"is_remote_call\", pybinding__is_remote_call, METH_VARARGS, \"Packet is a remote call\"},\n {\"pong\", pybinding__pong, METH_VARARGS, \"Return a ping\"},\n {\"make_fail_packet\", pybinding__make_fail_packetg, METH_VARARGS, \"Create a fail packet from an error message\"},\n {\"remote_call\", pybinding__remote_call, METH_VARARGS, \"Make a call to a remote cluster\"},\n {\"mlc_hash\", pybinding__mlc_hash, METH_VARARGS, \"Hash a value using xxhash\"},\n {\"mlc_save\", pybinding__mlc_save, METH_VARARGS, \"Save a value to file in msgpack format\"},\n {\"mlc_save_voidstar\", pybinding__mlc_save_voidstar, METH_VARARGS, \"Save a value to file in flat voidstar binary format\"},\n {\"mlc_save_json\", pybinding__mlc_save_json, METH_VARARGS, \"Save a value to file in JSON format\"},\n {\"mlc_load\", pybinding__mlc_load, METH_VARARGS, \"Load a value from file\"},\n {\"mlc_show\", pybinding__mlc_show, METH_VARARGS, \"Serialize a value to JSON string\"},\n {\"mlc_read\", pybinding__mlc_read, METH_VARARGS, \"Deserialize a JSON string to a value\"},\n {NULL, NULL, 0, NULL} // this is a sentinel value\n};\n\nstatic struct PyModuleDef pymorloc = {\n PyModuleDef_HEAD_INIT,\n \"pymorloc\",\n \"Python interface to Morloc binary and MessagePack data\",\n -1,\n Methods\n};\n\nPyMODINIT_FUNC PyInit_pymorloc(void) {\n return PyModule_Create(&pymorloc);\n}\n" }, { "path": "data/lang/py/setup.py", "content": "import os\nfrom setuptools import setup, Extension\nimport numpy as np\n\ntry:\n np_include_path = np.get_include()\nexcept AttributeError:\n raise RuntimeError(\"Numpy is required to build this extension\")\n\nmorloc_home = os.environ.get(\n 'MORLOC_HOME',\n os.path.expanduser('~/.local/share/morloc')\n)\n\nmodule = Extension(\n 'pymorloc',\n sources=['pymorloc.c'],\n include_dirs=[\n os.path.join(morloc_home, 'include'),\n np_include_path\n ],\n library_dirs=[os.path.join(morloc_home, 'lib')],\n runtime_library_dirs=[os.path.join(morloc_home, 'lib')],\n libraries=['morloc']\n)\n\nsetup(\n name='pymorloc',\n version='0.1',\n ext_modules=[module],\n extras_require={\n 'numpy': ['numpy']\n }\n)\n" }, { "path": "data/lang/r/init.sh", "content": "#!/bin/bash\nset -e\n\nMORLOC_HOME=\"$1\"\nBUILD_DIR=\"$2\"\nSANITIZE_FLAGS=\"$3\"\nINCLUDE_DIR=\"$MORLOC_HOME/include\"\nLIB_DIR=\"$MORLOC_HOME/lib\"\n\n# Write source to include dir (R CMD SHLIB expects it there)\ncp \"$BUILD_DIR/rmorloc.c\" \"$INCLUDE_DIR/\"\n\n# Compile directly with gcc so SANITIZE_FLAGS can be passed at both compile\n# and link time. R CMD SHLIB drops sanitizer flags from the link step, which\n# would leave librmorloc.so with unresolved ubsan symbols (R itself is not\n# built with ubsan, so dlopen would fail with:\n# \"undefined symbol: __ubsan_handle_type_mismatch_v1_abort\").\n#\n# We only link against R's core shared lib (libR). R's full --ldflags include\n# libs the R interpreter uses (pcre2, tirpc, icu...) that may not be present\n# as separate dev packages on the build machine; R-loadable shared objects\n# resolve those symbols through the loaded R interpreter at dlopen time, not\n# at link time.\nR_CPPFLAGS=$(R CMD config --cppflags)\nR_HOME=$(R RHOME)\ngcc $R_CPPFLAGS -I\"$INCLUDE_DIR\" $SANITIZE_FLAGS -fpic -O2 \\\n -c \"$INCLUDE_DIR/rmorloc.c\" -o \"$INCLUDE_DIR/rmorloc.o\"\ngcc -shared $SANITIZE_FLAGS \\\n -Wl,-Bsymbolic-functions -Wl,-z,relro \\\n -o \"$LIB_DIR/librmorloc.so\" \"$INCLUDE_DIR/rmorloc.o\" \\\n -L\"$LIB_DIR\" -Wl,-rpath,\"$LIB_DIR\" -lmorloc -lpthread \\\n -L\"$R_HOME/lib\" -lR\n\n# Clean up\nrm -f \"$INCLUDE_DIR/rmorloc.c\" \"$INCLUDE_DIR/rmorloc.o\"\n" }, { "path": "data/lang/r/lang.yaml", "content": "# R language descriptor for morloc compiler\n# Metadata fields (read by LangRegistry) + descriptor fields (read by generic translator)\n\n# Identity and metadata\nname: r\nextension: R\naliases: []\nis_compiled: false\nrun_command: [\"Rscript\"]\nserial_type: \"character\"\ncost: 4\npreamble:\n - 'dyn.load(\"{{home}}/lib/librmorloc.so\")'\n - '.morloc.srcdir <- normalizePath(file.path(dirname(sub(\"^--file=\", \"\", grep(\"^--file=\", commandArgs(FALSE), value=TRUE)[1])), \"..\", \"..\"), mustWork=FALSE)'\n - '.morloc.source <- function(p) source(ifelse(startsWith(p, \"/\"), p, file.path(.morloc.srcdir, p)), chdir=TRUE)'\n\n# Literals\nldBoolTrue: \"TRUE\"\nldBoolFalse: \"FALSE\"\nldNullLiteral: \"NULL\"\n\n# Constructors\nldListStyle: type_dependent\nldTupleConstructor: \"list\"\nldRecordConstructor: \"list\"\nldRecordSeparator: \"=\"\n\n# Access styles\nldIndexStyle: one_double_bracket\nldKeyAccess: \"double_bracket\"\nldFieldAccess: dollar\n\n# Serialize/deserialize\nldSerializeFn: \"morloc_put_value\"\nldDeserializeFn: \"morloc_get_value\"\nldIntrinsicPrefix: \"morloc_\"\n\n# Foreign call\nldForeignCallFn: \"morloc_foreign_call\"\nldForeignCallIntSuffix: \"L\"\nldIntLiteralSuffix: \"L\"\n\n# Remote call\nldRemoteCallFn: \"morloc_remote_call\"\n\n# Record handling\nldDictStyleRecords: false\nldQuoteRecordKeys: false\n\n# Import syntax\nldQualifiedImports: false\nldIncludeRelToFile: false\n\n# Template fields\nldAssignOp: \"<-\"\nldLambdaTemplate: \"function({{args}}) { {{body}} }\"\nldDoBlockExpr: \"(function() {{expr}})\"\nldDoBlockBlock: \"(function(){\\n{{body}}\\n})\"\nldPartialTemplate: \"function({{bound_args}}) { {{fn}}({{all_args}}) }\"\nldImportTemplate: \".morloc.source(\\\"{{path}}\\\")\"\nldSocketPathTemplate: \"paste0(global_state$tmpdir, \\\"/\\\", {{socket}})\"\nldResourcePackTemplate: \"list(memory={{mem}}L, time={{time}}L, cpus={{cpus}}L, gpus={{gpus}}L)\"\nldReturnTemplate: \"return({{expr}})\"\nldFuncDefHeader: \"{{name}} <- function({{args}})\"\nldBlockStyle: braces\nldBlockEnd: \"}\"\nldErrorWrapOpen: \"\"\nldErrorWrapClose: []\nldPatternStyle: concat_call\nldConcatFn: \"paste0\"\nldQuoteTerminator: '\"'\nldQuoteTerminatorEsc: '\\\"'\nldAtomicTypes: [\"integer\", \"numeric\", \"double\", \"logical\", \"character\"]\nldAtomicListFn: \"c\"\nldGenericListFn: \"list\"\nldMapStyle: apply_callback\nldDispatchLocalHeader: \".dispatch <- list()\"\nldDispatchLocalEntry: \".dispatch[[{{mid}}L]] <- {{name}}\"\nldDispatchLocalFooter: \"\"\nldDispatchRemoteHeader: \".remote_dispatch <- list()\"\nldDispatchRemoteEntry: \".remote_dispatch[[{{mid}}L]] <- {{name}}_remote\"\nldDispatchRemoteFooter: \"\"\n\n# Pool template (loaded from pool.R, left empty here)\nldPoolTemplate: \"\"\nldBreakMarker: \"# <<>>\"\nldCommentMarker: \"#\"\n" }, { "path": "data/lang/r/pool.R", "content": "# AUTO include sources start\n# <<>>\n# AUTO include sources end\n\nmorloc_is_ping <- function(...){ .Call(\"morloc_is_ping\", ...) }\nmorloc_pong <- function(...){ .Call(\"morloc_pong\", ...) }\nmorloc_is_local_call <- function(...){ .Call(\"morloc_is_local_call\", ...) }\nmorloc_is_remote_call <- function(...){ .Call(\"morloc_is_remote_call\", ...) }\nmorloc_make_fail_packet <- function(...){ .Call(\"morloc_make_fail_packet\", ...) }\nmorloc_wait_for_client <- function(...){ .Call(\"morloc_wait_for_client\", ...) }\nmorloc_stream_from_client <- function(...){ .Call(\"morloc_stream_from_client\", ...) }\nmorloc_read_morloc_call_packet <- function(...){ .Call(\"morloc_read_morloc_call_packet\", ...) }\nmorloc_send_packet_to_foreign_server <- function(...){ .Call(\"morloc_send_packet_to_foreign_server\", ...) }\nmorloc_close_socket <- function(...){ .Call(\"morloc_close_socket\", ...) }\nmorloc_start_daemon <- function(...){ .Call(\"morloc_start_daemon\", ...) }\nmorloc_shinit <- function(...){ .Call(\"morloc_shinit\", ...) }\nmorloc_foreign_call <- function(...){ .Call(\"morloc_foreign_call\", ...) }\nmorloc_get_value <- function(...){ .Call(\"morloc_get_value\", ...) }\nmorloc_put_value <- function(...){ .Call(\"morloc_put_value\", ...) }\nmorloc_mlc_show <- function(...){ .Call(\"morloc_mlc_show\", ...) }\nmorloc_socketpair <- function(...){ .Call(\"morloc_socketpair\", ...) }\nmorloc_fork <- function(...){ .Call(\"morloc_fork\", ...) }\nmorloc_send_fd <- function(...){ .Call(\"morloc_send_fd\", ...) }\nmorloc_recv_fd <- function(...){ .Call(\"morloc_recv_fd\", ...) }\nmorloc_kill <- function(...){ .Call(\"morloc_kill\", ...) }\nmorloc_waitpid <- function(...){ .Call(\"morloc_waitpid\", ...) }\nmorloc_install_sigterm_handler <- function(...){ .Call(\"morloc_install_sigterm_handler\", ...) }\nmorloc_is_shutting_down <- function(...){ .Call(\"morloc_is_shutting_down\", ...) }\nmorloc_waitpid_blocking <- function(...){ .Call(\"morloc_waitpid_blocking\", ...) }\nmorloc_detach_daemon <- function(...){ .Call(\"morloc_detach_daemon\", ...) }\nmorloc_shared_counter_create <- function(...){ .Call(\"morloc_shared_counter_create\", ...) }\nmorloc_shared_counter_inc <- function(...){ .Call(\"morloc_shared_counter_inc\", ...) }\nmorloc_shared_counter_dec <- function(...){ .Call(\"morloc_shared_counter_dec\", ...) }\nmorloc_shared_counter_read <- function(...){ .Call(\"morloc_shared_counter_read\", ...) }\nmorloc_pipe <- function(...){ .Call(\"morloc_pipe\", ...) }\nmorloc_write_byte <- function(...){ .Call(\"morloc_write_byte\", ...) }\nmorloc_close_fd <- function(...){ .Call(\"morloc_close_fd\", ...) }\nmorloc_worker_loop_c <- function(...){ .Call(\"morloc_worker_loop_c\", ...) }\nmorloc_set_line_buffered <- function(...){ .Call(\"morloc_set_line_buffered\", ...) }\nmorloc_exit <- function(...){ .Call(\"morloc_exit\", ...) }\n\nglobal_state <- list()\n\n# Dynamic worker spawning: monkey-patch morloc_foreign_call to track busy workers.\n# Workers atomically increment a shared counter before a foreign_call and\n# decrement after. When all workers are busy, a byte is written to a wake-up\n# pipe to tell the dispatcher to spawn a new worker.\n.orig_foreign_call <- morloc_foreign_call\n.busy_counter <- NULL\n.wakeup_fd <- NULL\n.n_workers_total <- 0L\n\nmorloc_foreign_call <- function(...) {\n val <- morloc_shared_counter_inc(.busy_counter)\n if (val >= .n_workers_total && !is.null(.wakeup_fd)) {\n tryCatch(morloc_write_byte(.wakeup_fd, as.raw(0x21)), error = function(e) NULL)\n }\n on.exit(morloc_shared_counter_dec(.busy_counter))\n .orig_foreign_call(...)\n}\n\n# AUTO include manifolds start\n# <<>>\n# AUTO include manifolds end\n\n# AUTO include dispatch start\n# <<>>\n# AUTO include dispatch end\n\nworker_loop <- function(pipe_fd) {\n morloc_worker_loop_c(pipe_fd, .dispatch, .remote_dispatch)\n}\n\nmain <- function(socket_path, tmpdir, shm_basename) {\n # Force line-buffered stdout/stderr so output from user functions is not lost\n # when the nexus kills the pool process group.\n morloc_set_line_buffered()\n morloc_install_sigterm_handler()\n\n daemon <- morloc_start_daemon(socket_path, tmpdir, shm_basename, 0xffff)\n n_workers <- 1L\n\n # Shared job queue: dispatcher writes fds to fd[1], workers read from fd[2].\n # Only idle workers (blocked in recvmsg) pick up jobs, preventing the\n # round-robin deadlock where a callback gets dispatched to a busy worker.\n job_queue <- morloc_socketpair()\n\n # Shared counter for dynamic worker spawning\n busy_counter <- morloc_shared_counter_create()\n wakeup <- morloc_pipe() # c(read_fd, write_fd)\n\n # Set globals so the monkey-patched morloc_foreign_call can use them.\n # Forked children inherit these values.\n .busy_counter <<- busy_counter\n .wakeup_fd <<- wakeup[2L]\n .n_workers_total <<- n_workers\n\n pids <- integer(n_workers)\n for (i in seq_len(n_workers)) {\n pid <- morloc_fork()\n if (pid == 0L) {\n morloc_detach_daemon(daemon)\n morloc_close_socket(job_queue[1L]) # child doesn't write\n morloc_close_fd(wakeup[1L]) # child doesn't read wakeup pipe\n worker_loop(job_queue[2L])\n morloc_exit(0L)\n }\n pids[i] <- pid\n }\n # Keep job_queue[2L] open so dynamically spawned children can use it\n\n on.exit({\n tryCatch(morloc_close_socket(job_queue[1L]), error = function(e) NULL)\n tryCatch(morloc_close_socket(job_queue[2L]), error = function(e) NULL)\n tryCatch(morloc_close_fd(wakeup[1L]), error = function(e) NULL)\n tryCatch(morloc_close_fd(wakeup[2L]), error = function(e) NULL)\n for (pid in pids) {\n if (pid > 0L) {\n tryCatch(morloc_kill(pid, 9L), error = function(e) NULL)\n tryCatch(morloc_waitpid_blocking(pid), error = function(e) NULL)\n }\n }\n })\n\n # Dispatch loop - idle workers pull from shared queue.\n # After each dispatch cycle, check if all workers are busy and spawn more.\n while (!morloc_is_shutting_down()) {\n client_fd <- morloc_wait_for_client(daemon)\n if (client_fd > 0L) {\n tryCatch({\n morloc_send_fd(job_queue[1L], client_fd)\n }, error = function(e) {\n cat(paste(\"Failed to dispatch job:\", e$message, \"\\n\"), file = stderr())\n }, finally = {\n morloc_close_socket(client_fd)\n })\n }\n\n # Dynamic worker spawning: if all workers are blocked in foreign_call,\n # spawn a new one so incoming callbacks can still be served.\n current_busy <- morloc_shared_counter_read(busy_counter)\n if (current_busy >= n_workers) {\n pid <- morloc_fork()\n if (pid == 0L) {\n morloc_detach_daemon(daemon)\n morloc_close_socket(job_queue[1L])\n morloc_close_fd(wakeup[1L])\n worker_loop(job_queue[2L])\n morloc_exit(0L)\n }\n pids <- c(pids, pid)\n n_workers <- n_workers + 1L\n .n_workers_total <<- n_workers\n }\n }\n}\n\nargs <- commandArgs(trailingOnly = TRUE)\n\n# Health check: confirm sources loaded and print version\nif (length(args) == 1 && args[1] == \"--health\") {\n cat('{\"status\":\"ok\",\"version\":\"__MORLOC_VERSION__\"}\\n')\n quit(status = 0)\n}\n\nif (length(args) != 3) {\n cat(\"Usage: Rscript pool.R \\n\", file=stderr())\n quit(status = 1)\n}\n\nsocket_path <- args[1]\ntmpdir <- args[2]\nshm_basename <- args[3]\n\nglobal_state$tmpdir <- tmpdir\n\ntryCatch(\n {\n main(socket_path, tmpdir, shm_basename)\n }, error = function(e) {\n stop(paste(\"Pool failed:\", e$message))\n })\n\n# Use _exit to avoid R cleanup which triggers heap corruption on glibc >= 2.39\n# (R's finalizers attempt to free objects in SHM-related C extensions)\nmorloc_exit(0L)\n" }, { "path": "data/lang/r/rmorloc.c", "content": "#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"morloc.h\"\n\n// {{{ macros\n\n#define MAYFAIL char* child_errmsg_ = NULL;\n\n#define R_TRY(fun, ...) \\\n fun(__VA_ARGS__ __VA_OPT__(,) &child_errmsg_); \\\n if(child_errmsg_ != NULL){ \\\n error(\"Error in R pool (%s:%d in %s):\\n%s\", __FILE__, __LINE__, __func__, child_errmsg_); \\\n }\n\n#define R_TRY_WITH(clean, fun, ...) \\\n fun(__VA_ARGS__ __VA_OPT__(,) &child_errmsg_); \\\n if(child_errmsg_ != NULL){ \\\n clean; \\\n error(\"Error in R pool (%s:%d in %s):\\n%s\", __FILE__, __LINE__, __func__, child_errmsg_); \\\n }\n\n#define MORLOC_ERROR(msg, ...) error(\"Error in R pool (%s:%d in %s):\" msg, __FILE__, __LINE__, __func__, ##__VA_ARGS__);\n\n/// }}}\n\n// {{{ to_voidstar\n\nstatic size_t get_shm_size(const Schema* schema, SEXP obj) {\n size_t size = 0;\n switch (schema->type) {\n case MORLOC_NIL:\n case MORLOC_BOOL:\n case MORLOC_SINT8:\n case MORLOC_SINT16:\n case MORLOC_SINT32:\n case MORLOC_SINT64:\n case MORLOC_UINT8:\n case MORLOC_UINT16:\n case MORLOC_UINT32:\n case MORLOC_UINT64:\n case MORLOC_FLOAT32:\n case MORLOC_FLOAT64:\n return schema->width;\n case MORLOC_STRING:\n case MORLOC_ARRAY:\n {\n size_t length = (size_t)LENGTH(obj);\n size = sizeof(Array);\n // worst-case cursor alignment padding for element data\n size += schema_alignment(schema->parameters[0]) - 1;\n const char* str;\n\n switch (TYPEOF(obj)) {\n case CHARSXP:\n str = CHAR(obj);\n size += (size_t)strlen(str); // Do not include null terminator\n break;\n case STRSXP:\n if (LENGTH(obj) == 1) {\n str = CHAR(STRING_ELT(obj, 0));\n size += (size_t)strlen(str); // Do not include null terminator\n } else {\n if(schema->parameters[0]->type == MORLOC_STRING){\n for(size_t i = 0; i < length; i++){\n size += get_shm_size(schema->parameters[0], STRING_ELT(obj, i));\n }\n } else {\n MORLOC_ERROR(\"Expected character vector of length 1, but got length %zu\", length);\n }\n }\n break;\n case VECSXP: // This handles lists\n for (int i = 0; i < length; i++) {\n size += get_shm_size(schema->parameters[0], VECTOR_ELT(obj, i));\n }\n break;\n case LGLSXP:\n case INTSXP:\n case REALSXP:\n case RAWSXP:\n size += length * schema->parameters[0]->width;\n break;\n default:\n MORLOC_ERROR(\"Unsupported type in get_shm_size array: %s\", type2char(TYPEOF(obj)));\n }\n return size;\n }\n\n case MORLOC_TUPLE:\n if (!isVectorList(obj)) {\n MORLOC_ERROR(\"Expected list for MORLOC_TUPLE, but got %s\", type2char(TYPEOF(obj)));\n }\n\n {\n size_t array_size = (size_t)xlength(obj);\n if (array_size != schema->size) {\n MORLOC_ERROR(\"Expected tuple of length %zu, but found list of length %zu\", schema->size, size);\n }\n size = schema->width;\n for (R_xlen_t i = 0; i < (R_xlen_t)array_size; ++i) {\n SEXP item = VECTOR_ELT(obj, i);\n size_t elem = get_shm_size(schema->parameters[i], item);\n if (elem > schema->parameters[i]->width) {\n size += elem - schema->parameters[i]->width;\n }\n }\n return size;\n }\n\n case MORLOC_MAP:\n {\n if (isNewList(obj)) {\n // Handle named list\n size = schema->width;\n SEXP names = getAttrib(obj, R_NamesSymbol);\n if (names == R_NilValue) {\n error(\"List must have names for MORLOC_MAP\");\n }\n for (size_t i = 0; i < schema->size; ++i) {\n SEXP key = PROTECT(mkChar(schema->keys[i]));\n int index = -1;\n for (int j = 0; j < length(obj); j++) {\n if (strcmp(CHAR(STRING_ELT(names, j)), CHAR(key)) == 0) {\n index = j;\n break;\n }\n }\n if (index != -1) {\n SEXP value = VECTOR_ELT(obj, index);\n size_t elem = get_shm_size(schema->parameters[i], value);\n if (elem > schema->parameters[i]->width) {\n size += elem - schema->parameters[i]->width;\n }\n }\n UNPROTECT(1);\n }\n return size;\n } else {\n error(\"Expected a named list for MORLOC_MAP\");\n }\n }\n\n case MORLOC_OPTIONAL:\n if (obj == R_NilValue) {\n return schema->width;\n }\n {\n size_t inner_size = get_shm_size(schema->parameters[0], obj);\n size = schema->width;\n if (inner_size > schema->parameters[0]->width) {\n size += inner_size - schema->parameters[0]->width;\n }\n return size;\n }\n\n case MORLOC_TENSOR:\n {\n size_t ndim = schema_tensor_ndim(schema);\n size_t elem_width = schema->parameters[0]->width;\n SEXP dim = getAttrib(obj, R_DimSymbol);\n size_t total = 1;\n if (dim != R_NilValue) {\n for (int i = 0; i < length(dim); i++)\n total *= (size_t)INTEGER(dim)[i];\n } else {\n total = (size_t)XLENGTH(obj);\n }\n size = sizeof(Tensor);\n size += _Alignof(int64_t) - 1;\n size += ndim * sizeof(int64_t);\n size += schema_alignment(schema->parameters[0]) - 1;\n size += total * elem_width;\n return size;\n }\n\n default:\n MORLOC_ERROR(\"Unhandled schema type\");\n break;\n }\n\n return size;\n}\n\n\n#define HANDLE_SINT_TYPE(CTYPE, MIN, MAX) \\\n do { \\\n if (!(isInteger(obj) || isReal(obj))) { \\\n MORLOC_ERROR(\"Expected integer for %s, but got %s\", #CTYPE, type2char(TYPEOF(obj))); \\\n } \\\n double value = asReal(obj); \\\n if (value < MIN || value > MAX) { \\\n MORLOC_ERROR(\"Integer overflow for %s\", #CTYPE); \\\n } \\\n *(CTYPE*)dest = (CTYPE)value; \\\n } while(0)\n\n#define HANDLE_UINT_TYPE(CTYPE, MAX) \\\n do { \\\n if (!(isInteger(obj) || isReal(obj))) { \\\n MORLOC_ERROR(\"Expected integer for %s, but got %s\", #CTYPE, type2char(TYPEOF(obj))); \\\n } \\\n double value = asReal(obj); \\\n if (value < 0 || value > MAX) { \\\n MORLOC_ERROR(\"Integer overflow for %s\", #CTYPE); \\\n } \\\n *(CTYPE*)dest = (CTYPE)value; \\\n } while(0)\n\nstatic void* to_voidstar_r(void* dest, void** cursor, SEXP obj, const Schema* schema){\n MAYFAIL\n\n switch (schema->type) {\n case MORLOC_NIL:\n if (obj != R_NilValue) {\n MORLOC_ERROR(\"Expected NULL for MORLOC_NIL, but got %s\", type2char(TYPEOF(obj)));\n }\n *((int8_t*)dest) = (int8_t)0;\n break;\n case MORLOC_BOOL:\n if (!isLogical(obj)) {\n MORLOC_ERROR(\"Expected logical for MORLOC_BOOL, but got %s\", type2char(TYPEOF(obj)));\n }\n *((uint8_t*)dest) = (uint8_t)((LOGICAL(obj)[0] == TRUE) ? 1 : 0);\n break;\n case MORLOC_SINT8:\n HANDLE_SINT_TYPE(int8_t, INT8_MIN, INT8_MAX);\n break;\n case MORLOC_SINT16:\n HANDLE_SINT_TYPE(int16_t, INT16_MIN, INT16_MAX);\n break;\n case MORLOC_SINT32:\n HANDLE_SINT_TYPE(int32_t, INT32_MIN, INT32_MAX);\n break;\n case MORLOC_SINT64:\n HANDLE_SINT_TYPE(int64_t, INT64_MIN, INT64_MAX);\n break;\n case MORLOC_UINT8:\n HANDLE_UINT_TYPE(uint8_t, UINT8_MAX);\n break;\n case MORLOC_UINT16:\n HANDLE_UINT_TYPE(uint16_t, UINT16_MAX);\n break;\n case MORLOC_UINT32:\n HANDLE_UINT_TYPE(uint32_t, UINT32_MAX);\n break;\n case MORLOC_UINT64:\n HANDLE_UINT_TYPE(uint64_t, UINT64_MAX);\n break;\n case MORLOC_FLOAT32:\n if (!(isReal(obj) || isInteger(obj))) {\n MORLOC_ERROR(\"Expected numeric for MORLOC_FLOAT32, but got %s\", type2char(TYPEOF(obj)));\n }\n *((float*)dest) = (float)asReal(obj);\n break;\n\n case MORLOC_FLOAT64:\n if (!(isReal(obj) || isInteger(obj))) {\n MORLOC_ERROR(\"Expected numeric for MORLOC_FLOAT64, but got %s\", type2char(TYPEOF(obj)));\n }\n *((double*)dest) = asReal(obj);\n break;\n case MORLOC_STRING:\n {\n const char* str = NULL;\n size_t length = 0;\n switch(TYPEOF(obj)){\n case CHARSXP:\n str = CHAR(obj);\n length = (size_t)strlen(str);\n break;\n case STRSXP:\n if (LENGTH(obj) == 1) {\n str = CHAR(STRING_ELT(obj, 0));\n length = (size_t)strlen(str);\n } else {\n MORLOC_ERROR(\"Expected character of length 1\");\n }\n break;\n case RAWSXP:\n str = RAW(obj);\n length = LENGTH(obj);\n break;\n default:\n MORLOC_ERROR(\"Expected a character type\");\n break;\n }\n Array* array = (Array*)dest;\n array->size = length; // Do not include null terminator\n if(length > 0){\n // align cursor for element data placement\n *cursor = (void*)ALIGN_UP((uintptr_t)*cursor, schema_alignment(schema->parameters[0]));\n array->data = R_TRY(abs2rel, *cursor);\n absptr_t tmp_ptr = R_TRY(rel2abs, array->data);\n memcpy(tmp_ptr, str, array->size);\n } else {\n array->data = RELNULL;\n }\n\n // move cursor to the location after the copied data\n *cursor = (void*)(*(char**)cursor + array->size);\n }\n break;\n case MORLOC_ARRAY:\n Array* array = (Array*)dest;\n array->size = (size_t)length(obj);\n if(array->size == 0){\n array->data = RELNULL;\n break;\n }\n\n // align cursor for element data placement\n *cursor = (void*)ALIGN_UP((uintptr_t)*cursor, schema_alignment(schema->parameters[0]));\n array->data = R_TRY(abs2rel, *cursor);\n Schema* element_schema = schema->parameters[0];\n char* start;\n\n switch (TYPEOF(obj)) {\n case STRSXP:\n {\n if(element_schema->type == MORLOC_STRING){\n // set the cursor the the location after the array headers\n *cursor = (void*)(*(char**)cursor + array->size * element_schema->width);\n start = R_TRY(rel2abs, array->data);\n for(size_t i = 0; i < array->size; i++){\n SEXP elem = STRING_ELT(obj, i);\n to_voidstar_r(start + i * element_schema->width, cursor, elem, element_schema);\n }\n } else {\n MORLOC_ERROR(\"Expected character vector of length 1, but got length %ld\", array->size);\n }\n }\n break;\n case RAWSXP: // Raw vectors\n if (element_schema->type != MORLOC_UINT8) {\n MORLOC_ERROR(\"Expected MORLOC_UINT8 for raw vector\");\n }\n absptr_t tmp_ptr = R_TRY(rel2abs, array->data);\n memcpy(tmp_ptr, RAW(obj), array->size * sizeof(uint8_t));\n *cursor = (void*)(*(char**)cursor + array->size * sizeof(uint8_t));\n break;\n case VECSXP: // This handles lists\n *cursor = (void*)(*(char**)cursor + array->size * element_schema->width);\n start = R_TRY(rel2abs, array->data);\n for (int i = 0; i < array->size; i++) {\n SEXP elem = VECTOR_ELT(obj, i);\n to_voidstar_r(start + i * element_schema->width, cursor, elem, element_schema);\n }\n break;\n case LGLSXP:\n *cursor = (void*)(*(char**)cursor + array->size * element_schema->width);\n start = R_TRY(rel2abs, array->data);\n for (int i = 0; i < array->size; i++) {\n SEXP elem = PROTECT(ScalarLogical(LOGICAL(obj)[i]));\n to_voidstar_r(start + i * element_schema->width, cursor, elem, element_schema);\n UNPROTECT(1);\n }\n break;\n case INTSXP:\n *cursor = (void*)(*(char**)cursor + array->size * element_schema->width);\n start = R_TRY(rel2abs, array->data);\n for (int i = 0; i < array->size; i++) {\n SEXP elem = PROTECT(ScalarInteger(INTEGER(obj)[i]));\n to_voidstar_r(start + i * element_schema->width, cursor, elem, element_schema);\n UNPROTECT(1);\n }\n break;\n case REALSXP:\n *cursor = (void*)(*(char**)cursor + array->size * element_schema->width);\n start = R_TRY(rel2abs, array->data);\n for (int i = 0; i < array->size; i++) {\n SEXP elem = PROTECT(ScalarReal(REAL(obj)[i]));\n to_voidstar_r(start + i * element_schema->width, cursor, elem, element_schema);\n UNPROTECT(1);\n }\n break;\n default:\n MORLOC_ERROR(\"Unsupported type in to_voidstar array: %s\", type2char(TYPEOF(obj)));\n }\n break;\n\n\n\n case MORLOC_TUPLE:\n if (!isVectorList(obj)) {\n MORLOC_ERROR(\"Expected list for MORLOC_TUPLE, but got %s\", type2char(TYPEOF(obj)));\n }\n\n {\n R_xlen_t size = xlength(obj);\n if ((size_t)size != schema->size) {\n MORLOC_ERROR(\"Expected tuple of length %zu, but found list of length %zu\", schema->size, size);\n }\n for (R_xlen_t i = 0; i < size; ++i) {\n SEXP item = VECTOR_ELT(obj, i);\n to_voidstar_r(dest + schema->offsets[i], cursor, item, schema->parameters[i]);\n }\n }\n break;\n\n case MORLOC_MAP:\n {\n if (isNewList(obj)) {\n // Handle named list\n SEXP names = getAttrib(obj, R_NamesSymbol);\n if (names == R_NilValue) {\n MORLOC_ERROR(\"List must have names for MORLOC_MAP\");\n }\n for (size_t i = 0; i < schema->size; ++i) {\n SEXP key = PROTECT(mkChar(schema->keys[i]));\n int index = -1;\n for (int j = 0; j < length(obj); j++) {\n if (strcmp(CHAR(STRING_ELT(names, j)), CHAR(key)) == 0) {\n index = j;\n break;\n }\n }\n if (index != -1) {\n SEXP value = VECTOR_ELT(obj, index);\n to_voidstar_r(dest + schema->offsets[i], cursor, value, schema->parameters[i]);\n }\n UNPROTECT(1);\n }\n } else {\n MORLOC_ERROR(\"Expected a named list for MORLOC_MAP\");\n }\n }\n break;\n\n case MORLOC_OPTIONAL:\n if (obj == R_NilValue) {\n *((uint8_t*)dest) = 0;\n memset((char*)dest + schema->offsets[0], 0, schema->parameters[0]->width);\n } else {\n *((uint8_t*)dest) = 1;\n to_voidstar_r((char*)dest + schema->offsets[0], cursor, obj, schema->parameters[0]);\n }\n break;\n\n case MORLOC_TENSOR:\n {\n size_t ndim = schema_tensor_ndim(schema);\n size_t elem_width = schema->parameters[0]->width;\n\n // Get shape from dim attribute (or length for 1D)\n SEXP dim = getAttrib(obj, R_DimSymbol);\n int64_t shape[5];\n size_t total = 1;\n if (dim != R_NilValue) {\n for (size_t i = 0; i < ndim; i++) {\n shape[i] = (int64_t)INTEGER(dim)[i];\n total *= (size_t)shape[i];\n }\n } else {\n shape[0] = (int64_t)XLENGTH(obj);\n total = (size_t)shape[0];\n }\n\n Tensor* tensor = (Tensor*)dest;\n tensor->total_elements = total;\n tensor->device_type = 0;\n tensor->device_id = 0;\n\n if (total == 0) {\n tensor->shape = RELNULL;\n tensor->data = RELNULL;\n break;\n }\n\n // Write shape\n *cursor = (void*)ALIGN_UP((uintptr_t)*cursor, _Alignof(int64_t));\n tensor->shape = R_TRY(abs2rel, (absptr_t)*cursor);\n int64_t* shape_dst = (int64_t*)*cursor;\n for (size_t i = 0; i < ndim; i++) shape_dst[i] = shape[i];\n *cursor = (char*)*cursor + ndim * sizeof(int64_t);\n\n // Write data: transpose from column-major (R) to row-major (C)\n size_t data_align = schema_alignment(schema->parameters[0]);\n *cursor = (void*)ALIGN_UP((uintptr_t)*cursor, data_align);\n tensor->data = R_TRY(abs2rel, (absptr_t)*cursor);\n\n // Coerce R object to match schema element type\n SEXP coerced = obj;\n int need_protect = 0;\n morloc_serial_type etype = schema->parameters[0]->type;\n if ((etype == MORLOC_FLOAT64 || etype == MORLOC_FLOAT32) && !isReal(obj)) {\n coerced = PROTECT(coerceVector(obj, REALSXP));\n need_protect = 1;\n } else if (etype != MORLOC_FLOAT64 && etype != MORLOC_FLOAT32 && etype != MORLOC_BOOL && !isInteger(obj)) {\n coerced = PROTECT(coerceVector(obj, INTSXP));\n need_protect = 1;\n }\n\n if (ndim == 1) {\n // 1D: no transpose needed\n if (isReal(coerced)) {\n memcpy(*cursor, REAL(coerced), total * elem_width);\n } else if (isInteger(coerced)) {\n memcpy(*cursor, INTEGER(coerced), total * elem_width);\n } else if (isLogical(coerced)) {\n int* src = LOGICAL(coerced);\n uint8_t* dst = (uint8_t*)*cursor;\n for (size_t i = 0; i < total; i++) dst[i] = (uint8_t)(src[i] != 0);\n }\n } else if (ndim == 2) {\n size_t nrows = (size_t)shape[0];\n size_t ncols = (size_t)shape[1];\n if (isReal(coerced)) {\n double* src = REAL(coerced);\n double* dst = (double*)*cursor;\n for (size_t r = 0; r < nrows; r++)\n for (size_t c = 0; c < ncols; c++)\n dst[r * ncols + c] = src[c * nrows + r];\n } else if (isInteger(coerced)) {\n int* src = INTEGER(coerced);\n int* dst = (int*)*cursor;\n for (size_t r = 0; r < nrows; r++)\n for (size_t c = 0; c < ncols; c++)\n dst[r * ncols + c] = src[c * nrows + r];\n }\n } else {\n size_t col_strides[5];\n col_strides[0] = 1;\n for (size_t d = 1; d < ndim; d++)\n col_strides[d] = col_strides[d-1] * (size_t)shape[d-1];\n size_t row_strides[5];\n row_strides[ndim-1] = 1;\n for (size_t d = ndim-1; d > 0; d--)\n row_strides[d-1] = row_strides[d] * (size_t)shape[d];\n\n if (isReal(coerced)) {\n double* src = REAL(coerced);\n double* dst = (double*)*cursor;\n for (size_t i = 0; i < total; i++) {\n size_t rem = i;\n size_t col_idx = 0;\n for (size_t d = 0; d < ndim; d++) {\n size_t coord = rem / row_strides[d];\n rem %= row_strides[d];\n col_idx += coord * col_strides[d];\n }\n dst[i] = src[col_idx];\n }\n } else if (isInteger(coerced)) {\n int* src = INTEGER(coerced);\n int* dst = (int*)*cursor;\n for (size_t i = 0; i < total; i++) {\n size_t rem = i;\n size_t col_idx = 0;\n for (size_t d = 0; d < ndim; d++) {\n size_t coord = rem / row_strides[d];\n rem %= row_strides[d];\n col_idx += coord * col_strides[d];\n }\n dst[i] = src[col_idx];\n }\n }\n }\n if (need_protect) UNPROTECT(1);\n *cursor = (char*)*cursor + total * elem_width;\n }\n break;\n\n default:\n MORLOC_ERROR(\"Unhandled schema type\");\n break;\n }\n\n return dest;\n\n}\n\n\n// NOTE: If to_voidstar_r calls error() (via MORLOC_ERROR or R_TRY), the shared\n// memory at dest leaks. This only happens on type mismatches (a development-time\n// bug) and the memory is reclaimed when the pool process exits.\nstatic void* to_voidstar(SEXP obj, const Schema* schema) {\n MAYFAIL\n\n size_t total_size = get_shm_size(schema, obj);\n\n void* dest = R_TRY(shmalloc, total_size);\n\n void* cursor = (void*)((char*)dest + schema->width);\n\n return to_voidstar_r(dest, &cursor, obj, schema);\n}\n\n// }}} to_voidstar\n\n// {{{ from_voidstar\n\nstatic SEXP from_voidstar(const void* data, const Schema* schema, const void* base_ptr) {\n MAYFAIL\n\n if(data == NULL){\n MORLOC_ERROR(\"NULL data (%s:%d in %s)\", __FILE__, __LINE__, __func__);\n }\n\n if(schema == NULL){\n MORLOC_ERROR(\"NULL schema (%s:%d in %s)\", __FILE__, __LINE__, __func__);\n }\n\n SEXP obj = R_NilValue;\n switch (schema->type) {\n case MORLOC_NIL:\n return R_NilValue;\n case MORLOC_BOOL:\n obj = ScalarLogical((bool)*(uint8_t*)data);\n break;\n case MORLOC_SINT8:\n obj = ScalarInteger((int)(*(int8_t*)data));\n break;\n case MORLOC_SINT16:\n obj = ScalarInteger((int)(*(int16_t*)data));\n break;\n case MORLOC_SINT32:\n obj = ScalarInteger(*(int32_t*)data);\n break;\n case MORLOC_SINT64:\n obj = ScalarReal((double)(*(int64_t*)data));\n break;\n case MORLOC_UINT8:\n obj = ScalarInteger((int)(*(uint8_t*)data));\n break;\n case MORLOC_UINT16:\n obj = ScalarInteger((int)(*(uint16_t*)data));\n break;\n case MORLOC_UINT32:\n obj = ScalarReal((double)(*(uint32_t*)data));\n break;\n case MORLOC_UINT64:\n obj = ScalarReal((double)(*(uint64_t*)data));\n break;\n case MORLOC_FLOAT32:\n obj = ScalarReal((double)(*(float*)data));\n break;\n case MORLOC_FLOAT64:\n obj = ScalarReal(*(double*)data);\n break;\n case MORLOC_STRING: {\n if (schema->hint != NULL && strcmp(schema->hint, \"raw\") == 0){\n Array* raw_array = (Array*)data;\n if(raw_array->size > 0){\n void* tmp_ptr = R_TRY(resolve_relptr, raw_array->data, base_ptr);\n obj = PROTECT(allocVector(RAWSXP, raw_array->size));\n memcpy(RAW(obj), tmp_ptr, raw_array->size);\n } else {\n obj = PROTECT(allocVector(RAWSXP, 0));\n }\n UNPROTECT(1);\n } else {\n Array* str_array = (Array*)data;\n if(str_array->size > 0){\n void* tmp_ptr = R_TRY(resolve_relptr, str_array->data, base_ptr);\n SEXP chr = PROTECT(mkCharLen(tmp_ptr, str_array->size));\n obj = PROTECT(ScalarString(chr));\n } else {\n SEXP chr = PROTECT(mkChar(\"\"));\n obj = PROTECT(ScalarString(chr));\n }\n UNPROTECT(2);\n }\n }\n break;\n case MORLOC_ARRAY:\n {\n Array* array = (Array*)data;\n Schema* element_schema = schema->parameters[0];\n char* start;\n\n switch(element_schema->type){\n case MORLOC_BOOL:\n obj = PROTECT(allocVector(LGLSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n LOGICAL(obj)[i] = (bool)*(uint8_t*)(start + i) ? TRUE : FALSE;\n }\n UNPROTECT(1);\n break;\n case MORLOC_SINT8:\n obj = PROTECT(allocVector(INTSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n INTEGER(obj)[i] = (int)(*(int8_t*)(start + i * sizeof(int8_t)));\n }\n UNPROTECT(1);\n break;\n case MORLOC_SINT16:\n obj = PROTECT(allocVector(INTSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n INTEGER(obj)[i] = (int)(*(int16_t*)(start + i * sizeof(int16_t)));\n }\n UNPROTECT(1);\n break;\n case MORLOC_SINT32:\n obj = PROTECT(allocVector(INTSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n {\n void* tmp_ptr = R_TRY(resolve_relptr, array->data, base_ptr);\n memcpy(INTEGER(obj), tmp_ptr, array->size * sizeof(int32_t));\n }\n UNPROTECT(1);\n break;\n case MORLOC_SINT64:\n obj = PROTECT(allocVector(REALSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n REAL(obj)[i] = (double)(*(int64_t*)(start + i * sizeof(int64_t)));\n }\n UNPROTECT(1);\n break;\n // Interpret the uint8 as a raw vector\n case MORLOC_UINT8:\n obj = PROTECT(allocVector(RAWSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n memcpy(RAW(obj), start, array->size * sizeof(uint8_t));\n UNPROTECT(1);\n break;\n case MORLOC_UINT16:\n obj = PROTECT(allocVector(INTSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n INTEGER(obj)[i] = (int)(*(uint16_t*)(start + i * sizeof(uint16_t)));\n }\n UNPROTECT(1);\n break;\n case MORLOC_UINT32:\n obj = PROTECT(allocVector(REALSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n REAL(obj)[i] = (double)(*(uint32_t*)(start + i * sizeof(uint32_t)));\n }\n UNPROTECT(1);\n break;\n case MORLOC_UINT64:\n // NOTE: the R integer cannot store a 64 bit int\n obj = PROTECT(allocVector(REALSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n REAL(obj)[i] = (double)(*(uint64_t*)(start + i * sizeof(uint64_t)));\n }\n UNPROTECT(1);\n break;\n case MORLOC_FLOAT32:\n obj = PROTECT(allocVector(REALSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n for (size_t i = 0; i < array->size; i++) {\n REAL(obj)[i] = (double)(*(float*)(start + i * sizeof(float)));\n }\n UNPROTECT(1);\n break;\n case MORLOC_FLOAT64:\n obj = PROTECT(allocVector(REALSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n memcpy(REAL(obj), start, array->size * sizeof(double));\n UNPROTECT(1);\n break;\n case MORLOC_STRING:\n {\n obj = PROTECT(allocVector(STRSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n size_t width = schema->width;\n for (size_t i = 0; i < array->size; i++) {\n Array* str_array = (Array*)(start + i * width);\n SEXP item;\n if(str_array->size == 0){\n item = PROTECT(mkCharLen(\"\", 0));\n } else {\n void* str_ptr = R_TRY_WITH(UNPROTECT(1), resolve_relptr, str_array->data, base_ptr);\n item = PROTECT(mkCharLen(str_ptr, str_array->size));\n }\n UNPROTECT(1);\n SET_STRING_ELT(obj, i, item);\n }\n UNPROTECT(1);\n }\n break;\n default:\n {\n obj = PROTECT(allocVector(VECSXP, array->size));\n if(array->size == 0) {\n UNPROTECT(1);\n break;\n }\n start = (char*)R_TRY(resolve_relptr, array->data, base_ptr);\n size_t width = element_schema->width;\n for (size_t i = 0; i < array->size; i++) {\n SEXP item = from_voidstar(start + width * i, element_schema, base_ptr);\n if (item == R_NilValue) {\n UNPROTECT(1);\n obj = R_NilValue;\n goto error;\n }\n SET_VECTOR_ELT(obj, i, item);\n }\n UNPROTECT(1);\n }\n break;\n }\n }\n break;\n case MORLOC_TUPLE: {\n obj = PROTECT(allocVector(VECSXP, schema->size));\n for (size_t i = 0; i < schema->size; i++) {\n void* item_ptr = (char*)data + schema->offsets[i];\n SEXP item = from_voidstar(item_ptr, schema->parameters[i], base_ptr);\n if (item == R_NilValue) {\n UNPROTECT(1);\n obj = R_NilValue;\n goto error;\n }\n SET_VECTOR_ELT(obj, i, item);\n }\n UNPROTECT(1);\n break;\n }\n case MORLOC_MAP: {\n obj = PROTECT(allocVector(VECSXP, schema->size));\n SEXP names = PROTECT(allocVector(STRSXP, schema->size));\n for (size_t i = 0; i < schema->size; i++) {\n void* item_ptr = (char*)data + schema->offsets[i];\n SEXP value = from_voidstar(item_ptr, schema->parameters[i], base_ptr);\n if (value == R_NilValue) {\n UNPROTECT(2);\n obj = R_NilValue;\n goto error;\n }\n SET_VECTOR_ELT(obj, i, value);\n SET_STRING_ELT(names, i, mkChar(schema->keys[i]));\n }\n setAttrib(obj, R_NamesSymbol, names);\n UNPROTECT(2);\n break;\n }\n case MORLOC_OPTIONAL: {\n uint8_t tag = *(const uint8_t*)data;\n if (tag == 0) {\n return R_NilValue;\n }\n obj = from_voidstar((const char*)data + schema->offsets[0], schema->parameters[0], base_ptr);\n break;\n }\n case MORLOC_TENSOR: {\n const Tensor* tensor = (const Tensor*)data;\n size_t ndim = schema_tensor_ndim(schema);\n size_t total = tensor->total_elements;\n\n if (total == 0) {\n if (isReal(obj)) {\n obj = PROTECT(allocVector(REALSXP, 0));\n } else {\n obj = PROTECT(allocVector(INTSXP, 0));\n }\n UNPROTECT(1);\n break;\n }\n\n const int64_t* shape = (const int64_t*)resolve_relptr(tensor->shape, base_ptr, NULL);\n const void* tdata = resolve_relptr(tensor->data, base_ptr, NULL);\n\n // Allocate R vector\n int sexptype;\n switch (schema->parameters[0]->type) {\n case MORLOC_FLOAT32:\n case MORLOC_FLOAT64: sexptype = REALSXP; break;\n case MORLOC_BOOL: sexptype = LGLSXP; break;\n default: sexptype = INTSXP; break;\n }\n\n obj = PROTECT(allocVector(sexptype, (R_xlen_t)total));\n\n if (ndim == 1) {\n // 1D: no transpose\n if (sexptype == REALSXP) {\n if (schema->parameters[0]->type == MORLOC_FLOAT32) {\n const float* src = (const float*)tdata;\n double* dst = REAL(obj);\n for (size_t i = 0; i < total; i++) dst[i] = (double)src[i];\n } else {\n memcpy(REAL(obj), tdata, total * sizeof(double));\n }\n } else if (sexptype == INTSXP) {\n size_t elem_w = schema->parameters[0]->width;\n if (elem_w == sizeof(int)) {\n memcpy(INTEGER(obj), tdata, total * sizeof(int));\n } else {\n // Widen or narrow to int\n int* dst = INTEGER(obj);\n const char* src = (const char*)tdata;\n for (size_t i = 0; i < total; i++) {\n int64_t v = 0;\n memcpy(&v, src + i * elem_w, elem_w);\n dst[i] = (int)v;\n }\n }\n } else if (sexptype == LGLSXP) {\n const uint8_t* src = (const uint8_t*)tdata;\n int* dst = LOGICAL(obj);\n for (size_t i = 0; i < total; i++) dst[i] = src[i] ? 1 : 0;\n }\n } else if (ndim == 2) {\n // 2D: row-major to col-major transpose\n size_t nrows = (size_t)shape[0];\n size_t ncols = (size_t)shape[1];\n if (sexptype == REALSXP) {\n const double* src = (const double*)tdata;\n double* dst = REAL(obj);\n for (size_t r = 0; r < nrows; r++)\n for (size_t c = 0; c < ncols; c++)\n dst[c * nrows + r] = src[r * ncols + c];\n } else if (sexptype == INTSXP) {\n const int* src = (const int*)tdata;\n int* dst = INTEGER(obj);\n for (size_t r = 0; r < nrows; r++)\n for (size_t c = 0; c < ncols; c++)\n dst[c * nrows + r] = src[r * ncols + c];\n }\n } else {\n // General N-D: row-major to col-major\n size_t col_strides[5];\n col_strides[0] = 1;\n for (size_t d = 1; d < ndim; d++)\n col_strides[d] = col_strides[d-1] * (size_t)shape[d-1];\n size_t row_strides[5];\n row_strides[ndim-1] = 1;\n for (size_t d = ndim-1; d > 0; d--)\n row_strides[d-1] = row_strides[d] * (size_t)shape[d];\n\n if (sexptype == REALSXP) {\n const double* src = (const double*)tdata;\n double* dst = REAL(obj);\n for (size_t i = 0; i < total; i++) {\n // i is row-major index, compute col-major index\n size_t rem = i;\n size_t col_idx = 0;\n for (size_t d = 0; d < ndim; d++) {\n size_t coord = rem / row_strides[d];\n rem %= row_strides[d];\n col_idx += coord * col_strides[d];\n }\n dst[col_idx] = src[i];\n }\n } else if (sexptype == INTSXP) {\n const int* src = (const int*)tdata;\n int* dst = INTEGER(obj);\n for (size_t i = 0; i < total; i++) {\n size_t rem = i;\n size_t col_idx = 0;\n for (size_t d = 0; d < ndim; d++) {\n size_t coord = rem / row_strides[d];\n rem %= row_strides[d];\n col_idx += coord * col_strides[d];\n }\n dst[col_idx] = src[i];\n }\n }\n }\n\n // Set dim attribute\n SEXP r_dim = PROTECT(allocVector(INTSXP, (R_xlen_t)ndim));\n for (size_t i = 0; i < ndim; i++)\n INTEGER(r_dim)[i] = (int)shape[i];\n setAttrib(obj, R_DimSymbol, r_dim);\n UNPROTECT(2);\n break;\n }\n default:\n MORLOC_ERROR(\"Unsupported schema type\");\n goto error;\n }\n\n return obj;\n\nerror:\n return R_NilValue;\n}\n\n// }}} from_voidstar\n\n// {{{ exported morloc API functions\n\n// PID of the process that created the daemon (set in morloc_start_daemon)\nstatic pid_t daemon_creator_pid = 0;\n\n// Close the daemon when the R object dies\nstatic void daemon_finalizer(SEXP ptr) {\n if (!R_ExternalPtrAddr(ptr)) return;\n // Skip cleanup in forked children -- they must not unlink the socket file\n if (daemon_creator_pid != 0 && getpid() != daemon_creator_pid) {\n R_ClearExternalPtr(ptr);\n return;\n }\n language_daemon_t* daemon = (language_daemon_t*)R_ExternalPtrAddr(ptr);\n if(daemon != NULL){\n close_daemon(&daemon);\n }\n R_ClearExternalPtr(ptr);\n}\n\n// Release daemon resources in a forked child WITHOUT unlinking the socket file.\n// Workers call this after fork so they don't hold the server_fd or accidentally\n// destroy the socket when they exit.\nSEXP morloc_detach_daemon(SEXP daemon_r) {\n if (!R_ExternalPtrAddr(daemon_r)) return R_NilValue;\n language_daemon_t* daemon = (language_daemon_t*)R_ExternalPtrAddr(daemon_r);\n if (daemon != NULL) {\n close_socket(daemon->server_fd);\n client_list_t *current = daemon->client_fds;\n while (current) {\n client_list_t *next = current->next;\n close(current->fd);\n free(current);\n current = next;\n }\n free(daemon->socket_path);\n free(daemon->tmpdir);\n free(daemon->shm_basename);\n free(daemon);\n }\n R_ClearExternalPtr(daemon_r);\n return R_NilValue;\n}\n\nSEXP morloc_start_daemon(\n SEXP socket_path_r,\n SEXP tmpdir_r,\n SEXP shm_basename_r,\n SEXP shm_default_size_r\n){ MAYFAIL\n const char* socket_path = CHAR(STRING_ELT(socket_path_r, 0));\n const char* tmpdir = CHAR(STRING_ELT(tmpdir_r, 0));\n const char* shm_basename = CHAR(STRING_ELT(shm_basename_r, 0));\n size_t shm_default_size = (size_t)asInteger(shm_default_size_r);\n\n language_daemon_t* daemon = R_TRY(\n start_daemon,\n socket_path,\n tmpdir,\n shm_basename,\n shm_default_size\n );\n\n // Wrap pointer in external pointer\n SEXP result = PROTECT(R_MakeExternalPtr(daemon, R_NilValue, R_NilValue));\n\n // Record which process owns the daemon (for the PID guard in daemon_finalizer)\n daemon_creator_pid = getpid();\n\n // Register finalizer with wrapper\n R_RegisterCFinalizerEx(result, daemon_finalizer, TRUE);\n\n // Set class attribute\n SEXP class_name = PROTECT(mkString(\"language_daemon\"));\n SET_CLASS(result, class_name);\n\n UNPROTECT(2);\n return result;\n}\n\n\n\nSEXP morloc_shinit(SEXP shm_basename_r, SEXP volume_index_r, SEXP shm_size_r) { MAYFAIL\n const char* shm_basename = CHAR(STRING_ELT(shm_basename_r, 0));\n size_t volume_index = (size_t)asInteger(volume_index_r);\n size_t shm_size = (size_t)asInteger(shm_size_r);\n\n R_TRY(shinit, shm_basename, volume_index, shm_size);\n\n return R_NilValue;\n}\n\n\n// {{{ signal handling for graceful shutdown\n\nstatic volatile sig_atomic_t r_shutting_down = 0;\n\nstatic void r_sigterm_handler(int sig) {\n (void)sig;\n r_shutting_down = 1;\n}\n\nSEXP morloc_install_sigterm_handler(void) {\n struct sigaction sa;\n sa.sa_handler = r_sigterm_handler;\n sigemptyset(&sa.sa_mask);\n sa.sa_flags = 0;\n sigaction(SIGTERM, &sa, NULL);\n return R_NilValue;\n}\n\nSEXP morloc_is_shutting_down(void) {\n return ScalarLogical(r_shutting_down != 0);\n}\n\nSEXP morloc_set_line_buffered(void) {\n // Only stderr - stdout is left fully buffered for performance\n // and flushed explicitly after each job.\n setvbuf(stderr, NULL, _IOLBF, 0);\n return R_NilValue;\n}\n\n// }}} signal handling\n\nSEXP morloc_wait_for_client(SEXP daemon_r){ MAYFAIL\n if (!R_ExternalPtrAddr(daemon_r)) {\n MORLOC_ERROR(\"Expected a daemon pointer\");\n }\n\n // Return immediately if shutdown was requested\n if (r_shutting_down) {\n return ScalarInteger(-1);\n }\n\n language_daemon_t* daemon = (language_daemon_t*)R_ExternalPtrAddr(daemon_r);\n\n // Use pselect directly (not wait_for_client_with_timeout) so we can\n // return immediately on EINTR from SIGTERM instead of retrying via WAIT\n fd_set read_fds;\n FD_ZERO(&read_fds);\n FD_SET(daemon->server_fd, &read_fds);\n int max_fd = daemon->server_fd;\n\n for (client_list_t* cl = daemon->client_fds; cl != NULL; cl = cl->next) {\n FD_SET(cl->fd, &read_fds);\n if (cl->fd > max_fd) max_fd = cl->fd;\n }\n\n // 100ms timeout -- short enough for responsive SIGTERM handling\n struct timespec ts = { .tv_sec = 0, .tv_nsec = 100000000 };\n sigset_t emptymask;\n sigemptyset(&emptymask);\n\n int ready = pselect(max_fd + 1, &read_fds, NULL, NULL, &ts, &emptymask);\n\n // Check shutdown after pselect (signal may have arrived during the call)\n if (r_shutting_down) {\n return ScalarInteger(-1);\n }\n\n // Timeout or interrupted -- return 0 (no client)\n if (ready <= 0) {\n return ScalarInteger(0);\n }\n\n // Accept new connection if server_fd is ready\n if (FD_ISSET(daemon->server_fd, &read_fds)) {\n int fd = accept(daemon->server_fd, NULL, NULL);\n if (fd >= 0) {\n fcntl(fd, F_SETFL, O_NONBLOCK);\n client_list_t* new_client = (client_list_t*)calloc(1, sizeof(client_list_t));\n if (new_client == NULL) {\n close(fd);\n MORLOC_ERROR(\"calloc failed\");\n }\n new_client->fd = fd;\n new_client->next = NULL;\n if (daemon->client_fds == NULL) {\n daemon->client_fds = new_client;\n } else {\n client_list_t* last = daemon->client_fds;\n while (last->next) last = last->next;\n last->next = new_client;\n }\n }\n }\n\n // Return first ready client fd\n if (daemon->client_fds != NULL) {\n client_list_t* first = daemon->client_fds;\n int client_fd = first->fd;\n daemon->client_fds = first->next;\n free(first);\n return ScalarInteger(client_fd);\n }\n\n return ScalarInteger(0);\n}\n\n\nSEXP morloc_read_morloc_call_packet(SEXP packet_r) { MAYFAIL\n uint8_t* packet = RAW(packet_r);\n morloc_call_t* call_packet = R_TRY(read_morloc_call_packet, packet);\n\n // Create two element R list\n // 1: manifold id\n // 2: argument list of raw packets\n SEXP r_list = PROTECT(allocVector(VECSXP, 2));\n\n // Convert midx to R integer\n SEXP r_mid = PROTECT(ScalarInteger(call_packet->midx));\n\n // Create arguments list\n SEXP r_args = PROTECT(allocVector(VECSXP, call_packet->nargs));\n\n for(size_t i = 0; i < call_packet->nargs; i++) {\n size_t arg_packet_size = R_TRY_WITH(UNPROTECT(3), morloc_packet_size, call_packet->args[i]);\n SEXP r_arg = PROTECT(allocVector(RAWSXP, arg_packet_size));\n memcpy(RAW(r_arg), call_packet->args[i], arg_packet_size);\n SET_VECTOR_ELT(r_args, i, r_arg);\n UNPROTECT(1); // r_arg\n }\n\n // Assemble final list\n SET_VECTOR_ELT(r_list, 0, r_mid);\n SET_VECTOR_ELT(r_list, 1, r_args);\n\n free_morloc_call(call_packet);\n\n UNPROTECT(3); // r_list, r_mid, r_args\n return r_list;\n}\n\n\nSEXP morloc_send_packet_to_foreign_server(SEXP client_fd_r, SEXP packet_r) { MAYFAIL\n if (TYPEOF(client_fd_r) != INTSXP || LENGTH(client_fd_r) != 1) {\n MORLOC_ERROR(\"client_fd must be a single integer\");\n }\n if (TYPEOF(packet_r) != RAWSXP) {\n MORLOC_ERROR(\"packet must be a raw vector\");\n }\n\n // Extract arguments\n int client_fd = INTEGER(client_fd_r)[0];\n uint8_t* packet = RAW(packet_r);\n size_t packet_size = (size_t)LENGTH(packet_r);\n\n // Call underlying implementation\n size_t bytes_sent = R_TRY(send_packet_to_foreign_server, client_fd, packet);\n\n // This could in theory be problematic, since int is smaller than size_t\n // In practice it should not be, since packets are typically small\n // However, if I refactor to send large packets in the future, this could be\n // problematic. Then I would need to convert to a double return.\n return ScalarInteger((int)bytes_sent);\n}\n\n\n// Read from socket returning raw vector of received data\nSEXP morloc_stream_from_client(SEXP client_fd_r) { MAYFAIL\n if (TYPEOF(client_fd_r) != INTSXP || LENGTH(client_fd_r) != 1) {\n MORLOC_ERROR(\"client_fd must be a single integer\");\n }\n\n int client_fd = INTEGER(client_fd_r)[0];\n\n // Read packet from socket\n uint8_t* packet = R_TRY(stream_from_client, client_fd);\n\n // Read the packet size from the header (free packet before longjmp on error)\n size_t packet_size = R_TRY_WITH(free(packet), morloc_packet_size, packet);\n\n // Create raw vector for result\n SEXP result = PROTECT(allocVector(RAWSXP, packet_size));\n memcpy(RAW(result), packet, packet_size);\n free(packet);\n\n UNPROTECT(1);\n return result;\n}\n\n\n// close_socket\nSEXP morloc_close_socket(SEXP socket_id_r) {\n if (TYPEOF(socket_id_r) != INTSXP || LENGTH(socket_id_r) != 1) {\n MORLOC_ERROR(\"socket_id must be a single integer\");\n }\n int socket_id = INTEGER(socket_id_r)[0];\n close_socket(socket_id);\n // Return invisible NULL\n return R_NilValue;\n}\n\n\n// put_value\nSEXP morloc_put_value(SEXP obj_r, SEXP schema_str_r) { MAYFAIL\n if (TYPEOF(schema_str_r) != STRSXP || LENGTH(schema_str_r) != 1) {\n MORLOC_ERROR(\"schema must be a single string\");\n }\n\n const char* schema_cstr = CHAR(STRING_ELT(schema_str_r, 0));\n\n char* schema_str = strdup(schema_cstr);\n Schema* schema = R_TRY_WITH(free(schema_str), parse_schema, schema_str);\n free(schema_str);\n\n // Arrow dispatch: if schema hint is \"arrow\", use Arrow C Data Interface\n if (schema->hint && strcmp(schema->hint, \"arrow\") == 0) {\n // Export R arrow RecordBatch via C Data Interface -> copy to shm -> packet\n // arrow::ExportRecordBatch(batch, array_ptr, schema_ptr)\n struct ArrowSchema arrow_schema;\n struct ArrowArray arrow_array;\n memset(&arrow_schema, 0, sizeof(arrow_schema));\n memset(&arrow_array, 0, sizeof(arrow_array));\n\n SEXP arrow_ns = PROTECT(R_FindNamespace(mkString(\"arrow\")));\n SEXP export_fn = PROTECT(findVarInFrame(arrow_ns, install(\"ExportRecordBatch\")));\n if (export_fn == R_UnboundValue) {\n UNPROTECT(2);\n free_schema(schema);\n MORLOC_ERROR(\"arrow::ExportRecordBatch not found; is the arrow package installed?\");\n }\n\n SEXP array_ptr_r = PROTECT(R_MakeExternalPtr(&arrow_array, R_NilValue, R_NilValue));\n SEXP schema_ptr_r = PROTECT(R_MakeExternalPtr(&arrow_schema, R_NilValue, R_NilValue));\n SEXP call = PROTECT(lang4(export_fn, obj_r, array_ptr_r, schema_ptr_r));\n eval(call, arrow_ns);\n UNPROTECT(5);\n\n char* errmsg = NULL;\n relptr_t relptr = arrow_to_shm(&arrow_array, &arrow_schema, &errmsg);\n\n if (arrow_schema.release) arrow_schema.release(&arrow_schema);\n if (arrow_array.release) arrow_array.release(&arrow_array);\n\n if (errmsg) {\n free_schema(schema);\n MORLOC_ERROR(\"Arrow export failed: %s\", errmsg);\n }\n\n uint8_t* packet = make_arrow_data_packet(relptr, schema);\n if (!packet) {\n free_schema(schema);\n MORLOC_ERROR(\"Failed to create arrow data packet\");\n }\n\n size_t packet_size = R_TRY_WITH({free(packet); free_schema(schema);}, morloc_packet_size, packet);\n SEXP result = PROTECT(allocVector(RAWSXP, packet_size));\n memcpy(RAW(result), packet, packet_size);\n free(packet);\n free_schema(schema);\n UNPROTECT(1);\n return result;\n }\n\n void* voidstar = to_voidstar(obj_r, schema);\n if (!voidstar) {\n free_schema(schema);\n MORLOC_ERROR(\"Failed to convert R object to internal representation\");\n }\n\n relptr_t relptr = R_TRY_WITH(free_schema(schema), abs2rel, voidstar);\n\n uint8_t* packet = R_TRY_WITH(free_schema(schema), make_data_packet_auto, voidstar, relptr, schema);\n\n const morloc_packet_header_t* hdr = (const morloc_packet_header_t*)packet;\n if (hdr->command.data.source != PACKET_SOURCE_RPTR) {\n // Data inlined in packet -- free SHM immediately\n char* free_err = NULL;\n shfree_by_schema((absptr_t)voidstar, schema, &free_err);\n if (free_err) { free(free_err); free_err = NULL; }\n shfree((absptr_t)voidstar, &free_err);\n if (free_err) { free(free_err); }\n }\n\n size_t packet_size = R_TRY_WITH({free(packet); free_schema(schema);}, morloc_packet_size, packet);\n\n SEXP result = PROTECT(allocVector(RAWSXP, packet_size));\n memcpy(RAW(result), packet, packet_size);\n free(packet);\n free_schema(schema);\n\n UNPROTECT(1);\n return result;\n}\n\n\n// mlc_show: serialize a value to a JSON string\nSEXP morloc_mlc_show(SEXP obj_r, SEXP schema_str_r) { MAYFAIL\n if (TYPEOF(schema_str_r) != STRSXP || LENGTH(schema_str_r) != 1) {\n MORLOC_ERROR(\"schema must be a single string\");\n }\n\n char* schema_str = strdup(CHAR(STRING_ELT(schema_str_r, 0)));\n Schema* schema = R_TRY_WITH(free(schema_str), parse_schema, schema_str);\n free(schema_str);\n\n void* voidstar = to_voidstar(obj_r, schema);\n if (!voidstar) {\n free_schema(schema);\n MORLOC_ERROR(\"Failed to convert R object to internal representation\");\n }\n\n char* json = R_TRY_WITH(free_schema(schema), mlc_show, voidstar, schema);\n\n {\n char* shfree_errmsg = NULL;\n shfree(voidstar, &shfree_errmsg);\n free(shfree_errmsg);\n }\n free_schema(schema);\n\n SEXP result = PROTECT(mkString(json));\n free(json);\n UNPROTECT(1);\n return result;\n}\n\n\nSEXP morloc_get_value(SEXP packet_r, SEXP schema_str_r) { MAYFAIL\n if (TYPEOF(packet_r) != RAWSXP) {\n MORLOC_ERROR(\"packet must be a raw vector\");\n }\n if (TYPEOF(schema_str_r) != STRSXP || LENGTH(schema_str_r) != 1) {\n MORLOC_ERROR(\"schema must be a single string\");\n }\n\n // Extract arguments\n uint8_t* packet = RAW(packet_r);\n size_t packet_size = (size_t)LENGTH(packet_r);\n\n const morloc_packet_header_t* header = (const morloc_packet_header_t*)packet;\n uint8_t source = header->command.data.source;\n uint8_t format = header->command.data.format;\n\n const char* schema_cstr = CHAR(STRING_ELT(schema_str_r, 0));\n\n char* schema_str = strdup(schema_cstr);\n Schema* schema = R_TRY_WITH(free(schema_str), parse_schema, schema_str);\n free(schema_str);\n\n // Arrow dispatch: if packet format is Arrow, import via C Data Interface\n if (format == PACKET_FORMAT_ARROW) {\n uint8_t* arrow_ptr = R_TRY_WITH(free_schema(schema),\n get_morloc_data_packet_value, packet, schema);\n const arrow_shm_header_t* arrow_hdr = (const arrow_shm_header_t*)arrow_ptr;\n\n struct ArrowSchema arrow_schema;\n struct ArrowArray arrow_array;\n char* arrow_err = NULL;\n arrow_from_shm(arrow_hdr, &arrow_schema, &arrow_array, &arrow_err);\n if (arrow_err) {\n if (arrow_schema.release) arrow_schema.release(&arrow_schema);\n if (arrow_array.release) arrow_array.release(&arrow_array);\n free_schema(schema);\n MORLOC_ERROR(\"Arrow import failed: %s\", arrow_err);\n }\n\n // Import via R arrow package: arrow::ImportRecordBatch(array_ptr, schema_ptr)\n SEXP arrow_ns = PROTECT(R_FindNamespace(mkString(\"arrow\")));\n SEXP import_fn = PROTECT(findVarInFrame(arrow_ns, install(\"ImportRecordBatch\")));\n if (import_fn == R_UnboundValue) {\n if (arrow_schema.release) arrow_schema.release(&arrow_schema);\n if (arrow_array.release) arrow_array.release(&arrow_array);\n UNPROTECT(2);\n free_schema(schema);\n MORLOC_ERROR(\"arrow::ImportRecordBatch not found; is the arrow package installed?\");\n }\n\n SEXP array_ptr_r = PROTECT(R_MakeExternalPtr(&arrow_array, R_NilValue, R_NilValue));\n SEXP schema_ptr_r = PROTECT(R_MakeExternalPtr(&arrow_schema, R_NilValue, R_NilValue));\n SEXP call = PROTECT(lang3(import_fn, array_ptr_r, schema_ptr_r));\n SEXP obj_r = PROTECT(eval(call, arrow_ns));\n UNPROTECT(6);\n\n // Incref shm so data stays alive\n char* incref_err = NULL;\n shincref((absptr_t)arrow_ptr, &incref_err);\n if (incref_err) { free(incref_err); }\n\n free_schema(schema);\n return obj_r;\n }\n\n // Fast path: inline voidstar -- read directly from packet, no SHM needed\n if (source == PACKET_SOURCE_MESG && format == PACKET_FORMAT_VOIDSTAR) {\n const uint8_t* payload = packet + sizeof(morloc_packet_header_t) + header->offset;\n SEXP obj_r = from_voidstar((const void*)payload, schema, (const void*)payload);\n free_schema(schema);\n if (obj_r == NULL) {\n MORLOC_ERROR(\"Failed to convert internal representation to R object\");\n }\n return obj_r;\n }\n\n // SHM paths\n uint8_t* voidstar = R_TRY_WITH(free_schema(schema), get_morloc_data_packet_value, packet, schema);\n\n SEXP obj_r = from_voidstar(voidstar, schema, NULL);\n if (obj_r == NULL) {\n free_schema(schema);\n MORLOC_ERROR(\"Failed to convert internal representation to R object\");\n }\n\n free_schema(schema);\n\n return obj_r;\n}\n\n\nSEXP morloc_foreign_call(SEXP socket_path_r, SEXP mid_r, SEXP args_r) { MAYFAIL\n // Validate inputs\n if (TYPEOF(socket_path_r) != STRSXP || LENGTH(socket_path_r) != 1) {\n MORLOC_ERROR(\"socket_path must be a single string\");\n }\n if (TYPEOF(mid_r) != INTSXP || LENGTH(mid_r) != 1) {\n MORLOC_ERROR(\"mid must be a single integer\");\n }\n if (TYPEOF(args_r) != VECSXP) {\n MORLOC_ERROR(\"args must be a list of raw vectors\");\n }\n\n // Extract arguments\n const char* socket_path = CHAR(STRING_ELT(socket_path_r, 0));\n int mid = INTEGER(mid_r)[0];\n size_t nargs = (size_t)LENGTH(args_r);\n\n // Allocate temporary storage\n const uint8_t** arg_packets = (const uint8_t**)R_alloc(nargs, sizeof(uint8_t*));\n\n // Convert R raw vectors to C buffers\n for (size_t i = 0; i < nargs; i++) {\n SEXP arg = VECTOR_ELT(args_r, i);\n if (TYPEOF(arg) != RAWSXP) {\n MORLOC_ERROR(\"All arguments must be raw vectors (argument %zu)\", i+1);\n }\n arg_packets[i] = RAW(arg);\n }\n\n // Create call packet\n uint8_t* packet = R_TRY(\n make_morloc_local_call_packet,\n (uint32_t)mid,\n arg_packets,\n nargs\n );\n\n // Send/receive over socket\n uint8_t* result = R_TRY_WITH(free(packet),\n send_and_receive_over_socket,\n socket_path,\n packet\n );\n\n // Get result size\n size_t result_length = R_TRY_WITH({free(packet); free(result);}, morloc_packet_size, result);\n\n // Create result raw vector\n SEXP result_r = PROTECT(allocVector(RAWSXP, result_length));\n memcpy(RAW(result_r), result, result_length);\n free(packet);\n free(result);\n\n // Cleanup\n UNPROTECT(1);\n return result_r;\n}\n\n\nSEXP morloc_is_ping(SEXP packet_r) { MAYFAIL\n if (TYPEOF(packet_r) != RAWSXP) {\n MORLOC_ERROR(\"packet must be a raw vector\");\n }\n\n bool is_ping = R_TRY(packet_is_ping, RAW(packet_r));\n\n return ScalarLogical(is_ping);\n}\n\n\nSEXP morloc_is_local_call(SEXP packet_r) { MAYFAIL\n if (TYPEOF(packet_r) != RAWSXP) {\n MORLOC_ERROR(\"packet must be a raw vector\");\n }\n\n bool is_local_call = R_TRY(packet_is_local_call, RAW(packet_r));\n\n return ScalarLogical(is_local_call);\n}\n\n\nSEXP morloc_is_remote_call(SEXP packet_r) { MAYFAIL\n if (TYPEOF(packet_r) != RAWSXP) {\n MORLOC_ERROR(\"packet must be a raw vector\");\n }\n\n bool is_remote_call = R_TRY(packet_is_remote_call, RAW(packet_r));\n\n return ScalarLogical(is_remote_call);\n}\n\n\nSEXP morloc_pong(SEXP packet_r) { MAYFAIL\n if (TYPEOF(packet_r) != RAWSXP) {\n MORLOC_ERROR(\"packet must be a raw vector\");\n }\n\n // Generate a response to ping\n uint8_t* pong = R_TRY(return_ping, RAW(packet_r));\n\n size_t pong_size = R_TRY_WITH(free(pong), morloc_packet_size, pong);\n\n SEXP result_r = PROTECT(allocVector(RAWSXP, pong_size));\n memcpy(RAW(result_r), pong, pong_size);\n free(pong);\n\n UNPROTECT(1);\n return result_r;\n}\n\n\nSEXP morloc_make_fail_packet(SEXP failure_message_r) { MAYFAIL\n const char* failure_message = CHAR(STRING_ELT(failure_message_r, 0));\n uint8_t* fail_packet = make_fail_packet(failure_message);\n\n size_t packet_size = R_TRY(morloc_packet_size, fail_packet);\n\n SEXP packet_r = PROTECT(allocVector(RAWSXP, packet_size));\n memcpy(RAW(packet_r), fail_packet, packet_size);\n free(fail_packet);\n\n UNPROTECT(1);\n return packet_r;\n}\n\n\nSEXP extract_element_by_name(SEXP list, const char* key) {\n // Ensure inputs are correct types\n if (TYPEOF(list) != VECSXP) MORLOC_ERROR(\"Input must be a list\");\n\n // Get list names attribute\n SEXP names = Rf_getAttrib(list, R_NamesSymbol);\n if (names == R_NilValue) MORLOC_ERROR(\"List must have names\");\n\n // Iterate through list elements\n for (int i = 0; i < Rf_length(list); i++) {\n const char *current_name = CHAR(STRING_ELT(names, i));\n\n if (strcmp(key, current_name) == 0) {\n return VECTOR_ELT(list, i); // Return matching element\n }\n }\n\n return R_NilValue; // Return NULL if name not found\n}\n\n\nSEXP morloc_remote_call(SEXP midx, SEXP socket_path, SEXP cache_path, SEXP resources, SEXP arg_packets) { MAYFAIL\n // Protect all R inputs immediately\n PROTECT(socket_path);\n PROTECT(cache_path);\n PROTECT(resources);\n PROTECT(arg_packets = coerceVector(arg_packets, VECSXP));\n\n // Convert basic parameters\n int c_midx = INTEGER(midx)[0];\n const char* c_socket_path = CHAR(STRING_ELT(socket_path, 0));\n const char* c_cache_path = CHAR(STRING_ELT(cache_path, 0));\n\n // Extract resources with validation\n resources_t c_resources;\n SEXP mem = extract_element_by_name(resources, \"memory\");\n SEXP tim = extract_element_by_name(resources, \"time\");\n SEXP cpu = extract_element_by_name(resources, \"cpus\");\n SEXP gpu = extract_element_by_name(resources, \"gpus\");\n if (mem == R_NilValue || tim == R_NilValue || cpu == R_NilValue || gpu == R_NilValue) {\n UNPROTECT(4);\n MORLOC_ERROR(\"Missing required resource field (memory, time, cpus, or gpus)\");\n }\n c_resources.memory = INTEGER(mem)[0];\n c_resources.time = INTEGER(tim)[0];\n c_resources.cpus = INTEGER(cpu)[0];\n c_resources.gpus = INTEGER(gpu)[0];\n\n // Process argument packets with type checking\n size_t nargs = LENGTH(arg_packets);\n const uint8_t** c_arg_packets = (const uint8_t**) R_alloc(nargs, sizeof(uint8_t*));\n\n for(size_t i = 0; i < nargs; i++) {\n SEXP raw_vec = VECTOR_ELT(arg_packets, i);\n if(TYPEOF(raw_vec) != RAWSXP) {\n UNPROTECT(4);\n MORLOC_ERROR(\"arg_packets must contain only raw vectors\");\n }\n c_arg_packets[i] = (uint8_t*)RAW(raw_vec);\n }\n\n // Execute remote call\n uint8_t* result_packet = R_TRY_WITH(UNPROTECT(4),\n remote_call,\n c_midx,\n c_socket_path,\n c_cache_path,\n &c_resources,\n c_arg_packets,\n nargs\n );\n\n // Validate and copy result\n size_t packet_size = R_TRY_WITH({free(result_packet); UNPROTECT(4);}, morloc_packet_size, result_packet);\n if(!result_packet || packet_size == 0) {\n if(result_packet) free(result_packet);\n UNPROTECT(4);\n MORLOC_ERROR(\"Invalid result packet from remote call\");\n }\n\n SEXP result_packet_r = PROTECT(allocVector(RAWSXP, packet_size));\n memcpy(RAW(result_packet_r), result_packet, packet_size);\n free(result_packet);\n\n // Cleanup and return\n UNPROTECT(5); // socket_path, cache_path, resources, arg_packets, result_packet_r\n return result_packet_r;\n}\n\n\n// {{{ fork and fd-passing functions\n\nSEXP morloc_socketpair(void) {\n int sv[2];\n if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) < 0) {\n error(\"socketpair failed: %s\", strerror(errno));\n }\n SEXP result = PROTECT(allocVector(INTSXP, 2));\n INTEGER(result)[0] = sv[0];\n INTEGER(result)[1] = sv[1];\n UNPROTECT(1);\n return result;\n}\n\nSEXP morloc_fork(void) {\n pid_t pid = fork();\n if (pid < 0) {\n error(\"fork failed: %s\", strerror(errno));\n }\n return ScalarInteger((int)pid);\n}\n\n// Immediately terminate the process without running any cleanup.\n// Must be used by forked worker children instead of R's quit().\n// R's quit() runs finalizers that try to free objects allocated by the\n// parent process, which corrupts the heap on glibc >= 2.39.\nSEXP morloc_exit(SEXP status_r) {\n int status = INTEGER(status_r)[0];\n _exit(status);\n return R_NilValue; // unreachable\n}\n\nSEXP morloc_send_fd(SEXP pipe_fd_r, SEXP client_fd_r) {\n int pipe_fd = INTEGER(pipe_fd_r)[0];\n int client_fd = INTEGER(client_fd_r)[0];\n\n struct msghdr msg = {0};\n struct iovec iov;\n char buf[1] = {0};\n char cmsgbuf[CMSG_SPACE(sizeof(int))];\n\n iov.iov_base = buf;\n iov.iov_len = 1;\n msg.msg_iov = &iov;\n msg.msg_iovlen = 1;\n msg.msg_control = cmsgbuf;\n msg.msg_controllen = sizeof(cmsgbuf);\n\n struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);\n cmsg->cmsg_level = SOL_SOCKET;\n cmsg->cmsg_type = SCM_RIGHTS;\n cmsg->cmsg_len = CMSG_LEN(sizeof(int));\n memcpy(CMSG_DATA(cmsg), &client_fd, sizeof(int));\n\n ssize_t n = sendmsg(pipe_fd, &msg, 0);\n if (n < 0) {\n error(\"sendmsg SCM_RIGHTS failed: %s\", strerror(errno));\n }\n return R_NilValue;\n}\n\nSEXP morloc_recv_fd(SEXP pipe_fd_r) {\n int pipe_fd = INTEGER(pipe_fd_r)[0];\n\n struct msghdr msg = {0};\n struct iovec iov;\n char buf[1];\n char cmsgbuf[CMSG_SPACE(sizeof(int))];\n\n iov.iov_base = buf;\n iov.iov_len = 1;\n msg.msg_iov = &iov;\n msg.msg_iovlen = 1;\n msg.msg_control = cmsgbuf;\n msg.msg_controllen = sizeof(cmsgbuf);\n\n ssize_t n = recvmsg(pipe_fd, &msg, 0);\n if (n <= 0) {\n return ScalarInteger(-1);\n }\n\n struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);\n if (cmsg == NULL || cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {\n return ScalarInteger(-1);\n }\n\n int fd;\n memcpy(&fd, CMSG_DATA(cmsg), sizeof(int));\n return ScalarInteger(fd);\n}\n\nSEXP morloc_kill(SEXP pid_r, SEXP sig_r) {\n pid_t pid = (pid_t)INTEGER(pid_r)[0];\n int sig = INTEGER(sig_r)[0];\n int ret = kill(pid, sig);\n return ScalarInteger(ret);\n}\n\nSEXP morloc_waitpid(SEXP pid_r) {\n pid_t pid = (pid_t)INTEGER(pid_r)[0];\n int status;\n pid_t result = waitpid(pid, &status, WNOHANG);\n return ScalarInteger((int)result);\n}\n\nSEXP morloc_waitpid_blocking(SEXP pid_r) {\n pid_t pid = (pid_t)INTEGER(pid_r)[0];\n int status;\n pid_t result = waitpid(pid, &status, 0);\n return ScalarInteger((int)result);\n}\n\n// }}} fork and fd-passing functions\n\n// {{{ shared counter functions (for dynamic worker spawning)\n\nstatic void shared_counter_finalizer(SEXP ptr) {\n int* p = (int*)R_ExternalPtrAddr(ptr);\n if (p != NULL) {\n munmap(p, sizeof(int));\n R_ClearExternalPtr(ptr);\n }\n}\n\nSEXP morloc_shared_counter_create(void) {\n int* p = (int*)mmap(NULL, sizeof(int),\n PROT_READ | PROT_WRITE,\n MAP_SHARED | MAP_ANONYMOUS, -1, 0);\n if (p == MAP_FAILED) {\n error(\"mmap failed for shared counter: %s\", strerror(errno));\n }\n *p = 0;\n SEXP ptr = PROTECT(R_MakeExternalPtr(p, R_NilValue, R_NilValue));\n R_RegisterCFinalizerEx(ptr, shared_counter_finalizer, TRUE);\n UNPROTECT(1);\n return ptr;\n}\n\nSEXP morloc_shared_counter_inc(SEXP ptr_r) {\n int* p = (int*)R_ExternalPtrAddr(ptr_r);\n if (p == NULL) error(\"shared counter is NULL\");\n int val = __atomic_add_fetch(p, 1, __ATOMIC_RELAXED);\n return ScalarInteger(val);\n}\n\nSEXP morloc_shared_counter_dec(SEXP ptr_r) {\n int* p = (int*)R_ExternalPtrAddr(ptr_r);\n if (p == NULL) error(\"shared counter is NULL\");\n int val = __atomic_sub_fetch(p, 1, __ATOMIC_RELAXED);\n return ScalarInteger(val);\n}\n\nSEXP morloc_shared_counter_read(SEXP ptr_r) {\n int* p = (int*)R_ExternalPtrAddr(ptr_r);\n if (p == NULL) error(\"shared counter is NULL\");\n int val = __atomic_load_n(p, __ATOMIC_RELAXED);\n return ScalarInteger(val);\n}\n\nSEXP morloc_pipe(void) {\n int fds[2];\n if (pipe(fds) != 0) {\n error(\"pipe failed: %s\", strerror(errno));\n }\n SEXP result = PROTECT(allocVector(INTSXP, 2));\n INTEGER(result)[0] = fds[0]; /* read end */\n INTEGER(result)[1] = fds[1]; /* write end */\n UNPROTECT(1);\n return result;\n}\n\nSEXP morloc_write_byte(SEXP fd_r, SEXP byte_r) {\n int fd = INTEGER(fd_r)[0];\n unsigned char b = (unsigned char)RAW(byte_r)[0];\n ssize_t n = write(fd, &b, 1);\n return ScalarInteger((int)n);\n}\n\nSEXP morloc_close_fd(SEXP fd_r) {\n int fd = INTEGER(fd_r)[0];\n close(fd);\n return R_NilValue;\n}\n\n// }}} shared counter functions\n\n// {{{ C-level worker loop\n\n// Receive a file descriptor over a Unix domain socket (C-level helper).\nstatic int recv_fd_c(int pipe_fd) {\n struct msghdr msg = {0};\n struct iovec iov;\n char buf[1];\n char cmsgbuf[CMSG_SPACE(sizeof(int))];\n\n iov.iov_base = buf;\n iov.iov_len = 1;\n msg.msg_iov = &iov;\n msg.msg_iovlen = 1;\n msg.msg_control = cmsgbuf;\n msg.msg_controllen = sizeof(cmsgbuf);\n\n ssize_t n = recvmsg(pipe_fd, &msg, 0);\n if (n <= 0) return -1;\n\n struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);\n if (!cmsg || cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS)\n return -1;\n\n int fd;\n memcpy(&fd, CMSG_DATA(cmsg), sizeof(int));\n return fd;\n}\n\n// Send a fail packet to the client (best-effort, ignores send errors).\nstatic void send_fail_to_client(int client_fd, const char* msg) {\n char* errmsg = NULL;\n uint8_t* fail = make_fail_packet(msg);\n send_packet_to_foreign_server(client_fd, fail, &errmsg);\n free(fail);\n}\n\n// Dispatch a call to a manifold function. All packet handling is in C;\n// only the manifold evaluation crosses into R via R_tryEval.\nstatic void dispatch_manifold_c(int client_fd, const uint8_t* packet,\n SEXP dispatch, const char* label) {\n char* errmsg = NULL;\n\n morloc_call_t* call = read_morloc_call_packet(packet, &errmsg);\n if (errmsg) {\n send_fail_to_client(client_fd, errmsg);\n return;\n }\n\n int midx = (int)call->midx;\n SEXP fn = (midx >= 1 && midx <= LENGTH(dispatch))\n ? VECTOR_ELT(dispatch, midx - 1) : R_NilValue;\n\n if (fn == R_NilValue) {\n char msg[128];\n snprintf(msg, sizeof(msg), \"%s function not found: m%d\", label, midx);\n send_fail_to_client(client_fd, msg);\n free_morloc_call(call);\n return;\n }\n\n // Build R pairlist of raw-vector arguments: fn(arg1, arg2, ...)\n int nprotect = 0;\n SEXP pairlist = R_NilValue;\n for (int i = (int)call->nargs - 1; i >= 0; i--) {\n size_t arg_size = morloc_packet_size(call->args[i], &errmsg);\n if (errmsg) {\n UNPROTECT(nprotect);\n send_fail_to_client(client_fd, errmsg);\n free_morloc_call(call);\n return;\n }\n SEXP r_arg = PROTECT(allocVector(RAWSXP, arg_size));\n nprotect++;\n memcpy(RAW(r_arg), call->args[i], arg_size);\n pairlist = PROTECT(Rf_cons(r_arg, pairlist));\n nprotect++;\n }\n free_morloc_call(call);\n\n SEXP r_call = PROTECT(Rf_lcons(fn, pairlist));\n nprotect++;\n\n // Single crossing into R: evaluate the manifold\n int eval_err = 0;\n SEXP result = R_tryEvalSilent(r_call, R_GlobalEnv, &eval_err);\n\n if (eval_err || result == R_NilValue || TYPEOF(result) != RAWSXP) {\n UNPROTECT(nprotect);\n send_fail_to_client(client_fd,\n eval_err ? R_curErrorBuf() : \"manifold returned non-raw result\");\n return;\n }\n\n PROTECT(result);\n nprotect++;\n\n send_packet_to_foreign_server(client_fd, RAW(result), &errmsg);\n UNPROTECT(nprotect);\n}\n\n// Process one client job entirely in C. Only crosses into R for\n// the actual manifold evaluation.\nstatic void run_job_c(int client_fd, SEXP dispatch, SEXP remote_dispatch) {\n char* errmsg = NULL;\n\n uint8_t* packet = stream_from_client(client_fd, &errmsg);\n if (errmsg) {\n send_fail_to_client(client_fd, errmsg);\n free(errmsg);\n close_socket(client_fd);\n return;\n }\n\n bool is_local = packet_is_local_call(packet, &errmsg);\n if (!errmsg && is_local) {\n dispatch_manifold_c(client_fd, packet, dispatch, \"Local\");\n } else if (!errmsg) {\n bool is_remote = packet_is_remote_call(packet, &errmsg);\n if (!errmsg && is_remote) {\n dispatch_manifold_c(client_fd, packet, remote_dispatch, \"Remote\");\n } else if (!errmsg) {\n bool is_ping_pkt = packet_is_ping(packet, &errmsg);\n if (!errmsg && is_ping_pkt) {\n uint8_t* pong = return_ping(packet, &errmsg);\n if (!errmsg) {\n send_packet_to_foreign_server(client_fd, pong, &errmsg);\n free(pong);\n }\n } else if (!errmsg) {\n send_fail_to_client(client_fd, \"Unexpected packet type\");\n }\n }\n }\n\n if (errmsg) {\n send_fail_to_client(client_fd, errmsg);\n }\n\n free(packet);\n close_socket(client_fd);\n}\n\n// Tight C worker loop. Receives fds from the job queue and processes them,\n// crossing into R only for manifold evaluation.\nSEXP morloc_worker_loop_c(SEXP pipe_fd_r, SEXP dispatch_r, SEXP remote_dispatch_r) {\n int pipe_fd = INTEGER(pipe_fd_r)[0];\n PROTECT(dispatch_r);\n PROTECT(remote_dispatch_r);\n\n while (!r_shutting_down) {\n int client_fd = recv_fd_c(pipe_fd);\n if (client_fd < 0) break;\n run_job_c(client_fd, dispatch_r, remote_dispatch_r);\n fflush(stdout);\n }\n\n UNPROTECT(2);\n return R_NilValue;\n}\n\n// }}} C-level worker loop\n\n// }}} exported functions\n\n\nvoid R_init_rmorloc(DllInfo *info) {\n R_CallMethodDef callMethods[] = {\n {\"morloc_start_daemon\", (DL_FUNC) &morloc_start_daemon, 4},\n {\"morloc_wait_for_client\", (DL_FUNC) &morloc_wait_for_client, 1},\n {\"morloc_read_morloc_call_packet\", (DL_FUNC) &morloc_read_morloc_call_packet, 1},\n {\"morloc_send_packet_to_foreign_server\", (DL_FUNC) &morloc_send_packet_to_foreign_server, 2},\n {\"morloc_stream_from_client\", (DL_FUNC) &morloc_stream_from_client, 1},\n {\"morloc_close_socket\", (DL_FUNC) &morloc_close_socket, 1},\n {\"morloc_foreign_call\", (DL_FUNC) &morloc_foreign_call, 3},\n {\"morloc_get_value\", (DL_FUNC) &morloc_get_value, 2},\n {\"morloc_put_value\", (DL_FUNC) &morloc_put_value, 2},\n {\"morloc_mlc_show\", (DL_FUNC) &morloc_mlc_show, 2},\n {\"morloc_is_ping\", (DL_FUNC) &morloc_is_ping, 1},\n {\"morloc_is_local_call\", (DL_FUNC) &morloc_is_local_call, 1},\n {\"morloc_is_remote_call\", (DL_FUNC) &morloc_is_remote_call, 1},\n {\"morloc_remote_call\", (DL_FUNC) &morloc_remote_call, 5},\n {\"morloc_pong\", (DL_FUNC) &morloc_pong, 1},\n {\"morloc_make_fail_packet\", (DL_FUNC) &morloc_make_fail_packet, 1},\n {\"morloc_shinit\", (DL_FUNC) &morloc_shinit, 3},\n {\"morloc_socketpair\", (DL_FUNC) &morloc_socketpair, 0},\n {\"morloc_fork\", (DL_FUNC) &morloc_fork, 0},\n {\"morloc_exit\", (DL_FUNC) &morloc_exit, 1},\n {\"morloc_send_fd\", (DL_FUNC) &morloc_send_fd, 2},\n {\"morloc_recv_fd\", (DL_FUNC) &morloc_recv_fd, 1},\n {\"morloc_kill\", (DL_FUNC) &morloc_kill, 2},\n {\"morloc_waitpid\", (DL_FUNC) &morloc_waitpid, 1},\n {\"morloc_waitpid_blocking\", (DL_FUNC) &morloc_waitpid_blocking, 1},\n {\"morloc_install_sigterm_handler\", (DL_FUNC) &morloc_install_sigterm_handler, 0},\n {\"morloc_set_line_buffered\", (DL_FUNC) &morloc_set_line_buffered, 0},\n {\"morloc_is_shutting_down\", (DL_FUNC) &morloc_is_shutting_down, 0},\n {\"morloc_detach_daemon\", (DL_FUNC) &morloc_detach_daemon, 1},\n {\"morloc_shared_counter_create\", (DL_FUNC) &morloc_shared_counter_create, 0},\n {\"morloc_shared_counter_inc\", (DL_FUNC) &morloc_shared_counter_inc, 1},\n {\"morloc_shared_counter_dec\", (DL_FUNC) &morloc_shared_counter_dec, 1},\n {\"morloc_shared_counter_read\", (DL_FUNC) &morloc_shared_counter_read, 1},\n {\"morloc_pipe\", (DL_FUNC) &morloc_pipe, 0},\n {\"morloc_write_byte\", (DL_FUNC) &morloc_write_byte, 2},\n {\"morloc_close_fd\", (DL_FUNC) &morloc_close_fd, 1},\n {\"morloc_worker_loop_c\", (DL_FUNC) &morloc_worker_loop_c, 3},\n {NULL, NULL, 0}\n };\n\n R_registerRoutines(info, NULL, callMethods, NULL, NULL);\n R_useDynamicSymbols(info, FALSE);\n}\n" }, { "path": "data/misc/mlccpptypes.hpp", "content": "" }, { "path": "data/morloc/morloc.h", "content": "// morloc.h -- C ABI contract for libmorloc.so\n//\n// This is the single public header for consumers of the morloc runtime library.\n// It defines all types and function declarations exported by libmorloc.so.\n\n#ifndef __MORLOC_H__\n#define __MORLOC_H__\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// ========================================================================\n// Section 1: System includes and basic typedefs\n// ========================================================================\n\n#include // va_list, va_start, va_arg, va_end (used by pool templates)\n#include \n#include \n#include \n#include // FILE* for read_binary_fd\n#include // fd_set\n#include \n#include // pid_t, ssize_t\n#include // struct sockaddr_un\n\n// Error message output parameter: all fallible functions take this as their\n// last argument. On failure the callee sets *errmsg_ to a heap-allocated\n// string describing the error. On success *errmsg_ is set to NULL.\ntypedef char** errmsg_;\n#define ERRMSG char** errmsg_\n\n// Exit codes used by several functions.\n#define EXIT_PASS 0\n#define EXIT_FAIL 1\n\n// Convenience macros used by language extensions (pymorloc.c, rmorloc.c, cppmorloc.cpp)\n#include // free\n#define FREE(ptr) if(ptr != NULL){ free(ptr); ptr = NULL; }\n\n// Size limits shared between library and consumers.\n#define MAX_FILENAME_SIZE 128\n#define MAX_ERRMSG_SIZE 1024\n#define MAX_PATH_SIZE 512\n#define BUFFER_SIZE 4096\n\n// Opaque JSON builder handle (Rust uses its own struct internally).\ntypedef void* json_buf_t;\n\n// ========================================================================\n// Section 2: Memory / SHM types\n// ========================================================================\n\n// Pointer types for the multi-volume shared memory pool.\n//\n// relptr_t -- index into the logical (multi-volume) pool; shared between\n// processes.\n// volptr_t -- index into a single volume (0 = first block after shm header).\n// absptr_t -- absolute virtual address in the current process.\ntypedef ssize_t relptr_t;\ntypedef ssize_t volptr_t;\ntypedef void* absptr_t;\n\n#define VOLNULL -1\n#define RELNULL -1\n\n// Magic numbers for integrity checks.\n#define SHM_MAGIC 0xFECA0DF0\n#define BLK_MAGIC 0x0CB10DF0\n\n#define MAX_VOLUME_NUMBER 32\n\n// Shared memory volume header (lives at the start of each mmap'd region).\ntypedef struct shm_s {\n unsigned int magic;\n char volume_name[MAX_FILENAME_SIZE];\n int volume_index;\n size_t volume_size;\n size_t relative_offset;\n // Note: pthread_rwlock_t is opaque; consumers should not access it directly.\n // It is included here so that sizeof(shm_t) is correct for mmap calculations.\n // On Linux x86_64 this is typically 56 bytes.\n char _rwlock_storage[56]; // placeholder for pthread_rwlock_t\n volptr_t cursor;\n} shm_t;\n\n// Block header preceding every allocation inside a shared memory volume.\n// Atomic reference count for thread safety. Layout is stable (no padding).\ntypedef struct block_header_s {\n unsigned int magic;\n unsigned int reference_count; // actually _Atomic in the C impl\n size_t size;\n} block_header_t;\n\n// ========================================================================\n// Section 3: Schema types\n// ========================================================================\n\ntypedef enum {\n MORLOC_NIL,\n MORLOC_BOOL,\n MORLOC_SINT8,\n MORLOC_SINT16,\n MORLOC_SINT32,\n MORLOC_SINT64,\n MORLOC_UINT8,\n MORLOC_UINT16,\n MORLOC_UINT32,\n MORLOC_UINT64,\n MORLOC_FLOAT32,\n MORLOC_FLOAT64,\n MORLOC_TENSOR,\n MORLOC_STRING,\n MORLOC_ARRAY,\n MORLOC_TUPLE,\n MORLOC_MAP,\n MORLOC_OPTIONAL\n} morloc_serial_type;\n\n// Single-character schema encoding tokens.\n#define SCHEMA_NIL 'z'\n#define SCHEMA_BOOL 'b'\n#define SCHEMA_SINT 'i'\n#define SCHEMA_UINT 'u'\n#define SCHEMA_FLOAT 'f'\n#define SCHEMA_STRING 's'\n#define SCHEMA_ARRAY 'a'\n#define SCHEMA_TENSOR 'T'\n#define SCHEMA_TUPLE 't'\n#define SCHEMA_MAP 'm'\n#define SCHEMA_OPTIONAL '?'\n\n// Schema: recursive type descriptor used for serialisation/deserialisation.\nstruct Schema;\ntypedef struct Schema {\n morloc_serial_type type;\n size_t size; // number of parameters\n size_t width; // bytes per element when stored in a fixed-width array\n size_t* offsets; // field offsets (tuples) or ndim (tensors, in offsets[0])\n char* hint;\n struct Schema** parameters;\n char** keys; // field names (records only)\n} Schema;\n\n// Variable-length array in voidstar representation.\ntypedef struct Array {\n size_t size;\n relptr_t data;\n} Array;\n\n// Dense N-dimensional tensor in voidstar representation (row-major / C order).\ntypedef struct Tensor {\n size_t total_elements;\n uint32_t device_type; // reserved: 0 = CPU\n uint32_t device_id; // reserved: 0\n relptr_t data; // relptr to contiguous element data\n relptr_t shape; // relptr to int64_t[ndim]\n} Tensor;\n\n// ========================================================================\n// Section 4: Packet types\n// ========================================================================\n\n#define MORLOC_PACKET_MAGIC 0x0707f86d\n\n// Packet type discriminator.\ntypedef uint8_t command_type_t;\n#define PACKET_TYPE_DATA ((command_type_t)0)\n#define PACKET_TYPE_CALL ((command_type_t)1)\n#define PACKET_TYPE_PING ((command_type_t)2)\n\n// Packed command sub-structs (all 8 bytes wide).\ntypedef struct __attribute__((packed)) packet_command_type_s {\n command_type_t type;\n uint8_t padding[7];\n} packet_command_type_t;\n\n#define PACKET_ENTRYPOINT_LOCAL 0x00\n#define PACKET_ENTRYPOINT_REMOTE_SFS 0x01\n\ntypedef struct __attribute__((packed)) packet_command_call_s {\n command_type_t type;\n uint8_t entrypoint;\n uint8_t padding[2];\n uint32_t midx;\n} packet_command_call_t;\n\n// Data source, format, compression, encryption, and status constants.\n#define PACKET_SOURCE_MESG 0x00\n#define PACKET_SOURCE_FILE 0x01\n#define PACKET_SOURCE_RPTR 0x02\n\n#define PACKET_FORMAT_JSON 0x00\n#define PACKET_FORMAT_MSGPACK 0x01\n#define PACKET_FORMAT_TEXT 0x02\n#define PACKET_FORMAT_DATA 0x03\n#define PACKET_FORMAT_VOIDSTAR 0x04\n#define PACKET_FORMAT_ARROW 0x05\n\n#define PACKET_COMPRESSION_NONE 0x00\n#define PACKET_ENCRYPTION_NONE 0x00\n\n#define PACKET_STATUS_PASS 0x00\n#define PACKET_STATUS_FAIL 0x01\n\ntypedef struct __attribute__((packed)) packet_command_data_s {\n command_type_t type;\n uint8_t source;\n uint8_t format;\n uint8_t compression;\n uint8_t encryption;\n uint8_t status;\n uint8_t padding[2];\n} packet_command_data_t;\n\ntypedef struct __attribute__((packed)) packet_command_ping_s {\n command_type_t type;\n uint8_t padding[7];\n} packet_command_ping_t;\n\ntypedef union __attribute__((packed)) packet_command_u {\n packet_command_type_t cmd_type;\n packet_command_call_t call;\n packet_command_data_t data;\n packet_command_ping_t ping;\n} packet_command_t;\n\n// 32-byte packet header (stable binary format, packed).\ntypedef struct __attribute__((packed)) morloc_packet_header_s {\n uint32_t magic;\n uint16_t plain;\n uint16_t version;\n uint16_t flavor;\n uint16_t mode;\n packet_command_t command;\n uint32_t offset;\n uint64_t length;\n} morloc_packet_header_t;\n\n// Inline threshold: voidstar data <= this size is embedded in packet payload.\n#define MORLOC_INLINE_THRESHOLD (64 * 1024)\n\n// Metadata sub-header in packet metadata sections.\n#define MORLOC_METADATA_TYPE_SCHEMA_STRING 0x01\n#define MORLOC_METADATA_TYPE_XXHASH 0x02\n#define MORLOC_METADATA_HEADER_MAGIC \"mmh\"\n\ntypedef struct __attribute__((packed)) morloc_metadata_header_s {\n char magic[3];\n uint8_t type;\n uint32_t size;\n} morloc_metadata_header_t;\n\n// ========================================================================\n// Section 5: Expression / eval types\n// ========================================================================\n\ntypedef struct argument_s {\n char* value;\n char** fields;\n char** default_fields;\n size_t size;\n} argument_t;\n\ntypedef enum {\n MORLOC_X_DAT,\n MORLOC_X_APP,\n MORLOC_X_LAM,\n MORLOC_X_BND,\n MORLOC_X_PAT,\n MORLOC_X_FMT,\n MORLOC_X_SHOW,\n MORLOC_X_READ\n} morloc_expression_type;\n\ntypedef enum { APPLY_PATTERN, APPLY_LAMBDA, APPLY_FORMAT } morloc_app_expression_type;\n\ntypedef enum { SELECT_BY_KEY, SELECT_BY_INDEX, SELECT_END } morloc_pattern_type;\n\n// Forward declarations.\ntypedef struct morloc_expression_s morloc_expression_t;\ntypedef struct morloc_app_expression_s morloc_app_expression_t;\ntypedef struct morloc_lam_expression_s morloc_lam_expression_t;\ntypedef struct morloc_data_s morloc_data_t;\ntypedef struct morloc_pattern_s morloc_pattern_t;\n\ntypedef union primitive_u {\n char* s;\n uint8_t z;\n bool b;\n int8_t i1;\n int16_t i2;\n int32_t i4;\n int64_t i8;\n uint8_t u1;\n uint16_t u2;\n uint32_t u4;\n uint64_t u8;\n float f4;\n double f8;\n} primitive_t;\n\ntypedef struct morloc_data_array_s {\n Schema* schema;\n size_t size;\n morloc_expression_t** values;\n} morloc_data_array_t;\n\ntypedef struct morloc_data_s {\n bool is_voidstar;\n union {\n primitive_t lit_val;\n morloc_expression_t** tuple_val;\n morloc_data_array_t* array_val;\n void* voidstar;\n } data;\n} morloc_data_t;\n\ntypedef struct morloc_app_expression_s {\n morloc_app_expression_type type;\n union {\n morloc_pattern_t* pattern;\n morloc_lam_expression_t* lambda;\n char** fmt;\n } function;\n morloc_expression_t** args;\n size_t nargs;\n} morloc_app_expression_t;\n\ntypedef struct morloc_lam_expression_s {\n size_t nargs;\n char** args;\n morloc_expression_t* body;\n} morloc_lam_expression_t;\n\ntypedef struct morloc_pattern_s {\n morloc_pattern_type type;\n size_t size;\n union {\n size_t* indices;\n char** keys;\n } fields;\n morloc_pattern_t** selectors;\n} morloc_pattern_t;\n\ntypedef struct morloc_expression_s {\n morloc_expression_type type;\n Schema* schema;\n union {\n morloc_app_expression_t* app_expr;\n morloc_lam_expression_t* lam_expr;\n char* bnd_expr;\n char** interpolation;\n morloc_pattern_t* pattern_expr;\n morloc_data_t* data_expr;\n morloc_expression_t* unary_expr;\n } expr;\n} morloc_expression_t;\n\n// ========================================================================\n// Section 6: Manifest types\n// ========================================================================\n\ntypedef struct {\n char* lang;\n char** exec; // NULL-terminated array\n char* socket; // socket basename\n} manifest_pool_t;\n\ntypedef enum {\n MARG_POS = 0,\n MARG_OPT,\n MARG_FLAG,\n MARG_GRP\n} manifest_arg_kind_t;\n\ntypedef struct manifest_arg_s manifest_arg_t;\n\ntypedef struct {\n char* key;\n manifest_arg_t* arg;\n} manifest_grp_entry_t;\n\nstruct manifest_arg_s {\n manifest_arg_kind_t kind;\n char** desc;\n char* metavar;\n char* type_desc;\n bool quoted;\n char short_opt;\n char* long_opt;\n char* long_rev;\n char* default_val;\n char grp_short;\n char* grp_long;\n manifest_grp_entry_t* entries;\n size_t n_entries;\n};\n\ntypedef struct {\n char* name;\n char** desc;\n} manifest_cmd_group_t;\n\ntypedef struct {\n char* name;\n bool is_pure;\n uint32_t mid;\n size_t pool_index;\n size_t* needed_pools;\n size_t n_needed_pools;\n char** arg_schemas;\n char* return_schema;\n char** desc;\n char* return_type;\n char** return_desc;\n manifest_arg_t* args;\n size_t n_args;\n morloc_expression_t* expr;\n char* group;\n} manifest_command_t;\n\ntypedef struct {\n char* type;\n char* host;\n int port;\n char* socket;\n} manifest_service_t;\n\ntypedef struct {\n int version;\n char* name;\n char* build_dir;\n manifest_pool_t* pools;\n size_t n_pools;\n manifest_command_t* commands;\n size_t n_commands;\n manifest_cmd_group_t* groups;\n size_t n_groups;\n manifest_service_t* service;\n} manifest_t;\n\n// ========================================================================\n// Section 7: Daemon / HTTP / Router types\n// ========================================================================\n\n// -- Call types --\n\ntypedef struct morloc_call_s {\n uint32_t midx;\n uint8_t** args;\n size_t nargs;\n int owns_args;\n} morloc_call_t;\n\ntypedef struct client_list_s {\n int fd;\n struct client_list_s* next;\n} client_list_t;\n\ntypedef struct language_daemon_s {\n char* socket_path;\n char* tmpdir;\n char* shm_basename;\n shm_t* shm;\n size_t shm_default_size;\n int server_fd;\n fd_set read_fds;\n client_list_t* client_fds;\n} language_daemon_t;\n\ntypedef struct morloc_socket_s {\n char* lang;\n char** syscmd;\n char* socket_filename;\n int pid;\n} morloc_socket_t;\n\n// -- Binding store --\n\ntypedef struct binding_entry_s {\n uint64_t hash;\n char* expr;\n char* artifact_dir;\n char* type_sig;\n char** names;\n size_t n_names;\n} binding_entry_t;\n\ntypedef struct binding_store_s {\n binding_entry_t* entries;\n size_t capacity;\n size_t count;\n char* base_dir;\n char* names_path;\n} binding_store_t;\n\n// -- Daemon config and request/response --\n\ntypedef void (*pool_check_fn_t)(morloc_socket_t* sockets, size_t n_pools);\ntypedef bool (*pool_alive_fn_t)(size_t pool_index);\n\ntypedef struct daemon_config_s {\n const char* unix_socket_path;\n int tcp_port;\n int http_port;\n pool_check_fn_t pool_check_fn;\n pool_alive_fn_t pool_alive_fn;\n size_t n_pools;\n int eval_timeout;\n} daemon_config_t;\n\ntypedef enum {\n DAEMON_CALL,\n DAEMON_DISCOVER,\n DAEMON_HEALTH,\n DAEMON_EVAL,\n DAEMON_TYPECHECK,\n DAEMON_BIND,\n DAEMON_BINDINGS,\n DAEMON_UNBIND\n} daemon_method_t;\n\ntypedef struct daemon_request_s {\n char* id;\n daemon_method_t method;\n char* command;\n char* args_json;\n char* expr;\n char* name;\n} daemon_request_t;\n\ntypedef struct daemon_response_s {\n char* id;\n bool success;\n char* result_json;\n char* error;\n} daemon_response_t;\n\n// -- HTTP types --\n\ntypedef enum {\n HTTP_GET,\n HTTP_POST,\n HTTP_DELETE,\n HTTP_OPTIONS\n} http_method_t;\n\ntypedef struct http_request_s {\n http_method_t method;\n char path[256];\n char* body;\n size_t body_len;\n} http_request_t;\n\n// -- Router types --\n\ntypedef struct router_program_s {\n char* name;\n char* manifest_path;\n manifest_t* manifest;\n pid_t daemon_pid;\n char daemon_socket[sizeof(((struct sockaddr_un*)0)->sun_path)];\n} router_program_t;\n\ntypedef struct router_s {\n router_program_t* programs;\n size_t n_programs;\n char* fdb_path;\n} router_t;\n\n// ========================================================================\n// Section 8: Pool types\n// ========================================================================\n\ntypedef uint8_t* (*pool_dispatch_fn_t)(\n uint32_t mid,\n const uint8_t** args,\n size_t nargs,\n void* ctx\n);\n\ntypedef enum {\n POOL_THREADS,\n POOL_FORK,\n POOL_SINGLE\n} pool_concurrency_t;\n\ntypedef struct {\n pool_dispatch_fn_t local_dispatch;\n pool_dispatch_fn_t remote_dispatch;\n void* dispatch_ctx;\n pool_concurrency_t concurrency;\n int initial_workers;\n bool dynamic_scaling;\n void (*post_fork_child)(void* ctx);\n} pool_config_t;\n\ntypedef struct pool_state_s pool_state_t;\n\n// ========================================================================\n// Section 9: Arrow types\n// ========================================================================\n\n#ifndef ARROW_C_DATA_INTERFACE\n#define ARROW_C_DATA_INTERFACE\n\nstruct ArrowSchema {\n const char* format;\n const char* name;\n const char* metadata;\n int64_t flags;\n int64_t n_children;\n struct ArrowSchema** children;\n struct ArrowSchema* dictionary;\n void (*release)(struct ArrowSchema*);\n void* private_data;\n};\n\nstruct ArrowArray {\n int64_t length;\n int64_t null_count;\n int64_t offset;\n int64_t n_buffers;\n int64_t n_children;\n const void** buffers;\n struct ArrowArray** children;\n struct ArrowArray* dictionary;\n void (*release)(struct ArrowArray*);\n void* private_data;\n};\n\n#endif // ARROW_C_DATA_INTERFACE\n\n#define ARROW_SHM_MAGIC 0xA770DA7A\n#define ARROW_BUFFER_ALIGN 64\n#define ARROW_ALIGN_UP(x) (((x) + ARROW_BUFFER_ALIGN - 1) & ~((size_t)ARROW_BUFFER_ALIGN - 1))\n\ntypedef struct arrow_column_desc {\n morloc_serial_type type;\n uint64_t length;\n uint64_t null_count;\n uint32_t name_offset;\n uint16_t name_length;\n uint64_t data_offset;\n uint64_t data_size;\n} arrow_column_desc_t;\n\ntypedef struct arrow_shm_header {\n uint32_t magic;\n uint32_t n_columns;\n uint64_t n_rows;\n uint64_t total_size;\n} arrow_shm_header_t;\n\n// ========================================================================\n// Section 10: Slurm / resource types\n// ========================================================================\n\n#define MAX_SLURM_COMMAND_LENGTH 1024\n\ntypedef struct resources_s {\n int memory; // in GB\n int time; // walltime in seconds\n int cpus;\n int gpus;\n} resources_t;\n\n// ========================================================================\n// Section 11: Function declarations -- Memory / SHM\n// ========================================================================\n\nshm_t* shinit(const char* shm_basename, size_t volume_index, size_t shm_size, ERRMSG);\nshm_t* shopen(size_t volume_index, ERRMSG);\nbool shclose(ERRMSG);\nvoid shm_set_fallback_dir(const char* dir);\nvoid* shmalloc(size_t size, ERRMSG);\nvoid* shmemcpy(void* src, size_t size, ERRMSG);\nbool shfree(absptr_t ptr, ERRMSG);\nbool shincref(absptr_t ptr, ERRMSG);\nbool shfree_by_schema(absptr_t ptr, const Schema* schema, ERRMSG);\nvoid* shcalloc(size_t nmemb, size_t size, ERRMSG);\nvoid* shrealloc(void* ptr, size_t size, ERRMSG);\nsize_t total_shm_size(void);\nvolptr_t rel2vol(relptr_t ptr, ERRMSG);\nabsptr_t rel2abs(relptr_t ptr, ERRMSG);\n\n// Convenience: resolve a relptr, using base_ptr if available (no SHM lookup needed).\nstatic inline void* resolve_relptr(relptr_t relptr, const void* base_ptr, ERRMSG) {\n if (base_ptr) {\n return (char*)base_ptr + relptr;\n }\n return rel2abs(relptr, errmsg_);\n}\nrelptr_t vol2rel(volptr_t ptr, shm_t* shm);\nabsptr_t vol2abs(volptr_t ptr, shm_t* shm);\nrelptr_t abs2rel(absptr_t ptr, ERRMSG);\nshm_t* abs2shm(absptr_t ptr, ERRMSG);\nblock_header_t* abs2blk(void* ptr, ERRMSG);\n\n// ========================================================================\n// Section 12: Function declarations -- Schema\n// ========================================================================\n\nSchema* parse_schema(const char* schema, ERRMSG);\nchar* schema_to_string(const Schema* schema);\nvoid* get_ptr(const Schema* schema, ERRMSG);\nvoid free_schema(Schema* schema);\nbool schema_is_fixed_width(const Schema* schema);\nsize_t schema_alignment(const Schema* schema);\nsize_t calculate_voidstar_size(const void* data, const Schema* schema, ERRMSG);\n\n// Inline helpers used by language extensions (pymorloc.c, rmorloc.c)\n#define ALIGN_UP(x, align) (((x) + (align) - 1) & ~((size_t)(align) - 1))\n\nstatic inline size_t schema_tensor_ndim(const Schema* schema) {\n if (schema == NULL || schema->size == 0) return 0;\n // ndim is stored in offsets[0] for tensor schemas\n return schema->offsets ? schema->offsets[0] : 0;\n}\n\n// ========================================================================\n// Section 13: Function declarations -- Serialisation (pack/unpack)\n// ========================================================================\n\nint pack(const void* mlc, const char* schema_str, char** mpkptr, size_t* mpk_size, ERRMSG);\nint pack_with_schema(const void* mlc, const Schema* schema, char** mpkptr, size_t* mpk_size, ERRMSG);\nint unpack_with_schema(const char* mpk, size_t mpk_size, const Schema* schema, void** mlcptr, ERRMSG);\n\n// ========================================================================\n// Section 14: Function declarations -- Packets\n// ========================================================================\n\nmorloc_packet_header_t* read_morloc_packet_header(const uint8_t* msg, ERRMSG);\nbool packet_is_ping(const uint8_t* packet, ERRMSG);\nbool packet_is_local_call(const uint8_t* packet, ERRMSG);\nbool packet_is_remote_call(const uint8_t* packet, ERRMSG);\nsize_t morloc_packet_size_from_header(const morloc_packet_header_t* header);\nsize_t morloc_packet_size(const uint8_t* packet, ERRMSG);\nuint8_t* return_ping(const uint8_t* packet, ERRMSG);\nuint8_t* make_ping_packet(void);\nuint8_t* make_standard_data_packet(relptr_t ptr, const Schema* schema);\nuint8_t* make_arrow_data_packet(relptr_t ptr, const Schema* schema);\nuint8_t* make_mpk_data_packet(const char* mpk_filename, const Schema* schema);\nuint8_t* make_data_packet_from_mpk(const char* mpk, size_t mpk_size, const Schema* schema);\nint get_data_packet_as_mpk(const uint8_t* packet, const Schema* schema, char** mpk_out, size_t* mpk_size_out, ERRMSG);\nchar* read_schema_from_packet_meta(const uint8_t* packet, ERRMSG);\nuint8_t* make_fail_packet(const char* failure_message);\nchar* get_morloc_data_packet_error_message(const uint8_t* data, ERRMSG);\nuint8_t* get_morloc_data_packet_value(const uint8_t* data, const Schema* schema, ERRMSG);\nuint8_t* make_morloc_local_call_packet(uint32_t midx, const uint8_t** arg_packets, size_t nargs, ERRMSG);\nuint8_t* make_morloc_remote_call_packet(uint32_t midx, const uint8_t** arg_packets, size_t nargs, ERRMSG);\nmorloc_call_t* read_morloc_call_packet(const uint8_t* packet, ERRMSG);\nvoid free_morloc_call(morloc_call_t* call);\nint print_morloc_data_packet(const uint8_t* packet, const Schema* schema, ERRMSG);\nint flatten_voidstar_to_buffer(const void* data, const Schema* schema, uint8_t** out_buf, size_t* out_size, ERRMSG);\nuint8_t* make_data_packet_auto(void* voidstar, relptr_t relptr, const Schema* schema, ERRMSG);\nint adjust_voidstar_relptrs(void* data, const Schema* schema, relptr_t base_rel, ERRMSG);\nvoid* read_voidstar_binary(const uint8_t* blob, size_t blob_size, const Schema* schema, ERRMSG);\nbool parse_morloc_call_arguments(uint8_t* packet, uint8_t** args, size_t* nargs, ERRMSG);\nbool hash_morloc_packet(const uint8_t* packet, const Schema* schema, uint64_t seed, uint64_t* hash, ERRMSG);\n\n// ========================================================================\n// Section 15: Function declarations -- Printing / output\n// ========================================================================\n\nchar* quoted(const char* input);\nbool print_voidstar(const void* voidstar, const Schema* schema, ERRMSG);\nbool pretty_print_voidstar(const void* voidstar, const Schema* schema, ERRMSG);\nbool print_arrow_as_json(const void* data, ERRMSG);\nbool print_arrow_as_table(const void* data, ERRMSG);\nbool print_hex_dump(const uint8_t* data, size_t size, ERRMSG);\nchar* voidstar_to_json_string(const void* voidstar, const Schema* schema, ERRMSG);\n\n// ========================================================================\n// Section 16: Function declarations -- Daemon / socket communication\n// ========================================================================\n\nvoid close_socket(int socket_id);\nvoid close_daemon(language_daemon_t** daemon_ptr);\nlanguage_daemon_t* start_daemon(\n const char* socket_path, const char* tmpdir,\n const char* shm_basename, size_t shm_default_size, ERRMSG);\nuint8_t* stream_from_client_wait(int client_fd, int pselect_timeout_us, int recv_timeout_us, ERRMSG);\nuint8_t* stream_from_client(int client_fd, ERRMSG);\nuint8_t* send_and_receive_over_socket_wait(\n const char* socket_path, const uint8_t* packet,\n int pselect_timeout_us, int recv_timeout_us, ERRMSG);\nuint8_t* send_and_receive_over_socket(const char* socket_path, const uint8_t* packet, ERRMSG);\nsize_t send_packet_to_foreign_server(int client_fd, uint8_t* packet, ERRMSG);\nint wait_for_client_with_timeout(language_daemon_t* daemon, int timeout_us, ERRMSG);\nint wait_for_client(language_daemon_t* daemon, ERRMSG);\n\n// Daemon event loop and dispatch.\nvoid daemon_run(daemon_config_t* config, manifest_t* manifest,\n morloc_socket_t* sockets, size_t n_pools,\n const char* shm_basename);\ndaemon_response_t* daemon_dispatch(manifest_t* manifest,\n daemon_request_t* request,\n morloc_socket_t* sockets,\n const char* shm_basename);\ndaemon_request_t* daemon_parse_request(const char* json, size_t len, ERRMSG);\ndaemon_response_t* daemon_parse_response(const char* json, size_t len, ERRMSG);\nchar* daemon_serialize_response(daemon_response_t* response, size_t* out_len);\nchar* daemon_build_discovery(manifest_t* manifest);\nvoid daemon_set_eval_timeout(int timeout_sec);\nvoid daemon_free_request(daemon_request_t* req);\nvoid daemon_free_response(daemon_response_t* resp);\n\n// Binding store (public types only; internal hash table functions are Rust-side).\nbinding_store_t* binding_store_init(const char* base_dir);\nvoid binding_store_free(binding_store_t* store);\n\n// ========================================================================\n// Section 17: Function declarations -- HTTP\n// ========================================================================\n\nhttp_request_t* http_parse_request(int fd, ERRMSG);\nbool http_write_response(int fd, int status, const char* content_type,\n const char* body, size_t body_len);\ndaemon_request_t* http_to_daemon_request(http_request_t* req, ERRMSG);\nvoid http_free_request(http_request_t* req);\n\n// ========================================================================\n// Section 18: Function declarations -- Router\n// ========================================================================\n\nrouter_t* router_init(const char* fdb_path, ERRMSG);\nvoid router_run(daemon_config_t* config, router_t* router);\nbool router_start_program(router_program_t* prog, ERRMSG);\ndaemon_response_t* router_forward(router_t* router, const char* program,\n daemon_request_t* request, ERRMSG);\nchar* router_build_discovery(router_t* router);\nvoid router_free(router_t* router);\n\n// ========================================================================\n// Section 19: Function declarations -- Pool\n// ========================================================================\n\nint pool_main(int argc, char** argv, pool_config_t* config);\nuint8_t* pool_dispatch_packet(\n const uint8_t* packet,\n pool_dispatch_fn_t local_dispatch,\n pool_dispatch_fn_t remote_dispatch,\n void* ctx);\nvoid pool_mark_busy(void);\nvoid pool_mark_idle(void);\n\n// ========================================================================\n// Section 20: Function declarations -- Arrow\n// ========================================================================\n\nsize_t arrow_element_size(morloc_serial_type type);\nconst char* arrow_format_string(morloc_serial_type type);\nmorloc_serial_type arrow_format_to_type(const char* format);\nrelptr_t arrow_to_shm(const struct ArrowArray* array, const struct ArrowSchema* schema, ERRMSG);\nint arrow_validate(const arrow_shm_header_t* header, const Schema* schema, ERRMSG);\nconst void* arrow_column_data(const arrow_shm_header_t* header, uint32_t col_index);\nconst arrow_column_desc_t* arrow_column_desc(const arrow_shm_header_t* header, uint32_t col_index);\nconst char* arrow_column_name(const arrow_shm_header_t* header, uint32_t col_index);\nint arrow_from_shm(const arrow_shm_header_t* header,\n struct ArrowSchema* out_schema,\n struct ArrowArray* out_array, ERRMSG);\n\n// ========================================================================\n// Section 21: Function declarations -- Cache\n// ========================================================================\n\nchar* put_cache_packet(const uint8_t* voidstar, const Schema* schema, uint64_t key, const char* cache_path, ERRMSG);\nuint8_t* get_cache_packet(uint64_t key, const char* cache_path, ERRMSG);\nbool del_cache_packet(uint64_t key, const char* cache_path, ERRMSG);\nchar* check_cache_packet(uint64_t key, const char* cache_path, ERRMSG);\n\n// ========================================================================\n// Section 22: Function declarations -- CLI / argument parsing\n// ========================================================================\n\nargument_t* initialize_positional(char* value);\nargument_t* initialize_unrolled(size_t size, char* default_value, char** fields, char** default_fields);\nvoid free_argument_t(argument_t* arg);\nuint8_t* parse_cli_data_argument(uint8_t* dest, const argument_t* arg, const Schema* schema, ERRMSG);\nuint8_t* make_call_packet_from_cli(\n uint8_t* dest, uint32_t mid,\n argument_t** args, char** arg_schema_strs, ERRMSG);\nvoid* load_morloc_data_file(const char* path, uint8_t* data, size_t data_size, const Schema* schema, ERRMSG);\n\n// ========================================================================\n// Section 23: Function declarations -- Expression evaluation\n// ========================================================================\n\nmorloc_expression_t* make_morloc_bound_var(const char* schema_str, char* varname, ERRMSG);\nmorloc_expression_t* make_morloc_literal(const char* schema_str, primitive_t lit, ERRMSG);\nmorloc_expression_t* make_morloc_pattern(const char* schema_str, morloc_pattern_t* pattern, ERRMSG);\nmorloc_pattern_t* make_morloc_pattern_end(void);\nabsptr_t morloc_eval(\n morloc_expression_t* expr, Schema* return_schema,\n uint8_t** arg_voidstar, Schema** arg_schemas, size_t nargs, ERRMSG);\n\n// ========================================================================\n// Section 24: Function declarations -- Manifest\n// ========================================================================\n\nmanifest_t* parse_manifest(const char* text, ERRMSG);\nmanifest_t* read_manifest(const char* path, ERRMSG);\nvoid free_manifest(manifest_t* manifest);\nmorloc_expression_t* build_manifest_expr(const char* json_str, ERRMSG);\nchar* manifest_to_discovery_json(const manifest_t* manifest);\n\n// ========================================================================\n// Section 25: Function declarations -- Intrinsics\n// ========================================================================\n\nint mlc_save(const absptr_t data, const Schema* schema, const char* path, ERRMSG);\nint mlc_save_json(const absptr_t data, const Schema* schema, const char* path, ERRMSG);\nint mlc_save_voidstar(const absptr_t data, const Schema* schema, const char* path, ERRMSG);\nvoid* mlc_load(const char* path, const Schema* schema, ERRMSG);\nchar* mlc_hash(const absptr_t data, const Schema* schema, ERRMSG);\nchar* mlc_show(const absptr_t data, const Schema* schema, ERRMSG);\nvoid* mlc_read(const char* json_str, const Schema* schema, ERRMSG);\nrelptr_t write_voidstar_binary(int fd, const void* data, const Schema* schema, ERRMSG);\n\n// ========================================================================\n// Section 26: Function declarations -- Slurm\n// ========================================================================\n\nsize_t parse_slurm_time(const char* time_str, ERRMSG);\nchar* write_slurm_time(int seconds);\nbool slurm_job_is_complete(uint32_t job_id);\nuint32_t submit_morloc_slurm_job(\n const char* nexus_path,\n const char* socket_basename,\n const char* call_packet_filename,\n const char* result_cache_filename,\n const char* output_filename,\n const char* error_filename,\n const resources_t* resources,\n ERRMSG);\nuint8_t* remote_call(\n int midx,\n const char* socket_basename,\n const char* cache_path,\n const resources_t* resources,\n const uint8_t** arg_packets,\n size_t nargs,\n ERRMSG);\n\n// ========================================================================\n// Section 27: Function declarations -- Utility\n// ========================================================================\n\nvoid hex(const void* ptr, size_t size);\nbool file_exists(const char* filename);\nint mkdir_p(const char* path, ERRMSG);\nvoid delete_directory(const char* path);\nbool has_suffix(const char* x, const char* suffix);\nint write_atomic(const char* filename, const uint8_t* data, size_t size, ERRMSG);\nint write_binary_fd(int fd, const char* buf, size_t count, ERRMSG);\nint print_binary(const char* buf, size_t count, ERRMSG);\nuint8_t* read_binary_fd(FILE* file, size_t* file_size, ERRMSG);\nuint8_t* read_binary_file(const char* filename, size_t* file_size, ERRMSG);\n\n// ========================================================================\n// Section 28: Function declarations -- Hashing\n// ========================================================================\n\nuint64_t morloc_xxh64(const void* input, size_t length, uint64_t seed);\n\n// ========================================================================\n// Section 29: Function declarations -- JSON reader\n// ========================================================================\n\nuint8_t* read_json_with_schema(uint8_t* voidstar, char* json_data, const Schema* schema, ERRMSG);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // __MORLOC_H__\n" }, { "path": "data/rust/.gitignore", "content": "target/\n" }, { "path": "data/rust/Cargo.toml", "content": "[workspace]\nmembers = [\"morloc-manifest\", \"morloc-runtime\", \"morloc-nexus\", \"morloc-manager\"]\nresolver = \"2\"\n\n[workspace.dependencies]\nlibc = \"0.2\"\nserde = { version = \"1\", features = [\"derive\"] }\nserde_json = \"1\"\nrmp-serde = \"1\"\ntwox-hash = \"2\"\nnix = { version = \"0.29\", features = [\"signal\", \"socket\", \"mman\", \"process\", \"fs\"] }\nclap = { version = \"4\", features = [\"derive\"] }\nthiserror = \"2\"\n\n[profile.release]\nopt-level = 2\nlto = \"thin\"\n" }, { "path": "data/rust/morloc-manager/Cargo.toml", "content": "[package]\nname = \"morloc-manager\"\nversion = \"0.23.1\"\nedition = \"2021\"\ndescription = \"Container lifecycle manager for Morloc\"\n\n[[bin]]\nname = \"morloc-manager\"\npath = \"src/main.rs\"\n\n[dependencies]\nserde = { workspace = true }\nserde_json = { workspace = true }\nnix = { version = \"0.29\", features = [\"signal\", \"socket\", \"mman\", \"process\", \"fs\", \"user\"] }\nclap = { workspace = true }\nthiserror = { workspace = true }\nsha2 = \"0.10\"\nchrono = { version = \"0.4\", features = [\"serde\"] }\ndirs = \"6\"\n\n[dev-dependencies]\ntempfile = \"3\"\n" }, { "path": "data/rust/morloc-manager/src/config.rs", "content": "use std::fs;\nuse std::os::unix::fs::{OpenOptionsExt, PermissionsExt};\nuse std::path::{Path, PathBuf};\nuse std::process::Command as StdCommand;\n\nuse crate::error::{ManagerError, Result};\nuse crate::types::*;\n\n// ======================================================================\n// Path utilities\n// ======================================================================\n\npub fn config_dir(scope: Scope) -> PathBuf {\n match scope {\n Scope::Local => dirs::config_dir()\n .unwrap_or_else(|| PathBuf::from(\"~/.config\"))\n .join(\"morloc\"),\n Scope::System => PathBuf::from(\"/etc/morloc\"),\n }\n}\n\npub fn config_path(scope: Scope) -> PathBuf {\n config_dir(scope).join(\"config.json\")\n}\n\npub fn data_dir(scope: Scope) -> PathBuf {\n match scope {\n Scope::Local => dirs::data_dir()\n .unwrap_or_else(|| PathBuf::from(\"~/.local/share\"))\n .join(\"morloc\"),\n Scope::System => PathBuf::from(\"/usr/local/share/morloc\"),\n }\n}\n\n// Environment paths\n\npub fn env_config_dir(scope: Scope, name: &str) -> PathBuf {\n config_dir(scope).join(\"environments\").join(name)\n}\n\npub fn env_config_path(scope: Scope, name: &str) -> PathBuf {\n env_config_dir(scope, name).join(\"env.json\")\n}\n\npub fn env_dockerfile_path(scope: Scope, name: &str) -> PathBuf {\n env_config_dir(scope, name).join(\"Dockerfile\")\n}\n\npub fn env_flags_path(scope: Scope, name: &str) -> PathBuf {\n env_config_dir(scope, name).join(\"env.flags\")\n}\n\npub fn env_data_dir(scope: Scope, name: &str) -> PathBuf {\n data_dir(scope).join(\"environments\").join(name)\n}\n\n// ======================================================================\n// Reading configuration\n// ======================================================================\n\npub fn read_config(path: &Path) -> Result {\n let bytes = fs::read(path).map_err(|e| {\n if e.kind() == std::io::ErrorKind::PermissionDenied {\n ManagerError::ConfigPermissionDenied(path.display().to_string())\n } else {\n ManagerError::ConfigNotFound(path.display().to_string())\n }\n })?;\n serde_json::from_slice(&bytes).map_err(|e| ManagerError::ConfigParseError {\n path: path.display().to_string(),\n msg: e.to_string(),\n })\n}\n\npub fn read_active_config() -> Option {\n let local_path = config_path(Scope::Local);\n if let Ok(cfg) = read_config::(&local_path) {\n return Some(cfg);\n }\n let system_path = config_path(Scope::System);\n read_config::(&system_path).ok()\n}\n\npub fn read_env_config(scope: Scope, name: &str) -> Result {\n read_config(&env_config_path(scope, name))\n}\n\n// ======================================================================\n// Writing configuration\n// ======================================================================\n\npub fn write_config(path: &Path, val: &T) -> Result<()> {\n let dir = path.parent().unwrap();\n fs::create_dir_all(dir).map_err(|e| ManagerError::ConfigParseError {\n path: path.display().to_string(),\n msg: e.to_string(),\n })?;\n best_effort_chmod(dir, 0o755);\n\n let lock_path = format!(\"{}.lock\", path.display());\n with_file_lock(&lock_path, || {\n // Atomic write: temp file then rename\n let tmp_path = path.with_extension(\"tmp\");\n let json = serde_json::to_vec(val).map_err(|e| ManagerError::ConfigParseError {\n path: path.display().to_string(),\n msg: e.to_string(),\n })?;\n fs::write(&tmp_path, &json).map_err(|e| ManagerError::ConfigParseError {\n path: path.display().to_string(),\n msg: e.to_string(),\n })?;\n fs::rename(&tmp_path, path).map_err(|e| ManagerError::ConfigParseError {\n path: path.display().to_string(),\n msg: e.to_string(),\n })?;\n best_effort_chmod(path, 0o644);\n Ok(())\n })\n}\n\npub fn write_env_config(scope: Scope, name: &str, ec: &EnvironmentConfig) -> Result<()> {\n write_config(&env_config_path(scope, name), ec)\n}\n\n// ======================================================================\n// Scope utilities\n// ======================================================================\n\n/// Find which scope an environment lives in. Checks local first, then system.\npub fn find_env_scope(name: &str) -> Result {\n let local_path = env_config_path(Scope::Local, name);\n if local_path.is_file() {\n return Ok(Scope::Local);\n }\n let sys_path = env_config_path(Scope::System, name);\n if sys_path.is_file() {\n return Ok(Scope::System);\n }\n Err(ManagerError::EnvironmentNotFound(name.to_string()))\n}\n\n/// List environment names in a given scope.\npub fn list_env_names(scope: Scope) -> Vec {\n let env_dir = config_dir(scope).join(\"environments\");\n if !env_dir.is_dir() {\n return Vec::new();\n }\n let Ok(entries) = fs::read_dir(&env_dir) else {\n return Vec::new();\n };\n entries\n .filter_map(|e| e.ok())\n .filter(|e| e.path().join(\"env.json\").is_file())\n .filter_map(|e| e.file_name().into_string().ok())\n .collect()\n}\n\n\n// ======================================================================\n// Flags files\n// ======================================================================\n\npub fn read_flags_file(path: &Path) -> Vec {\n let Ok(contents) = fs::read_to_string(path) else {\n return Vec::new();\n };\n contents\n .lines()\n .map(|line| line.trim())\n .filter(|line| !line.is_empty() && !line.starts_with('#'))\n .flat_map(shell_expand_line)\n .collect()\n}\n\n/// Expand a single flagfile line through the shell, getting glob expansion,\n/// environment variable expansion, tilde expansion, and quote handling.\n/// Falls back to simple whitespace splitting if the shell invocation fails.\nfn shell_expand_line(line: &str) -> Vec {\n let output = StdCommand::new(\"sh\")\n .args([\"-c\", &format!(\"printf '%s\\\\0' {}\", line)])\n .output();\n match output {\n Ok(out) if out.status.success() => {\n let stdout = String::from_utf8_lossy(&out.stdout);\n let tokens: Vec = stdout\n .split('\\0')\n .filter(|s| !s.is_empty())\n .map(|s| s.to_string())\n .collect();\n if tokens.is_empty() {\n line.split_whitespace().map(|s| s.to_string()).collect()\n } else {\n tokens\n }\n }\n _ => line.split_whitespace().map(|s| s.to_string()).collect(),\n }\n}\n\n/// Read flags file preserving one line per entry (for display).\npub fn read_flags_file_lines(path: &Path) -> Vec {\n let Ok(contents) = fs::read_to_string(path) else {\n return Vec::new();\n };\n contents\n .lines()\n .map(|line| line.trim())\n .filter(|line| !line.is_empty() && !line.starts_with('#'))\n .map(|s| s.to_string())\n .collect()\n}\n\n// ======================================================================\n// File locking\n// ======================================================================\n\nfn with_file_lock(lock_path: &str, action: F) -> Result\nwhere\n F: FnOnce() -> Result,\n{\n if let Some(parent) = Path::new(lock_path).parent() {\n let _ = fs::create_dir_all(parent);\n }\n let file = std::fs::OpenOptions::new()\n .write(true)\n .create(true)\n .truncate(false)\n .mode(0o644)\n .open(lock_path)\n .map_err(|e| {\n if e.kind() == std::io::ErrorKind::PermissionDenied {\n ManagerError::ConfigPermissionDenied(format!(\n \"{}. Use sudo for system-scope operations\", lock_path\n ))\n } else {\n ManagerError::ConfigParseError {\n path: lock_path.to_string(),\n msg: format!(\"Failed to open lock file: {e}\"),\n }\n }\n })?;\n\n use nix::fcntl::FlockArg;\n let locked = nix::fcntl::Flock::lock(file, FlockArg::LockExclusive).map_err(\n |(_file, errno)| ManagerError::ConfigParseError {\n path: lock_path.to_string(),\n msg: format!(\"Failed to acquire lock: {errno}\"),\n },\n )?;\n\n let result = action();\n\n // Lock is released when Flock is dropped\n drop(locked);\n result\n}\n\n// ======================================================================\n// Internal\n// ======================================================================\n\nfn best_effort_chmod(path: &Path, mode: u32) {\n let _ = fs::set_permissions(path, fs::Permissions::from_mode(mode));\n}\n" }, { "path": "data/rust/morloc-manager/src/container.rs", "content": "use std::io;\nuse std::process::{Command, ExitStatus, Stdio};\n\nuse crate::types::ContainerEngine;\n\n// ======================================================================\n// Configuration records\n// ======================================================================\n\n#[derive(Debug, Clone)]\npub struct RunConfig {\n pub image: String,\n pub bind_mounts: Vec<(String, String)>,\n pub ports: Vec<(u16, u16)>,\n pub env: Vec<(String, String)>,\n pub read_only: bool,\n pub interactive: bool,\n pub remove_after: bool,\n pub name: Option,\n pub shm_size: Option,\n pub command: Option>,\n pub work_dir: Option,\n pub selinux_suffix: String,\n pub extra_flags: Vec,\n}\n\nimpl RunConfig {\n pub fn new(image: &str) -> Self {\n Self {\n image: image.to_string(),\n bind_mounts: Vec::new(),\n ports: Vec::new(),\n env: Vec::new(),\n read_only: false,\n interactive: false,\n remove_after: true,\n name: None,\n shm_size: None,\n command: None,\n work_dir: None,\n selinux_suffix: String::new(),\n extra_flags: Vec::new(),\n }\n }\n}\n\n#[derive(Debug, Clone)]\npub struct BuildConfig {\n pub dockerfile: String,\n pub context: String,\n pub tag: String,\n pub build_args: Vec<(String, String)>,\n}\n\n// ======================================================================\n// Engine detection\n// ======================================================================\n\npub fn engine_executable(engine: ContainerEngine) -> &'static str {\n match engine {\n ContainerEngine::Docker => \"docker\",\n ContainerEngine::Podman => \"podman\",\n }\n}\n\n// ======================================================================\n// Operations\n// ======================================================================\n\npub fn container_run(engine: ContainerEngine, cfg: &RunConfig) -> (ExitStatus, String, String) {\n let exe = engine_executable(engine);\n let extra = engine_specific_run_flags_io(engine);\n let args = build_run_args(engine, &extra, cfg);\n run_process(exe, &args)\n}\n\n/// Like `container_run` but captures both stdout and stderr (no streaming).\npub fn container_run_quiet(engine: ContainerEngine, cfg: &RunConfig) -> (ExitStatus, String, String) {\n let exe = engine_executable(engine);\n let extra = engine_specific_run_flags_io(engine);\n let args = build_run_args(engine, &extra, cfg);\n run_process_quiet(exe, &args)\n}\n\npub fn container_run_passthrough(\n engine: ContainerEngine,\n verbose: bool,\n shell: bool,\n cfg: &RunConfig,\n) -> ExitStatus {\n let exe = engine_executable(engine);\n let extra = engine_specific_run_flags_io(engine);\n let args = build_run_args(engine, &extra, cfg);\n\n if verbose || shell {\n let quoted: Vec = args\n .iter()\n .map(|a| {\n if a.contains(' ') {\n format!(\"'{a}'\")\n } else {\n a.clone()\n }\n })\n .collect();\n eprintln!(\"[morloc-manager] {exe} {}\", quoted.join(\" \"));\n }\n\n Command::new(exe)\n .args(&args)\n .stdin(Stdio::inherit())\n .stdout(Stdio::inherit())\n .stderr(Stdio::inherit())\n .status()\n .unwrap_or_else(|_| std::process::exit(1))\n}\n\npub fn container_build(engine: ContainerEngine, cfg: &BuildConfig) -> (ExitStatus, String, String) {\n let exe = engine_executable(engine);\n let args = build_build_args(cfg);\n run_process(exe, &args)\n}\n\npub fn container_pull(engine: ContainerEngine, image: &str) -> (ExitStatus, String, String) {\n let exe = engine_executable(engine);\n run_process(exe, &[\"pull\".to_string(), image.to_string()])\n}\n\n/// Build a container image with all output (stdout+stderr) redirected to stderr.\n/// Use for IO () commands where stdout must stay clean.\npub fn container_build_visible(engine: ContainerEngine, cfg: &BuildConfig) -> ExitStatus {\n let exe = engine_executable(engine);\n let args = build_build_args(cfg);\n run_process_to_stderr(exe, &args)\n}\n\n/// Pull a container image with all output (stdout+stderr) redirected to stderr.\n/// Use for IO () commands where stdout must stay clean.\npub fn container_pull_visible(engine: ContainerEngine, image: &str) -> ExitStatus {\n let exe = engine_executable(engine);\n run_process_to_stderr(exe, &[\"pull\".to_string(), image.to_string()])\n}\n\npub fn image_exists_locally(engine: ContainerEngine, image: &str) -> bool {\n let exe = engine_executable(engine);\n Command::new(exe)\n .args([\"image\", \"inspect\", image])\n .stdout(Stdio::null())\n .stderr(Stdio::null())\n .status()\n .map(|s| s.success())\n .unwrap_or(false)\n}\n\n/// Run `image inspect` and return the stderr if it fails.\n/// Returns None on success, Some(stderr) on failure.\npub fn image_inspect_stderr(engine: ContainerEngine, image: &str) -> Option {\n let exe = engine_executable(engine);\n let output = Command::new(exe)\n .args([\"image\", \"inspect\", image])\n .stdout(Stdio::null())\n .output()\n .ok()?;\n if output.status.success() {\n None\n } else {\n Some(String::from_utf8_lossy(&output.stderr).to_string())\n }\n}\n\n/// Result of checking whether a remote image exists.\npub enum RemoteImageStatus {\n /// The image exists on the registry.\n Exists,\n /// The registry was reached but the image/tag was not found.\n NotFound,\n /// The check failed for an unknown reason (network, auth, etc).\n /// Contains the stderr output from the container engine.\n Unknown(String),\n}\n\npub fn check_remote_image(engine: ContainerEngine, image: &str) -> RemoteImageStatus {\n let exe = engine_executable(engine);\n let output = Command::new(exe)\n .args([\"manifest\", \"inspect\", image])\n .stdout(Stdio::null())\n .output();\n\n match output {\n Ok(o) if o.status.success() => RemoteImageStatus::Exists,\n Ok(o) => {\n let stderr = String::from_utf8_lossy(&o.stderr).to_string();\n let lower = stderr.to_lowercase();\n // \"manifest unknown\" / \"not found\" / \"name unknown\" indicate\n // the registry was reachable but the image doesn't exist.\n if lower.contains(\"manifest unknown\")\n || lower.contains(\"not found\")\n || lower.contains(\"name unknown\")\n {\n RemoteImageStatus::NotFound\n } else {\n RemoteImageStatus::Unknown(stderr)\n }\n }\n Err(e) => RemoteImageStatus::Unknown(format!(\"Failed to execute {exe}: {e}\")),\n }\n}\n\npub fn container_stop(engine: ContainerEngine, name_or_id: &str) -> (ExitStatus, String) {\n let exe = engine_executable(engine);\n let (code, _, err) = run_process(exe, &[\"stop\".to_string(), name_or_id.to_string()]);\n (code, err)\n}\n\npub fn container_remove(engine: ContainerEngine, name_or_id: &str) -> ExitStatus {\n let exe = engine_executable(engine);\n let (code, _, _) = run_process(exe, &[\"rm\".to_string(), \"-f\".to_string(), name_or_id.to_string()]);\n code\n}\n\n/// Quiet container removal: suppresses stderr (for pre-emptive cleanup).\npub fn container_remove_quiet(engine: ContainerEngine, name_or_id: &str) -> ExitStatus {\n let exe = engine_executable(engine);\n let (code, _, _) = run_process_quiet(exe, &[\"rm\".to_string(), \"-f\".to_string(), name_or_id.to_string()]);\n code\n}\n\n/// Check whether a container with this name exists (running or stopped).\npub fn container_exists(engine: ContainerEngine, name: &str) -> bool {\n let exe = engine_executable(engine);\n Command::new(exe)\n .args([\"container\", \"inspect\", name])\n .stdout(Stdio::null())\n .stderr(Stdio::null())\n .status()\n .map(|s| s.success())\n .unwrap_or(false)\n}\n\npub fn remove_image(engine: ContainerEngine, tag: &str) -> bool {\n let exe = engine_executable(engine);\n let (status, _, _) = run_process(exe, &[\"rmi\".to_string(), tag.to_string()]);\n status.success()\n}\n\n// ======================================================================\n// CLI argument construction\n// ======================================================================\n\npub fn build_run_args(\n engine: ContainerEngine,\n extra_engine_flags: &[String],\n cfg: &RunConfig,\n) -> Vec {\n let mut args = vec![\"run\".to_string()];\n args.extend(extra_engine_flags.iter().cloned());\n\n if cfg.remove_after {\n args.push(\"--rm\".to_string());\n }\n if cfg.read_only {\n args.push(\"--read-only\".to_string());\n // Docker does not auto-mount a tmpfs at /tmp when --read-only is used\n // (podman does). Pool daemons need a writable /tmp for temp files.\n if engine == ContainerEngine::Docker {\n args.push(\"--tmpfs\".to_string());\n args.push(\"/tmp\".to_string());\n }\n }\n // Always attach stdin so piped input works; only allocate a TTY for\n // interactive (shell) sessions.\n args.push(\"-i\".to_string());\n if cfg.interactive {\n args.push(\"-t\".to_string());\n }\n if let Some(ref n) = cfg.name {\n args.push(\"--name\".to_string());\n args.push(n.clone());\n }\n if let Some(ref s) = cfg.shm_size {\n args.push(\"--shm-size\".to_string());\n args.push(s.clone());\n }\n if let Some(ref w) = cfg.work_dir {\n args.push(\"-w\".to_string());\n args.push(w.clone());\n }\n for (host, container) in &cfg.bind_mounts {\n args.push(\"-v\".to_string());\n args.push(format!(\"{host}:{container}{}\", cfg.selinux_suffix));\n }\n for (host_port, container_port) in &cfg.ports {\n args.push(\"-p\".to_string());\n args.push(format!(\"{host_port}:{container_port}\"));\n }\n for (key, val) in &cfg.env {\n args.push(\"-e\".to_string());\n args.push(format!(\"{key}={val}\"));\n }\n args.extend(cfg.extra_flags.iter().cloned());\n args.push(cfg.image.clone());\n if let Some(ref cmd) = cfg.command {\n args.extend(cmd.iter().cloned());\n }\n args\n}\n\npub fn engine_specific_run_flags_io(engine: ContainerEngine) -> Vec {\n let uid = nix::unistd::getuid();\n match engine {\n ContainerEngine::Podman => {\n if uid.is_root() {\n Vec::new()\n } else {\n vec![\"--userns=keep-id\".to_string()]\n }\n }\n ContainerEngine::Docker => {\n if uid.is_root() {\n Vec::new()\n } else {\n let gid = nix::unistd::getgid();\n vec![\"--user\".to_string(), format!(\"{}:{}\", uid, gid)]\n }\n }\n }\n}\n\n/// Pure version for testing.\n#[cfg(test)]\npub fn engine_specific_run_flags(engine: ContainerEngine) -> Vec {\n match engine {\n ContainerEngine::Podman => vec![\"--userns=keep-id\".to_string()],\n ContainerEngine::Docker => Vec::new(),\n }\n}\n\npub fn build_build_args(cfg: &BuildConfig) -> Vec {\n let mut args = vec![\n \"build\".to_string(),\n \"-f\".to_string(),\n cfg.dockerfile.clone(),\n \"-t\".to_string(),\n cfg.tag.clone(),\n ];\n for (key, val) in &cfg.build_args {\n args.push(\"--build-arg\".to_string());\n args.push(format!(\"{key}={val}\"));\n }\n args.push(cfg.context.clone());\n args\n}\n\n// ======================================================================\n// Process execution\n// ======================================================================\n\n/// Run a process with both stdout and stderr redirected to our stderr.\n/// Returns only the exit status. Use for IO () commands where morloc-manager's\n/// stdout must stay clean but the user should see all container output.\nfn run_process_to_stderr(exe: &str, args: &[String]) -> ExitStatus {\n let mut child = Command::new(exe)\n .args(args)\n .stdin(Stdio::null())\n .stdout(Stdio::piped())\n .stderr(Stdio::inherit())\n .spawn()\n .unwrap_or_else(|e| {\n eprintln!(\"Failed to execute {exe}: {e}\");\n std::process::exit(1);\n });\n // Pump child stdout -> our stderr\n if let Some(mut child_stdout) = child.stdout.take() {\n let stderr = io::stderr();\n let _ = io::copy(&mut child_stdout, &mut stderr.lock());\n }\n child.wait().unwrap_or_else(|e| {\n eprintln!(\"Failed to wait for {exe}: {e}\");\n std::process::exit(1);\n })\n}\n\n/// Run a process with stderr streamed live to the terminal.\n/// Returns (exit_status, captured_stdout, \"\").\nfn run_process(exe: &str, args: &[String]) -> (ExitStatus, String, String) {\n let output = Command::new(exe)\n .args(args)\n .stdin(Stdio::null())\n .stdout(Stdio::piped())\n .stderr(Stdio::inherit())\n .output()\n .unwrap_or_else(|e| {\n eprintln!(\"Failed to execute {exe}: {e}\");\n std::process::exit(1);\n });\n (\n output.status,\n String::from_utf8_lossy(&output.stdout).to_string(),\n String::new(),\n )\n}\n\n/// Run a process with all output captured (no streaming).\n/// Used when stderr must be parsed (e.g., for error classification).\nfn run_process_quiet(exe: &str, args: &[String]) -> (ExitStatus, String, String) {\n let output = Command::new(exe)\n .args(args)\n .stdin(Stdio::null())\n .output()\n .unwrap_or_else(|e| {\n eprintln!(\"Failed to execute {exe}: {e}\");\n std::process::exit(1);\n });\n (\n output.status,\n String::from_utf8_lossy(&output.stdout).to_string(),\n String::from_utf8_lossy(&output.stderr).to_string(),\n )\n}\n\n// ======================================================================\n// Helpers\n// ======================================================================\n\npub fn exit_code_to_int(status: ExitStatus) -> i32 {\n status.code().unwrap_or(1)\n}\n" }, { "path": "data/rust/morloc-manager/src/doctor.rs", "content": "use std::fs;\nuse std::path::Path;\nuse std::process::Command;\n\nuse crate::config as cfg;\nuse crate::container::{container_run_quiet, engine_executable, image_exists_locally, RunConfig};\nuse crate::environment;\nuse crate::error::Result;\nuse crate::types::*;\n\nconst MANIFEST_MARKER: &str = \"### MANIFEST ###\";\n\n#[derive(serde::Serialize)]\npub struct CheckResult {\n pub category: String,\n pub result: String,\n pub message: String,\n}\n\n#[derive(serde::Serialize)]\npub struct DoctorSummary {\n pub ok: u32,\n pub warnings: u32,\n pub errors: u32,\n}\n\nstruct Counts {\n ok: u32,\n warn: u32,\n fail: u32,\n json_mode: bool,\n current_category: String,\n checks: Vec,\n}\n\nimpl Counts {\n fn new(json_mode: bool) -> Self {\n Self { ok: 0, warn: 0, fail: 0, json_mode, current_category: String::new(), checks: Vec::new() }\n }\n\n fn set_category(&mut self, cat: &str) {\n self.current_category = cat.to_string();\n }\n\n fn pass(&mut self, msg: &str) {\n self.ok += 1;\n if self.json_mode {\n self.checks.push(CheckResult {\n category: self.current_category.clone(),\n result: \"ok\".to_string(),\n message: msg.to_string(),\n });\n } else {\n println!(\" [ok] {msg}\");\n }\n }\n\n fn warn(&mut self, msg: &str) {\n self.warn += 1;\n if self.json_mode {\n self.checks.push(CheckResult {\n category: self.current_category.clone(),\n result: \"warning\".to_string(),\n message: msg.to_string(),\n });\n } else {\n println!(\" [!!] {msg}\");\n }\n }\n\n fn fail(&mut self, msg: &str) {\n self.fail += 1;\n if self.json_mode {\n self.checks.push(CheckResult {\n category: self.current_category.clone(),\n result: \"error\".to_string(),\n message: msg.to_string(),\n });\n } else {\n println!(\" [EE] {msg}\");\n }\n }\n\n fn skip(&mut self, msg: &str) {\n if self.json_mode {\n self.checks.push(CheckResult {\n category: self.current_category.clone(),\n result: \"skipped\".to_string(),\n message: msg.to_string(),\n });\n } else {\n println!(\" [--] {msg}\");\n }\n }\n}\n\npub fn doctor(\n engine: ContainerEngine,\n verbose: bool,\n env_name: &str,\n scope: Scope,\n ec: &EnvironmentConfig,\n deep: bool,\n strict: bool,\n json_mode: bool,\n) -> Result<()> {\n let scope_str = match scope {\n Scope::Local => \"local\",\n Scope::System => \"system\",\n };\n let engine_str = match engine {\n ContainerEngine::Docker => \"docker\",\n ContainerEngine::Podman => \"podman\",\n };\n\n if !json_mode {\n println!(\"Environment: {env_name} ({scope_str})\");\n println!(\"Engine: {engine_str}\");\n println!();\n }\n\n let mut c = Counts::new(json_mode);\n let data_dir = cfg::env_data_dir(scope, env_name);\n\n // ==== Prerequisites ====\n if !json_mode { println!(\"Prerequisites\"); }\n c.set_category(\"prerequisites\");\n check_engine(&mut c, engine);\n check_base_image(&mut c, engine, &ec.base_image);\n check_built_image(&mut c, engine, ec, scope, env_name);\n check_data_dirs(&mut c, &data_dir);\n check_file_readability(&mut c, &data_dir);\n\n // ==== Manifests ====\n if !json_mode { println!(\"\\nManifests\"); }\n c.set_category(\"manifests\");\n check_manifests(&mut c, &data_dir, ec.morloc_version.as_ref());\n\n // ==== Deep checks ====\n c.set_category(\"deep\");\n if deep {\n if !json_mode { println!(\"\\nDeep checks\"); }\n check_morloc_version(&mut c, engine, ec);\n check_programs_deep(&mut c, engine, verbose, ec, &data_dir);\n } else {\n if !json_mode { println!(\"\\nDeep checks\"); }\n c.skip(\"Use --deep to run container-side checks\");\n }\n\n let fail_count = c.fail;\n let warn_count = c.warn;\n\n if json_mode {\n #[derive(serde::Serialize)]\n struct DoctorOutput {\n environment: String,\n scope: String,\n engine: String,\n checks: Vec,\n summary: DoctorSummary,\n }\n let output = DoctorOutput {\n environment: env_name.to_string(),\n scope: scope_str.to_string(),\n engine: engine_str.to_string(),\n checks: c.checks,\n summary: DoctorSummary { ok: c.ok, warnings: warn_count, errors: fail_count },\n };\n println!(\"{}\", serde_json::to_string_pretty(&output).unwrap());\n } else {\n // ==== Summary ====\n println!();\n println!(\n \"{} passed, {} warnings, {} errors\",\n c.ok, warn_count, fail_count\n );\n }\n\n if fail_count > 0 {\n return Err(crate::error::ManagerError::DoctorFailed(fail_count));\n }\n if strict && warn_count > 0 {\n return Err(crate::error::ManagerError::DoctorFailed(warn_count));\n }\n Ok(())\n}\n\n// ======================================================================\n// Individual checks\n// ======================================================================\n\nfn check_engine(c: &mut Counts, engine: ContainerEngine) {\n let exe = engine_executable(engine);\n let fmt = match engine {\n ContainerEngine::Podman => \"{{.Version.Version}}\",\n ContainerEngine::Docker => \"{{.ServerVersion}}\",\n };\n let output = Command::new(exe)\n .args([\"info\", \"--format\", fmt])\n .output();\n match output {\n Ok(o) if o.status.success() => {\n let ver = String::from_utf8_lossy(&o.stdout).trim().to_string();\n if ver.is_empty() {\n c.pass(&format!(\"{exe} engine reachable\"));\n } else {\n c.pass(&format!(\"{exe} engine reachable ({ver})\"));\n }\n }\n Ok(o) => {\n let err = String::from_utf8_lossy(&o.stderr).trim().to_string();\n if err.contains(\"permission denied\") || err.contains(\"Permission denied\") {\n c.fail(&format!(\"{exe} permission denied -- add user to {exe} group?\"));\n } else {\n c.fail(&format!(\"{exe} not reachable: {err}\"));\n }\n }\n Err(e) => {\n c.fail(&format!(\"{exe} not found: {e}\"));\n }\n }\n}\n\nfn check_base_image(c: &mut Counts, engine: ContainerEngine, base_image: &str) {\n if image_exists_locally(engine, base_image) {\n c.pass(&format!(\"Base image {base_image}\"));\n } else {\n c.fail(&format!(\n \"Base image {base_image} not found locally\\n \\\n Run: morloc-manager run -- morloc --version (triggers pull)\"\n ));\n }\n}\n\nfn check_built_image(c: &mut Counts, engine: ContainerEngine, ec: &EnvironmentConfig, scope: Scope, env_name: &str) {\n if ec.dockerfile.is_none() {\n return;\n }\n // Check if the Dockerfile file itself still exists\n let df_path = cfg::env_dockerfile_path(scope, env_name);\n if !df_path.exists() {\n c.warn(&format!(\n \"Dockerfile configured but file is missing: {}\\n \\\n Remove stale config or recreate the file, then run: morloc-manager update\",\n df_path.display()\n ));\n return;\n }\n match &ec.built_image {\n Some(img) => {\n if image_exists_locally(engine, img) {\n c.pass(&format!(\"Built image {img}\"));\n } else {\n c.fail(&format!(\n \"Built image {img} not found locally\\n \\\n Run: morloc-manager update\"\n ));\n }\n }\n None => {\n c.warn(\"Dockerfile configured but no image built yet\\n \\\n Run: morloc-manager update\");\n }\n }\n}\n\nfn check_data_dirs(c: &mut Counts, data_dir: &Path) {\n let expected = [\"lib\", \"bin\", \"opt\", \"fdb\", \"src/morloc/plane\", \"exe\"];\n\n let mut missing: Vec<&str> = Vec::new();\n for dir in &expected {\n if !data_dir.join(dir).is_dir() {\n missing.push(dir);\n }\n }\n if missing.is_empty() {\n c.pass(\"Data directories intact\");\n } else {\n c.fail(&format!(\n \"Missing directories: {}\\n \\\n Run: morloc-manager run -- morloc init -f\",\n missing.join(\", \")\n ));\n }\n}\n\n/// Walk exe/ and other data subdirectories, warning about files unreadable\n/// by the current user (which would cause freeze to fail).\nfn check_file_readability(c: &mut Counts, data_dir: &Path) {\n let dirs_to_check = [\"exe\", \"bin\", \"lib\"];\n let mut unreadable: Vec = Vec::new();\n for dir in &dirs_to_check {\n let dir_path = data_dir.join(dir);\n if dir_path.is_dir() {\n collect_unreadable(&dir_path, &mut unreadable);\n }\n }\n if unreadable.is_empty() {\n c.pass(\"All data files readable\");\n } else {\n let shown: Vec<&str> = unreadable.iter().take(5).map(|s| s.as_str()).collect();\n let suffix = if unreadable.len() > 5 {\n format!(\" (and {} more)\", unreadable.len() - 5)\n } else {\n String::new()\n };\n c.fail(&format!(\n \"Unreadable files (freeze will fail): {}{suffix}\\n \\\n Fix with: chmod -R a+rX \",\n shown.join(\", \")\n ));\n }\n}\n\nfn collect_unreadable(dir: &Path, out: &mut Vec) {\n let Ok(entries) = fs::read_dir(dir) else {\n out.push(dir.display().to_string());\n return;\n };\n for entry in entries.flatten() {\n let path = entry.path();\n if path.is_dir() {\n collect_unreadable(&path, out);\n } else if fs::File::open(&path).is_err() {\n out.push(path.display().to_string());\n }\n }\n}\n\nfn check_manifests(\n c: &mut Counts,\n data_dir: &Path,\n expected_version: Option<&Version>,\n) {\n let fdb_dir = data_dir.join(\"fdb\");\n if !fdb_dir.is_dir() {\n c.warn(\"No fdb/ directory found\");\n return;\n }\n\n let entries = match fs::read_dir(&fdb_dir) {\n Ok(e) => e,\n Err(e) => {\n c.fail(&format!(\"Cannot read fdb/: {e}\"));\n return;\n }\n };\n\n let mut found_any = false;\n for entry in entries.flatten() {\n let name = entry.file_name();\n let name_str = name.to_string_lossy();\n if !name_str.ends_with(\".manifest\") {\n continue;\n }\n found_any = true;\n let prog_name = &name_str[..name_str.len() - \".manifest\".len()];\n check_one_manifest(c, &entry.path(), prog_name, data_dir, expected_version);\n }\n\n if !found_any {\n c.warn(\"No program manifests found in fdb/\");\n }\n}\n\nfn check_one_manifest(\n c: &mut Counts,\n path: &Path,\n prog_name: &str,\n data_dir: &Path,\n expected_version: Option<&Version>,\n) {\n let content = match fs::read_to_string(path) {\n Ok(s) => s,\n Err(e) => {\n c.fail(&format!(\"{prog_name} -- cannot read manifest: {e}\"));\n return;\n }\n };\n\n let json_str = if content.starts_with(\"#!\") {\n if let Some(marker_pos) = content.find(MANIFEST_MARKER) {\n let after_marker = &content[marker_pos..];\n let json_start = after_marker\n .find('\\n')\n .map(|i| marker_pos + i + 1)\n .unwrap_or(content.len());\n &content[json_start..]\n } else {\n c.fail(&format!(\"{prog_name} -- manifest missing ### MANIFEST ### marker\"));\n return;\n }\n } else {\n content.as_str()\n };\n\n let manifest: serde_json::Value = match serde_json::from_str(json_str) {\n Ok(v) => v,\n Err(e) => {\n c.fail(&format!(\"{prog_name} -- invalid manifest JSON: {e}\"));\n return;\n }\n };\n\n let mut issues: Vec = Vec::new();\n\n // Check morloc_version\n let build_ver = manifest\n .get(\"build\")\n .and_then(|b| b.get(\"morloc_version\"))\n .and_then(|v| v.as_str());\n\n if let (Some(build_ver_str), Some(expected)) = (build_ver, expected_version) {\n let expected_str = expected.show();\n if build_ver_str != expected_str {\n issues.push(format!(\n \"built with {build_ver_str}, expected {expected_str}\"\n ));\n }\n }\n\n // Check build.path exists\n let build_path = manifest\n .get(\"build\")\n .and_then(|b| b.get(\"path\"))\n .and_then(|v| v.as_str());\n\n if let Some(bp) = build_path {\n // Build paths inside containers are /opt/morloc/exe/..., on host they're\n // under data_dir/exe/... Try the host path first.\n let host_path = data_dir.join(\"exe\").join(prog_name);\n if !host_path.is_dir() && !Path::new(bp).is_dir() {\n issues.push(\"build directory missing\".to_string());\n }\n } else {\n issues.push(\"no build.path in manifest\".to_string());\n }\n\n // Check pool files exist\n let pool_count = manifest\n .get(\"pools\")\n .and_then(|p| p.as_array())\n .map(|a| a.len())\n .unwrap_or(0);\n\n if pool_count == 0 {\n issues.push(\"no pools defined\".to_string());\n }\n\n if issues.is_empty() {\n let ver_str = build_ver.unwrap_or(\"unknown\");\n c.pass(&format!(\n \"{prog_name} -- built with {ver_str}, {pool_count} pools\"\n ));\n } else {\n for issue in &issues {\n c.warn(&format!(\n \"{prog_name} -- {issue}\\n \\\n Recompile: morloc-manager run -- morloc make --install\"\n ));\n }\n }\n}\n\nfn check_morloc_version(c: &mut Counts, engine: ContainerEngine, ec: &EnvironmentConfig) {\n let image = ec.active_image();\n match environment::detect_morloc_version(engine, image) {\n Ok(detected) => {\n if let Some(ref expected) = ec.morloc_version {\n if detected == *expected {\n c.pass(&format!(\"morloc {} (matches config)\", detected.show()));\n } else {\n c.warn(&format!(\n \"morloc {} in container, config says {}\",\n detected.show(),\n expected.show()\n ));\n }\n } else {\n c.pass(&format!(\"morloc {} (no version in config to compare)\", detected.show()));\n }\n }\n Err(e) => {\n c.fail(&format!(\"Cannot run morloc in container: {e}\"));\n }\n }\n}\n\nfn check_programs_deep(\n c: &mut Counts,\n engine: ContainerEngine,\n verbose: bool,\n ec: &EnvironmentConfig,\n data_dir: &Path,\n) {\n let image = ec.active_image();\n let mh = \"/opt/morloc\";\n let bind_mounts = vec![(data_dir.to_string_lossy().to_string(), mh.to_string())];\n let env = vec![\n (\"MORLOC_HOME\".to_string(), mh.to_string()),\n ];\n\n // Scan programs from fdb/ to get program names\n let fdb_dir = format!(\"{mh}/fdb\");\n let cfg = RunConfig {\n command: Some(vec![\"ls\".to_string(), fdb_dir.clone()]),\n bind_mounts: bind_mounts.clone(),\n env: env.clone(),\n ..RunConfig::new(image)\n };\n let (status, stdout, _) = container_run_quiet(engine, &cfg);\n if !status.success() {\n c.fail(\"Cannot list programs in container\");\n return;\n }\n\n let programs: Vec = stdout\n .lines()\n .filter(|l| l.ends_with(\".manifest\"))\n .map(|l| {\n let name = l.strip_suffix(\".manifest\").unwrap_or(l);\n ProgramEntry {\n name: name.to_string(),\n commands: Vec::new(),\n }\n })\n .collect();\n\n if programs.is_empty() {\n c.warn(\"No programs found in container\");\n return;\n }\n\n if !c.json_mode {\n println!(\"Running smoke tests for {} programs...\", programs.len());\n }\n for prog in &programs {\n let exe_path = format!(\"{mh}/bin/{}\", prog.name);\n let cfg = RunConfig {\n command: Some(vec![exe_path.clone(), \"--help\".to_string()]),\n bind_mounts: bind_mounts.clone(),\n env: env.clone(),\n ..RunConfig::new(image)\n };\n if verbose {\n let exe = engine_executable(engine);\n eprintln!(\"[morloc-manager] {exe} run --rm {image} {exe_path} --help\");\n }\n let (status, _, stderr) = container_run_quiet(engine, &cfg);\n if status.success() {\n c.pass(&format!(\"{} -- smoke test passed\", prog.name));\n } else {\n let snippet: String = stderr.lines().take(3).collect::>().join(\"\\n \");\n c.fail(&format!(\"{} -- smoke test failed: {snippet}\", prog.name));\n }\n }\n}\n" }, { "path": "data/rust/morloc-manager/src/environment.rs", "content": "use std::fs;\nuse std::path::{Path, PathBuf};\nuse std::process::Command;\n\nuse serde::Serialize;\nuse sha2::{Digest, Sha256};\n\nuse crate::config;\nuse crate::container::{\n self, check_remote_image, container_build_visible,\n container_pull_visible, engine_executable, exit_code_to_int,\n image_exists_locally, BuildConfig, RemoteImageStatus,\n};\nuse crate::error::{ManagerError, Result};\nuse crate::serve;\nuse crate::types::*;\n\n// ======================================================================\n// Public types\n// ======================================================================\n\n/// Options for creating or updating an environment.\n/// For `new` (is_new=true): all Option fields that are None use defaults.\n/// For `update` (is_new=false): None means keep the existing value.\npub struct ApplyOptions {\n pub name: String,\n pub scope: Scope,\n pub is_new: bool,\n pub base_image: Option,\n pub original_image: Option,\n pub morloc_version: Option,\n pub dockerfile: Option,\n pub includes: Vec,\n pub flagfile: Option,\n pub engine_args: Vec,\n pub engine: Option,\n pub shm_size: Option,\n pub skip_dockerfile_build: bool,\n pub verbose: bool,\n}\n\n/// Info returned by list_environments.\n#[derive(Serialize)]\npub struct EnvInfo {\n pub name: String,\n pub morloc_version: Option,\n pub active: bool,\n}\n\n// ======================================================================\n// Image resolution\n// ======================================================================\n\nconst MORLOC_IMAGE_PREFIX: &str = \"ghcr.io/morloc-project/morloc/morloc-full\";\n\n/// Recognize engine errors that mean \"cannot chdir into the current working\n/// directory\" and rewrite them into a clearer message. This commonly happens\n/// when running `sudo -u morloc-manager ...` from a directory\n/// that cannot access (e.g., /root or another user's $HOME).\n/// Without this hint, the error bubbles up as \"Failed to check registry...\"\n/// which misleads users toward debugging network/auth problems.\nfn cwd_access_hint(stderr: &str) -> Option {\n let lower = stderr.to_lowercase();\n let looks_like_cwd_denied = (lower.contains(\"chdir\") || lower.contains(\"getwd\")\n || lower.contains(\"current working directory\"))\n && (lower.contains(\"permission denied\") || lower.contains(\"no such file\"));\n if looks_like_cwd_denied {\n Some(format!(\n \"Cannot change into the current working directory as the target user. \\\n Run morloc-manager from a directory the target user can access \\\n (for example /tmp or the user's home directory).\\nOriginal error: {}\",\n stderr.trim()\n ))\n } else {\n None\n }\n}\n\n/// Resolve a morloc version string to a registry image reference.\npub fn version_to_image(ver: &Version) -> String {\n format!(\"{MORLOC_IMAGE_PREFIX}:{}\", ver.show())\n}\n\n/// Pull an image by tag from the morloc registry, detect its version, and\n/// return (image_ref, version). The tag can be a semver string (\"0.77.0\"),\n/// a named tag (\"edge\", \"nightly\"), or any other valid container tag.\npub fn pull_tagged_image(engine: ContainerEngine, tag: &str) -> Result<(String, Version)> {\n let image_ref = format!(\"{MORLOC_IMAGE_PREFIX}:{tag}\");\n\n if !image_exists_locally(engine, &image_ref) {\n match check_remote_image(engine, &image_ref) {\n RemoteImageStatus::Exists => {}\n RemoteImageStatus::NotFound => {\n return Err(ManagerError::EnvError(format!(\n \"No container image found for tag '{tag}'\"\n )));\n }\n RemoteImageStatus::Unknown(stderr) => {\n if let Some(hint) = cwd_access_hint(&stderr) {\n return Err(ManagerError::EnvError(hint));\n }\n return Err(ManagerError::EnvError(format!(\n \"Failed to check registry for tag '{tag}': {}\",\n stderr.trim()\n )));\n }\n }\n\n eprintln!(\"Pulling {image_ref}...\");\n let status = container_pull_visible(engine, &image_ref);\n if !status.success() {\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(status),\n stderr: \"Pull failed (see output above)\".to_string(),\n });\n }\n } else {\n eprintln!(\"Using local copy of {image_ref}\");\n }\n\n let ver = detect_morloc_version(engine, &image_ref)?;\n\n // Also tag with the detected version so future --version lookups find it\n let versioned_image = version_to_image(&ver);\n if versioned_image != image_ref {\n let exe = engine_executable(engine);\n let _ = Command::new(exe)\n .args([\"tag\", &image_ref, &versioned_image])\n .output();\n }\n\n Ok((versioned_image, ver))\n}\n\n/// Pull the :edge image. Convenience wrapper around pull_tagged_image.\npub fn resolve_latest(engine: ContainerEngine) -> Result<(String, Version)> {\n pull_tagged_image(engine, \"edge\")\n}\n\n/// Pull a specific version image from the morloc registry.\npub fn pull_version_image(engine: ContainerEngine, ver: &Version) -> Result {\n let (img, _) = pull_tagged_image(engine, &ver.show())?;\n Ok(img)\n}\n\n/// Detect the morloc version by running `morloc --version` inside the image.\npub fn detect_morloc_version(engine: ContainerEngine, image: &str) -> Result {\n let exe = engine_executable(engine);\n let output = Command::new(exe)\n .args([\"run\", \"--rm\", image, \"morloc\", \"--version\"])\n .stdin(std::process::Stdio::null())\n .stdout(std::process::Stdio::piped())\n .stderr(std::process::Stdio::piped())\n .output()\n .map_err(|e| ManagerError::EnvError(format!(\"Failed to run container: {e}\")))?;\n\n if !output.status.success() {\n return Err(ManagerError::EnvError(format!(\n \"Image '{image}' does not have a working morloc binary: {}\",\n String::from_utf8_lossy(&output.stderr).trim()\n )));\n }\n\n let ver_out = String::from_utf8_lossy(&output.stdout).trim().to_string();\n let ver_str = ver_out.split_whitespace().last().unwrap_or(&ver_out);\n ver_str.parse().map_err(|_| {\n ManagerError::EnvError(format!(\n \"Could not parse morloc version from image '{image}' output: {ver_out}\"\n ))\n })\n}\n\n/// Pull a custom image (not from morloc registry).\npub fn pull_custom_image(engine: ContainerEngine, image: &str) -> Result<()> {\n if image_exists_locally(engine, image) {\n eprintln!(\"Using local copy of {image}\");\n return Ok(());\n }\n\n eprintln!(\"Pulling {image}...\");\n let status = container_pull_visible(engine, image);\n if !status.success() {\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(status),\n stderr: \"Pull failed (see output above)\".to_string(),\n });\n }\n Ok(())\n}\n\n// ======================================================================\n// Core operations\n// ======================================================================\n\n/// Create or update an environment.\n///\n/// When `is_new` is true: validates name uniqueness, creates data directories.\n/// Validate that an environment name contains only allowed characters.\npub fn validate_env_name(name: &str) -> Result<()> {\n if name.is_empty()\n || !name\n .chars()\n .all(|c| c.is_alphanumeric() || c == '-' || c == '_' || c == '.')\n {\n return Err(ManagerError::EnvError(format!(\n \"Invalid environment name '{name}': must contain only alphanumeric characters, hyphens, underscores, or dots\"\n )));\n }\n Ok(())\n}\n\n/// Parse an include spec into (resolved_source, destination).\n///\n/// Supports two forms:\n/// - `path` — copies to cfg_dir/basename(path)\n/// - `src:dest` — copies src to cfg_dir/dest\n///\n/// Rules for dest:\n/// - Must be relative (no leading `/`)\n/// - Cannot contain `..`\n/// - If dest ends with `/`, src's basename is appended\n///\n/// Source symlinks are resolved via canonicalize().\nfn parse_include_spec(spec: &str, cfg_dir: &Path) -> Result<(PathBuf, PathBuf)> {\n let (src_str, dest_rel) = if let Some(idx) = spec.find(':') {\n let s = &spec[..idx];\n let d = &spec[idx + 1..];\n if s.is_empty() || d.is_empty() {\n return Err(ManagerError::EnvError(format!(\n \"Invalid include spec: '{spec}'\"\n )));\n }\n (s, d.to_string())\n } else {\n let src_path = Path::new(spec);\n let fname = src_path.file_name().ok_or_else(|| {\n ManagerError::EnvError(format!(\"Invalid include path: {spec}\"))\n })?;\n (spec.as_ref(), fname.to_string_lossy().to_string())\n };\n\n // Validate dest constraints\n if dest_rel.starts_with('/') {\n return Err(ManagerError::EnvError(format!(\n \"Include destination must be relative, not absolute: '{dest_rel}'\"\n )));\n }\n if dest_rel.contains(\"..\") {\n return Err(ManagerError::EnvError(format!(\n \"Include destination cannot contain '..': '{dest_rel}'\"\n )));\n }\n\n // Resolve src (canonicalize follows symlinks, errors if path doesn't exist)\n let real_src = Path::new(src_str).canonicalize().map_err(|e| {\n ManagerError::EnvError(format!(\"Cannot resolve include path '{src_str}': {e}\"))\n })?;\n\n // Compute final dest\n let dest = cfg_dir.join(&dest_rel);\n let final_dest = if dest_rel.ends_with('/') {\n dest.join(real_src.file_name().unwrap_or_default())\n } else {\n dest\n };\n\n Ok((real_src, final_dest))\n}\n\n/// When `is_new` is false: loads existing config, applies overrides.\npub fn apply_environment(opts: &ApplyOptions) -> Result<()> {\n let scope = opts.scope;\n let name = &opts.name;\n\n validate_env_name(name)?;\n\n // Load existing config or start fresh\n let mut ec = if opts.is_new {\n let cfg_path = config::env_config_path(scope, name);\n if cfg_path.is_file() {\n return Err(ManagerError::EnvError(format!(\n \"Environment '{name}' already exists\"\n )));\n }\n // Create data directories\n let data_dir = config::env_data_dir(scope, name);\n for sub in &[\"bin\", \"lib\", \"fdb\", \"include\", \"opt\", \"tmp\"] {\n fs::create_dir_all(data_dir.join(sub)).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to create directory: {e}\"))\n })?;\n }\n if scope == Scope::System {\n use std::os::unix::fs::PermissionsExt;\n let dirs: Vec<_> = std::iter::once(data_dir.clone())\n .chain(\n [\"bin\", \"lib\", \"fdb\", \"include\", \"opt\", \"tmp\"]\n .iter()\n .map(|d| data_dir.join(d)),\n )\n .collect();\n for d in dirs {\n let _ = fs::set_permissions(&d, fs::Permissions::from_mode(0o2775));\n }\n }\n // Start with required fields from opts; the rest will be applied below\n EnvironmentConfig {\n name: name.clone(),\n base_image: opts.base_image.clone().unwrap_or_default(),\n original_image: None,\n dockerfile: None,\n content_hash: None,\n built_image: None,\n engine: opts.engine.unwrap_or(ContainerEngine::Podman),\n shm_size: \"512m\".to_string(),\n morloc_version: None,\n }\n } else {\n config::read_env_config(scope, name)\n .map_err(|_| ManagerError::EnvironmentNotFound(name.to_string()))?\n };\n\n // Apply overrides\n if let Some(ref img) = opts.base_image {\n ec.base_image = img.clone();\n }\n if let Some(ref img) = opts.original_image {\n ec.original_image = Some(img.clone());\n }\n if let Some(ref ver) = opts.morloc_version {\n ec.morloc_version = Some(ver.clone());\n }\n if let Some(engine) = opts.engine {\n ec.engine = engine;\n }\n if let Some(ref shm) = opts.shm_size {\n if !is_valid_shm_size(shm) {\n return Err(ManagerError::EnvError(format!(\n \"Invalid --shm-size '{shm}'. Use format like: 512m, 1g, 2048k\"\n )));\n }\n ec.shm_size = shm.clone();\n }\n\n // Copy Dockerfile if a new one was provided\n let dockerfile_changed = if let Some(ref src) = opts.dockerfile {\n let dest = config::env_dockerfile_path(scope, name);\n let dest_dir = dest.parent().unwrap();\n fs::create_dir_all(dest_dir).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to create config dir: {e}\"))\n })?;\n fs::copy(src, &dest).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to copy Dockerfile '{}': {e}\", src))\n })?;\n ec.dockerfile = Some(\"Dockerfile\".to_string());\n true\n } else {\n false\n };\n\n // Copy included files/directories into build context.\n // Supports src:dest syntax (like Docker volume mounts) for explicit placement.\n let cfg_dir = config::env_config_dir(scope, name);\n fs::create_dir_all(&cfg_dir).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to create config dir: {e}\"))\n })?;\n for spec in &opts.includes {\n let (real_src, final_dest) = parse_include_spec(spec, &cfg_dir)?;\n if let Some(parent) = final_dest.parent() {\n fs::create_dir_all(parent).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to create directory: {e}\"))\n })?;\n }\n if real_src.is_dir() {\n let status = Command::new(\"cp\")\n .args([\"-a\", &real_src.to_string_lossy(), &final_dest.to_string_lossy()])\n .stdin(std::process::Stdio::null())\n .stdout(std::process::Stdio::null())\n .stderr(std::process::Stdio::inherit())\n .status()\n .map_err(|e| ManagerError::EnvError(format!(\"Failed to copy '{spec}': {e}\")))?;\n if !status.success() {\n return Err(ManagerError::EnvError(format!(\n \"Failed to copy directory '{spec}'\"\n )));\n }\n } else {\n fs::copy(&real_src, &final_dest).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to copy '{spec}': {e}\"))\n })?;\n }\n }\n\n // Write flags file: for new envs or when flagfile is provided, write fresh.\n // For updates with only engine_args, append to existing.\n let flags_path = config::env_flags_path(scope, name);\n if opts.is_new || opts.flagfile.is_some() {\n let mut flag_lines: Vec = Vec::new();\n if let Some(ref src) = opts.flagfile {\n let content = fs::read_to_string(src).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to read flagfile '{}': {e}\", src))\n })?;\n flag_lines.extend(\n content\n .lines()\n .map(|l| l.trim().to_string())\n .filter(|l| !l.is_empty() && !l.starts_with('#')),\n );\n }\n flag_lines.extend(opts.engine_args.iter().cloned());\n let flags_content = if flag_lines.is_empty() {\n String::new()\n } else {\n flag_lines.join(\"\\n\") + \"\\n\"\n };\n fs::write(&flags_path, &flags_content).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to write flags file: {e}\"))\n })?;\n } else if !opts.engine_args.is_empty() {\n // Append engine_args to existing flags file\n let mut existing = config::read_flags_file_lines(&flags_path);\n existing.extend(opts.engine_args.iter().cloned());\n let flags_content = existing.join(\"\\n\") + \"\\n\";\n fs::write(&flags_path, &flags_content).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to write flags file: {e}\"))\n })?;\n }\n\n // Build Dockerfile layer if present and not skipped\n let has_dockerfile = ec.dockerfile.is_some();\n let should_build = has_dockerfile\n && !opts.skip_dockerfile_build\n && (opts.is_new || dockerfile_changed || !opts.includes.is_empty()\n || opts.base_image.is_some() || opts.engine.is_some()\n // For update with no specific changes, rebuild if Dockerfile exists\n || (!opts.is_new && opts.dockerfile.is_none() && opts.includes.is_empty()));\n\n if should_build {\n let tag = format!(\"localhost/morloc-env:{name}\");\n let df_path = config::env_dockerfile_path(scope, name);\n if df_path.exists() {\n let hash = hash_file(&df_path)?;\n // Skip rebuild when nothing has actually changed: same Dockerfile\n // hash, no new includes, no base-image change, tagged image still\n // present. Without this, `update` with no arguments silently\n // re-runs the full build every time.\n let unchanged = !opts.is_new\n && !dockerfile_changed\n && opts.includes.is_empty()\n && opts.base_image.is_none()\n && ec.content_hash.as_deref() == Some(hash.as_str())\n && ec.built_image.as_ref()\n .map(|img| image_exists_locally(ec.engine, img))\n .unwrap_or(false);\n if unchanged {\n eprintln!(\"Dockerfile unchanged; skipping rebuild.\");\n } else {\n let build_cfg = BuildConfig {\n dockerfile: df_path.to_string_lossy().to_string(),\n context: cfg_dir.to_string_lossy().to_string(),\n tag: tag.clone(),\n build_args: vec![(\"CONTAINER_BASE\".to_string(), ec.base_image.clone())],\n };\n if opts.verbose {\n let exe = engine_executable(ec.engine);\n eprintln!(\n \"[morloc-manager] {exe} build -f {} -t {} {}\",\n build_cfg.dockerfile, build_cfg.tag, build_cfg.context\n );\n }\n let status = container_build_visible(ec.engine, &build_cfg);\n if !status.success() {\n return Err(ManagerError::EngineError {\n engine: ec.engine,\n code: exit_code_to_int(status),\n stderr: \"Build failed (see output above)\".to_string(),\n });\n }\n ec.built_image = Some(tag);\n ec.content_hash = Some(hash);\n }\n }\n }\n\n // Always reconcile the stored morloc version against the actual image.\n // - For `new --version 0.77.0-rc.6`, the binary reports \"0.77.0\" (stack\n // does not expose prerelease tags), so keep the recorded value when\n // major.minor.patch match — the recorded tag is more informative.\n // - For `new --image ` or `update --image ...`, nothing was\n // recorded yet, so store the detected version.\n // - If the image has no morloc binary (e.g., a bare base image staged\n // for a Dockerfile layer not yet built), silently leave the field\n // unchanged rather than failing the whole operation.\n let detect_target = ec.built_image.clone().unwrap_or_else(|| ec.base_image.clone());\n if !detect_target.is_empty() {\n if let Ok(detected) = detect_morloc_version(ec.engine, &detect_target) {\n ec.morloc_version = Some(match ec.morloc_version.take() {\n Some(recorded) if recorded.major == detected.major\n && recorded.minor == detected.minor\n && recorded.patch == detected.patch => recorded,\n _ => detected,\n });\n }\n }\n\n // Write environment config\n config::write_env_config(scope, name, &ec)?;\n\n Ok(())\n}\n\n/// Remove an environment and all its data.\npub fn remove_environment(engine: ContainerEngine, scope: Scope, name: &str) -> Result<()> {\n let ec = config::read_env_config(scope, name)\n .map_err(|_| ManagerError::EnvironmentNotFound(name.to_string()))?;\n\n // Stop and remove any running serve container for this environment before\n // removing its image. If we skipped this, the serve container would keep\n // running and be unreachable through morloc-manager.\n let serve_name = serve::serve_container_name(name);\n if container::container_exists(engine, &serve_name) {\n let _ = container::container_stop(engine, &serve_name);\n let _ = container::container_remove_quiet(engine, &serve_name);\n }\n\n // Remove built Dockerfile layer image\n if let Some(ref img) = ec.built_image {\n if image_exists_locally(engine, img) {\n container::remove_image(engine, img);\n }\n }\n\n // Remove config directory\n let cfg_dir = config::env_config_dir(scope, name);\n if cfg_dir.is_dir() {\n let _ = fs::remove_dir_all(&cfg_dir);\n }\n\n // Remove data directory\n let data_dir = config::env_data_dir(scope, name);\n if data_dir.is_dir() {\n let _ = fs::remove_dir_all(&data_dir);\n }\n\n // If the active env was this one, clear it in both local and system configs\n for cfg_scope in [Scope::Local, Scope::System] {\n let cfg_path = config::config_path(cfg_scope);\n if let Ok(cfg) = config::read_config::(&cfg_path) {\n if cfg.active_env.as_deref() == Some(name) {\n let new_cfg = Config {\n active_env: None,\n ..cfg\n };\n let _ = config::write_config(&cfg_path, &new_cfg);\n }\n }\n }\n\n Ok(())\n}\n\n/// List environments in the given scope.\npub fn list_environments(scope: Scope, active_env: Option<&str>) -> Vec {\n let names = config::list_env_names(scope);\n let mut result = Vec::new();\n for name in names {\n if let Ok(ec) = config::read_env_config(scope, &name) {\n result.push(EnvInfo {\n name: name.clone(),\n morloc_version: ec.morloc_version,\n active: active_env == Some(name.as_str()),\n });\n }\n }\n result\n}\n\n/// Select an environment by writing active_env to the given write_scope config.\npub fn select_environment(name: &str, write_scope: Scope) -> Result<()> {\n // Verify the environment exists somewhere\n config::find_env_scope(name)?;\n\n let cfg_path = config::config_path(write_scope);\n let base_cfg = config::read_config::(&cfg_path)\n .or_else(|_| config::read_config::(&config::config_path(Scope::System)))\n .unwrap_or_default();\n let new_cfg = Config {\n active_env: Some(name.to_string()),\n ..base_cfg\n };\n config::write_config(&cfg_path, &new_cfg)\n}\n\n/// Resolve the active environment. Checks local config first, then system.\n/// Returns (name, scope where env config lives, EnvironmentConfig).\npub fn resolve_active_environment() -> Result<(String, Scope, EnvironmentConfig)> {\n // Find active_env name from config (local first, then system)\n let name = resolve_active_env_name()?;\n\n // Find which scope has the environment config\n let scope = config::find_env_scope(&name)?;\n let ec = config::read_env_config(scope, &name)?;\n Ok((name, scope, ec))\n}\n\n/// Resolve just the active environment name from config.\n/// Skips names that don't resolve to an actual environment (e.g., stale\n/// entries from old config formats).\nfn resolve_active_env_name() -> Result {\n if let Ok(cfg) = config::read_config::(&config::config_path(Scope::Local)) {\n if let Some(ref name) = cfg.active_env {\n if config::find_env_scope(name).is_ok() {\n return Ok(name.clone());\n }\n }\n }\n if let Ok(cfg) = config::read_config::(&config::config_path(Scope::System)) {\n if let Some(ref name) = cfg.active_env {\n if config::find_env_scope(name).is_ok() {\n return Ok(name.clone());\n }\n }\n }\n // Check if any environments exist to give a better suggestion\n let local_envs = config::list_env_names(Scope::Local);\n let system_envs = config::list_env_names(Scope::System);\n if local_envs.is_empty() && system_envs.is_empty() {\n Err(ManagerError::NoActiveEnvironment)\n } else {\n // Label each entry with its scope so same-named envs are distinguishable.\n // System envs are flagged with --system to disambiguate in select.\n let mut available: Vec = local_envs\n .iter()\n .map(|n| format!(\"{n} (local)\"))\n .collect();\n available.extend(system_envs.iter().map(|n| format!(\"{n} (system)\")));\n Err(ManagerError::EnvError(format!(\n \"No active environment. Select one with: morloc-manager select \\n\\\n Available: {}\",\n available.join(\", \")\n )))\n }\n}\n\n// ======================================================================\n// Internal\n// ======================================================================\n\npub fn is_valid_shm_size(s: &str) -> bool {\n if s.is_empty() {\n return false;\n }\n let (digits, suffix) = if s.ends_with(|c: char| \"bkmgBKMG\".contains(c)) {\n (&s[..s.len() - 1], true)\n } else {\n (s, false)\n };\n !digits.is_empty() && digits.chars().all(|c| c.is_ascii_digit()) && (suffix || !digits.is_empty())\n}\n\nfn hash_file(path: &Path) -> Result {\n let contents = fs::read(path).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to read file: {e}\"))\n })?;\n let digest = Sha256::digest(&contents);\n Ok(hex_encode(&digest))\n}\n\nfn hex_encode(bytes: &[u8]) -> String {\n bytes.iter().map(|b| format!(\"{b:02x}\")).collect()\n}\n" }, { "path": "data/rust/morloc-manager/src/error.rs", "content": "use std::fmt;\nuse thiserror::Error;\n\nuse crate::types::{ContainerEngine, Scope, Version};\n\n#[derive(Debug, Error, PartialEq, Eq)]\npub enum ManagerError {\n #[error(\"Configuration not found: {0}\")]\n ConfigNotFound(String),\n\n #[error(\"Permission denied: {0}\")]\n ConfigPermissionDenied(String),\n\n #[error(\"Invalid configuration in {path}: {msg}\")]\n ConfigParseError { path: String, msg: String },\n\n #[error(\"No active environment. Run: morloc-manager new\")]\n NoActiveEnvironment,\n\n #[error(\"Environment not found: {0}\")]\n EnvironmentNotFound(String),\n\n #[error(\"Environment error: {0}\")]\n EnvError(String),\n\n #[error(\"Invalid version: {0}. Expected format: MAJOR.MINOR.PATCH. For named tags like 'edge', use --tag instead.\")]\n InvalidVersion(String),\n\n #[error(\"No command specified. Use --shell or provide a command after --.\")]\n NoCommand,\n\n #[error(\"No container engine found. Install podman or docker.\")]\n EngineNotFound,\n\n #[error(\"Container engine ({engine}) failed with exit code {code}:\\n{stderr}\")]\n EngineError {\n engine: ContainerEngine,\n code: i32,\n stderr: String,\n },\n\n #[error(\"Freeze failed: {0}\")]\n FreezeError(String),\n\n #[error(\"Unfreeze failed: {0}\")]\n UnfreezeError(String),\n\n #[error(\"SELinux error: {0}\")]\n SELinuxError(String),\n\n #[error(\"Doctor found {0} error(s)\")]\n DoctorFailed(u32),\n\n #[error(\"{}\", match .0 {\n Scope::Local => \"No local configuration found. Run: morloc-manager new\",\n Scope::System => \"No system configuration found. Run: sudo morloc-manager new --system\",\n })]\n SetupNotComplete(Scope),\n}\n\nimpl fmt::Display for Version {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(f, \"{}.{}.{}\", self.major, self.minor, self.patch)\n }\n}\n\nimpl fmt::Display for ContainerEngine {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match self {\n ContainerEngine::Docker => write!(f, \"Docker\"),\n ContainerEngine::Podman => write!(f, \"Podman\"),\n }\n }\n}\n\npub type Result = std::result::Result;\n" }, { "path": "data/rust/morloc-manager/src/freeze.rs", "content": "use std::fs;\nuse std::path::Path;\nuse std::process::{Command, Stdio};\n\nuse chrono::Utc;\nuse sha2::{Digest, Sha256};\nuse crate::config;\nuse crate::error::{ManagerError, Result};\nuse crate::types::*;\n\npub fn freeze_from_dir(\n scope: Scope,\n ver: Version,\n engine: ContainerEngine,\n image: &str,\n v_data_dir: &str,\n output_dir: &str,\n verbose: bool,\n) -> Result<()> {\n fs::create_dir_all(output_dir)\n .map_err(|e| ManagerError::FreezeError(format!(\"Failed to create output dir: {e}\")))?;\n\n if !Path::new(v_data_dir).is_dir() {\n return Err(ManagerError::FreezeError(format!(\n \"Data directory does not exist: {v_data_dir}\"\n )));\n }\n\n // Validate programs exist before writing any files\n let modules = scan_modules(&format!(\"{v_data_dir}/fdb\"));\n let programs = scan_programs(&format!(\"{v_data_dir}/fdb\"));\n if programs.is_empty() {\n return Err(ManagerError::FreezeError(\n \"No morloc programs are installed. Compile and install with 'morloc make --install' before freezing.\".to_string()\n ));\n }\n\n // Validate programs work before freezing\n let mh = \"/opt/morloc\";\n let bind_mounts = vec![(v_data_dir.to_string(), mh.to_string())];\n crate::serve::validate_programs(engine, image, &programs, bind_mounts, verbose)?;\n\n eprintln!(\"Freezing installed state from {v_data_dir}...\");\n let tar_path = Path::new(output_dir).join(\"state.tar.gz\");\n let tar_path = tar_path.to_string_lossy();\n let mut tar_dirs: Vec<&str> = Vec::new();\n for dir in &[\"lib\", \"fdb\", \"bin\", \"exe\", \"opt\", \"src\"] {\n if Path::new(&format!(\"{v_data_dir}/{dir}\")).is_dir() {\n tar_dirs.push(dir);\n }\n }\n\n // Pre-flight: verify all files are readable before invoking tar\n for dir in &tar_dirs {\n check_readable_recursive(&Path::new(v_data_dir).join(dir))?;\n }\n\n let tar_status = Command::new(\"tar\")\n .args([\"-czf\", &tar_path, \"-C\", v_data_dir])\n .args(&tar_dirs)\n .stdin(Stdio::null())\n .stdout(Stdio::null())\n .stderr(Stdio::inherit())\n .status()\n .map_err(|e| ManagerError::FreezeError(format!(\"tar failed: {e}\")))?;\n\n if !tar_status.success() {\n return Err(ManagerError::FreezeError(\n \"tar failed (see error output above)\".to_string()\n ));\n }\n eprintln!(\"Created {tar_path}\");\n let now = Utc::now();\n\n // Get base image from the active environment config.\n // Check local config first, then system config for the active env name\n // (mirrors resolve_active_env_name in environment.rs).\n let active_env_name: Option = config::read_active_config()\n .and_then(|c| c.active_env)\n .or_else(|| {\n let sys_path = config::config_path(Scope::System);\n config::read_config::(&sys_path)\n .ok()\n .and_then(|c| c.active_env)\n });\n\n let (base_img, env_layer) = if let Some(ref env_name) = active_env_name {\n let env_scope = config::find_env_scope(env_name).unwrap_or(scope);\n match config::read_env_config(env_scope, env_name) {\n Ok(ec) => {\n let base = ec.base_image.clone();\n // Capture env layer info if there's a Dockerfile\n let layer = if ec.dockerfile.is_some() {\n let df_path = config::env_dockerfile_path(env_scope, env_name);\n if df_path.exists() {\n let df_contents = fs::read_to_string(&df_path).unwrap_or_default();\n let content_hash = ec.content_hash.unwrap_or_default();\n // Use the tagged image reference (not digest) so that\n // unfreeze can resolve it locally without network access.\n // Digest references like localhost/morloc-env@sha256:...\n // cause BuildKit to attempt HTTPS to localhost.\n let image_tag = ec.built_image.clone();\n Some(FrozenEnvLayer {\n name: env_name.to_string(),\n dockerfile: df_contents,\n content_hash,\n image_tag,\n })\n } else {\n None\n }\n } else {\n None\n };\n (base, layer)\n }\n Err(_) => (\"unknown\".to_string(), None),\n }\n } else {\n (\"unknown\".to_string(), None)\n };\n\n let manifest = FreezeManifest {\n morloc_version: ver,\n frozen_at: now,\n modules,\n programs,\n base_image: base_img,\n env_layer,\n env_vars: Vec::new(),\n };\n let manifest_path = Path::new(output_dir).join(\"freeze-manifest.json\");\n let manifest_path = manifest_path.to_string_lossy();\n write_freeze_manifest(&manifest_path, &manifest)?;\n eprintln!(\"Wrote {manifest_path}\");\n eprintln!(\"Frozen state written to {output_dir}\");\n Ok(())\n}\n\npub fn write_freeze_manifest(path: &str, manifest: &FreezeManifest) -> Result<()> {\n let json = serde_json::to_vec(manifest)\n .map_err(|e| ManagerError::FreezeError(format!(\"JSON encode failed: {e}\")))?;\n fs::write(path, json)\n .map_err(|e| ManagerError::FreezeError(format!(\"Write failed: {e}\")))?;\n Ok(())\n}\n\npub fn read_freeze_manifest(path: &str) -> Result {\n let bytes =\n fs::read(path).map_err(|e| ManagerError::FreezeError(format!(\"Read failed: {e}\")))?;\n serde_json::from_slice(&bytes)\n .map_err(|e| ManagerError::FreezeError(format!(\"Invalid manifest: {e}\")))\n}\n\n// ======================================================================\n// Internal: scanning installed state\n// ======================================================================\n\nfn scan_modules(fdb_dir: &str) -> Vec {\n let fdb_path = Path::new(fdb_dir);\n if !fdb_path.is_dir() {\n return Vec::new();\n }\n let Ok(entries) = fs::read_dir(fdb_path) else {\n return Vec::new();\n };\n\n #[derive(serde::Deserialize)]\n struct ModuleStub {\n name: String,\n #[serde(default)]\n version: Option,\n }\n\n entries\n .flatten()\n .filter(|e| {\n e.file_name()\n .to_string_lossy()\n .ends_with(\".module\")\n })\n .filter_map(|e| {\n let bytes = fs::read(e.path()).ok()?;\n let stub: ModuleStub = serde_json::from_slice(&bytes).ok()?;\n let digest = Sha256::digest(&bytes);\n let sha256: String = digest.iter().map(|b| format!(\"{b:02x}\")).collect();\n Some(ModuleEntry {\n name: stub.name,\n version: stub.version,\n sha256,\n })\n })\n .collect()\n}\n\nfn scan_programs(fdb_dir: &str) -> Vec {\n let fdb_path = Path::new(fdb_dir);\n if !fdb_path.is_dir() {\n return Vec::new();\n }\n let Ok(entries) = fs::read_dir(fdb_path) else {\n return Vec::new();\n };\n entries\n .flatten()\n .filter(|e| {\n e.file_name()\n .to_string_lossy()\n .ends_with(\".manifest\")\n })\n .map(|e| {\n let filename = e.file_name().to_string_lossy().to_string();\n let prog_name = filename.strip_suffix(\".manifest\").unwrap_or(&filename);\n let commands = parse_manifest_commands(&e.path());\n ProgramEntry {\n name: prog_name.to_string(),\n commands,\n }\n })\n .collect()\n}\n\nfn parse_manifest_commands(path: &Path) -> Vec {\n let Ok(bytes) = fs::read(path) else {\n return Vec::new();\n };\n #[derive(serde::Deserialize)]\n struct ManifestStub {\n #[serde(default)]\n commands: Vec,\n }\n #[derive(serde::Deserialize)]\n struct ManifestStubCmd {\n name: String,\n }\n match serde_json::from_slice::(&bytes) {\n Ok(stub) => stub.commands.into_iter().map(|c| c.name).collect(),\n Err(_) => Vec::new(),\n }\n}\n\n/// Walk a directory tree and verify every file is readable by the current user.\nfn check_readable_recursive(dir: &Path) -> Result<()> {\n if !dir.is_dir() {\n return Ok(());\n }\n let entries = fs::read_dir(dir).map_err(|e| {\n ManagerError::FreezeError(format!(\"Cannot read directory {}: {e}\", dir.display()))\n })?;\n for entry in entries {\n let entry = entry.map_err(|e| {\n ManagerError::FreezeError(format!(\n \"Cannot read entry in {}: {e}\",\n dir.display()\n ))\n })?;\n let path = entry.path();\n if path.is_dir() {\n check_readable_recursive(&path)?;\n } else if fs::File::open(&path).is_err() {\n return Err(ManagerError::FreezeError(format!(\n \"Unreadable file: {}. Fix permissions or remove before freezing.\",\n path.display()\n )));\n }\n }\n Ok(())\n}\n" }, { "path": "data/rust/morloc-manager/src/main.rs", "content": "mod config;\nmod container;\nmod doctor;\nmod environment;\nmod error;\nmod freeze;\nmod selinux;\nmod serve;\nmod types;\n\nuse std::collections::HashSet;\nuse std::fs;\nuse std::io::{self, IsTerminal, Write};\nuse std::process::{Command, ExitCode, Stdio};\n\nuse clap::builder::styling::Style;\nuse clap::{CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum};\n\nuse crate::config as cfg;\nuse crate::container::{container_run_passthrough, RunConfig};\nuse crate::error::{ManagerError, Result};\nuse crate::selinux::{detect_selinux, volume_suffix, SELinuxMode};\nuse crate::types::*;\n\n// ======================================================================\n// CLI types\n// ======================================================================\n\nfn build_help_template() -> String {\n let b = Style::new().bold().render();\n let bu = Style::new().bold().underline().render();\n let r = \"\\x1b[0m\"; // full ANSI reset\n\n format!(\n \"\\\n{{name}} - {{about}}\n\n{{usage-heading}} {{usage}}\n\n{bu}Development{r}\n {b}setup{r} Configure the default container engine\n {b}new{r} Build a new morloc environment\n {b}run{r} Run a command in the active environment\n {b}rm{r} Remove a morloc environment\n {b}ls{r} List morloc environments\n {b}info{r} Show configuration and installed environments\n {b}select{r} Select an environment\n {b}update{r} Rebuild an environment\n {b}nuke{r} Remove all morloc environments\n\n{bu}Deployment{r}\n {b}start{r} Serve an environment over the network\n {b}stop{r} Stop a running serve container\n {b}logs{r} Stream logs from a running serve container\n {b}freeze{r} Export installed state as a frozen artifact\n {b}unfreeze{r} Build a portable serve image from frozen state\n {b}status{r} List running serve containers\n {b}doctor{r} Check environment health and diagnose issues\n\n{bu}Options{r}\n{{options}}\"\n )\n}\n\n#[derive(Parser)]\n#[command(name = \"morloc-manager\")]\n#[command(about = \"container lifecycle manager for Morloc\")]\n#[command(long_about = \"Manage containerized Morloc installations, dependency layers, and deployments\")]\n#[command(disable_version_flag = true)]\n#[command(arg_required_else_help = true)]\n#[command(hide_possible_values = true)]\nstruct Cli {\n /// Print container commands to stderr before executing\n #[arg(short, long, global = true)]\n verbose: bool,\n\n /// Output machine-readable JSON instead of human-readable text\n #[arg(long, global = true)]\n json: bool,\n\n /// Print version and exit\n #[arg(long)]\n version: bool,\n\n #[command(subcommand)]\n command: Option,\n}\n\n#[derive(Subcommand)]\nenum Cmd {\n // -- Development --\n /// Configure the default container engine\n #[command(display_order = 0)]\n #[command(after_help = \"Examples:\\n morloc-manager setup --engine podman\\n morloc-manager setup --engine docker\\n sudo morloc-manager setup --engine podman --system\")]\n Setup {\n /// Container engine: podman or docker\n #[arg(long, value_enum)]\n engine: Option,\n /// Apply to system scope (requires root)\n #[arg(long)]\n system: bool,\n },\n /// Build a new morloc environment\n #[command(display_order = 1)]\n #[command(after_help = \"Examples:\\n morloc-manager new\\n morloc-manager new myenv --version 0.73.0\\n morloc-manager new myenv --tag edge\\n morloc-manager new myenv --image ubuntu:22.04 --dockerfile ./Dockerfile\\n\\nDefault (when --version, --tag, and --image are all omitted): pulls the\\n:edge tag from the morloc registry and records the resolved version.\\n\\nIn non-interactive mode (no TTY), if no name is given, the latest edge\\nimage is pulled and the environment is named after the detected morloc\\nversion.\")]\n New {\n /// Environment name (default: derived from base image version)\n name: Option,\n /// Base image from Docker Hub or a registry\n #[arg(long)]\n image: Option,\n /// Morloc version (MAJOR.MINOR.PATCH, leading 'v' stripped automatically)\n #[arg(long)]\n version: Option,\n /// Container image tag (e.g., 'edge', 'nightly')\n #[arg(long, conflicts_with_all = [\"version\", \"image\"])]\n tag: Option,\n /// Dockerfile to layer on top of the base image\n #[arg(long)]\n dockerfile: Option,\n /// Generate a stub Dockerfile for customization\n #[arg(long)]\n dockerfile_stub: bool,\n /// Force overwrite of existing Dockerfile stub\n #[arg(long)]\n force: bool,\n /// Include file/dir in build context; use src:dest for explicit placement (repeatable)\n #[arg(short = 'i', long = \"include\")]\n include: Vec,\n /// Path to a file with one engine argument per line\n #[arg(long)]\n flagfile: Option,\n /// A single engine flag (may be repeated)\n #[arg(short = 'x', long = \"engine-arg\", allow_hyphen_values = true)]\n engine_arg: Vec,\n /// Container engine: podman or docker\n #[arg(long, value_enum)]\n engine: Option,\n /// Shared memory size (default: 512m)\n #[arg(long)]\n shm_size: Option,\n /// Create in system scope (requires root)\n #[arg(long)]\n system: bool,\n /// Skip morloc init after creation\n #[arg(long)]\n no_init: bool,\n /// Skip interactive wizard, use defaults for unspecified options\n #[arg(long)]\n non_interactive: bool,\n },\n /// Run a command in the active environment\n #[command(display_order = 2)]\n #[command(after_help = \"\\\nExamples:\n morloc-manager run -- morloc --version\n morloc-manager run -- morloc make -o svc svc.loc\n morloc-manager run -- morloc install math\n morloc-manager run --shell\n\nUse -- to separate morloc-manager flags from the container command.\nWithout --, flags like --version are interpreted by morloc-manager itself.\")]\n Run {\n /// Command to run inside the container\n command: Vec,\n /// Start an interactive shell\n #[arg(long)]\n shell: bool,\n /// Pass environment variable to the container (KEY=VALUE)\n #[arg(short, long = \"env\")]\n env_vars: Vec,\n /// Read environment variables from a file (one KEY=VALUE per line)\n #[arg(long)]\n env_file: Option,\n },\n /// Remove a morloc environment\n #[command(display_order = 3)]\n #[command(after_help = \"Examples:\\n morloc-manager rm myenv\\n sudo morloc-manager rm myenv --system\")]\n Rm {\n /// Environment name(s) to remove\n names: Vec,\n /// Remove from system scope (requires root)\n #[arg(long)]\n system: bool,\n /// Remove even if active (deactivates first)\n #[arg(long)]\n force: bool,\n },\n /// Remove all morloc environments\n #[command(display_order = 8)]\n #[command(after_help = \"Examples:\\n morloc-manager nuke\\n morloc-manager nuke --yes\\n morloc-manager nuke --images\\n sudo morloc-manager nuke --system\\n sudo morloc-manager nuke --system --images --yes\")]\n Nuke {\n /// Remove system-scope environments instead of local (requires root)\n #[arg(long)]\n system: bool,\n /// Also remove base container images\n #[arg(long)]\n images: bool,\n /// Skip confirmation prompt\n #[arg(long)]\n yes: bool,\n },\n /// List morloc environments\n #[command(display_order = 4)]\n #[command(after_help = \"Examples:\\n morloc-manager ls\\n morloc-manager ls --system\")]\n Ls {\n /// Show only system environments\n #[arg(long)]\n system: bool,\n /// Show only local environments\n #[arg(long)]\n local: bool,\n },\n /// Show configuration and installed environments\n #[command(display_order = 5)]\n #[command(after_help = \"Examples:\\n morloc-manager info\\n morloc-manager info myenv\")]\n Info {\n /// Environment name (show details for this environment)\n name: Option,\n /// Look up the system-scope environment (when name is shadowed locally)\n #[arg(long)]\n system: bool,\n },\n /// Select an environment\n #[command(display_order = 6)]\n #[command(after_help = \"Examples:\\n morloc-manager select myenv\\n sudo morloc-manager select myenv --system\")]\n Select {\n /// Environment name\n name: String,\n /// Write to system config instead of local (requires root)\n #[arg(long)]\n system: bool,\n },\n\n /// Rebuild an environment\n #[command(display_order = 7)]\n #[command(after_help = \"Examples:\\n morloc-manager update # rebuild active environment\\n morloc-manager update myenv # rebuild a specific environment\\n morloc-manager update --shm-size 1g\\n morloc-manager update --dockerfile ./new.Dockerfile -i ./data\\n morloc-manager update myenv --reinit # re-run morloc init in myenv\")]\n Update {\n /// Environment name (default: active environment)\n name: Option,\n /// Change the base image\n #[arg(long)]\n image: Option,\n /// Change to a specific morloc version (MAJOR.MINOR.PATCH, leading 'v' stripped)\n #[arg(long)]\n version: Option,\n /// Container image tag (e.g., 'edge', 'nightly')\n #[arg(long, conflicts_with_all = [\"version\", \"image\"])]\n tag: Option,\n /// Replace the Dockerfile\n #[arg(long)]\n dockerfile: Option,\n /// Include file/dir in build context; use src:dest for explicit placement (repeatable)\n #[arg(short = 'i', long = \"include\")]\n include: Vec,\n /// Replace the flags file\n #[arg(long)]\n flagfile: Option,\n /// Add an engine flag (repeatable; appends unless --flagfile replaces)\n #[arg(short = 'x', long = \"engine-arg\", allow_hyphen_values = true)]\n engine_arg: Vec,\n /// Change the container engine\n #[arg(long, value_enum)]\n engine: Option,\n /// Change shared memory size\n #[arg(long)]\n shm_size: Option,\n /// Generate a stub Dockerfile (fails if one already exists)\n #[arg(long)]\n dockerfile_stub: bool,\n /// Force overwrite of existing Dockerfile stub\n #[arg(long)]\n force: bool,\n /// Skip Dockerfile build\n #[arg(long)]\n no_build: bool,\n /// Re-run morloc init\n #[arg(long)]\n reinit: bool,\n /// Accepted for scripting uniformity with `new` (no effect)\n #[arg(long, hide = true)]\n non_interactive: bool,\n },\n\n // -- Deployment --\n /// Serve an environment over the network\n #[command(display_order = 20)]\n #[command(after_help = \"Examples:\\n morloc-manager start # serve active environment\\n morloc-manager start myenv -p 9090:8080\")]\n Start {\n /// Environment name (default: active environment)\n name: Option,\n /// Port mapping HOST:CONTAINER (default: 8080:8080)\n #[arg(short, long, value_parser = parse_port)]\n port: Vec<(u16, u16)>,\n /// Pass environment variable to the container (KEY=VALUE)\n #[arg(short, long = \"env\")]\n env_vars: Vec,\n /// Read environment variables from a file (one KEY=VALUE per line)\n #[arg(long)]\n env_file: Option,\n /// Replace an already-running serve container\n #[arg(long)]\n force: bool,\n },\n /// Stop a running serve container\n #[command(display_order = 21)]\n #[command(after_help = \"Examples:\\n morloc-manager stop # stop active environment\\n morloc-manager stop myenv\")]\n Stop {\n /// Environment name (default: active environment)\n name: Option,\n },\n /// Stream logs from a running serve container\n #[command(display_order = 22)]\n #[command(after_help = \"Examples:\\n morloc-manager logs # logs from only running serve container\\n morloc-manager logs myenv\\n morloc-manager logs -f myenv # follow mode\")]\n Logs {\n /// Environment name (default: auto-detect running container)\n name: Option,\n /// Follow log output\n #[arg(short, long)]\n follow: bool,\n },\n /// Export installed state as a frozen artifact\n #[command(display_order = 23)]\n #[command(after_help = \"Examples:\\n morloc-manager freeze\\n morloc-manager freeze myenv\\n morloc-manager freeze -o ./my-freeze\\n\\nRequires at least one program compiled with 'morloc make --install'.\")]\n Freeze {\n /// Environment name (default: active environment)\n name: Option,\n /// Output directory (default: ./morloc-freeze)\n #[arg(short, long)]\n output: Option,\n /// Overwrite existing output directory\n #[arg(long)]\n force: bool,\n },\n /// Build a serve image from frozen state\n #[command(display_order = 24)]\n #[command(after_help = \"Examples:\\n morloc-manager unfreeze --from ./morloc-freeze/state.tar.gz -t myservice:v1\\n morloc-manager unfreeze --from ./state.tar.gz -t svc:v1 --engine docker\")]\n Unfreeze {\n /// Path to state.tar.gz from freeze\n #[arg(long)]\n from: String,\n /// Image tag\n #[arg(short, long)]\n tag: String,\n /// Base image override\n #[arg(long)]\n base: Option,\n /// Container engine override (default: configured engine).\n /// Images frozen with engine-specific flags may not work with a different engine.\n #[arg(long, value_enum)]\n engine: Option,\n /// Rebuild image even if it already exists locally\n #[arg(long)]\n rebuild: bool,\n },\n /// Evaluate a morloc expression against a running serve container\n #[command(display_order = 25)]\n #[command(after_help = \"Examples:\\n morloc-manager eval 'add 1 2'\\n morloc-manager eval myenv 'map (add 1) [1,2,3]'\\n morloc-manager eval -p 9090 'greet \\\"world\\\"'\")]\n Eval {\n /// Expression to evaluate (or environment name if two positional args)\n first: String,\n /// Expression to evaluate (when first arg is environment name)\n second: Option,\n /// Port of the serve container (default: 8080)\n #[arg(short, long, default_value = \"8080\")]\n port: u16,\n },\n /// List running serve containers\n #[command(display_order = 26)]\n #[command(after_help = \"Examples:\\n morloc-manager status\")]\n Status,\n /// Check environment health and diagnose issues\n #[command(display_order = 26)]\n #[command(after_help = \"Examples:\\n morloc-manager doctor\\n morloc-manager doctor myenv\\n morloc-manager doctor --deep\")]\n Doctor {\n /// Environment name (default: active)\n name: Option,\n /// Check system-scope environment\n #[arg(long)]\n system: bool,\n /// Run checks inside the container (slower, more thorough)\n #[arg(long)]\n deep: bool,\n /// Treat warnings as errors (non-zero exit on warnings)\n #[arg(long)]\n strict: bool,\n },\n}\n\n#[derive(Clone, ValueEnum)]\nenum EngineArg {\n Docker,\n Podman,\n}\n\nimpl From for ContainerEngine {\n fn from(e: EngineArg) -> Self {\n match e {\n EngineArg::Docker => ContainerEngine::Docker,\n EngineArg::Podman => ContainerEngine::Podman,\n }\n }\n}\n\nfn parse_port(s: &str) -> std::result::Result<(u16, u16), String> {\n let parts: Vec<&str> = s.splitn(2, ':').collect();\n if parts.len() != 2 {\n return Err(format!(\"Expected HOST:CONTAINER format, got: {s}\"));\n }\n let host: u16 = parts[0]\n .parse()\n .map_err(|_| format!(\"Invalid host port: {}\", parts[0]))?;\n let container: u16 = parts[1]\n .parse()\n .map_err(|_| format!(\"Invalid container port: {}\", parts[1]))?;\n Ok((host, container))\n}\n\n/// Parse env vars from --env flags and --env-file, returning (key, value) pairs.\nfn collect_env_vars(\n env_flags: &[String],\n env_file: Option<&str>,\n) -> Result> {\n let mut result = Vec::new();\n\n if let Some(path) = env_file {\n let contents = std::fs::read_to_string(path).map_err(|e| {\n ManagerError::EnvError(format!(\"Cannot read env file {path}: {e}\"))\n })?;\n for line in contents.lines() {\n let trimmed = line.trim();\n if trimmed.is_empty() || trimmed.starts_with('#') {\n continue;\n }\n if let Some((k, v)) = trimmed.split_once('=') {\n result.push((k.to_string(), v.to_string()));\n }\n }\n }\n\n for entry in env_flags {\n if let Some((k, v)) = entry.split_once('=') {\n result.push((k.to_string(), v.to_string()));\n } else {\n // Bare key — pass through from host environment\n if let Ok(v) = std::env::var(entry) {\n result.push((entry.clone(), v));\n } else {\n eprintln!(\"Warning: env var '{entry}' not set in host environment, skipping\");\n }\n }\n }\n\n Ok(result)\n}\n\n// ======================================================================\n// Main\n// ======================================================================\n\nfn main() -> ExitCode {\n #[cfg(unix)]\n {\n use nix::sys::signal::{signal, SigHandler, Signal};\n unsafe { let _ = signal(Signal::SIGPIPE, SigHandler::SigDfl); }\n }\n\n let matches = match Cli::command()\n .help_template(build_help_template())\n .try_get_matches()\n {\n Ok(m) => m,\n Err(e) => {\n // Detect missing -- separator for the run subcommand\n let rendered = e.to_string();\n if rendered.contains(\"unrecognized\") || rendered.contains(\"unexpected\") {\n let args: Vec = std::env::args().collect();\n if args.len() > 1 && args[1] == \"run\" {\n let inner: Vec<&str> = args[2..].iter()\n .filter(|a| *a != \"--shell\")\n .map(|a| a.as_str())\n .collect();\n if !inner.is_empty() {\n eprintln!(\"Error: unrecognized arguments for 'run'.\");\n eprintln!();\n eprintln!(\"Use -- to separate morloc-manager flags from the container command:\");\n eprintln!(\" morloc-manager run -- {}\", inner.join(\" \"));\n return ExitCode::from(2);\n }\n }\n }\n e.exit();\n }\n };\n let cli = Cli::from_arg_matches(&matches).unwrap();\n if cli.version {\n println!(\"morloc-manager {}\", env!(\"CARGO_PKG_VERSION\"));\n return ExitCode::SUCCESS;\n }\n let Some(cmd) = cli.command else {\n Cli::command()\n .help_template(build_help_template())\n .print_help()\n .ok();\n return ExitCode::from(2);\n };\n match dispatch(cli.verbose, cli.json, cmd) {\n Ok(()) => ExitCode::SUCCESS,\n Err(err) => {\n if cli.json {\n println!(\"{}\", serde_json::json!({\"error\": format!(\"{err}\")}));\n } else {\n eprintln!(\"{err}\");\n }\n if let ManagerError::EngineError { code, .. } = &err {\n ExitCode::from(*code as u8)\n } else {\n ExitCode::FAILURE\n }\n }\n }\n}\n\nfn resolve_scope(system: bool) -> Scope {\n if system { Scope::System } else { Scope::Local }\n}\n\nfn check_system_write_access() -> Result<()> {\n let sys_dir = cfg::config_dir(Scope::System);\n if sys_dir.exists() {\n let test_path = sys_dir.join(\".write-check\");\n match fs::write(&test_path, b\"\") {\n Ok(_) => { let _ = fs::remove_file(&test_path); Ok(()) }\n Err(_) => Err(ManagerError::ConfigPermissionDenied(format!(\n \"{}. System-scope operations require root. Re-run with sudo\",\n sys_dir.display()\n )))\n }\n } else {\n match fs::create_dir_all(&sys_dir) {\n Ok(_) => Ok(()),\n Err(_) => Err(ManagerError::ConfigPermissionDenied(format!(\n \"{}. System-scope operations require root. Re-run with sudo\",\n sys_dir.display()\n )))\n }\n }\n}\n\n/// Resolve an environment by explicit name or fall back to the active environment.\nfn resolve_env_or_active(name: Option) -> Result<(String, Scope, EnvironmentConfig)> {\n match name {\n Some(n) => {\n let scope = cfg::find_env_scope(&n)?;\n let ec = cfg::read_env_config(scope, &n)?;\n Ok((n, scope, ec))\n }\n None => environment::resolve_active_environment(),\n }\n}\n\nfn ensure_engine() -> Result {\n if let Some(cfg) = cfg::read_active_config() {\n return Ok(cfg.engine);\n }\n Err(ManagerError::SetupNotComplete(Scope::Local))\n}\n\nfn which(name: &str) -> bool {\n Command::new(\"which\")\n .arg(name)\n .stdout(Stdio::null())\n .stderr(Stdio::null())\n .status()\n .map(|s| s.success())\n .unwrap_or(false)\n}\n\nfn display_engine(engine: ContainerEngine) -> &'static str {\n match engine {\n ContainerEngine::Docker => \"docker\",\n ContainerEngine::Podman => \"podman\",\n }\n}\n\nfn bold_green(msg: &str) -> String {\n if io::stderr().is_terminal() {\n format!(\"\\x1b[1;32m{msg}\\x1b[0m\")\n } else {\n msg.to_string()\n }\n}\n\nfn check_docker_socket(engine: ContainerEngine) {\n use std::path::Path;\n if engine != ContainerEngine::Docker {\n return;\n }\n let socket = Path::new(\"/var/run/docker.sock\");\n if !socket.exists() {\n eprintln!(\"Warning: Docker socket not found at /var/run/docker.sock\");\n eprintln!(\" Docker may not be installed or the daemon may not be running.\");\n } else if nix::unistd::access(socket, nix::unistd::AccessFlags::R_OK).is_err() {\n eprintln!(\"Warning: Cannot access Docker socket. You may need to:\");\n eprintln!(\" sudo usermod -aG docker $USER # then log out and back in\");\n }\n}\n\n/// Returns Err with a clear message if Docker is selected but its socket is unreachable.\nfn require_docker_socket(engine: ContainerEngine) -> Result<()> {\n use std::path::Path;\n if engine != ContainerEngine::Docker {\n return Ok(());\n }\n let socket = Path::new(\"/var/run/docker.sock\");\n if !socket.exists() {\n return Err(ManagerError::EnvError(\n \"Docker socket not found at /var/run/docker.sock. Ensure Docker is installed and the daemon is running.\".to_string()\n ));\n }\n if nix::unistd::access(socket, nix::unistd::AccessFlags::R_OK).is_err() {\n return Err(ManagerError::EnvError(\n \"Cannot access Docker socket. Add your user to the docker group:\\n \\\n sudo usermod -aG docker $USER # then log out and back in\".to_string()\n ));\n }\n Ok(())\n}\n\n/// Check if Podman is configured to see rootful images from rootless contexts.\n/// Returns true if additionalimagestore is configured (or not needed).\nfn check_podman_additional_stores(engine: ContainerEngine) -> bool {\n if engine != ContainerEngine::Podman {\n return true;\n }\n // Root doesn't need additional stores — it owns the store\n if nix::unistd::getuid().is_root() {\n return true;\n }\n let rootful_store = std::path::Path::new(\"/var/lib/containers/storage\");\n if !rootful_store.is_dir() {\n // No rootful store exists, nothing to configure\n return true;\n }\n // Check system and user storage.conf for additionalimagestores\n for path in &[\n \"/etc/containers/storage.conf\",\n &format!(\n \"{}/.config/containers/storage.conf\",\n dirs::home_dir()\n .unwrap_or_default()\n .to_string_lossy()\n ),\n ] {\n if let Ok(contents) = fs::read_to_string(path) {\n if contents.contains(\"/var/lib/containers/storage\") {\n return true;\n }\n }\n }\n false\n}\n\nfn warn_podman_additional_stores() {\n eprintln!(\"Warning: Podman is not configured to see system (rootful) images.\");\n eprintln!(\" Non-root users will not be able to run system environments.\");\n eprintln!(\" Option 1 (recommended): Use Docker for system environments.\");\n eprintln!(\" Option 2: Add to [storage.options] in /etc/containers/storage.conf:\");\n eprintln!();\n eprintln!(\" additionalimagestores = [\\\"/var/lib/containers/storage\\\"]\");\n eprintln!();\n eprintln!(\" Note: Option 2 may cause storage locking conflicts on Fedora and Debian.\");\n}\n\n// ======================================================================\n// Dispatch\n// ======================================================================\n\nfn dispatch(verbose: bool, json: bool, cmd: Cmd) -> Result<()> {\n match cmd {\n // ---- setup ----\n Cmd::Setup { engine, system } => {\n // With no --engine, show the current engine settings\n if engine.is_none() {\n let local = cfg::read_config::(&cfg::config_path(Scope::Local)).ok();\n let sys = cfg::read_config::(&cfg::config_path(Scope::System)).ok();\n println!(\"Local engine: {}\",\n local.as_ref().map(|c| display_engine(c.engine)).unwrap_or(\"unset\"));\n println!(\"System engine: {}\",\n sys.as_ref().map(|c| display_engine(c.engine)).unwrap_or(\"unset\"));\n println!();\n println!(\"Set with: morloc-manager setup --engine \");\n return Ok(());\n }\n if system { check_system_write_access()?; }\n let scope = resolve_scope(system);\n let eng: ContainerEngine = engine.unwrap().into();\n check_docker_socket(eng);\n let cfg_path = cfg::config_path(scope);\n let base_cfg = cfg::read_config::(&cfg_path).unwrap_or_default();\n let new_cfg = Config {\n engine: eng,\n ..base_cfg\n };\n cfg::write_config(&cfg_path, &new_cfg)?;\n eprintln!(\"Engine set to: {}\", display_engine(eng));\n Ok(())\n }\n\n // ---- new ----\n Cmd::New {\n name,\n image,\n version,\n tag,\n dockerfile,\n dockerfile_stub,\n force,\n include,\n flagfile,\n engine_arg,\n engine,\n shm_size,\n system,\n no_init,\n non_interactive,\n } => {\n if system { check_system_write_access()?; }\n let scope = resolve_scope(system);\n\n // Resolve engine: explicit flag > config default > auto-detect single > error\n // For --system, prefer system config so the env uses the system engine.\n let resolved_engine = if let Some(e) = engine {\n let eng: ContainerEngine = e.into();\n check_docker_socket(eng);\n eng\n } else if let Some(cfg) = if system {\n // System scope: check system config first, then local\n cfg::read_config::(&cfg::config_path(Scope::System)).ok()\n .or_else(|| cfg::read_active_config())\n } else {\n cfg::read_active_config()\n } {\n cfg.engine\n } else {\n // No config — try auto-detection\n let has_podman = which(\"podman\");\n let has_docker = which(\"docker\");\n match (has_podman, has_docker) {\n (true, false) => ContainerEngine::Podman,\n (false, true) => {\n check_docker_socket(ContainerEngine::Docker);\n ContainerEngine::Docker\n }\n (true, true) => {\n let scope_flag = if system { \" --system\" } else { \"\" };\n return Err(ManagerError::EnvError(format!(\n \"Both podman and docker are installed and no default is set.\\n\\\n Pick one with:\\n \\\n morloc-manager setup --engine podman{scope_flag}\\n \\\n morloc-manager setup --engine docker{scope_flag}\\n\\\n Or pass --engine to this command directly.\"\n )));\n }\n (false, false) => return Err(ManagerError::EngineNotFound),\n }\n };\n\n // Ensure config exists (write default if first run)\n if cfg::read_active_config().is_none() {\n let cfg_path = cfg::config_path(scope);\n let new_cfg = Config {\n active_env: None,\n engine: resolved_engine,\n };\n cfg::write_config(&cfg_path, &new_cfg)?;\n }\n\n let interactive = !non_interactive && io::stdin().is_terminal();\n if !non_interactive && !interactive {\n eprintln!(\"Note: No TTY detected, running in non-interactive mode.\");\n }\n\n // Step 1: Resolve name (ask first so user isn't surprised after a long pull)\n let env_name = if let Some(n) = name {\n if cfg::env_config_path(scope, &n).is_file() {\n return Err(ManagerError::EnvError(format!(\n \"Environment '{n}' already exists\"\n )));\n }\n n\n } else if interactive {\n loop {\n eprint!(\"Environment name: \");\n io::stderr().flush().ok();\n let mut name_input = String::new();\n io::stdin().read_line(&mut name_input).ok();\n let n = name_input.trim().to_string();\n if n.is_empty() {\n eprintln!(\"Name cannot be empty.\");\n continue;\n }\n if cfg::env_config_path(scope, &n).is_file() {\n eprintln!(\"Environment '{n}' already exists. Choose a different name.\");\n continue;\n }\n break n;\n }\n } else {\n // Non-interactive without a name: will be filled in after\n // version resolution below (default to version string)\n String::new()\n };\n\n // Validate name early (before potentially slow image pull)\n if !env_name.is_empty() {\n environment::validate_env_name(&env_name)?;\n }\n\n if version.is_some() && image.is_some() {\n return Err(ManagerError::EnvError(\n \"--version and --image are mutually exclusive\".to_string()\n ));\n }\n\n // Validate cheap-to-check parameters before any I/O\n if let Some(ref shm) = shm_size {\n if !environment::is_valid_shm_size(shm) {\n return Err(ManagerError::EnvError(format!(\n \"Invalid --shm-size '{shm}'. Use format like: 512m, 1g, 2048k\"\n )));\n }\n }\n\n // Step 2: Resolve base image and version\n let (base_image, original_image, morloc_ver) = if let Some(ref ver_str) = version {\n // Strip leading 'v' for convenience (e.g., \"v0.77.0\" -> \"0.77.0\")\n let clean = ver_str.strip_prefix('v').unwrap_or(ver_str);\n let ver: Version = clean.parse().map_err(|_| {\n ManagerError::InvalidVersion(ver_str.clone())\n })?;\n let img = environment::pull_version_image(resolved_engine, &ver)?;\n (img, None, Some(ver))\n } else if let Some(ref t) = tag {\n let (img, ver) = environment::pull_tagged_image(resolved_engine, t)?;\n (img, None, Some(ver))\n } else if let Some(ref img) = image {\n environment::pull_custom_image(resolved_engine, img)?;\n (img.clone(), None, None)\n } else if interactive {\n eprintln!(\"Choose a base image:\");\n eprintln!(\" [1] Latest morloc release (recommended)\");\n eprintln!(\" [2] Specific morloc version\");\n eprintln!(\" [3] Custom image\");\n eprint!(\"Choose [1]: \");\n io::stderr().flush().ok();\n let mut input = String::new();\n io::stdin().read_line(&mut input).ok();\n match input.trim() {\n \"2\" => {\n eprint!(\"Morloc version: \");\n io::stderr().flush().ok();\n let mut ver_input = String::new();\n io::stdin().read_line(&mut ver_input).ok();\n let ver: Version = ver_input.trim().parse().map_err(|_| {\n ManagerError::InvalidVersion(ver_input.trim().to_string())\n })?;\n let img = environment::pull_version_image(resolved_engine, &ver)?;\n (img, None, Some(ver))\n }\n \"3\" => {\n eprint!(\"Image reference: \");\n io::stderr().flush().ok();\n let mut img_input = String::new();\n io::stdin().read_line(&mut img_input).ok();\n let img = img_input.trim().to_string();\n if img.is_empty() {\n return Err(ManagerError::EnvError(\"No image specified\".to_string()));\n }\n environment::pull_custom_image(resolved_engine, &img)?;\n (img, None, None)\n }\n _ => {\n let (img, ver) = environment::resolve_latest(resolved_engine)?;\n (img.clone(), Some(img), Some(ver))\n }\n }\n } else {\n let (img, ver) = environment::resolve_latest(resolved_engine)?;\n (img.clone(), Some(img), Some(ver))\n };\n\n // Fill in name for non-interactive mode if it wasn't provided\n let env_name = if env_name.is_empty() {\n if let Some(ref ver) = morloc_ver {\n let default_name = ver.show();\n if cfg::env_config_path(scope, &default_name).is_file() {\n return Err(ManagerError::EnvError(format!(\n \"Environment '{}' already exists. Specify a different name: morloc-manager new ...\",\n default_name\n )));\n }\n default_name\n } else {\n return Err(ManagerError::EnvError(\n \"Environment name required in non-interactive mode\".to_string(),\n ));\n }\n } else {\n env_name\n };\n\n // Resolve dockerfile: explicit path takes precedence, then stub generation\n let resolved_dockerfile = if dockerfile.is_some() {\n if dockerfile_stub {\n return Err(ManagerError::EnvError(\n \"Cannot use both --dockerfile and --dockerfile-stub\".to_string(),\n ));\n }\n dockerfile\n } else if dockerfile_stub {\n let df_path = cfg::env_dockerfile_path(scope, &env_name);\n if df_path.exists() && !force {\n return Err(ManagerError::EnvError(format!(\n \"Dockerfile already exists: {}\\nUse --force to overwrite.\",\n df_path.display()\n )));\n }\n let stub_dir = cfg::data_dir(scope).join(\"tmp\");\n fs::create_dir_all(&stub_dir).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to create tmp dir: {e}\"))\n })?;\n let stub_path = stub_dir.join(format!(\"{env_name}.Dockerfile\"));\n let stub_content = format!(\n \"# morloc environment: {env_name}\\n\\\n # Edit this file, then rebuild with: morloc-manager update\\n\\\n \\n\\\n # CONTAINER_BASE is replaced at build time with the environment's base image\\n\\\n ARG CONTAINER_BASE=scratch\\n\\\n FROM ${{CONTAINER_BASE}}\\n\\\n \\n\\\n # Example: install system packages\\n\\\n # RUN apt-get update && apt-get install -y jq && rm -rf /var/lib/apt/lists/*\\n\\\n \\n\\\n # Example: install Python packages\\n\\\n # RUN pip install scikit-learn pandas\\n\\\n \\n\\\n # Example: install R packages\\n\\\n # RUN R -e \\\"install.packages('ggplot2', repos='https://cloud.r-project.org')\\\"\\n\"\n );\n fs::write(&stub_path, &stub_content).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to write stub Dockerfile: {e}\"))\n })?;\n Some(stub_path.to_string_lossy().to_string())\n } else {\n None\n };\n\n let opts = environment::ApplyOptions {\n name: env_name.clone(),\n scope,\n is_new: true,\n base_image: Some(base_image),\n original_image,\n morloc_version: morloc_ver,\n dockerfile: resolved_dockerfile,\n includes: include,\n flagfile,\n engine_args: engine_arg,\n engine: Some(resolved_engine),\n shm_size: Some(shm_size.unwrap_or_else(|| \"512m\".to_string())),\n skip_dockerfile_build: dockerfile_stub,\n verbose,\n };\n\n environment::apply_environment(&opts)?;\n\n if dockerfile_stub {\n let df_path = cfg::env_dockerfile_path(scope, &env_name);\n eprintln!(\"Stub Dockerfile: {}\", df_path.display());\n eprintln!(\"Edit it, then run: morloc-manager update {env_name}\");\n }\n\n eprintln!(\"Created environment: {env_name}\");\n\n // Run morloc init, passing the env explicitly (no active env needed)\n if !no_init {\n let ec = cfg::read_env_config(scope, &env_name)?;\n run_morloc_init_for(Some((env_name.clone(), scope, ec)), verbose)?;\n } else {\n eprintln!(\"Warning: --no-init was used. Run 'morloc-manager run -- morloc init -f' before building morloc programs.\");\n }\n\n eprintln!(\"{}\", bold_green(&format!(\"Environment '{env_name}' is ready.\")));\n eprintln!(\"Activate it with: morloc-manager select {env_name}\");\n\n if system && !check_podman_additional_stores(resolved_engine) {\n eprintln!();\n warn_podman_additional_stores();\n }\n\n Ok(())\n }\n\n // ---- run ----\n Cmd::Run { command, shell, env_vars, env_file } => {\n if !shell && command.is_empty() {\n return Err(ManagerError::NoCommand);\n }\n let user_env = collect_env_vars(&env_vars, env_file.as_deref())?;\n run_in_container(verbose, shell, &command, &user_env).map_err(|e| match e {\n ManagerError::EnvironmentNotFound(msg) => ManagerError::EnvironmentNotFound(\n format!(\"{msg}. Run 'morloc-manager new' to create an environment\")\n ),\n other => other,\n })\n }\n\n // ---- rm ----\n Cmd::Rm { names, system, force } => {\n if system { check_system_write_access()?; }\n if names.is_empty() {\n return Err(ManagerError::EnvError(\"No environment names specified\".to_string()));\n }\n // Capture current active env for post-removal feedback\n let was_active = cfg::read_active_config().and_then(|c| c.active_env);\n // Attempt each removal; collect failures, continue past errors\n let mut failures: Vec = Vec::new();\n for name in &names {\n let result: Result<()> = (|| {\n let scope = if system {\n Scope::System\n } else {\n cfg::find_env_scope(name)?\n };\n if scope == Scope::System && !system {\n check_system_write_access()?;\n }\n if !force {\n if let Some(cfg) = cfg::read_active_config() {\n if cfg.active_env.as_deref() == Some(name.as_str()) {\n return Err(ManagerError::EnvError(format!(\n \"active environment (use --force)\"\n )));\n }\n }\n }\n let ec = cfg::read_env_config(scope, name)\n .map_err(|_| ManagerError::EnvironmentNotFound(name.to_string()))?;\n environment::remove_environment(ec.engine, scope, name)?;\n Ok(())\n })();\n match result {\n Ok(()) => {\n // Check if removed env was active and report new state\n if was_active.as_deref() == Some(name.as_str()) {\n match environment::resolve_active_environment() {\n Ok((new_active, _, _)) => {\n // Persist the fallback as the new active environment\n let _ = environment::select_environment(&new_active, Scope::Local);\n eprintln!(\"Removed environment: {name}. Active environment is now: {new_active}\");\n }\n Err(_) => {\n eprintln!(\"Removed environment: {name}. No active environment. Use: morloc-manager select \");\n }\n }\n } else {\n eprintln!(\"Removed environment: {name}\");\n }\n }\n Err(e) => failures.push(format!(\"{name}: {e}\")),\n }\n }\n if !failures.is_empty() {\n eprintln!();\n eprintln!(\"Failed to remove {} environment(s):\", failures.len());\n for f in &failures {\n eprintln!(\" {f}\");\n }\n return Err(ManagerError::EnvError(format!(\n \"{} of {} removals failed\",\n failures.len(),\n names.len()\n )));\n }\n Ok(())\n }\n\n // ---- nuke ----\n Cmd::Nuke { system, images, yes } => {\n let scope = if system { Scope::System } else { Scope::Local };\n let scope_label = if system { \"system\" } else { \"local\" };\n\n if system {\n check_system_write_access()?;\n }\n\n // Confirm before removing all environments\n let env_names = cfg::list_env_names(scope);\n if env_names.is_empty() {\n eprintln!(\"No {scope_label} environments found.\");\n return Ok(());\n }\n\n if !yes {\n eprintln!(\"This will remove {} {scope_label} environment(s):\", env_names.len());\n for n in &env_names {\n eprintln!(\" {n}\");\n }\n if io::stdin().is_terminal() {\n eprint!(\"Continue? [y/N] \");\n io::stderr().flush().ok();\n let mut answer = String::new();\n io::stdin().read_line(&mut answer).ok();\n if !matches!(answer.trim(), \"y\" | \"yes\" | \"Y\" | \"YES\") {\n eprintln!(\"Aborted.\");\n return Ok(());\n }\n } else {\n return Err(ManagerError::EnvError(\n \"nuke requires --yes for non-interactive use\".to_string(),\n ));\n }\n }\n\n eprintln!(\"Removing all {scope_label} morloc environments...\");\n\n // Collect env info before removal (configs are deleted during removal)\n let mut env_list: Vec<(String, ContainerEngine)> = Vec::new();\n let mut base_images: HashSet = HashSet::new();\n\n for name in cfg::list_env_names(scope) {\n if let Ok(ec) = cfg::read_env_config(scope, &name) {\n if images {\n base_images.insert(ec.base_image.clone());\n if let Some(ref orig) = ec.original_image {\n base_images.insert(orig.clone());\n }\n }\n env_list.push((name, ec.engine));\n }\n }\n\n if env_list.is_empty() {\n eprintln!(\"No {scope_label} environments found.\");\n } else {\n let mut removed = 0usize;\n let mut failures: Vec = Vec::new();\n\n for (name, engine) in &env_list {\n eprintln!(\"Removing environment: {name}...\");\n match environment::remove_environment(*engine, scope, name) {\n Ok(()) => {\n eprintln!(\" Removed: {name}\");\n removed += 1;\n }\n Err(e) => {\n eprintln!(\" Failed: {name}: {e}\");\n failures.push(format!(\"{name}: {e}\"));\n }\n }\n }\n\n // Clear active_env in the targeted scope's config\n let cfg_path = cfg::config_path(scope);\n if let Ok(cfg_data) = cfg::read_config::(&cfg_path) {\n if cfg_data.active_env.is_some() {\n let new_cfg = Config { active_env: None, ..cfg_data };\n let _ = cfg::write_config(&cfg_path, &new_cfg);\n eprintln!(\"Cleared active environment.\");\n }\n }\n\n eprintln!(\"Removed {removed} environment(s).\");\n\n if !failures.is_empty() {\n eprintln!();\n eprintln!(\"Failed to remove {} environment(s):\", failures.len());\n for f in &failures {\n eprintln!(\" {f}\");\n }\n return Err(ManagerError::EnvError(format!(\n \"{} of {} removals failed\",\n failures.len(),\n env_list.len()\n )));\n }\n }\n\n // Remove base images if --images\n if images && !base_images.is_empty() {\n let engine = ensure_engine().unwrap_or(ContainerEngine::Docker);\n eprintln!(\"Removing base images...\");\n for img in &base_images {\n if container::image_exists_locally(engine, img) {\n eprintln!(\" Removing image: {img}...\");\n if container::remove_image(engine, img) {\n eprintln!(\" Removed: {img}\");\n } else {\n eprintln!(\" Failed to remove: {img}\");\n }\n }\n }\n }\n\n // Hint about the other scope\n let other_scope = if system { Scope::Local } else { Scope::System };\n let other_envs = cfg::list_env_names(other_scope);\n if !other_envs.is_empty() {\n if system {\n eprintln!(\n \"{} local environment(s) remain. Use: morloc-manager nuke\",\n other_envs.len()\n );\n } else {\n eprintln!(\n \"{} system environment(s) remain. Use: sudo morloc-manager nuke --system\",\n other_envs.len()\n );\n }\n }\n\n Ok(())\n }\n\n // ---- ls ----\n Cmd::Ls { system, local } => {\n let active_env = cfg::read_active_config()\n .and_then(|c| c.active_env);\n let active_str = active_env.as_deref();\n\n // Determine which scope effectively owns the active environment.\n // Local takes priority (same resolution as run/select).\n let active_in_local = active_str\n .map(|name| cfg::env_config_path(Scope::Local, name).is_file())\n .unwrap_or(false);\n\n let show_local = !system || local;\n let show_system = !local || system;\n\n let local_envs = if show_local {\n let local_active = if active_in_local { active_str } else { None };\n environment::list_environments(Scope::Local, local_active)\n } else {\n Vec::new()\n };\n let system_envs = if show_system {\n let system_active = if active_in_local { None } else { active_str };\n environment::list_environments(Scope::System, system_active)\n } else {\n Vec::new()\n };\n\n if json {\n #[derive(serde::Serialize)]\n struct LsOutput {\n local: Vec,\n system: Vec,\n }\n let output = LsOutput { local: local_envs, system: system_envs };\n println!(\"{}\", serde_json::to_string_pretty(&output).unwrap());\n } else {\n let total = local_envs.len() + system_envs.len();\n if !local_envs.is_empty() {\n println!(\"Local environments:\");\n for e in &local_envs {\n let active_mark = if e.active { \" (active)\" } else { \"\" };\n let ver_mark = e.morloc_version.as_ref()\n .map(|v| format!(\" [{}]\", v.show()))\n .unwrap_or_default();\n println!(\" {}{}{}\", e.name, ver_mark, active_mark);\n }\n }\n if !system_envs.is_empty() {\n if !local_envs.is_empty() {\n println!();\n }\n println!(\"System environments:\");\n for e in &system_envs {\n let active_mark = if e.active { \" (active)\" } else { \"\" };\n let ver_mark = e.morloc_version.as_ref()\n .map(|v| format!(\" [{}]\", v.show()))\n .unwrap_or_default();\n println!(\" {}{}{}\", e.name, ver_mark, active_mark);\n }\n }\n if total == 0 {\n println!(\"No environments found. Create one with: morloc-manager new\");\n }\n }\n Ok(())\n }\n\n // ---- info ----\n Cmd::Info { name, system } => {\n if let Some(env_name) = name {\n // Detailed info for a specific environment\n let scope = if system {\n if !cfg::env_config_path(Scope::System, &env_name).is_file() {\n return Err(ManagerError::EnvironmentNotFound(format!(\n \"{env_name} (in system scope)\"\n )));\n }\n Scope::System\n } else {\n cfg::find_env_scope(&env_name)?\n };\n let ec = cfg::read_env_config(scope, &env_name)?;\n let data_dir = cfg::env_data_dir(scope, &env_name);\n let active = cfg::read_active_config()\n .and_then(|c| c.active_env)\n .as_deref() == Some(env_name.as_str());\n\n if json {\n #[derive(serde::Serialize)]\n struct InfoDetail {\n name: String,\n scope: String,\n active: bool,\n base_image: String,\n built_image: Option,\n morloc_version: Option,\n engine: String,\n shm_size: String,\n dockerfile: Option,\n flags: Vec,\n data_dir: String,\n }\n let df_str = ec.dockerfile.as_ref().map(|_| {\n let df_path = cfg::env_dockerfile_path(scope, &env_name);\n df_path.display().to_string()\n });\n let flags_path = cfg::env_flags_path(scope, &env_name);\n let flags = cfg::read_flags_file_lines(&flags_path);\n let output = InfoDetail {\n name: ec.name.clone(),\n scope: match scope { Scope::Local => \"local\", Scope::System => \"system\" }.to_string(),\n active,\n base_image: ec.base_image.clone(),\n built_image: ec.built_image.clone(),\n morloc_version: ec.morloc_version.clone(),\n engine: display_engine(ec.engine).to_string(),\n shm_size: ec.shm_size.clone(),\n dockerfile: df_str,\n flags,\n data_dir: data_dir.display().to_string(),\n };\n println!(\"{}\", serde_json::to_string_pretty(&output).unwrap());\n } else {\n println!(\"Name: {}\", ec.name);\n println!(\"Scope: {}\", match scope { Scope::Local => \"local\", Scope::System => \"system\" });\n println!(\"Active: {}\", if active { \"yes\" } else { \"no\" });\n println!(\"Base image: {}\", ec.base_image);\n if let Some(ref img) = ec.built_image {\n println!(\"Built image: {img}\");\n }\n if let Some(ref ver) = ec.morloc_version {\n println!(\"Morloc version: {}\", ver.show());\n }\n println!(\"Engine: {}\", display_engine(ec.engine));\n println!(\"SHM size: {}\", ec.shm_size);\n println!(\"Dockerfile: {}\", match ec.dockerfile {\n Some(_) => {\n let df_path = cfg::env_dockerfile_path(scope, &env_name);\n if df_path.exists() {\n df_path.display().to_string()\n } else {\n format!(\"{} (MISSING)\", df_path.display())\n }\n }\n None => \"none\".to_string(),\n });\n let flags_path = cfg::env_flags_path(scope, &env_name);\n println!(\"Flags: {}\", flags_path.display());\n let flags = cfg::read_flags_file_lines(&flags_path);\n for flag in &flags {\n println!(\" {flag}\");\n }\n println!(\"Data dir: {}\", data_dir.display());\n }\n } else {\n // Overview\n let local_cfg = cfg::read_config::(&cfg::config_path(Scope::Local)).ok();\n let system_cfg = cfg::read_config::(&cfg::config_path(Scope::System)).ok();\n let se_mode = detect_selinux();\n\n let active_env = environment::resolve_active_environment()\n .map(|(name, _, _)| name)\n .unwrap_or_else(|_| \"none\".to_string());\n\n let se_str = match se_mode {\n SELinuxMode::Enforcing => \"enforcing\",\n SELinuxMode::Permissive => \"permissive\",\n SELinuxMode::Disabled => \"not detected\",\n };\n\n if json {\n #[derive(serde::Serialize)]\n struct DirInfo { path: String, exists: bool }\n #[derive(serde::Serialize)]\n struct InfoOverview {\n active: String,\n local_engine: String,\n system_engine: String,\n selinux: String,\n directories: std::collections::BTreeMap,\n local: Vec,\n system: Vec,\n }\n let active_str = if active_env == \"none\" { None } else { Some(active_env.as_str()) };\n let mut directories = std::collections::BTreeMap::new();\n for (label, path) in [\n (\"config_local\", cfg::config_dir(Scope::Local)),\n (\"data_local\", cfg::data_dir(Scope::Local)),\n (\"config_system\", cfg::config_dir(Scope::System)),\n (\"data_system\", cfg::data_dir(Scope::System)),\n ] {\n directories.insert(label.to_string(), DirInfo {\n path: path.display().to_string(),\n exists: path.is_dir(),\n });\n }\n let output = InfoOverview {\n active: active_env.clone(),\n local_engine: local_cfg.as_ref().map(|c| display_engine(c.engine)).unwrap_or(\"unset\").to_string(),\n system_engine: system_cfg.as_ref().map(|c| display_engine(c.engine)).unwrap_or(\"unset\").to_string(),\n selinux: se_str.to_string(),\n directories,\n local: environment::list_environments(Scope::Local, active_str),\n system: environment::list_environments(Scope::System, active_str),\n };\n println!(\"{}\", serde_json::to_string_pretty(&output).unwrap());\n } else {\n println!(\"Active: {active_env}\");\n println!(\"Local engine: {}\",\n local_cfg.as_ref().map(|c| display_engine(c.engine)).unwrap_or(\"unset\"));\n println!(\"System engine: {}\",\n system_cfg.as_ref().map(|c| display_engine(c.engine)).unwrap_or(\"unset\"));\n println!(\"SELinux: {se_str}\");\n\n let dirs = [\n (\"Config (local)\", cfg::config_dir(Scope::Local)),\n (\"Data (local)\", cfg::data_dir(Scope::Local)),\n (\"Config (system)\", cfg::config_dir(Scope::System)),\n (\"Data (system)\", cfg::data_dir(Scope::System)),\n ];\n println!(\"\\nDirectories:\");\n for (label, path) in &dirs {\n let status = if path.is_dir() { \"exists\" } else { \"not found\" };\n println!(\" {:<20} {} ({})\", label, path.display(), status);\n }\n\n let active_str = if active_env == \"none\" { None } else { Some(active_env.as_str()) };\n\n // Check if active env lives in local scope (local takes priority)\n let active_in_local = active_str\n .map(|name| cfg::env_config_path(Scope::Local, name).is_file())\n .unwrap_or(false);\n\n let local_envs = environment::list_environments(Scope::Local, active_str);\n println!(\"\\nLocal environments:\");\n if local_envs.is_empty() {\n println!(\" (none)\");\n } else {\n for e in &local_envs {\n let active_mark = if e.active { \" (active)\" } else { \"\" };\n let ver_mark = e.morloc_version.as_ref()\n .map(|v| format!(\" [{}]\", v.show()))\n .unwrap_or_default();\n println!(\" {}{}{}\", e.name, ver_mark, active_mark);\n }\n }\n\n let system_envs = environment::list_environments(Scope::System, active_str);\n if !system_envs.is_empty() {\n println!(\"\\nSystem environments:\");\n for e in &system_envs {\n let active_mark = if e.active && active_in_local {\n \" (active - shadowed)\"\n } else if e.active {\n \" (active)\"\n } else {\n \"\"\n };\n let ver_mark = e.morloc_version.as_ref()\n .map(|v| format!(\" [{}]\", v.show()))\n .unwrap_or_default();\n println!(\" {}{}{}\", e.name, ver_mark, active_mark);\n }\n }\n }\n }\n Ok(())\n }\n\n // ---- select ----\n Cmd::Select { name, system } => {\n if system { check_system_write_access()?; }\n let write_scope = resolve_scope(system);\n environment::select_environment(&name, write_scope)?;\n if system {\n eprintln!(\"Set system default environment: {name}\");\n } else {\n eprintln!(\"Selected environment: {name}\");\n }\n Ok(())\n }\n\n // ---- update ----\n Cmd::Update {\n name, image, version, tag, dockerfile, dockerfile_stub, force, include, flagfile,\n engine_arg, engine, shm_size, no_build, reinit, non_interactive: _,\n } => {\n let (env_name, env_scope) = match name {\n Some(n) => {\n let scope = cfg::find_env_scope(&n)?;\n (n, scope)\n }\n None => {\n let (n, s, _) = environment::resolve_active_environment()?;\n (n, s)\n }\n };\n if env_scope == Scope::System {\n check_system_write_access()?;\n }\n\n // Handle --dockerfile-stub: generate stub if no Dockerfile exists\n let resolved_dockerfile = if dockerfile.is_some() && dockerfile_stub {\n return Err(ManagerError::EnvError(\n \"Cannot use both --dockerfile and --dockerfile-stub\".to_string(),\n ));\n } else if dockerfile_stub {\n let df_path = cfg::env_dockerfile_path(env_scope, &env_name);\n if df_path.exists() && !force {\n return Err(ManagerError::EnvError(format!(\n \"Dockerfile already exists: {}\\nUse --force to overwrite.\",\n df_path.display()\n )));\n }\n let stub_dir = cfg::data_dir(env_scope).join(\"tmp\");\n fs::create_dir_all(&stub_dir).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to create tmp dir: {e}\"))\n })?;\n let stub_path = stub_dir.join(format!(\"{env_name}.Dockerfile\"));\n let stub_content = format!(\n \"# morloc environment: {env_name}\\n\\\n # Edit this file, then rebuild with: morloc-manager update\\n\\\n \\n\\\n # CONTAINER_BASE is replaced at build time with the environment's base image\\n\\\n ARG CONTAINER_BASE=scratch\\n\\\n FROM ${{CONTAINER_BASE}}\\n\\\n \\n\\\n # Example: install system packages\\n\\\n # RUN apt-get update && apt-get install -y jq && rm -rf /var/lib/apt/lists/*\\n\\\n \\n\\\n # Example: install Python packages\\n\\\n # RUN pip install scikit-learn pandas\\n\\\n \\n\\\n # Example: install R packages\\n\\\n # RUN R -e \\\"install.packages('ggplot2', repos='https://cloud.r-project.org')\\\"\\n\"\n );\n fs::write(&stub_path, &stub_content).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to write stub Dockerfile: {e}\"))\n })?;\n Some(stub_path.to_string_lossy().to_string())\n } else {\n dockerfile\n };\n\n if version.is_some() && image.is_some() {\n return Err(ManagerError::EnvError(\n \"--version and --image are mutually exclusive\".to_string()\n ));\n }\n\n // Resolve base image if --version, --tag, or --image provided\n let (base_image, original_image, morloc_ver) = if let Some(ref ver_str) = version {\n let ec = cfg::read_env_config(env_scope, &env_name)?;\n let clean = ver_str.strip_prefix('v').unwrap_or(ver_str);\n let ver: Version = clean.parse().map_err(|_| {\n ManagerError::InvalidVersion(ver_str.clone())\n })?;\n let img = environment::pull_version_image(ec.engine, &ver)?;\n (Some(img), None, Some(ver))\n } else if let Some(ref t) = tag {\n let ec = cfg::read_env_config(env_scope, &env_name)?;\n let (img, ver) = environment::pull_tagged_image(ec.engine, t)?;\n (Some(img), None, Some(ver))\n } else if let Some(ref img) = image {\n let ec = cfg::read_env_config(env_scope, &env_name)?;\n environment::pull_custom_image(ec.engine, img)?;\n // Detect version from the new image so it doesn't stay stale\n let detected_ver = environment::detect_morloc_version(ec.engine, img).ok();\n (Some(img.clone()), None, detected_ver)\n } else {\n (None, None, None)\n };\n\n eprintln!(\"Updating environment: {env_name}\");\n let opts = environment::ApplyOptions {\n name: env_name.clone(),\n scope: env_scope,\n is_new: false,\n base_image,\n original_image,\n morloc_version: morloc_ver,\n dockerfile: resolved_dockerfile,\n includes: include,\n flagfile,\n engine_args: engine_arg,\n engine: engine.map(|e| e.into()),\n shm_size,\n skip_dockerfile_build: no_build || dockerfile_stub,\n verbose,\n };\n environment::apply_environment(&opts)?;\n\n if dockerfile_stub {\n let df_path = cfg::env_dockerfile_path(env_scope, &env_name);\n eprintln!(\"Stub Dockerfile: {}\", df_path.display());\n eprintln!(\"Edit it, then run: morloc-manager update {env_name}\");\n }\n\n // --version, --tag, and --image imply --reinit (ABI may have changed)\n if reinit || version.is_some() || tag.is_some() || image.is_some() {\n // Re-read the config (apply_environment may have updated it)\n let ec = cfg::read_env_config(env_scope, &env_name)?;\n\n // Check for running serve container -- reinit replaces morloc-nexus\n // which will fail with \"Text file busy\" if the container has it open.\n let serve_name = serve::serve_container_name(&env_name);\n let running = serve::find_running_serve_containers(ec.engine);\n if running.iter().any(|n| n == &serve_name) {\n return Err(ManagerError::EnvError(format!(\n \"Cannot reinit environment '{env_name}' while its serve container is running.\\n \\\n Run 'morloc-manager stop {env_name}' first.\"\n )));\n }\n\n run_morloc_init_for(Some((env_name.clone(), env_scope, ec)), verbose)?;\n }\n\n eprintln!(\"{}\", bold_green(&format!(\"Environment '{env_name}' updated.\")));\n\n if env_scope == Scope::System && !check_podman_additional_stores(\n cfg::read_env_config(env_scope, &env_name)\n .map(|ec| ec.engine)\n .unwrap_or(ContainerEngine::Podman),\n ) {\n eprintln!();\n warn_podman_additional_stores();\n }\n\n Ok(())\n }\n\n // ---- freeze ----\n Cmd::Freeze { name, output, force } => {\n let output_dir = output.as_deref().unwrap_or(\"./morloc-freeze\");\n // Protect against silently overwriting a previous freeze\n let existing_tar = std::path::Path::new(output_dir).join(\"state.tar.gz\");\n if existing_tar.exists() && !force {\n return Err(ManagerError::FreezeError(format!(\n \"Output directory already contains a freeze: {}\\n \\\n Use --force to overwrite, or specify a different -o path.\",\n existing_tar.display()\n )));\n }\n let (env_name, env_scope, ec) = resolve_env_or_active(name)?;\n let engine = ec.engine;\n // Detect the version from the container binary for sanity check.\n // The morloc binary can't report prerelease tags (stack limitation),\n // so if major.minor.patch match, keep the recorded version which has\n // the full tag from the image.\n eprintln!(\"Detecting morloc version from image...\");\n let detected = environment::detect_morloc_version(ec.engine, ec.active_image())?;\n let ver = if let Some(ref recorded) = ec.morloc_version {\n if recorded.major == detected.major\n && recorded.minor == detected.minor\n && recorded.patch == detected.patch\n {\n recorded.clone()\n } else {\n eprintln!(\n \"Warning: recorded morloc version ({}) does not match image ({}).\",\n recorded.show(), detected.show()\n );\n detected\n }\n } else {\n detected\n };\n let data_dir = cfg::env_data_dir(env_scope, &env_name);\n let image = ec.active_image().to_string();\n let result = freeze::freeze_from_dir(env_scope, ver.clone(), engine, &image, &data_dir.to_string_lossy(), output_dir, verbose);\n if result.is_ok() && ec.morloc_version.as_ref() != Some(&ver) {\n let mut updated = ec.clone();\n updated.morloc_version = Some(ver);\n let _ = cfg::write_env_config(env_scope, &env_name, &updated);\n }\n result\n }\n\n // ---- unfreeze ----\n Cmd::Unfreeze { from, tag, base, engine: engine_override, rebuild } => {\n let from = {\n let p = std::path::Path::new(&from);\n if p.is_dir() {\n let tar = p.join(\"state.tar.gz\");\n if tar.is_file() {\n tar.to_string_lossy().to_string()\n } else {\n return Err(ManagerError::UnfreezeError(format!(\n \"Directory '{}' does not contain state.tar.gz. \\\n Pass the path to state.tar.gz directly, or the directory containing it.\",\n from\n )));\n }\n } else if p.is_file() {\n from\n } else {\n return Err(ManagerError::UnfreezeError(format!(\n \"Input not found: {from}. \\\n Pass the path to state.tar.gz or the directory containing it.\"\n )));\n }\n };\n // Read version and engine from the freeze manifest so unfreeze\n // works on deployment machines with no morloc environments.\n let tarball_dir = std::path::Path::new(&from)\n .parent()\n .unwrap_or(std::path::Path::new(\".\"));\n let manifest_path = tarball_dir.join(\"freeze-manifest.json\");\n let manifest = freeze::read_freeze_manifest(&manifest_path.to_string_lossy())\n .map_err(|_| ManagerError::UnfreezeError(format!(\n \"Cannot read freeze manifest at {}. Ensure state.tar.gz and freeze-manifest.json are in the same directory.\",\n manifest_path.display()\n )))?;\n let engine = match engine_override {\n Some(EngineArg::Docker) => ContainerEngine::Docker,\n Some(EngineArg::Podman) => ContainerEngine::Podman,\n None => {\n let e = ensure_engine()?;\n eprintln!(\n \"Note: using {} engine from global config. Override with --engine if needed.\",\n display_engine(e)\n );\n e\n }\n };\n serve::build_serve_image(engine, verbose, &from, &tag, manifest.morloc_version, base.as_deref(), rebuild, &manifest.programs)\n }\n\n // ---- start ----\n Cmd::Start { name, port, env_vars, env_file, force } => {\n let (env_name, env_scope, ec) = resolve_env_or_active(name)?;\n let image = ec.active_image().to_string();\n let data_dir = cfg::env_data_dir(env_scope, &env_name);\n let container_name = serve::serve_container_name(&env_name);\n // Warn if a Dockerfile is configured but the layered image hasn't been built\n if ec.dockerfile.is_some() && ec.built_image.is_none() {\n eprintln!(\"Warning: Dockerfile is configured but image has not been built. Using base image.\");\n eprintln!(\" Run 'morloc-manager update {env_name}' to build the Dockerfile layer.\");\n }\n // Refuse to replace a running container unless --force is passed\n if container::container_exists(ec.engine, &container_name) {\n if !force {\n return Err(ManagerError::EnvError(format!(\n \"Serve container already running for '{env_name}'. Use --force to replace.\"\n )));\n }\n eprintln!(\"Warning: replacing existing serve container '{container_name}'\");\n }\n let port_mappings = if port.is_empty() {\n vec![(8080, 8080)]\n } else {\n port\n };\n let flags_path = cfg::env_flags_path(env_scope, &env_name);\n let extra_flags = cfg::read_flags_file(&flags_path);\n let user_env = collect_env_vars(&env_vars, env_file.as_deref())?;\n serve::serve_environment(\n ec.engine, verbose, &image,\n &data_dir.to_string_lossy(), &container_name,\n &port_mappings, &extra_flags, &Some(ec.shm_size.clone()),\n &user_env,\n )\n }\n\n // ---- stop ----\n Cmd::Stop { name } => {\n let (env_name, _, ec) = resolve_env_or_active(name)?;\n let container_name = serve::serve_container_name(&env_name);\n if crate::container::container_exists(ec.engine, &container_name) {\n serve::stop_serve_container(ec.engine, verbose, &container_name)?;\n eprintln!(\"Stopped serving environment: {env_name}\");\n } else {\n return Err(ManagerError::EnvError(\n format!(\"No serve container running for environment '{env_name}'\")\n ));\n }\n Ok(())\n }\n\n // ---- logs ----\n Cmd::Logs { name, follow } => {\n let (container_name, engine) = if let Some(ref n) = name {\n let (_, _, ec) = resolve_env_or_active(Some(n.clone()))?;\n let cname = serve::serve_container_name(n);\n if !container::container_exists(ec.engine, &cname) {\n return Err(ManagerError::EnvError(\n format!(\"No serve container running for environment '{n}'\")\n ));\n }\n (cname, ec.engine)\n } else {\n find_running_serve_container()?\n };\n let exe = match engine {\n ContainerEngine::Podman => \"podman\",\n ContainerEngine::Docker => \"docker\",\n };\n let mut cmd_args = vec![\"logs\"];\n if follow {\n cmd_args.push(\"-f\");\n }\n cmd_args.push(&container_name);\n // Log content is the primary data of this command, so both the\n // container's original stdout and stderr should go to our stdout.\n // docker/podman logs preserves the original stream split; we merge\n // them so that `morloc-manager logs | grep ERROR` works.\n let stdout_handle = std::io::stdout();\n let status = std::process::Command::new(exe)\n .args(&cmd_args)\n .stdin(Stdio::null())\n .stdout(Stdio::inherit())\n .stderr(Stdio::from(stdout_handle))\n .status()\n .map_err(|e| ManagerError::EnvError(format!(\"Failed to run {exe} logs: {e}\")))?;\n if !status.success() {\n return Err(ManagerError::EngineError {\n engine,\n code: status.code().unwrap_or(1),\n stderr: String::new(),\n });\n }\n Ok(())\n }\n\n // ---- eval ----\n Cmd::Eval { first, second, port } => {\n let expr = if let Some(ref expr_arg) = second {\n // first is env name — validate it exists and its serve container is running\n let (env_name, _, ec) = resolve_env_or_active(Some(first))?;\n let container_name = serve::serve_container_name(&env_name);\n if !container::container_exists(ec.engine, &container_name) {\n return Err(ManagerError::EnvError(format!(\n \"No serve container running for '{env_name}'. Start with: morloc-manager start {env_name}\"\n )));\n }\n expr_arg.clone()\n } else {\n first\n };\n use std::io::{Read as IoRead, Write as IoWrite};\n let body = format!(\"{{\\\"expr\\\":{}}}\", serde_json::to_string(&expr).unwrap_or_default());\n let request = format!(\n \"POST /eval HTTP/1.1\\r\\nHost: localhost\\r\\nContent-Type: application/json\\r\\nContent-Length: {}\\r\\nConnection: close\\r\\n\\r\\n{}\",\n body.len(), body\n );\n let addr = format!(\"127.0.0.1:{port}\");\n let mut stream = std::net::TcpStream::connect(&addr).map_err(|e| {\n ManagerError::EnvError(format!(\n \"Cannot connect to serve container on {addr}: {e}\\n Is a serve container running? Start with: morloc-manager start\"\n ))\n })?;\n stream.write_all(request.as_bytes()).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to send request: {e}\"))\n })?;\n let mut response = String::new();\n stream.read_to_string(&mut response).map_err(|e| {\n ManagerError::EnvError(format!(\"Failed to read response: {e}\"))\n })?;\n // Extract body from HTTP response (after \\r\\n\\r\\n)\n if let Some(pos) = response.find(\"\\r\\n\\r\\n\") {\n let body = &response[pos + 4..];\n println!(\"{body}\");\n } else {\n println!(\"{response}\");\n }\n Ok(())\n }\n\n // ---- status ----\n Cmd::Status => {\n let mut all_containers: Vec = Vec::new();\n let mut any_engine = false;\n for engine in [ContainerEngine::Podman, ContainerEngine::Docker] {\n let exe = match engine {\n ContainerEngine::Podman => \"podman\",\n ContainerEngine::Docker => \"docker\",\n };\n if which(exe) {\n any_engine = true;\n if let Ok(containers) = serve::query_serve_containers(engine, verbose) {\n all_containers.extend(containers);\n }\n }\n }\n if !any_engine {\n return Err(ManagerError::EngineNotFound);\n }\n if json {\n #[derive(serde::Serialize)]\n struct StatusOutput { containers: Vec }\n let output = StatusOutput { containers: all_containers };\n println!(\"{}\", serde_json::to_string_pretty(&output).unwrap());\n } else if all_containers.is_empty() {\n println!(\"No morloc serve containers running.\");\n } else {\n println!(\"Running servers:\");\n for c in &all_containers {\n println!(\" {} {} ({}) [{}]\", c.name, c.ports, c.env, c.status);\n }\n }\n Ok(())\n }\n\n // ---- doctor ----\n Cmd::Doctor { name, system, deep, strict } => {\n let (env_name, env_scope, ec) = if let Some(ref n) = name {\n let s = if system { Scope::System } else { cfg::find_env_scope(n)? };\n let c = cfg::read_env_config(s, n)?;\n (n.clone(), s, c)\n } else {\n resolve_env_or_active(None)?\n };\n doctor::doctor(ec.engine, verbose, &env_name, env_scope, &ec, deep, strict, json)\n }\n\n }\n}\n\n// ======================================================================\n// Serve container discovery\n// ======================================================================\n\n/// Find exactly one running morloc-serve-* container across all engines.\n/// Returns (container_name, engine). Errors if zero or multiple found.\nfn find_running_serve_container() -> Result<(String, ContainerEngine)> {\n let mut found: Vec<(String, ContainerEngine)> = Vec::new();\n for engine in [ContainerEngine::Podman, ContainerEngine::Docker] {\n let exe = match engine {\n ContainerEngine::Podman => \"podman\",\n ContainerEngine::Docker => \"docker\",\n };\n if which(exe) {\n for name in serve::find_running_serve_containers(engine) {\n found.push((name, engine));\n }\n }\n }\n match found.len() {\n 0 => Err(ManagerError::EnvError(\n \"No morloc serve containers running\".to_string(),\n )),\n 1 => Ok(found.into_iter().next().unwrap()),\n _ => {\n let names: Vec = found.iter().map(|(n, _)| n.clone()).collect();\n Err(ManagerError::EnvError(format!(\n \"Multiple serve containers running. Specify one explicitly:\\n {}\",\n names.join(\"\\n \")\n )))\n }\n }\n}\n\n// ======================================================================\n// Container run\n// ======================================================================\n\nfn run_in_container(\n verbose: bool,\n shell: bool,\n args: &[String],\n user_env: &[(String, String)],\n) -> Result<()> {\n run_in_container_for(None, verbose, shell, args, user_env)\n}\n\nfn run_in_container_for(\n target: Option<(String, Scope, EnvironmentConfig)>,\n verbose: bool,\n shell: bool,\n args: &[String],\n user_env: &[(String, String)],\n) -> Result<()> {\n let (env_name, env_scope, ec) = match target {\n Some(t) => t,\n None => environment::resolve_active_environment()?,\n };\n let engine = ec.engine;\n let image = ec.active_image().to_string();\n let data_dir = cfg::env_data_dir(env_scope, &env_name);\n let v_data_dir = data_dir.to_string_lossy().to_string();\n\n // Warn if a Dockerfile is configured but the layered image hasn't been built\n if ec.dockerfile.is_some() && ec.built_image.is_none() {\n eprintln!(\"Warning: Dockerfile is configured but image has not been built. Using base image.\");\n eprintln!(\" Run 'morloc-manager update {env_name}' to build the Dockerfile layer.\");\n }\n\n // Fail fast with a clear message if docker socket is unreachable\n require_docker_socket(engine)?;\n\n // Verify the image is accessible before attempting to run\n if !container::image_exists_locally(engine, &image) {\n // Show the raw container engine error before our hint\n if let Some(raw_err) = container::image_inspect_stderr(engine, &image) {\n let trimmed = raw_err.trim();\n if !trimmed.is_empty() {\n eprintln!(\"{trimmed}\");\n }\n }\n if env_scope == Scope::System && !check_podman_additional_stores(engine) {\n return Err(ManagerError::EnvError(format!(\n \"Image '{image}' not found. The environment '{env_name}' is a system environment \\\n but Podman is not configured to see rootful images.\\n\\\n Option 1 (recommended): Use Docker for system environments.\\n\\\n Option 2: Add to [storage.options] in /etc/containers/storage.conf:\\n\\n \\\n additionalimagestores = [\\\"/var/lib/containers/storage\\\"]\\n\\n\\\n Note: Option 2 may cause storage locking conflicts on Fedora and Debian.\\n\"\n )));\n }\n let hint = if env_scope == Scope::System {\n format!(\"Ask your administrator to run: sudo morloc-manager update {env_name}\")\n } else {\n format!(\"Run 'morloc-manager update {env_name}' to build it.\")\n };\n return Err(ManagerError::EnvError(format!(\n \"Image '{image}' not found locally. {hint}\"\n )));\n }\n\n let se_mode = detect_selinux();\n let suffix = volume_suffix(se_mode);\n let home = dirs::home_dir()\n .unwrap_or_default()\n .to_string_lossy()\n .to_string();\n let cwd = std::env::current_dir()\n .unwrap_or_default()\n .to_string_lossy()\n .to_string();\n\n // Refuse to run from the root directory — container engines cannot\n // bind-mount \"/\" and the resulting error is opaque.\n if !shell && cwd == \"/\" {\n return Err(ManagerError::EnvError(\n \"Cannot run from the root directory (/). \\\n Change to a subdirectory first (e.g., cd /tmp).\".to_string()\n ));\n }\n\n // Read flags from the environment's flags file\n let flags_path = cfg::env_flags_path(env_scope, &env_name);\n let extra_flags = cfg::read_flags_file(&flags_path);\n\n let is_init = matches!(args, [a, b, ..] if a == \"morloc\" && b == \"init\");\n let is_home_dir = normalize_trailing(&cwd) == normalize_trailing(&home);\n\n if !is_init && !suffix.is_empty() && !is_home_dir {\n selinux::validate_mount_path(&cwd)?;\n run_with_config(\n engine, verbose, &image, &v_data_dir, &home, &cwd, suffix,\n shell, args, false, &ec.shm_size, &extra_flags, user_env,\n )\n } else {\n let (cwd_final, skip_work_mount) = if is_home_dir && !suffix.is_empty() && !is_init {\n eprintln!(\"Warning: running from home directory with SELinux; working directory mount skipped.\");\n eprintln!(\"Workaround: create a project subdirectory and work from there:\");\n eprintln!(\" mkdir ~/myproject && cd ~/myproject\");\n (home.clone(), true)\n } else {\n (cwd, false)\n };\n run_with_config(\n engine, verbose, &image, &v_data_dir, &home, &cwd_final, suffix,\n shell, args, is_init || skip_work_mount, &ec.shm_size, &extra_flags, user_env,\n )\n }\n}\n\nfn run_with_config(\n engine: ContainerEngine,\n verbose: bool,\n image: &str,\n v_data_dir: &str,\n home: &str,\n cwd: &str,\n suffix: &str,\n shell: bool,\n args: &[String],\n is_init: bool,\n shm_size: &str,\n extra_flags: &[String],\n user_env: &[(String, String)],\n) -> Result<()> {\n if shell {\n if !io::stdin().is_terminal() || !io::stdout().is_terminal() {\n eprintln!(\"Error: --shell requires an interactive terminal (TTY).\");\n eprintln!(\"If connecting over SSH, use: ssh -t morloc-manager run --shell\");\n std::process::exit(1);\n }\n }\n\n // Mount data at /opt/morloc — matching the serve container (start).\n // The compiler reads MORLOC_HOME to resolve all generated paths.\n let mh = \"/opt/morloc\";\n let base_mounts = vec![\n (v_data_dir.to_string(), mh.to_string()),\n ];\n let work_mount = if is_init {\n Vec::new()\n } else {\n vec![(cwd.to_string(), cwd.to_string())]\n };\n let all_mounts: Vec<(String, String)> = base_mounts.into_iter().chain(work_mount).collect();\n let work_dir = if is_init {\n mh.to_string()\n } else {\n cwd.to_string()\n };\n let mut env_vars = vec![\n (\"HOME\".to_string(), home.to_string()),\n (\"MORLOC_HOME\".to_string(), mh.to_string()),\n (\n \"PATH\".to_string(),\n format!(\"{mh}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"),\n ),\n ];\n env_vars.extend(user_env.iter().cloned());\n let cmd = if shell {\n Some(vec![\"/bin/bash\".to_string()])\n } else if args.is_empty() {\n None\n } else {\n Some(args.to_vec())\n };\n\n let cfg = RunConfig {\n image: image.to_string(),\n bind_mounts: all_mounts,\n env: env_vars,\n interactive: shell,\n shm_size: Some(shm_size.to_string()),\n work_dir: Some(work_dir),\n selinux_suffix: suffix.to_string(),\n command: cmd,\n extra_flags: extra_flags.to_vec(),\n ..RunConfig::new(image)\n };\n\n let status = container_run_passthrough(engine, verbose, shell, &cfg);\n let code = status.code().unwrap_or(1);\n if status.success() {\n Ok(())\n } else if code >= 125 {\n // Exit 125+ = container engine error (not the user's program)\n Err(ManagerError::EngineError {\n engine,\n code,\n stderr: \"Container engine error\".to_string(),\n })\n } else {\n // Exit 1-124 = program exited with non-zero, pass through silently\n std::process::exit(code);\n }\n}\n\nfn run_morloc_init_for(\n target: Option<(String, Scope, EnvironmentConfig)>,\n verbose: bool,\n) -> Result<()> {\n let init_args: Vec = if verbose {\n [\"morloc\", \"init\", \"-f\"].iter().map(|s| s.to_string()).collect()\n } else {\n [\"morloc\", \"init\", \"-f\", \"-q\"].iter().map(|s| s.to_string()).collect()\n };\n eprintln!(\"Initializing morloc (this may take several minutes)...\");\n run_in_container_for(target, verbose, false, &init_args, &[])\n}\n\nfn normalize_trailing(p: &str) -> String {\n let mut s = p.to_string();\n if !s.ends_with('/') {\n s.push('/');\n }\n s\n}\n\n// ======================================================================\n// Tests\n// ======================================================================\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use crate::container::{build_build_args, build_run_args, engine_executable, engine_specific_run_flags, BuildConfig};\n\n // ---- Type tests ----\n\n #[test]\n fn show_version_formats_correctly() {\n assert_eq!(Version::new(0, 67, 0).show(), \"0.67.0\");\n }\n\n #[test]\n fn parse_version_round_trips() {\n assert_eq!(\"0.67.0\".parse::().ok(), Some(Version::new(0, 67, 0)));\n }\n\n #[test]\n fn parse_version_rejects_invalid() {\n assert!(\"abc\".parse::().is_err());\n }\n\n #[test]\n fn parse_version_rejects_incomplete() {\n assert!(\"0.67\".parse::().is_err());\n }\n\n #[test]\n fn version_ordering_is_semantic() {\n assert!(Version::new(1, 0, 0) > Version::new(0, 99, 99));\n }\n\n #[test]\n fn version_ordering_minor() {\n assert!(Version::new(0, 2, 0) > Version::new(0, 1, 99));\n }\n\n #[test]\n fn version_equality() {\n assert_eq!(Version::new(0, 67, 0), Version::new(0, 67, 0));\n }\n\n #[test]\n fn parse_version_with_prerelease() {\n for (input, expected_pre) in [\n (\"0.77.0-rc.1\", \"rc.1\"),\n (\"1.0.0-alpha\", \"alpha\"),\n (\"1.0.0-beta.2\", \"beta.2\"),\n (\"0.1.0-dev.20260414\", \"dev.20260414\"),\n ] {\n let ver: Version = input.parse().unwrap();\n assert_eq!(ver.prerelease, Some(expected_pre.to_string()), \"input: {input}\");\n assert_eq!(ver.show(), input, \"round-trip failed for: {input}\");\n }\n }\n\n #[test]\n fn prerelease_sorts_before_release() {\n let rc: Version = \"0.77.0-rc.1\".parse().unwrap();\n let release = Version::new(0, 77, 0);\n assert!(rc < release);\n }\n\n // ---- Error message tests ----\n\n #[test]\n fn invalid_version_renders() {\n let err = ManagerError::InvalidVersion(\"abc\".to_string());\n assert!(err.to_string().contains(\"Invalid version\"));\n }\n\n #[test]\n fn no_command_renders() {\n let err = ManagerError::NoCommand;\n assert!(err.to_string().contains(\"No command\"));\n }\n\n #[test]\n fn no_active_environment_suggests_new() {\n let err = ManagerError::NoActiveEnvironment;\n assert!(err.to_string().contains(\"new\"));\n }\n\n #[test]\n fn config_permission_denied_mentions_permissions() {\n let err = ManagerError::ConfigPermissionDenied(\"/etc/morloc/config.json\".to_string());\n assert!(err.to_string().contains(\"Permission\"));\n }\n\n #[test]\n fn freeze_error_renders() {\n let err = ManagerError::FreezeError(\"tar error\".to_string());\n assert!(err.to_string().contains(\"Freeze failed\"));\n }\n\n // ---- Config default tests ----\n\n #[test]\n fn default_config_has_no_active_env() {\n assert_eq!(Config::default().active_env, None);\n }\n\n #[test]\n fn default_config_uses_podman() {\n assert_eq!(Config::default().engine, ContainerEngine::Podman);\n }\n\n // ---- Config JSON round-trip tests ----\n\n #[test]\n fn config_json_round_trip() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"config.json\");\n let cfg = Config {\n active_env: Some(\"ml\".to_string()),\n engine: ContainerEngine::Docker,\n };\n cfg::write_config(&path, &cfg).unwrap();\n let cfg2: Config = cfg::read_config(&path).unwrap();\n assert_eq!(cfg2.active_env.as_deref(), Some(\"ml\"));\n assert_eq!(cfg2.engine, ContainerEngine::Docker);\n }\n\n #[test]\n fn config_read_missing_returns_not_found() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"nonexistent.json\");\n let result = cfg::read_config::(&path);\n assert!(matches!(result, Err(ManagerError::ConfigNotFound(_))));\n }\n\n #[test]\n fn config_read_invalid_json_returns_parse_error() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"bad.json\");\n fs::write(&path, \"not json at all\").unwrap();\n let result = cfg::read_config::(&path);\n assert!(matches!(result, Err(ManagerError::ConfigParseError { .. })));\n }\n\n #[test]\n fn env_config_json_round_trip() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"env.json\");\n let ec = EnvironmentConfig {\n name: \"test\".to_string(),\n base_image: \"ghcr.io/morloc-project/morloc/morloc-full:0.67.0\".to_string(),\n original_image: None,\n dockerfile: None,\n content_hash: None,\n built_image: None,\n engine: ContainerEngine::Podman,\n shm_size: \"1g\".to_string(),\n morloc_version: Some(Version::new(0, 67, 0)),\n };\n cfg::write_config(&path, &ec).unwrap();\n let ec2: EnvironmentConfig = cfg::read_config(&path).unwrap();\n assert_eq!(ec2.name, \"test\");\n assert_eq!(ec2.shm_size, \"1g\");\n assert_eq!(ec2.morloc_version, Some(Version::new(0, 67, 0)));\n }\n\n #[test]\n fn freeze_manifest_json_round_trip() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"fm.json\");\n let fm = FreezeManifest {\n morloc_version: Version::new(0, 67, 0),\n frozen_at: chrono::Utc::now(),\n modules: vec![ModuleEntry {\n name: \"math\".to_string(),\n version: Some(\"0.3.0\".to_string()),\n sha256: \"abc123\".to_string(),\n }],\n programs: vec![ProgramEntry {\n name: \"svc\".to_string(),\n commands: vec![\"hello\".to_string(), \"compute\".to_string()],\n }],\n base_image: \"morloc-full:0.67.0\".to_string(),\n env_layer: Some(FrozenEnvLayer {\n name: \"ml\".to_string(),\n dockerfile: \"FROM scratch\".to_string(),\n content_hash: \"abc\".to_string(),\n image_tag: None,\n }),\n env_vars: Vec::new(),\n };\n cfg::write_config(&path, &fm).unwrap();\n let fm2: FreezeManifest = cfg::read_config(&path).unwrap();\n assert_eq!(fm2.morloc_version, Version::new(0, 67, 0));\n assert_eq!(fm2.modules.len(), 1);\n assert_eq!(fm2.programs.len(), 1);\n assert_eq!(fm2.programs[0].commands, vec![\"hello\", \"compute\"]);\n // env_vars is no longer written but can still be read from old manifests\n assert!(fm2.env_vars.is_empty());\n }\n\n #[test]\n fn freeze_manifest_reads_legacy_env_vars() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"legacy.json\");\n // Simulate an old manifest that included env_vars\n let json = r#\"{\n \"morloc_version\": {\"major\":0,\"minor\":67,\"patch\":0,\"pre\":null},\n \"frozen_at\": \"2025-01-01T00:00:00Z\",\n \"modules\": [],\n \"programs\": [],\n \"base_image\": \"morloc-full:0.67.0\",\n \"env_layer\": null,\n \"env_vars\": [\"API_KEY\", \"DB_URL\"]\n }\"#;\n std::fs::write(&path, json).unwrap();\n let fm: FreezeManifest = cfg::read_config(&path).unwrap();\n assert_eq!(fm.env_vars, vec![\"API_KEY\", \"DB_URL\"]);\n }\n\n // ---- Config flags tests ----\n\n #[test]\n fn read_flags_file_parses() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"test.flags\");\n fs::write(\n &path,\n \"# This is a comment\\n--gpus all\\n\\n -v /data:/data \\n# another comment\\n--network host\\n\",\n )\n .unwrap();\n let flags = cfg::read_flags_file(&path);\n assert_eq!(\n flags,\n vec![\"--gpus\", \"all\", \"-v\", \"/data:/data\", \"--network\", \"host\"]\n );\n }\n\n #[test]\n fn read_flags_file_missing() {\n let dir = tempfile::tempdir().unwrap();\n let flags = cfg::read_flags_file(&dir.path().join(\"nope.flags\"));\n assert!(flags.is_empty());\n }\n\n #[test]\n fn read_flags_file_expands_env_vars() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"test.flags\");\n fs::write(&path, \"-v $HOME/data:/data\\n\").unwrap();\n let flags = cfg::read_flags_file(&path);\n let home = std::env::var(\"HOME\").unwrap();\n assert_eq!(flags, vec![\"-v\", &format!(\"{home}/data:/data\")]);\n }\n\n #[test]\n fn read_flags_file_expands_tilde() {\n let dir = tempfile::tempdir().unwrap();\n let path = dir.path().join(\"test.flags\");\n fs::write(&path, \"-v ~/data:/data\\n\").unwrap();\n let flags = cfg::read_flags_file(&path);\n let home = std::env::var(\"HOME\").unwrap();\n assert_eq!(flags, vec![\"-v\", &format!(\"{home}/data:/data\")]);\n }\n\n // ---- Container CLI argument tests ----\n\n #[test]\n fn engine_executable_docker() {\n assert_eq!(engine_executable(ContainerEngine::Docker), \"docker\");\n }\n\n #[test]\n fn engine_executable_podman() {\n assert_eq!(engine_executable(ContainerEngine::Podman), \"podman\");\n }\n\n #[test]\n fn build_run_args_minimal() {\n let cfg = RunConfig::new(\"myimage:latest\");\n let args = build_run_args(\n ContainerEngine::Docker,\n &engine_specific_run_flags(ContainerEngine::Docker),\n &cfg,\n );\n assert_eq!(args[0], \"run\");\n assert!(args.contains(&\"--rm\".to_string()));\n assert!(args.contains(&\"myimage:latest\".to_string()));\n assert!(!args.contains(&\"-it\".to_string()));\n }\n\n #[test]\n fn build_run_args_podman_userns() {\n let cfg = RunConfig::new(\"myimage:latest\");\n let args = build_run_args(\n ContainerEngine::Podman,\n &engine_specific_run_flags(ContainerEngine::Podman),\n &cfg,\n );\n assert!(args.contains(&\"--userns=keep-id\".to_string()));\n }\n\n #[test]\n fn build_run_args_interactive() {\n let mut cfg = RunConfig::new(\"img\");\n cfg.interactive = true;\n let args = build_run_args(\n ContainerEngine::Docker,\n &engine_specific_run_flags(ContainerEngine::Docker),\n &cfg,\n );\n assert!(args.contains(&\"-it\".to_string()));\n }\n\n #[test]\n fn build_run_args_selinux_suffix() {\n let mut cfg = RunConfig::new(\"img\");\n cfg.bind_mounts = vec![(\"/host\".to_string(), \"/container\".to_string())];\n cfg.selinux_suffix = \":z\".to_string();\n let args = build_run_args(\n ContainerEngine::Docker,\n &engine_specific_run_flags(ContainerEngine::Docker),\n &cfg,\n );\n assert!(args.contains(&\"-v\".to_string()));\n assert!(args.contains(&\"/host:/container:z\".to_string()));\n }\n\n #[test]\n fn build_run_args_workdir() {\n let mut cfg = RunConfig::new(\"img\");\n cfg.work_dir = Some(\"/work\".to_string());\n let args = build_run_args(\n ContainerEngine::Docker,\n &engine_specific_run_flags(ContainerEngine::Docker),\n &cfg,\n );\n assert!(args.contains(&\"-w\".to_string()));\n assert!(args.contains(&\"/work\".to_string()));\n }\n\n #[test]\n fn build_run_args_read_only() {\n let mut cfg = RunConfig::new(\"img\");\n cfg.read_only = true;\n let args = build_run_args(\n ContainerEngine::Docker,\n &engine_specific_run_flags(ContainerEngine::Docker),\n &cfg,\n );\n assert!(args.contains(&\"--read-only\".to_string()));\n }\n\n #[test]\n fn build_run_args_command_at_end() {\n let mut cfg = RunConfig::new(\"img\");\n cfg.command = Some(vec![\n \"morloc\".to_string(),\n \"make\".to_string(),\n \"-o\".to_string(),\n \"svc\".to_string(),\n \"svc.loc\".to_string(),\n ]);\n let args = build_run_args(\n ContainerEngine::Docker,\n &engine_specific_run_flags(ContainerEngine::Docker),\n &cfg,\n );\n let img_idx = args.iter().position(|a| a == \"img\").unwrap();\n let cmd_idx = args.iter().position(|a| a == \"morloc\").unwrap();\n assert!(img_idx < cmd_idx);\n }\n\n #[test]\n fn build_build_args_includes_tag_and_dockerfile() {\n let cfg = BuildConfig {\n dockerfile: \"/tmp/Dockerfile\".to_string(),\n context: \"/tmp/ctx\".to_string(),\n tag: \"test:v1\".to_string(),\n build_args: vec![(\"BASE\".to_string(), \"ubuntu:22.04\".to_string())],\n };\n let args = build_build_args(&cfg);\n assert_eq!(args[0], \"build\");\n assert!(args.contains(&\"-f\".to_string()));\n assert!(args.contains(&\"-t\".to_string()));\n assert!(args.contains(&\"--build-arg\".to_string()));\n assert_eq!(args.last().unwrap(), \"/tmp/ctx\");\n }\n\n // ---- SELinux tests ----\n\n #[test]\n fn root_is_unsafe() {\n assert!(!selinux::is_safe_to_relabel(\"/\"));\n }\n\n #[test]\n fn tmp_is_unsafe() {\n assert!(!selinux::is_safe_to_relabel(\"/tmp\"));\n }\n\n #[test]\n fn tmp_subdir_is_unsafe() {\n assert!(!selinux::is_safe_to_relabel(\"/tmp/foo\"));\n }\n\n #[test]\n fn home_subdir_is_safe() {\n assert!(selinux::is_safe_to_relabel(\"/home/user/project\"));\n }\n\n #[test]\n fn var_tmp_is_unsafe() {\n assert!(!selinux::is_safe_to_relabel(\"/var/tmp\"));\n }\n}\n" }, { "path": "data/rust/morloc-manager/src/selinux.rs", "content": "use std::path::Path;\nuse std::process::Command;\n\nuse crate::error::{ManagerError, Result};\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum SELinuxMode {\n Enforcing,\n Permissive,\n Disabled,\n}\n\npub fn detect_selinux() -> SELinuxMode {\n if !Path::new(\"/usr/sbin/getenforce\").exists() {\n return SELinuxMode::Disabled;\n }\n let Ok(output) = Command::new(\"getenforce\").output() else {\n return SELinuxMode::Disabled;\n };\n if !output.status.success() {\n return SELinuxMode::Disabled;\n }\n let stdout = String::from_utf8_lossy(&output.stdout);\n let first_line = stdout.lines().next().unwrap_or(\"\");\n match first_line {\n \"Enforcing\" => SELinuxMode::Enforcing,\n \"Permissive\" => SELinuxMode::Permissive,\n _ => SELinuxMode::Disabled,\n }\n}\n\npub fn volume_suffix(mode: SELinuxMode) -> &'static str {\n match mode {\n SELinuxMode::Enforcing => \":z\",\n SELinuxMode::Permissive | SELinuxMode::Disabled => \"\",\n }\n}\n\npub fn is_safe_to_relabel(path: &str) -> bool {\n let home = dirs::home_dir().unwrap_or_default();\n let norm = normalize(path);\n let home_norm = normalize_trailing(&home.to_string_lossy());\n let is_home_root = normalize_trailing(&norm) == home_norm;\n !is_unsafe_system_path(&norm) && !is_home_root\n}\n\npub fn validate_mount_path(path: &str) -> Result<()> {\n if is_safe_to_relabel(path) {\n Ok(())\n } else {\n Err(ManagerError::SELinuxError(format!(\n \"Cannot bind-mount {path} with SELinux relabeling. \\\n This path is unsafe to relabel. \\\n Use a subdirectory instead (e.g., {path}/project/).\"\n )))\n }\n}\n\nfn is_unsafe_system_path(p: &str) -> bool {\n let norm = normalize_trailing(p);\n norm == \"/\" || norm.starts_with(\"/tmp/\") || norm == \"/tmp/\" || norm.starts_with(\"/var/tmp/\") || norm == \"/var/tmp/\"\n}\n\nfn normalize(p: &str) -> String {\n // Simple normalization: resolve . and remove trailing /\n let path = Path::new(p);\n path.to_string_lossy().to_string()\n}\n\nfn normalize_trailing(p: &str) -> String {\n let mut s = normalize(p);\n if !s.ends_with('/') {\n s.push('/');\n }\n s\n}\n" }, { "path": "data/rust/morloc-manager/src/serve.rs", "content": "use std::fs;\nuse std::path::Path;\nuse std::process::{Command, Stdio};\nuse std::thread;\nuse std::time::Duration;\n\nuse crate::container::{\n container_build, container_pull, container_run, container_run_quiet, container_stop,\n container_remove, engine_executable, exit_code_to_int, image_exists_locally,\n BuildConfig, RunConfig,\n};\nuse crate::error::{ManagerError, Result};\nuse crate::types::*;\n\npub fn build_serve_image(\n engine: ContainerEngine,\n verbose: bool,\n state_tarball: &str,\n tag: &str,\n ver: Version,\n base_override: Option<&str>,\n rebuild: bool,\n programs: &[ProgramEntry],\n) -> Result<()> {\n if !Path::new(state_tarball).exists() {\n return Err(ManagerError::UnfreezeError(format!(\n \"Tarball not found: {state_tarball}\"\n )));\n }\n\n if !rebuild && image_exists_locally(engine, tag) {\n eprintln!(\"Image '{tag}' already exists locally; skipping build (use --rebuild to force)\");\n return Ok(());\n }\n\n let tarball_dir = Path::new(state_tarball)\n .parent()\n .unwrap_or(Path::new(\".\"));\n let manifest_path = tarball_dir.join(\"freeze-manifest.json\");\n let m_manifest = if manifest_path.exists() {\n crate::freeze::read_freeze_manifest(&manifest_path.to_string_lossy()).ok()\n } else {\n None\n };\n\n let base_image = match base_override {\n Some(b) => b.to_string(),\n None => resolve_base_from_manifest(engine, m_manifest.as_ref(), ver),\n };\n\n eprintln!(\"Using base image: {base_image}\");\n if !image_exists_locally(engine, &base_image) {\n let exe = engine_executable(engine);\n if verbose {\n eprintln!(\"[morloc-manager] {exe} pull {base_image}\");\n }\n let (pull_status, _, pull_err) = container_pull(engine, &base_image);\n if !pull_status.success() {\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(pull_status),\n stderr: pull_err,\n });\n }\n }\n\n let context_dir = tarball_dir.join(\"serve-build\");\n fs::create_dir_all(&context_dir)\n .map_err(|e| ManagerError::UnfreezeError(format!(\"mkdir failed: {e}\")))?;\n\n eprintln!(\"Extracting frozen state...\");\n let tar_status = Command::new(\"tar\")\n .args([\"-xzf\", state_tarball, \"-C\", &context_dir.to_string_lossy()])\n .stdin(Stdio::null())\n .stdout(Stdio::null())\n .stderr(Stdio::inherit())\n .status()\n .map_err(|e| ManagerError::UnfreezeError(format!(\"tar extract failed: {e}\")))?;\n if !tar_status.success() {\n return Err(ManagerError::UnfreezeError(\n \"tar extract failed (see error output above)\".to_string()\n ));\n }\n\n // Rewrite build.path in each manifest so the nexus chdirs to the\n // container-internal path instead of the original host path.\n rewrite_manifest_paths(&context_dir)?;\n\n let dockerfile_path = context_dir.join(\"Dockerfile\");\n let has_exe = context_dir.join(\"exe\").is_dir()\n && fs::read_dir(context_dir.join(\"exe\"))\n .map(|mut d| d.next().is_some())\n .unwrap_or(false);\n let has_opt = context_dir.join(\"opt\").is_dir()\n && fs::read_dir(context_dir.join(\"opt\"))\n .map(|mut d| d.next().is_some())\n .unwrap_or(false);\n let has_src = context_dir.join(\"src\").is_dir()\n && fs::read_dir(context_dir.join(\"src\"))\n .map(|mut d| d.next().is_some())\n .unwrap_or(false);\n let mh = CONTAINER_MORLOC_HOME;\n let exe_line = if has_exe {\n format!(\"COPY exe/ {mh}/exe/\\n\")\n } else {\n String::new()\n };\n let opt_line = if has_opt {\n format!(\"COPY opt/ {mh}/opt/\\n\")\n } else {\n String::new()\n };\n let src_line = if has_src {\n format!(\"COPY src/ {mh}/src/\\n\")\n } else {\n String::new()\n };\n // Podman's OCI format drops HEALTHCHECK; omit it to avoid warnings.\n let healthcheck = if engine == ContainerEngine::Docker {\n \"# Health check for container orchestrators\\n\\\n HEALTHCHECK --interval=30s --timeout=5s --retries=3 \\\\\\n\\\n CMD curl -sf http://localhost:8080/health || exit 1\\n\\\n \\n\"\n .to_string()\n } else {\n String::new()\n };\n let dockerfile_content = format!(\n \"# Auto-generated by morloc-manager serve-image\\n\\\n FROM {base_image}\\n\\\n \\n\\\n # Ensure morloc binaries are on PATH\\n\\\n ENV PATH=\\\"{mh}/bin:${{PATH}}\\\"\\n\\\n \\n\\\n # Morloc home for pool path resolution\\n\\\n ENV MORLOC_HOME=\\\"{mh}\\\"\\n\\\n \\n\\\n # Copy frozen morloc state (modules, manifests, binaries, pools)\\n\\\n COPY lib/ {mh}/lib/\\n\\\n COPY fdb/ {mh}/fdb/\\n\\\n COPY bin/ {mh}/bin/\\n\\\n {exe_line}\\\n {opt_line}\\\n {src_line}\\\n RUN chmod -R a+rX {mh}\\n\\\n \\n\\\n {healthcheck}\\\n # Entrypoint: nexus router aggregates all installed programs\\n\\\n ENTRYPOINT [\\\"morloc-nexus\\\", \\\"--router\\\", \\\\\\n\\\n \\\"--fdb\\\", \\\"{mh}/fdb\\\", \\\\\\n\\\n \\\"--http-port\\\", \\\"8080\\\"]\\n\"\n );\n fs::write(&dockerfile_path, &dockerfile_content)\n .map_err(|e| ManagerError::UnfreezeError(format!(\"Write Dockerfile failed: {e}\")))?;\n\n eprintln!(\"Building serve image {tag} (base: {base_image})...\");\n let build_cfg = BuildConfig {\n dockerfile: dockerfile_path.to_string_lossy().to_string(),\n context: context_dir.to_string_lossy().to_string(),\n tag: tag.to_string(),\n build_args: Vec::new(),\n };\n if verbose {\n let exe = engine_executable(engine);\n eprintln!(\n \"[morloc-manager] {exe} build -f {} -t {tag} {}\",\n build_cfg.dockerfile, build_cfg.context\n );\n }\n let (status, _, build_err) = container_build(engine, &build_cfg);\n if !status.success() {\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(status),\n stderr: build_err,\n });\n }\n eprintln!(\"Built serve image: {tag}\");\n\n // Validate programs work inside the built image\n validate_programs(engine, tag, programs, Vec::new(), verbose)?;\n\n // Clean up the temporary build context\n if let Err(e) = fs::remove_dir_all(&context_dir) {\n eprintln!(\"Warning: failed to clean up {}: {e}\", context_dir.display());\n }\n\n Ok(())\n}\n\n#[allow(dead_code)]\npub fn run_serve_container(\n engine: ContainerEngine,\n verbose: bool,\n image: &str,\n name: &str,\n ports: &[(u16, u16)],\n) -> Result<()> {\n // Clean up any existing dead container with this name (silently)\n let _ = crate::container::container_remove_quiet(engine, name);\n\n let port_str: Vec = ports\n .iter()\n .map(|(h, c)| format!(\"{h}:{c}\"))\n .collect();\n eprintln!(\n \"Starting serve container {name} on ports {}...\",\n port_str.join(\", \")\n );\n\n let mut cfg = RunConfig::new(image);\n cfg.read_only = true;\n cfg.remove_after = false;\n cfg.name = Some(name.to_string());\n cfg.ports = ports.to_vec();\n cfg.extra_flags = vec![\"-d\".to_string()];\n\n if verbose {\n let exe = engine_executable(engine);\n let extra = crate::container::engine_specific_run_flags_io(engine);\n let args = crate::container::build_run_args(engine, &extra, &cfg);\n let quoted: Vec = args.iter().map(|a| {\n if a.contains(' ') { format!(\"'{a}'\") } else { a.clone() }\n }).collect();\n eprintln!(\"[morloc-manager] {exe} {}\", quoted.join(\" \"));\n }\n\n let (status, _stdout, run_err) = container_run(engine, &cfg);\n if !status.success() {\n let _ = crate::container::container_remove_quiet(engine, name);\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(status),\n stderr: run_err,\n });\n }\n\n // Verify container reached running state\n thread::sleep(Duration::from_secs(1));\n let exe = engine_executable(engine);\n let insp_output = Command::new(exe)\n .args([\"inspect\", \"--format\", \"{{.State.Status}}\", name])\n .output();\n match insp_output {\n Ok(o) if o.status.success() => {\n let state = String::from_utf8_lossy(&o.stdout).trim().to_string();\n if state == \"running\" {\n eprintln!(\"Container {name} started\");\n eprintln!(\" Logs: morloc-manager logs\");\n eprintln!(\" Stop: morloc-manager stop {name}\");\n eprintln!(\" Status: morloc-manager status\");\n Ok(())\n } else {\n let log_output = Command::new(exe).args([\"logs\", name]).output();\n let logs = log_output\n .map(|o| {\n let stdout = String::from_utf8_lossy(&o.stdout);\n let stderr = String::from_utf8_lossy(&o.stderr);\n format!(\"{stdout}{stderr}\")\n })\n .unwrap_or_default();\n // Clean up the dead container to prevent name conflicts on retry\n let _ = container_remove(engine, name);\n Err(ManagerError::EngineError {\n engine,\n code: 1,\n stderr: format!(\"Container failed to start (state: {state}):\\n{logs}\"),\n })\n }\n }\n _ => Err(ManagerError::EngineError {\n engine,\n code: 1,\n stderr: \"Failed to inspect container state\".to_string(),\n }),\n }\n}\n\n/// Serve an environment by bind-mounting its data directory into the container.\npub fn serve_environment(\n engine: ContainerEngine,\n verbose: bool,\n image: &str,\n data_dir: &str,\n container_name: &str,\n ports: &[(u16, u16)],\n extra_flags: &[String],\n shm_size: &Option,\n user_env: &[(String, String)],\n) -> Result<()> {\n // Clean up any existing dead container with this name (silently)\n let _ = crate::container::container_remove_quiet(engine, container_name);\n\n let port_str: Vec = ports\n .iter()\n .map(|(h, c)| format!(\"{h}:{c}\"))\n .collect();\n eprintln!(\n \"Starting serve container {container_name} on ports {}...\",\n port_str.join(\", \")\n );\n\n let mut cfg = RunConfig::new(image);\n cfg.read_only = true;\n cfg.remove_after = false;\n cfg.name = Some(container_name.to_string());\n cfg.ports = ports.to_vec();\n let mh = CONTAINER_MORLOC_HOME;\n cfg.bind_mounts = vec![(data_dir.to_string(), mh.to_string())];\n cfg.env = vec![\n (\"PATH\".to_string(), format!(\"{mh}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\")),\n (\"MORLOC_HOME\".to_string(), mh.to_string()),\n ];\n cfg.env.extend(user_env.iter().cloned());\n cfg.command = Some(vec![\n \"morloc-nexus\".to_string(),\n \"--router\".to_string(),\n \"--fdb\".to_string(), format!(\"{mh}/fdb\"),\n \"--http-port\".to_string(), \"8080\".to_string(),\n ]);\n cfg.shm_size = shm_size.clone();\n cfg.extra_flags = vec![\"-d\".to_string()];\n cfg.extra_flags.extend(extra_flags.iter().cloned());\n\n if verbose {\n let exe = engine_executable(engine);\n let extra = crate::container::engine_specific_run_flags_io(engine);\n let args = crate::container::build_run_args(engine, &extra, &cfg);\n let quoted: Vec = args.iter().map(|a| {\n if a.contains(' ') { format!(\"'{a}'\") } else { a.clone() }\n }).collect();\n eprintln!(\"[morloc-manager] {exe} {}\", quoted.join(\" \"));\n }\n\n let (status, _stdout, run_err) = container_run(engine, &cfg);\n if !status.success() {\n // `container_run` may have left a partially-created container behind\n // (e.g., port conflict after container creation). Clean it up so the\n // next `start` doesn't fail on a name collision.\n let _ = crate::container::container_remove_quiet(engine, container_name);\n\n // Detect port conflict and provide a friendlier error message\n let lower = run_err.to_lowercase();\n if lower.contains(\"address already in use\") || lower.contains(\"port is already allocated\")\n || lower.contains(\"pasta failed\")\n {\n // Try to extract the port number from the error\n let port_hint = ports.first()\n .map(|(h, _)| format!(\" Port {h} is already in use.\"))\n .unwrap_or_default();\n return Err(ManagerError::EnvError(format!(\n \"{port_hint}\\n \\\n Another container or process is using this port.\\n \\\n Use '-p :8080' to choose a different host port, or\\n \\\n check running containers with 'morloc-manager status'.\"\n )));\n }\n\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(status),\n stderr: run_err,\n });\n }\n\n // Verify container reached running state\n thread::sleep(Duration::from_secs(1));\n let exe = engine_executable(engine);\n let insp_output = Command::new(exe)\n .args([\"inspect\", \"--format\", \"{{.State.Status}}\", container_name])\n .output();\n match insp_output {\n Ok(o) if o.status.success() => {\n let state = String::from_utf8_lossy(&o.stdout).trim().to_string();\n if state == \"running\" {\n eprintln!(\"Container {container_name} started\");\n eprintln!(\" Logs: morloc-manager logs\");\n eprintln!(\" Stop: morloc-manager stop\");\n eprintln!(\" Status: morloc-manager status\");\n Ok(())\n } else {\n let log_output = Command::new(exe).args([\"logs\", container_name]).output();\n let logs = log_output\n .map(|o| {\n let stdout = String::from_utf8_lossy(&o.stdout);\n let stderr = String::from_utf8_lossy(&o.stderr);\n format!(\"{stdout}{stderr}\")\n })\n .unwrap_or_default();\n let _ = container_remove(engine, container_name);\n Err(ManagerError::EngineError {\n engine,\n code: 1,\n stderr: format!(\"Container failed to start (state: {state}):\\n{logs}\"),\n })\n }\n }\n _ => Err(ManagerError::EngineError {\n engine,\n code: 1,\n stderr: \"Failed to inspect container state\".to_string(),\n }),\n }\n}\n\npub fn stop_serve_container(engine: ContainerEngine, verbose: bool, name: &str) -> Result<()> {\n if !crate::container::container_exists(engine, name) {\n return Err(ManagerError::EnvError(format!(\n \"No serve container running for '{name}'\"\n )));\n }\n if verbose {\n let exe = engine_executable(engine);\n eprintln!(\"[morloc-manager] {exe} stop {name}\");\n }\n let (status, err) = container_stop(engine, name);\n let _ = crate::container::container_remove_quiet(engine, name);\n if !status.success() {\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(status),\n stderr: err,\n });\n }\n Ok(())\n}\n\n/// Build the serve container name for an environment.\n/// Format: morloc-serve--\npub fn serve_container_name(env_name: &str) -> String {\n let user = std::env::var(\"USER\")\n .or_else(|_| std::env::var(\"LOGNAME\"))\n .unwrap_or_else(|_| \"unknown\".to_string());\n format!(\"morloc-serve-{user}-{env_name}\")\n}\n\n/// The prefix used to filter all serve containers for the current user.\npub fn serve_container_prefix() -> String {\n let user = std::env::var(\"USER\")\n .or_else(|_| std::env::var(\"LOGNAME\"))\n .unwrap_or_else(|_| \"unknown\".to_string());\n format!(\"morloc-serve-{user}-\")\n}\n\n/// Extract the environment name from a serve container name.\npub fn env_name_from_container(container_name: &str) -> &str {\n let prefix = serve_container_prefix();\n container_name.strip_prefix(&prefix).unwrap_or(container_name)\n}\n\n#[derive(serde::Serialize)]\npub struct ServeContainerInfo {\n pub name: String,\n pub env: String,\n pub ports: String,\n pub status: String,\n}\n\n/// Query running serve containers and return structured info.\npub fn query_serve_containers(engine: ContainerEngine, verbose: bool) -> Result> {\n let exe = engine_executable(engine);\n let fmt = \"{{.Names}}\\t{{.Status}}\\t{{.Ports}}\";\n let prefix = serve_container_prefix();\n let filter = format!(\"name={prefix}\");\n if verbose {\n eprintln!(\"[morloc-manager] {exe} ps -a --filter {filter} --format '{fmt}'\");\n }\n let output = Command::new(exe)\n .args([\n \"ps\", \"-a\", \"--filter\", &filter, \"--format\", fmt,\n ])\n // Use /tmp as cwd to avoid podman \"cannot chdir\" failures when the\n // current directory is inaccessible (e.g. another user's home).\n .current_dir(\"/tmp\")\n .output()\n .map_err(|e| ManagerError::EngineError {\n engine,\n code: 1,\n stderr: format!(\"Failed to list containers: {e}\"),\n })?;\n if !output.status.success() {\n return Err(ManagerError::EngineError {\n engine,\n code: exit_code_to_int(output.status),\n stderr: String::from_utf8_lossy(&output.stderr).to_string(),\n });\n }\n let text = String::from_utf8_lossy(&output.stdout).trim().to_string();\n let mut result = Vec::new();\n for line in text.lines() {\n let parts: Vec<&str> = line.split('\\t').collect();\n if parts.len() >= 3 {\n let name = parts[0];\n let status = parts[1];\n let ports = parts[2];\n let env = env_name_from_container(name);\n result.push(ServeContainerInfo {\n name: name.to_string(),\n env: env.to_string(),\n ports: if ports.is_empty() { \"-\".to_string() } else { ports.to_string() },\n status: status.to_string(),\n });\n }\n }\n Ok(result)\n}\n\n/// Find running serve container names for the current user.\npub fn find_running_serve_containers(engine: ContainerEngine) -> Vec {\n let exe = engine_executable(engine);\n let filter = format!(\"name={}\", serve_container_prefix());\n let output = Command::new(exe)\n .args([\"ps\", \"--filter\", &filter, \"--format\", \"{{.Names}}\"])\n .current_dir(\"/tmp\")\n .output();\n match output {\n Ok(o) if o.status.success() => {\n String::from_utf8_lossy(&o.stdout)\n .lines()\n .filter(|l| !l.is_empty())\n .map(|l| l.to_string())\n .collect()\n }\n _ => Vec::new(),\n }\n}\n\n// ======================================================================\n// Program validation\n// ======================================================================\n\n/// Run `--help` for each installed program inside a container image to\n/// verify that pool processes start correctly (e.g. all imports resolve).\n///\n/// `bind_mounts` should be non-empty for pre-freeze validation (where the\n/// data dir is on the host) and empty for post-unfreeze validation (where\n/// everything is baked into the image).\npub fn validate_programs(\n engine: ContainerEngine,\n image: &str,\n programs: &[ProgramEntry],\n bind_mounts: Vec<(String, String)>,\n verbose: bool,\n) -> Result<()> {\n if programs.is_empty() {\n return Ok(());\n }\n eprintln!(\"Validating installed programs...\");\n let mut any_failed = false;\n for prog in programs {\n let exe_path = format!(\"{}/bin/{}\", CONTAINER_MORLOC_HOME, prog.name);\n if verbose {\n let exe = engine_executable(engine);\n eprintln!(\"[morloc-manager] {exe} run --rm --entrypoint '' {image} {exe_path} --help\");\n }\n let cfg = RunConfig {\n bind_mounts: bind_mounts.clone(),\n command: Some(vec![exe_path, \"--help\".to_string()]),\n env: vec![\n (\"MORLOC_HOME\".to_string(), CONTAINER_MORLOC_HOME.to_string()),\n ],\n // Override the image ENTRYPOINT so the command runs directly\n // instead of being appended to the router entrypoint.\n extra_flags: vec![\"--entrypoint\".to_string(), \"\".to_string()],\n ..RunConfig::new(image)\n };\n let (status, _stdout, stderr) = container_run_quiet(engine, &cfg);\n if status.success() {\n let n = prog.commands.len();\n eprintln!(\" [ok] {} ({} commands)\", prog.name, n);\n } else {\n let snippet: String = stderr.lines().take(5).collect::>().join(\"\\n \");\n eprintln!(\" [FAIL] {}: {}\", prog.name, snippet);\n any_failed = true;\n }\n }\n if any_failed {\n return Err(ManagerError::FreezeError(\n \"Some programs failed validation (see errors above)\".to_string(),\n ));\n }\n Ok(())\n}\n\n// ======================================================================\n// Manifest path rewriting for frozen images\n// ======================================================================\n\nconst CONTAINER_MORLOC_HOME: &str = \"/opt/morloc\";\nconst MANIFEST_MARKER: &str = \"### MANIFEST ###\";\n\n/// Rewrite `build.path` in every `.manifest` file under `fdb/` so the\n/// nexus inside the container chdirs to the correct location instead of\n/// the original host path.\nfn rewrite_manifest_paths(context_dir: &Path) -> Result<()> {\n let fdb_dir = context_dir.join(\"fdb\");\n if !fdb_dir.is_dir() {\n return Ok(());\n }\n let entries = fs::read_dir(&fdb_dir)\n .map_err(|e| ManagerError::UnfreezeError(format!(\"read fdb/: {e}\")))?;\n for entry in entries {\n let entry = entry\n .map_err(|e| ManagerError::UnfreezeError(format!(\"read fdb/ entry: {e}\")))?;\n let path = entry.path();\n let name = entry.file_name();\n let name_str = name.to_string_lossy();\n if !name_str.ends_with(\".manifest\") {\n continue;\n }\n let prog_name = &name_str[..name_str.len() - \".manifest\".len()];\n let container_build_path = format!(\"{}/exe/{}\", CONTAINER_MORLOC_HOME, prog_name);\n rewrite_one_manifest(&path, &container_build_path)?;\n }\n Ok(())\n}\n\n/// Rewrite the `build.path` field in a single manifest wrapper script.\nfn rewrite_one_manifest(path: &Path, new_build_path: &str) -> Result<()> {\n let content = fs::read_to_string(path)\n .map_err(|e| ManagerError::UnfreezeError(format!(\"read {}: {e}\", path.display())))?;\n\n let (prefix, json_str) = if content.starts_with(\"#!\") {\n if let Some(marker_pos) = content.find(MANIFEST_MARKER) {\n let after_marker = &content[marker_pos..];\n let json_start = after_marker\n .find('\\n')\n .map(|i| marker_pos + i + 1)\n .unwrap_or(content.len());\n (&content[..json_start], &content[json_start..])\n } else {\n return Ok(()); // no marker, skip\n }\n } else {\n (\"\", content.as_str())\n };\n\n let mut manifest: serde_json::Value = serde_json::from_str(json_str)\n .map_err(|e| ManagerError::UnfreezeError(format!(\"parse {}: {e}\", path.display())))?;\n\n if let Some(build) = manifest.get_mut(\"build\") {\n if let Some(p) = build.get_mut(\"path\") {\n *p = serde_json::Value::String(new_build_path.to_string());\n }\n }\n\n let new_json = serde_json::to_string(&manifest)\n .map_err(|e| ManagerError::UnfreezeError(format!(\"serialize {}: {e}\", path.display())))?;\n\n let new_content = format!(\"{}{}\\n\", prefix, new_json);\n fs::write(path, new_content)\n .map_err(|e| ManagerError::UnfreezeError(format!(\"write {}: {e}\", path.display())))?;\n Ok(())\n}\n\n// ======================================================================\n// Manifest and image resolution\n// ======================================================================\n\nfn resolve_base_from_manifest(\n engine: ContainerEngine,\n m_manifest: Option<&FreezeManifest>,\n ver: Version,\n) -> String {\n let ghcr_fallback = format!(\n \"ghcr.io/morloc-project/morloc/morloc-full:{}\",\n ver.show()\n );\n let Some(fm) = m_manifest else {\n return ghcr_fallback;\n };\n\n // Resolve the effective base image: use manifest's base_image if it exists\n // locally, otherwise fall back to the GHCR image. The manifest may record a\n // locally-retagged image (e.g. localhost/morloc:0.69.0) that won't exist on\n // other machines.\n let effective_base = if image_exists_locally(engine, &fm.base_image) {\n fm.base_image.clone()\n } else {\n eprintln!(\n \"Base image '{}' not found locally, trying GHCR fallback...\",\n fm.base_image\n );\n ghcr_fallback\n };\n\n match &fm.env_layer {\n None => effective_base,\n Some(fel) => {\n // Fast path: env image tag exists locally\n if let Some(ref tag) = fel.image_tag {\n let exe = engine_executable(engine);\n let check = Command::new(exe)\n .args([\"image\", \"inspect\", tag])\n .stdout(Stdio::null())\n .stderr(Stdio::null())\n .status();\n if check.map(|s| s.success()).unwrap_or(false) {\n return tag.clone();\n }\n }\n // Rebuild env layer from stored Dockerfile using effective base\n rebuild_env_image(engine, &effective_base, fm, fel)\n }\n }\n}\n\nfn rebuild_env_image(\n engine: ContainerEngine,\n effective_base: &str,\n fm: &FreezeManifest,\n fel: &FrozenEnvLayer,\n) -> String {\n let env_tag = format!(\n \"localhost/morloc-env:{}-{}\",\n fm.morloc_version.show(),\n fel.name\n );\n let exe = engine_executable(engine);\n // Check if tagged image exists locally\n let check = Command::new(exe)\n .args([\"image\", \"inspect\", &env_tag])\n .stdout(Stdio::null())\n .stderr(Stdio::null())\n .status();\n if check.map(|s| s.success()).unwrap_or(false) {\n return env_tag;\n }\n\n eprintln!(\"Building deployment image (environment layer: {})\", fel.name);\n let build_dir = \"/tmp/morloc-env-rebuild\";\n let _ = fs::create_dir_all(build_dir);\n let df_path = format!(\"{build_dir}/Dockerfile\");\n let _ = fs::write(&df_path, &fel.dockerfile);\n let build_cfg = BuildConfig {\n dockerfile: df_path,\n context: build_dir.to_string(),\n tag: env_tag.clone(),\n build_args: vec![(\"CONTAINER_BASE\".to_string(), effective_base.to_string())],\n };\n let (status, _, build_err) = container_build(engine, &build_cfg);\n if status.success() {\n env_tag\n } else {\n eprintln!(\n \"Warning: env rebuild failed, falling back to base image: {build_err}\"\n );\n effective_base.to_string()\n }\n}\n" }, { "path": "data/rust/morloc-manager/src/types.rs", "content": "use serde::{Deserialize, Deserializer, Serialize, Serializer};\nuse std::cmp::Ordering;\nuse std::str::FromStr;\n\n// ======================================================================\n// Core enumerations\n// ======================================================================\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]\npub enum Scope {\n Local,\n System,\n}\n\nimpl Serialize for Scope {\n fn serialize(&self, serializer: S) -> Result {\n match self {\n Scope::Local => serializer.serialize_str(\"local\"),\n Scope::System => serializer.serialize_str(\"system\"),\n }\n }\n}\n\nimpl<'de> Deserialize<'de> for Scope {\n fn deserialize>(deserializer: D) -> Result {\n let s = String::deserialize(deserializer)?;\n match s.as_str() {\n \"local\" => Ok(Scope::Local),\n \"system\" => Ok(Scope::System),\n _ => Err(serde::de::Error::custom(format!(\"Unknown scope: {s}\"))),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum ContainerEngine {\n Docker,\n Podman,\n}\n\nimpl Serialize for ContainerEngine {\n fn serialize(&self, serializer: S) -> Result {\n match self {\n ContainerEngine::Docker => serializer.serialize_str(\"docker\"),\n ContainerEngine::Podman => serializer.serialize_str(\"podman\"),\n }\n }\n}\n\nimpl<'de> Deserialize<'de> for ContainerEngine {\n fn deserialize>(deserializer: D) -> Result {\n let s = String::deserialize(deserializer)?;\n match s.as_str() {\n \"docker\" => Ok(ContainerEngine::Docker),\n \"podman\" => Ok(ContainerEngine::Podman),\n _ => Err(serde::de::Error::custom(format!(\n \"Unknown container engine: {s}\"\n ))),\n }\n }\n}\n\n// ======================================================================\n// Version\n// ======================================================================\n\n#[derive(Debug, Clone, PartialEq, Eq)]\npub struct Version {\n pub major: u32,\n pub minor: u32,\n pub patch: u32,\n pub prerelease: Option,\n}\n\nimpl Version {\n #[cfg(test)]\n pub fn new(major: u32, minor: u32, patch: u32) -> Self {\n Self {\n major,\n minor,\n patch,\n prerelease: None,\n }\n }\n\n pub fn show(&self) -> String {\n match &self.prerelease {\n Some(pre) => format!(\"{}.{}.{}-{}\", self.major, self.minor, self.patch, pre),\n None => format!(\"{}.{}.{}\", self.major, self.minor, self.patch),\n }\n }\n}\n\nimpl Ord for Version {\n fn cmp(&self, other: &Self) -> Ordering {\n self.major\n .cmp(&other.major)\n .then(self.minor.cmp(&other.minor))\n .then(self.patch.cmp(&other.patch))\n .then(match (&self.prerelease, &other.prerelease) {\n (None, None) => Ordering::Equal,\n (Some(_), None) => Ordering::Less, // pre-release < release\n (None, Some(_)) => Ordering::Greater, // release > pre-release\n (Some(a), Some(b)) => a.cmp(b),\n })\n }\n}\n\nimpl PartialOrd for Version {\n fn partial_cmp(&self, other: &Self) -> Option {\n Some(self.cmp(other))\n }\n}\n\nimpl FromStr for Version {\n type Err = String;\n\n fn from_str(s: &str) -> Result {\n // Split off pre-release suffix on first '-': \"0.77.0-rc.1\" -> (\"0.77.0\", Some(\"rc.1\"))\n let (version_part, prerelease) = match s.find('-') {\n Some(idx) => (&s[..idx], Some(s[idx + 1..].to_string())),\n None => (s, None),\n };\n let parts: Vec<&str> = version_part.split('.').collect();\n if parts.len() != 3 {\n return Err(format!(\"Invalid version: {s}. Expected format: MAJOR.MINOR.PATCH[-PRERELEASE]\"));\n }\n let major = parts[0]\n .parse()\n .map_err(|_| format!(\"Invalid major version: {}\", parts[0]))?;\n let minor = parts[1]\n .parse()\n .map_err(|_| format!(\"Invalid minor version: {}\", parts[1]))?;\n let patch = parts[2]\n .parse()\n .map_err(|_| format!(\"Invalid patch version: {}\", parts[2]))?;\n Ok(Version {\n major,\n minor,\n patch,\n prerelease,\n })\n }\n}\n\nimpl Serialize for Version {\n fn serialize(&self, serializer: S) -> Result {\n serializer.serialize_str(&self.show())\n }\n}\n\nimpl<'de> Deserialize<'de> for Version {\n fn deserialize>(deserializer: D) -> Result {\n let s = String::deserialize(deserializer)?;\n s.parse().map_err(serde::de::Error::custom)\n }\n}\n\n// ======================================================================\n// Configuration\n// ======================================================================\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\npub struct Config {\n /// Name of the active environment.\n pub active_env: Option,\n /// Default container engine.\n #[serde(default = \"default_engine\")]\n pub engine: ContainerEngine,\n}\n\nfn default_engine() -> ContainerEngine {\n ContainerEngine::Podman\n}\n\nimpl Default for Config {\n fn default() -> Self {\n Self {\n active_env: None,\n engine: ContainerEngine::Podman,\n }\n }\n}\n\n// ======================================================================\n// Environment configuration\n// ======================================================================\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\npub struct EnvironmentConfig {\n /// Human-readable name (also the directory name).\n pub name: String,\n /// Base container image reference.\n pub base_image: String,\n /// Original pullable image reference (e.g., :edge tag) before local re-tagging.\n #[serde(default)]\n pub original_image: Option,\n /// Filename of the custom Dockerfile layer (within the env config dir).\n #[serde(default)]\n pub dockerfile: Option,\n /// SHA256 hash of the Dockerfile content (for rebuild detection).\n #[serde(default)]\n pub content_hash: Option,\n /// Built image tag after applying the Dockerfile layer.\n /// None when only the base image is used.\n #[serde(default)]\n pub built_image: Option,\n /// Container engine for this environment.\n pub engine: ContainerEngine,\n /// Shared memory size for container runs.\n #[serde(default = \"default_shm_size\")]\n pub shm_size: String,\n /// Morloc version this environment was created from.\n #[serde(default)]\n pub morloc_version: Option,\n}\n\nfn default_shm_size() -> String {\n \"512m\".to_string()\n}\n\nimpl EnvironmentConfig {\n /// Returns the image to use for running containers.\n /// Prefers the built Dockerfile layer image, falls back to base_image.\n pub fn active_image(&self) -> &str {\n self.built_image.as_deref().unwrap_or(&self.base_image)\n }\n}\n\n// ======================================================================\n// Freeze manifest\n// ======================================================================\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\npub struct FreezeManifest {\n pub morloc_version: Version,\n pub frozen_at: chrono::DateTime,\n pub modules: Vec,\n pub programs: Vec,\n pub base_image: String,\n pub env_layer: Option,\n /// Deprecated: previously held expected env var names. Retained for backward\n /// compatibility when reading older freeze manifests.\n #[serde(default, skip_serializing)]\n #[allow(dead_code)]\n pub env_vars: Vec,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\npub struct FrozenEnvLayer {\n pub name: String,\n pub dockerfile: String,\n pub content_hash: String,\n /// Container image tag (e.g. localhost/morloc-env:0.79.2-dnd).\n /// Named image_tag because it stores a mutable tag, not a content-addressed digest.\n #[serde(alias = \"image_digest\")]\n pub image_tag: Option,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\npub struct ModuleEntry {\n pub name: String,\n pub version: Option,\n pub sha256: String,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\npub struct ProgramEntry {\n pub name: String,\n pub commands: Vec,\n}\n" }, { "path": "data/rust/morloc-manifest/Cargo.toml", "content": "[package]\nname = \"morloc-manifest\"\nversion = \"0.81.0\"\nedition = \"2021\"\ndescription = \"Morloc manifest schema (v2): shared Rust types for the .manifest JSON blob produced by the morloc compiler and consumed by the nexus, runtime, and any future tooling.\"\n# Version is intentionally synchronized with the morloc compiler version\n# (see ../../../package.yaml). Manifests record the morloc_version that\n# built them; consumers reject manifests whose version differs from\n# CARGO_PKG_VERSION at parse time. Bumping the morloc compiler version\n# requires bumping this version in lockstep.\n\n[dependencies]\nserde = { workspace = true }\nserde_json = { workspace = true }\n" }, { "path": "data/rust/morloc-manifest/src/lib.rs", "content": "//! Morloc manifest schema (v2) -- canonical Rust types.\n//!\n//! The morloc compiler emits a `.manifest` JSON blob describing every\n//! exported command's interface. This crate is the **single source of\n//! truth** for that schema's Rust representation. Both the CLI nexus\n//! (`morloc-nexus`) and the C-FFI runtime (`morloc-runtime`) depend on\n//! these types so neither has to maintain its own deserialization\n//! logic.\n//!\n//! ## Versioning\n//!\n//! The manifest does not carry a dedicated schema version. Manifests\n//! are transient build artifacts (always regenerated on `morloc make`,\n//! never stored in version control), so the morloc compiler version\n//! recorded in the [`Build`] sub-object serves as the staleness\n//! indicator. The check happens in [`parse_manifest`].\n//!\n//! Version coupling: this crate's `CARGO_PKG_VERSION` is intentionally\n//! kept in lockstep with the morloc Haskell compiler version (see\n//! `package.yaml`). The same is true of `morloc-nexus` and\n//! `morloc-runtime` -- bumping the morloc compiler requires bumping\n//! all three Rust crates in the same commit.\n//!\n//! ## Extension slots\n//!\n//! Every entity object (manifest, pool, command, arg, return, group,\n//! service) carries:\n//!\n//! - `constraints: Vec` -- enforceable rules. Currently\n//! the compiler emits only the `kind` constraint on named-type args\n//! (record/object/table). Future constraints (`min`, `max`, `regex`,\n//! `length`, `non_empty`, `row_count`, ...) will append to this\n//! list without any schema change.\n//!\n//! - `metadata: BTreeMap` -- free-form\n//! informational key-value pairs. Always emitted as `{}` today;\n//! reserved so consumers never have to check whether the field\n//! exists. Future doc hints, studio annotations, telemetry tags,\n//! etc. live here until they stabilize into first-class fields.\n//!\n//! Many of these slots are `#[allow(dead_code)]` because no current\n//! consumer reads them. They are deliberate forward-compatible\n//! placeholders, not vestigial fields.\n//!\n//! ## Unknown-field tolerance\n//!\n//! All structs use `#[serde(default)]` on optional fields and silently\n//! ignore unknown JSON keys. A manifest written by a newer morloc\n//! compiler will still parse with an older nexus (modulo the version\n//! mismatch error in [`parse_manifest`]).\n\nuse serde::Deserialize;\nuse std::collections::BTreeMap;\n\n/// Convenient alias for the `metadata` extension slot. Using\n/// `BTreeMap` (rather than `HashMap` or raw `serde_json::Value`) gives\n/// us (a) compile-time enforcement that metadata is always a JSON\n/// object, and (b) deterministic iteration order for stable diffs.\npub type Metadata = BTreeMap;\n\n// -- Top-level manifest -------------------------------------------------------\n\n/// The top-level manifest object. Embedded in every built nexus binary\n/// as a JSON blob after the `### MANIFEST ###` marker.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct Manifest {\n /// Program identifier -- comes from the morloc `module` declaration.\n pub name: String,\n /// Compiler-sourced build metadata (path, timestamp, version).\n /// Distinct from the user-sourced top-level `metadata` slot.\n pub build: Build,\n /// Language pool daemons that this program dispatches to.\n #[serde(default)]\n pub pools: Vec,\n /// Exported commands the user can invoke.\n #[serde(default)]\n pub commands: Vec,\n /// Command groups for organizing CLI subcommands in help output.\n #[serde(default)]\n pub groups: Vec,\n /// Daemon-mode service configuration. None for normal CLI mode.\n #[serde(default)]\n pub service: Option,\n /// Module-level description lines (from docstrings before `module`).\n /// Shown after \"Usage:\" and before \"Nexus options\" in top-level help.\n #[serde(default)]\n pub desc: Vec,\n /// Epilogue blocks shown at the end of top-level help output.\n #[serde(default)]\n pub epilogues: Vec>,\n /// **Reserved.** User-sourced free-form annotations on the module.\n /// Always emitted as `{}` today. Distinct from `build` (which is\n /// compiler-sourced).\n #[serde(default)]\n pub metadata: Metadata,\n}\n\n/// Compiler-sourced metadata about how this manifest was produced.\n///\n/// Future build fields (`hash`, `source_hash`, `host`, `user`, `system`,\n/// `dependencies`, `cflags`, `reproducible`, ...) will be added directly\n/// to this struct as additive non-breaking changes -- no sub-metadata\n/// nesting required.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct Build {\n /// Absolute path to the build directory containing this program's\n /// pool executables and generated source files. The nexus chdirs\n /// here at startup so relative pool exec paths resolve.\n pub path: String,\n /// Unix timestamp at which the manifest was generated.\n pub time: i64,\n /// Version of the morloc compiler that produced this manifest. The\n /// nexus compares this against its own compile-time\n /// `CARGO_PKG_VERSION` (which is intentionally synchronized with\n /// the morloc compiler version) in [`parse_manifest`]; a mismatch\n /// produces an actionable \"rebuild with the current compiler\"\n /// error.\n pub morloc_version: String,\n}\n\n/// A single language pool daemon. Each pool is one OS process that\n/// hosts the language-specific implementations of source functions.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct Pool {\n /// Language tag (e.g. `\"py\"`, `\"cpp\"`, `\"r\"`, `\"julia\"`).\n pub lang: String,\n /// argv used to spawn the pool process (e.g. `[\"python3\", \"pool.py\"]`).\n pub exec: Vec,\n /// Unix domain socket basename (under tmpdir) for IPC.\n pub socket: String,\n /// **Reserved.** Per-pool metadata. Future slots: `resource`\n /// (cpu/memory limits), `env` (environment variables),\n /// `startup_timeout`, `health_check`.\n #[serde(default)]\n pub metadata: Metadata,\n}\n\n// -- Commands -----------------------------------------------------------------\n\n/// Discriminator for the command kind. Closed enum so adding a new\n/// variant requires explicit code changes everywhere it's matched.\n#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)]\n#[serde(rename_all = \"lowercase\")]\npub enum CmdType {\n /// Dispatched to a language pool process via IPC.\n Remote,\n /// Evaluated inline by the nexus from an embedded expression tree.\n Pure,\n}\n\n/// One exported morloc function the user can invoke as a CLI subcommand.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct Command {\n /// CLI subcommand name (defaults to the morloc function name; can\n /// be overridden via a `--' name:` docstring directive).\n pub name: String,\n /// Discriminator: [`CmdType::Remote`] (dispatch to a pool) or\n /// [`CmdType::Pure`] (evaluate inline via the manifest's `expr`\n /// tree).\n #[serde(rename = \"type\")]\n pub cmd_type: CmdType,\n\n // -- Remote-only dispatch info ----------------------------------------\n /// Manifold ID -- the integer key under which the pool's dispatch\n /// table contains this function's entry. Remote commands only.\n #[serde(default)]\n pub mid: u32,\n /// Index into [`Manifest::pools`] for the primary pool that hosts\n /// this command's top-level function. Remote commands only.\n #[serde(default, rename = \"pool\")]\n pub pool_index: usize,\n /// Indices of every pool transitively required to execute this\n /// command (the primary pool plus any pools called as foreign\n /// functions from inside it). Remote commands only.\n #[serde(default)]\n pub needed_pools: Vec,\n\n // -- Common fields ----------------------------------------------------\n /// Description lines shown in CLI help. The first line is the\n /// summary used in subcommand listings.\n #[serde(default)]\n pub desc: Vec,\n /// Argument list, in declaration order. Each entry is a\n /// discriminated union -- see [`Arg`].\n #[serde(default)]\n pub args: Vec,\n /// Return-value descriptor. Always present, even for nullary\n /// returns (use a Nil schema in that case).\n #[serde(default, rename = \"return\")]\n pub ret: Return,\n /// **Reserved.** Command-level constraints -- invariants that span\n /// multiple arguments (e.g. `equal_length` of two list args).\n /// Empty in v2; populated when the constraint system rolls out.\n #[serde(default)]\n pub constraints: Vec,\n /// **Reserved.** Per-command metadata. Future slots: `effects`\n /// (declared I/O / network / filesystem effect set), `resource`\n /// (CPU/memory/time limits), `auth` (required capabilities),\n /// `version` (semantic version of the command's API),\n /// `deprecated` (migration notice).\n #[serde(default)]\n pub metadata: Metadata,\n\n // -- Pure-only evaluation info ----------------------------------------\n /// Embedded expression tree (NexusExpr JSON) for pure commands.\n /// Evaluated inline by the nexus instead of being dispatched to a\n /// pool. Absent on remote commands.\n #[serde(default)]\n pub expr: Option,\n\n // -- Command group ----------------------------------------------------\n /// Optional name of the command group this command belongs to. Used\n /// to organize subcommands in help output. The Haskell emitter now\n /// writes a real JSON null for absent groups (see\n /// `Morloc.CodeGenerator.Nexus.cmdGroupField`), so no custom\n /// deserializer is needed.\n #[serde(default)]\n pub group: Option,\n}\n\nimpl Command {\n pub fn is_pure(&self) -> bool {\n self.cmd_type == CmdType::Pure\n }\n}\n\n/// Return-value descriptor. Structurally similar to a typed [`Arg`]\n/// minus the CLI-specific fields (kind, metavar, quoted, short/long,\n/// default). Always present on every command.\n#[derive(Debug, Deserialize, Default)]\n#[allow(dead_code)]\npub struct Return {\n /// Morloc serialization schema string for the return type. The\n /// nexus uses this to deserialize the bytes coming back from the\n /// pool process.\n #[serde(default)]\n pub schema: String,\n /// User-facing type name as written in the morloc source (e.g.\n /// `\"Int\"`, `\"Config\"`, `\"[Int]\"`). Used in help output and error\n /// messages. JSON key is `type`; the Rust field is `type_desc`\n /// because `type` is a reserved keyword.\n #[serde(default, rename = \"type\")]\n pub type_desc: String,\n /// Description lines for the return value, parsed from `--' return:`\n /// docstrings.\n #[serde(default)]\n pub desc: Vec,\n /// **Reserved.** Constraints on the return value. Currently used\n /// only for `kind: record|object|table` on named return types;\n /// future constraints (min/max/regex/...) will live here.\n #[serde(default)]\n pub constraints: Vec,\n /// **Reserved.** Per-return metadata; same forward-compatibility\n /// rationale as the per-arg slot.\n #[serde(default)]\n pub metadata: Metadata,\n}\n\n// -- Constraints --------------------------------------------------------------\n\n/// A single constraint entry attached to an arg, return value, or\n/// command. Discriminated by `type`.\n///\n/// **Currently emitted constraint types**:\n///\n/// - `kind`: marks a named type as `record` / `object` / `table`. The\n/// `value` payload is the lowercased name. The CLI help renderer\n/// uses this to partition into Record Schemas / Table Schemas\n/// sections.\n///\n/// **Reserved constraint types** (defined shapes, not yet emitted by\n/// any compiler pass -- names and payloads chosen so the schema\n/// doesn't need to bump when the constraint feature lands):\n///\n/// - `min`, `max`: numeric bounds with `value: `.\n/// - `length`: `value: ` for a fixed length, or `{min, max}` for\n/// a bounded range.\n/// - `non_empty`: list/string must be non-empty (no payload).\n/// - `regex`: string must match `value: \"\"`.\n/// - `enum`: value must be one of `value: [...]`.\n/// - `unique`: list elements must be pairwise distinct (no payload).\n/// - `row_count`: fixed/bounded row count for `table`-tagged args.\n/// - `sorted`: `value: \"asc\"|\"desc\"`.\n///\n/// **Extensibility rules**:\n///\n/// 1. Unknown `type` values MUST be silently ignored by readers.\n/// 2. Adding a new constraint type does not bump the manifest version.\n/// 3. Changing the payload shape of an existing type DOES bump.\n/// 4. Constraints are unordered.\n/// 5. Multiple constraints of the same type on the same entity are\n/// allowed.\n#[derive(Debug, Deserialize, Clone)]\n#[allow(dead_code)]\npub struct Constraint {\n /// Constraint discriminator. JSON key is `type`; the Rust field is\n /// `ctype` because `type` is a reserved keyword.\n #[serde(rename = \"type\")]\n pub ctype: String,\n /// Constraint-specific payload. Shape depends on `ctype`. Some\n /// constraint types (`non_empty`, `unique`) carry no payload.\n #[serde(default)]\n pub value: Option,\n}\n\n// -- Arguments ----------------------------------------------------------------\n\n/// CLI argument variants. Each command's argument list is a sequence\n/// of these, in declaration order.\n///\n/// Three of the four variants (`Positional`, `Optional`, `Group`)\n/// carry type/schema/constraints information because they represent\n/// typed values that flow through to a pool. `Flag` is a pure boolean\n/// toggle with no associated type -- and therefore no `schema`,\n/// `type_desc`, or `constraints` slot.\n///\n/// `#[allow(dead_code)]` covers the `metadata` slots that are reserved\n/// for future use and not read by any current consumer.\n#[derive(Debug, Deserialize)]\n#[serde(tag = \"kind\")]\n#[allow(dead_code)]\npub enum Arg {\n /// A positional CLI argument.\n #[serde(rename = \"pos\")]\n Positional {\n /// Morloc serialization schema string. Used at dispatch time\n /// to parse the user's CLI input into a binary data packet.\n #[serde(default)]\n schema: Option,\n /// User-facing type name (e.g. `\"Int\"`, `\"Config\"`). The Rust\n /// field is `type_desc` because `type` is a reserved keyword.\n #[serde(default, rename = \"type\")]\n type_desc: Option,\n /// Display placeholder shown in help (e.g. `\"FILE\"`). None\n /// falls back to a generic `ARG` placeholder.\n #[serde(default)]\n metavar: Option,\n /// If true, the user's CLI value is JSON-wrapped before being\n /// passed to the pool. Used for `Str`-typed positionals\n /// declared with `--' literal: true`.\n #[serde(default)]\n quoted: bool,\n /// Description lines from `--' desc:` docstrings.\n #[serde(default)]\n desc: Vec,\n /// Per-argument enforceable invariants. Currently only the\n /// `kind` constraint is emitted (for named-type args). Future\n /// constraints (min/max/regex/length/...) will populate this.\n #[serde(default)]\n constraints: Vec,\n /// **Reserved.** Per-argument informational metadata. Always\n /// emitted as `{}` in v2; not yet read by any consumer.\n #[serde(default)]\n metadata: Metadata,\n },\n /// An optional CLI argument with a long/short option name.\n #[serde(rename = \"opt\")]\n Optional {\n /// Morloc serialization schema for the option's value type.\n #[serde(default)]\n schema: Option,\n /// User-facing type name. JSON key is `type`.\n #[serde(default, rename = \"type\")]\n type_desc: Option,\n /// Required display placeholder (e.g. `\"FILE\"`).\n #[serde(default)]\n metavar: Option,\n /// JSON-wrap flag for `Str`-typed literal options.\n #[serde(default)]\n quoted: bool,\n /// Single-character short option (e.g. `\"f\"` for `-f`).\n #[serde(default, rename = \"short\")]\n short_opt: Option,\n /// Long option name (e.g. `\"file\"` for `--file`).\n #[serde(default, rename = \"long\")]\n long_opt: Option,\n /// Default value used when the user does not pass the option.\n /// Always present (declared via `--' default:` docstring).\n #[serde(default, rename = \"default\")]\n default_val: Option,\n /// Description lines.\n #[serde(default)]\n desc: Vec,\n /// Per-argument constraints -- see [`Arg::Positional`]'s\n /// `constraints` field.\n #[serde(default)]\n constraints: Vec,\n /// **Reserved.** Per-argument metadata. Not read in v2.\n #[serde(default)]\n metadata: Metadata,\n },\n /// A pure boolean flag toggle. Carries no type, schema, or\n /// constraints because it has no payload -- flipping the flag\n /// produces the value `true` or `false`.\n #[serde(rename = \"flag\")]\n Flag {\n /// Single-character short option (e.g. `\"v\"` for `-v`).\n #[serde(default, rename = \"short\")]\n short_opt: Option,\n /// Long option name (e.g. `\"verbose\"` for `--verbose`).\n #[serde(default, rename = \"long\")]\n long_opt: Option,\n /// Long option name that flips the flag in the opposite\n /// direction (e.g. `\"no-verbose\"` for `--no-verbose`).\n #[serde(default)]\n long_rev: Option,\n /// Default value when the flag is not present on the CLI.\n /// String form: `\"true\"` or `\"false\"`.\n #[serde(default, rename = \"default\")]\n default_val: Option,\n /// Description lines.\n #[serde(default)]\n desc: Vec,\n /// **Reserved.** Per-flag metadata. Not read in v2.\n #[serde(default)]\n metadata: Metadata,\n },\n /// A record-typed argument that has been \"unrolled\" into a flat\n /// collection of CLI flags/options, one per record field. The\n /// group's top-level `schema` is the schema of the whole record;\n /// dispatch sends the assembled record to the pool, so individual\n /// entries never need their own schemas.\n #[serde(rename = \"grp\")]\n Group {\n /// Morloc schema for the whole record (a `Map` schema).\n #[serde(default)]\n schema: Option,\n /// User-facing record type name (e.g. `\"SysConfig\"`).\n #[serde(default, rename = \"type\")]\n type_desc: Option,\n /// Display placeholder for the group as a whole.\n #[serde(default)]\n metavar: Option,\n /// Description lines for the group.\n #[serde(default)]\n desc: Vec,\n /// Optional CLI option that accepts the entire record as a\n /// single JSON value (e.g. `--sys-config '{...}'`).\n #[serde(default)]\n group_opt: Option,\n /// Flattened per-field options/flags. Each entry's `arg` is a\n /// nested [`Arg`] (typically `Optional` or `Flag`) that has\n /// no schema of its own -- only the group's top-level schema\n /// matters at dispatch time.\n #[serde(default)]\n entries: Vec,\n /// Per-group constraints. Currently the `kind` constraint\n /// (almost always `record`) is emitted.\n #[serde(default)]\n constraints: Vec,\n /// **Reserved.** Per-group metadata. Not read in v2.\n #[serde(default)]\n metadata: Metadata,\n },\n}\n\n/// Nested CLI option that accepts the entire record (associated with\n/// an [`Arg::Group`]) as a single JSON value.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct GroupOpt {\n /// Short option char that accepts the whole record as one JSON value.\n #[serde(default, rename = \"short\")]\n pub short_opt: Option,\n /// Long option name that accepts the whole record as one JSON value.\n #[serde(default, rename = \"long\")]\n pub long_opt: Option,\n}\n\n/// One entry inside an [`Arg::Group`] -- pairs a record field name\n/// with the CLI flag/option that backs it.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct GroupEntry {\n /// Record field name.\n pub key: String,\n /// CLI binding for this field. Always a [`Arg::Optional`] or\n /// [`Arg::Flag`] in practice; never carries its own schema (the\n /// containing group's schema covers all fields).\n pub arg: Arg,\n}\n\n/// CLI command group -- purely organizational metadata used to bucket\n/// related subcommands together in the help output.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct CmdGroup {\n /// Group name; matches `Command::group` on member commands.\n pub name: String,\n /// Group description lines for the help output.\n #[serde(default)]\n pub desc: Vec,\n /// **Reserved.** Per-group metadata. Not read in v2.\n #[serde(default)]\n pub metadata: Metadata,\n}\n\n/// Daemon-mode service configuration. Present only when the program\n/// is configured to run as a long-lived service rather than a one-shot\n/// CLI invocation.\n#[derive(Debug, Deserialize)]\n#[allow(dead_code)]\npub struct Service {\n /// Transport type: typically `\"http\"`, `\"tcp\"`, or `\"unix\"`.\n #[serde(rename = \"type\")]\n pub service_type: Option,\n /// Listening host address (TCP/HTTP).\n pub host: Option,\n /// Listening port (TCP/HTTP).\n pub port: Option,\n /// Unix socket path (when `service_type` is `\"unix\"`).\n pub socket: Option,\n /// **Reserved.** Per-service metadata. Not read in v2.\n #[serde(default)]\n pub metadata: Metadata,\n}\n\n// -- I/O ----------------------------------------------------------------------\n\n/// Read the manifest payload from a built-nexus wrapper script. The\n/// nexus binary is wrapped in a shell script that contains a\n/// `### MANIFEST ###` marker followed by the JSON blob. Plain JSON\n/// files (no shebang) are returned as-is.\npub fn read_manifest_payload(path: &str) -> Result {\n let content = std::fs::read_to_string(path)\n .map_err(|e| format!(\"Cannot open manifest file '{}': {}\", path, e))?;\n\n if content.starts_with(\"#!\") {\n if let Some(pos) = content.find(\"### MANIFEST ###\") {\n let after_marker = &content[pos..];\n let payload_start = after_marker\n .find('\\n')\n .map(|i| pos + i + 1)\n .unwrap_or(content.len());\n Ok(content[payload_start..].to_string())\n } else {\n Err(\"No ### MANIFEST ### marker found in wrapper script\".into())\n }\n } else {\n Ok(content)\n }\n}\n\n/// Parse a manifest JSON payload into a [`Manifest`]. Performs a\n/// staleness check on `build.morloc_version` against this crate's own\n/// `CARGO_PKG_VERSION` (which is intentionally pinned to match the\n/// morloc compiler version). Mismatched versions return a clean\n/// \"rebuild with the current compiler\" error rather than silently\n/// misinterpreting the manifest.\npub fn parse_manifest(payload: &str) -> Result {\n let m: Manifest = serde_json::from_str(payload)\n .map_err(|e| format!(\"Failed to parse manifest JSON: {}\", e))?;\n let crate_version = env!(\"CARGO_PKG_VERSION\");\n if m.build.morloc_version != crate_version {\n return Err(format!(\n \"manifest built with morloc {}, runtime is {}; rebuild with the current compiler\",\n m.build.morloc_version, crate_version\n ));\n }\n Ok(m)\n}\n\n// -- Arg accessors ------------------------------------------------------------\n//\n// Variant-agnostic helpers for the fields that exist on multiple Arg\n// variants. Callers in the nexus and runtime use these instead of\n// pattern-matching at every site.\n\nimpl Arg {\n /// Single-character short option (e.g. `'f'` for `-f`). Returns\n /// None for positional and group args.\n pub fn short_opt_char(&self) -> Option {\n let s = match self {\n Arg::Optional { short_opt, .. } => short_opt.as_deref(),\n Arg::Flag { short_opt, .. } => short_opt.as_deref(),\n _ => None,\n };\n s.and_then(|s| s.chars().next())\n }\n\n /// Long option name (e.g. `\"verbose\"` for `--verbose`). Returns\n /// None for positional and group args.\n pub fn long_opt_str(&self) -> Option<&str> {\n match self {\n Arg::Optional { long_opt, .. } => long_opt.as_deref(),\n Arg::Flag { long_opt, .. } => long_opt.as_deref(),\n _ => None,\n }\n }\n\n /// True if this arg is a boolean flag toggle.\n pub fn is_flag(&self) -> bool {\n matches!(self, Arg::Flag { .. })\n }\n\n /// True if the user's CLI value should be JSON-wrapped before\n /// being passed to the pool. Used for `Str`-typed arguments\n /// declared with `--' literal: true`. Always false for flags and\n /// groups.\n pub fn is_quoted(&self) -> bool {\n match self {\n Arg::Positional { quoted, .. } | Arg::Optional { quoted, .. } => *quoted,\n _ => false,\n }\n }\n\n /// Default CLI value when the user does not pass the argument.\n /// Returns None for positional args (which are always required)\n /// and groups.\n pub fn default_val(&self) -> Option<&str> {\n match self {\n Arg::Optional { default_val, .. } => default_val.as_deref(),\n Arg::Flag { default_val, .. } => default_val.as_deref(),\n _ => None,\n }\n }\n\n /// CLI display placeholder (e.g. `\"FILE\"`, `\"INT\"`). None for\n /// flags and for positional args without an explicit metavar.\n pub fn metavar_str(&self) -> Option<&str> {\n match self {\n Arg::Positional { metavar, .. } => metavar.as_deref(),\n Arg::Optional { metavar, .. } => metavar.as_deref(),\n Arg::Group { metavar, .. } => metavar.as_deref(),\n _ => None,\n }\n }\n\n /// Description lines from the source-level docstring. Always\n /// available regardless of variant.\n pub fn desc_lines(&self) -> &[String] {\n match self {\n Arg::Positional { desc, .. }\n | Arg::Optional { desc, .. }\n | Arg::Flag { desc, .. }\n | Arg::Group { desc, .. } => desc,\n }\n }\n\n /// User-facing type name for typed args (e.g. `\"Int\"`,\n /// `\"Config\"`). Returns None for flags, which carry no type.\n pub fn type_desc_str(&self) -> Option<&str> {\n match self {\n Arg::Positional { type_desc, .. }\n | Arg::Optional { type_desc, .. }\n | Arg::Group { type_desc, .. } => type_desc.as_deref(),\n Arg::Flag { .. } => None,\n }\n }\n\n /// Morloc serialization schema string for typed args. Returns\n /// None for flags. The schema drives both dispatch (how to encode\n /// the value into a packet) and help rendering (how to extract\n /// record field layouts for the Record/Table Schemas sections).\n pub fn schema_str(&self) -> Option<&str> {\n match self {\n Arg::Positional { schema, .. }\n | Arg::Optional { schema, .. }\n | Arg::Group { schema, .. } => schema.as_deref(),\n Arg::Flag { .. } => None,\n }\n }\n\n /// All constraints attached to this arg. Empty for flags. The\n /// caller is responsible for filtering by constraint type and\n /// silently ignoring unknown types (per the extensibility rules\n /// on [`Constraint`]).\n pub fn constraints(&self) -> &[Constraint] {\n match self {\n Arg::Positional { constraints, .. }\n | Arg::Optional { constraints, .. }\n | Arg::Group { constraints, .. } => constraints,\n Arg::Flag { .. } => &[],\n }\n }\n\n /// Convenience accessor: extract the `value` of the `kind`\n /// constraint as a string slice. Returns the lowercased\n /// `\"record\"`, `\"object\"`, or `\"table\"` for named-type args.\n /// None for everything else (including untagged primitive types).\n pub fn kind_constraint(&self) -> Option<&str> {\n self.constraints()\n .iter()\n .find(|c| c.ctype == \"kind\")\n .and_then(|c| c.value.as_ref().and_then(|v| v.as_str()))\n }\n}\n\n// -- Tests --------------------------------------------------------------------\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n /// Wrap a v2 command body with the required top-level fields so\n /// each test fixture stays compact.\n fn wrap(commands_json: &str) -> String {\n let v = env!(\"CARGO_PKG_VERSION\");\n format!(\n r#\"{{\n \"name\": \"main\",\n \"build\": {{\n \"path\": \"/tmp/test\",\n \"time\": 0,\n \"morloc_version\": \"{}\"\n }},\n \"pools\": [\n {{\"lang\": \"py\", \"exec\": [\"python3\", \"pool.py\"], \"socket\": \"pipe-py\", \"metadata\": {{}}}}\n ],\n \"commands\": {},\n \"groups\": [],\n \"metadata\": {{}}\n }}\"#,\n v, commands_json\n )\n }\n\n #[test]\n fn test_parse_simple_manifest() {\n let json = wrap(\n r#\"[\n {\n \"name\": \"f\",\n \"type\": \"remote\",\n \"mid\": 1,\n \"pool\": 0,\n \"needed_pools\": [0],\n \"desc\": [],\n \"args\": [\n {\n \"kind\": \"pos\",\n \"schema\": \"s\",\n \"type\": \"Str\",\n \"metavar\": null,\n \"quoted\": false,\n \"desc\": [],\n \"constraints\": [],\n \"metadata\": {}\n }\n ],\n \"return\": {\n \"schema\": \"s\",\n \"type\": \"Str\",\n \"desc\": [],\n \"constraints\": [],\n \"metadata\": {}\n },\n \"constraints\": [],\n \"metadata\": {},\n \"group\": null\n }\n ]\"#,\n );\n let m = parse_manifest(&json).unwrap();\n assert_eq!(m.pools.len(), 1);\n assert_eq!(m.pools[0].lang, \"py\");\n assert_eq!(m.commands.len(), 1);\n assert_eq!(m.commands[0].name, \"f\");\n assert!(!m.commands[0].is_pure());\n assert_eq!(m.commands[0].mid, 1);\n assert!(m.commands[0].group.is_none());\n assert_eq!(m.commands[0].args.len(), 1);\n assert_eq!(m.commands[0].args[0].schema_str(), Some(\"s\"));\n assert_eq!(m.commands[0].args[0].type_desc_str(), Some(\"Str\"));\n assert_eq!(m.commands[0].ret.schema, \"s\");\n assert_eq!(m.commands[0].ret.type_desc, \"Str\");\n }\n\n #[test]\n fn test_parse_pure_command() {\n let json = wrap(\n r#\"[\n {\n \"name\": \"greet\",\n \"type\": \"pure\",\n \"desc\": [\"Say hello\"],\n \"args\": [\n {\n \"kind\": \"pos\",\n \"schema\": \"s\",\n \"type\": \"Str\",\n \"metavar\": \"NAME\",\n \"quoted\": true,\n \"desc\": [\"name\"],\n \"constraints\": [],\n \"metadata\": {}\n }\n ],\n \"return\": {\n \"schema\": \"s\",\n \"type\": \"Str\",\n \"desc\": [],\n \"constraints\": [],\n \"metadata\": {}\n },\n \"expr\": {\"tag\": \"lit\", \"schema\": \"s\", \"lit_type\": \"str\", \"value\": \"hello\"},\n \"constraints\": [],\n \"metadata\": {},\n \"group\": null\n }\n ]\"#,\n );\n let m = parse_manifest(&json).unwrap();\n assert!(m.commands[0].is_pure());\n assert!(m.commands[0].expr.is_some());\n }\n\n #[test]\n fn test_parse_kind_constraint() {\n let json = wrap(\n r#\"[\n {\n \"name\": \"process\",\n \"type\": \"remote\",\n \"mid\": 1,\n \"pool\": 0,\n \"needed_pools\": [0],\n \"desc\": [],\n \"args\": [\n {\n \"kind\": \"pos\",\n \"schema\": \"m24nameas3ageai4\",\n \"type\": \"People\",\n \"metavar\": null,\n \"quoted\": false,\n \"desc\": [],\n \"constraints\": [\n {\"type\": \"kind\", \"value\": \"table\"}\n ],\n \"metadata\": {}\n }\n ],\n \"return\": {\n \"schema\": \"i4\",\n \"type\": \"Int\",\n \"desc\": [],\n \"constraints\": [],\n \"metadata\": {}\n },\n \"constraints\": [],\n \"metadata\": {},\n \"group\": null\n }\n ]\"#,\n );\n let m = parse_manifest(&json).unwrap();\n assert_eq!(m.commands[0].args[0].kind_constraint(), Some(\"table\"));\n }\n\n #[test]\n fn test_version_mismatch_rejected() {\n let json = r#\"{\n \"name\": \"main\",\n \"build\": {\"path\": \"/tmp/x\", \"time\": 0, \"morloc_version\": \"0.0.1-stale\"},\n \"pools\": [],\n \"commands\": [],\n \"groups\": [],\n \"metadata\": {}\n }\"#;\n let err = parse_manifest(json).unwrap_err();\n assert!(\n err.contains(\"rebuild with the current compiler\"),\n \"got: {}\",\n err\n );\n }\n}\n" }, { "path": "data/rust/morloc-nexus/Cargo.toml", "content": "[package]\nname = \"morloc-nexus\"\nversion = \"0.81.0\"\nedition = \"2021\"\ndescription = \"Morloc nexus: CLI dispatcher for multi-language pool orchestration\"\n# Version is intentionally synchronized with the morloc compiler version\n# (see ../../../package.yaml). Manifests record the morloc_version that\n# built them; the nexus rejects manifests whose version differs from its\n# own CARGO_PKG_VERSION at parse time. Bumping the morloc compiler\n# version requires bumping this version in lockstep.\n\n[[bin]]\nname = \"morloc-nexus\"\npath = \"src/main.rs\"\n\n[dependencies]\nmorloc-manifest = { path = \"../morloc-manifest\" }\nmorloc-runtime = { path = \"../morloc-runtime\" }\nlibc = { workspace = true }\nserde = { workspace = true }\nserde_json = { workspace = true }\nnix = { workspace = true }\nclap = { workspace = true }\nthiserror = { workspace = true }\n" }, { "path": "data/rust/morloc-nexus/build.rs", "content": "fn main() {\n // Use MORLOC_HOME at build time if set, else fall back to $HOME default.\n // This is only for the compile-time link search path.\n let morloc_lib = std::env::var(\"MORLOC_HOME\")\n .map(|h| format!(\"{}/lib\", h))\n .unwrap_or_else(|_| {\n format!(\n \"{}/.local/share/morloc/lib\",\n std::env::var(\"HOME\").unwrap_or_else(|_| \"/root\".into())\n )\n });\n println!(\"cargo:rustc-link-search=native={}\", morloc_lib);\n println!(\"cargo:rustc-link-lib=dylib=morloc\");\n\n // Embed $ORIGIN-relative rpaths so the nexus finds libmorloc.so\n // regardless of install location:\n // $ORIGIN/../lib covers /opt/morloc/bin -> /opt/morloc/lib\n // $ORIGIN/../share/morloc/lib covers ~/.local/bin -> ~/.local/share/morloc/lib\n println!(\"cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/../lib\");\n println!(\"cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/../share/morloc/lib\");\n\n // The morloc compiler version is sourced from CARGO_PKG_VERSION\n // (this crate's Cargo.toml), which is intentionally kept in\n // lockstep with the morloc Haskell package.yaml. No build-time\n // extraction needed -- Cargo guarantees CARGO_PKG_VERSION is set\n // and rebuilds when Cargo.toml changes.\n}\n" }, { "path": "data/rust/morloc-nexus/src/dispatch.rs", "content": "//! Command dispatch: CLI argument parsing and routing to pools.\n//!\n//! Replaces the dispatch_command, dispatch, run_command, and run_pure_command\n//! functions from nexus.c. Uses the C libmorloc for packet construction and\n//! serialization until Phase 2/3 replaces those.\n//!\n//! For Phase 1, the nexus links against the C libmorloc.so for:\n//! - make_call_packet_from_cli, parse_cli_data_argument\n//! - send_and_receive_over_socket\n//! - pack_with_schema, print_voidstar, etc.\n//! - morloc_eval for pure commands\n\nuse crate::help;\nuse crate::manifest::{Arg, Command, Manifest};\nuse crate::process::{self, PoolSocket};\n\n/// Output format enum.\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum OutputFormat {\n Json,\n MessagePack,\n VoidStar,\n Packet,\n}\n\n/// Nexus configuration parsed from CLI options.\n#[derive(Debug, Clone)]\npub struct NexusConfig {\n pub help_flag: bool,\n pub print_flag: bool,\n pub packet_path: Option,\n pub socket_base: Option,\n pub output_path: Option,\n pub output_format: OutputFormat,\n pub daemon_flag: bool,\n pub router_flag: bool,\n pub unix_socket_path: Option,\n pub tcp_port: Option,\n pub http_port: Option,\n pub fdb_path: Option,\n pub eval_timeout: i32,\n}\n\nimpl Default for NexusConfig {\n fn default() -> Self {\n NexusConfig {\n help_flag: false,\n print_flag: false,\n packet_path: None,\n socket_base: None,\n output_path: None,\n output_format: OutputFormat::Json,\n daemon_flag: false,\n router_flag: false,\n unix_socket_path: None,\n tcp_port: None,\n http_port: None,\n fdb_path: None,\n eval_timeout: 30,\n }\n }\n}\n\n/// Emit a uniform error when pool communication fails, then exit.\n///\n/// The pool's stderr was inherited by the nexus, so any traceback the pool\n/// printed before dying is already on the user's terminal. This helper\n/// reports the communication error plus the pool's exit status (if it has\n/// been reaped) so the user can correlate the two.\n///\n/// Race condition: the pool process may still be writing its error output\n/// (traceback, panic message, etc.) to stderr when the nexus detects the\n/// broken connection. If we call clean_exit immediately, it sends SIGTERM\n/// to the pool process group, which can kill the pool before its stderr\n/// buffer is flushed. We insert a brief drain window to let any in-flight\n/// stderr from the dying pool reach the terminal before tearing everything\n/// down. This is best-effort: a pool killed by SIGKILL (OOM killer, etc.)\n/// won't have pending output, and a pool stuck in a blocking syscall won't\n/// flush within the window. But for the common case of a Python exception\n/// traceback, this is enough.\nfn die_with_pool_error(\n socket: &PoolSocket,\n pool_index: usize,\n context: &str,\n comm_err: &dyn std::fmt::Display,\n) -> ! {\n // Give the dying pool process time to flush its stderr/stdout before\n // we tear down the process group. Without this, a Python traceback or\n // error message that is still in a pipe buffer gets lost when\n // clean_exit sends SIGTERM/SIGKILL to the pool's process group.\n std::thread::sleep(std::time::Duration::from_millis(100));\n\n eprintln!(\"Error: {}: {}\", context, comm_err);\n if let Some(info) = process::pool_death_info(pool_index) {\n eprintln!(\"Pool '{}' {}\", socket.lang, info);\n }\n process::clean_exit(1);\n}\n\n/// Parse nexus-level options from argv. Returns the index of the first\n/// non-option argument (the manifest path or subcommand).\npub fn parse_nexus_options(args: &[String], config: &mut NexusConfig) -> usize {\n let mut i = 1; // skip argv[0]\n while i < args.len() {\n let arg = &args[i];\n match arg.as_str() {\n \"-h\" | \"--help\" => {\n config.help_flag = true;\n i += 1;\n }\n \"-p\" | \"--print\" => {\n config.print_flag = true;\n i += 1;\n }\n \"-c\" | \"--call-packet\" => {\n i += 1;\n if i < args.len() {\n config.packet_path = Some(args[i].clone());\n i += 1;\n }\n }\n \"-s\" | \"--socket-base\" => {\n i += 1;\n if i < args.len() {\n config.socket_base = Some(args[i].clone());\n i += 1;\n }\n }\n \"-o\" | \"--output-file\" => {\n i += 1;\n if i < args.len() {\n config.output_path = Some(args[i].clone());\n i += 1;\n }\n }\n \"-f\" | \"--output-form\" => {\n i += 1;\n if i < args.len() {\n config.output_format = parse_output_format(&args[i]);\n i += 1;\n }\n }\n \"--daemon\" => {\n config.daemon_flag = true;\n i += 1;\n }\n \"--router\" => {\n config.router_flag = true;\n i += 1;\n }\n \"--socket\" => {\n i += 1;\n if i < args.len() {\n config.unix_socket_path = Some(args[i].clone());\n i += 1;\n }\n }\n \"--port\" => {\n i += 1;\n if i < args.len() {\n config.tcp_port = args[i].parse().ok();\n i += 1;\n }\n }\n \"--http-port\" => {\n i += 1;\n if i < args.len() {\n config.http_port = args[i].parse().ok();\n i += 1;\n }\n }\n \"--fdb\" => {\n i += 1;\n if i < args.len() {\n config.fdb_path = Some(args[i].clone());\n i += 1;\n }\n }\n \"--eval-timeout\" => {\n i += 1;\n if i < args.len() {\n config.eval_timeout = args[i].parse().unwrap_or(30);\n i += 1;\n }\n }\n _ => {\n // Handle --key=value forms\n if let Some(val) = arg.strip_prefix(\"--socket=\") {\n config.unix_socket_path = Some(val.to_string());\n i += 1;\n } else if let Some(val) = arg.strip_prefix(\"--port=\") {\n config.tcp_port = val.parse().ok();\n i += 1;\n } else if let Some(val) = arg.strip_prefix(\"--http-port=\") {\n config.http_port = val.parse().ok();\n i += 1;\n } else if let Some(val) = arg.strip_prefix(\"--fdb=\") {\n config.fdb_path = Some(val.to_string());\n i += 1;\n } else if let Some(val) = arg.strip_prefix(\"--eval-timeout=\") {\n config.eval_timeout = val.parse().unwrap_or(30);\n i += 1;\n } else {\n // Not a nexus option - stop parsing\n break;\n }\n }\n }\n }\n i\n}\n\n/// Extract daemon/server long options from argv in single-command mode.\n/// Removes matched options from the args vector.\npub fn extract_global_options(args: &mut Vec, config: &mut NexusConfig) {\n let mut i = 1;\n while i < args.len() {\n if args[i] == \"--\" {\n break;\n }\n\n let mut matched = false;\n let mut consumed = 1;\n\n match args[i].as_str() {\n \"--daemon\" => {\n config.daemon_flag = true;\n matched = true;\n }\n \"--socket\" if i + 1 < args.len() => {\n config.unix_socket_path = Some(args[i + 1].clone());\n consumed = 2;\n matched = true;\n }\n \"--port\" if i + 1 < args.len() => {\n config.tcp_port = args[i + 1].parse().ok();\n consumed = 2;\n matched = true;\n }\n \"--http-port\" if i + 1 < args.len() => {\n config.http_port = args[i + 1].parse().ok();\n consumed = 2;\n matched = true;\n }\n \"--fdb\" if i + 1 < args.len() => {\n config.fdb_path = Some(args[i + 1].clone());\n consumed = 2;\n matched = true;\n }\n \"--eval-timeout\" if i + 1 < args.len() => {\n config.eval_timeout = args[i + 1].parse().unwrap_or(30);\n consumed = 2;\n matched = true;\n }\n _ => {\n // Check --key=value forms\n if let Some(val) = args[i].strip_prefix(\"--socket=\") {\n config.unix_socket_path = Some(val.to_string());\n matched = true;\n } else if let Some(val) = args[i].strip_prefix(\"--port=\") {\n config.tcp_port = val.parse().ok();\n matched = true;\n } else if let Some(val) = args[i].strip_prefix(\"--http-port=\") {\n config.http_port = val.parse().ok();\n matched = true;\n } else if let Some(val) = args[i].strip_prefix(\"--fdb=\") {\n config.fdb_path = Some(val.to_string());\n matched = true;\n } else if let Some(val) = args[i].strip_prefix(\"--eval-timeout=\") {\n config.eval_timeout = val.parse().unwrap_or(30);\n matched = true;\n }\n }\n }\n\n if matched {\n for _ in 0..consumed {\n args.remove(i);\n }\n } else {\n i += 1;\n }\n }\n}\n\nfn parse_output_format(s: &str) -> OutputFormat {\n match s {\n \"json\" => OutputFormat::Json,\n \"mpk\" => OutputFormat::MessagePack,\n \"voidstar\" => OutputFormat::VoidStar,\n \"packet\" => OutputFormat::Packet,\n _ => {\n eprintln!(\"Invalid output format: {}\", s);\n std::process::exit(1);\n }\n }\n}\n\n/// Wrap a string in JSON quotes (for literal string arguments).\npub fn quoted(s: &str) -> String {\n // JSON-escape the string\n let escaped = serde_json::to_string(s).unwrap_or_else(|_| format!(\"\\\"{}\\\"\", s));\n escaped\n}\n\n/// Main dispatch entry point. Routes to the correct command based on argv.\npub fn dispatch(\n args: &[String],\n arg_start: usize,\n _shm_basename: &str,\n config: &NexusConfig,\n manifest: &Manifest,\n sockets: &mut [PoolSocket],\n prog_name: &str,\n) {\n if arg_start >= args.len() {\n help::print_usage(prog_name, manifest);\n }\n\n let cmd_name = &args[arg_start];\n let next = arg_start + 1;\n\n // Check if it matches a group name\n for grp in &manifest.groups {\n if grp.name == *cmd_name {\n if next >= args.len() {\n help::print_group_usage(prog_name, manifest, cmd_name);\n }\n let subcmd = &args[next];\n if subcmd == \"-h\" || subcmd == \"--help\" {\n help::print_group_usage(prog_name, manifest, cmd_name);\n }\n // Find command within this group\n for cmd in &manifest.commands {\n if cmd.group.as_deref() == Some(cmd_name.as_str()) && cmd.name == *subcmd {\n dispatch_command(args, next + 1, config, manifest, cmd, sockets, prog_name);\n return;\n }\n }\n eprintln!(\"Unrecognized command '{}' in group '{}'\", subcmd, cmd_name);\n process::clean_exit(1);\n }\n }\n\n // Try ungrouped commands\n for cmd in &manifest.commands {\n if cmd.name == *cmd_name && cmd.group.is_none() {\n dispatch_command(args, next, config, manifest, cmd, sockets, prog_name);\n return;\n }\n }\n\n eprintln!(\"Unrecognized command '{}'\", cmd_name);\n process::clean_exit(1);\n}\n\n/// Dispatch a single command: parse its args, start needed daemons, execute.\npub fn dispatch_command(\n args: &[String],\n arg_start: usize,\n config: &NexusConfig,\n manifest: &Manifest,\n cmd: &Command,\n sockets: &mut [PoolSocket],\n prog_name: &str,\n) {\n let single_cmd = manifest.commands.len() == 1 && manifest.groups.is_empty();\n\n // Parse command-specific arguments\n let (parsed_args, _remaining_start) =\n parse_command_args(args, arg_start, cmd, config, single_cmd, prog_name);\n\n // Start daemons for remote commands\n if !cmd.is_pure() {\n if let Err(e) = process::start_daemons(sockets, &cmd.needed_pools) {\n eprintln!(\"Error: {}\", e);\n process::clean_exit(1);\n }\n }\n\n // Execute the command\n if cmd.is_pure() {\n run_pure_command(cmd, &parsed_args, config);\n } else {\n run_remote_command(cmd, &parsed_args, sockets, config);\n }\n}\n\n/// Parsed CLI argument value for a manifest arg slot.\n#[derive(Debug)]\npub enum ArgValue {\n /// A value string (already quoted if needed).\n Value(String),\n /// Null/absent value.\n Null,\n /// Group argument with per-entry values.\n Group {\n grp_val: Option,\n fields: Vec>,\n defaults: Vec>,\n },\n}\n\n/// Parse command-specific arguments from argv.\nfn parse_command_args(\n args: &[String],\n pos: usize,\n cmd: &Command,\n _config: &NexusConfig,\n single_cmd: bool,\n prog_name: &str,\n) -> (Vec, usize) {\n let mut parsed = Vec::with_capacity(cmd.args.len());\n // Simple option tracking: collect all --opt=val and -o val\n let mut opt_values: std::collections::HashMap = std::collections::HashMap::new();\n let mut flag_values: std::collections::HashMap = std::collections::HashMap::new();\n let mut positional_idx = 0;\n let mut positionals: Vec = Vec::new();\n\n // First pass: separate options from positionals\n let mut i = pos;\n while i < args.len() {\n let arg = &args[i];\n if arg == \"--\" {\n i += 1;\n // Everything after -- is positional\n while i < args.len() {\n positionals.push(args[i].clone());\n i += 1;\n }\n break;\n }\n if arg == \"-h\" || arg == \"--help\" {\n if single_cmd {\n help::print_command_help_single(prog_name, cmd);\n } else {\n help::print_command_help(prog_name, cmd);\n }\n }\n if arg.starts_with(\"--\") && arg.len() > 2 {\n // Long option\n if let Some(eq_pos) = arg.find('=') {\n let key = &arg[2..eq_pos];\n let val = &arg[eq_pos + 1..];\n opt_values.insert(key.to_string(), val.to_string());\n i += 1;\n } else {\n let key = &arg[2..];\n // Check if it's a flag\n if is_flag_opt(cmd, key) {\n flag_values.insert(key.to_string(), flag_forward_value(cmd, key));\n i += 1;\n } else if is_rev_flag(cmd, key) {\n if let Some(orig) = find_flag_by_rev(cmd, key) {\n flag_values.insert(orig, flag_reverse_value_by_rev(cmd, key));\n }\n i += 1;\n } else if i + 1 < args.len() {\n opt_values.insert(key.to_string(), args[i + 1].clone());\n i += 2;\n } else {\n eprintln!(\"Error: option --{} requires a value\", key);\n process::clean_exit(1);\n }\n }\n } else if arg.starts_with('-') && arg.len() == 2 && arg.as_bytes()[1].is_ascii_alphabetic() {\n let ch = arg.chars().nth(1).unwrap();\n if is_short_flag(cmd, ch) {\n flag_values.insert(\n short_to_long(cmd, ch).unwrap_or_else(|| ch.to_string()),\n flag_forward_value_by_short(cmd, ch),\n );\n i += 1;\n } else if i + 1 < args.len() {\n opt_values.insert(\n short_to_long(cmd, ch).unwrap_or_else(|| ch.to_string()),\n args[i + 1].clone(),\n );\n i += 2;\n } else {\n eprintln!(\"Error: option -{} requires a value\", ch);\n process::clean_exit(1);\n }\n } else {\n positionals.push(arg.clone());\n i += 1;\n }\n }\n\n // Second pass: build ArgValue for each manifest arg\n for arg_def in &cmd.args {\n match arg_def {\n Arg::Positional { quoted, .. } => {\n if positional_idx < positionals.len() {\n let val = if *quoted {\n self::quoted(&positionals[positional_idx])\n } else {\n positionals[positional_idx].clone()\n };\n parsed.push(ArgValue::Value(val));\n positional_idx += 1;\n } else {\n eprintln!(\"Error: too few positional arguments\");\n process::clean_exit(1);\n }\n }\n Arg::Optional {\n long_opt,\n short_opt,\n default_val,\n quoted,\n ..\n } => {\n let key = long_opt\n .as_deref()\n .or_else(|| short_opt.as_deref())\n .unwrap_or(\"\");\n let user_val = opt_values.get(key);\n if let Some(val) = user_val {\n let v = if *quoted { self::quoted(val) } else { val.clone() };\n parsed.push(ArgValue::Value(v));\n } else if let Some(def) = default_val {\n parsed.push(ArgValue::Value(def.clone()));\n } else {\n parsed.push(ArgValue::Null);\n }\n }\n Arg::Flag {\n long_opt,\n default_val,\n ..\n } => {\n let key = long_opt.as_deref().unwrap_or(\"\");\n if let Some(val) = flag_values.get(key) {\n parsed.push(ArgValue::Value(val.clone()));\n } else {\n parsed.push(ArgValue::Value(\n default_val.as_deref().unwrap_or(\"false\").to_string(),\n ));\n }\n }\n Arg::Group {\n entries,\n group_opt,\n ..\n } => {\n let grp_val = group_opt.as_ref().and_then(|go| {\n go.long_opt\n .as_deref()\n .and_then(|k| opt_values.get(k))\n .cloned()\n });\n let mut fields = Vec::new();\n let mut defaults = Vec::new();\n for entry in entries {\n // Look up by long option name or short option character\n let long_key = entry.arg.long_opt_str().unwrap_or(\"\");\n let short_key = entry.arg.short_opt_char()\n .map(|c| c.to_string())\n .unwrap_or_default();\n let user = opt_values\n .get(long_key)\n .or_else(|| opt_values.get(&short_key))\n .or_else(|| flag_values.get(long_key))\n .or_else(|| flag_values.get(&short_key))\n .map(|v| {\n if entry.arg.is_quoted() {\n self::quoted(v)\n } else {\n v.clone()\n }\n });\n fields.push(user);\n defaults.push(entry.arg.default_val().map(|s| s.to_string()));\n }\n parsed.push(ArgValue::Group {\n grp_val,\n fields,\n defaults,\n });\n }\n }\n }\n\n if positional_idx < positionals.len() {\n eprintln!(\"Error: too many positional arguments given\");\n process::clean_exit(1);\n }\n\n (parsed, i)\n}\n\n// -- Command execution ------------------------------------------------------\n\n/// Execute a remote command by sending a call packet to the pool.\nfn run_remote_command(\n cmd: &Command,\n args: &[ArgValue],\n sockets: &[PoolSocket],\n config: &NexusConfig,\n) {\n use morloc_runtime::packet;\n use morloc_runtime::schema::{parse_schema, SerialType};\n use std::io::{Read, Write};\n use std::os::unix::net::UnixStream;\n\n // C library functions from libmorloc.so\n extern \"C\" {\n fn parse_cli_data_argument(\n dest: *mut u8, arg: *const std::ffi::c_void,\n schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> *mut u8;\n fn initialize_positional(value: *mut std::ffi::c_char) -> *mut std::ffi::c_void;\n fn free_argument_t(arg: *mut std::ffi::c_void);\n fn morloc_packet_size(packet: *const u8, errmsg: *mut *mut std::ffi::c_char) -> usize;\n fn make_morloc_local_call_packet(\n midx: u32, arg_packets: *const *const u8, nargs: usize,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> *mut u8;\n fn get_morloc_data_packet_value(\n data: *const u8, schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> *mut u8;\n }\n\n let socket = &sockets[cmd.pool_index];\n\n // Parse return schema\n let return_schema = match parse_schema(&cmd.ret.schema) {\n Ok(s) => s,\n Err(e) => {\n eprintln!(\"Error: failed to parse return schema '{}': {}\", cmd.ret.schema, e);\n process::clean_exit(1);\n }\n };\n\n // The parsed `args` list and `cmd.args` are index-aligned 1:1 in\n // declaration order: parse_command_args pushes one ArgValue for\n // EVERY arg (including flags). The Haskell compiler emits one\n // schema per arg position too. Walk both lists in lockstep; for\n // flags, schema_str() returns None and the flag's ArgValue is\n // already a ready-to-send \"true\"/\"false\" string that doesn't need\n // packet conversion -- but the original v1 dispatch path still\n // ran flags through parse_cli_data_argument with the flag's bool\n // schema, so we mirror that to keep the wire format consistent.\n let mut arg_packets: Vec> = Vec::new();\n for (i, (arg_val, arg_def)) in args.iter().zip(cmd.args.iter()).enumerate() {\n let schema_str = arg_def.schema_str().unwrap_or(\"b\");\n let schema = match parse_schema(schema_str) {\n Ok(s) => s,\n Err(e) => {\n eprintln!(\"Error: failed to parse arg schema #{}: {}\", i, e);\n process::clean_exit(1);\n }\n };\n\n let c_schema = morloc_runtime::cschema::CSchema::from_rust(&schema);\n let mut errmsg: *mut std::ffi::c_char = std::ptr::null_mut();\n\n let c_arg;\n match arg_val {\n ArgValue::Group { grp_val, fields, defaults } => {\n // Group arg: use initialize_unrolled (matches C nexus behavior)\n extern \"C\" {\n fn initialize_unrolled(\n size: usize, default_value: *mut std::ffi::c_char,\n fields: *mut *mut std::ffi::c_char,\n default_fields: *mut *mut std::ffi::c_char,\n ) -> *mut std::ffi::c_void;\n }\n let n = fields.len();\n let grp_val_c = grp_val.as_ref()\n .map(|s| std::ffi::CString::new(s.as_str()).unwrap().into_raw())\n .unwrap_or(std::ptr::null_mut());\n let mut c_fields: Vec<*mut std::ffi::c_char> = fields.iter()\n .map(|f| f.as_ref()\n .map(|s| std::ffi::CString::new(s.as_str()).unwrap().into_raw())\n .unwrap_or(std::ptr::null_mut()))\n .collect();\n let mut c_defaults: Vec<*mut std::ffi::c_char> = defaults.iter()\n .map(|d| d.as_ref()\n .map(|s| std::ffi::CString::new(s.as_str()).unwrap().into_raw())\n .unwrap_or(std::ptr::null_mut()))\n .collect();\n c_arg = unsafe {\n initialize_unrolled(n, grp_val_c, c_fields.as_mut_ptr(), c_defaults.as_mut_ptr())\n };\n }\n _ => {\n let json_str = match arg_val {\n ArgValue::Value(s) => s.clone(),\n ArgValue::Null => \"null\".to_string(),\n _ => unreachable!(),\n };\n let json_c = std::ffi::CString::new(json_str.as_str()).unwrap();\n c_arg = unsafe { initialize_positional(json_c.into_raw()) };\n }\n }\n\n let c_pkt = unsafe { parse_cli_data_argument(std::ptr::null_mut(), c_arg, c_schema, &mut errmsg) };\n unsafe { free_argument_t(c_arg) };\n unsafe { morloc_runtime::cschema::CSchema::free(c_schema) };\n\n if c_pkt.is_null() {\n let msg = if !errmsg.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut std::ffi::c_void) };\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: failed to parse argument #{}: {}\", i, msg);\n process::clean_exit(1);\n }\n\n // Get packet size and copy to Vec\n let pkt_size = unsafe { morloc_packet_size(c_pkt, &mut errmsg) };\n let data_pkt = unsafe { std::slice::from_raw_parts(c_pkt, pkt_size).to_vec() };\n unsafe { libc::free(c_pkt as *mut std::ffi::c_void) };\n arg_packets.push(data_pkt);\n }\n\n // Build call packet via C library\n let arg_ptrs: Vec<*const u8> = arg_packets.iter().map(|p| p.as_ptr()).collect();\n let mut errmsg_call: *mut std::ffi::c_char = std::ptr::null_mut();\n let c_call = unsafe {\n make_morloc_local_call_packet(cmd.mid, arg_ptrs.as_ptr(), arg_packets.len(), &mut errmsg_call)\n };\n if c_call.is_null() {\n eprintln!(\"Error: failed to create call packet\");\n process::clean_exit(1);\n }\n\n // Get call packet size\n let call_size = unsafe {\n let mut e: *mut std::ffi::c_char = std::ptr::null_mut();\n morloc_packet_size(c_call, &mut e)\n };\n let call_packet = unsafe { std::slice::from_raw_parts(c_call, call_size).to_vec() };\n unsafe { libc::free(c_call as *mut std::ffi::c_void) };\n\n // Send to pool and receive response\n let mut stream = match UnixStream::connect(&socket.socket_path) {\n Ok(s) => s,\n Err(e) => {\n die_with_pool_error(\n socket,\n cmd.pool_index,\n &format!(\"failed to connect to pool '{}'\", socket.lang),\n &e,\n );\n }\n };\n\n if let Err(e) = stream.write_all(&call_packet) {\n die_with_pool_error(\n socket,\n cmd.pool_index,\n &format!(\"failed to send call packet to pool '{}'\", socket.lang),\n &e,\n );\n }\n\n // Read response header\n let mut resp_header_bytes = [0u8; 32];\n if let Err(e) = stream.read_exact(&mut resp_header_bytes) {\n die_with_pool_error(\n socket,\n cmd.pool_index,\n &format!(\"failed to read response header from pool '{}'\", socket.lang),\n &e,\n );\n }\n\n let resp_header = match packet::PacketHeader::from_bytes(&resp_header_bytes) {\n Ok(h) => h,\n Err(e) => {\n eprintln!(\"Error: invalid response packet: {}\", e);\n process::clean_exit(1);\n }\n };\n\n // Read full response (metadata + payload)\n let offset = { resp_header.offset } as usize;\n let length = { resp_header.length } as usize;\n let remaining = offset + length;\n let mut resp_body = vec![0u8; remaining];\n if remaining > 0 {\n if let Err(e) = stream.read_exact(&mut resp_body) {\n die_with_pool_error(\n socket,\n cmd.pool_index,\n &format!(\"failed to read response body from pool '{}'\", socket.lang),\n &e,\n );\n }\n }\n\n // Reconstruct full packet (header + body)\n let mut full_packet = Vec::with_capacity(32 + remaining);\n full_packet.extend_from_slice(&resp_header_bytes);\n full_packet.extend_from_slice(&resp_body);\n\n // Check for error\n match packet::get_error_message(&full_packet) {\n Ok(Some(err_msg)) => {\n eprintln!(\"Error: run failed\\n{}\", err_msg);\n process::clean_exit(1);\n }\n Ok(None) => {}\n Err(e) => {\n eprintln!(\"Error: failed to parse response: {}\", e);\n process::clean_exit(1);\n }\n }\n\n // Extract and print via C library for correct voidstar handling\n let c_schema = morloc_runtime::cschema::CSchema::from_rust(&return_schema);\n let mut errmsg: *mut std::ffi::c_char = std::ptr::null_mut();\n let result_ptr = unsafe {\n get_morloc_data_packet_value(full_packet.as_ptr(), c_schema, &mut errmsg)\n };\n if result_ptr.is_null() {\n let msg = if !errmsg.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut std::ffi::c_void) };\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: failed to extract result: {}\", msg);\n unsafe { morloc_runtime::cschema::CSchema::free(c_schema) };\n process::clean_exit(1);\n }\n\n // Check if response is Arrow format\n let is_arrow = resp_header.is_data() && unsafe { resp_header.command.data.format } == packet::PACKET_FORMAT_ARROW;\n\n // Print using the C library for correct output.\n // Suppress \"null\" for Unit-returning commands (CLI convention).\n if return_schema.serial_type != SerialType::Nil {\n print_result_c(result_ptr, c_schema, &full_packet, is_arrow, config);\n }\n unsafe { morloc_runtime::cschema::CSchema::free(c_schema) };\n}\n\n/// Print using the C library functions for correct voidstar handling.\nfn print_result_c(\n ptr: *mut u8,\n schema: *const morloc_runtime::cschema::CSchema,\n full_packet: &[u8],\n is_arrow: bool,\n config: &NexusConfig,\n) {\n extern \"C\" {\n fn print_voidstar(\n voidstar: *const std::ffi::c_void,\n schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> bool;\n fn pretty_print_voidstar(\n voidstar: *const std::ffi::c_void,\n schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> bool;\n fn print_arrow_as_json(\n data: *const std::ffi::c_void,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> bool;\n fn print_arrow_as_table(\n data: *const std::ffi::c_void,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> bool;\n fn pack_with_schema(\n mlc: *const std::ffi::c_void,\n schema: *const morloc_runtime::cschema::CSchema,\n mpkptr: *mut *mut std::ffi::c_char,\n mpk_size: *mut usize,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> i32;\n }\n\n let mut errmsg: *mut std::ffi::c_char = std::ptr::null_mut();\n\n match config.output_format {\n OutputFormat::Json => {\n let ok = unsafe {\n if is_arrow && config.print_flag {\n print_arrow_as_table(ptr as *const std::ffi::c_void, &mut errmsg)\n } else if is_arrow {\n print_arrow_as_json(ptr as *const std::ffi::c_void, &mut errmsg)\n } else if config.print_flag {\n pretty_print_voidstar(ptr as *const std::ffi::c_void, schema, &mut errmsg)\n } else {\n print_voidstar(ptr as *const std::ffi::c_void, schema, &mut errmsg)\n }\n };\n if !ok {\n let msg = if !errmsg.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut std::ffi::c_void) };\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: {}\", msg);\n process::clean_exit(1);\n }\n }\n OutputFormat::MessagePack => {\n let mut mpk_ptr: *mut std::ffi::c_char = std::ptr::null_mut();\n let mut mpk_size: usize = 0;\n let rc = unsafe {\n pack_with_schema(\n ptr as *const std::ffi::c_void,\n schema,\n &mut mpk_ptr,\n &mut mpk_size,\n &mut errmsg,\n )\n };\n if rc != 0 {\n eprintln!(\"Error: msgpack serialization failed\");\n process::clean_exit(1);\n }\n if config.print_flag {\n let bytes = unsafe { std::slice::from_raw_parts(mpk_ptr as *const u8, mpk_size) };\n for (i, b) in bytes.iter().enumerate() {\n if i > 0 && i % 16 == 0 { println!(); }\n print!(\"{:02x} \", b);\n }\n println!();\n } else {\n use std::io::Write;\n let bytes = unsafe { std::slice::from_raw_parts(mpk_ptr as *const u8, mpk_size) };\n let _ = std::io::stdout().lock().write_all(bytes);\n }\n if !mpk_ptr.is_null() {\n unsafe { libc::free(mpk_ptr as *mut std::ffi::c_void) };\n }\n }\n OutputFormat::VoidStar => {\n extern \"C\" {\n fn print_morloc_data_packet(\n packet: *const u8,\n schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> i32;\n }\n if config.print_flag {\n // Hex dump\n for (i, b) in full_packet.iter().enumerate() {\n if i > 0 && i % 4 == 0 {\n if i % 24 == 0 { println!(); } else { print!(\" \"); }\n }\n print!(\"{:02X}\", b);\n }\n if !full_packet.is_empty() { println!(); }\n } else {\n let mut errmsg2: *mut std::ffi::c_char = std::ptr::null_mut();\n unsafe { print_morloc_data_packet(full_packet.as_ptr(), schema, &mut errmsg2) };\n }\n }\n OutputFormat::Packet => {\n // Packet format: write raw binary packet to stdout (used by SLURM)\n use std::io::Write;\n let _ = std::io::stdout().lock().write_all(&full_packet);\n }\n }\n process::clean_exit(0);\n}\n\n/// Print using Rust-native functions (kept for reference, currently unused).\n#[allow(dead_code)]\nfn print_result(\n ptr: morloc_runtime::shm::AbsPtr,\n schema: &morloc_runtime::Schema,\n config: &NexusConfig,\n) {\n use morloc_runtime::{json, mpack};\n\n match config.output_format {\n OutputFormat::Json => {\n if config.print_flag {\n if let Err(e) = json::pretty_print_voidstar(ptr, schema) {\n eprintln!(\"Error: {}\", e);\n process::clean_exit(1);\n }\n } else {\n if let Err(e) = json::print_voidstar(ptr, schema) {\n eprintln!(\"Error: {}\", e);\n process::clean_exit(1);\n }\n }\n }\n OutputFormat::MessagePack => {\n let mpk = match mpack::pack_with_schema(ptr, schema) {\n Ok(m) => m,\n Err(e) => {\n eprintln!(\"Error: {}\", e);\n process::clean_exit(1);\n }\n };\n if config.print_flag {\n // Hex dump for human-readable msgpack\n for (i, byte) in mpk.iter().enumerate() {\n if i > 0 && i % 16 == 0 {\n println!();\n }\n print!(\"{:02x} \", byte);\n }\n println!();\n } else {\n use std::io::Write;\n let stdout = std::io::stdout();\n let mut handle = stdout.lock();\n let _ = handle.write_all(&mpk);\n }\n }\n OutputFormat::VoidStar | OutputFormat::Packet => {\n eprintln!(\"Error: voidstar/packet output not supported in Rust-native print path\");\n process::clean_exit(1);\n }\n }\n process::clean_exit(0);\n}\n\n/// Execute a pure command by evaluating the expression via C library.\nfn run_pure_command(cmd: &Command, args: &[ArgValue], config: &NexusConfig) {\n use morloc_runtime::schema::{parse_schema, SerialType};\n\n extern \"C\" {\n fn build_manifest_expr(\n json_str: *const std::ffi::c_char,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> *mut std::ffi::c_void; // morloc_expression_t*\n fn morloc_eval(\n expr: *mut std::ffi::c_void,\n return_schema: *const morloc_runtime::cschema::CSchema,\n arg_voidstar: *const *mut u8,\n arg_schemas: *const *const morloc_runtime::cschema::CSchema,\n nargs: usize,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> *mut std::ffi::c_void; // absptr_t\n fn parse_cli_data_argument(\n dest: *mut u8, arg: *const std::ffi::c_void,\n schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> *mut u8;\n fn initialize_positional(value: *mut std::ffi::c_char) -> *mut std::ffi::c_void;\n fn free_argument_t(arg: *mut std::ffi::c_void);\n fn get_morloc_data_packet_value(\n data: *const u8, schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut std::ffi::c_char,\n ) -> *mut u8;\n fn make_standard_data_packet(\n relptr: isize,\n schema: *const morloc_runtime::cschema::CSchema,\n ) -> *mut u8;\n fn abs2rel(ptr: *mut std::ffi::c_void, errmsg: *mut *mut std::ffi::c_char) -> isize;\n }\n\n // Build expression tree from manifest JSON\n let expr_json = match &cmd.expr {\n Some(v) => serde_json::to_string(v).unwrap_or_default(),\n None => {\n eprintln!(\"Error: pure command '{}' has no expression\", cmd.name);\n process::clean_exit(1);\n }\n };\n let expr_c = std::ffi::CString::new(expr_json.as_str()).unwrap();\n let mut errmsg: *mut std::ffi::c_char = std::ptr::null_mut();\n let expr = unsafe { build_manifest_expr(expr_c.as_ptr(), &mut errmsg) };\n if expr.is_null() {\n let msg = unsafe_errmsg_to_string(errmsg);\n eprintln!(\"Error: failed to build expression: {}\", msg);\n process::clean_exit(1);\n }\n\n // Parse return schema\n let return_schema = match parse_schema(&cmd.ret.schema) {\n Ok(s) => s,\n Err(e) => {\n eprintln!(\"Error: failed to parse return schema '{}': {}\", cmd.ret.schema, e);\n process::clean_exit(1);\n }\n };\n let c_return_schema = morloc_runtime::cschema::CSchema::from_rust(&return_schema);\n\n // The parsed `args` list and `cmd.args` are index-aligned 1:1 in\n // declaration order: parse_command_args pushes one ArgValue for\n // EVERY arg (including flags). The Haskell compiler emits one\n // schema per arg position too. Walk both lists in lockstep; for\n // flags, the schema_str() accessor returns None and we fall back\n // to the bool schema \"b\" so the wire format stays consistent.\n let mut c_arg_schemas: Vec<*const morloc_runtime::cschema::CSchema> = Vec::new();\n let mut c_arg_voidstars: Vec<*mut u8> = Vec::new();\n\n for (i, (arg_val, arg_def)) in args.iter().zip(cmd.args.iter()).enumerate() {\n let schema_str = arg_def.schema_str().unwrap_or(\"b\");\n let schema = match parse_schema(schema_str) {\n Ok(s) => s,\n Err(e) => {\n eprintln!(\"Error: failed to parse arg schema #{}: {}\", i, e);\n process::clean_exit(1);\n }\n };\n let c_schema = morloc_runtime::cschema::CSchema::from_rust(&schema);\n\n let json_str = match arg_val {\n ArgValue::Value(s) => s.clone(),\n ArgValue::Null => \"null\".to_string(),\n ArgValue::Group { .. } => \"null\".to_string(),\n };\n\n // Parse CLI arg to data packet, then extract voidstar\n let json_c = std::ffi::CString::new(json_str.as_str()).unwrap();\n let c_arg = unsafe { initialize_positional(json_c.into_raw()) };\n let c_pkt = unsafe { parse_cli_data_argument(std::ptr::null_mut(), c_arg, c_schema, &mut errmsg) };\n unsafe { free_argument_t(c_arg) };\n\n if c_pkt.is_null() {\n let msg = unsafe_errmsg_to_string(errmsg);\n eprintln!(\"Error: failed to parse argument #{}: {}\", i, msg);\n process::clean_exit(1);\n }\n\n let voidstar = unsafe { get_morloc_data_packet_value(c_pkt, c_schema, &mut errmsg) };\n unsafe { libc::free(c_pkt as *mut std::ffi::c_void) };\n if voidstar.is_null() {\n let msg = unsafe_errmsg_to_string(errmsg);\n eprintln!(\"Error: failed to extract argument #{}: {}\", i, msg);\n process::clean_exit(1);\n }\n\n c_arg_schemas.push(c_schema);\n c_arg_voidstars.push(voidstar);\n }\n\n // Call morloc_eval\n let result = unsafe {\n morloc_eval(\n expr,\n c_return_schema,\n c_arg_voidstars.as_ptr(),\n c_arg_schemas.as_ptr(),\n c_arg_voidstars.len(),\n &mut errmsg,\n )\n };\n\n if result.is_null() {\n let msg = unsafe_errmsg_to_string(errmsg);\n eprintln!(\"Error: evaluation failed: {}\", msg);\n process::clean_exit(1);\n }\n\n // Convert result to relptr and make a data packet for printing\n let result_relptr = unsafe { abs2rel(result, &mut errmsg) };\n let result_packet = unsafe { make_standard_data_packet(result_relptr, c_return_schema) };\n\n if result_packet.is_null() {\n eprintln!(\"Error: failed to create result packet\");\n process::clean_exit(1);\n }\n\n // Get packet as bytes for print_result_c\n extern \"C\" {\n fn morloc_packet_size(packet: *const u8, errmsg: *mut *mut std::ffi::c_char) -> usize;\n }\n let pkt_size = unsafe { morloc_packet_size(result_packet, &mut errmsg) };\n let pkt_bytes = unsafe { std::slice::from_raw_parts(result_packet, pkt_size).to_vec() };\n\n // Extract voidstar value from the result packet\n let result_ptr = unsafe { get_morloc_data_packet_value(pkt_bytes.as_ptr(), c_return_schema, &mut errmsg) };\n\n if return_schema.serial_type != SerialType::Nil {\n print_result_c(result_ptr, c_return_schema, &pkt_bytes, false, config);\n }\n\n // Cleanup\n for cs in &c_arg_schemas {\n unsafe { morloc_runtime::cschema::CSchema::free(*cs as *mut morloc_runtime::cschema::CSchema) };\n }\n unsafe {\n morloc_runtime::cschema::CSchema::free(c_return_schema);\n libc::free(result_packet as *mut std::ffi::c_void);\n }\n}\n\nfn unsafe_errmsg_to_string(errmsg: *mut std::ffi::c_char) -> String {\n if errmsg.is_null() {\n \"unknown error\".into()\n } else {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut std::ffi::c_void) };\n s\n }\n}\n\n// -- Helpers for command argument parsing ------------------------------------\n\nfn is_flag_opt(cmd: &Command, long_name: &str) -> bool {\n cmd.args.iter().any(|a| match a {\n Arg::Flag { long_opt, .. } => long_opt.as_deref() == Some(long_name),\n Arg::Group { entries, .. } => entries.iter().any(|e| match &e.arg {\n Arg::Flag { long_opt, .. } => long_opt.as_deref() == Some(long_name),\n _ => false,\n }),\n _ => false,\n })\n}\n\nfn is_rev_flag(cmd: &Command, name: &str) -> bool {\n cmd.args.iter().any(|a| match a {\n Arg::Flag { long_rev, .. } => long_rev.as_deref() == Some(name),\n Arg::Group { entries, .. } => entries.iter().any(|e| match &e.arg {\n Arg::Flag { long_rev, .. } => long_rev.as_deref() == Some(name),\n _ => false,\n }),\n _ => false,\n })\n}\n\nfn find_flag_by_rev(cmd: &Command, rev_name: &str) -> Option {\n for a in &cmd.args {\n match a {\n Arg::Flag { long_opt, long_rev, .. } => {\n if long_rev.as_deref() == Some(rev_name) {\n return long_opt.clone();\n }\n }\n Arg::Group { entries, .. } => {\n for e in entries {\n if let Arg::Flag { long_opt, long_rev, .. } = &e.arg {\n if long_rev.as_deref() == Some(rev_name) {\n return long_opt.clone();\n }\n }\n }\n }\n _ => {}\n }\n }\n None\n}\n\nfn flag_forward_value(cmd: &Command, long_name: &str) -> String {\n for a in &cmd.args {\n if let Arg::Flag {\n long_opt,\n default_val,\n ..\n } = a\n {\n if long_opt.as_deref() == Some(long_name) {\n let def = default_val.as_deref().unwrap_or(\"false\");\n return if def == \"true\" {\n \"false\".into()\n } else {\n \"true\".into()\n };\n }\n }\n }\n \"true\".into()\n}\n\nfn flag_forward_value_by_short(cmd: &Command, ch: char) -> String {\n for a in &cmd.args {\n if let Arg::Flag {\n short_opt,\n default_val,\n ..\n } = a\n {\n if short_opt.as_deref().and_then(|s| s.chars().next()) == Some(ch) {\n let def = default_val.as_deref().unwrap_or(\"false\");\n return if def == \"true\" {\n \"false\".into()\n } else {\n \"true\".into()\n };\n }\n }\n }\n \"true\".into()\n}\n\nfn flag_reverse_value_by_rev(cmd: &Command, rev_name: &str) -> String {\n // Search top-level and group entries\n let check = |long_rev: &Option, default_val: &Option| -> Option {\n if long_rev.as_deref() == Some(rev_name) {\n let def = default_val.as_deref().unwrap_or(\"false\");\n Some(if def == \"true\" { \"true\".into() } else { \"false\".into() })\n } else {\n None\n }\n };\n for a in &cmd.args {\n match a {\n Arg::Flag { long_rev, default_val, .. } => {\n if let Some(v) = check(long_rev, default_val) { return v; }\n }\n Arg::Group { entries, .. } => {\n for e in entries {\n if let Arg::Flag { long_rev, default_val, .. } = &e.arg {\n if let Some(v) = check(long_rev, default_val) { return v; }\n }\n }\n }\n _ => {}\n }\n }\n \"false\".into()\n}\n\nfn is_short_flag(cmd: &Command, ch: char) -> bool {\n cmd.args.iter().any(|a| match a {\n Arg::Flag { short_opt, .. } => {\n short_opt.as_deref().and_then(|s| s.chars().next()) == Some(ch)\n }\n _ => false,\n })\n}\n\nfn short_to_long(cmd: &Command, ch: char) -> Option {\n for a in &cmd.args {\n let (s, l) = match a {\n Arg::Optional {\n short_opt,\n long_opt,\n ..\n } => (short_opt.as_deref(), long_opt.clone()),\n Arg::Flag {\n short_opt,\n long_opt,\n ..\n } => (short_opt.as_deref(), long_opt.clone()),\n Arg::Group { entries, .. } => {\n // Search inside group entries\n for entry in entries {\n let (es, el) = match &entry.arg {\n Arg::Optional { short_opt, long_opt, .. } => (short_opt.as_deref(), long_opt.clone()),\n Arg::Flag { short_opt, long_opt, .. } => (short_opt.as_deref(), long_opt.clone()),\n _ => (None, None),\n };\n if es.and_then(|s| s.chars().next()) == Some(ch) {\n return el.or_else(|| Some(ch.to_string()));\n }\n }\n (None, None)\n }\n _ => (None, None),\n };\n if s.and_then(|s| s.chars().next()) == Some(ch) {\n return l;\n }\n }\n None\n}\n" }, { "path": "data/rust/morloc-nexus/src/help.rs", "content": "//! Help text generation matching the C nexus output format.\n\nuse crate::manifest::{Arg, Command, GroupEntry, Manifest};\n\n/// Print nexus-level usage (no manifest loaded).\npub fn print_nexus_usage(prog_name: &str) -> ! {\n eprintln!(\"Usage: {} [OPTION...] COMMAND [ARG...]\", prog_name);\n eprintln!();\n eprintln!(\"morloc-nexus is the morloc program dispatcher.\");\n eprintln!();\n eprintln!(\"Arguments:\");\n eprintln!(\" Path to a .manifest file or wrapper script\");\n eprintln!();\n eprintln!(\"Nexus options:\");\n eprintln!(\" -h, --help Print this help message\");\n eprintln!(\" -p, --print Pretty-print output for human consumption\");\n eprintln!(\" -o, --output-file Print to this file instead of STDOUT\");\n eprintln!(\" -f, --output-format Output format [json|mpk|voidstar]\");\n eprintln!();\n eprintln!(\"Daemon mode:\");\n eprintln!(\" --daemon Run as a long-lived daemon\");\n eprintln!(\" --http-port PORT Listen on HTTP port\");\n eprintln!(\" --port PORT Listen on TCP port\");\n eprintln!(\" --socket PATH Listen on Unix socket\");\n eprintln!(\" --eval-timeout SECS Timeout for /eval requests (default: 30)\");\n eprintln!();\n eprintln!(\"Router mode:\");\n eprintln!(\" --router Run as a multi-program router\");\n eprintln!(\" --fdb Path to fdb manifest directory\");\n std::process::exit(0);\n}\n\n/// Print usage for a multi-command program.\npub fn print_usage(prog_name: &str, manifest: &Manifest) -> ! {\n eprintln!(\"Usage: {} [OPTION...] COMMAND [ARG...]\", prog_name);\n\n // Module-level description\n if !manifest.desc.is_empty() {\n eprintln!();\n for line in &manifest.desc {\n eprintln!(\"{}\", line);\n }\n }\n\n eprintln!();\n eprintln!(\"Nexus options:\");\n eprintln!(\" -h, --help Print this help message\");\n eprintln!(\" -p, --print Pretty-print output for human consumption\");\n eprintln!(\" -o, --output-file Print to this file instead of STDOUT\");\n eprintln!(\" -f, --output-format Output format [json|mpk|voidstar]\");\n eprintln!();\n eprintln!(\"Daemon mode:\");\n eprintln!(\" --daemon Run as a long-lived daemon\");\n eprintln!(\" --http-port PORT Listen on HTTP port\");\n eprintln!(\" --port PORT Listen on TCP port\");\n eprintln!(\" --socket PATH Listen on Unix socket\");\n eprintln!();\n\n // Ungrouped commands\n let ungrouped: Vec<&Command> = manifest\n .commands\n .iter()\n .filter(|c| c.group.is_none())\n .collect();\n\n if !ungrouped.is_empty() {\n eprintln!(\"Commands (call with -h/--help for more info):\");\n let longest = ungrouped.iter().map(|c| c.name.len()).max().unwrap_or(0);\n for cmd in &ungrouped {\n eprint!(\" {}\", cmd.name);\n if let Some(first) = cmd.desc.first() {\n let pad = longest - cmd.name.len() + 2;\n eprint!(\"{:pad$}{}\", \"\", first, pad = pad);\n }\n eprintln!();\n }\n }\n\n if !manifest.groups.is_empty() {\n if !ungrouped.is_empty() {\n eprintln!();\n }\n eprintln!(\"Command groups (call with -h/--help for more info):\");\n let longest = manifest.groups.iter().map(|g| g.name.len()).max().unwrap_or(0);\n for grp in &manifest.groups {\n eprint!(\" {}\", grp.name);\n if let Some(first) = grp.desc.first() {\n let pad = longest - grp.name.len() + 2;\n eprint!(\"{:pad$}{}\", \"\", first, pad = pad);\n }\n eprintln!();\n }\n }\n\n // Epilogues\n for epilogue in &manifest.epilogues {\n eprintln!();\n for line in epilogue {\n eprintln!(\"{}\", line);\n }\n }\n\n std::process::exit(0);\n}\n\n/// Print usage for a command group.\npub fn print_group_usage(prog_name: &str, manifest: &Manifest, group_name: &str) -> ! {\n let grp = manifest.groups.iter().find(|g| g.name == group_name);\n\n eprintln!(\"Usage: {} {} COMMAND [ARG...]\", prog_name, group_name);\n if let Some(g) = grp {\n if !g.desc.is_empty() {\n eprintln!();\n for line in &g.desc {\n eprintln!(\"{}\", line);\n }\n }\n }\n eprintln!(\"\\nCommands:\");\n\n let cmds: Vec<&Command> = manifest\n .commands\n .iter()\n .filter(|c| c.group.as_deref() == Some(group_name))\n .collect();\n\n let longest = cmds.iter().map(|c| c.name.len()).max().unwrap_or(0);\n for cmd in &cmds {\n eprint!(\" {}\", cmd.name);\n if let Some(first) = cmd.desc.first() {\n let pad = longest - cmd.name.len() + 2;\n eprint!(\"{:pad$}{}\", \"\", first, pad = pad);\n }\n eprintln!();\n }\n\n std::process::exit(0);\n}\n\n/// Print help for a specific subcommand.\npub fn print_command_help(prog_name: &str, cmd: &Command) -> ! {\n // Usage line\n if let Some(ref group) = cmd.group {\n eprint!(\"Usage: {} {} {}\", prog_name, group, cmd.name);\n } else {\n eprint!(\"Usage: {} {}\", prog_name, cmd.name);\n }\n print_usage_suffix(cmd);\n eprintln!();\n if !cmd.desc.is_empty() {\n eprintln!();\n }\n\n print_command_body(cmd);\n std::process::exit(0);\n}\n\n/// Print help for a single-command program.\npub fn print_command_help_single(prog_name: &str, cmd: &Command) -> ! {\n eprint!(\"Usage: {}\", prog_name);\n print_usage_suffix(cmd);\n eprintln!();\n\n // Description\n if !cmd.desc.is_empty() {\n eprintln!();\n for (i, line) in cmd.desc.iter().enumerate() {\n if i == 0 && line.is_empty() {\n continue;\n }\n eprintln!(\"{}\", line);\n }\n }\n\n // Nexus options\n eprintln!(\"\\nNexus options:\");\n eprintln!(\" --print Pretty-print output for human consumption\");\n eprintln!(\" --output-file Print to this file instead of STDOUT\");\n eprintln!(\" --output-form Output format [json|mpk|voidstar]\");\n eprintln!(\"\\nDaemon mode:\");\n eprintln!(\" --daemon Run as a long-lived daemon\");\n eprintln!(\" --http-port PORT Listen on HTTP port\");\n eprintln!(\" --port PORT Listen on TCP port\");\n eprintln!(\" --socket PATH Listen on UNIX socket\");\n\n print_args_body(cmd);\n print_type_definitions(cmd);\n print_return_info(cmd);\n std::process::exit(0);\n}\n\n// -- Helpers ----------------------------------------------------------------\n\nfn print_usage_suffix(cmd: &Command) {\n let has_opts = cmd.args.iter().any(|a| !matches!(a, Arg::Positional { .. }));\n if has_opts {\n eprint!(\" [OPTION...]\");\n }\n for arg in &cmd.args {\n if let Arg::Positional { metavar, .. } = arg {\n eprint!(\" {}\", metavar.as_deref().unwrap_or(\"ARG\"));\n }\n }\n}\n\nfn print_command_body(cmd: &Command) {\n // Description\n if !cmd.desc.is_empty() {\n for (i, line) in cmd.desc.iter().enumerate() {\n if i == 0 && line.is_empty() {\n continue;\n }\n eprintln!(\"{}\", line);\n }\n }\n\n print_args_body(cmd);\n print_type_definitions(cmd);\n print_return_info(cmd);\n}\n\nfn print_args_body(cmd: &Command) {\n // Positional arguments\n let has_pos = cmd.args.iter().any(|a| matches!(a, Arg::Positional { .. }));\n if has_pos {\n eprintln!(\"\\nPositional arguments:\");\n for arg in &cmd.args {\n if let Arg::Positional {\n metavar,\n type_desc,\n desc,\n ..\n } = arg\n {\n eprint!(\" {}\", metavar.as_deref().unwrap_or(\"ARG\"));\n if let Some(first) = desc.first() {\n eprint!(\" {}\", first);\n }\n eprintln!();\n if let Some(td) = type_desc {\n eprintln!(\" type: {}\", td);\n }\n }\n }\n }\n\n // Optional arguments (opts and flags)\n let has_opt = cmd\n .args\n .iter()\n .any(|a| matches!(a, Arg::Optional { .. } | Arg::Flag { .. }));\n if has_opt {\n eprintln!(\"\\nOptional arguments:\");\n for arg in &cmd.args {\n print_opt_or_flag(arg);\n }\n }\n\n // Group arguments\n for arg in &cmd.args {\n if let Arg::Group {\n metavar,\n desc,\n group_opt,\n entries,\n ..\n } = arg\n {\n eprintln!(\"\\nGroup arguments:\");\n eprint!(\" {}\", metavar.as_deref().unwrap_or(\"\"));\n if let Some(first) = desc.first() {\n eprint!(\": {}\", first);\n }\n eprintln!();\n\n if let Some(go) = group_opt {\n eprint!(\" \");\n if let Some(ref s) = go.short_opt {\n eprint!(\"-{}, \", s);\n }\n if let Some(ref l) = go.long_opt {\n eprint!(\"--{} {}\", l, metavar.as_deref().unwrap_or(\"\"));\n }\n eprintln!();\n eprintln!(\" provide record as file or JSON string\");\n }\n\n for entry in entries {\n print_group_entry(entry);\n }\n }\n }\n}\n\nfn print_opt_or_flag(arg: &Arg) {\n match arg {\n Arg::Optional {\n short_opt,\n long_opt,\n metavar,\n default_val,\n desc,\n type_desc,\n ..\n } => {\n eprint!(\" \");\n match (short_opt.as_deref(), long_opt.as_deref()) {\n (Some(s), Some(l)) => eprint!(\n \"-{}, --{} {}\",\n s,\n l,\n metavar.as_deref().unwrap_or(\"\")\n ),\n (Some(s), None) => {\n eprint!(\"-{} {}\", s, metavar.as_deref().unwrap_or(\"\"))\n }\n (None, Some(l)) => eprint!(\n \"--{} {}\",\n l,\n metavar.as_deref().unwrap_or(\"\")\n ),\n _ => {}\n }\n eprintln!();\n if let Some(d) = default_val {\n eprintln!(\" default: {}\", d);\n }\n for d in desc {\n eprintln!(\" {}\", d);\n }\n if let Some(td) = type_desc {\n eprintln!(\" type: {}\", td);\n }\n }\n Arg::Flag {\n short_opt,\n long_opt,\n long_rev,\n default_val,\n desc,\n ..\n } => {\n eprint!(\" \");\n match (short_opt.as_deref(), long_opt.as_deref()) {\n (Some(s), Some(l)) => eprint!(\"-{}, --{}\", s, l),\n (Some(s), None) => eprint!(\"-{}\", s),\n (None, Some(l)) => eprint!(\"--{}\", l),\n _ => {}\n }\n eprintln!();\n if let Some(rev) = long_rev {\n eprintln!(\" --{}\", rev);\n }\n if let Some(d) = default_val {\n eprintln!(\" default: {}\", d);\n }\n for d in desc {\n eprintln!(\" {}\", d);\n }\n }\n _ => {}\n }\n}\n\nfn print_group_entry(entry: &GroupEntry) {\n let ea = &entry.arg;\n eprint!(\" \");\n match ea {\n Arg::Optional {\n short_opt,\n long_opt,\n metavar,\n default_val,\n desc,\n ..\n } => {\n match (short_opt.as_deref(), long_opt.as_deref()) {\n (Some(s), Some(l)) => {\n eprint!(\"-{}, --{}\", s, l);\n if let Some(m) = metavar {\n eprint!(\" {}\", m);\n }\n }\n (Some(s), None) => {\n eprint!(\"-{}\", s);\n if let Some(m) = metavar {\n eprint!(\" {}\", m);\n }\n }\n (None, Some(l)) => {\n eprint!(\"--{}\", l);\n if let Some(m) = metavar {\n eprint!(\" {}\", m);\n }\n }\n _ => {}\n }\n eprintln!();\n if let Some(d) = default_val {\n eprintln!(\" default: {}\", d);\n }\n for d in desc {\n eprintln!(\" {}\", d);\n }\n }\n Arg::Flag {\n short_opt,\n long_opt,\n default_val,\n desc,\n ..\n } => {\n match (short_opt.as_deref(), long_opt.as_deref()) {\n (Some(s), Some(l)) => eprint!(\"-{}, --{}\", s, l),\n (Some(s), None) => eprint!(\"-{}\", s),\n (None, Some(l)) => eprint!(\"--{}\", l),\n _ => {}\n }\n eprintln!();\n if let Some(d) = default_val {\n eprintln!(\" default: {}\", d);\n }\n for d in desc {\n eprintln!(\" {}\", d);\n }\n }\n _ => {}\n }\n}\n\nfn print_return_info(cmd: &Command) {\n eprintln!(\"\\nReturn: {}\", cmd.ret.type_desc);\n for line in &cmd.ret.desc {\n eprintln!(\" {}\", line);\n }\n}\n\n// -- Schema-walking renderer for the Record / Table Schemas sections ---------\n//\n// In v2 the manifest no longer carries a parallel `type_definitions` list.\n// The same information is reconstructed at help-render time by walking\n// each command's args + return value: for every typed entry whose schema\n// parses to a Map at the top level, we treat the entry's `type` name as\n// the named-type label and its schema's keys + parameter schemas as the\n// field list. The record-vs-table distinction comes from the entry's\n// `kind` constraint.\n\n/// A rendered named-type layout for the help output, sourced from a\n/// parsed schema.\nstruct TypeLayout<'a> {\n name: &'a str,\n /// \"record\" | \"object\" | \"table\" -- comes from the `kind` constraint.\n kind: &'a str,\n /// (field_name, rendered_type)\n fields: Vec<(String, String)>,\n}\n\n/// Pretty-render a parsed `Schema` as a morloc-flavored type string,\n/// suitable for the field-type column in the schemas block.\nfn render_schema_type(s: &morloc_runtime::schema::Schema) -> String {\n use morloc_runtime::schema::SerialType::*;\n match s.serial_type {\n Nil => \"()\".into(),\n Bool => \"Bool\".into(),\n Sint8 => \"Int8\".into(),\n Sint16 => \"Int16\".into(),\n Sint32 => \"Int\".into(),\n Sint64 => \"Int64\".into(),\n Uint8 => \"UInt8\".into(),\n Uint16 => \"UInt16\".into(),\n Uint32 => \"UInt32\".into(),\n Uint64 => \"UInt64\".into(),\n Float32 => \"Float32\".into(),\n Float64 => \"Real\".into(),\n String => \"Str\".into(),\n Array => format!(\n \"[{}]\",\n s.parameters\n .first()\n .map(render_schema_type)\n .unwrap_or_else(|| \"?\".into())\n ),\n Tuple => {\n // `String` here is fully qualified because the surrounding\n // match brings `SerialType::String` into scope as a variant,\n // shadowing the std `String` type.\n let inner: Vec =\n s.parameters.iter().map(render_schema_type).collect();\n format!(\"({})\", inner.join(\", \"))\n }\n Map => {\n // A nested record-ish thing. Use the hint string when present\n // (which carries the language-specific concrete type name);\n // otherwise show an inline placeholder. Either way, the\n // nested record will be listed separately in the same\n // schema block if its name appears as another arg's type.\n s.hint.clone().unwrap_or_else(|| \"{..}\".into())\n }\n Optional => format!(\n \"?{}\",\n s.parameters\n .first()\n .map(render_schema_type)\n .unwrap_or_else(|| \"?\".into())\n ),\n Tensor => format!(\n \"Tensor<{}>\",\n s.parameters\n .first()\n .map(render_schema_type)\n .unwrap_or_else(|| \"?\".into())\n ),\n }\n}\n\n/// Try to extract a `TypeLayout` from a (name, schema_string, kind)\n/// triple. Returns None if any input is missing or the schema does not\n/// parse to a top-level Map. Tables (whose fields are arrays in the wire\n/// schema) render their fields by the array's element type, mirroring\n/// how the user wrote them in the source.\nfn extract_named_layout<'a>(\n type_name: Option<&'a str>,\n schema_str: Option<&str>,\n kind: Option<&'a str>,\n) -> Option> {\n use morloc_runtime::schema::SerialType;\n let name = type_name?;\n let schema = schema_str?;\n let kind = kind?;\n let parsed = morloc_runtime::schema::parse_schema(schema).ok()?;\n if parsed.serial_type != SerialType::Map {\n return None;\n }\n // For a table, every field's wire schema is an Array -- peel one layer\n // off so the user sees `name :: Str` instead of `name :: [Str]`.\n let strip_array = kind == \"table\";\n let fields = parsed\n .keys\n .iter()\n .zip(parsed.parameters.iter())\n .map(|(k, p)| {\n let inner = if strip_array && p.serial_type == SerialType::Array {\n p.parameters.first().unwrap_or(p)\n } else {\n p\n };\n (k.clone(), render_schema_type(inner))\n })\n .collect();\n Some(TypeLayout { name, kind, fields })\n}\n\n/// Walk every arg + the return of a command. For each typed entry, try\n/// to build a layout. Deduplicate by type name, preserving discovery\n/// order so the rendering matches the order types appear in the\n/// signature.\nfn collect_command_layouts<'a>(cmd: &'a Command) -> Vec> {\n use std::collections::HashSet;\n let mut seen: HashSet<&str> = HashSet::new();\n let mut out: Vec> = Vec::new();\n\n for arg in &cmd.args {\n // Skip unrolled groups without a group_opt: each field already\n // appears as its own flag in the usage, so the schema is redundant.\n // Keep the schema when group_opt is present (the user can pass the\n // entire record as JSON and needs the full field spec).\n if let Arg::Group { group_opt: None, .. } = arg {\n continue;\n }\n if let Some(layout) =\n extract_named_layout(arg.type_desc_str(), arg.schema_str(), arg.kind_constraint())\n {\n if seen.insert(layout.name) {\n out.push(layout);\n }\n }\n }\n\n let ret_kind = cmd\n .ret\n .constraints\n .iter()\n .find(|c| c.ctype == \"kind\")\n .and_then(|c| c.value.as_ref().and_then(|v| v.as_str()));\n if let Some(layout) =\n extract_named_layout(Some(&cmd.ret.type_desc), Some(&cmd.ret.schema), ret_kind)\n {\n if seen.insert(layout.name) {\n out.push(layout);\n }\n }\n\n out\n}\n\n/// Print the Record Schemas / Table Schemas sections for any named\n/// types referenced in this command's signature. The whole block is\n/// skipped when there are none.\nfn print_type_definitions(cmd: &Command) {\n let layouts = collect_command_layouts(cmd);\n if layouts.is_empty() {\n return;\n }\n\n let records: Vec<&TypeLayout> = layouts.iter().filter(|l| l.kind != \"table\").collect();\n let tables: Vec<&TypeLayout> = layouts.iter().filter(|l| l.kind == \"table\").collect();\n\n if !records.is_empty() {\n eprintln!(\"\\nRecord Schemas:\");\n print_layouts(&records);\n }\n if !tables.is_empty() {\n eprintln!(\"\\nTable Schemas:\");\n print_layouts(&tables);\n }\n}\n\n/// Render a list of layouts. Each layout shows its type name on its own\n/// line followed by the field list with `::`-aligned column widths.\n/// Definitions are separated by blank lines.\nfn print_layouts(defs: &[&TypeLayout]) {\n for (i, def) in defs.iter().enumerate() {\n if i > 0 {\n eprintln!();\n }\n eprintln!(\" {}\", def.name);\n\n let name_width = def\n .fields\n .iter()\n .map(|(k, _)| k.len())\n .max()\n .unwrap_or(0);\n for (k, v) in &def.fields {\n eprintln!(\" {:width$} :: {}\", k, v, width = name_width);\n }\n }\n}\n" }, { "path": "data/rust/morloc-nexus/src/main.rs", "content": "//! Morloc Nexus: CLI dispatcher for multi-language pool orchestration.\n//!\n//! Replaces data/nexus.c. Entry point for all morloc programs.\n//! Reads a .manifest JSON, spawns language pool daemons, and routes\n//! function calls to them over Unix sockets.\n\nmod dispatch;\nmod help;\nmod manifest;\nmod process;\n\nuse dispatch::NexusConfig;\n\n/// Resolve the morloc data directory: MORLOC_HOME if set, else ~/.local/share/morloc.\nfn morloc_home() -> String {\n std::env::var(\"MORLOC_HOME\").unwrap_or_else(|_| {\n format!(\n \"{}/.local/share/morloc\",\n std::env::var(\"HOME\").unwrap_or_else(|_| \"/root\".into())\n )\n })\n}\n\nfn main() {\n let args: Vec = std::env::args().collect();\n\n let mut config = NexusConfig::default();\n\n // First pass: parse nexus-level options\n let opt_end = dispatch::parse_nexus_options(&args, &mut config);\n\n // Handle --router mode (no manifest needed)\n if config.router_flag {\n run_router(&config);\n std::process::exit(0);\n }\n\n // If -h with no manifest argument, show nexus help\n let prog_name = args.first().map(|s| s.as_str()).unwrap_or(\"morloc-nexus\");\n if config.help_flag && opt_end >= args.len() {\n help::print_nexus_usage(prog_name);\n }\n\n // Manifest path: either an explicit argument or derived from argv[0].\n // In daemon mode (`./test --daemon`), the manifest is at `.manifest`.\n // In normal mode (`./test add 1 2`), argv[0] is also the manifest source.\n // An explicit path argument is only needed for multi-command mode.\n let manifest_path = if opt_end < args.len() {\n args[opt_end].clone()\n } else if config.daemon_flag || config.router_flag {\n // Daemon/router: derive from argv[0]\n args[0].clone()\n } else {\n help::print_nexus_usage(prog_name)\n };\n let prog_name = std::path::Path::new(&manifest_path)\n .file_name()\n .and_then(|n| n.to_str())\n .unwrap_or(&manifest_path)\n .to_string();\n let mut arg_cursor = if opt_end < args.len() { opt_end + 1 } else { args.len() };\n\n // Read and parse manifest\n let payload = match manifest::read_manifest_payload(&manifest_path) {\n Ok(p) => p,\n Err(e) => {\n eprintln!(\"Failed to load manifest '{}': {}\", manifest_path, e);\n std::process::exit(1);\n }\n };\n\n let manifest = match manifest::parse_manifest(&payload) {\n Ok(m) => m,\n Err(e) => {\n eprintln!(\"Failed to parse manifest '{}': {}\", manifest_path, e);\n std::process::exit(1);\n }\n };\n\n let single_command = manifest.commands.len() == 1 && manifest.groups.is_empty();\n\n // Second pass: parse options after manifest path (skip in single-command mode)\n let mut remaining_args = args.clone();\n if !single_command {\n arg_cursor = dispatch::parse_nexus_options(&args[opt_end..], &mut config) + opt_end;\n } else {\n // In single-command mode, extract daemon/server long options manually\n dispatch::extract_global_options(&mut remaining_args, &mut config);\n }\n\n // Pool paths in the manifest are absolute, so no chdir is needed.\n // This lets user programs resolve file paths relative to the caller's CWD.\n // Source imports in pools resolve via __file__-relative paths (Python sys.path)\n // or script-relative paths (R .morloc.source) rather than depending on CWD.\n\n // Validate pool executables exist\n if let Err(e) = process::validate_pools(&manifest.pools) {\n eprintln!(\"Error: {}\", e);\n std::process::exit(1);\n }\n\n // Handle help flag with manifest loaded\n if config.help_flag {\n if single_command {\n help::print_command_help_single(&prog_name, &manifest.commands[0]);\n } else {\n help::print_usage(&prog_name, &manifest);\n }\n }\n\n // Setup tmpdir and SHM\n let tmpdir = match process::make_tmpdir() {\n Ok(t) => t,\n Err(e) => {\n eprintln!(\"Error: {}\", e);\n std::process::exit(1);\n }\n };\n process::set_tmpdir(tmpdir.clone());\n\n let job_hash = process::make_job_hash(42);\n let shm_basename = format!(\"morloc-{}\", job_hash);\n\n // Initialize shared memory via libmorloc.so using dlsym.\n // CRITICAL: We must use dlsym to call the CDYLIB's shinit, not the rlib's.\n // The rlib and cdylib have separate static globals (VOLUMES, ALLOC_MUTEX, etc.).\n // All SHM operations in pool-facing C code go through the cdylib's globals.\n // If we call the rlib's shinit, the cdylib's globals stay uninitialized.\n {\n let _lib = unsafe { libc::dlopen(std::ptr::null(), libc::RTLD_NOW) };\n // RTLD_DEFAULT (NULL handle) searches in order: executable, then loaded libs\n // But the rlib symbols come first. Use RTLD_NEXT-style lookup via the .so path.\n let lib_path = std::ffi::CString::new(\n format!(\"{}/lib/libmorloc.so\", morloc_home())\n ).unwrap();\n let lib = unsafe { libc::dlopen(lib_path.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL) };\n if lib.is_null() {\n let err = unsafe { libc::dlerror() };\n let err_msg = if err.is_null() {\n \"unknown error\".to_string()\n } else {\n unsafe { std::ffi::CStr::from_ptr(err) }.to_string_lossy().into_owned()\n };\n eprintln!(\"Error: failed to load libmorloc.so: {}\", err_msg);\n process::clean_exit(1);\n }\n\n type ShmSetFallbackFn = unsafe extern \"C\" fn(*const std::ffi::c_char);\n type ShinitFn = unsafe extern \"C\" fn(*const std::ffi::c_char, usize, usize, *mut *mut std::ffi::c_char) -> *mut std::ffi::c_void;\n\n let set_fb_sym = std::ffi::CString::new(\"shm_set_fallback_dir\").unwrap();\n let shinit_sym = std::ffi::CString::new(\"shinit\").unwrap();\n\n let set_fb: ShmSetFallbackFn = unsafe { std::mem::transmute(libc::dlsym(lib, set_fb_sym.as_ptr())) };\n let do_shinit: ShinitFn = unsafe { std::mem::transmute(libc::dlsym(lib, shinit_sym.as_ptr())) };\n\n let tmpdir_c = std::ffi::CString::new(tmpdir.as_str()).unwrap();\n let basename_c = std::ffi::CString::new(shm_basename.as_str()).unwrap();\n let mut errmsg: *mut std::ffi::c_char = std::ptr::null_mut();\n unsafe {\n set_fb(tmpdir_c.as_ptr());\n let shm = do_shinit(basename_c.as_ptr(), 0, 0xffff, &mut errmsg);\n if shm.is_null() {\n let msg = if !errmsg.is_null() {\n let s = std::ffi::CStr::from_ptr(errmsg).to_string_lossy().into_owned();\n libc::free(errmsg as *mut std::ffi::c_void);\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: failed to initialize shared memory: {}\", msg);\n process::clean_exit(1);\n }\n }\n unsafe { libc::dlclose(lib) };\n }\n\n // Become subreaper for orphaned grandchildren\n process::set_child_subreaper();\n\n // Install signal handlers\n process::install_signal_handlers();\n\n // Setup sockets\n let mut sockets = process::setup_sockets(&manifest.pools, &tmpdir, &shm_basename);\n\n // Daemon mode\n if config.daemon_flag {\n let all_indices: Vec = (0..manifest.pools.len()).collect();\n if let Err(e) = process::start_daemons(&mut sockets, &all_indices) {\n eprintln!(\"Error: {}\", e);\n process::clean_exit(1);\n }\n\n // Build DaemonConfig and call daemon_run in libmorloc.so\n run_daemon(&config, &mut sockets, &shm_basename, &payload);\n process::clean_exit(0);\n }\n\n // Normal CLI mode\n if config.packet_path.is_none() {\n if single_command {\n // Single-command: dispatch directly to the command, no subcommand lookup\n // Allow optional command name prefix for backward compatibility\n let mut cmd_arg_start = arg_cursor;\n if cmd_arg_start < remaining_args.len()\n && remaining_args[cmd_arg_start] == manifest.commands[0].name\n {\n cmd_arg_start += 1;\n }\n dispatch::dispatch_command(\n &remaining_args,\n cmd_arg_start,\n &config,\n &manifest,\n &manifest.commands[0],\n &mut sockets,\n &prog_name,\n );\n } else {\n if arg_cursor >= remaining_args.len() {\n help::print_usage(&prog_name, &manifest);\n }\n dispatch::dispatch(\n &remaining_args,\n arg_cursor,\n &shm_basename,\n &config,\n &manifest,\n &mut sockets,\n &prog_name,\n );\n }\n } else {\n // Call-packet mode: read a pre-built call packet from file,\n // send to the appropriate pool, write result as MessagePack.\n // Used by SLURM workers on remote compute nodes.\n run_call_packet(&config, &tmpdir);\n }\n\n process::clean_exit(0);\n}\n\n/// Run the daemon event loop by calling daemon_run in libmorloc.so.\nfn run_daemon(\n config: &dispatch::NexusConfig,\n sockets: &mut [process::PoolSocket],\n shm_basename: &str,\n manifest_payload: &str,\n) {\n use std::ffi::{c_char, c_void, CString};\n use std::ptr;\n\n // daemon_run and parse_manifest signatures from libmorloc.so\n extern \"C\" {\n fn daemon_run(\n config: *mut c_void, // *mut DaemonConfig\n manifest: *mut c_void, // *mut Manifest (opaque)\n sockets: *mut c_void, // *mut MorlocSocket\n n_pools: usize,\n shm_basename: *const c_char,\n );\n fn parse_manifest(text: *const c_char, errmsg: *mut *mut c_char) -> *mut c_void;\n }\n\n // Build C MorlocSocket array (matches daemon_ffi::MorlocSocket layout)\n #[repr(C)]\n struct CMorlocSocket {\n lang: *mut c_char,\n syscmd: *mut *mut c_char,\n socket_filename: *mut c_char,\n pid: i32,\n }\n\n let n_pools = sockets.len();\n let mut c_sockets: Vec = Vec::with_capacity(n_pools);\n // Keep CStrings alive for the duration\n let mut _keepalive: Vec> = Vec::new();\n\n for sock in sockets.iter() {\n let lang_c = CString::new(sock.lang.as_str()).unwrap();\n let socket_c = CString::new(sock.socket_path.as_str()).unwrap();\n\n // Build NULL-terminated syscmd array\n let mut cmd_ptrs: Vec<*mut c_char> = Vec::new();\n let mut cmd_strs: Vec = Vec::new();\n for arg in &sock.syscmd {\n let c = CString::new(arg.to_bytes()).unwrap();\n cmd_ptrs.push(c.as_ptr() as *mut c_char);\n cmd_strs.push(c);\n }\n cmd_ptrs.push(ptr::null_mut());\n\n c_sockets.push(CMorlocSocket {\n lang: lang_c.as_ptr() as *mut c_char,\n syscmd: cmd_ptrs.as_ptr() as *mut *mut c_char,\n socket_filename: socket_c.as_ptr() as *mut c_char,\n pid: sock.pid,\n });\n\n // Keep everything alive\n _keepalive.push(cmd_strs);\n _keepalive.push(vec![lang_c, socket_c]);\n }\n\n // Build C DaemonConfig (matches daemon_ffi::DaemonConfig layout)\n #[repr(C)]\n struct CDaemonConfig {\n unix_socket_path: *const c_char,\n tcp_port: i32,\n http_port: i32,\n pool_check_fn: *const c_void, // Option as null\n pool_alive_fn: *const c_void, // Option as null\n n_pools: usize,\n eval_timeout: i32,\n }\n\n let unix_socket_cstr = config.unix_socket_path.as_ref()\n .map(|p| CString::new(p.as_str()).unwrap());\n\n let mut daemon_config = CDaemonConfig {\n unix_socket_path: unix_socket_cstr.as_ref()\n .map_or(ptr::null(), |c| c.as_ptr()),\n tcp_port: config.tcp_port.unwrap_or(0),\n http_port: config.http_port.unwrap_or(0),\n pool_check_fn: ptr::null(),\n pool_alive_fn: process::pool_is_alive_ptr(),\n n_pools,\n eval_timeout: config.eval_timeout,\n };\n\n // Parse manifest via the C FFI (so daemon_run gets the C-layout manifest).\n // The payload was already extracted from the wrapper script by the main flow.\n let manifest_c_str = CString::new(manifest_payload).unwrap();\n let mut errmsg: *mut c_char = ptr::null_mut();\n let c_manifest = unsafe { parse_manifest(manifest_c_str.as_ptr(), &mut errmsg) };\n if c_manifest.is_null() {\n let msg = if !errmsg.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut c_void) };\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: failed to parse manifest for daemon: {}\", msg);\n process::clean_exit(1);\n }\n\n let shm_c = CString::new(shm_basename).unwrap();\n\n unsafe {\n daemon_run(\n &mut daemon_config as *mut CDaemonConfig as *mut c_void,\n c_manifest,\n c_sockets.as_mut_ptr() as *mut c_void,\n n_pools,\n shm_c.as_ptr(),\n );\n }\n}\n\n/// Run the multi-program router daemon.\n/// Scans the fdb directory for .manifest files and serves them all via HTTP/TCP/Unix.\nfn run_router(config: &dispatch::NexusConfig) {\n use std::ffi::{c_char, c_void, CString};\n use std::ptr;\n\n extern \"C\" {\n fn router_init(fdb_path: *const c_char, errmsg: *mut *mut c_char) -> *mut c_void;\n fn router_run(config: *mut c_void, router: *mut c_void);\n fn router_free(router: *mut c_void);\n }\n\n let fdb_path = config.fdb_path.clone().unwrap_or_else(|| {\n format!(\"{}/fdb\", morloc_home())\n });\n let fdb_c = CString::new(fdb_path.as_str()).unwrap();\n\n let mut errmsg: *mut c_char = ptr::null_mut();\n let router = unsafe { router_init(fdb_c.as_ptr(), &mut errmsg) };\n if router.is_null() {\n let msg = if !errmsg.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut c_void) };\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: failed to initialize router: {}\", msg);\n std::process::exit(1);\n }\n\n // Build DaemonConfig for the router\n #[repr(C)]\n struct CDaemonConfig {\n unix_socket_path: *const c_char,\n tcp_port: i32,\n http_port: i32,\n pool_check_fn: *const c_void,\n pool_alive_fn: *const c_void,\n n_pools: usize,\n eval_timeout: i32,\n }\n\n let unix_cstr = config.unix_socket_path.as_ref()\n .map(|p| CString::new(p.as_str()).unwrap());\n\n let mut dc = CDaemonConfig {\n unix_socket_path: unix_cstr.as_ref().map_or(ptr::null(), |c| c.as_ptr()),\n tcp_port: config.tcp_port.unwrap_or(0),\n http_port: config.http_port.unwrap_or(0),\n pool_check_fn: ptr::null(),\n pool_alive_fn: ptr::null(),\n n_pools: 0,\n eval_timeout: if config.eval_timeout > 0 { config.eval_timeout } else { 30 },\n };\n\n unsafe {\n router_run(&mut dc as *mut CDaemonConfig as *mut c_void, router);\n router_free(router);\n }\n}\n\n/// Run a pre-built call packet on a remote worker node (SLURM mode).\n/// Reads a call packet from file, sends it to the pool, writes result as MessagePack.\nfn run_call_packet(config: &dispatch::NexusConfig, tmpdir: &str) {\n use std::ffi::{c_char, c_void, CString};\n use std::ptr;\n\n extern \"C\" {\n fn read_binary_file(\n filename: *const c_char, file_size: *mut usize, errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn send_and_receive_over_socket(\n socket_path: *const c_char, packet: *const u8, errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn get_morloc_data_packet_error_message(\n data: *const u8, errmsg: *mut *mut c_char,\n ) -> *mut c_char;\n fn read_schema_from_packet_meta(\n packet: *const u8, errmsg: *mut *mut c_char,\n ) -> *mut c_char;\n fn parse_schema(\n schema_str: *const c_char, errmsg: *mut *mut c_char,\n ) -> *mut morloc_runtime::cschema::CSchema;\n fn get_morloc_data_packet_value(\n data: *const u8, schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn pack_with_schema(\n mlc: *const c_void, schema: *const morloc_runtime::cschema::CSchema,\n mpkptr: *mut *mut c_char, mpk_size: *mut usize, errmsg: *mut *mut c_char,\n ) -> i32;\n fn write_atomic(\n filename: *const c_char, data: *const u8, size: usize, errmsg: *mut *mut c_char,\n ) -> i32;\n fn print_morloc_data_packet(\n packet: *const u8, schema: *const morloc_runtime::cschema::CSchema,\n errmsg: *mut *mut c_char,\n ) -> i32;\n }\n\n let packet_path = config.packet_path.as_ref().unwrap();\n let socket_base = match &config.socket_base {\n Some(s) => s.clone(),\n None => {\n eprintln!(\"Error: --socket-base required for call-packet mode\");\n process::clean_exit(1);\n }\n };\n let socket_path = format!(\"{}/{}\", tmpdir, socket_base);\n\n let packet_c = CString::new(packet_path.as_str()).unwrap();\n let socket_c = CString::new(socket_path.as_str()).unwrap();\n let mut errmsg: *mut c_char = ptr::null_mut();\n\n // Read call packet from file\n let mut packet_size: usize = 0;\n let call_packet = unsafe { read_binary_file(packet_c.as_ptr(), &mut packet_size, &mut errmsg) };\n if call_packet.is_null() || !errmsg.is_null() {\n let msg = if !errmsg.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut c_void) };\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: failed to read call packet '{}': {}\", packet_path, msg);\n process::clean_exit(1);\n }\n\n // Send to pool and receive response\n let result_packet = unsafe {\n send_and_receive_over_socket(socket_c.as_ptr(), call_packet, &mut errmsg)\n };\n unsafe { libc::free(call_packet as *mut c_void) };\n if result_packet.is_null() || !errmsg.is_null() {\n let msg = if !errmsg.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(errmsg) }.to_string_lossy().into_owned();\n unsafe { libc::free(errmsg as *mut c_void) };\n s\n } else {\n \"unknown error\".into()\n };\n eprintln!(\"Error: run failed: {}\", msg);\n process::clean_exit(1);\n }\n\n // Check for error in response\n let run_err = unsafe { get_morloc_data_packet_error_message(result_packet, &mut errmsg) };\n if !run_err.is_null() {\n let s = unsafe { std::ffi::CStr::from_ptr(run_err) }.to_string_lossy().into_owned();\n unsafe { libc::free(run_err as *mut c_void) };\n eprintln!(\"Error: run failed: {}\", s);\n process::clean_exit(1);\n }\n\n // If output-form is \"packet\", write raw packet to output file\n if config.output_format == dispatch::OutputFormat::Packet {\n if let Some(ref output_path) = config.output_path {\n let schema_str = unsafe { read_schema_from_packet_meta(result_packet, &mut errmsg) };\n let schema = if !schema_str.is_null() {\n unsafe { parse_schema(schema_str, &mut errmsg) }\n } else {\n ptr::null_mut()\n };\n unsafe {\n print_morloc_data_packet(result_packet, schema, &mut errmsg);\n };\n // Also write as msgpack file\n if !schema.is_null() {\n let mlc = unsafe { get_morloc_data_packet_value(result_packet, schema, &mut errmsg) };\n if !mlc.is_null() && errmsg.is_null() {\n let mut mpk_data: *mut c_char = ptr::null_mut();\n let mut mpk_size: usize = 0;\n unsafe { pack_with_schema(mlc as *const c_void, schema, &mut mpk_data, &mut mpk_size, &mut errmsg) };\n if !mpk_data.is_null() && errmsg.is_null() {\n let mpk_filename = format!(\"{}.mpk\", output_path);\n let mpk_c = CString::new(mpk_filename.as_str()).unwrap();\n unsafe { write_atomic(mpk_c.as_ptr(), mpk_data as *const u8, mpk_size, &mut errmsg) };\n unsafe { libc::free(mpk_data as *mut c_void) };\n }\n }\n }\n }\n }\n\n unsafe { libc::free(result_packet as *mut c_void) };\n}\n" }, { "path": "data/rust/morloc-nexus/src/manifest.rs", "content": "//! Manifest types -- thin re-export shim over the canonical schema\n//! definitions in the `morloc-manifest` crate.\n//!\n//! This module exists only so that existing code in this crate (and\n//! its consumers) can keep saying `crate::manifest::Manifest` without\n//! caring whether the types live here or in a sibling crate. The\n//! actual schema, with full doc comments and parsing logic, lives in\n//! `data/rust/morloc-manifest/src/lib.rs`.\n\npub use morloc_manifest::{\n parse_manifest, read_manifest_payload, Arg, Command,\n GroupEntry, Manifest, Pool,\n};\n" }, { "path": "data/rust/morloc-nexus/src/process.rs", "content": "//! Pool daemon process management, signal handling, and lifecycle.\n//!\n//! Replaces the fork/exec, SIGCHLD, SIGTERM, clean_exit logic from nexus.c.\n\nuse std::ffi::CString;\nuse std::path::Path;\nuse std::sync::atomic::{AtomicBool, AtomicI32, Ordering};\nuse std::time::Duration;\n\nuse crate::manifest::Pool;\n\npub const MAX_DAEMONS: usize = 32;\n\nconst INITIAL_PING_TIMEOUT: Duration = Duration::from_millis(10);\nconst INITIAL_RETRY_DELAY: Duration = Duration::from_millis(1);\nconst RETRY_MULTIPLIER: f64 = 1.25;\nconst MAX_RETRIES: usize = 16;\n\n// ── Global state for signal handlers ───────────────────────────────────────\n\n/// PIDs of spawned pool daemons. 0 = unused, -1 = reaped.\nstatic PIDS: [AtomicI32; MAX_DAEMONS] = {\n const INIT: AtomicI32 = AtomicI32::new(0);\n [INIT; MAX_DAEMONS]\n};\n\n/// Process group IDs for cleanup.\nstatic PGIDS: [AtomicI32; MAX_DAEMONS] = {\n const INIT: AtomicI32 = AtomicI32::new(0);\n [INIT; MAX_DAEMONS]\n};\n\n/// Exit statuses saved by SIGCHLD handler.\nstatic EXIT_STATUSES: [AtomicI32; MAX_DAEMONS] = {\n const INIT: AtomicI32 = AtomicI32::new(0);\n [INIT; MAX_DAEMONS]\n};\n\n/// Re-entrancy guard for clean_exit.\nstatic CLEANING_UP: AtomicBool = AtomicBool::new(false);\n\n/// Global tmpdir path (set once in main, read during cleanup).\nstatic TMPDIR: std::sync::OnceLock = std::sync::OnceLock::new();\n\n/// Socket info for each pool.\n///\n/// Pool stderr and stdout are intentionally NOT captured or intercepted by\n/// the nexus: a core morloc guarantee is that anything a sourced function\n/// prints to stderr/stdout is passed through unchanged. Raised exceptions\n/// are caught inside each pool's dispatch wrapper (see pool.py/pool.cpp/\n/// pool.R/pool.jl) and returned as morloc error packets, which the nexus\n/// then annotates with call-site context when bubbling them up.\npub struct PoolSocket {\n pub lang: String,\n pub socket_path: String,\n pub syscmd: Vec,\n pub pid: i32,\n}\n\n// ── Signal handlers (async-signal-safe) ────────────────────────────────────\n\n/// SIGCHLD handler: reap terminated children.\nextern \"C\" fn sigchld_handler(_sig: libc::c_int) {\n #[cfg(target_os = \"linux\")]\n let saved_errno = unsafe { *libc::__errno_location() };\n #[cfg(target_os = \"macos\")]\n let saved_errno = unsafe { *libc::__error() };\n loop {\n let mut status: libc::c_int = 0;\n let pid = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) };\n if pid <= 0 {\n break;\n }\n for i in 0..MAX_DAEMONS {\n if PIDS[i].load(Ordering::Relaxed) == pid {\n EXIT_STATUSES[i].store(status, Ordering::Relaxed);\n PIDS[i].store(-1, Ordering::Relaxed);\n break;\n }\n }\n }\n #[cfg(target_os = \"linux\")]\n unsafe { *libc::__errno_location() = saved_errno };\n #[cfg(target_os = \"macos\")]\n unsafe { *libc::__error() = saved_errno };\n}\n\n/// SIGTERM/SIGINT handler: clean shutdown.\nextern \"C\" fn signal_exit_handler(sig: libc::c_int) {\n if CLEANING_UP.load(Ordering::Relaxed) {\n unsafe { libc::_exit(128 + sig) };\n }\n clean_exit(128 + sig);\n}\n\n/// Install signal handlers.\npub fn install_signal_handlers() {\n unsafe {\n // SIGCHLD\n let mut sa: libc::sigaction = std::mem::zeroed();\n sa.sa_sigaction = sigchld_handler as *const () as usize;\n libc::sigemptyset(&mut sa.sa_mask);\n sa.sa_flags = libc::SA_RESTART | libc::SA_NOCLDSTOP;\n libc::sigaction(libc::SIGCHLD, &sa, std::ptr::null_mut());\n\n // SIGTERM and SIGINT\n let mut sa_exit: libc::sigaction = std::mem::zeroed();\n sa_exit.sa_sigaction = signal_exit_handler as *const () as usize;\n libc::sigemptyset(&mut sa_exit.sa_mask);\n sa_exit.sa_flags = 0;\n libc::sigaction(libc::SIGTERM, &sa_exit, std::ptr::null_mut());\n libc::sigaction(libc::SIGINT, &sa_exit, std::ptr::null_mut());\n }\n}\n\n/// Set the global tmpdir for cleanup.\npub fn set_tmpdir(path: String) {\n let _ = TMPDIR.set(path);\n}\n\n/// Get the tmpdir path.\npub fn get_tmpdir() -> Option<&'static str> {\n TMPDIR.get().map(|s| s.as_str())\n}\n\n// ── Clean exit ─────────────────────────────────────────────────────────────\n\n/// Terminate all pool daemons and clean up resources.\n///\n/// Race condition with stderr output: when a pool process is dying (e.g.,\n/// Python printing a traceback), its stderr writes may still be in a pipe\n/// buffer or mid-syscall when we send SIGTERM. The pool's signal handler\n/// (or SIG_DFL) may kill the process before its output reaches the\n/// terminal. We mitigate this by:\n/// 1. Flushing the nexus's own stderr first (so our error message is out)\n/// 2. Giving pools 200ms after SIGTERM before escalating to SIGKILL\n/// (up from the previous 50ms, which was too short for Python's\n/// atexit handlers and multiprocessing cleanup to flush buffers)\npub fn clean_exit(exit_code: i32) -> ! {\n CLEANING_UP.store(true, Ordering::SeqCst);\n\n // Flush nexus stderr so our error messages are visible even if\n // the process is killed by a parent (e.g., shell pipeline).\n unsafe { libc::fsync(2) };\n\n // Block SIGCHLD during cleanup\n unsafe {\n let mut block_chld: libc::sigset_t = std::mem::zeroed();\n libc::sigemptyset(&mut block_chld);\n libc::sigaddset(&mut block_chld, libc::SIGCHLD);\n libc::sigprocmask(libc::SIG_BLOCK, &block_chld, std::ptr::null_mut());\n }\n\n // Send SIGTERM to all pool process groups\n for i in 0..MAX_DAEMONS {\n let pgid = PGIDS[i].load(Ordering::Relaxed);\n if pgid > 0 {\n unsafe { libc::kill(-pgid, libc::SIGTERM) };\n }\n }\n\n // Wait for groups to exit (up to 200ms per group, then SIGKILL).\n // The 200ms window serves two purposes:\n // - Lets pool signal handlers run (Python's signal_handler in pool.py\n // calls close_daemon and cleans up shared memory)\n // - Lets any pending stderr writes (tracebacks, error messages) drain\n // to the terminal before the process is force-killed\n for i in 0..MAX_DAEMONS {\n let pgid = PGIDS[i].load(Ordering::Relaxed);\n if pgid <= 0 {\n continue;\n }\n\n // Reap any available children\n while unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) } > 0 {}\n if unsafe { libc::kill(-pgid, 0) } == -1 {\n continue;\n }\n\n let mut group_dead = false;\n for _ in 0..100 {\n while unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) } > 0 {}\n if unsafe { libc::kill(-pgid, 0) } == -1 {\n group_dead = true;\n break;\n }\n std::thread::sleep(Duration::from_millis(2));\n }\n\n if !group_dead {\n unsafe { libc::kill(-pgid, libc::SIGKILL) };\n std::thread::sleep(Duration::from_millis(50));\n }\n }\n\n // Final reap\n while unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) } > 0 {}\n\n // Clean up shared memory segments\n extern \"C\" {\n fn shclose(errmsg: *mut *mut std::ffi::c_char) -> bool;\n }\n unsafe {\n let mut err: *mut std::ffi::c_char = std::ptr::null_mut();\n shclose(&mut err);\n if !err.is_null() {\n libc::free(err as *mut libc::c_void);\n }\n }\n\n // Clean up tmpdir\n if let Some(dir) = get_tmpdir() {\n let _ = std::fs::remove_dir_all(dir);\n }\n\n std::process::exit(exit_code);\n}\n\n// ── Pool daemon spawning ───────────────────────────────────────────────────\n\n/// Setup socket descriptors for all pools from the manifest.\npub fn setup_sockets(pools: &[Pool], tmpdir: &str, shm_basename: &str) -> Vec {\n pools\n .iter()\n .map(|pool| {\n let socket_path = format!(\"{}/{}\", tmpdir, pool.socket);\n\n // Build syscmd: exec_args... socket_path tmpdir shm_basename\n let mut syscmd: Vec = pool\n .exec\n .iter()\n .map(|s| CString::new(s.as_str()).unwrap())\n .collect();\n syscmd.push(CString::new(socket_path.as_str()).unwrap());\n syscmd.push(CString::new(tmpdir).unwrap());\n syscmd.push(CString::new(shm_basename).unwrap());\n\n PoolSocket {\n lang: pool.lang.clone(),\n socket_path,\n syscmd,\n pid: 0,\n }\n })\n .collect()\n}\n\n/// Fork and exec a language pool daemon. Returns child PID.\n///\n/// The child inherits the nexus's stdin/stdout/stderr unchanged: anything a\n/// sourced function prints must reach the terminal byte-for-byte without\n/// morloc interposing. Runtime errors raised inside the pool are caught by\n/// the pool's own dispatch wrapper and returned as morloc error packets.\nfn start_language_server(socket: &PoolSocket) -> Result {\n let pid = unsafe { libc::fork() };\n\n if pid == 0 {\n // Child process\n unsafe { libc::setpgid(0, 0) };\n\n let argv: Vec<*const libc::c_char> = socket\n .syscmd\n .iter()\n .map(|s| s.as_ptr())\n .chain(std::iter::once(std::ptr::null()))\n .collect();\n\n unsafe {\n libc::execvp(argv[0], argv.as_ptr());\n }\n // Only reached if exec fails.\n eprintln!(\n \"execvp failed for {}: {}\",\n socket.lang,\n std::io::Error::last_os_error()\n );\n unsafe { libc::_exit(127) };\n } else if pid > 0 {\n // Parent: ensure child is in its own process group\n unsafe { libc::setpgid(pid, pid) };\n Ok(pid)\n } else {\n Err(format!(\"fork failed: {}\", std::io::Error::last_os_error()))\n }\n}\n\n/// Start pool daemons for the given socket indices and wait for them to respond to pings.\npub fn start_daemons(sockets: &mut [PoolSocket], indices: &[usize]) -> Result<(), String> {\n for &idx in indices {\n let pid = start_language_server(&sockets[idx])?;\n sockets[idx].pid = pid;\n PIDS[idx].store(pid, Ordering::Relaxed);\n PGIDS[idx].store(pid, Ordering::Relaxed);\n }\n\n // Wait for each daemon to respond to pings\n for &idx in indices {\n wait_for_daemon(&sockets[idx], idx)?;\n }\n\n Ok(())\n}\n\n/// Ping a daemon with exponential backoff until it responds.\n/// Matches the C nexus behavior: initial delay 1ms, multiplier 1.25,\n/// plus socket timeout that doubles from 10ms to ~10s.\nfn wait_for_daemon(socket: &PoolSocket, pool_index: usize) -> Result<(), String> {\n use morloc_runtime::packet::PacketHeader;\n use std::os::unix::net::UnixStream;\n use std::io::{Read, Write};\n\n let ping = PacketHeader::ping();\n let ping_bytes = ping.to_bytes();\n let mut retry_delay = INITIAL_RETRY_DELAY.as_secs_f64();\n let mut ping_timeout = INITIAL_PING_TIMEOUT;\n\n for attempt in 0..=MAX_RETRIES {\n // Check if child already died. The pool's stderr was inherited\n // directly, so any traceback it printed is already on the user's\n // terminal; the nexus just reports the exit status here.\n if PIDS[pool_index].load(Ordering::Relaxed) == -1 {\n let status = EXIT_STATUSES[pool_index].load(Ordering::Relaxed);\n return Err(format!(\n \"Pool process for '{}' died unexpectedly (status: {})\",\n socket.lang, status\n ));\n }\n\n // Try to connect and ping\n match UnixStream::connect(&socket.socket_path) {\n Ok(mut stream) => {\n let _ = stream.set_read_timeout(Some(ping_timeout));\n let _ = stream.set_write_timeout(Some(ping_timeout));\n\n if stream.write_all(&ping_bytes).is_ok() {\n let mut resp = [0u8; 32];\n if stream.read_exact(&mut resp).is_ok() {\n if let Ok(hdr) = PacketHeader::from_bytes(&resp) {\n if hdr.is_ping() {\n return Ok(());\n }\n }\n }\n }\n }\n Err(_) => {}\n }\n\n if attempt == MAX_RETRIES {\n return Err(format!(\n \"Failed to ping pool '{}' at {} after {} retries\",\n socket.lang, socket.socket_path, MAX_RETRIES\n ));\n }\n\n // Sleep with exponential backoff\n // Use the larger of retry_delay or ping_timeout to ensure we wait\n // long enough for slow-starting pools (R, Python)\n let wait = retry_delay.max(ping_timeout.as_secs_f64());\n let secs = wait as u64;\n let nanos = ((wait - secs as f64) * 1e9) as u32;\n std::thread::sleep(Duration::new(secs, nanos));\n retry_delay *= RETRY_MULTIPLIER;\n ping_timeout = ping_timeout * 2;\n }\n\n unreachable!()\n}\n\n/// Return a C-compatible function pointer for pool_is_alive.\npub fn pool_is_alive_ptr() -> *const std::ffi::c_void {\n extern \"C\" fn pool_alive_c(pool_index: usize) -> bool {\n pool_is_alive(pool_index)\n }\n pool_alive_c as *const std::ffi::c_void\n}\n\n/// Check if a pool at given index is alive.\npub fn pool_is_alive(pool_index: usize) -> bool {\n if pool_index >= MAX_DAEMONS {\n return false;\n }\n let pid = PIDS[pool_index].load(Ordering::Relaxed);\n if pid <= 0 {\n return false;\n }\n unsafe { libc::kill(pid, 0) == 0 }\n}\n\n/// Get the exit status of a reaped pool, returning signal/exit info.\npub fn pool_death_info(pool_index: usize) -> Option {\n if PIDS[pool_index].load(Ordering::Relaxed) != -1 {\n return None;\n }\n let st = EXIT_STATUSES[pool_index].load(Ordering::Relaxed);\n if libc::WIFSIGNALED(st) {\n let sig = libc::WTERMSIG(st);\n Some(format!(\"Pool process crashed with signal {sig}\"))\n } else if libc::WIFEXITED(st) {\n let code = libc::WEXITSTATUS(st);\n Some(format!(\"Pool process exited with status {code}\"))\n } else {\n Some(\"Pool process died unexpectedly\".into())\n }\n}\n\n/// Validate that all pool executables exist.\npub fn validate_pools(pools: &[Pool]) -> Result<(), String> {\n for pool in pools {\n if let Some(exec) = pool.exec.last() {\n if !Path::new(exec).exists() {\n return Err(format!(\n \"Build artifacts missing or stale. Pool file '{}' not found. Re-run `morloc make`.\",\n exec\n ));\n }\n }\n }\n Ok(())\n}\n\n/// Create a temporary directory for this nexus session.\npub fn make_tmpdir() -> Result {\n let template = CString::new(\"/tmp/morloc.XXXXXX\").unwrap();\n let mut buf = template.into_bytes_with_nul();\n let ptr = buf.as_mut_ptr() as *mut libc::c_char;\n let result = unsafe { libc::mkdtemp(ptr) };\n if result.is_null() {\n return Err(format!(\n \"Failed to create temporary directory: {}\",\n std::io::Error::last_os_error()\n ));\n }\n let cstr = unsafe { std::ffi::CStr::from_ptr(result) };\n Ok(cstr.to_string_lossy().into_owned())\n}\n\n/// Generate a job hash from seed, pid, and timestamps.\npub fn make_job_hash(seed: u64) -> u64 {\n use morloc_runtime::hash::xxh64;\n\n let pid = std::process::id() as u64;\n let now = std::time::SystemTime::now()\n .duration_since(std::time::UNIX_EPOCH)\n .unwrap_or_default();\n let epoch_ns = now.as_nanos() as u64;\n\n let data = format!(\"{}:{}:{}\", pid, epoch_ns, seed);\n xxh64(data.as_bytes())\n}\n\n/// Become a subreaper so orphaned grandchildren get reparented to us.\n/// Only available on Linux; no-op on other platforms.\npub fn set_child_subreaper() {\n #[cfg(target_os = \"linux\")]\n unsafe {\n libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0);\n }\n}\n" }, { "path": "data/rust/morloc-runtime/Cargo.toml", "content": "[package]\nname = \"morloc-runtime\"\nversion = \"0.81.0\"\nedition = \"2021\"\ndescription = \"Morloc runtime library: shared memory, serialization, IPC, packet protocol\"\n# Version is intentionally synchronized with the morloc compiler version\n# (see ../../../package.yaml). The C-FFI manifest reader rejects\n# manifests whose recorded morloc_version differs from this crate's\n# CARGO_PKG_VERSION at parse time. Bumping the morloc compiler version\n# requires bumping this version in lockstep.\n\n[lib]\ncrate-type = [\"cdylib\", \"staticlib\", \"rlib\"]\n\n[dependencies]\nmorloc-manifest = { path = \"../morloc-manifest\" }\nlibc = { workspace = true }\nserde = { workspace = true }\nserde_json = { workspace = true }\nrmp = \"0.8\"\nrmp-serde = { workspace = true }\ntwox-hash = { workspace = true }\nnix = { workspace = true }\nthiserror = { workspace = true }\n\n[build-dependencies]\ncbindgen = \"0.28\"\n# cc crate no longer needed -- all C files ported to Rust\n" }, { "path": "data/rust/morloc-runtime/build.rs", "content": "fn main() {\n let target = std::env::var(\"TARGET\").unwrap_or_default();\n println!(\"cargo:rustc-link-lib=pthread\");\n if target.contains(\"linux\") {\n println!(\"cargo:rustc-link-lib=rt\");\n }\n // The morloc compiler version is sourced from CARGO_PKG_VERSION\n // (this crate's Cargo.toml), which is intentionally kept in\n // lockstep with the morloc Haskell package.yaml.\n}\n" }, { "path": "data/rust/morloc-runtime/cbindgen.toml", "content": "language = \"C\"\nheader = \"/* Generated by cbindgen - do not edit manually */\"\nautogen_warning = \"\"\ninclude_version = false\ndocumentation_style = \"c99\"\nno_includes = true\n\n[export]\nprefix = \"\"\n\n[fn]\nprefix = \"\"\n\n[parse]\nparse_deps = false\n" }, { "path": "data/rust/morloc-runtime/src/arrow_ffi.rs", "content": "//! Arrow C Data Interface implementation.\n//! Replaces arrow.c. Provides SHM ↔ Arrow conversion.\n\nuse std::ffi::{c_char, c_void};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::shm::{self, RelPtr};\n\nconst ARROW_SHM_MAGIC: u32 = 0xA770DA7A;\nconst ARROW_BUFFER_ALIGN: usize = 64;\n\nfn arrow_align_up(x: usize) -> usize {\n (x + ARROW_BUFFER_ALIGN - 1) & !(ARROW_BUFFER_ALIGN - 1)\n}\n\n// ── Arrow C Data Interface structs (matching Apache spec) ────────────────────\n\n#[repr(C)]\npub struct ArrowSchema {\n pub format: *const c_char,\n pub name: *const c_char,\n pub metadata: *const c_char,\n pub flags: i64,\n pub n_children: i64,\n pub children: *mut *mut ArrowSchema,\n pub dictionary: *mut ArrowSchema,\n pub release: Option,\n pub private_data: *mut c_void,\n}\n\n#[repr(C)]\npub struct ArrowArray {\n pub length: i64,\n pub null_count: i64,\n pub offset: i64,\n pub n_buffers: i64,\n pub n_children: i64,\n pub buffers: *mut *const c_void,\n pub children: *mut *mut ArrowArray,\n pub dictionary: *mut ArrowArray,\n pub release: Option,\n pub private_data: *mut c_void,\n}\n\n// ── SHM header types ─────────────────────────────────────────────────────────\n\n#[repr(C)]\npub struct ArrowColumnDesc {\n pub col_type: u32, // morloc_serial_type\n pub length: u64,\n pub null_count: u64,\n pub name_offset: u32,\n pub name_length: u16,\n pub data_offset: u64,\n pub data_size: u64,\n}\n\n#[repr(C)]\npub struct ArrowShmHeader {\n pub magic: u32,\n pub n_columns: u32,\n pub n_rows: u64,\n pub total_size: u64,\n}\n\n// ── Type mapping ─────────────────────────────────────────────────────────────\n\n// Serial type constants matching C enum\nconst MORLOC_NIL: u32 = 0;\nconst MORLOC_BOOL: u32 = 1;\nconst MORLOC_SINT8: u32 = 2;\nconst MORLOC_SINT16: u32 = 3;\nconst MORLOC_SINT32: u32 = 4;\nconst MORLOC_SINT64: u32 = 5;\nconst MORLOC_UINT8: u32 = 6;\nconst MORLOC_UINT16: u32 = 7;\nconst MORLOC_UINT32: u32 = 8;\nconst MORLOC_UINT64: u32 = 9;\nconst MORLOC_FLOAT32: u32 = 10;\nconst MORLOC_FLOAT64: u32 = 11;\nconst MORLOC_STRING: u32 = 12;\n\n#[no_mangle]\npub extern \"C\" fn arrow_element_size(serial_type: u32) -> usize {\n match serial_type {\n MORLOC_BOOL | MORLOC_SINT8 | MORLOC_UINT8 => 1,\n MORLOC_SINT16 | MORLOC_UINT16 => 2,\n MORLOC_SINT32 | MORLOC_UINT32 | MORLOC_FLOAT32 => 4,\n MORLOC_SINT64 | MORLOC_UINT64 | MORLOC_FLOAT64 => 8,\n _ => 0,\n }\n}\n\n#[no_mangle]\npub extern \"C\" fn arrow_format_string(serial_type: u32) -> *const c_char {\n match serial_type {\n MORLOC_BOOL => b\"b\\0\".as_ptr() as *const c_char,\n MORLOC_SINT8 => b\"c\\0\".as_ptr() as *const c_char,\n MORLOC_UINT8 => b\"C\\0\".as_ptr() as *const c_char,\n MORLOC_SINT16 => b\"s\\0\".as_ptr() as *const c_char,\n MORLOC_UINT16 => b\"S\\0\".as_ptr() as *const c_char,\n MORLOC_SINT32 => b\"i\\0\".as_ptr() as *const c_char,\n MORLOC_UINT32 => b\"I\\0\".as_ptr() as *const c_char,\n MORLOC_SINT64 => b\"l\\0\".as_ptr() as *const c_char,\n MORLOC_UINT64 => b\"L\\0\".as_ptr() as *const c_char,\n MORLOC_FLOAT32 => b\"f\\0\".as_ptr() as *const c_char,\n MORLOC_FLOAT64 => b\"g\\0\".as_ptr() as *const c_char,\n MORLOC_STRING => b\"u\\0\".as_ptr() as *const c_char,\n _ => ptr::null(),\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn arrow_format_to_type(format: *const c_char) -> u32 {\n if format.is_null() || *format == 0 || *format.add(1) != 0 {\n return MORLOC_NIL;\n }\n match *format as u8 {\n b'b' => MORLOC_BOOL,\n b'c' => MORLOC_SINT8,\n b'C' => MORLOC_UINT8,\n b's' => MORLOC_SINT16,\n b'S' => MORLOC_UINT16,\n b'i' => MORLOC_SINT32,\n b'I' => MORLOC_UINT32,\n b'l' => MORLOC_SINT64,\n b'L' => MORLOC_UINT64,\n b'f' => MORLOC_FLOAT32,\n b'g' => MORLOC_FLOAT64,\n b'u' => MORLOC_STRING,\n _ => MORLOC_NIL,\n }\n}\n\n// ── Column accessors (used by arrow_json.c) ──────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn arrow_column_desc(\n header: *const ArrowShmHeader,\n col_index: u32,\n) -> *const ArrowColumnDesc {\n if header.is_null() || col_index >= (*header).n_columns {\n return ptr::null();\n }\n let descs = (header as *const u8).add(std::mem::size_of::()) as *const ArrowColumnDesc;\n descs.add(col_index as usize)\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn arrow_column_data(\n header: *const ArrowShmHeader,\n col_index: u32,\n) -> *const c_void {\n let desc = arrow_column_desc(header, col_index);\n if desc.is_null() { return ptr::null(); }\n (header as *const u8).add((*desc).data_offset as usize) as *const c_void\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn arrow_column_name(\n header: *const ArrowShmHeader,\n col_index: u32,\n) -> *const c_char {\n let desc = arrow_column_desc(header, col_index);\n if desc.is_null() { return ptr::null(); }\n (header as *const u8).add((*desc).name_offset as usize) as *const c_char\n}\n\n// ── arrow_to_shm ─────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn arrow_to_shm(\n array: *const ArrowArray,\n schema: *const ArrowSchema,\n errmsg: *mut *mut c_char,\n) -> RelPtr {\n clear_errmsg(errmsg);\n\n if array.is_null() || schema.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"NULL array or schema\".into()));\n return shm::RELNULL;\n }\n\n // Verify struct type\n if (*schema).format.is_null() || libc::strcmp((*schema).format, b\"+s\\0\".as_ptr() as *const c_char) != 0 {\n set_errmsg(errmsg, &MorlocError::Other(\"Expected struct schema (format '+s')\".into()));\n return shm::RELNULL;\n }\n\n let n_cols = (*schema).n_children as usize;\n let n_rows = (*array).length as usize;\n\n if n_cols == 0 {\n set_errmsg(errmsg, &MorlocError::Other(\"Arrow struct must have at least one column\".into()));\n return shm::RELNULL;\n }\n\n let header_size = std::mem::size_of::();\n let descs_size = n_cols * std::mem::size_of::();\n\n let mut names_size: usize = 0;\n for i in 0..n_cols {\n let child_schema = *(*schema).children.add(i);\n let name = (*child_schema).name;\n if !name.is_null() { names_size += libc::strlen(name); }\n }\n\n let data_start = arrow_align_up(header_size + descs_size + names_size);\n let mut total_size = data_start;\n\n for i in 0..n_cols {\n let child_schema = *(*schema).children.add(i);\n let col_type = arrow_format_to_type((*child_schema).format);\n let elem_size = arrow_element_size(col_type);\n if col_type == MORLOC_STRING {\n let child = *(*array).children.add(i);\n let offsets = if (*child).n_buffers >= 2 { *(*child).buffers.add(1) as *const i32 } else { ptr::null() };\n let str_data_size = if !offsets.is_null() {\n let off = (*child).offset as usize;\n (*offsets.add(off + n_rows) - *offsets.add(off)) as usize\n } else { 0 };\n total_size = arrow_align_up(total_size)\n + (n_rows + 1) * std::mem::size_of::()\n + str_data_size;\n } else {\n if elem_size == 0 {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Unsupported Arrow column type for column {}\", i)));\n return shm::RELNULL;\n }\n total_size = arrow_align_up(total_size) + elem_size * n_rows;\n }\n }\n\n let shm_ptr = match shm::shmalloc(total_size) {\n Ok(p) => p,\n Err(e) => { set_errmsg(errmsg, &e); return shm::RELNULL; }\n };\n ptr::write_bytes(shm_ptr, 0, total_size);\n\n // Write header\n let header = &mut *(shm_ptr as *mut ArrowShmHeader);\n header.magic = ARROW_SHM_MAGIC;\n header.n_columns = n_cols as u32;\n header.n_rows = n_rows as u64;\n header.total_size = total_size as u64;\n\n let descs = shm_ptr.add(header_size) as *mut ArrowColumnDesc;\n let mut name_cursor = header_size + descs_size;\n let mut data_cursor = data_start;\n\n for i in 0..n_cols {\n let child_schema = *(*schema).children.add(i);\n let child_array = *(*array).children.add(i);\n let col_type = arrow_format_to_type((*child_schema).format);\n\n data_cursor = arrow_align_up(data_cursor);\n\n let name = if (*child_schema).name.is_null() { b\"\\0\".as_ptr() as *const c_char } else { (*child_schema).name };\n let name_len = libc::strlen(name);\n\n let desc = &mut *descs.add(i);\n desc.col_type = col_type;\n desc.length = n_rows as u64;\n desc.null_count = (*child_array).null_count as u64;\n desc.name_offset = name_cursor as u32;\n desc.name_length = name_len as u16;\n desc.data_offset = data_cursor as u64;\n\n if name_len > 0 {\n ptr::copy_nonoverlapping(name as *const u8, shm_ptr.add(name_cursor), name_len);\n }\n name_cursor += name_len;\n\n if col_type == MORLOC_STRING {\n let src_offsets = if (*child_array).n_buffers >= 2 { *(*child_array).buffers.add(1) as *const i32 } else { ptr::null() };\n let src_data = if (*child_array).n_buffers >= 3 { *(*child_array).buffers.add(2) as *const u8 } else { ptr::null() };\n let arr_offset = (*child_array).offset as usize;\n\n let dst_offsets = shm_ptr.add(data_cursor) as *mut i32;\n let base = if !src_offsets.is_null() { *src_offsets.add(arr_offset) } else { 0 };\n for r in 0..=n_rows {\n *dst_offsets.add(r) = if !src_offsets.is_null() { *src_offsets.add(arr_offset + r) - base } else { 0 };\n }\n let offsets_size = (n_rows + 1) * std::mem::size_of::();\n\n let str_data_size = if !src_offsets.is_null() {\n (*src_offsets.add(arr_offset + n_rows) - base) as usize\n } else { 0 };\n if str_data_size > 0 && !src_data.is_null() {\n ptr::copy_nonoverlapping(src_data.add(base as usize), shm_ptr.add(data_cursor + offsets_size), str_data_size);\n }\n\n let buf_size = offsets_size + str_data_size;\n desc.data_size = buf_size as u64;\n data_cursor += buf_size;\n } else {\n let elem_size = arrow_element_size(col_type);\n let buf_size = elem_size * n_rows;\n desc.data_size = buf_size as u64;\n\n if (*child_array).n_buffers >= 2 && !(*(*child_array).buffers.add(1)).is_null() {\n let src = (*(*child_array).buffers.add(1) as *const u8).add((*child_array).offset as usize * elem_size);\n if buf_size > 0 {\n ptr::copy_nonoverlapping(src, shm_ptr.add(data_cursor), buf_size);\n }\n }\n data_cursor += buf_size;\n }\n }\n\n match shm::abs2rel(shm_ptr) {\n Ok(r) => r,\n Err(e) => { set_errmsg(errmsg, &e); shm::RELNULL }\n }\n}\n\n// ── arrow_validate ───────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn arrow_validate(\n header: *const ArrowShmHeader,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n if header.is_null() { set_errmsg(errmsg, &MorlocError::Other(\"NULL arrow header\".into())); return 1; }\n if (*header).magic != ARROW_SHM_MAGIC { set_errmsg(errmsg, &MorlocError::Other(\"Invalid arrow SHM magic\".into())); return 1; }\n if schema.is_null() { set_errmsg(errmsg, &MorlocError::Other(\"NULL schema for arrow validation\".into())); return 1; }\n // MORLOC_MAP = 16\n if (*schema).serial_type != crate::schema::SerialType::Map as u32 { set_errmsg(errmsg, &MorlocError::Other(\"Expected MORLOC_MAP schema for arrow table\".into())); return 1; }\n\n let n_cols = (*header).n_columns as usize;\n if n_cols != (*schema).size {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Column count mismatch: arrow has {}, schema has {}\", n_cols, (*schema).size)));\n return 1;\n }\n\n for i in 0..n_cols {\n let desc = arrow_column_desc(header, i as u32);\n if desc.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"NULL column descriptor at index {}\", i)));\n return 1;\n }\n let param_schema = *(*schema).parameters.add(i);\n if (*desc).col_type != (*param_schema).serial_type as u32 {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Column {} type mismatch\", i)));\n return 1;\n }\n }\n\n 0\n}\n\n// ── Release callbacks for arrow_from_shm ─────────────────────────────────────\n\nunsafe extern \"C\" fn arrow_shm_child_schema_release(schema: *mut ArrowSchema) {\n if schema.is_null() { return; }\n if !(*schema).name.is_null() { libc::free((*schema).name as *mut c_void); }\n (*schema).name = ptr::null();\n (*schema).release = None;\n}\n\nunsafe extern \"C\" fn arrow_shm_child_array_release(array: *mut ArrowArray) {\n if array.is_null() { return; }\n if !(*array).buffers.is_null() { libc::free((*array).buffers as *mut c_void); }\n (*array).buffers = ptr::null_mut();\n (*array).release = None;\n}\n\nunsafe extern \"C\" fn arrow_shm_schema_release(schema: *mut ArrowSchema) {\n if schema.is_null() { return; }\n for i in 0..(*schema).n_children as usize {\n let child = *(*schema).children.add(i);\n if !child.is_null() {\n if let Some(release) = (*child).release { release(child); }\n libc::free(child as *mut c_void);\n }\n }\n libc::free((*schema).children as *mut c_void);\n (*schema).children = ptr::null_mut();\n (*schema).release = None;\n}\n\nunsafe extern \"C\" fn arrow_shm_array_release(array: *mut ArrowArray) {\n if array.is_null() { return; }\n for i in 0..(*array).n_children as usize {\n let child = *(*array).children.add(i);\n if !child.is_null() {\n if let Some(release) = (*child).release { release(child); }\n libc::free(child as *mut c_void);\n }\n }\n libc::free((*array).children as *mut c_void);\n (*array).children = ptr::null_mut();\n if !(*array).buffers.is_null() { libc::free((*array).buffers as *mut c_void); }\n (*array).buffers = ptr::null_mut();\n (*array).release = None;\n}\n\n// ── arrow_from_shm ───────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn arrow_from_shm(\n header: *const ArrowShmHeader,\n out_schema: *mut ArrowSchema,\n out_array: *mut ArrowArray,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n if header.is_null() { set_errmsg(errmsg, &MorlocError::Other(\"NULL arrow header\".into())); return 1; }\n if (*header).magic != ARROW_SHM_MAGIC { set_errmsg(errmsg, &MorlocError::Other(\"Invalid arrow SHM magic\".into())); return 1; }\n\n let n_cols = (*header).n_columns as usize;\n let n_rows = (*header).n_rows as i64;\n\n // Parent schema (struct type)\n ptr::write_bytes(out_schema, 0, 1);\n (*out_schema).format = b\"+s\\0\".as_ptr() as *const c_char;\n (*out_schema).n_children = n_cols as i64;\n (*out_schema).children = libc::calloc(n_cols, std::mem::size_of::<*mut ArrowSchema>()) as *mut *mut ArrowSchema;\n (*out_schema).release = Some(arrow_shm_schema_release);\n\n // Parent array\n ptr::write_bytes(out_array, 0, 1);\n (*out_array).length = n_rows;\n (*out_array).n_buffers = 1;\n (*out_array).buffers = libc::calloc(1, std::mem::size_of::<*const c_void>()) as *mut *const c_void;\n (*out_array).n_children = n_cols as i64;\n (*out_array).children = libc::calloc(n_cols, std::mem::size_of::<*mut ArrowArray>()) as *mut *mut ArrowArray;\n (*out_array).release = Some(arrow_shm_array_release);\n\n for i in 0..n_cols {\n let desc = arrow_column_desc(header, i as u32);\n\n // Child schema\n let child_s = libc::calloc(1, std::mem::size_of::()) as *mut ArrowSchema;\n (*child_s).release = Some(arrow_shm_child_schema_release);\n *(*out_schema).children.add(i) = child_s;\n\n (*child_s).format = arrow_format_string((*desc).col_type);\n let raw_name = arrow_column_name(header, i as u32);\n let name_len = (*desc).name_length as usize;\n let name_copy = libc::calloc(name_len + 1, 1) as *mut c_char;\n if !raw_name.is_null() && name_len > 0 {\n ptr::copy_nonoverlapping(raw_name as *const u8, name_copy as *mut u8, name_len);\n }\n (*child_s).name = name_copy;\n\n // Child array\n let child_a = libc::calloc(1, std::mem::size_of::()) as *mut ArrowArray;\n (*child_a).release = Some(arrow_shm_child_array_release);\n *(*out_array).children.add(i) = child_a;\n\n (*child_a).length = n_rows;\n (*child_a).null_count = (*desc).null_count as i64;\n\n let col_buf = arrow_column_data(header, i as u32);\n if (*desc).col_type == MORLOC_STRING {\n (*child_a).n_buffers = 3;\n (*child_a).buffers = libc::calloc(3, std::mem::size_of::<*const c_void>()) as *mut *const c_void;\n *(*child_a).buffers.add(1) = col_buf; // offsets\n *(*child_a).buffers.add(2) = (col_buf as *const u8).add((n_rows as usize + 1) * std::mem::size_of::()) as *const c_void;\n } else {\n (*child_a).n_buffers = 2;\n (*child_a).buffers = libc::calloc(2, std::mem::size_of::<*const c_void>()) as *mut *const c_void;\n *(*child_a).buffers.add(1) = col_buf; // zero-copy data\n }\n }\n\n 0\n}\n\n// ── Arrow JSON/Table output (replaces arrow_json.c) ──────────────────────────\n\nunsafe fn print_arrow_value(desc: *const ArrowColumnDesc, col_data: *const u8, row: u64) {\n let row = row as usize;\n match (*desc).col_type {\n MORLOC_BOOL => {\n let v = *col_data.add(row);\n if v != 0 { libc::printf(b\"true\\0\".as_ptr() as *const c_char); }\n else { libc::printf(b\"false\\0\".as_ptr() as *const c_char); }\n }\n MORLOC_SINT8 => { libc::printf(b\"%d\\0\".as_ptr() as *const c_char, *(col_data as *const i8).add(row) as i32); }\n MORLOC_SINT16 => { libc::printf(b\"%d\\0\".as_ptr() as *const c_char, *(col_data as *const i16).add(row) as i32); }\n MORLOC_SINT32 => { libc::printf(b\"%d\\0\".as_ptr() as *const c_char, *(col_data as *const i32).add(row)); }\n MORLOC_SINT64 => { libc::printf(b\"%ld\\0\".as_ptr() as *const c_char, *(col_data as *const i64).add(row)); }\n MORLOC_UINT8 => { libc::printf(b\"%u\\0\".as_ptr() as *const c_char, *col_data.add(row) as u32); }\n MORLOC_UINT16 => { libc::printf(b\"%u\\0\".as_ptr() as *const c_char, *(col_data as *const u16).add(row) as u32); }\n MORLOC_UINT32 => { libc::printf(b\"%u\\0\".as_ptr() as *const c_char, *(col_data as *const u32).add(row)); }\n MORLOC_UINT64 => { libc::printf(b\"%lu\\0\".as_ptr() as *const c_char, *(col_data as *const u64).add(row)); }\n MORLOC_FLOAT32 => { libc::printf(b\"%.7g\\0\".as_ptr() as *const c_char, *(col_data as *const f32).add(row) as f64); }\n MORLOC_FLOAT64 => { libc::printf(b\"%.15g\\0\".as_ptr() as *const c_char, *(col_data as *const f64).add(row)); }\n MORLOC_STRING => {\n let offsets = col_data as *const i32;\n let str_data = offsets.add((*desc).length as usize + 1) as *const u8;\n let start = *offsets.add(row) as usize;\n let end = *offsets.add(row + 1) as usize;\n libc::putchar(b'\"' as i32);\n for i in start..end {\n let c = *str_data.add(i);\n match c {\n b'\"' => { libc::printf(b\"\\\\\\\"\\0\".as_ptr() as *const c_char); }\n b'\\\\' => { libc::printf(b\"\\\\\\\\\\0\".as_ptr() as *const c_char); }\n b'\\n' => { libc::printf(b\"\\\\n\\0\".as_ptr() as *const c_char); }\n b'\\r' => { libc::printf(b\"\\\\r\\0\".as_ptr() as *const c_char); }\n b'\\t' => { libc::printf(b\"\\\\t\\0\".as_ptr() as *const c_char); }\n _ if c < 32 => { libc::printf(b\"\\\\u%04x\\0\".as_ptr() as *const c_char, c as u32); }\n _ => { libc::putchar(c as i32); }\n }\n }\n libc::putchar(b'\"' as i32);\n }\n _ => { libc::printf(b\"null\\0\".as_ptr() as *const c_char); }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn print_arrow_as_json(\n data: *const c_void,\n errmsg: *mut *mut c_char,\n) -> bool {\n use crate::error::{clear_errmsg, set_errmsg, MorlocError};\n clear_errmsg(errmsg);\n\n let header = data as *const ArrowShmHeader;\n if header.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"NULL arrow data\".into()));\n return false;\n }\n if (*header).magic != ARROW_SHM_MAGIC {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Invalid arrow SHM magic: 0x{:08x}\", (*header).magic)));\n return false;\n }\n\n let n_cols = (*header).n_columns;\n let n_rows = (*header).n_rows;\n\n libc::putchar(b'[' as i32);\n for r in 0..n_rows {\n if r > 0 { libc::putchar(b',' as i32); }\n libc::putchar(b'{' as i32);\n for c in 0..n_cols {\n if c > 0 { libc::putchar(b',' as i32); }\n let desc = arrow_column_desc(header, c);\n let name = arrow_column_name(header, c);\n let col_data = arrow_column_data(header, c);\n if !name.is_null() {\n libc::printf(b\"\\\"%.*s\\\":\\0\".as_ptr() as *const c_char, (*desc).name_length as i32, name);\n }\n if !desc.is_null() && !col_data.is_null() {\n print_arrow_value(desc, col_data as *const u8, r);\n } else {\n libc::printf(b\"null\\0\".as_ptr() as *const c_char);\n }\n }\n libc::putchar(b'}' as i32);\n }\n libc::printf(b\"]\\n\\0\".as_ptr() as *const c_char);\n true\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn print_arrow_as_table(\n data: *const c_void,\n errmsg: *mut *mut c_char,\n) -> bool {\n use crate::error::{clear_errmsg, set_errmsg, MorlocError};\n clear_errmsg(errmsg);\n\n let header = data as *const ArrowShmHeader;\n if header.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"NULL arrow data\".into()));\n return false;\n }\n if (*header).magic != ARROW_SHM_MAGIC {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Invalid arrow SHM magic: 0x{:08x}\", (*header).magic)));\n return false;\n }\n\n let n_cols = (*header).n_columns;\n let n_rows = (*header).n_rows;\n\n for c in 0..n_cols {\n if c > 0 { libc::putchar(b'\\t' as i32); }\n let desc = arrow_column_desc(header, c);\n let name = arrow_column_name(header, c);\n if !name.is_null() && !desc.is_null() {\n libc::printf(b\"%.*s\\0\".as_ptr() as *const c_char, (*desc).name_length as i32, name);\n }\n }\n libc::putchar(b'\\n' as i32);\n\n for r in 0..n_rows {\n for c in 0..n_cols {\n if c > 0 { libc::putchar(b'\\t' as i32); }\n let desc = arrow_column_desc(header, c);\n let col_data = arrow_column_data(header, c);\n if !desc.is_null() && !col_data.is_null() {\n print_arrow_value(desc, col_data as *const u8, r);\n }\n }\n libc::putchar(b'\\n' as i32);\n }\n true\n}\n" }, { "path": "data/rust/morloc-runtime/src/cache.rs", "content": "//! File-based packet caching with xxHash keys.\n//! Replaces cache.c.\n\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::hash;\nuse crate::shm;\n\n// ── hash_voidstar ──────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn hash_voidstar(\n data: *const c_void,\n schema: *const CSchema,\n seed: u64,\n errmsg: *mut *mut c_char,\n) -> u64 {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n match hash_voidstar_inner(data as *const u8, &rs, seed) {\n Ok(h) => h,\n Err(e) => {\n set_errmsg(errmsg, &e);\n 0\n }\n }\n}\n\nfn hash_voidstar_inner(\n data: *const u8,\n schema: &crate::schema::Schema,\n seed: u64,\n) -> Result {\n use crate::schema::SerialType;\n\n // SAFETY: data points to voidstar data in SHM with layout described by schema.\n // All reads (Array headers, element data) are within schema-defined bounds.\n unsafe {\n match schema.serial_type {\n SerialType::String | SerialType::Array => {\n let arr = &*(data as *const shm::Array);\n let elem_width = if schema.parameters.is_empty() {\n 1 // string bytes\n } else {\n schema.parameters[0].width\n };\n let elem_data = shm::rel2abs(arr.data)?;\n\n if schema.is_fixed_width() || schema.serial_type == SerialType::String {\n let total = elem_width * arr.size;\n let bytes = std::slice::from_raw_parts(elem_data, total);\n Ok(hash::xxh64_with_seed(bytes, seed))\n } else {\n let mut h = seed;\n for i in 0..arr.size {\n h = hash_voidstar_inner(\n elem_data.add(i * elem_width),\n &schema.parameters[0],\n h,\n )?;\n }\n Ok(h)\n }\n }\n SerialType::Tuple | SerialType::Map => {\n if schema.is_fixed_width() {\n let bytes = std::slice::from_raw_parts(data, schema.width);\n Ok(hash::xxh64_with_seed(bytes, seed))\n } else {\n let mut h = seed;\n for i in 0..schema.parameters.len() {\n h = hash_voidstar_inner(\n data.add(schema.offsets[i]),\n &schema.parameters[i],\n h,\n )?;\n }\n Ok(h)\n }\n }\n _ => {\n let bytes = std::slice::from_raw_parts(data, schema.width);\n Ok(hash::xxh64_with_seed(bytes, seed))\n }\n }\n }\n}\n\n// ── hash_morloc_packet ─────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn hash_morloc_packet(\n packet: *const u8,\n schema: *const CSchema,\n seed: u64,\n hash_out: *mut u64,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n *hash_out = 0;\n\n extern \"C\" {\n fn read_morloc_packet_header(\n msg: *const u8,\n errmsg: *mut *mut c_char,\n ) -> *const crate::packet::PacketHeader;\n fn morloc_packet_size(packet: *const u8, errmsg: *mut *mut c_char) -> usize;\n fn get_morloc_data_packet_value(\n data: *const u8,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n let header = read_morloc_packet_header(packet, &mut err);\n if header.is_null() {\n if !err.is_null() {\n *errmsg = err;\n }\n return false;\n }\n\n let cmd_type = (*header).command_type();\n if cmd_type == crate::packet::PACKET_TYPE_CALL {\n let midx = { (*header).command.call.midx };\n *hash_out = crate::utility::mix(seed, midx as u64);\n let offset = { (*header).offset } as usize;\n let length = { (*header).length } as usize;\n let arg_data = packet.add(32 + offset);\n let mut arg_start = 0usize;\n while arg_start < length {\n let arg_size = morloc_packet_size(arg_data.add(arg_start), &mut err);\n if !err.is_null() {\n *errmsg = err;\n return false;\n }\n let arg_bytes = std::slice::from_raw_parts(arg_data.add(arg_start), arg_size);\n *hash_out = crate::utility::mix(*hash_out, hash::xxh64_with_seed(arg_bytes, *hash_out));\n arg_start += arg_size;\n }\n } else if cmd_type == crate::packet::PACKET_TYPE_DATA {\n let voidstar = get_morloc_data_packet_value(packet, schema, &mut err);\n if voidstar.is_null() {\n if !err.is_null() {\n *errmsg = err;\n }\n return false;\n }\n let rs = CSchema::to_rust(schema);\n match hash_voidstar_inner(voidstar, &rs, seed) {\n Ok(h) => *hash_out = h,\n Err(e) => {\n set_errmsg(errmsg, &e);\n return false;\n }\n }\n } else {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"Cannot hash packet with command 0x{:02x}\", cmd_type)),\n );\n return false;\n }\n\n true\n}\n\n// ── Cache filename generation ──────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_cache_filename_ext(\n key: u64,\n cache_path: *const c_char,\n ext: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n let path = CStr::from_ptr(cache_path).to_string_lossy();\n let extension = CStr::from_ptr(ext).to_string_lossy();\n let filename = format!(\"{}/{:016x}{}\", path, key, extension);\n match CString::new(filename) {\n Ok(cs) => cs.into_raw(),\n Err(_) => {\n set_errmsg(errmsg, &MorlocError::Other(\"CString conversion failed\".into()));\n ptr::null_mut()\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_cache_filename(\n key: u64,\n cache_path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n let ext = CString::new(\".packet\").unwrap();\n make_cache_filename_ext(key, cache_path, ext.as_ptr(), errmsg)\n}\n\n// ── Cache operations ───────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn put_cache_packet(\n voidstar: *const u8,\n schema: *const CSchema,\n key: u64,\n cache_path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn make_mpk_data_packet(filename: *const c_char, schema: *const CSchema) -> *mut u8;\n fn morloc_packet_size(packet: *const u8, errmsg: *mut *mut c_char) -> usize;\n fn pack_with_schema(\n mlc: *const c_void, schema: *const CSchema,\n mpk: *mut *mut c_char, mpk_size: *mut usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n fn write_atomic(\n filename: *const c_char, data: *const u8, size: usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n\n // Generate filenames\n let pkt_filename = make_cache_filename(key, cache_path, &mut err);\n if pkt_filename.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n\n let dat_ext = CString::new(\".dat\").unwrap();\n let dat_filename = make_cache_filename_ext(key, cache_path, dat_ext.as_ptr(), &mut err);\n if dat_filename.is_null() {\n libc::free(pkt_filename as *mut c_void);\n *errmsg = err;\n return ptr::null_mut();\n }\n\n // Create data packet pointing to the .dat file\n let data_packet = make_mpk_data_packet(dat_filename, schema);\n if data_packet.is_null() {\n libc::free(pkt_filename as *mut c_void);\n libc::free(dat_filename as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to create data packet\".into()));\n return ptr::null_mut();\n }\n\n let pkt_size = morloc_packet_size(data_packet, &mut err);\n\n // Pack voidstar to msgpack\n let mut mpk_data: *mut c_char = ptr::null_mut();\n let mut mpk_size: usize = 0;\n let rc = pack_with_schema(voidstar as *const c_void, schema, &mut mpk_data, &mut mpk_size, &mut err);\n if rc != 0 {\n libc::free(data_packet as *mut c_void);\n libc::free(pkt_filename as *mut c_void);\n libc::free(dat_filename as *mut c_void);\n *errmsg = err;\n return ptr::null_mut();\n }\n\n // Write packet file\n write_atomic(pkt_filename, data_packet, pkt_size, &mut err);\n libc::free(data_packet as *mut c_void);\n if !err.is_null() {\n libc::free(mpk_data as *mut c_void);\n libc::free(pkt_filename as *mut c_void);\n libc::free(dat_filename as *mut c_void);\n *errmsg = err;\n return ptr::null_mut();\n }\n\n // Write data file\n write_atomic(dat_filename, mpk_data as *const u8, mpk_size, &mut err);\n libc::free(mpk_data as *mut c_void);\n libc::free(dat_filename as *mut c_void);\n if !err.is_null() {\n libc::free(pkt_filename as *mut c_void);\n *errmsg = err;\n return ptr::null_mut();\n }\n\n // Return the packet filename\n let result = libc::strdup(pkt_filename);\n libc::free(pkt_filename as *mut c_void);\n result\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn get_cache_packet(\n key: u64,\n cache_path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n\n let filename = make_cache_filename(key, cache_path, &mut err);\n if filename.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n\n extern \"C\" {\n fn read_binary_file(\n filename: *const c_char, file_size: *mut usize,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n }\n\n let mut file_size: usize = 0;\n let data = read_binary_file(filename, &mut file_size, &mut err);\n libc::free(filename as *mut c_void);\n if data.is_null() {\n *errmsg = err;\n }\n data\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn del_cache_packet(\n key: u64,\n cache_path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n\n let filename = make_cache_filename(key, cache_path, &mut err);\n if filename.is_null() {\n *errmsg = err;\n return false;\n }\n\n let rc = libc::unlink(filename);\n if rc != 0 {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"Failed to delete cache file '{}'\",\n CStr::from_ptr(filename).to_string_lossy()\n )),\n );\n libc::free(filename as *mut c_void);\n return false;\n }\n libc::free(filename as *mut c_void);\n true\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn check_cache_packet(\n key: u64,\n cache_path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n\n let filename = make_cache_filename(key, cache_path, &mut err);\n if filename.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n\n let mut sb: libc::stat = std::mem::zeroed();\n if libc::stat(filename, &mut sb) == 0 {\n let result = libc::strdup(filename);\n libc::free(filename as *mut c_void);\n return result;\n }\n libc::free(filename as *mut c_void);\n ptr::null_mut() // Not an error — cache miss\n}\n" }, { "path": "data/rust/morloc-runtime/src/cli.rs", "content": "//! CLI argument handling and voidstar utility functions.\n//! Replaces cli.c.\n\nuse std::ffi::{c_char, c_void, CStr};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::packet;\nuse crate::shm;\n\n// ── argument_t lifecycle ───────────────────────────────────────────────────\n\n// argument_t is defined in eval.h (C). We use it opaquely via libc pointers.\n// The struct: { value: *mut c_char, fields: *mut *mut c_char, default_fields: *mut *mut c_char, size: usize }\n#[repr(C)]\npub struct ArgumentT {\n pub value: *mut c_char,\n pub fields: *mut *mut c_char,\n pub default_fields: *mut *mut c_char,\n pub size: usize,\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn initialize_positional(value: *mut c_char) -> *mut ArgumentT {\n let arg = libc::calloc(1, std::mem::size_of::()) as *mut ArgumentT;\n if arg.is_null() {\n return ptr::null_mut();\n }\n (*arg).value = if value.is_null() {\n ptr::null_mut()\n } else {\n libc::strdup(value)\n };\n (*arg).size = 0;\n arg\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn initialize_unrolled(\n size: usize,\n default_value: *mut c_char,\n fields: *mut *mut c_char,\n default_fields: *mut *mut c_char,\n) -> *mut ArgumentT {\n let arg = libc::calloc(1, std::mem::size_of::()) as *mut ArgumentT;\n if arg.is_null() {\n return ptr::null_mut();\n }\n (*arg).value = if default_value.is_null() {\n ptr::null_mut()\n } else {\n libc::strdup(default_value)\n };\n (*arg).size = size;\n\n (*arg).fields = libc::calloc(size, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n for i in 0..size {\n let f = *fields.add(i);\n if !f.is_null() {\n *(*arg).fields.add(i) = libc::strdup(f);\n }\n }\n\n (*arg).default_fields =\n libc::calloc(size, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n for i in 0..size {\n let d = *default_fields.add(i);\n if !d.is_null() {\n *(*arg).default_fields.add(i) = libc::strdup(d);\n }\n }\n\n arg\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn free_argument_t(arg: *mut ArgumentT) {\n if arg.is_null() {\n return;\n }\n if !(*arg).value.is_null() {\n libc::free((*arg).value as *mut c_void);\n }\n if !(*arg).fields.is_null() {\n for i in 0..(*arg).size {\n let f = *(*arg).fields.add(i);\n if !f.is_null() {\n libc::free(f as *mut c_void);\n }\n }\n libc::free((*arg).fields as *mut c_void);\n }\n if !(*arg).default_fields.is_null() {\n for i in 0..(*arg).size {\n let d = *(*arg).default_fields.add(i);\n if !d.is_null() {\n libc::free(d as *mut c_void);\n }\n }\n libc::free((*arg).default_fields as *mut c_void);\n }\n libc::free(arg as *mut c_void);\n}\n\n// ── shfree_by_schema ───────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn shfree_by_schema(\n ptr: *mut c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n if ptr.is_null() || schema.is_null() {\n return true;\n }\n let rs = CSchema::to_rust(schema);\n match shfree_by_schema_inner(ptr as *mut u8, &rs) {\n Ok(_) => true,\n Err(e) => {\n set_errmsg(errmsg, &e);\n false\n }\n }\n}\n\nfn shfree_by_schema_inner(\n ptr: *mut u8,\n schema: &crate::schema::Schema,\n) -> Result<(), MorlocError> {\n use crate::schema::SerialType;\n\n // SAFETY: ptr points to voidstar data in SHM with layout described by schema.\n // We recursively visit sub-structures and zero metadata before the parent shfree.\n unsafe {\n match schema.serial_type {\n SerialType::String | SerialType::Array => {\n let arr = &*(ptr as *const shm::Array);\n if arr.data > 0 {\n if !schema.parameters.is_empty() && !schema.parameters[0].is_fixed_width() {\n let arr_data = shm::rel2abs(arr.data)?;\n let elem_width = schema.parameters[0].width;\n for i in 0..arr.size {\n shfree_by_schema_inner(\n arr_data.add(i * elem_width),\n &schema.parameters[0],\n )?;\n }\n }\n }\n }\n SerialType::Tuple | SerialType::Map => {\n for i in 0..schema.parameters.len() {\n let child = ptr.add(schema.offsets[i]);\n shfree_by_schema_inner(child, &schema.parameters[i])?;\n }\n }\n SerialType::Tensor => {\n // shape and data are inline, freed by parent shfree\n }\n _ => {\n // fixed-size: no sub-data\n }\n }\n // Zero this node's metadata\n std::ptr::write_bytes(ptr, 0, schema.width);\n }\n Ok(())\n}\n\n// ── adjust_voidstar_relptrs ────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn adjust_voidstar_relptrs(\n data: *mut c_void,\n schema: *const CSchema,\n base_rel: shm::RelPtr,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n match adjust_relptrs_inner(data as *mut u8, &rs, base_rel) {\n Ok(_) => 0,\n Err(e) => {\n set_errmsg(errmsg, &e);\n 1\n }\n }\n}\n\nfn adjust_relptrs_inner(\n data: *mut u8,\n schema: &crate::schema::Schema,\n base_rel: shm::RelPtr,\n) -> Result<(), MorlocError> {\n use crate::schema::SerialType;\n\n // SAFETY: data points to a voidstar blob in SHM. We adjust relptrs in-place;\n // all pointer arithmetic stays within the blob's bounds as defined by schema.\n unsafe {\n match schema.serial_type {\n SerialType::String | SerialType::Array => {\n let arr = &mut *(data as *mut shm::Array);\n arr.data += base_rel;\n if !schema.parameters.is_empty() && !schema.parameters[0].is_fixed_width() {\n let arr_data = shm::rel2abs(arr.data)?;\n let elem_width = schema.parameters[0].width;\n for i in 0..arr.size {\n adjust_relptrs_inner(\n arr_data.add(i * elem_width),\n &schema.parameters[0],\n base_rel,\n )?;\n }\n }\n }\n SerialType::Tuple | SerialType::Map => {\n for i in 0..schema.parameters.len() {\n adjust_relptrs_inner(\n data.add(schema.offsets[i]),\n &schema.parameters[i],\n base_rel,\n )?;\n }\n }\n SerialType::Optional => {\n let tag = *data;\n if tag != 0 && !schema.parameters.is_empty() {\n let inner_offset = schema.offsets.first().copied().unwrap_or(\n shm::align_up(1, schema.parameters[0].alignment().max(1)),\n );\n adjust_relptrs_inner(\n data.add(inner_offset),\n &schema.parameters[0],\n base_rel,\n )?;\n }\n }\n SerialType::Tensor => {\n let tensor = &mut *(data as *mut shm::Tensor);\n if tensor.total_elements > 0 {\n tensor.shape += base_rel;\n tensor.data += base_rel;\n }\n }\n _ => {}\n }\n }\n Ok(())\n}\n\n// ── read_voidstar_binary ───────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_voidstar_binary(\n blob: *const u8,\n blob_size: usize,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n\n let base = match shm::shmalloc(blob_size) {\n Ok(p) => p,\n Err(e) => {\n set_errmsg(errmsg, &e);\n return ptr::null_mut();\n }\n };\n std::ptr::copy_nonoverlapping(blob, base, blob_size);\n\n let base_rel = match shm::abs2rel(base) {\n Ok(r) => r,\n Err(e) => {\n let _ = shm::shfree(base);\n set_errmsg(errmsg, &e);\n return ptr::null_mut();\n }\n };\n\n if let Err(e) = adjust_relptrs_inner(base, &rs, base_rel) {\n let _ = shm::shfree(base);\n set_errmsg(errmsg, &e);\n return ptr::null_mut();\n }\n\n base as *mut c_void\n}\n\n// ── load_morloc_data_file ──────────────────────────────────────────────────\n// This function is complex and calls many C functions (read_json_with_schema,\n// unpack_with_schema). Keep delegating to C for now via extern declarations.\n\n#[no_mangle]\npub unsafe extern \"C\" fn load_morloc_data_file(\n path: *const c_char,\n data: *mut u8,\n data_size: usize,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn read_json_with_schema(\n dest: *mut u8, json: *mut c_char, schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn unpack_with_schema(\n mpk: *const c_char, mpk_size: usize, schema: *const CSchema,\n mlcptr: *mut *mut c_void, errmsg: *mut *mut c_char,\n ) -> i32;\n }\n\n if data_size == 0 {\n set_errmsg(errmsg, &MorlocError::Other(\"Cannot parse 0-length data\".into()));\n return ptr::null_mut();\n }\n\n let path_str = CStr::from_ptr(path).to_string_lossy();\n let mut err: *mut c_char = ptr::null_mut();\n\n // 1. Extension-based dispatch\n if path_str.ends_with(\".json\") {\n let json_buf = libc::realloc(data as *mut c_void, data_size + 1) as *mut u8;\n if json_buf.is_null() {\n libc::free(data as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"realloc failed\".into()));\n return ptr::null_mut();\n }\n *json_buf.add(data_size) = 0;\n let result = read_json_with_schema(ptr::null_mut(), json_buf as *mut c_char, schema, &mut err);\n if !err.is_null() {\n libc::free(json_buf as *mut c_void);\n *errmsg = err;\n return ptr::null_mut();\n }\n libc::free(json_buf as *mut c_void);\n return result as *mut c_void;\n }\n\n if path_str.ends_with(\".mpk\") || path_str.ends_with(\".msgpack\") {\n let mut result: *mut c_void = ptr::null_mut();\n unpack_with_schema(data as *const c_char, data_size, schema, &mut result, &mut err);\n libc::free(data as *mut c_void);\n if !err.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n return result;\n }\n\n // 2. Check for morloc packet header\n if data_size >= 32 {\n let magic = *(data as *const u32);\n if magic == packet::PACKET_MAGIC {\n let header_bytes: &[u8; 32] = &*(data as *const [u8; 32]);\n if let Ok(header) = packet::PacketHeader::from_bytes(header_bytes) {\n if !header.is_data() {\n libc::free(data as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Expected data packet in '{}'\", path_str)));\n return ptr::null_mut();\n }\n let offset = { header.offset } as usize;\n let length = { header.length } as usize;\n let payload = data.add(32 + offset);\n let format = { header.command.data.format };\n\n if format == packet::PACKET_FORMAT_VOIDSTAR {\n let result = read_voidstar_binary(payload, length, schema, &mut err);\n libc::free(data as *mut c_void);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n return result;\n } else if format == packet::PACKET_FORMAT_MSGPACK {\n let mut result: *mut c_void = ptr::null_mut();\n unpack_with_schema(payload as *const c_char, length, schema, &mut result, &mut err);\n libc::free(data as *mut c_void);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n return result;\n } else {\n libc::free(data as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Unsupported format 0x{:02x} in '{}'\", format, path_str)));\n return ptr::null_mut();\n }\n }\n }\n }\n\n // 3. Try JSON\n let first_byte = *data;\n let may_be_json = matches!(first_byte,\n b'\\'' | b'\"' | b'[' | b'{' | b't' | b'f' | b'n' |\n b'\\t' | b'\\n' | b'\\r' | b' ' |\n b'0'..=b'9' | b'-'\n );\n\n if (data_size > 1 && may_be_json) || (data_size == 1 && first_byte >= b'0' && first_byte <= b'9') {\n let json_buf = libc::realloc(data as *mut c_void, data_size + 1) as *mut u8;\n if !json_buf.is_null() {\n *json_buf.add(data_size) = 0;\n let result = read_json_with_schema(ptr::null_mut(), json_buf as *mut c_char, schema, &mut err);\n if err.is_null() && !result.is_null() {\n libc::free(json_buf as *mut c_void);\n return result as *mut c_void;\n }\n if !err.is_null() { libc::free(err as *mut c_void); err = ptr::null_mut(); }\n // Fall through to try msgpack\n // Note: data pointer may have been invalidated by realloc\n // Use json_buf as the data pointer going forward\n let mut result: *mut c_void = ptr::null_mut();\n unpack_with_schema(json_buf as *const c_char, data_size, schema, &mut result, &mut err);\n libc::free(json_buf as *mut c_void);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n return result;\n }\n }\n\n // 4. Try msgpack\n let mut result: *mut c_void = ptr::null_mut();\n unpack_with_schema(data as *const c_char, data_size, schema, &mut result, &mut err);\n libc::free(data as *mut c_void);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n result\n}\n\n// ── upload_packet (static helper) ────────────────────────────────────────────\n\n/// Copy a voidstar packet into SHM, adjusting relptrs.\n///\n/// # Safety\n/// `dest` must point to schema.width writable bytes in SHM.\n/// `data` must point to a valid voidstar blob within [data, data_end].\nunsafe fn upload_packet(\n dest: *mut u8,\n data: *const u8,\n data_end: usize,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n\n match upload_packet_inner(dest, data, data_end, schema, &rs) {\n Ok(_) => 0,\n Err(e) => {\n set_errmsg(errmsg, &e);\n 1\n }\n }\n}\n\nunsafe fn upload_packet_inner(\n dest: *mut u8,\n data: *const u8,\n data_end: usize,\n schema: *const CSchema,\n rs: &crate::schema::Schema,\n) -> Result<(), MorlocError> {\n use crate::schema::SerialType;\n\n match rs.serial_type {\n SerialType::String | SerialType::Array => {\n if (data as usize + rs.width - 1) <= data_end {\n return Err(MorlocError::Packet(\"Data is too small to store an array header\".into()));\n }\n ptr::copy_nonoverlapping(data, dest, rs.width);\n let arr = &mut *(dest as *mut shm::Array);\n let arr_data_offset = arr.data as usize;\n let arr_data = data.add(arr_data_offset);\n let elem_width = rs.parameters[0].width;\n let arr_size = arr.size * elem_width;\n\n if (arr_data as usize + arr_size - 1) > data_end {\n return Err(MorlocError::Packet(\"Data is too small to contain array values\".into()));\n }\n\n let data_ptr = shm::shmemcpy(arr_data, arr_size)?;\n\n if !rs.is_fixed_width() {\n let elem_schema = &rs.parameters[0];\n // Need the C schema for each element\n let elem_c_schema = (*schema).parameters;\n if !elem_c_schema.is_null() {\n let elem_cs = *elem_c_schema;\n for i in 0..arr.size {\n upload_packet_inner(\n data_ptr.add(i * elem_width),\n arr_data.add(i * elem_width),\n data_end,\n elem_cs,\n elem_schema,\n )?;\n }\n }\n }\n\n arr.data = shm::abs2rel(data_ptr)?;\n }\n SerialType::Tuple | SerialType::Map => {\n for i in 0..rs.parameters.len() {\n let elem_cs = if (*schema).parameters.is_null() {\n return Err(MorlocError::Packet(\"NULL parameters in schema\".into()));\n } else {\n *(*schema).parameters.add(i)\n };\n upload_packet_inner(\n dest.add(rs.offsets[i]),\n data.add(rs.offsets[i]),\n data_end,\n elem_cs,\n &rs.parameters[i],\n )?;\n }\n }\n _ => {\n if (data as usize + rs.width - 1) > data_end {\n return Err(MorlocError::Packet(\"Given data packet is too small\".into()));\n }\n ptr::copy_nonoverlapping(data, dest, rs.width);\n }\n }\n Ok(())\n}\n\n// ── parse_cli_data_argument_singular ─────────────────────────────────────────\n\nunsafe fn parse_cli_data_argument_singular(\n mut dest: *mut u8,\n arg: *mut c_char,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn read_json_with_schema(\n dest: *mut u8, json: *mut c_char, schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn file_exists(filename: *const c_char) -> bool;\n fn read_binary_fd(file: *mut libc::FILE, file_size: *mut usize, errmsg: *mut *mut c_char) -> *mut u8;\n }\n\n let rs = CSchema::to_rust(schema);\n let mut err: *mut c_char = ptr::null_mut();\n let mut fd: *mut libc::FILE = ptr::null_mut();\n\n // handle STDIN\n let stdin_path = b\"/dev/stdin\\0\";\n let dash_path = b\"-\\0\";\n if libc::strcmp(arg, stdin_path.as_ptr() as *const c_char) == 0\n || libc::strcmp(arg, dash_path.as_ptr() as *const c_char) == 0\n {\n fd = libc::fdopen(libc::STDIN_FILENO, b\"rb\\0\".as_ptr() as *const c_char);\n } else if file_exists(arg) {\n fd = libc::fopen(arg, b\"rb\\0\".as_ptr() as *const c_char);\n if fd.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\n format!(\"The argument '{}' is a filename, but it can't be read\",\n CStr::from_ptr(arg).to_string_lossy())\n ));\n return ptr::null_mut();\n }\n }\n\n if fd.is_null() {\n // Literal JSON data\n if dest.is_null() {\n match shm::shcalloc(1, rs.width) {\n Ok(p) => dest = p,\n Err(e) => { set_errmsg(errmsg, &e); return ptr::null_mut(); }\n }\n }\n dest = read_json_with_schema(dest, arg, schema, &mut err);\n if !err.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n return dest;\n }\n\n // File or stdin\n let mut data_size: usize = 0;\n let data = read_binary_fd(fd, &mut data_size, &mut err);\n // Don't close stdin\n if fd != libc::fdopen(libc::STDIN_FILENO, b\"rb\\0\".as_ptr() as *const c_char) {\n libc::fclose(fd);\n }\n if !err.is_null() {\n if !data.is_null() { libc::free(data as *mut c_void); }\n *errmsg = err;\n return ptr::null_mut();\n }\n\n // Special case: RPTR packets\n if data_size >= 32 {\n let magic = *(data as *const u32);\n if magic == packet::PACKET_MAGIC {\n let header = &*(data as *const packet::PacketHeader);\n let source = header.command.data.source;\n let format = header.command.data.format;\n if source == packet::PACKET_SOURCE_RPTR && format == packet::PACKET_FORMAT_VOIDSTAR {\n if dest.is_null() {\n match shm::shcalloc(1, rs.width) {\n Ok(p) => dest = p,\n Err(e) => {\n libc::free(data as *mut c_void);\n set_errmsg(errmsg, &e);\n return ptr::null_mut();\n }\n }\n }\n let voidstar_ptr = data.add(32 + header.offset as usize);\n if upload_packet(dest, voidstar_ptr, voidstar_ptr as usize + data_size - 1, schema, &mut err) != 0 {\n libc::free(data as *mut c_void);\n *errmsg = err;\n return ptr::null_mut();\n }\n libc::free(data as *mut c_void);\n return dest;\n }\n }\n }\n\n // All other formats: canonical file loader (takes ownership of data)\n dest = load_morloc_data_file(arg, data, data_size, schema, &mut err) as *mut u8;\n if !err.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n dest\n}\n\n// ── parse_cli_data_argument_unrolled ─────────────────────────────────────────\n\nunsafe fn parse_cli_data_argument_unrolled(\n mut dest: *mut u8,\n default_value: *mut c_char,\n fields: *mut *mut c_char,\n default_fields: *mut *mut c_char,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n let mut err: *mut c_char = ptr::null_mut();\n let mut using_record_default = false;\n\n if dest.is_null() {\n match shm::shcalloc(1, rs.width) {\n Ok(p) => dest = p,\n Err(e) => { set_errmsg(errmsg, &e); return ptr::null_mut(); }\n }\n }\n\n if !default_value.is_null() {\n dest = parse_cli_data_argument_singular(dest, default_value, schema, &mut err);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n using_record_default = true;\n }\n\n use crate::schema::SerialType;\n match rs.serial_type {\n SerialType::Tuple | SerialType::Map => {\n for i in 0..rs.parameters.len() {\n let element_dest = dest.add(rs.offsets[i]);\n let field_val = *fields.add(i);\n let elem_cs = *(*schema).parameters.add(i);\n\n if !field_val.is_null() {\n // Free memory from default for this field\n shfree_by_schema(element_dest as *mut c_void, elem_cs, &mut err);\n if !err.is_null() { libc::free(err as *mut c_void); err = ptr::null_mut(); }\n\n let result = parse_cli_data_argument_singular(\n element_dest, field_val, elem_cs, &mut err,\n );\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n let _ = result; // result writes into element_dest\n } else if using_record_default {\n continue;\n } else {\n let default_field = *default_fields.add(i);\n if !default_field.is_null() {\n let result = parse_cli_data_argument_singular(\n element_dest, default_field, elem_cs, &mut err,\n );\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n let _ = result;\n } else {\n set_errmsg(errmsg, &MorlocError::Other(\n format!(\"Field {} missing with no default or default record\", i)\n ));\n return ptr::null_mut();\n }\n }\n }\n }\n _ => {\n set_errmsg(errmsg, &MorlocError::Other(\"Only record and tuple types may be unrolled\".into()));\n return ptr::null_mut();\n }\n }\n dest\n}\n\n// ── parse_cli_data_argument ──────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn parse_cli_data_argument(\n dest: *mut u8,\n arg: *const ArgumentT,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n\n let result = if (*arg).fields.is_null() {\n parse_cli_data_argument_singular(dest, (*arg).value, schema, &mut err)\n } else {\n parse_cli_data_argument_unrolled(\n dest, (*arg).value, (*arg).fields, (*arg).default_fields, schema, &mut err,\n )\n };\n\n if !err.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n if result.is_null() {\n return ptr::null_mut();\n }\n\n let relptr = match shm::abs2rel(result) {\n Ok(r) => r,\n Err(e) => { set_errmsg(errmsg, &e); return ptr::null_mut(); }\n };\n\n // Call the Rust make_standard_data_packet FFI\n crate::packet_ffi::make_standard_data_packet(relptr, schema)\n}\n\n// ── make_call_packet_from_cli ────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_call_packet_from_cli(\n dest: *mut u8,\n mid: u32,\n args: *mut *mut ArgumentT, // NULL-terminated\n arg_schema_strs: *mut *mut c_char, // NULL-terminated\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n\n // Count and parse schemas\n let mut nschemas: usize = 0;\n while !(*arg_schema_strs.add(nschemas)).is_null() {\n nschemas += 1;\n }\n\n let mut schemas: Vec<*mut CSchema> = Vec::with_capacity(nschemas);\n for i in 0..nschemas {\n let schema = crate::ffi::parse_schema(*arg_schema_strs.add(i), &mut err);\n if !err.is_null() {\n for s in &schemas { CSchema::free(*s); }\n *errmsg = err;\n return ptr::null_mut();\n }\n schemas.push(schema);\n }\n\n // Count args\n let mut nargs: usize = 0;\n while !(*args.add(nargs)).is_null() {\n nargs += 1;\n }\n\n // Parse each argument into a data packet\n let mut packet_args: Vec<*const u8> = Vec::with_capacity(nargs);\n for i in 0..nargs {\n let packet = parse_cli_data_argument(dest, *args.add(i), schemas[i], &mut err);\n if !err.is_null() {\n for p in &packet_args { libc::free(*p as *mut c_void); }\n for s in &schemas { CSchema::free(*s); }\n *errmsg = err;\n return ptr::null_mut();\n }\n packet_args.push(packet as *const u8);\n }\n\n // Build call packet\n let call_packet = crate::packet_ffi::make_morloc_local_call_packet(\n mid, packet_args.as_ptr(), nargs, &mut err,\n );\n\n for p in &packet_args { libc::free(*p as *mut c_void); }\n for s in &schemas { CSchema::free(*s); }\n\n if !err.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n call_packet\n}\n" }, { "path": "data/rust/morloc-runtime/src/cschema.rs", "content": "//! C-compatible Schema type for FFI.\n//! This module is always compiled (even with no-ffi-exports feature).\n\nuse std::ffi::{c_char, CStr, CString};\nuse std::ptr;\n\nuse crate::schema::{Schema, SerialType};\n\n/// C-compatible Schema struct matching the C `Schema` layout.\n#[repr(C)]\npub struct CSchema {\n pub serial_type: u32,\n pub size: usize,\n pub width: usize,\n pub offsets: *mut usize,\n pub hint: *mut c_char,\n pub parameters: *mut *mut CSchema,\n pub keys: *mut *mut c_char,\n}\n\nimpl CSchema {\n pub fn from_rust(schema: &Schema) -> *mut CSchema {\n let cs = Box::new(CSchema {\n serial_type: schema.serial_type as u32,\n size: schema.size,\n width: schema.width,\n offsets: if schema.offsets.is_empty() {\n ptr::null_mut()\n } else {\n let mut v = schema.offsets.clone().into_boxed_slice();\n let p = v.as_mut_ptr();\n std::mem::forget(v);\n p\n },\n hint: match &schema.hint {\n Some(s) => CString::new(s.as_str()).unwrap_or_default().into_raw(),\n None => ptr::null_mut(),\n },\n parameters: if schema.parameters.is_empty() {\n ptr::null_mut()\n } else {\n let mut ptrs: Vec<*mut CSchema> = schema\n .parameters\n .iter()\n .map(|p| CSchema::from_rust(p))\n .collect();\n let p = ptrs.as_mut_ptr();\n std::mem::forget(ptrs);\n p\n },\n keys: if schema.keys.is_empty() {\n ptr::null_mut()\n } else {\n let mut ptrs: Vec<*mut c_char> = schema\n .keys\n .iter()\n .map(|k| CString::new(k.as_str()).unwrap_or_default().into_raw())\n .collect();\n let p = ptrs.as_mut_ptr();\n std::mem::forget(ptrs);\n p\n },\n });\n Box::into_raw(cs)\n }\n\n /// Convert a C-allocated CSchema to a Rust Schema by deep-copying all data.\n ///\n /// # Safety\n /// `cs` must be null or a valid pointer to a CSchema allocated by `from_rust`\n /// or equivalent C code. All child pointers must be valid for `cs.size` entries.\n pub unsafe fn to_rust(cs: *const CSchema) -> Schema {\n if cs.is_null() {\n return Schema::primitive(SerialType::Nil);\n }\n let cs = &*cs;\n // SAFETY: SerialType is #[repr(u32)] and cs.serial_type was set from a valid SerialType.\n let serial_type = std::mem::transmute::(cs.serial_type);\n\n let offsets = if cs.offsets.is_null() || cs.size == 0 {\n Vec::new()\n } else {\n let n = match serial_type {\n SerialType::Tuple | SerialType::Map => cs.size,\n SerialType::Optional | SerialType::Tensor => 1,\n _ => 0,\n };\n if n > 0 {\n std::slice::from_raw_parts(cs.offsets, n).to_vec()\n } else {\n Vec::new()\n }\n };\n\n let parameters = if cs.parameters.is_null() || cs.size == 0 {\n Vec::new()\n } else {\n (0..cs.size)\n .map(|i| CSchema::to_rust(*cs.parameters.add(i)))\n .collect()\n };\n\n let keys = if cs.keys.is_null() || cs.size == 0 {\n Vec::new()\n } else {\n (0..cs.size)\n .filter_map(|i| {\n let p = *cs.keys.add(i);\n if p.is_null() { None }\n else { Some(CStr::from_ptr(p).to_string_lossy().into_owned()) }\n })\n .collect()\n };\n\n let hint = if cs.hint.is_null() {\n None\n } else {\n Some(CStr::from_ptr(cs.hint).to_string_lossy().into_owned())\n };\n\n Schema {\n serial_type,\n size: cs.size,\n width: cs.width,\n offsets,\n hint,\n parameters,\n keys,\n }\n }\n\n /// Free a CSchema and all its children (same logic as ffi::free_schema).\n ///\n /// # Safety\n /// `schema` must be null or a valid pointer previously returned by `from_rust`.\n pub unsafe fn free(schema: *mut CSchema) {\n if schema.is_null() { return; }\n let cs = Box::from_raw(schema);\n // SAFETY: cs.serial_type was set from a valid SerialType in from_rust.\n let st = std::mem::transmute::(cs.serial_type);\n if !cs.offsets.is_null() {\n let n = match st {\n SerialType::Tuple | SerialType::Map => cs.size,\n SerialType::Optional | SerialType::Tensor => 1,\n _ => 0,\n };\n if n > 0 { let _ = Vec::from_raw_parts(cs.offsets, n, n); }\n }\n if !cs.hint.is_null() { let _ = CString::from_raw(cs.hint); }\n if !cs.parameters.is_null() && cs.size > 0 {\n let ptrs = Vec::from_raw_parts(cs.parameters, cs.size, cs.size);\n for p in ptrs { CSchema::free(p); }\n }\n if !cs.keys.is_null() && cs.size > 0 {\n let ptrs = Vec::from_raw_parts(cs.keys, cs.size, cs.size);\n for p in ptrs { if !p.is_null() { let _ = CString::from_raw(p); } }\n }\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/daemon_ffi.rs", "content": "//! C ABI wrappers for daemon subsystems.\n//! Replaces daemon.c. Uses serde_json, HashMap, VecDeque, and std::thread.\n\nuse std::collections::HashMap;\nuse std::collections::VecDeque;\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\nuse std::sync::atomic::{AtomicBool, AtomicI32, Ordering};\nuse std::sync::{Arc, Condvar, Mutex};\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::hash;\nuse crate::http_ffi::{DaemonMethod, DaemonRequest, HttpRequest};\n\n// -- Constants ----------------------------------------------------------------\n\nconst DEFAULT_XXHASH_SEED: u64 = 0;\nconst MAX_LP_MESSAGE: u32 = 64 * 1024 * 1024;\n\n// -- Global state -------------------------------------------------------------\n\nstatic SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false);\nstatic G_EVAL_TIMEOUT: AtomicI32 = AtomicI32::new(30);\n\n// SAFETY: These globals are set once during daemon_run initialization (single-threaded)\n// and only read afterwards. The daemon is single-threaded for request dispatch.\nstatic mut G_POOL_ALIVE_FN: Option bool> = None;\nstatic mut G_N_POOLS: usize = 0;\nstatic mut G_BINDING_STORE: *mut BindingStore = ptr::null_mut();\n\n// -- C-compatible types -------------------------------------------------------\n\n/// Matches morloc_socket_t from call.h\n#[repr(C)]\npub struct MorlocSocket {\n pub lang: *mut c_char,\n pub syscmd: *mut *mut c_char,\n pub socket_filename: *mut c_char,\n pub pid: i32,\n}\n\n/// Matches daemon_config_t from daemon.h\n#[repr(C)]\npub struct DaemonConfig {\n pub unix_socket_path: *const c_char,\n pub tcp_port: i32,\n pub http_port: i32,\n pub pool_check_fn: Option,\n pub pool_alive_fn: Option bool>,\n pub n_pools: usize,\n pub eval_timeout: i32,\n}\n\n/// Matches daemon_response_t from daemon.h\n#[repr(C)]\npub struct DaemonResponse {\n pub id: *mut c_char,\n pub success: bool,\n pub result_json: *mut c_char,\n pub error: *mut c_char,\n}\n\n// -- Binding store (replaces linear-probe hash table with HashMap) ------------\n\nstruct BindingEntry {\n hash: u64,\n expr: String,\n #[allow(dead_code)]\n artifact_dir: String,\n type_sig: Option,\n names: Vec,\n}\n\nstruct BindingStore {\n entries: HashMap,\n /// Index from name -> hash for name-based lookup\n name_index: HashMap,\n base_dir: String,\n}\n\nimpl BindingStore {\n fn new(base_dir: &str) -> Self {\n let _ = std::fs::create_dir_all(base_dir);\n BindingStore {\n entries: HashMap::new(),\n name_index: HashMap::new(),\n base_dir: base_dir.to_string(),\n }\n }\n\n fn lookup_hash(&self, hash: u64) -> Option<&BindingEntry> {\n self.entries.get(&hash)\n }\n\n fn lookup_name(&self, name: &str) -> Option<&BindingEntry> {\n let hash = self.name_index.get(name)?;\n self.entries.get(hash)\n }\n\n fn add_name(&mut self, hash: u64, name: &str) {\n if let Some(entry) = self.entries.get_mut(&hash) {\n if !entry.names.contains(&name.to_string()) {\n entry.names.push(name.to_string());\n }\n }\n self.name_index.insert(name.to_string(), hash);\n }\n\n fn bind(&mut self, expr: &str, name: Option<&str>, eval_timeout: i32) -> Option {\n let hv = hash::xxh64_with_seed(expr.as_bytes(), DEFAULT_XXHASH_SEED);\n\n if self.entries.contains_key(&hv) {\n if let Some(n) = name {\n self.add_name(hv, n);\n }\n return Some(hv);\n }\n\n let hash_hex = format!(\"{:016x}\", hv);\n let artifact_dir = format!(\"{}/{}\", self.base_dir, hash_hex);\n\n // Fork morloc eval --save\n unsafe {\n let mut stdout_pipe = [0i32; 2];\n let mut stderr_pipe = [0i32; 2];\n if libc::pipe(stdout_pipe.as_mut_ptr()) != 0\n || libc::pipe(stderr_pipe.as_mut_ptr()) != 0\n {\n return None;\n }\n\n let pid = libc::fork();\n if pid < 0 {\n libc::close(stdout_pipe[0]);\n libc::close(stdout_pipe[1]);\n libc::close(stderr_pipe[0]);\n libc::close(stderr_pipe[1]);\n return None;\n }\n\n if pid == 0 {\n // Child\n libc::close(stdout_pipe[0]);\n libc::close(stderr_pipe[0]);\n libc::dup2(stdout_pipe[1], libc::STDOUT_FILENO);\n libc::dup2(stderr_pipe[1], libc::STDERR_FILENO);\n libc::close(stdout_pipe[1]);\n libc::close(stderr_pipe[1]);\n\n if eval_timeout > 0 {\n let cpu_limit = libc::rlimit {\n rlim_cur: eval_timeout as libc::rlim_t,\n rlim_max: (eval_timeout + 5) as libc::rlim_t,\n };\n libc::setrlimit(libc::RLIMIT_CPU, &cpu_limit);\n let as_limit = libc::rlimit {\n rlim_cur: 2 * 1024 * 1024 * 1024,\n rlim_max: 2 * 1024 * 1024 * 1024,\n };\n libc::setrlimit(libc::RLIMIT_AS, &as_limit);\n }\n\n let cmd = CString::new(\"morloc\").unwrap();\n let arg_eval = CString::new(\"eval\").unwrap();\n let arg_save = CString::new(\"--save\").unwrap();\n let arg_hex = CString::new(hash_hex.as_str()).unwrap();\n let arg_expr = CString::new(expr).unwrap();\n libc::execlp(\n cmd.as_ptr(),\n cmd.as_ptr(),\n arg_eval.as_ptr(),\n arg_save.as_ptr(),\n arg_hex.as_ptr(),\n arg_expr.as_ptr(),\n ptr::null::(),\n );\n libc::_exit(127);\n }\n\n // Parent\n libc::close(stdout_pipe[1]);\n libc::close(stderr_pipe[1]);\n\n let mut stderr_buf = vec![0u8; 4096];\n let mut stderr_len: usize = 0;\n loop {\n let n = libc::read(\n stderr_pipe[0],\n stderr_buf.as_mut_ptr().add(stderr_len) as *mut c_void,\n stderr_buf.len() - stderr_len - 1,\n );\n if n <= 0 {\n break;\n }\n stderr_len += n as usize;\n }\n libc::close(stdout_pipe[0]);\n libc::close(stderr_pipe[0]);\n\n let mut status: i32 = 0;\n libc::waitpid(pid, &mut status, 0);\n\n if !libc::WIFEXITED(status) || libc::WEXITSTATUS(status) != 0 {\n stderr_buf.truncate(stderr_len);\n let msg = String::from_utf8_lossy(&stderr_buf);\n eprintln!(\"binding_store_bind: morloc eval --save failed: {}\", msg);\n return None;\n }\n }\n\n let entry = BindingEntry {\n hash: hv,\n expr: expr.to_string(),\n artifact_dir,\n type_sig: None,\n names: Vec::new(),\n };\n self.entries.insert(hv, entry);\n if let Some(n) = name {\n self.add_name(hv, n);\n }\n\n Some(hv)\n }\n\n fn list_json(&self) -> String {\n #[derive(serde::Serialize)]\n struct BindingInfo {\n hash: String,\n expr: String,\n #[serde(skip_serializing_if = \"Option::is_none\")]\n r#type: Option,\n names: Vec,\n }\n #[derive(serde::Serialize)]\n struct BindingsList {\n bindings: Vec,\n }\n let bindings: Vec = self\n .entries\n .values()\n .map(|e| BindingInfo {\n hash: format!(\"{:016x}\", e.hash),\n expr: e.expr.clone(),\n r#type: e.type_sig.clone(),\n names: e.names.clone(),\n })\n .collect();\n serde_json::to_string(&BindingsList { bindings }).unwrap_or_default()\n }\n\n fn unbind(&mut self, name: &str) -> bool {\n let hash = match self.name_index.remove(name) {\n Some(h) => h,\n None => return false,\n };\n if let Some(entry) = self.entries.get_mut(&hash) {\n entry.names.retain(|n| n != name);\n }\n true\n }\n}\n\n// -- C-exported binding store functions ---------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn binding_store_init(base_dir: *const c_char) -> *mut c_void {\n let dir = CStr::from_ptr(base_dir).to_string_lossy().into_owned();\n let store = Box::new(BindingStore::new(&dir));\n Box::into_raw(store) as *mut c_void\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn binding_store_free(store: *mut c_void) {\n if !store.is_null() {\n drop(Box::from_raw(store as *mut BindingStore));\n }\n}\n\n// -- Request parsing (serde_json) ---------------------------------------------\n\n#[derive(serde::Deserialize)]\nstruct JsonRequest {\n id: Option,\n method: Option,\n command: Option,\n args: Option,\n expr: Option,\n name: Option,\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_parse_request(\n json: *const c_char,\n len: usize,\n errmsg: *mut *mut c_char,\n) -> *mut DaemonRequest {\n clear_errmsg(errmsg);\n\n let slice = std::slice::from_raw_parts(json as *const u8, len);\n let text = match std::str::from_utf8(slice) {\n Ok(s) => s,\n Err(_) => {\n set_errmsg(errmsg, &MorlocError::Other(\"Invalid UTF-8 in request\".into()));\n return ptr::null_mut();\n }\n };\n\n let parsed: JsonRequest = match serde_json::from_str(text) {\n Ok(r) => r,\n Err(e) => {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"Failed to parse request JSON: {}\", e)),\n );\n return ptr::null_mut();\n }\n };\n\n let req = libc::calloc(1, std::mem::size_of::()) as *mut DaemonRequest;\n if req.is_null() {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to allocate daemon_request_t\".into()),\n );\n return ptr::null_mut();\n }\n\n if let Some(id) = &parsed.id {\n let c = CString::new(id.as_str()).unwrap_or_default();\n (*req).id = libc::strdup(c.as_ptr());\n }\n\n if let Some(method) = &parsed.method {\n (*req).method = match method.as_str() {\n \"call\" => DaemonMethod::Call,\n \"discover\" => DaemonMethod::Discover,\n \"health\" => DaemonMethod::Health,\n \"eval\" => DaemonMethod::Eval,\n \"typecheck\" => DaemonMethod::Typecheck,\n \"bind\" => DaemonMethod::Bind,\n \"bindings\" => DaemonMethod::Bindings,\n \"unbind\" => DaemonMethod::Unbind,\n _ => {\n daemon_free_request(req);\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"Unknown method: {}\", method)),\n );\n return ptr::null_mut();\n }\n };\n }\n\n if let Some(cmd) = &parsed.command {\n let c = CString::new(cmd.as_str()).unwrap_or_default();\n (*req).command = libc::strdup(c.as_ptr());\n }\n\n if let Some(args) = &parsed.args {\n let args_str = serde_json::to_string(args).unwrap_or_default();\n let c = CString::new(args_str).unwrap_or_default();\n (*req).args_json = libc::strdup(c.as_ptr());\n }\n\n if let Some(expr) = &parsed.expr {\n let c = CString::new(expr.as_str()).unwrap_or_default();\n (*req).expr = libc::strdup(c.as_ptr());\n }\n\n if let Some(name) = &parsed.name {\n let c = CString::new(name.as_str()).unwrap_or_default();\n (*req).name = libc::strdup(c.as_ptr());\n }\n\n req\n}\n\n// -- Response parsing (serde_json) --------------------------------------------\n\n#[derive(serde::Deserialize)]\nstruct JsonResponse {\n id: Option,\n status: Option,\n result: Option,\n error: Option,\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_parse_response(\n json: *const c_char,\n len: usize,\n errmsg: *mut *mut c_char,\n) -> *mut DaemonResponse {\n clear_errmsg(errmsg);\n\n let slice = std::slice::from_raw_parts(json as *const u8, len);\n let text = match std::str::from_utf8(slice) {\n Ok(s) => s,\n Err(_) => {\n set_errmsg(errmsg, &MorlocError::Other(\"Invalid UTF-8 in response\".into()));\n return ptr::null_mut();\n }\n };\n\n let parsed: JsonResponse = match serde_json::from_str(text) {\n Ok(r) => r,\n Err(e) => {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"Failed to parse response JSON: {}\", e)),\n );\n return ptr::null_mut();\n }\n };\n\n let resp = libc::calloc(1, std::mem::size_of::()) as *mut DaemonResponse;\n if resp.is_null() {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to allocate daemon_response_t\".into()),\n );\n return ptr::null_mut();\n }\n\n if let Some(id) = &parsed.id {\n let c = CString::new(id.as_str()).unwrap_or_default();\n (*resp).id = libc::strdup(c.as_ptr());\n }\n\n (*resp).success = parsed\n .status\n .as_deref()\n .map(|s| s == \"ok\")\n .unwrap_or(false);\n\n if let Some(result) = &parsed.result {\n let s = serde_json::to_string(result).unwrap_or_default();\n let c = CString::new(s).unwrap_or_default();\n (*resp).result_json = libc::strdup(c.as_ptr());\n }\n\n if let Some(error) = &parsed.error {\n let c = CString::new(error.as_str()).unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n }\n\n resp\n}\n\n// -- Free functions -----------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_free_request(req: *mut DaemonRequest) {\n if req.is_null() {\n return;\n }\n if !(*req).id.is_null() {\n libc::free((*req).id as *mut c_void);\n }\n if !(*req).command.is_null() {\n libc::free((*req).command as *mut c_void);\n }\n if !(*req).args_json.is_null() {\n libc::free((*req).args_json as *mut c_void);\n }\n if !(*req).expr.is_null() {\n libc::free((*req).expr as *mut c_void);\n }\n if !(*req).name.is_null() {\n libc::free((*req).name as *mut c_void);\n }\n libc::free(req as *mut c_void);\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_free_response(resp: *mut DaemonResponse) {\n if resp.is_null() {\n return;\n }\n if !(*resp).id.is_null() {\n libc::free((*resp).id as *mut c_void);\n }\n if !(*resp).result_json.is_null() {\n libc::free((*resp).result_json as *mut c_void);\n }\n if !(*resp).error.is_null() {\n libc::free((*resp).error as *mut c_void);\n }\n libc::free(resp as *mut c_void);\n}\n\n// -- Response serialization (serde_json) --------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_serialize_response(\n response: *mut DaemonResponse,\n out_len: *mut usize,\n) -> *mut c_char {\n let mut map = serde_json::Map::new();\n\n if !(*response).id.is_null() {\n let id = CStr::from_ptr((*response).id).to_string_lossy();\n map.insert(\"id\".into(), serde_json::Value::String(id.into_owned()));\n }\n\n map.insert(\n \"status\".into(),\n serde_json::Value::String(\n if (*response).success { \"ok\" } else { \"error\" }.into(),\n ),\n );\n\n if (*response).success && !(*response).result_json.is_null() {\n let raw = CStr::from_ptr((*response).result_json).to_string_lossy();\n // Try to parse as JSON value; if it fails, store as raw string\n match serde_json::from_str::(&raw) {\n Ok(v) => {\n map.insert(\"result\".into(), v);\n }\n Err(_) => {\n map.insert(\"result\".into(), serde_json::Value::String(raw.into_owned()));\n }\n }\n }\n\n if !(*response).success && !(*response).error.is_null() {\n let err = CStr::from_ptr((*response).error).to_string_lossy();\n map.insert(\"error\".into(), serde_json::Value::String(err.into_owned()));\n }\n\n let json_str = serde_json::to_string(&map).unwrap_or_else(|_| \"{}\".into());\n if !out_len.is_null() {\n *out_len = json_str.len();\n }\n let c = CString::new(json_str).unwrap_or_default();\n libc::strdup(c.as_ptr())\n}\n\n// -- Discovery ----------------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_build_discovery(manifest: *mut c_void) -> *mut c_char {\n extern \"C\" {\n fn manifest_to_discovery_json(manifest: *const c_void) -> *mut c_char;\n }\n manifest_to_discovery_json(manifest)\n}\n\n// -- Eval timeout -------------------------------------------------------------\n\n#[no_mangle]\npub extern \"C\" fn daemon_set_eval_timeout(timeout_sec: i32) {\n let t = if timeout_sec > 0 { timeout_sec } else { 30 };\n G_EVAL_TIMEOUT.store(t, Ordering::Relaxed);\n}\n\n// -- Fork-based eval/typecheck ------------------------------------------------\n\n/// Fork `morloc `, capture stdout/stderr, return a DaemonResponse.\nunsafe fn fork_morloc_command(subcmd: &str, expr: *const c_char) -> *mut DaemonResponse {\n let resp = libc::calloc(1, std::mem::size_of::()) as *mut DaemonResponse;\n\n let mut stdout_pipe = [0i32; 2];\n let mut stderr_pipe = [0i32; 2];\n if libc::pipe(stdout_pipe.as_mut_ptr()) != 0 || libc::pipe(stderr_pipe.as_mut_ptr()) != 0 {\n (*resp).success = false;\n let c = CString::new(format!(\"Failed to create pipes for {}\", subcmd)).unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n\n let pid = libc::fork();\n if pid < 0 {\n (*resp).success = false;\n let c = CString::new(format!(\"Failed to fork for {}\", subcmd)).unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n libc::close(stdout_pipe[0]);\n libc::close(stdout_pipe[1]);\n libc::close(stderr_pipe[0]);\n libc::close(stderr_pipe[1]);\n return resp;\n }\n\n if pid == 0 {\n // Child\n libc::close(stdout_pipe[0]);\n libc::close(stderr_pipe[0]);\n libc::dup2(stdout_pipe[1], libc::STDOUT_FILENO);\n libc::dup2(stderr_pipe[1], libc::STDERR_FILENO);\n libc::close(stdout_pipe[1]);\n libc::close(stderr_pipe[1]);\n\n let timeout = G_EVAL_TIMEOUT.load(Ordering::Relaxed);\n if timeout > 0 {\n let cpu_limit = libc::rlimit {\n rlim_cur: timeout as libc::rlim_t,\n rlim_max: (timeout + 5) as libc::rlim_t,\n };\n libc::setrlimit(libc::RLIMIT_CPU, &cpu_limit);\n let as_limit = libc::rlimit {\n rlim_cur: 2 * 1024 * 1024 * 1024,\n rlim_max: 2 * 1024 * 1024 * 1024,\n };\n libc::setrlimit(libc::RLIMIT_AS, &as_limit);\n }\n\n let cmd = CString::new(\"morloc\").unwrap();\n let arg_subcmd = CString::new(subcmd).unwrap();\n libc::execlp(\n cmd.as_ptr(),\n cmd.as_ptr(),\n arg_subcmd.as_ptr(),\n expr,\n ptr::null::(),\n );\n libc::_exit(127);\n }\n\n // Parent\n libc::close(stdout_pipe[1]);\n libc::close(stderr_pipe[1]);\n\n let stdout_buf = read_fd_to_vec(stdout_pipe[0]);\n libc::close(stdout_pipe[0]);\n let stderr_buf = read_fd_to_vec(stderr_pipe[0]);\n libc::close(stderr_pipe[0]);\n\n let mut status: i32 = 0;\n libc::waitpid(pid, &mut status, 0);\n\n if libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 {\n let mut out = String::from_utf8_lossy(&stdout_buf).into_owned();\n // Trim trailing newlines\n while out.ends_with('\\n') || out.ends_with('\\r') {\n out.pop();\n }\n (*resp).success = true;\n let c = CString::new(out).unwrap_or_default();\n (*resp).result_json = libc::strdup(c.as_ptr());\n } else {\n (*resp).success = false;\n let errmsg = if !stderr_buf.is_empty() {\n String::from_utf8_lossy(&stderr_buf).into_owned()\n } else if libc::WIFSIGNALED(status) {\n format!(\"morloc {} killed by signal {}\", subcmd, libc::WTERMSIG(status))\n } else {\n let code = if libc::WIFEXITED(status) {\n libc::WEXITSTATUS(status)\n } else {\n -1\n };\n format!(\"morloc {} exited with code {}\", subcmd, code)\n };\n let c = CString::new(errmsg).unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n }\n\n resp\n}\n\n/// Read an fd to completion into a Vec.\nunsafe fn read_fd_to_vec(fd: i32) -> Vec {\n let mut buf = Vec::with_capacity(65536);\n let mut tmp = [0u8; 8192];\n loop {\n let n = libc::read(fd, tmp.as_mut_ptr() as *mut c_void, tmp.len());\n if n <= 0 {\n break;\n }\n buf.extend_from_slice(&tmp[..n as usize]);\n }\n buf\n}\n\n// -- Dispatch -----------------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_dispatch(\n manifest: *mut c_void,\n request: *mut DaemonRequest,\n sockets: *mut MorlocSocket,\n _shm_basename: *const c_char,\n) -> *mut DaemonResponse {\n let resp = libc::calloc(1, std::mem::size_of::()) as *mut DaemonResponse;\n\n // Echo request id\n if !(*request).id.is_null() {\n (*resp).id = libc::strdup((*request).id);\n }\n\n match (*request).method {\n DaemonMethod::Health => {\n (*resp).success = true;\n if let Some(alive_fn) = G_POOL_ALIVE_FN {\n let mut arr = Vec::with_capacity(G_N_POOLS);\n for i in 0..G_N_POOLS {\n arr.push(serde_json::Value::Bool(alive_fn(i)));\n }\n let json = serde_json::to_string(&arr).unwrap_or_default();\n let c = CString::new(json).unwrap_or_default();\n (*resp).result_json = libc::strdup(c.as_ptr());\n }\n return resp;\n }\n DaemonMethod::Discover => {\n (*resp).success = true;\n (*resp).result_json = daemon_build_discovery(manifest);\n return resp;\n }\n DaemonMethod::Eval => {\n if (*request).expr.is_null() {\n (*resp).success = false;\n let c = CString::new(\"Missing 'expr' field in eval request\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n\n // Check binding store for cached expression\n if !G_BINDING_STORE.is_null() {\n let expr_str = CStr::from_ptr((*request).expr).to_string_lossy();\n let store = &*G_BINDING_STORE;\n let hv = hash::xxh64_with_seed(expr_str.as_bytes(), DEFAULT_XXHASH_SEED);\n let _cached = store\n .lookup_hash(hv)\n .or_else(|| store.lookup_name(&expr_str));\n // TODO: direct binary execution for bound functions\n }\n\n let eval_resp = fork_morloc_command(\"eval\", (*request).expr);\n if !(*request).id.is_null() {\n (*eval_resp).id = libc::strdup((*request).id);\n }\n libc::free(resp as *mut c_void);\n return eval_resp;\n }\n DaemonMethod::Typecheck => {\n if (*request).expr.is_null() {\n (*resp).success = false;\n let c = CString::new(\"Missing 'expr' field in typecheck request\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n let tc_resp = fork_morloc_command(\"typecheck\", (*request).expr);\n if !(*request).id.is_null() {\n (*tc_resp).id = libc::strdup((*request).id);\n }\n libc::free(resp as *mut c_void);\n return tc_resp;\n }\n DaemonMethod::Bind => {\n if (*request).expr.is_null() {\n (*resp).success = false;\n let c = CString::new(\"Missing 'expr' field in bind request\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n if G_BINDING_STORE.is_null() {\n (*resp).success = false;\n let c = CString::new(\"Binding store not initialized\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n let store = &mut *G_BINDING_STORE;\n let expr_str = CStr::from_ptr((*request).expr).to_string_lossy().into_owned();\n let name = if (*request).name.is_null() {\n None\n } else {\n Some(CStr::from_ptr((*request).name).to_string_lossy().into_owned())\n };\n let timeout = G_EVAL_TIMEOUT.load(Ordering::Relaxed);\n match store.bind(&expr_str, name.as_deref(), timeout) {\n Some(hv) => {\n let mut map = serde_json::Map::new();\n map.insert(\n \"hash\".into(),\n serde_json::Value::String(format!(\"{:016x}\", hv)),\n );\n map.insert(\"expr\".into(), serde_json::Value::String(expr_str));\n if let Some(n) = &name {\n map.insert(\"name\".into(), serde_json::Value::String(n.clone()));\n }\n if let Some(entry) = store.lookup_hash(hv) {\n if let Some(ref ts) = entry.type_sig {\n map.insert(\"type\".into(), serde_json::Value::String(ts.clone()));\n }\n }\n let json = serde_json::to_string(&map).unwrap_or_default();\n (*resp).success = true;\n let c = CString::new(json).unwrap_or_default();\n (*resp).result_json = libc::strdup(c.as_ptr());\n }\n None => {\n (*resp).success = false;\n let c =\n CString::new(\"Failed to compile and bind expression\").unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n }\n }\n return resp;\n }\n DaemonMethod::Bindings => {\n (*resp).success = true;\n if G_BINDING_STORE.is_null() {\n let c = CString::new(\"{\\\"bindings\\\":[]}\").unwrap();\n (*resp).result_json = libc::strdup(c.as_ptr());\n } else {\n let store = &*G_BINDING_STORE;\n let json = store.list_json();\n let c = CString::new(json).unwrap_or_default();\n (*resp).result_json = libc::strdup(c.as_ptr());\n }\n return resp;\n }\n DaemonMethod::Unbind => {\n let name_ptr = if !(*request).command.is_null() {\n (*request).command\n } else {\n (*request).name\n };\n if name_ptr.is_null() {\n (*resp).success = false;\n let c = CString::new(\"Missing binding name\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n if G_BINDING_STORE.is_null() {\n (*resp).success = false;\n let c = CString::new(\"Binding store not initialized\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n let store = &mut *G_BINDING_STORE;\n let name = CStr::from_ptr(name_ptr).to_string_lossy();\n if store.unbind(&name) {\n (*resp).success = true;\n let c = CString::new(\"{\\\"removed\\\":true}\").unwrap();\n (*resp).result_json = libc::strdup(c.as_ptr());\n } else {\n (*resp).success = false;\n let c = CString::new(format!(\"Binding not found: {}\", name)).unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n }\n return resp;\n }\n DaemonMethod::Call => {\n // Fall through to call dispatch below\n }\n }\n\n // DAEMON_CALL\n if (*request).command.is_null() {\n (*resp).success = false;\n let c = CString::new(\"Missing 'command' field in call request\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n\n // Delegate to the C functions that handle manifest lookup, arg parsing,\n // schema handling, and pool communication. These are all already ported\n // to Rust in other _ffi modules, so we declare them as extern \"C\".\n extern \"C\" {\n fn parse_schema(schema: *const c_char, errmsg: *mut *mut c_char) -> *mut CSchema;\n fn free_schema(schema: *mut CSchema);\n fn initialize_positional(value: *mut c_char) -> *mut c_void;\n fn free_argument_t(arg: *mut c_void);\n fn parse_cli_data_argument(\n dest: *mut u8,\n arg: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn make_call_packet_from_cli(\n dest: *mut u8,\n mid: u32,\n args: *mut *mut c_void,\n arg_schema_strs: *mut *mut c_char,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn send_and_receive_over_socket(\n socket_path: *const c_char,\n packet: *const u8,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn get_morloc_data_packet_error_message(\n data: *const u8,\n errmsg: *mut *mut c_char,\n ) -> *mut c_char;\n fn get_morloc_data_packet_value(\n data: *const u8,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn voidstar_to_json_string(\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut c_char;\n fn morloc_eval(\n expr: *mut c_void, // actually *mut MorlocExpression\n return_schema: *mut CSchema,\n arg_voidstar: *mut *mut u8,\n arg_schemas: *mut *mut CSchema,\n nargs: usize,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n }\n\n // The manifest is the canonical v2 C struct from manifest_ffi.rs.\n // No local mirror needed -- import the real type and walk it.\n use crate::manifest_ffi::{Manifest as ManifestC, ManifestArgKind, ManifestCommand};\n\n let mv = manifest as *const ManifestC;\n let command_name = CStr::from_ptr((*request).command);\n let mut cmd: *const ManifestCommand = ptr::null();\n for i in 0..(*mv).n_commands {\n let c = &*(*mv).commands.add(i);\n if CStr::from_ptr(c.name) == command_name {\n cmd = c;\n break;\n }\n }\n\n if cmd.is_null() {\n (*resp).success = false;\n let msg = format!(\n \"Unknown command: {}\",\n command_name.to_string_lossy()\n );\n let c = CString::new(msg).unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n\n let cmd = &*cmd;\n let expected_nargs = cmd.n_args;\n\n // Parse JSON args into argument_t** array\n let mut err: *mut c_char = ptr::null_mut();\n let args: *mut *mut c_void;\n\n if !(*request).args_json.is_null() {\n // Parse the JSON array\n let args_str = CStr::from_ptr((*request).args_json).to_string_lossy();\n let parsed_args: Vec = match serde_json::from_str(&args_str) {\n Ok(v) => v,\n Err(e) => {\n (*resp).success = false;\n let c = CString::new(format!(\"Failed to parse args: {}\", e)).unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n };\n\n if parsed_args.len() != expected_nargs {\n (*resp).success = false;\n let c = CString::new(format!(\n \"Expected {} arguments, got {}\",\n expected_nargs,\n parsed_args.len()\n ))\n .unwrap_or_default();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n\n args = libc::calloc(expected_nargs + 1, std::mem::size_of::<*mut c_void>())\n as *mut *mut c_void;\n for (i, val) in parsed_args.iter().enumerate() {\n let val_str = match val {\n serde_json::Value::String(s) => format!(\"\\\"{}\\\"\", s),\n other => other.to_string(),\n };\n let c = CString::new(val_str).unwrap_or_default();\n let dup = libc::strdup(c.as_ptr());\n *args.add(i) = initialize_positional(dup);\n libc::free(dup as *mut c_void);\n }\n *args.add(expected_nargs) = ptr::null_mut();\n } else {\n if expected_nargs > 0 {\n // Check if any are positional (required)\n // For simplicity, match the C behavior: require args if n_args > 0\n (*resp).success = false;\n let c = CString::new(\"Missing 'args' field in call request\").unwrap();\n (*resp).error = libc::strdup(c.as_ptr());\n return resp;\n }\n args =\n libc::calloc(1, std::mem::size_of::<*mut c_void>()) as *mut *mut c_void;\n *args = ptr::null_mut();\n }\n\n if cmd.is_pure {\n // Pure command: evaluate expression tree\n let mut nargs: usize = 0;\n while !(*args.add(nargs)).is_null() {\n nargs += 1;\n }\n\n // v2: schemas live on each ManifestArg. Walk cmd.args in\n // declaration order, INCLUDING flags (they consume an arg\n // slot in the parsed list and need a corresponding schema\n // entry to keep alignment). For flags, fall back to the\n // boolean schema \"b\".\n static FLAG_SCHEMA: &[u8] = b\"b\\0\";\n let mut arg_schema_strs: Vec<*mut c_char> = Vec::with_capacity(nargs);\n for i in 0..cmd.n_args {\n let a = &*cmd.args.add(i);\n let s = if a.kind == ManifestArgKind::Flag || a.schema.is_null() {\n FLAG_SCHEMA.as_ptr() as *mut c_char\n } else {\n a.schema\n };\n arg_schema_strs.push(s);\n }\n\n let arg_schemas_arr =\n libc::calloc(nargs, std::mem::size_of::<*mut CSchema>()) as *mut *mut CSchema;\n let arg_packets =\n libc::calloc(nargs, std::mem::size_of::<*mut u8>()) as *mut *mut u8;\n let arg_voidstars =\n libc::calloc(nargs, std::mem::size_of::<*mut u8>()) as *mut *mut u8;\n\n let mut cleanup_and_fail = false;\n\n for i in 0..nargs {\n let schema_str = arg_schema_strs.get(i).copied().unwrap_or(ptr::null_mut());\n *arg_schemas_arr.add(i) = parse_schema(schema_str, &mut err);\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n cleanup_and_fail = true;\n break;\n }\n\n *arg_packets.add(i) = parse_cli_data_argument(\n ptr::null_mut(),\n *args.add(i),\n *arg_schemas_arr.add(i),\n &mut err,\n );\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n cleanup_and_fail = true;\n break;\n }\n\n *arg_voidstars.add(i) = get_morloc_data_packet_value(\n *arg_packets.add(i),\n *arg_schemas_arr.add(i),\n &mut err,\n );\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n cleanup_and_fail = true;\n break;\n }\n }\n\n if !cleanup_and_fail {\n let return_schema = parse_schema(cmd.ret.schema, &mut err);\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n } else {\n let result_abs = morloc_eval(\n cmd.expr as *mut c_void,\n return_schema,\n arg_voidstars,\n arg_schemas_arr,\n nargs,\n &mut err,\n );\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n } else {\n let json = voidstar_to_json_string(\n result_abs as *const c_void,\n return_schema as *const CSchema,\n &mut err,\n );\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n } else {\n (*resp).success = true;\n (*resp).result_json = json;\n }\n }\n free_schema(return_schema);\n }\n }\n\n // Cleanup\n for i in 0..nargs {\n let s = *arg_schemas_arr.add(i);\n if !s.is_null() {\n free_schema(s);\n }\n let p = *arg_packets.add(i);\n if !p.is_null() {\n libc::free(p as *mut c_void);\n }\n }\n libc::free(arg_schemas_arr as *mut c_void);\n libc::free(arg_packets as *mut c_void);\n libc::free(arg_voidstars as *mut c_void);\n } else {\n // Remote command: send call packet to pool. v2 stores schemas\n // per-arg, but make_call_packet_from_cli wants a NULL-terminated\n // flat array. ManifestCommand exposes a helper that materializes\n // the flat view; the outer pointer array is owned by us and\n // freed below, but the inner C strings remain owned by the\n // ManifestArg objects.\n let arg_schemas_flat = cmd.build_arg_schemas_array();\n let call_packet = make_call_packet_from_cli(\n ptr::null_mut(),\n cmd.mid,\n args,\n arg_schemas_flat,\n &mut err,\n );\n libc::free(arg_schemas_flat as *mut c_void);\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n } else {\n let socket_path = (*sockets.add(cmd.pool_index)).socket_filename;\n let result_packet =\n send_and_receive_over_socket(socket_path, call_packet, &mut err);\n libc::free(call_packet as *mut c_void);\n\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n } else {\n let packet_error =\n get_morloc_data_packet_error_message(result_packet, &mut err);\n if !packet_error.is_null() {\n (*resp).success = false;\n (*resp).error = libc::strdup(packet_error);\n libc::free(result_packet as *mut c_void);\n } else if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n libc::free(result_packet as *mut c_void);\n } else {\n let return_schema = parse_schema(cmd.ret.schema, &mut err);\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n libc::free(result_packet as *mut c_void);\n } else {\n let packet_value = get_morloc_data_packet_value(\n result_packet,\n return_schema as *const CSchema,\n &mut err,\n );\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n } else {\n let json = voidstar_to_json_string(\n packet_value as *const c_void,\n return_schema as *const CSchema,\n &mut err,\n );\n if !err.is_null() {\n (*resp).success = false;\n (*resp).error = err;\n } else {\n (*resp).success = true;\n (*resp).result_json = json;\n }\n }\n free_schema(return_schema);\n libc::free(result_packet as *mut c_void);\n }\n }\n }\n }\n }\n\n // Free args\n let mut i = 0;\n while !(*args.add(i)).is_null() {\n free_argument_t(*args.add(i));\n i += 1;\n }\n libc::free(args as *mut c_void);\n\n resp\n}\n\n// -- Length-prefixed message protocol -----------------------------------------\n\nunsafe fn read_lp_message(\n fd: i32,\n out_len: *mut usize,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n\n let mut len_buf = [0u8; 4];\n let n = libc::recv(\n fd,\n len_buf.as_mut_ptr() as *mut c_void,\n 4,\n libc::MSG_WAITALL,\n );\n if n != 4 {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to read message length prefix\".into()),\n );\n return ptr::null_mut();\n }\n\n let msg_len = ((len_buf[0] as u32) << 24)\n | ((len_buf[1] as u32) << 16)\n | ((len_buf[2] as u32) << 8)\n | (len_buf[3] as u32);\n\n if msg_len > MAX_LP_MESSAGE {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"Message too large: {} bytes\", msg_len)),\n );\n return ptr::null_mut();\n }\n\n let msg = libc::malloc(msg_len as usize + 1) as *mut c_char;\n if msg.is_null() {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to allocate message buffer\".into()),\n );\n return ptr::null_mut();\n }\n\n let mut total: usize = 0;\n while total < msg_len as usize {\n let n = libc::recv(\n fd,\n msg.add(total) as *mut c_void,\n msg_len as usize - total,\n 0,\n );\n if n <= 0 {\n libc::free(msg as *mut c_void);\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"Failed to read message body (got {} of {} bytes)\",\n total, msg_len\n )),\n );\n return ptr::null_mut();\n }\n total += n as usize;\n }\n *msg.add(msg_len as usize) = 0;\n\n if !out_len.is_null() {\n *out_len = msg_len as usize;\n }\n msg\n}\n\nunsafe fn write_lp_message(\n fd: i32,\n data: *const c_char,\n len: usize,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n\n let len_buf: [u8; 4] = [\n ((len >> 24) & 0xFF) as u8,\n ((len >> 16) & 0xFF) as u8,\n ((len >> 8) & 0xFF) as u8,\n (len & 0xFF) as u8,\n ];\n\n let n = libc::send(\n fd,\n len_buf.as_ptr() as *const c_void,\n 4,\n crate::utility::SEND_NOSIGNAL,\n );\n if n != 4 {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to write message length prefix\".into()),\n );\n return false;\n }\n\n let mut total: usize = 0;\n while total < len {\n let n = libc::send(\n fd,\n (data as *const u8).add(total) as *const c_void,\n len - total,\n crate::utility::SEND_NOSIGNAL,\n );\n if n <= 0 {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to write message body\".into()),\n );\n return false;\n }\n total += n as usize;\n }\n\n true\n}\n\n// -- Connection handlers ------------------------------------------------------\n\nunsafe fn handle_lp_connection(\n client_fd: i32,\n manifest: *mut c_void,\n sockets: *mut MorlocSocket,\n shm_basename: *const c_char,\n) {\n let mut errmsg: *mut c_char = ptr::null_mut();\n let mut msg_len: usize = 0;\n\n // Peek to distinguish a probe connection (immediate EOF) from a real\n // client. The router's readiness check connects then closes without\n // sending data; silently ignore those.\n let mut peek_buf = [0u8; 1];\n let peek_n = libc::recv(client_fd, peek_buf.as_mut_ptr() as *mut c_void, 1, libc::MSG_PEEK);\n if peek_n == 0 {\n // Clean EOF — probe connection, silently close.\n libc::close(client_fd);\n return;\n }\n\n let msg = read_lp_message(client_fd, &mut msg_len, &mut errmsg);\n if !errmsg.is_null() {\n let err_str = CStr::from_ptr(errmsg).to_string_lossy();\n eprintln!(\"daemon: read error: {}\", err_str);\n libc::free(errmsg as *mut c_void);\n libc::close(client_fd);\n return;\n }\n\n let req = daemon_parse_request(msg, msg_len, &mut errmsg);\n libc::free(msg as *mut c_void);\n if !errmsg.is_null() {\n let mut err_resp: DaemonResponse = std::mem::zeroed();\n err_resp.success = false;\n err_resp.error = errmsg;\n let mut resp_len: usize = 0;\n let resp_json = daemon_serialize_response(&mut err_resp, &mut resp_len);\n let mut write_err: *mut c_char = ptr::null_mut();\n write_lp_message(client_fd, resp_json, resp_len, &mut write_err);\n libc::free(resp_json as *mut c_void);\n if !write_err.is_null() {\n libc::free(write_err as *mut c_void);\n }\n libc::free(errmsg as *mut c_void);\n libc::close(client_fd);\n return;\n }\n\n let resp = daemon_dispatch(manifest, req, sockets, shm_basename);\n\n let mut resp_len: usize = 0;\n let resp_json = daemon_serialize_response(resp, &mut resp_len);\n\n let mut write_err: *mut c_char = ptr::null_mut();\n write_lp_message(client_fd, resp_json, resp_len, &mut write_err);\n if !write_err.is_null() {\n let err_str = CStr::from_ptr(write_err).to_string_lossy();\n eprintln!(\"daemon: write error: {}\", err_str);\n libc::free(write_err as *mut c_void);\n }\n\n libc::free(resp_json as *mut c_void);\n daemon_free_request(req);\n daemon_free_response(resp);\n libc::close(client_fd);\n}\n\nunsafe fn handle_http_connection(\n client_fd: i32,\n manifest: *mut c_void,\n sockets: *mut MorlocSocket,\n shm_basename: *const c_char,\n) {\n extern \"C\" {\n fn http_parse_request(fd: i32, errmsg: *mut *mut c_char) -> *mut HttpRequest;\n fn http_free_request(req: *mut HttpRequest);\n fn http_write_response(\n fd: i32,\n status: i32,\n content_type: *const c_char,\n body: *const c_char,\n body_len: usize,\n ) -> bool;\n fn http_to_daemon_request(\n req: *mut HttpRequest,\n errmsg: *mut *mut c_char,\n ) -> *mut DaemonRequest;\n }\n\n let mut errmsg: *mut c_char = ptr::null_mut();\n let http_req = http_parse_request(client_fd, &mut errmsg);\n if !errmsg.is_null() {\n let body = b\"{\\\"status\\\":\\\"error\\\",\\\"error\\\":\\\"Bad request\\\"}\\0\";\n let ct = b\"application/json\\0\";\n http_write_response(\n client_fd,\n 400,\n ct.as_ptr() as *const c_char,\n body.as_ptr() as *const c_char,\n body.len() - 1,\n );\n libc::free(errmsg as *mut c_void);\n libc::close(client_fd);\n return;\n }\n\n let req = http_to_daemon_request(http_req, &mut errmsg);\n if !errmsg.is_null() {\n let body = b\"{\\\"status\\\":\\\"error\\\",\\\"error\\\":\\\"Invalid request\\\"}\\0\";\n let ct = b\"application/json\\0\";\n http_write_response(\n client_fd,\n 400,\n ct.as_ptr() as *const c_char,\n body.as_ptr() as *const c_char,\n body.len() - 1,\n );\n http_free_request(http_req);\n libc::free(errmsg as *mut c_void);\n libc::close(client_fd);\n return;\n }\n http_free_request(http_req);\n\n let resp = daemon_dispatch(manifest, req, sockets, shm_basename);\n\n let mut resp_len: usize = 0;\n let resp_json = daemon_serialize_response(resp, &mut resp_len);\n\n // Append newline for terminal-friendly output\n let resp_body = libc::malloc(resp_len + 2) as *mut u8;\n ptr::copy_nonoverlapping(resp_json as *const u8, resp_body, resp_len);\n *resp_body.add(resp_len) = b'\\n';\n *resp_body.add(resp_len + 1) = 0;\n\n let status = if (*resp).success { 200 } else { 500 };\n let ct = b\"application/json\\0\";\n http_write_response(\n client_fd,\n status,\n ct.as_ptr() as *const c_char,\n resp_body as *const c_char,\n resp_len + 1,\n );\n\n libc::free(resp_body as *mut c_void);\n libc::free(resp_json as *mut c_void);\n daemon_free_request(req);\n daemon_free_response(resp);\n libc::close(client_fd);\n}\n\n// -- Thread pool (VecDeque + Condvar instead of linked list + pthread) --------\n\n#[derive(Clone, Copy)]\nstruct DaemonJob {\n client_fd: i32,\n conn_type: i32, // 0 = length-prefixed (unix/tcp), 2 = http\n}\n\nstruct JobQueue {\n jobs: VecDeque,\n}\n\nstruct WorkerContext {\n queue: Mutex,\n cond: Condvar,\n manifest: *mut c_void,\n sockets: *mut MorlocSocket,\n shm_basename: *const c_char,\n}\n\n// SAFETY: WorkerContext is shared between threads but all raw pointers\n// within it point to read-only or thread-safe C data.\nunsafe impl Send for WorkerContext {}\nunsafe impl Sync for WorkerContext {}\n\nfn set_socket_timeouts(fd: i32, timeout_sec: i32) {\n unsafe {\n let tv = libc::timeval {\n tv_sec: timeout_sec as _,\n tv_usec: 0,\n };\n libc::setsockopt(\n fd,\n libc::SOL_SOCKET,\n libc::SO_RCVTIMEO,\n &tv as *const libc::timeval as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n libc::setsockopt(\n fd,\n libc::SOL_SOCKET,\n libc::SO_SNDTIMEO,\n &tv as *const libc::timeval as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n }\n}\n\n// -- Main daemon event loop ---------------------------------------------------\n\nconst MAX_LISTENERS: usize = 3;\n\n#[no_mangle]\npub unsafe extern \"C\" fn daemon_run(\n config: *mut DaemonConfig,\n manifest: *mut c_void,\n sockets: *mut MorlocSocket,\n n_pools: usize,\n shm_basename: *const c_char,\n) {\n // Set globals\n G_POOL_ALIVE_FN = (*config).pool_alive_fn;\n G_N_POOLS = n_pools;\n let timeout = if (*config).eval_timeout > 0 {\n (*config).eval_timeout\n } else {\n 30\n };\n G_EVAL_TIMEOUT.store(timeout, Ordering::Relaxed);\n\n // Initialize binding store\n if G_BINDING_STORE.is_null() {\n let store = Box::new(BindingStore::new(\"/tmp/morloc-bindings\"));\n G_BINDING_STORE = Box::into_raw(store);\n }\n\n // Install signal handlers\n SHUTDOWN_REQUESTED.store(false, Ordering::Relaxed);\n let handler: libc::sighandler_t =\n std::mem::transmute::(daemon_signal_handler_fn);\n libc::signal(libc::SIGTERM, handler);\n libc::signal(libc::SIGINT, handler);\n\n let mut fds = [libc::pollfd {\n fd: -1,\n events: 0,\n revents: 0,\n }; MAX_LISTENERS];\n let mut fd_types = [0i32; MAX_LISTENERS]; // 0=unix, 1=tcp, 2=http\n let mut nfds: usize = 0;\n\n // Unix socket\n if !(*config).unix_socket_path.is_null() {\n let sock_fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0);\n if sock_fd < 0 {\n eprintln!(\"daemon: failed to create unix socket\");\n return;\n }\n let mut addr: libc::sockaddr_un = std::mem::zeroed();\n addr.sun_family = libc::AF_UNIX as libc::sa_family_t;\n let path_bytes = CStr::from_ptr((*config).unix_socket_path).to_bytes();\n let copy_len = path_bytes.len().min(addr.sun_path.len() - 1);\n ptr::copy_nonoverlapping(\n path_bytes.as_ptr() as *const c_char,\n addr.sun_path.as_mut_ptr(),\n copy_len,\n );\n libc::unlink((*config).unix_socket_path);\n if libc::bind(\n sock_fd,\n &addr as *const libc::sockaddr_un as *const libc::sockaddr,\n std::mem::size_of::() as libc::socklen_t,\n ) < 0\n {\n eprintln!(\"daemon: failed to bind unix socket\");\n libc::close(sock_fd);\n return;\n }\n libc::listen(sock_fd, 64);\n fds[nfds].fd = sock_fd;\n fds[nfds].events = libc::POLLIN as i16;\n fd_types[nfds] = 0;\n nfds += 1;\n }\n\n // TCP\n if (*config).tcp_port > 0 {\n let tcp_fd = libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0);\n if tcp_fd < 0 {\n eprintln!(\"daemon: failed to create tcp socket\");\n return;\n }\n let opt: i32 = 1;\n libc::setsockopt(\n tcp_fd,\n libc::SOL_SOCKET,\n libc::SO_REUSEADDR,\n &opt as *const i32 as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n let mut addr: libc::sockaddr_in = std::mem::zeroed();\n addr.sin_family = libc::AF_INET as libc::sa_family_t;\n addr.sin_addr.s_addr = u32::from_be(0x7f000001); // INADDR_LOOPBACK\n addr.sin_port = ((*config).tcp_port as u16).to_be();\n if libc::bind(\n tcp_fd,\n &addr as *const libc::sockaddr_in as *const libc::sockaddr,\n std::mem::size_of::() as libc::socklen_t,\n ) < 0\n {\n eprintln!(\n \"daemon: failed to bind tcp port {}\",\n (*config).tcp_port\n );\n libc::close(tcp_fd);\n return;\n }\n libc::listen(tcp_fd, 64);\n fds[nfds].fd = tcp_fd;\n fds[nfds].events = libc::POLLIN as i16;\n fd_types[nfds] = 1;\n nfds += 1;\n }\n\n // HTTP\n if (*config).http_port > 0 {\n let http_fd = libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0);\n if http_fd < 0 {\n eprintln!(\"daemon: failed to create http socket\");\n return;\n }\n let opt: i32 = 1;\n libc::setsockopt(\n http_fd,\n libc::SOL_SOCKET,\n libc::SO_REUSEADDR,\n &opt as *const i32 as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n let mut addr: libc::sockaddr_in = std::mem::zeroed();\n addr.sin_family = libc::AF_INET as libc::sa_family_t;\n // HTTP router is externally reachable; bind to all interfaces so that\n // container port mappings (docker -p) can reach it.\n addr.sin_addr.s_addr = libc::INADDR_ANY.to_be();\n addr.sin_port = ((*config).http_port as u16).to_be();\n if libc::bind(\n http_fd,\n &addr as *const libc::sockaddr_in as *const libc::sockaddr,\n std::mem::size_of::() as libc::socklen_t,\n ) < 0\n {\n eprintln!(\n \"daemon: failed to bind http port {}\",\n (*config).http_port\n );\n libc::close(http_fd);\n return;\n }\n libc::listen(http_fd, 64);\n fds[nfds].fd = http_fd;\n fds[nfds].events = libc::POLLIN as i16;\n fd_types[nfds] = 2;\n nfds += 1;\n }\n\n if nfds == 0 {\n eprintln!(\"daemon: no listeners configured, exiting\");\n return;\n }\n\n // Start worker thread pool\n let ctx = Arc::new(WorkerContext {\n queue: Mutex::new(JobQueue {\n jobs: VecDeque::new(),\n }),\n cond: Condvar::new(),\n manifest,\n sockets,\n shm_basename,\n });\n\n let n_workers = n_pools.saturating_add(4).clamp(4, 32);\n let mut workers = Vec::with_capacity(n_workers);\n for _ in 0..n_workers {\n let ctx = Arc::clone(&ctx);\n workers.push(std::thread::spawn(move || {\n daemon_worker_fn(ctx);\n }));\n }\n\n // Main event loop\n while !SHUTDOWN_REQUESTED.load(Ordering::Relaxed) {\n let ready = libc::poll(fds.as_mut_ptr(), nfds as libc::nfds_t, 1000);\n if ready < 0 {\n if crate::utility::errno_val() == libc::EINTR {\n continue;\n }\n eprintln!(\"daemon: poll error\");\n break;\n }\n\n // Check and restart crashed pools\n if let Some(check_fn) = (*config).pool_check_fn {\n check_fn(sockets, n_pools);\n }\n\n if ready == 0 {\n continue;\n }\n\n for i in 0..nfds {\n if fds[i].revents & libc::POLLIN as i16 == 0 {\n continue;\n }\n let client_fd = libc::accept(fds[i].fd, ptr::null_mut(), ptr::null_mut());\n if client_fd < 0 {\n if crate::utility::errno_val() == libc::EINTR\n || crate::utility::errno_val() == libc::EAGAIN\n {\n continue;\n }\n eprintln!(\"daemon: accept error\");\n continue;\n }\n crate::utility::set_nosigpipe(client_fd);\n set_socket_timeouts(client_fd, 30);\n\n let job = DaemonJob {\n client_fd,\n conn_type: fd_types[i],\n };\n let mut q = ctx.queue.lock().unwrap();\n q.jobs.push_back(job);\n ctx.cond.notify_one();\n }\n }\n\n // Wake all workers and join\n ctx.cond.notify_all();\n for w in workers {\n let _ = w.join();\n }\n\n // Drain remaining jobs\n {\n let mut q = ctx.queue.lock().unwrap();\n while let Some(job) = q.jobs.pop_front() {\n libc::close(job.client_fd);\n }\n }\n\n // Close listener sockets\n for i in 0..nfds {\n libc::close(fds[i].fd);\n }\n\n if !(*config).unix_socket_path.is_null() {\n libc::unlink((*config).unix_socket_path);\n }\n}\n\nfn daemon_worker_fn(ctx: Arc) {\n loop {\n if SHUTDOWN_REQUESTED.load(Ordering::Relaxed) {\n break;\n }\n\n let job = {\n let mut q = ctx.queue.lock().unwrap();\n loop {\n if let Some(job) = q.jobs.pop_front() {\n break Some(job);\n }\n if SHUTDOWN_REQUESTED.load(Ordering::Relaxed) {\n break None;\n }\n // Wait with timeout so we recheck shutdown\n let (guard, _timeout) = ctx\n .cond\n .wait_timeout(q, std::time::Duration::from_millis(100))\n .unwrap();\n q = guard;\n }\n };\n\n let job = match job {\n Some(j) => j,\n None => continue,\n };\n\n unsafe {\n if job.conn_type == 2 {\n handle_http_connection(\n job.client_fd,\n ctx.manifest,\n ctx.sockets,\n ctx.shm_basename,\n );\n } else {\n handle_lp_connection(\n job.client_fd,\n ctx.manifest,\n ctx.sockets,\n ctx.shm_basename,\n );\n }\n }\n }\n}\n\n// Signal handler (must be async-signal-safe)\nextern \"C\" fn daemon_signal_handler_fn(_sig: i32) {\n SHUTDOWN_REQUESTED.store(true, Ordering::Relaxed);\n}\n" }, { "path": "data/rust/morloc-runtime/src/error.rs", "content": "use std::ffi::{CString, c_char};\n\n#[derive(Debug, thiserror::Error)]\npub enum MorlocError {\n #[error(\"shared memory error: {0}\")]\n Shm(String),\n #[error(\"packet error: {0}\")]\n Packet(String),\n #[error(\"schema error: {0}\")]\n Schema(String),\n #[error(\"serialization error: {0}\")]\n Serialization(String),\n #[error(\"IPC error: {0}\")]\n Ipc(String),\n #[error(\"IO error: {0}\")]\n Io(#[from] std::io::Error),\n #[error(\"null pointer\")]\n NullPointer,\n #[error(\"{0}\")]\n Other(String),\n}\n\n/// Write a MorlocError into the C ERRMSG convention.\n///\n/// # Safety\n/// `errmsg` must be a valid, non-dangling pointer to a `*mut c_char` (i.e., `char**`).\n/// The caller is responsible for freeing the allocated string via `CString::from_raw`\n/// or `libc::free`.\npub unsafe fn set_errmsg(errmsg: *mut *mut c_char, err: &MorlocError) {\n if !errmsg.is_null() {\n if let Ok(cstr) = CString::new(err.to_string()) {\n *errmsg = cstr.into_raw();\n }\n }\n}\n\n/// Clear the ERRMSG pointer (must be called at FFI entry).\n///\n/// # Safety\n/// `errmsg` must be a valid pointer to a `*mut c_char`.\npub unsafe fn clear_errmsg(errmsg: *mut *mut c_char) {\n if !errmsg.is_null() {\n *errmsg = std::ptr::null_mut();\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/eval_ffi.rs", "content": "//! Expression evaluator and constructor functions.\n//! Replaces eval.c. Uses HashMap instead of linked-list dict_t.\n\nuse std::collections::HashMap;\nuse std::ffi::{c_char, c_void, CStr};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::manifest_ffi::*;\nuse crate::shm::{self, AbsPtr, RelPtr};\n\n// ── Constructor functions (called by manifest_ffi.rs and daemon.c) ───────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_morloc_bound_var(\n schema_str: *const c_char,\n varname: *mut c_char,\n errmsg: *mut *mut c_char,\n) -> *mut MorlocExpression {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n let schema = crate::ffi::parse_schema(schema_str, &mut err);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n if expr.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to allocate bound variable expression\".into()));\n return ptr::null_mut();\n }\n (*expr).etype = MorlocExpressionType::Bnd;\n (*expr).schema = schema;\n (*expr).expr.bnd_expr = varname;\n expr\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_morloc_literal(\n schema_str: *const c_char,\n lit: Primitive,\n errmsg: *mut *mut c_char,\n) -> *mut MorlocExpression {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n let schema = crate::ffi::parse_schema(schema_str, &mut err);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n\n let data = libc::malloc(std::mem::size_of::()) as *mut MorlocData;\n if data.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to allocate literal data\".into()));\n return ptr::null_mut();\n }\n (*data).is_voidstar = false;\n (*data).data = DataUnion { lit_val: std::mem::ManuallyDrop::new(lit) };\n\n let expr = libc::malloc(std::mem::size_of::()) as *mut MorlocExpression;\n if expr.is_null() {\n libc::free(data as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to allocate literal expression\".into()));\n return ptr::null_mut();\n }\n (*expr).etype = MorlocExpressionType::Dat;\n (*expr).schema = schema;\n (*expr).expr.data_expr = data;\n expr\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_morloc_pattern(\n schema_str: *const c_char,\n pattern: *mut MorlocPattern,\n errmsg: *mut *mut c_char,\n) -> *mut MorlocExpression {\n clear_errmsg(errmsg);\n let mut err: *mut c_char = ptr::null_mut();\n let schema = crate::ffi::parse_schema(schema_str, &mut err);\n if !err.is_null() { *errmsg = err; return ptr::null_mut(); }\n\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n if expr.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to allocate pattern expression\".into()));\n return ptr::null_mut();\n }\n (*expr).etype = MorlocExpressionType::Pat;\n (*expr).schema = schema;\n (*expr).expr.pattern_expr = pattern;\n expr\n}\n\n#[no_mangle]\npub extern \"C\" fn make_morloc_pattern_end() -> *mut MorlocPattern {\n // SAFETY: calloc returns zeroed memory suitable for MorlocPattern.\n // We initialize all fields before returning.\n unsafe {\n let pat = libc::calloc(1, std::mem::size_of::()) as *mut MorlocPattern;\n if pat.is_null() { return ptr::null_mut(); }\n (*pat).ptype = MorlocPatternType::End;\n (*pat).size = 0;\n (*pat).fields = PatternFields { indices: ptr::null_mut() };\n (*pat).selectors = ptr::null_mut();\n pat\n }\n}\n\n// Varargs constructors (make_morloc_container, make_morloc_app,\n// make_morloc_lambda, make_morloc_interpolation, make_morloc_pattern_idx,\n// make_morloc_pattern_key) are only used by generated C++ pool code.\n// They cannot be implemented in stable Rust due to C-variadic ABI.\n// The C-gcc build path (morloc init) provides them from the original eval.c.\n// The Rust hybrid build does not call them (only morloc_eval and the\n// non-varargs constructors are needed).\n\n// ── Core evaluator ───────────────────────────────────────────────────────────\n\ntype BndVars<'a> = HashMap<&'a str, AbsPtr>;\n\n/// Convert key-based pattern selectors to index-based using the schema's key names.\n///\n/// # Safety\n/// `pattern` and `schema` must be valid, non-null pointers to C-allocated structures.\n/// `schema` keys array must have `schema.size` entries.\nunsafe fn convert_keys_to_indices(\n pattern: *mut MorlocPattern,\n schema: *const CSchema,\n) -> Result<(), MorlocError> {\n let pat = &mut *pattern;\n let n_params = (*schema).size;\n\n if n_params > 1 {\n for i in 0..pat.size {\n let child_schema = *(*schema).parameters.add(i);\n convert_keys_to_indices(*pat.selectors.add(i), child_schema)?;\n }\n }\n\n if pat.ptype == MorlocPatternType::ByKey {\n let indices = libc::calloc(n_params, std::mem::size_of::()) as *mut usize;\n for i in 0..pat.size {\n let key = CStr::from_ptr(*pat.fields.keys.add(i)).to_str().unwrap_or(\"\");\n let mut found = false;\n for j in 0..n_params {\n let record_key = CStr::from_ptr(*(*schema).keys.add(j)).to_str().unwrap_or(\"\");\n if key == record_key {\n found = true;\n *indices.add(i) = j;\n break;\n }\n }\n if !found {\n libc::free(indices as *mut c_void);\n return Err(MorlocError::Other(format!(\"Pattern contains key missing in schema: {}\", key)));\n }\n libc::free(*pat.fields.keys.add(i) as *mut c_void);\n }\n pat.ptype = MorlocPatternType::ByIndex;\n libc::free(pat.fields.keys as *mut c_void);\n pat.fields.indices = indices;\n }\n\n Ok(())\n}\n\n/// Extract fields from a voidstar value using a pattern, copying them into dest.\n///\n/// # Safety\n/// All pointer arguments must be valid and point to correctly-typed C structures.\n/// `value` must point to voidstar data matching `value_schema`.\nunsafe fn apply_getter(\n dest: AbsPtr,\n return_index: &mut usize,\n return_schema: *const CSchema,\n pattern: *mut MorlocPattern,\n value_schema: *const CSchema,\n value: AbsPtr,\n) -> Result {\n let pat = &*pattern;\n\n match pat.ptype {\n MorlocPatternType::ByIndex => {\n for i in 0..pat.size {\n let idx = *pat.fields.indices.add(i);\n apply_getter(\n dest, return_index, return_schema,\n *pat.selectors.add(i),\n *(*value_schema).parameters.add(idx),\n value.add(*(*value_schema).offsets.add(idx)),\n )?;\n }\n }\n MorlocPatternType::ByKey => {\n convert_keys_to_indices(pattern, value_schema)?;\n return apply_getter(dest, return_index, return_schema, pattern, value_schema, value);\n }\n MorlocPatternType::End => {\n let (element_dest, element_width) = if (*return_schema).size > 1 {\n (dest.add(*(*return_schema).offsets.add(*return_index)),\n (*(*(*return_schema).parameters.add(*return_index))).width)\n } else {\n (dest, (*return_schema).width)\n };\n *return_index += 1;\n ptr::copy_nonoverlapping(value, element_dest, element_width);\n }\n }\n\n Ok(dest)\n}\n\n/// Copy value into dest, preserving fields not selected by pattern.\n///\n/// # Safety\n/// All pointer arguments must be valid. Schema sizes must match.\nunsafe fn apply_setter_copy(\n dest: AbsPtr,\n return_schema: *const CSchema,\n pattern: *mut MorlocPattern,\n value_schema: *const CSchema,\n value: AbsPtr,\n) -> Result<(), MorlocError> {\n let pat = &*pattern;\n match pat.ptype {\n MorlocPatternType::ByKey => {\n convert_keys_to_indices(pattern, value_schema)?;\n return apply_setter_copy(dest, return_schema, pattern, value_schema, value);\n }\n MorlocPatternType::ByIndex => {\n if (*value_schema).size != (*return_schema).size {\n return Err(MorlocError::Other(\"Expected setter return and input sizes to be the same\".into()));\n }\n for i in 0..(*value_schema).size {\n let new_dest = dest.add(*(*return_schema).offsets.add(i));\n let new_value = value.add(*(*value_schema).offsets.add(i));\n let mut changed = false;\n for j in 0..pat.size {\n if i == *pat.fields.indices.add(j) {\n apply_setter_copy(\n new_dest,\n *(*return_schema).parameters.add(i),\n *pat.selectors.add(j),\n *(*value_schema).parameters.add(i),\n new_value,\n )?;\n changed = true;\n break;\n }\n }\n if !changed {\n let w = (*(*(*value_schema).parameters.add(i))).width;\n ptr::copy_nonoverlapping(new_value, new_dest, w);\n }\n }\n }\n MorlocPatternType::End => {}\n }\n Ok(())\n}\n\n/// Overwrite pattern-selected fields in dest with provided set_values.\n///\n/// # Safety\n/// All pointer arguments must be valid. set_values must have enough entries.\nunsafe fn apply_setter_set(\n dest: AbsPtr,\n return_schema: *const CSchema,\n pattern: *mut MorlocPattern,\n value_schema: *const CSchema,\n value: AbsPtr,\n set_schemas: *mut *mut CSchema,\n set_values: *mut AbsPtr,\n set_idx: &mut usize,\n) -> Result<(), MorlocError> {\n let pat = &*pattern;\n match pat.ptype {\n MorlocPatternType::ByIndex => {\n for pi in 0..pat.size {\n let di = *pat.fields.indices.add(pi);\n apply_setter_set(\n dest.add(*(*return_schema).offsets.add(di)),\n *(*return_schema).parameters.add(di),\n *pat.selectors.add(pi),\n *(*value_schema).parameters.add(di),\n value.add(*(*value_schema).offsets.add(di)),\n set_schemas, set_values, set_idx,\n )?;\n }\n }\n MorlocPatternType::End => {\n ptr::copy_nonoverlapping(*set_values.add(*set_idx), dest, (*return_schema).width);\n *set_idx += 1;\n }\n MorlocPatternType::ByKey => {\n return Err(MorlocError::Other(\"Key patterns should have been resolved in copy step\".into()));\n }\n }\n Ok(())\n}\n\n/// Recursively evaluate a morloc expression, writing results into SHM.\n///\n/// # Safety\n/// `expr` must be a valid MorlocExpression pointer (or null for error).\n/// If `dest` is non-null, it must point to `width` bytes of writable SHM.\nunsafe fn morloc_eval_r(\n expr: *mut MorlocExpression,\n dest: AbsPtr,\n width: usize,\n bndvars: &mut BndVars,\n) -> Result {\n if expr.is_null() {\n return Err(MorlocError::Other(\"Empty expression\".into()));\n }\n\n let schema = (*expr).schema;\n let (dest, width) = if dest.is_null() {\n let w = (*schema).width;\n let d = shm::shcalloc(1, w)?;\n (d, w)\n } else {\n if width != (*schema).width {\n return Err(MorlocError::Other(\"Unexpected data size\".into()));\n }\n (dest, width)\n };\n\n match (*expr).etype {\n MorlocExpressionType::Dat => {\n let data = (*expr).expr.data_expr;\n if (*data).is_voidstar {\n return Ok((*data).data.voidstar as AbsPtr);\n }\n\n let stype = (*schema).serial_type;\n if stype == crate::schema::SerialType::String as u32 {\n // String: allocate in SHM\n let s = std::mem::ManuallyDrop::into_inner(ptr::read(&(*data).data.lit_val)).s;\n let str_size = if s.is_null() { 0 } else { libc::strlen(s) };\n let str_relptr: RelPtr = if str_size > 0 {\n let abs = shm::shmemcpy(s as *const u8, str_size)?;\n shm::abs2rel(abs)?\n } else {\n -1isize as RelPtr\n };\n let arr = shm::Array { size: str_size, data: str_relptr };\n ptr::copy_nonoverlapping(&arr as *const shm::Array as *const u8, dest, width);\n } else if stype == crate::schema::SerialType::Array as u32 {\n let arr_data = (*data).data.array_val;\n let arr_size = (*arr_data).size;\n let elem_schema = (*arr_data).schema;\n let elem_width = (*elem_schema).width;\n let arr_reldata: RelPtr = if arr_size > 0 {\n let arr_abs = shm::shcalloc(arr_size, elem_width)?;\n for i in 0..arr_size {\n morloc_eval_r(\n *(*arr_data).values.add(i),\n arr_abs.add(i * elem_width),\n elem_width,\n bndvars,\n )?;\n }\n shm::abs2rel(arr_abs)?\n } else {\n -1isize as RelPtr\n };\n let arr = shm::Array { size: arr_size, data: arr_reldata };\n ptr::copy_nonoverlapping(&arr as *const shm::Array as *const u8, dest, width);\n } else if stype == crate::schema::SerialType::Tuple as u32\n || stype == crate::schema::SerialType::Map as u32 {\n for i in 0..(*schema).size {\n let elem_width = (*(*(*schema).parameters.add(i))).width;\n let elem_dest = dest.add(*(*schema).offsets.add(i));\n let element = *(*data).data.tuple_val.add(i);\n morloc_eval_r(element, elem_dest, elem_width, bndvars)?;\n }\n } else {\n // All primitives: just copy width bytes from the union\n ptr::copy_nonoverlapping(\n &(*data).data as *const DataUnion as *const u8,\n dest,\n width,\n );\n }\n }\n\n MorlocExpressionType::App => {\n let app = (*expr).expr.app_expr;\n let nargs = (*app).nargs;\n\n // Evaluate all arguments\n let mut arg_results: Vec = Vec::with_capacity(nargs);\n for i in 0..nargs {\n let r = morloc_eval_r(*(*app).args.add(i), ptr::null_mut(), 0, bndvars)?;\n arg_results.push(r);\n }\n\n match (*app).atype {\n MorlocAppExpressionType::Pattern => {\n if nargs == 1 {\n let mut return_index: usize = 0;\n apply_getter(\n dest, &mut return_index, schema,\n (*app).function.pattern,\n (*(*(*app).args)).schema,\n arg_results[0],\n )?;\n } else if nargs > 1 {\n // Setter: first arg is the value, rest are set values\n let mut set_schemas: Vec<*mut CSchema> = Vec::with_capacity(nargs - 1);\n for i in 1..nargs {\n set_schemas.push((*(*(*app).args.add(i))).schema);\n }\n apply_setter_copy(\n dest, schema, (*app).function.pattern,\n (*(*(*app).args)).schema, arg_results[0],\n )?;\n let mut set_idx: usize = 0;\n apply_setter_set(\n dest, schema, (*app).function.pattern,\n (*(*(*app).args)).schema, arg_results[0],\n set_schemas.as_mut_ptr(), arg_results[1..].as_ptr() as *mut AbsPtr,\n &mut set_idx,\n )?;\n } else {\n return Err(MorlocError::Other(\"No arguments provided to pattern\".into()));\n }\n }\n\n MorlocAppExpressionType::Lambda => {\n let lam = (*app).function.lambda;\n // Bind arguments\n for i in 0..nargs {\n let var = CStr::from_ptr(*(*lam).args.add(i)).to_str().unwrap_or(\"\");\n bndvars.insert(var, arg_results[i]);\n }\n morloc_eval_r((*lam).body, dest, width, bndvars)?;\n // Clean up bindings\n for i in 0..nargs {\n let var = CStr::from_ptr(*(*lam).args.add(i)).to_str().unwrap_or(\"\");\n bndvars.remove(var);\n }\n }\n\n MorlocAppExpressionType::Format => {\n let strings = (*app).function.fmt;\n let mut result_size: usize = 0;\n let mut string_lengths: Vec = Vec::with_capacity(nargs + 1);\n\n for i in 0..=nargs {\n let len = libc::strlen(*strings.add(i));\n string_lengths.push(len);\n result_size += len;\n }\n for i in 0..nargs {\n let arr = &*(arg_results[i] as *const shm::Array);\n result_size += arr.size;\n }\n\n let new_string = shm::shmalloc(result_size)?;\n let result_array = &mut *(dest as *mut shm::Array);\n result_array.size = result_size;\n result_array.data = shm::abs2rel(new_string)?;\n\n let mut cursor = new_string;\n for i in 0..=nargs {\n ptr::copy_nonoverlapping(*strings.add(i) as *const u8, cursor, string_lengths[i]);\n cursor = cursor.add(string_lengths[i]);\n if i < nargs {\n let arr = &*(arg_results[i] as *const shm::Array);\n if arr.size > 0 {\n let arr_data = shm::rel2abs(arr.data)?;\n ptr::copy_nonoverlapping(arr_data, cursor, arr.size);\n cursor = cursor.add(arr.size);\n }\n }\n }\n }\n }\n }\n\n MorlocExpressionType::Bnd => {\n let varname = CStr::from_ptr((*expr).expr.bnd_expr).to_str().unwrap_or(\"\");\n let bnd_value = bndvars.get(varname).copied()\n .ok_or_else(|| MorlocError::Other(format!(\"Unbound variable {}\", varname)))?;\n ptr::copy_nonoverlapping(bnd_value, dest, (*schema).width);\n }\n\n MorlocExpressionType::Show => {\n // Serialize child to JSON string\n let child = (*expr).expr.unary_expr;\n let child_schema = (*child).schema;\n let child_result = morloc_eval_r(child, ptr::null_mut(), 0, bndvars)?;\n\n extern \"C\" {\n fn voidstar_to_json_string(data: *const c_void, schema: *const CSchema, errmsg: *mut *mut c_char) -> *mut c_char;\n }\n let mut err: *mut c_char = ptr::null_mut();\n let json = voidstar_to_json_string(child_result as *const c_void, child_schema, &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n\n let json_len = libc::strlen(json);\n let str_relptr: RelPtr = if json_len > 0 {\n let abs = shm::shmemcpy(json as *const u8, json_len)?;\n libc::free(json as *mut c_void);\n shm::abs2rel(abs)?\n } else {\n libc::free(json as *mut c_void);\n -1isize as RelPtr\n };\n let arr = shm::Array { size: json_len, data: str_relptr };\n ptr::copy_nonoverlapping(&arr as *const shm::Array as *const u8, dest, width);\n }\n\n MorlocExpressionType::Read => {\n // Deserialize JSON string to typed data, return optional\n let child = (*expr).expr.unary_expr;\n let child_result = morloc_eval_r(child, ptr::null_mut(), 0, bndvars)?;\n let str_arr = &*(child_result as *const shm::Array);\n\n let opt_dest = dest;\n let inner_schema = *(*schema).parameters;\n\n if str_arr.size > 0 {\n let str_abs = shm::rel2abs(str_arr.data)?;\n let json_str = libc::malloc(str_arr.size + 1) as *mut c_char;\n if json_str.is_null() {\n return Err(MorlocError::Other(\"Failed to allocate for @read\".into()));\n }\n ptr::copy_nonoverlapping(str_abs, json_str as *mut u8, str_arr.size);\n *json_str.add(str_arr.size) = 0;\n\n extern \"C\" {\n fn read_json_with_schema(dest: *mut u8, json: *mut c_char, schema: *const CSchema, errmsg: *mut *mut c_char) -> *mut u8;\n }\n let inner_offset = *(*schema).offsets;\n let mut parse_err: *mut c_char = ptr::null_mut();\n let parsed = read_json_with_schema(opt_dest.add(inner_offset), json_str, inner_schema, &mut parse_err);\n libc::free(json_str as *mut c_void);\n if !parse_err.is_null() {\n libc::free(parse_err as *mut c_void);\n *opt_dest = 0; // None\n } else {\n *opt_dest = if parsed.is_null() { 0 } else { 1 };\n }\n } else {\n *opt_dest = 0; // None\n }\n }\n\n MorlocExpressionType::Hash => {\n // Hash child data and return hex string\n let child = (*expr).expr.unary_expr;\n let child_schema = (*child).schema;\n let child_result = morloc_eval_r(child, ptr::null_mut(), 0, bndvars)?;\n\n extern \"C\" {\n fn mlc_hash(data: *const c_void, schema: *const CSchema, errmsg: *mut *mut c_char) -> *mut c_char;\n }\n let mut err: *mut c_char = ptr::null_mut();\n let hex = mlc_hash(child_result as *const c_void, child_schema, &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n\n let hex_len = libc::strlen(hex);\n let str_relptr: RelPtr = if hex_len > 0 {\n let abs = shm::shmemcpy(hex as *const u8, hex_len)?;\n libc::free(hex as *mut c_void);\n shm::abs2rel(abs)?\n } else {\n libc::free(hex as *mut c_void);\n -1isize as RelPtr\n };\n let arr = shm::Array { size: hex_len, data: str_relptr };\n ptr::copy_nonoverlapping(&arr as *const shm::Array as *const u8, dest, width);\n }\n\n MorlocExpressionType::Save => {\n // Save value to file at path\n let save = (*expr).expr.save_expr;\n let value_expr = (*save).value;\n let path_expr = (*save).path;\n let fmt = CStr::from_ptr((*save).format).to_str().unwrap_or(\"voidstar\");\n\n let value_schema = (*value_expr).schema;\n let value_result = morloc_eval_r(value_expr, ptr::null_mut(), 0, bndvars)?;\n let path_result = morloc_eval_r(path_expr, ptr::null_mut(), 0, bndvars)?;\n\n // Extract path string from voidstar Array\n let path_arr = &*(path_result as *const shm::Array);\n let path_abs = shm::rel2abs(path_arr.data)?;\n let path_cstr = libc::malloc(path_arr.size + 1) as *mut c_char;\n if path_cstr.is_null() {\n return Err(MorlocError::Other(\"Failed to allocate for @save path\".into()));\n }\n ptr::copy_nonoverlapping(path_abs, path_cstr as *mut u8, path_arr.size);\n *path_cstr.add(path_arr.size) = 0;\n\n extern \"C\" {\n fn mlc_save(data: *const c_void, schema: *const CSchema, path: *const c_char, errmsg: *mut *mut c_char) -> i32;\n fn mlc_save_json(data: *const c_void, schema: *const CSchema, path: *const c_char, errmsg: *mut *mut c_char) -> i32;\n fn mlc_save_voidstar(data: *const c_void, schema: *const CSchema, path: *const c_char, errmsg: *mut *mut c_char) -> i32;\n }\n let mut err: *mut c_char = ptr::null_mut();\n let rc = match fmt {\n \"json\" => mlc_save_json(value_result as *const c_void, value_schema, path_cstr, &mut err),\n \"msgpack\" => mlc_save(value_result as *const c_void, value_schema, path_cstr, &mut err),\n _ => mlc_save_voidstar(value_result as *const c_void, value_schema, path_cstr, &mut err),\n };\n libc::free(path_cstr as *mut c_void);\n if rc != 0 && !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n // Return unit (zero-fill dest)\n ptr::write_bytes(dest, 0, width);\n }\n\n MorlocExpressionType::Load => {\n // Load data from file, return optional\n let child = (*expr).expr.unary_expr;\n let child_result = morloc_eval_r(child, ptr::null_mut(), 0, bndvars)?;\n\n // Extract path string from voidstar Array\n let path_arr = &*(child_result as *const shm::Array);\n let path_abs = shm::rel2abs(path_arr.data)?;\n let path_cstr = libc::malloc(path_arr.size + 1) as *mut c_char;\n if path_cstr.is_null() {\n return Err(MorlocError::Other(\"Failed to allocate for @load path\".into()));\n }\n ptr::copy_nonoverlapping(path_abs, path_cstr as *mut u8, path_arr.size);\n *path_cstr.add(path_arr.size) = 0;\n\n extern \"C\" {\n fn mlc_load(path: *const c_char, schema: *const CSchema, errmsg: *mut *mut c_char) -> *mut c_void;\n }\n let opt_dest = dest;\n let inner_schema = *(*schema).parameters;\n let inner_offset = *(*schema).offsets;\n\n let mut err: *mut c_char = ptr::null_mut();\n let loaded = mlc_load(path_cstr, inner_schema, &mut err);\n libc::free(path_cstr as *mut c_void);\n\n if loaded.is_null() {\n if !err.is_null() {\n libc::free(err as *mut c_void);\n }\n *opt_dest = 0; // None\n } else {\n // Copy loaded voidstar data into the optional's inner slot\n let inner_width = (*inner_schema).width;\n ptr::copy_nonoverlapping(loaded as *const u8, opt_dest.add(inner_offset), inner_width);\n libc::free(loaded as *mut c_void);\n *opt_dest = 1; // Some\n }\n }\n\n _ => {\n return Err(MorlocError::Other(\"Illegal top expression\".into()));\n }\n }\n\n Ok(dest)\n}\n\n// ── Public entry point ────���─────────────────────────────────��────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn morloc_eval(\n expr: *mut MorlocExpression,\n return_schema: *mut CSchema,\n arg_voidstar: *mut *mut u8,\n arg_schemas: *mut *mut CSchema,\n nargs: usize,\n errmsg: *mut *mut c_char,\n) -> AbsPtr {\n clear_errmsg(errmsg);\n\n let mut bndvars: BndVars = HashMap::new();\n let new_expr: *mut MorlocExpression;\n let mut allocated_wrappers: Vec<*mut c_void> = Vec::new();\n\n let eval_expr = match (*expr).etype {\n MorlocExpressionType::Lam | MorlocExpressionType::Pat => {\n // Wrap CLI args as voidstar data expressions and apply\n let arg_exprs = libc::calloc(nargs, std::mem::size_of::<*mut MorlocExpression>()) as *mut *mut MorlocExpression;\n allocated_wrappers.push(arg_exprs as *mut c_void);\n\n for i in 0..nargs {\n let ae = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*ae).etype = MorlocExpressionType::Dat;\n (*ae).schema = *arg_schemas.add(i);\n let ad = libc::calloc(1, std::mem::size_of::()) as *mut MorlocData;\n (*ad).is_voidstar = true;\n (*ad).data.voidstar = *arg_voidstar.add(i) as *mut c_void;\n (*ae).expr.data_expr = ad;\n *arg_exprs.add(i) = ae;\n allocated_wrappers.push(ad as *mut c_void);\n allocated_wrappers.push(ae as *mut c_void);\n }\n\n let app = libc::calloc(1, std::mem::size_of::()) as *mut MorlocAppExpression;\n (*app).atype = if (*expr).etype == MorlocExpressionType::Lam {\n (*app).function.lambda = (*expr).expr.lam_expr;\n MorlocAppExpressionType::Lambda\n } else {\n (*app).function.pattern = (*expr).expr.pattern_expr;\n MorlocAppExpressionType::Pattern\n };\n (*app).args = arg_exprs;\n (*app).nargs = nargs;\n allocated_wrappers.push(app as *mut c_void);\n\n new_expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*new_expr).etype = MorlocExpressionType::App;\n (*new_expr).schema = return_schema;\n (*new_expr).expr.app_expr = app;\n allocated_wrappers.push(new_expr as *mut c_void);\n\n new_expr\n }\n _ => expr,\n };\n\n let result = morloc_eval_r(eval_expr, ptr::null_mut(), 0, &mut bndvars);\n\n // Free wrapper nodes\n for p in &allocated_wrappers {\n libc::free(*p);\n }\n\n match result {\n Ok(ptr) => ptr,\n Err(e) => {\n set_errmsg(errmsg, &e);\n ptr::null_mut()\n }\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/ffi.rs", "content": "//! C ABI wrappers for libmorloc.so\n//!\n//! These `extern \"C\"` functions match the signatures in morloc.h.\n//! Internally they call the Rust implementations and convert between\n//! Rust Result and the C ERRMSG convention (char** last arg).\n\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\n\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::schema::{self};\nuse crate::shm::{self, AbsPtr, BlockHeader, RelPtr, ShmHeader, VolPtr};\npub use crate::cschema::CSchema;\n\n// ── Macro for ERRMSG-pattern FFI wrappers ──────────────────────────────────\n\n/// Wrap a Rust Result-returning expression into the C ERRMSG convention.\n/// On success: clears errmsg, returns the value.\n/// On error: sets errmsg, returns $fail.\nmacro_rules! ffi_try {\n ($errmsg:expr, $fail:expr, $body:expr) => {{\n unsafe { clear_errmsg($errmsg) };\n match $body {\n Ok(val) => val,\n Err(e) => {\n unsafe { set_errmsg($errmsg, &e) };\n $fail\n }\n }\n }};\n}\n\n// CSchema type and conversions are in cschema.rs (always compiled).\n\n// ── SHM functions ──────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn shinit(\n shm_basename: *const c_char,\n volume_index: usize,\n shm_size: usize,\n errmsg: *mut *mut c_char,\n) -> *mut ShmHeader {\n let basename = CStr::from_ptr(shm_basename).to_string_lossy();\n ffi_try!(errmsg, ptr::null_mut(), shm::shinit(&basename, volume_index, shm_size))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shopen(\n volume_index: usize,\n errmsg: *mut *mut c_char,\n) -> *mut ShmHeader {\n ffi_try!(\n errmsg,\n ptr::null_mut(),\n shm::shopen(volume_index).and_then(|opt| opt.ok_or(MorlocError::Shm(\"volume not found\".into())))\n )\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shclose(errmsg: *mut *mut c_char) -> bool {\n ffi_try!(errmsg, false, shm::shclose().map(|_| true))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shm_set_fallback_dir(dir: *const c_char) {\n if !dir.is_null() {\n let d = CStr::from_ptr(dir).to_string_lossy();\n shm::shm_set_fallback_dir(&d);\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shmalloc(size: usize, errmsg: *mut *mut c_char) -> *mut c_void {\n ffi_try!(errmsg, ptr::null_mut(), shm::shmalloc(size).map(|p| p as *mut c_void))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shmemcpy(\n src: *mut c_void,\n size: usize,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n ffi_try!(\n errmsg,\n ptr::null_mut(),\n shm::shmemcpy(src as *const u8, size).map(|p| p as *mut c_void)\n )\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shcalloc(\n nmemb: usize,\n size: usize,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n ffi_try!(errmsg, ptr::null_mut(), shm::shcalloc(nmemb, size).map(|p| p as *mut c_void))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shrealloc(\n ptr: *mut c_void,\n size: usize,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n // TODO: implement shrealloc in shm.rs\n let _ = (ptr, size);\n set_errmsg(errmsg, &MorlocError::Shm(\"shrealloc not yet implemented\".into()));\n ptr::null_mut()\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shfree(ptr: *mut c_void, errmsg: *mut *mut c_char) -> bool {\n ffi_try!(errmsg, false, shm::shfree(ptr as AbsPtr).map(|_| true))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn shincref(ptr: *mut c_void, errmsg: *mut *mut c_char) -> bool {\n ffi_try!(errmsg, false, shm::shincref(ptr as AbsPtr).map(|_| true))\n}\n\n// shfree_by_schema is provided by cli.c\n\n#[no_mangle]\npub unsafe extern \"C\" fn total_shm_size() -> usize {\n shm::total_shm_size()\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn rel2abs(ptr: RelPtr, errmsg: *mut *mut c_char) -> *mut c_void {\n ffi_try!(errmsg, ptr::null_mut(), shm::rel2abs(ptr).map(|p| p as *mut c_void))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn abs2rel(ptr: *mut c_void, errmsg: *mut *mut c_char) -> RelPtr {\n ffi_try!(errmsg, shm::RELNULL, shm::abs2rel(ptr as AbsPtr))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn abs2shm(ptr: *mut c_void, errmsg: *mut *mut c_char) -> *mut ShmHeader {\n ffi_try!(errmsg, ptr::null_mut(), shm::abs2shm(ptr as AbsPtr))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn abs2blk(ptr: *mut c_void, errmsg: *mut *mut c_char) -> *mut BlockHeader {\n clear_errmsg(errmsg);\n if ptr.is_null() {\n set_errmsg(errmsg, &MorlocError::NullPointer);\n return ptr::null_mut();\n }\n let blk = (ptr as *mut u8).sub(std::mem::size_of::()) as *mut BlockHeader;\n if (*blk).magic != shm::BLK_MAGIC {\n set_errmsg(errmsg, &MorlocError::Shm(\"Bad block magic\".into()));\n return ptr::null_mut();\n }\n blk\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn vol2rel(ptr: VolPtr, shm_ptr: *const ShmHeader) -> RelPtr {\n shm::vol2rel(ptr, &*shm_ptr)\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn vol2abs(ptr: VolPtr, shm_ptr: *const ShmHeader) -> *mut c_void {\n shm::vol2abs(ptr, shm_ptr) as *mut c_void\n}\n\n// ── Schema functions ───────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn parse_schema(\n schema_str: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut CSchema {\n clear_errmsg(errmsg);\n if schema_str.is_null() {\n set_errmsg(errmsg, &MorlocError::NullPointer);\n return ptr::null_mut();\n }\n let s = CStr::from_ptr(schema_str).to_string_lossy();\n match schema::parse_schema(&s) {\n Ok(schema) => CSchema::from_rust(&schema),\n Err(e) => {\n set_errmsg(errmsg, &e);\n ptr::null_mut()\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn schema_to_string(schema: *const CSchema) -> *mut c_char {\n if schema.is_null() {\n return ptr::null_mut();\n }\n let rs = CSchema::to_rust(schema);\n let s = schema::schema_to_string(&rs);\n match CString::new(s) {\n Ok(cs) => cs.into_raw(),\n Err(_) => ptr::null_mut(),\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn free_schema(schema: *mut CSchema) {\n CSchema::free(schema);\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn schema_is_fixed_width(schema: *const CSchema) -> bool {\n if schema.is_null() {\n return true;\n }\n let rs = CSchema::to_rust(schema);\n rs.is_fixed_width()\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn schema_alignment(schema: *const CSchema) -> usize {\n if schema.is_null() {\n return 1;\n }\n let rs = CSchema::to_rust(schema);\n rs.alignment()\n}\n\n// Hash: morloc_xxh64 is provided by utility.c (via xxhash.h inline)\n\n// ── Serialization ──────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn pack_with_schema(\n mlc: *const c_void,\n schema: *const CSchema,\n mpkptr: *mut *mut c_char,\n mpk_size: *mut usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n *mpkptr = ptr::null_mut();\n *mpk_size = 0;\n\n let rs = CSchema::to_rust(schema);\n match crate::mpack::pack_with_schema(mlc as AbsPtr, &rs) {\n Ok(data) => {\n *mpk_size = data.len();\n let buf = libc::malloc(data.len()) as *mut u8;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Shm(\"malloc failed\".into()));\n return 1;\n }\n std::ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len());\n *mpkptr = buf as *mut c_char;\n 0\n }\n Err(e) => {\n set_errmsg(errmsg, &e);\n 1\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn pack(\n mlc: *const c_void,\n schema_str: *const c_char,\n mpkptr: *mut *mut c_char,\n mpk_size: *mut usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n let s = CStr::from_ptr(schema_str).to_string_lossy();\n let schema = match schema::parse_schema(&s) {\n Ok(s) => s,\n Err(e) => {\n set_errmsg(errmsg, &e);\n return 1;\n }\n };\n let cs = CSchema::from_rust(&schema);\n let result = pack_with_schema(mlc, cs, mpkptr, mpk_size, errmsg);\n free_schema(cs);\n result\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn unpack_with_schema(\n mpk: *const c_char,\n mpk_size: usize,\n schema: *const CSchema,\n mlcptr: *mut *mut c_void,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n *mlcptr = ptr::null_mut();\n\n let data = std::slice::from_raw_parts(mpk as *const u8, mpk_size);\n let rs = CSchema::to_rust(schema);\n match crate::mpack::unpack_with_schema(data, &rs) {\n Ok(ptr) => {\n *mlcptr = ptr as *mut c_void;\n 0\n }\n Err(e) => {\n set_errmsg(errmsg, &e);\n 1\n }\n }\n}\n\n// quoted, print_voidstar, pretty_print_voidstar, read_json_with_schema\n// are provided by json.c\n\n// ── Schema utility functions needed by C code ──────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn calculate_voidstar_size(\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> usize {\n clear_errmsg(errmsg);\n if data.is_null() || schema.is_null() {\n return 0;\n }\n let rs = CSchema::to_rust(schema);\n match calc_voidstar_size_inner(data as *const u8, &rs) {\n Ok(size) => size,\n Err(e) => {\n set_errmsg(errmsg, &e);\n 0\n }\n }\n}\n\npub fn calc_voidstar_size_inner(\n data: *const u8,\n schema: &crate::schema::Schema,\n) -> Result {\n use crate::schema::SerialType;\n use crate::shm::{self, Array, Tensor};\n\n // SAFETY: data points to voidstar data in SHM with layout described by schema.\n // We only read Array/Tensor headers and follow relptrs to compute total size.\n unsafe {\n match schema.serial_type {\n SerialType::String => {\n let arr = &*(data as *const Array);\n Ok(std::mem::size_of::() + arr.size)\n }\n SerialType::Array => {\n let arr = &*(data as *const Array);\n let mut size = std::mem::size_of::();\n if arr.size == 0 {\n return Ok(size);\n }\n let elem_schema = &schema.parameters[0];\n let elem_width = elem_schema.width;\n size += elem_schema.alignment().saturating_sub(1);\n\n if schema.is_fixed_width() {\n size += elem_width * arr.size;\n } else {\n let elem_data = shm::rel2abs(arr.data)?;\n for i in 0..arr.size {\n size += calc_voidstar_size_inner(\n elem_data.add(i * elem_width),\n elem_schema,\n )?;\n }\n }\n Ok(size)\n }\n SerialType::Optional => {\n let tag = *data;\n let mut size = schema.width;\n if tag != 0 {\n let inner_offset = schema.offsets.first().copied().unwrap_or(\n shm::align_up(1, schema.parameters[0].alignment().max(1)),\n );\n let inner_total = calc_voidstar_size_inner(\n data.add(inner_offset),\n &schema.parameters[0],\n )?;\n if inner_total > schema.parameters[0].width {\n size += inner_total - schema.parameters[0].width;\n }\n }\n Ok(size)\n }\n SerialType::Tensor => {\n let tensor = &*(data as *const Tensor);\n let ndim = schema.offsets.first().copied().unwrap_or(0);\n let elem_width = schema.parameters[0].width;\n let mut size = std::mem::size_of::();\n size += schema.parameters[0].alignment().saturating_sub(1);\n size += ndim * std::mem::size_of::();\n size += schema.parameters[0].alignment().saturating_sub(1);\n size += tensor.total_elements * elem_width;\n Ok(size)\n }\n SerialType::Tuple | SerialType::Map => {\n if schema.is_fixed_width() {\n Ok(schema.width)\n } else {\n let mut size = schema.width;\n for i in 0..schema.parameters.len() {\n let elem_total = calc_voidstar_size_inner(\n data.add(schema.offsets[i]),\n &schema.parameters[i],\n )?;\n if elem_total > schema.parameters[i].width {\n size += elem_total - schema.parameters[i].width;\n }\n }\n Ok(size)\n }\n }\n _ => Ok(schema.width),\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn get_ptr(\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n clear_errmsg(errmsg);\n if schema.is_null() {\n return ptr::null_mut();\n }\n let rs = CSchema::to_rust(schema);\n ffi_try!(errmsg, ptr::null_mut(), shm::shmalloc(rs.width).map(|p| p as *mut c_void))\n}\n" }, { "path": "data/rust/morloc-runtime/src/hash.rs", "content": "//! xxHash64 wrapper using the twox-hash crate.\n//! Replaces the 1500-line xxhash.h header.\n\nuse std::hash::Hasher;\nuse twox_hash::XxHash64;\n\nconst DEFAULT_SEED: u64 = 0;\n\n/// Compute xxHash64 of a byte slice with the default seed.\npub fn xxh64(data: &[u8]) -> u64 {\n let mut hasher = XxHash64::with_seed(DEFAULT_SEED);\n hasher.write(data);\n hasher.finish()\n}\n\n/// Compute xxHash64 with a custom seed.\npub fn xxh64_with_seed(data: &[u8], seed: u64) -> u64 {\n let mut hasher = XxHash64::with_seed(seed);\n hasher.write(data);\n hasher.finish()\n}\n\n/// Mix multiple hash values together (for composite keys).\npub fn mix(a: u64, b: u64) -> u64 {\n // Use xxHash to mix two 64-bit values\n let mut hasher = XxHash64::with_seed(a);\n hasher.write(&b.to_le_bytes());\n hasher.finish()\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_xxh64_empty() {\n let h = xxh64(b\"\");\n assert_ne!(h, 0); // xxHash of empty with seed 0 is a known non-zero value\n }\n\n #[test]\n fn test_xxh64_deterministic() {\n let a = xxh64(b\"hello\");\n let b = xxh64(b\"hello\");\n assert_eq!(a, b);\n }\n\n #[test]\n fn test_xxh64_different_inputs() {\n let a = xxh64(b\"hello\");\n let b = xxh64(b\"world\");\n assert_ne!(a, b);\n }\n\n #[test]\n fn test_mix_commutative_ish() {\n // mix is not commutative by design\n let ab = mix(1, 2);\n let ba = mix(2, 1);\n assert_ne!(ab, ba);\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/http_ffi.rs", "content": "//! C ABI wrappers for HTTP request/response handling.\n//! Replaces http.c.\n\nuse std::ffi::{c_char, c_void};\nuse std::ptr;\n\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\n\nconst HTTP_MAX_HEADERS: usize = 8192;\nconst HTTP_MAX_REQUEST: usize = 4 * 1024 * 1024;\n\n// ── C-compatible types ───────────────────────────────────────────────────────\n\n#[repr(C)]\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum HttpMethod {\n Get = 0,\n Post = 1,\n Delete = 2,\n Options = 3,\n}\n\n#[repr(C)]\npub struct HttpRequest {\n pub method: HttpMethod,\n pub path: [c_char; 256],\n pub body: *mut c_char,\n pub body_len: usize,\n}\n\n#[repr(C)]\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum DaemonMethod {\n Call = 0,\n Discover = 1,\n Health = 2,\n Eval = 3,\n Typecheck = 4,\n Bind = 5,\n Bindings = 6,\n Unbind = 7,\n}\n\n#[repr(C)]\npub struct DaemonRequest {\n pub id: *mut c_char,\n pub method: DaemonMethod,\n pub command: *mut c_char,\n pub args_json: *mut c_char,\n pub expr: *mut c_char,\n pub name: *mut c_char,\n}\n\n// ── http_parse_request ───────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn http_parse_request(\n fd: i32,\n errmsg: *mut *mut c_char,\n) -> *mut HttpRequest {\n clear_errmsg(errmsg);\n\n // Read headers byte by byte until \\r\\n\\r\\n\n let mut header_buf = vec![0u8; HTTP_MAX_HEADERS];\n let mut header_len: usize = 0;\n let mut header_end_pos: Option = None;\n\n while header_len < HTTP_MAX_HEADERS - 1 {\n let n = libc::recv(fd, header_buf.as_mut_ptr().add(header_len) as *mut c_void, 1, 0);\n if n <= 0 {\n set_errmsg(errmsg, &MorlocError::Other(\"Connection closed while reading HTTP headers\".into()));\n return ptr::null_mut();\n }\n header_len += 1;\n\n if header_len >= 4 {\n let tail = &header_buf[header_len - 4..header_len];\n if tail == b\"\\r\\n\\r\\n\" {\n header_end_pos = Some(header_len - 4);\n break;\n }\n }\n }\n\n let header_end = match header_end_pos {\n Some(p) => p,\n None => {\n set_errmsg(errmsg, &MorlocError::Other(\"HTTP headers too large or malformed\".into()));\n return ptr::null_mut();\n }\n };\n\n let header_str = std::str::from_utf8(&header_buf[..header_len]).unwrap_or(\"\");\n\n // Allocate request\n let req = libc::calloc(1, std::mem::size_of::()) as *mut HttpRequest;\n if req.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to allocate http_request_t\".into()));\n return ptr::null_mut();\n }\n\n // Parse method\n if header_str.starts_with(\"GET \") {\n (*req).method = HttpMethod::Get;\n } else if header_str.starts_with(\"POST \") {\n (*req).method = HttpMethod::Post;\n } else if header_str.starts_with(\"DELETE \") {\n (*req).method = HttpMethod::Delete;\n } else if header_str.starts_with(\"OPTIONS \") {\n (*req).method = HttpMethod::Options;\n } else {\n libc::free(req as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Unsupported HTTP method\".into()));\n return ptr::null_mut();\n }\n\n // Parse path\n let first_space = header_str.find(' ').unwrap_or(0) + 1;\n let path_end = header_str[first_space..].find(' ').map(|p| first_space + p).unwrap_or(first_space);\n let path = &header_str[first_space..path_end];\n // Strip query string\n let path = path.split('?').next().unwrap_or(path);\n let path_len = path.len().min(255);\n ptr::copy_nonoverlapping(path.as_ptr(), (*req).path.as_mut_ptr() as *mut u8, path_len);\n (*req).path[path_len] = 0;\n\n // Find Content-Length\n let mut content_length: usize = 0;\n let header_lower = header_str.to_ascii_lowercase();\n if let Some(pos) = header_lower.find(\"content-length:\") {\n let after = &header_str[pos + 15..];\n let trimmed = after.trim_start();\n if let Some(end) = trimmed.find(|c: char| !c.is_ascii_digit()) {\n content_length = trimmed[..end].parse().unwrap_or(0);\n } else {\n content_length = trimmed.parse().unwrap_or(0);\n }\n }\n\n // Read body\n if content_length > 0 {\n if content_length > HTTP_MAX_REQUEST {\n libc::free(req as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(format!(\"HTTP body too large: {} bytes\", content_length)));\n return ptr::null_mut();\n }\n\n let body = libc::malloc(content_length + 1) as *mut u8;\n if body.is_null() {\n libc::free(req as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to allocate HTTP body buffer\".into()));\n return ptr::null_mut();\n }\n\n // Some body bytes may be in header_buf after \\r\\n\\r\\n\n let after_headers = header_end + 4;\n let already_read = (header_len - after_headers).min(content_length);\n if already_read > 0 {\n ptr::copy_nonoverlapping(header_buf.as_ptr().add(after_headers), body, already_read);\n }\n\n let mut total = already_read;\n while total < content_length {\n let n = libc::recv(fd, body.add(total) as *mut c_void, content_length - total, 0);\n if n <= 0 {\n libc::free(body as *mut c_void);\n libc::free(req as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Connection closed while reading HTTP body\".into()));\n return ptr::null_mut();\n }\n total += n as usize;\n }\n *body.add(content_length) = 0;\n (*req).body = body as *mut c_char;\n (*req).body_len = content_length;\n }\n\n req\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn http_free_request(req: *mut HttpRequest) {\n if req.is_null() { return; }\n if !(*req).body.is_null() {\n libc::free((*req).body as *mut c_void);\n }\n libc::free(req as *mut c_void);\n}\n\n// ── http_write_response ──────────────────────────────────────────────────────\n\nfn http_status_text(status: i32) -> &'static str {\n match status {\n 200 => \"OK\",\n 400 => \"Bad Request\",\n 404 => \"Not Found\",\n 405 => \"Method Not Allowed\",\n 500 => \"Internal Server Error\",\n _ => \"Unknown\",\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn http_write_response(\n fd: i32,\n status: i32,\n content_type: *const c_char,\n body: *const c_char,\n body_len: usize,\n) -> bool {\n let ct = if content_type.is_null() {\n \"application/json\"\n } else {\n std::ffi::CStr::from_ptr(content_type).to_str().unwrap_or(\"application/json\")\n };\n\n let header = format!(\n \"HTTP/1.1 {} {}\\r\\n\\\n Content-Type: {}\\r\\n\\\n Content-Length: {}\\r\\n\\\n Connection: close\\r\\n\\\n Access-Control-Allow-Origin: *\\r\\n\\\n Access-Control-Allow-Methods: GET, POST, OPTIONS\\r\\n\\\n Access-Control-Allow-Headers: Content-Type\\r\\n\\\n \\r\\n\",\n status, http_status_text(status), ct, body_len\n );\n\n let n = libc::send(fd, header.as_ptr() as *const c_void, header.len(), crate::utility::SEND_NOSIGNAL);\n if n < 0 { return false; }\n\n if !body.is_null() && body_len > 0 {\n let mut total: usize = 0;\n while total < body_len {\n let n = libc::send(fd, (body as *const u8).add(total) as *const c_void, body_len - total, crate::utility::SEND_NOSIGNAL);\n if n <= 0 { return false; }\n total += n as usize;\n }\n }\n\n true\n}\n\n// ── http_to_daemon_request ───────────────────────────────────────────────────\n\n/// Extract a JSON string value after a key like \"expr\": \"...\"\nfn extract_json_string(body: &str, key: &str) -> Option {\n let search = format!(\"\\\"{}\\\"\", key);\n let pos = body.find(&search)?;\n let after = &body[pos + search.len()..];\n let after = after.trim_start();\n let after = after.strip_prefix(':')?;\n let after = after.trim_start();\n if !after.starts_with('\"') { return None; }\n let after = &after[1..]; // skip opening quote\n let mut result = String::new();\n let mut chars = after.chars();\n loop {\n match chars.next() {\n Some('\\\\') => {\n if let Some(c) = chars.next() {\n result.push(c);\n }\n }\n Some('\"') => break,\n Some(c) => result.push(c),\n None => break,\n }\n }\n Some(result)\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn http_to_daemon_request(\n req: *mut HttpRequest,\n errmsg: *mut *mut c_char,\n) -> *mut DaemonRequest {\n clear_errmsg(errmsg);\n\n let dreq = libc::calloc(1, std::mem::size_of::()) as *mut DaemonRequest;\n if dreq.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to allocate daemon_request_t\".into()));\n return ptr::null_mut();\n }\n\n let path = std::ffi::CStr::from_ptr((*req).path.as_ptr())\n .to_str().unwrap_or(\"\");\n let method = (*req).method;\n\n let body_str = if !(*req).body.is_null() && (*req).body_len > 0 {\n std::str::from_utf8(std::slice::from_raw_parts((*req).body as *const u8, (*req).body_len))\n .unwrap_or(\"\")\n } else {\n \"\"\n };\n\n // GET /health\n if method == HttpMethod::Get && path == \"/health\" {\n (*dreq).method = DaemonMethod::Health;\n return dreq;\n }\n\n // GET /discover\n if method == HttpMethod::Get && path == \"/discover\" {\n (*dreq).method = DaemonMethod::Discover;\n return dreq;\n }\n\n // POST /eval\n if method == HttpMethod::Post && path == \"/eval\" {\n (*dreq).method = DaemonMethod::Eval;\n if let Some(expr) = extract_json_string(body_str, \"expr\") {\n (*dreq).expr = libc::strdup(expr.as_ptr() as *const c_char);\n // strdup from Rust string - need null terminated\n let c = std::ffi::CString::new(expr).unwrap_or_default();\n (*dreq).expr = libc::strdup(c.as_ptr());\n }\n if (*dreq).expr.is_null() {\n libc::free(dreq as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Missing 'expr' field in /eval request body\".into()));\n return ptr::null_mut();\n }\n return dreq;\n }\n\n // POST /typecheck\n if method == HttpMethod::Post && path == \"/typecheck\" {\n (*dreq).method = DaemonMethod::Typecheck;\n if let Some(expr) = extract_json_string(body_str, \"expr\") {\n let c = std::ffi::CString::new(expr).unwrap_or_default();\n (*dreq).expr = libc::strdup(c.as_ptr());\n }\n if (*dreq).expr.is_null() {\n libc::free(dreq as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Missing 'expr' field in /typecheck request body\".into()));\n return ptr::null_mut();\n }\n return dreq;\n }\n\n // POST /bind\n if method == HttpMethod::Post && path == \"/bind\" {\n (*dreq).method = DaemonMethod::Bind;\n if let Some(expr) = extract_json_string(body_str, \"expr\") {\n let c = std::ffi::CString::new(expr).unwrap_or_default();\n (*dreq).expr = libc::strdup(c.as_ptr());\n }\n if let Some(name) = extract_json_string(body_str, \"name\") {\n let c = std::ffi::CString::new(name).unwrap_or_default();\n (*dreq).name = libc::strdup(c.as_ptr());\n }\n if (*dreq).expr.is_null() {\n libc::free(dreq as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Missing 'expr' field in /bind request body\".into()));\n return ptr::null_mut();\n }\n return dreq;\n }\n\n // GET /bindings\n if method == HttpMethod::Get && path == \"/bindings\" {\n (*dreq).method = DaemonMethod::Bindings;\n return dreq;\n }\n\n // DELETE /bindings/\n if method == HttpMethod::Delete && path.starts_with(\"/bindings/\") {\n let name = &path[10..];\n if name.is_empty() {\n libc::free(dreq as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Missing binding name in /bindings/ path\".into()));\n return ptr::null_mut();\n }\n (*dreq).method = DaemonMethod::Unbind;\n let c = std::ffi::CString::new(name).unwrap_or_default();\n (*dreq).name = libc::strdup(c.as_ptr());\n return dreq;\n }\n\n // POST /call/\n if method == HttpMethod::Post && path.starts_with(\"/call/\") {\n let cmd_name = &path[6..];\n if cmd_name.is_empty() {\n libc::free(dreq as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"Missing command name in /call/ path\".into()));\n return ptr::null_mut();\n }\n (*dreq).method = DaemonMethod::Call;\n let c = std::ffi::CString::new(cmd_name).unwrap_or_default();\n (*dreq).command = libc::strdup(c.as_ptr());\n\n // Parse body\n let trimmed = body_str.trim();\n if trimmed.starts_with('[') {\n let c = std::ffi::CString::new(trimmed).unwrap_or_default();\n (*dreq).args_json = libc::strdup(c.as_ptr());\n } else if trimmed.starts_with('{') {\n // Extract \"args\" array\n if let Some(args_pos) = trimmed.find(\"\\\"args\\\"\") {\n let after = &trimmed[args_pos + 6..];\n let after = after.trim_start().strip_prefix(':').unwrap_or(after).trim_start();\n if after.starts_with('[') {\n // Find matching ]\n let mut depth = 0i32;\n let mut in_string = false;\n let mut end = 0;\n for (i, ch) in after.chars().enumerate() {\n if in_string {\n if ch == '\\\\' { continue; }\n if ch == '\"' { in_string = false; }\n } else {\n if ch == '\"' { in_string = true; }\n else if ch == '[' { depth += 1; }\n else if ch == ']' { depth -= 1; if depth == 0 { end = i + 1; break; } }\n }\n }\n if end > 0 {\n let arr = &after[..end];\n let c = std::ffi::CString::new(arr).unwrap_or_default();\n (*dreq).args_json = libc::strdup(c.as_ptr());\n }\n }\n }\n }\n return dreq;\n }\n\n // OPTIONS (CORS preflight)\n if method == HttpMethod::Options {\n (*dreq).method = DaemonMethod::Health;\n return dreq;\n }\n\n libc::free(dreq as *mut c_void);\n let method_str = match method {\n HttpMethod::Get => \"GET\",\n HttpMethod::Post => \"POST\",\n HttpMethod::Delete => \"DELETE\",\n HttpMethod::Options => \"OPTIONS\",\n };\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Unknown HTTP endpoint: {} {}\", method_str, path)));\n ptr::null_mut()\n}\n" }, { "path": "data/rust/morloc-runtime/src/intrinsics.rs", "content": "//! Intrinsic functions for morloc: save/load/show/read/hash.\n//! Replaces intrinsics.c. These are thin wrappers around serialization functions.\n\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\n\n// ── mlc_save: serialize to msgpack file ────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn mlc_save(\n data: *const c_void,\n schema: *const CSchema,\n path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn pack_with_schema(\n mlc: *const c_void, schema: *const CSchema,\n mpk: *mut *mut c_char, mpk_size: *mut usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n fn write_atomic(\n filename: *const c_char, data: *const u8, size: usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n let mut mpk: *mut c_char = ptr::null_mut();\n let mut mpk_size: usize = 0;\n\n let rc = pack_with_schema(data, schema, &mut mpk, &mut mpk_size, &mut err);\n if rc != 0 {\n *errmsg = err;\n return 1;\n }\n\n let wrc = write_atomic(path, mpk as *const u8, mpk_size, &mut err);\n libc::free(mpk as *mut c_void);\n if wrc != 0 {\n *errmsg = err;\n return 1;\n }\n 0\n}\n\n// ── mlc_save_json: serialize to JSON file ──────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn mlc_save_json(\n data: *const c_void,\n schema: *const CSchema,\n path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn voidstar_to_json_string(\n data: *const c_void, schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut c_char;\n fn write_atomic(\n filename: *const c_char, data: *const u8, size: usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n let json = voidstar_to_json_string(data, schema, &mut err);\n if json.is_null() {\n *errmsg = err;\n return 1;\n }\n\n let json_len = libc::strlen(json);\n let wrc = write_atomic(path, json as *const u8, json_len, &mut err);\n libc::free(json as *mut c_void);\n if wrc != 0 {\n *errmsg = err;\n return 1;\n }\n 0\n}\n\n// ── mlc_save_voidstar: serialize to binary voidstar file ───────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn mlc_save_voidstar(\n data: *const c_void,\n schema: *const CSchema,\n path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn flatten_voidstar_to_buffer(\n data: *const c_void, schema: *const CSchema,\n out_buf: *mut *mut u8, out_size: *mut usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n fn write_binary_fd(\n fd: i32, buf: *const c_char, count: usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n\n // Get directory for temp file\n let path_str = CStr::from_ptr(path).to_string_lossy();\n let parent = std::path::Path::new(path_str.as_ref()).parent();\n let dir = match parent {\n Some(p) if !p.as_os_str().is_empty() => p.to_string_lossy().into_owned(),\n _ => \".\".to_string(),\n };\n\n let tmp_template = format!(\"{}/morloc-tmp_XXXXXX\\0\", dir);\n let mut tmp_buf: Vec = tmp_template.into_bytes();\n let fd = libc::mkstemp(tmp_buf.as_mut_ptr() as *mut c_char);\n if fd < 0 {\n set_errmsg(errmsg, &MorlocError::Io(std::io::Error::last_os_error()));\n return 1;\n }\n\n // Write packet header placeholder\n let header_size = std::mem::size_of::();\n let zeros = vec![0u8; header_size];\n if write_binary_fd(fd, zeros.as_ptr() as *const c_char, header_size, &mut err) != 0 {\n libc::close(fd);\n libc::unlink(tmp_buf.as_ptr() as *const c_char);\n *errmsg = err;\n return 1;\n }\n\n // Flatten voidstar\n let mut blob: *mut u8 = ptr::null_mut();\n let mut blob_size: usize = 0;\n if flatten_voidstar_to_buffer(data, schema, &mut blob, &mut blob_size, &mut err) != 0 {\n libc::close(fd);\n libc::unlink(tmp_buf.as_ptr() as *const c_char);\n *errmsg = err;\n return 1;\n }\n\n // Write flattened data\n if write_binary_fd(fd, blob as *const c_char, blob_size, &mut err) != 0 {\n libc::free(blob as *mut c_void);\n libc::close(fd);\n libc::unlink(tmp_buf.as_ptr() as *const c_char);\n *errmsg = err;\n return 1;\n }\n libc::free(blob as *mut c_void);\n\n // Seek back and write real header\n libc::lseek(fd, 0, libc::SEEK_SET);\n let header = crate::packet::PacketHeader::data_mesg(\n crate::packet::PACKET_FORMAT_VOIDSTAR,\n blob_size as u64,\n );\n let hdr_bytes = header.to_bytes();\n write_binary_fd(fd, hdr_bytes.as_ptr() as *const c_char, hdr_bytes.len(), &mut err);\n\n libc::fsync(fd);\n libc::close(fd);\n\n // Atomic rename\n if libc::rename(tmp_buf.as_ptr() as *const c_char, path) != 0 {\n libc::unlink(tmp_buf.as_ptr() as *const c_char);\n set_errmsg(errmsg, &MorlocError::Io(std::io::Error::last_os_error()));\n return 1;\n }\n\n 0\n}\n\n// ── mlc_load: load from file (auto-detect format) ─────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn mlc_load(\n path: *const c_char,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn file_exists(filename: *const c_char) -> bool;\n fn read_binary_file(\n filename: *const c_char, file_size: *mut usize,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n fn load_morloc_data_file(\n path: *const c_char, data: *mut u8, data_size: usize,\n schema: *const CSchema, errmsg: *mut *mut c_char,\n ) -> *mut c_void;\n }\n\n if !file_exists(path) {\n return ptr::null_mut();\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n let mut file_size: usize = 0;\n let data = read_binary_file(path, &mut file_size, &mut err);\n if data.is_null() {\n if !err.is_null() {\n let path_str = CStr::from_ptr(path).to_string_lossy();\n let err_str = CStr::from_ptr(err).to_string_lossy();\n eprintln!(\"@load warning ({}): {}\", path_str, err_str);\n libc::free(err as *mut libc::c_void);\n }\n return ptr::null_mut();\n }\n\n let result = load_morloc_data_file(path, data, file_size, schema, &mut err);\n if result.is_null() && !err.is_null() {\n let path_str = CStr::from_ptr(path).to_string_lossy();\n let err_str = CStr::from_ptr(err).to_string_lossy();\n eprintln!(\"@load warning ({}): {}\", path_str, err_str);\n libc::free(err as *mut libc::c_void);\n }\n result\n}\n\n// ── mlc_hash: hash voidstar data ───────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn mlc_hash(\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n\n let mut err: *mut c_char = ptr::null_mut();\n let hash = crate::cache::hash_voidstar(data, schema, 0, &mut err);\n if !err.is_null() {\n *errmsg = err;\n return ptr::null_mut();\n }\n\n let hex = format!(\"{:016x}\", hash);\n match CString::new(hex) {\n Ok(cs) => cs.into_raw(),\n Err(_) => {\n set_errmsg(errmsg, &MorlocError::Other(\"CString error\".into()));\n ptr::null_mut()\n }\n }\n}\n\n// ── mlc_show: serialize to JSON string ─────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn mlc_show(\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn voidstar_to_json_string(\n data: *const c_void, schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut c_char;\n }\n\n voidstar_to_json_string(data, schema, errmsg)\n}\n\n// ── mlc_read: deserialize from JSON string ─────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn mlc_read(\n json_str: *const c_char,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_void {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn read_json_with_schema(\n dest: *mut u8, json: *mut c_char, schema: *const CSchema,\n errmsg: *mut *mut c_char,\n ) -> *mut u8;\n }\n\n let json_copy = libc::strdup(json_str);\n if json_copy.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"strdup failed\".into()));\n return ptr::null_mut();\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n let result = read_json_with_schema(ptr::null_mut(), json_copy, schema, &mut err);\n libc::free(json_copy as *mut c_void);\n if result.is_null() {\n if !err.is_null() {\n libc::free(err as *mut c_void);\n }\n }\n result as *mut c_void\n}\n\n// write_voidstar_binary is provided by packet.c (still C)\n// It will be ported when packet.c is ported to Rust.\n\n// Unused Rust implementation kept for future use\n#[allow(dead_code)]\nunsafe fn _write_voidstar_binary_rust(\n fd: i32,\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> isize {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn flatten_voidstar_to_buffer(\n data: *const c_void, schema: *const CSchema,\n out_buf: *mut *mut u8, out_size: *mut usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n fn write_binary_fd(\n fd: i32, buf: *const c_char, count: usize,\n errmsg: *mut *mut c_char,\n ) -> i32;\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n let mut blob: *mut u8 = ptr::null_mut();\n let mut blob_size: usize = 0;\n\n if flatten_voidstar_to_buffer(data, schema, &mut blob, &mut blob_size, &mut err) != 0 {\n *errmsg = err;\n return -1;\n }\n\n if write_binary_fd(fd, blob as *const c_char, blob_size, &mut err) != 0 {\n libc::free(blob as *mut c_void);\n *errmsg = err;\n return -1;\n }\n\n libc::free(blob as *mut c_void);\n blob_size as isize\n}\n" }, { "path": "data/rust/morloc-runtime/src/ipc.rs", "content": "//! Unix domain socket IPC for communication between nexus and language pools.\n//! Replaces ipc.c.\n\nuse crate::error::MorlocError;\nuse crate::packet::PacketHeader;\nuse std::io::{Read, Write};\nuse std::os::unix::net::UnixStream;\nuse std::path::Path;\n\n/// Send a packet (header + payload) over a Unix stream socket and receive the response.\npub fn send_and_receive(\n socket_path: &Path,\n header: &PacketHeader,\n payload: &[u8],\n) -> Result<(PacketHeader, Vec), MorlocError> {\n let mut stream = UnixStream::connect(socket_path).map_err(|e| {\n MorlocError::Ipc(format!(\n \"failed to connect to {}: {e}\",\n socket_path.display()\n ))\n })?;\n\n // Send header\n let header_bytes = header.to_bytes();\n stream\n .write_all(&header_bytes)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to send header: {e}\")))?;\n\n // Send payload\n if !payload.is_empty() {\n stream\n .write_all(payload)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to send payload: {e}\")))?;\n }\n\n // Read response header\n let mut resp_header_bytes = [0u8; 32];\n stream\n .read_exact(&mut resp_header_bytes)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to read response header: {e}\")))?;\n\n let resp_header = PacketHeader::from_bytes(&resp_header_bytes)?;\n\n // Read response payload\n let payload_len = resp_header.length as usize;\n let mut resp_payload = vec![0u8; payload_len];\n if payload_len > 0 {\n stream\n .read_exact(&mut resp_payload)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to read response payload: {e}\")))?;\n }\n\n Ok((resp_header, resp_payload))\n}\n\n/// Read a single packet from a connected stream.\npub fn read_packet(stream: &mut UnixStream) -> Result<(PacketHeader, Vec), MorlocError> {\n let mut header_bytes = [0u8; 32];\n stream\n .read_exact(&mut header_bytes)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to read packet header: {e}\")))?;\n\n let header = PacketHeader::from_bytes(&header_bytes)?;\n\n // Skip metadata between header and payload\n let skip = header.offset as usize - 32;\n if skip > 0 {\n let mut discard = vec![0u8; skip];\n stream\n .read_exact(&mut discard)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to skip metadata: {e}\")))?;\n }\n\n let payload_len = header.length as usize;\n let mut payload = vec![0u8; payload_len];\n if payload_len > 0 {\n stream\n .read_exact(&mut payload)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to read payload: {e}\")))?;\n }\n\n Ok((header, payload))\n}\n\n/// Send a packet over a connected stream.\npub fn send_packet(\n stream: &mut UnixStream,\n header: &PacketHeader,\n payload: &[u8],\n) -> Result<(), MorlocError> {\n let header_bytes = header.to_bytes();\n stream\n .write_all(&header_bytes)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to send header: {e}\")))?;\n if !payload.is_empty() {\n stream\n .write_all(payload)\n .map_err(|e| MorlocError::Ipc(format!(\"failed to send payload: {e}\")))?;\n }\n Ok(())\n}\n" }, { "path": "data/rust/morloc-runtime/src/ipc_ffi.rs", "content": "//! C ABI wrappers for IPC functions.\n//! Replaces ipc.c with Rust implementations of Unix domain socket operations.\n\nuse std::ffi::{c_char, c_void, CStr};\nuse std::ptr;\n\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\n\n// ── C types from call.h ──────────────────────────────────────────────────────\n\n#[repr(C)]\npub struct ClientList {\n pub fd: i32,\n pub next: *mut ClientList,\n}\n\n// language_daemon_t has fd_set which is 128 bytes on Linux.\n// We represent it as an opaque struct and use libc calls.\n#[repr(C)]\npub struct LanguageDaemon {\n pub socket_path: *mut c_char,\n pub tmpdir: *mut c_char,\n pub shm_basename: *mut c_char,\n pub shm: *mut crate::shm::ShmHeader,\n pub shm_default_size: usize,\n pub server_fd: i32,\n pub read_fds: libc::fd_set,\n pub client_fds: *mut ClientList,\n}\n\nconst BUFFER_SIZE: usize = 65536;\n\n// ── close_socket / close_daemon ──────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn close_socket(socket_id: i32) {\n if socket_id >= 0 {\n libc::close(socket_id);\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn close_daemon(daemon_ptr: *mut *mut LanguageDaemon) {\n if daemon_ptr.is_null() || (*daemon_ptr).is_null() {\n return;\n }\n let daemon = *daemon_ptr;\n\n close_socket((*daemon).server_fd);\n\n // Free client list\n let mut current = (*daemon).client_fds;\n while !current.is_null() {\n let next = (*current).next;\n libc::close((*current).fd);\n libc::free(current as *mut c_void);\n current = next;\n }\n\n if !(*daemon).socket_path.is_null() {\n libc::unlink((*daemon).socket_path);\n libc::free((*daemon).socket_path as *mut c_void);\n }\n if !(*daemon).tmpdir.is_null() {\n libc::free((*daemon).tmpdir as *mut c_void);\n }\n if !(*daemon).shm_basename.is_null() {\n libc::free((*daemon).shm_basename as *mut c_void);\n }\n\n libc::free(daemon as *mut c_void);\n *daemon_ptr = ptr::null_mut();\n}\n\n// ── Socket helpers ───────────────────────────────────────────────────────────\n\nunsafe fn new_socket(errmsg: *mut *mut c_char) -> i32 {\n clear_errmsg(errmsg);\n let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0);\n if fd < 0 {\n set_errmsg(errmsg, &MorlocError::Ipc(\"Error creating socket\".into()));\n return -1;\n }\n crate::utility::set_nosigpipe(fd);\n fd\n}\n\nunsafe fn new_server_addr(socket_path: *const c_char) -> libc::sockaddr_un {\n let mut addr: libc::sockaddr_un = std::mem::zeroed();\n addr.sun_family = libc::AF_UNIX as libc::sa_family_t;\n let path_bytes = CStr::from_ptr(socket_path).to_bytes();\n let copy_len = path_bytes.len().min(addr.sun_path.len() - 1);\n ptr::copy_nonoverlapping(\n path_bytes.as_ptr() as *const c_char,\n addr.sun_path.as_mut_ptr(),\n copy_len,\n );\n addr\n}\n\nunsafe fn new_server(socket_path: *const c_char, errmsg: *mut *mut c_char) -> i32 {\n let server_fd = new_socket(errmsg);\n if server_fd < 0 {\n return -1;\n }\n\n let addr = new_server_addr(socket_path);\n\n // Remove any existing socket file\n libc::unlink(socket_path);\n\n if libc::bind(server_fd, &addr as *const libc::sockaddr_un as *const libc::sockaddr,\n std::mem::size_of::() as u32) < 0 {\n close_socket(server_fd);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Error binding socket\".into()));\n return -1;\n }\n\n if libc::listen(server_fd, 16) < 0 {\n close_socket(server_fd);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Error listening on socket\".into()));\n return -1;\n }\n\n server_fd\n}\n\n// ── start_daemon ─────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn start_daemon(\n socket_path: *const c_char,\n tmpdir: *const c_char,\n shm_basename: *const c_char,\n shm_default_size: usize,\n errmsg: *mut *mut c_char,\n) -> *mut LanguageDaemon {\n clear_errmsg(errmsg);\n\n let daemon = libc::calloc(1, std::mem::size_of::()) as *mut LanguageDaemon;\n if daemon.is_null() {\n set_errmsg(errmsg, &MorlocError::Ipc(\"Calloc for language_daemon_t failed\".into()));\n return ptr::null_mut();\n }\n\n (*daemon).server_fd = -1;\n (*daemon).socket_path = libc::strdup(socket_path);\n (*daemon).tmpdir = libc::strdup(tmpdir);\n (*daemon).shm_basename = libc::strdup(shm_basename);\n\n if (*daemon).socket_path.is_null() || (*daemon).tmpdir.is_null() || (*daemon).shm_basename.is_null() {\n close_daemon(&mut (daemon as *mut LanguageDaemon));\n set_errmsg(errmsg, &MorlocError::Ipc(\"strdup failed in start_daemon\".into()));\n return ptr::null_mut();\n }\n\n (*daemon).shm_default_size = shm_default_size;\n (*daemon).client_fds = ptr::null_mut();\n libc::FD_ZERO(&mut (*daemon).read_fds);\n\n // Set fallback dir for file-backed SHM\n crate::shm::shm_set_fallback_dir(&CStr::from_ptr(tmpdir).to_string_lossy());\n\n // Init shared memory\n let mut err: *mut c_char = ptr::null_mut();\n let shm = crate::ffi::shinit(shm_basename, 0, shm_default_size, &mut err);\n if !err.is_null() {\n close_daemon(&mut (daemon as *mut LanguageDaemon));\n *errmsg = err;\n return ptr::null_mut();\n }\n (*daemon).shm = shm;\n\n // Create server socket\n (*daemon).server_fd = new_server(socket_path, &mut err);\n if !err.is_null() {\n close_daemon(&mut (daemon as *mut LanguageDaemon));\n *errmsg = err;\n return ptr::null_mut();\n }\n\n // Set non-blocking mode\n let flags = libc::fcntl((*daemon).server_fd, libc::F_GETFL);\n if flags == -1 || libc::fcntl((*daemon).server_fd, libc::F_SETFL, flags | libc::O_NONBLOCK) == -1 {\n let errno_msg = std::ffi::CStr::from_ptr(libc::strerror(crate::utility::errno_val()))\n .to_string_lossy().into_owned();\n close_daemon(&mut (daemon as *mut LanguageDaemon));\n set_errmsg(errmsg, &MorlocError::Ipc(format!(\"Failed to set non-blocking mode: {}\", errno_msg)));\n return ptr::null_mut();\n }\n\n daemon\n}\n\n// ── stream_from_client_wait ──────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn stream_from_client_wait(\n client_fd: i32,\n pselect_timeout_us: i32,\n recv_timeout_us: i32,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n\n if libc::fcntl(client_fd, libc::F_GETFD) == -1 {\n set_errmsg(errmsg, &MorlocError::Ipc(\"Invalid file descriptor\".into()));\n return ptr::null_mut();\n }\n\n let buffer = libc::calloc(BUFFER_SIZE, 1) as *mut u8;\n if buffer.is_null() {\n set_errmsg(errmsg, &MorlocError::Ipc(\"calloc failed for buffer\".into()));\n return ptr::null_mut();\n }\n\n let mut read_fds: libc::fd_set = std::mem::zeroed();\n let max_fd = client_fd;\n\n // Timeout setup\n let mut ts_loop: libc::timespec = std::mem::zeroed();\n let timeout_ptr = if pselect_timeout_us > 0 {\n ts_loop.tv_sec = (pselect_timeout_us / 1000000) as i64;\n ts_loop.tv_nsec = ((pselect_timeout_us % 1000000) * 1000) as i64;\n &ts_loop as *const libc::timespec\n } else {\n ptr::null()\n };\n\n // Signal mask setup\n let mut mask: libc::sigset_t = std::mem::zeroed();\n let mut origmask: libc::sigset_t = std::mem::zeroed();\n libc::sigemptyset(&mut mask);\n libc::sigaddset(&mut mask, libc::SIGINT);\n libc::pthread_sigmask(libc::SIG_SETMASK, &mask, &mut origmask);\n\n // Initial receive with timeout\n let mut ready;\n loop {\n libc::FD_ZERO(&mut read_fds);\n libc::FD_SET(client_fd, &mut read_fds);\n ready = libc::pselect(max_fd + 1, &mut read_fds, ptr::null_mut(), ptr::null_mut(), timeout_ptr, &origmask);\n if !(ready < 0 && crate::utility::errno_val() == libc::EINTR) {\n break;\n }\n }\n libc::pthread_sigmask(libc::SIG_SETMASK, &origmask, ptr::null_mut());\n\n if ready == 0 {\n libc::free(buffer as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Timeout waiting for initial data\".into()));\n return ptr::null_mut();\n }\n if ready < 0 {\n libc::free(buffer as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"pselect error\".into()));\n return ptr::null_mut();\n }\n if !libc::FD_ISSET(client_fd, &read_fds) {\n libc::free(buffer as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Bad client file descriptor\".into()));\n return ptr::null_mut();\n }\n\n let recv_length = libc::recv(client_fd, buffer as *mut c_void, BUFFER_SIZE, 0);\n if recv_length == 0 {\n libc::free(buffer as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Connection closed by peer\".into()));\n return ptr::null_mut();\n }\n if recv_length < 0 && crate::utility::errno_val() != libc::EWOULDBLOCK && crate::utility::errno_val() != libc::EAGAIN {\n libc::free(buffer as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Recv error\".into()));\n return ptr::null_mut();\n }\n\n // Get packet size from header\n let mut packet_err: *mut c_char = ptr::null_mut();\n let packet_length = crate::packet_ffi::morloc_packet_size(buffer, &mut packet_err);\n if !packet_err.is_null() {\n libc::free(buffer as *mut c_void);\n *errmsg = packet_err;\n return ptr::null_mut();\n }\n\n let result = libc::calloc(packet_length, 1) as *mut u8;\n if result.is_null() {\n libc::free(buffer as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"calloc failure\".into()));\n return ptr::null_mut();\n }\n\n let copy_length = (recv_length as usize).min(packet_length);\n ptr::copy_nonoverlapping(buffer, result, copy_length);\n let mut data_ptr = result.add(copy_length);\n libc::free(buffer as *mut c_void);\n\n let attempts = 10;\n while (data_ptr as usize - result as usize) < packet_length {\n let mut packet_received = false;\n for attempt in 0..attempts {\n libc::FD_ZERO(&mut read_fds);\n libc::FD_SET(client_fd, &mut read_fds);\n\n let recv_timeout_ptr = if recv_timeout_us > 0 {\n let total_us = recv_timeout_us as i64 * (attempt as i64 + 1);\n ts_loop.tv_sec = total_us / 1000000;\n ts_loop.tv_nsec = (total_us % 1000000) * 1000;\n &ts_loop as *const libc::timespec\n } else {\n ptr::null()\n };\n\n libc::pthread_sigmask(libc::SIG_SETMASK, &mask, ptr::null_mut());\n ready = libc::pselect(max_fd + 1, &mut read_fds, ptr::null_mut(), ptr::null_mut(), recv_timeout_ptr, &origmask);\n libc::pthread_sigmask(libc::SIG_SETMASK, &origmask, ptr::null_mut());\n\n if ready == 0 {\n libc::free(result as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Timeout waiting for remaining data\".into()));\n return ptr::null_mut();\n }\n if ready < 0 && crate::utility::errno_val() != libc::EINTR {\n libc::free(result as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"pselect error\".into()));\n return ptr::null_mut();\n }\n if ready <= 0 { continue; }\n\n if libc::FD_ISSET(client_fd, &read_fds) {\n let remaining = packet_length - (data_ptr as usize - result as usize);\n let recv_size = remaining.min(BUFFER_SIZE);\n let n = libc::recv(client_fd, data_ptr as *mut c_void, recv_size, 0);\n if n > 0 {\n data_ptr = data_ptr.add(n as usize);\n packet_received = true;\n break;\n }\n if n == 0 {\n libc::free(result as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Connection closed early\".into()));\n return ptr::null_mut();\n }\n if n < 0 && crate::utility::errno_val() != libc::EWOULDBLOCK && crate::utility::errno_val() != libc::EAGAIN {\n libc::free(result as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Recv error\".into()));\n return ptr::null_mut();\n }\n }\n }\n if !packet_received {\n libc::free(result as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Ipc(\"Failed to retrieve packet\".into()));\n return ptr::null_mut();\n }\n }\n\n result\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn stream_from_client(\n client_fd: i32,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n stream_from_client_wait(client_fd, 0, 0, errmsg)\n}\n\n// ── send_and_receive_over_socket ─────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn send_and_receive_over_socket_wait(\n socket_path: *const c_char,\n packet: *const u8,\n pselect_timeout_us: i32,\n recv_timeout_us: i32,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n\n let mut err: *mut c_char = ptr::null_mut();\n let client_fd = new_socket(&mut err);\n if client_fd < 0 {\n *errmsg = err;\n return ptr::null_mut();\n }\n\n let addr = new_server_addr(socket_path);\n\n // Connect with retry (matching C WAIT macro behavior)\n let mut retcode;\n let mut attempts = 0;\n loop {\n retcode = libc::connect(client_fd, &addr as *const libc::sockaddr_un as *const libc::sockaddr,\n std::mem::size_of::() as u32);\n if retcode == 0 { break; }\n attempts += 1;\n if attempts > 300 { // ~30 seconds with 100ms sleep\n close_socket(client_fd);\n set_errmsg(errmsg, &MorlocError::Ipc(format!(\n \"Failed to connect to pipe '{}'\",\n CStr::from_ptr(socket_path).to_string_lossy()\n )));\n return ptr::null_mut();\n }\n libc::usleep(100_000); // 100ms\n }\n\n let packet_size = crate::packet_ffi::morloc_packet_size(packet, &mut err);\n if !err.is_null() {\n close_socket(client_fd);\n *errmsg = err;\n return ptr::null_mut();\n }\n\n // Send packet in loop\n let mut total_sent: usize = 0;\n while total_sent < packet_size {\n let bytes_sent = libc::send(\n client_fd,\n packet.add(total_sent) as *const c_void,\n packet_size - total_sent,\n crate::utility::SEND_NOSIGNAL,\n );\n if bytes_sent <= 0 {\n close_socket(client_fd);\n set_errmsg(errmsg, &MorlocError::Ipc(format!(\n \"Failed to send data to '{}'\",\n CStr::from_ptr(socket_path).to_string_lossy()\n )));\n return ptr::null_mut();\n }\n total_sent += bytes_sent as usize;\n }\n\n let result = stream_from_client_wait(client_fd, pselect_timeout_us, recv_timeout_us, &mut err);\n if !err.is_null() {\n close_socket(client_fd);\n *errmsg = err;\n return ptr::null_mut();\n }\n\n close_socket(client_fd);\n result\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn send_and_receive_over_socket(\n socket_path: *const c_char,\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n send_and_receive_over_socket_wait(socket_path, packet, 0, 0, errmsg)\n}\n\n// ── send_packet_to_foreign_server ────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn send_packet_to_foreign_server(\n client_fd: i32,\n packet: *mut u8,\n errmsg: *mut *mut c_char,\n) -> usize {\n clear_errmsg(errmsg);\n\n let mut err: *mut c_char = ptr::null_mut();\n let size = crate::packet_ffi::morloc_packet_size(packet, &mut err);\n if !err.is_null() {\n *errmsg = err;\n return 0;\n }\n\n let mut total_sent: usize = 0;\n while total_sent < size {\n let bytes_sent = libc::send(\n client_fd,\n packet.add(total_sent) as *const c_void,\n size - total_sent,\n crate::utility::SEND_NOSIGNAL,\n );\n if bytes_sent <= 0 {\n set_errmsg(errmsg, &MorlocError::Ipc(format!(\n \"Failed to send over client {}\", client_fd\n )));\n return 0;\n }\n total_sent += bytes_sent as usize;\n }\n\n total_sent\n}\n\n// ── wait_for_client ──────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn wait_for_client_with_timeout(\n daemon: *mut LanguageDaemon,\n timeout_us: i32,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n\n libc::FD_ZERO(&mut (*daemon).read_fds);\n libc::FD_SET((*daemon).server_fd, &mut (*daemon).read_fds);\n\n let mut max_fd = (*daemon).server_fd;\n\n // Add existing client fds\n let mut client = (*daemon).client_fds;\n while !client.is_null() {\n libc::FD_SET((*client).fd, &mut (*daemon).read_fds);\n if (*client).fd > max_fd {\n max_fd = (*client).fd;\n }\n client = (*client).next;\n }\n\n // Timeout\n let mut ts: libc::timespec = std::mem::zeroed();\n let timeout_ptr = if timeout_us > 0 {\n ts.tv_sec = (timeout_us / 1000000) as i64;\n ts.tv_nsec = ((timeout_us % 1000000) * 1000) as i64;\n &ts as *const libc::timespec\n } else {\n ptr::null()\n };\n\n let mut emptymask: libc::sigset_t = std::mem::zeroed();\n libc::sigemptyset(&mut emptymask);\n\n let ready = libc::pselect(max_fd + 1, &mut (*daemon).read_fds, ptr::null_mut(), ptr::null_mut(), timeout_ptr, &emptymask);\n if ready < 0 {\n if crate::utility::errno_val() == libc::EINTR {\n return 0;\n }\n set_errmsg(errmsg, &MorlocError::Ipc(\"pselect error\".into()));\n return -1;\n }\n if ready == 0 {\n return 0;\n }\n\n // Check for new connection\n if libc::FD_ISSET((*daemon).server_fd, &(*daemon).read_fds) {\n let selected_fd = libc::accept((*daemon).server_fd, ptr::null_mut(), ptr::null_mut());\n if selected_fd >= 0 {\n crate::utility::set_nosigpipe(selected_fd);\n libc::fcntl(selected_fd, libc::F_SETFL, libc::O_NONBLOCK);\n\n let new_client = libc::calloc(1, std::mem::size_of::()) as *mut ClientList;\n (*new_client).fd = selected_fd;\n (*new_client).next = ptr::null_mut();\n\n if (*daemon).client_fds.is_null() {\n (*daemon).client_fds = new_client;\n } else {\n let mut last = (*daemon).client_fds;\n while !(*last).next.is_null() {\n last = (*last).next;\n }\n (*last).next = new_client;\n }\n }\n // Ignore EAGAIN/EWOULDBLOCK on accept\n }\n\n if (*daemon).client_fds.is_null() {\n return 0; // spurious wakeup\n }\n\n // Dequeue first client\n let client_node = (*daemon).client_fds;\n let return_fd = (*client_node).fd;\n (*daemon).client_fds = (*client_node).next;\n libc::free(client_node as *mut c_void);\n\n return_fd\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn wait_for_client(\n daemon: *mut LanguageDaemon,\n errmsg: *mut *mut c_char,\n) -> i32 {\n wait_for_client_with_timeout(daemon, 0, errmsg)\n}\n" }, { "path": "data/rust/morloc-runtime/src/json.rs", "content": "//! JSON <-> Voidstar conversion.\n//!\n//! # Safety model\n//!\n//! All SHM pointer operations are encapsulated in `ShmWriter` (write) and\n//! `ShmReader` (read). Each has a single `unsafe fn new()` constructor;\n//! all subsequent reads/writes through the handle are safe methods.\n//! The only remaining `unsafe` blocks are `libc::snprintf` for float\n//! formatting and constructing readers/writers at known-valid offsets.\n\nuse crate::error::MorlocError;\nuse crate::schema::{Schema, SerialType};\nuse crate::shm::{self, AbsPtr, Array, RELNULL};\n\n// ── Safe SHM abstractions ────────────────────────────────────────────────────\n\n/// Write handle to a region of shared memory.\n///\n/// # Safety invariant\n/// `ptr` was obtained from `shmalloc` and points to at least `len` writable bytes.\nstruct ShmWriter {\n ptr: *mut u8,\n #[cfg(debug_assertions)]\n len: usize,\n}\n\nimpl ShmWriter {\n /// # Safety\n /// `ptr` must point to `len` bytes of valid, writable SHM.\n unsafe fn new(ptr: *mut u8, len: usize) -> Self {\n let _ = len;\n ShmWriter { ptr, #[cfg(debug_assertions)] len }\n }\n\n fn as_ptr(&self) -> *mut u8 { self.ptr }\n\n fn write_bytes(&self, offset: usize, src: &[u8]) {\n debug_assert!(offset + src.len() <= { #[cfg(debug_assertions)] { self.len } #[cfg(not(debug_assertions))] { usize::MAX } });\n unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), self.ptr.add(offset), src.len()); }\n }\n\n fn zero(&self, offset: usize, count: usize) {\n unsafe { std::ptr::write_bytes(self.ptr.add(offset), 0, count); }\n }\n\n fn write_val(&self, offset: usize, val: T) {\n unsafe { (self.ptr.add(offset) as *mut T).write_unaligned(val); }\n }\n\n fn write_array_header(&self, offset: usize, size: usize, data_rel: shm::RelPtr) {\n let arr = Array { size, data: data_rel };\n unsafe {\n std::ptr::copy_nonoverlapping(\n &arr as *const Array as *const u8, self.ptr.add(offset),\n std::mem::size_of::(),\n );\n }\n }\n\n fn sub(&self, offset: usize, sub_len: usize) -> ShmWriter {\n unsafe { ShmWriter::new(self.ptr.add(offset), sub_len) }\n }\n}\n\n/// Read handle to SHM data.\n///\n/// # Safety invariant\n/// `ptr` was obtained from `rel2abs`/`shmalloc` and points to valid readable SHM.\nstruct ShmReader { ptr: *const u8 }\n\nimpl ShmReader {\n /// # Safety\n /// `ptr` must point to valid, readable shared memory.\n unsafe fn new(ptr: *const u8) -> Self { ShmReader { ptr } }\n\n fn read_val(&self, offset: usize) -> T {\n unsafe { (self.ptr.add(offset) as *const T).read_unaligned() }\n }\n fn read_u8(&self, offset: usize) -> u8 { self.read_val(offset) }\n fn read_array(&self, offset: usize) -> Array { unsafe { *(self.ptr.add(offset) as *const Array) } }\n\n fn read_str(&self, offset: usize, len: usize) -> &str {\n unsafe {\n std::str::from_utf8(std::slice::from_raw_parts(self.ptr.add(offset), len)).unwrap_or(\"\")\n }\n }\n\n fn at(&self, offset: usize) -> ShmReader {\n unsafe { ShmReader::new(self.ptr.add(offset)) }\n }\n}\n\n// ── JSON -> Voidstar ───────────────────────────────────────────────────────\n\npub fn read_json_with_schema(json_str: &str, schema: &Schema) -> Result {\n read_json_with_schema_dest(None, json_str, schema)\n}\n\npub fn read_json_with_schema_dest(\n dest: Option, json_str: &str, schema: &Schema,\n) -> Result {\n let value: serde_json::Value = serde_json::from_str(json_str)\n .map_err(|e| MorlocError::Serialization(format!(\"JSON parse error: {}\", e)))?;\n json_to_voidstar(&value, schema, dest)\n}\n\nfn alloc(dest: Option, size: usize) -> Result {\n let ptr = match dest { Some(p) => p, None => shm::shmalloc(size)? };\n // SAFETY: ptr from shmalloc or caller-provided valid SHM of sufficient size\n Ok(unsafe { ShmWriter::new(ptr, size) })\n}\n\nfn json_to_voidstar(\n value: &serde_json::Value, schema: &Schema, dest: Option,\n) -> Result {\n match schema.serial_type {\n SerialType::Nil => { let w = alloc(dest, 1)?; w.write_val::(0, 0); Ok(w.as_ptr()) }\n SerialType::Bool => {\n let b = value.as_bool().ok_or_else(|| err(\"expected bool\"))?;\n let w = alloc(dest, 1)?; w.write_val::(0, b as u8); Ok(w.as_ptr())\n }\n SerialType::Sint8 => { let w = alloc(dest, 1)?; w.write_val::(0, as_i64(value)? as i8); Ok(w.as_ptr()) }\n SerialType::Sint16 => { let w = alloc(dest, 2)?; w.write_val::(0, as_i64(value)? as i16); Ok(w.as_ptr()) }\n SerialType::Sint32 => { let w = alloc(dest, 4)?; w.write_val::(0, as_i64(value)? as i32); Ok(w.as_ptr()) }\n SerialType::Sint64 => { let w = alloc(dest, 8)?; w.write_val::(0, as_i64(value)?); Ok(w.as_ptr()) }\n SerialType::Uint8 => { let w = alloc(dest, 1)?; w.write_val::(0, as_u64(value)? as u8); Ok(w.as_ptr()) }\n SerialType::Uint16 => { let w = alloc(dest, 2)?; w.write_val::(0, as_u64(value)? as u16); Ok(w.as_ptr()) }\n SerialType::Uint32 => { let w = alloc(dest, 4)?; w.write_val::(0, as_u64(value)? as u32); Ok(w.as_ptr()) }\n SerialType::Uint64 => { let w = alloc(dest, 8)?; w.write_val::(0, as_u64(value)?); Ok(w.as_ptr()) }\n SerialType::Float32 => { let w = alloc(dest, 4)?; w.write_val::(0, as_f64(value)? as f32); Ok(w.as_ptr()) }\n SerialType::Float64 => { let w = alloc(dest, 8)?; w.write_val::(0, as_f64(value)?); Ok(w.as_ptr()) }\n\n SerialType::String => {\n let s = value.as_str().ok_or_else(|| err(\"expected string\"))?;\n let bytes = s.as_bytes();\n let hdr = std::mem::size_of::();\n\n let (w, data_rel) = if dest.is_some() {\n let w = alloc(dest, hdr)?;\n let data_rel = if bytes.is_empty() { RELNULL } else {\n shm::abs2rel(shm::shmemcpy(bytes.as_ptr(), bytes.len())?)?\n };\n (w, data_rel)\n } else {\n let w = alloc(None, hdr + bytes.len())?;\n w.write_bytes(hdr, bytes);\n // SAFETY: data is hdr bytes into the same shmalloc block\n let data_rel = shm::abs2rel(unsafe { w.as_ptr().add(hdr) })?;\n (w, data_rel)\n };\n w.write_array_header(0, bytes.len(), data_rel);\n Ok(w.as_ptr())\n }\n\n SerialType::Array => {\n let arr_val = value.as_array().ok_or_else(|| err(\"expected array\"))?;\n let es = schema.parameters.first().ok_or_else(|| err(\"array has no element type\"))?;\n let n = arr_val.len();\n let ew = es.width;\n let hdr = std::mem::size_of::();\n\n let (hw, data_ptr) = if dest.is_some() {\n let hw = alloc(dest, hdr)?;\n let dp = if n > 0 { shm::shmalloc(n * ew)? } else { std::ptr::null_mut() };\n (hw, dp)\n } else {\n let w = alloc(None, hdr + n * ew)?;\n // SAFETY: data is hdr bytes into the same shmalloc block\n let dp = unsafe { w.as_ptr().add(hdr) };\n (w, dp)\n };\n let data_rel = if data_ptr.is_null() { RELNULL } else { shm::abs2rel(data_ptr)? };\n\n for (i, elem) in arr_val.iter().enumerate() {\n // SAFETY: data_ptr + i * ew is within the data allocation\n let ep = unsafe { data_ptr.add(i * ew) };\n json_to_voidstar(elem, es, Some(ep))?;\n }\n hw.write_array_header(0, n, data_rel);\n Ok(hw.as_ptr())\n }\n\n SerialType::Tuple | SerialType::Map => {\n let fields = extract_fields(value, schema)?;\n if fields.len() != schema.parameters.len() {\n return Err(err(&format!(\"expected {} fields, got {}\", schema.parameters.len(), fields.len())));\n }\n let w = alloc(dest, schema.width)?;\n w.zero(0, schema.width);\n for (i, (fv, fs)) in fields.iter().zip(schema.parameters.iter()).enumerate() {\n let sub = w.sub(schema.offsets[i], fs.width);\n json_to_voidstar(fv, fs, Some(sub.as_ptr()))?;\n }\n Ok(w.as_ptr())\n }\n\n SerialType::Optional => {\n let inner = schema.parameters.first().ok_or_else(|| err(\"optional has no inner type\"))?;\n let off = shm::align_up(1, inner.alignment().max(1));\n let total = off + inner.width;\n let w = alloc(dest, total)?;\n if value.is_null() {\n w.zero(0, total);\n } else {\n w.write_val::(0, 1);\n json_to_voidstar(value, inner, Some(w.sub(off, inner.width).as_ptr()))?;\n }\n Ok(w.as_ptr())\n }\n\n SerialType::Tensor => Err(err(\"Tensor JSON parsing not yet implemented\")),\n }\n}\n\nfn extract_fields(value: &serde_json::Value, schema: &Schema) -> Result, MorlocError> {\n if schema.serial_type == SerialType::Map && value.is_object() {\n let obj = value.as_object().unwrap();\n Ok(schema.keys.iter().map(|k| obj.get(k).cloned().unwrap_or(serde_json::Value::Null)).collect())\n } else {\n value.as_array().ok_or_else(|| err(\"expected array for tuple/map\")).cloned()\n }\n}\n\n// ── Voidstar -> JSON ───────────────────────────────────────────────────────\n\npub fn voidstar_to_json_string(ptr: AbsPtr, schema: &Schema) -> Result {\n let mut buf = String::new();\n // SAFETY: ptr from shmalloc/rel2abs — valid SHM\n let r = unsafe { ShmReader::new(ptr) };\n to_json(&r, schema, &mut buf)?;\n Ok(buf)\n}\n\npub fn print_voidstar(ptr: AbsPtr, schema: &Schema) -> Result<(), MorlocError> {\n println!(\"{}\", voidstar_to_json_string(ptr, schema)?);\n Ok(())\n}\n\npub fn pretty_print_voidstar(ptr: AbsPtr, schema: &Schema) -> Result<(), MorlocError> {\n let json = voidstar_to_json_string(ptr, schema)?;\n let v: serde_json::Value = serde_json::from_str(&json).map_err(|e| err(&e.to_string()))?;\n match &v {\n // Print strings as raw text (unescaped, no quotes)\n serde_json::Value::String(s) => println!(\"{}\", s),\n // Print numbers and bools as plain values\n serde_json::Value::Number(n) => println!(\"{}\", n),\n serde_json::Value::Bool(b) => println!(\"{}\", b),\n serde_json::Value::Null => println!(\"null\"),\n // Print arrays and objects as indented JSON\n _ => println!(\"{}\", serde_json::to_string_pretty(&v).map_err(|e| err(&e.to_string()))?),\n }\n Ok(())\n}\n\nfn to_json(r: &ShmReader, schema: &Schema, buf: &mut String) -> Result<(), MorlocError> {\n match schema.serial_type {\n SerialType::Nil => buf.push_str(\"null\"),\n SerialType::Bool => buf.push_str(if r.read_u8(0) != 0 { \"true\" } else { \"false\" }),\n SerialType::Sint8 => buf.push_str(&(r.read_val::(0)).to_string()),\n SerialType::Sint16 => buf.push_str(&(r.read_val::(0)).to_string()),\n SerialType::Sint32 => buf.push_str(&(r.read_val::(0)).to_string()),\n SerialType::Sint64 => buf.push_str(&(r.read_val::(0)).to_string()),\n SerialType::Uint8 => buf.push_str(&r.read_u8(0).to_string()),\n SerialType::Uint16 => buf.push_str(&(r.read_val::(0)).to_string()),\n SerialType::Uint32 => buf.push_str(&(r.read_val::(0)).to_string()),\n SerialType::Uint64 => buf.push_str(&(r.read_val::(0)).to_string()),\n SerialType::Float32 => write_float(buf, r.read_val::(0) as f64, b\"%.7g\\0\"),\n SerialType::Float64 => write_float(buf, r.read_val::(0), b\"%.15g\\0\"),\n\n SerialType::String => {\n let arr = r.read_array(0);\n if arr.size == 0 || arr.data == RELNULL {\n buf.push_str(\"\\\"\\\"\");\n } else {\n // SAFETY: arr.data resolved to valid SHM string bytes\n let dr = unsafe { ShmReader::new(shm::rel2abs(arr.data)?) };\n json_escape(dr.read_str(0, arr.size), buf);\n }\n }\n SerialType::Array => {\n let arr = r.read_array(0);\n let es = &schema.parameters[0];\n buf.push('[');\n if arr.size > 0 && arr.data != RELNULL {\n let data = shm::rel2abs(arr.data)?;\n for i in 0..arr.size {\n if i > 0 { buf.push(','); }\n // SAFETY: data + i * es.width within array data block\n let er = unsafe { ShmReader::new(data.add(i * es.width)) };\n to_json(&er, es, buf)?;\n }\n }\n buf.push(']');\n }\n SerialType::Tuple => {\n buf.push('[');\n for (i, fs) in schema.parameters.iter().enumerate() {\n if i > 0 { buf.push(','); }\n to_json(&r.at(schema.offsets[i]), fs, buf)?;\n }\n buf.push(']');\n }\n SerialType::Map => {\n buf.push('{');\n for (i, fs) in schema.parameters.iter().enumerate() {\n if i > 0 { buf.push(','); }\n if i < schema.keys.len() { buf.push('\"'); buf.push_str(&schema.keys[i]); buf.push_str(\"\\\":\"); }\n to_json(&r.at(schema.offsets[i]), fs, buf)?;\n }\n buf.push('}');\n }\n SerialType::Optional => {\n if r.read_u8(0) == 0 {\n buf.push_str(\"null\");\n } else {\n let inner = &schema.parameters[0];\n to_json(&r.at(shm::align_up(1, inner.alignment().max(1))), inner, buf)?;\n }\n }\n SerialType::Tensor => {\n // SAFETY: reading Tensor struct from SHM\n let tensor = unsafe { &*(r.ptr as *const shm::Tensor) };\n if tensor.total_elements == 0 {\n buf.push_str(\"[]\");\n } else {\n let ndim = schema.offsets.first().copied().unwrap_or(1);\n let sp = shm::rel2abs(tensor.shape)?;\n // SAFETY: sp points to ndim i64 values in SHM\n let shape: Vec = (0..ndim).map(|i| unsafe { *((sp as *const i64).add(i)) } as usize).collect();\n let dp = shm::rel2abs(tensor.data)?;\n let es = &schema.parameters[0];\n tensor_to_json(buf, dp, &shape, tensor.total_elements, es)?;\n }\n }\n }\n Ok(())\n}\n\nfn tensor_to_json(\n buf: &mut String, data: *const u8, shape: &[usize], stride: usize, es: &Schema,\n) -> Result<(), MorlocError> {\n buf.push('[');\n if shape.len() == 1 {\n for i in 0..shape[0] {\n if i > 0 { buf.push(','); }\n // SAFETY: data + i * es.width within tensor data\n let r = unsafe { ShmReader::new(data.add(i * es.width)) };\n to_json(&r, es, buf)?;\n }\n } else {\n let inner = stride / shape[0];\n for i in 0..shape[0] {\n if i > 0 { buf.push(','); }\n tensor_to_json(buf, data.wrapping_add(i * inner * es.width), &shape[1..], inner, es)?;\n }\n }\n buf.push(']');\n Ok(())\n}\n\n// ── Helpers ────────────────────────────────────────────────────────────────\n\nfn json_escape(s: &str, buf: &mut String) {\n buf.push('\"');\n for ch in s.chars() {\n match ch {\n '\"' => buf.push_str(\"\\\\\\\"\"), '\\\\' => buf.push_str(\"\\\\\\\\\"), '/' => buf.push_str(\"\\\\/\"),\n '\\x08' => buf.push_str(\"\\\\b\"), '\\x0c' => buf.push_str(\"\\\\f\"),\n '\\n' => buf.push_str(\"\\\\n\"), '\\r' => buf.push_str(\"\\\\r\"), '\\t' => buf.push_str(\"\\\\t\"),\n c if c < '\\x20' => buf.push_str(&format!(\"\\\\u{:04x}\", c as u32)),\n c => buf.push(c),\n }\n }\n buf.push('\"');\n}\n\nfn err(msg: &str) -> MorlocError { MorlocError::Serialization(msg.into()) }\nfn as_i64(v: &serde_json::Value) -> Result { v.as_i64().ok_or_else(|| err(\"expected integer\")) }\nfn as_u64(v: &serde_json::Value) -> Result { v.as_u64().ok_or_else(|| err(\"expected unsigned integer\")) }\nfn as_f64(v: &serde_json::Value) -> Result { v.as_f64().ok_or_else(|| err(\"expected number\")) }\n\nfn write_float(buf: &mut String, f: f64, fmt: &[u8]) {\n if f.is_nan() || f.is_infinite() { buf.push_str(\"null\"); return; }\n let mut cbuf = [0u8; 64];\n // SAFETY: snprintf writes to stack-local buffer with explicit size limit\n let n = unsafe { libc::snprintf(cbuf.as_mut_ptr() as *mut libc::c_char, cbuf.len(), fmt.as_ptr() as *const libc::c_char, f) };\n if n > 0 && (n as usize) < cbuf.len() {\n buf.push_str(std::str::from_utf8(&cbuf[..n as usize]).unwrap_or(\"0\"));\n } else {\n buf.push_str(\"0\");\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use crate::schema::parse_schema;\n fn setup() { crate::init_test_shm(); }\n\n #[test] fn test_int() { setup(); let s = parse_schema(\"i4\").unwrap(); let p = read_json_with_schema(\"42\", &s).unwrap(); assert_eq!(voidstar_to_json_string(p, &s).unwrap(), \"42\"); }\n #[test] fn test_string() { setup(); let s = parse_schema(\"s\").unwrap(); let p = read_json_with_schema(\"\\\"hello\\\"\", &s).unwrap(); assert_eq!(voidstar_to_json_string(p, &s).unwrap(), \"\\\"hello\\\"\"); }\n #[test] fn test_bool() { setup(); let s = parse_schema(\"b\").unwrap(); let p = read_json_with_schema(\"true\", &s).unwrap(); assert_eq!(voidstar_to_json_string(p, &s).unwrap(), \"true\"); }\n #[test] fn test_array() { setup(); let s = parse_schema(\"ai4\").unwrap(); let p = read_json_with_schema(\"[1,2,3]\", &s).unwrap(); assert_eq!(voidstar_to_json_string(p, &s).unwrap(), \"[1,2,3]\"); }\n #[test] fn test_opt_some(){ setup(); let s = parse_schema(\"?i4\").unwrap(); let p = read_json_with_schema(\"5\", &s).unwrap(); assert_eq!(voidstar_to_json_string(p, &s).unwrap(), \"5\"); }\n #[test] fn test_opt_null(){ setup(); let s = parse_schema(\"?i4\").unwrap(); let p = read_json_with_schema(\"null\", &s).unwrap(); assert_eq!(voidstar_to_json_string(p, &s).unwrap(), \"null\"); }\n}\n" }, { "path": "data/rust/morloc-runtime/src/json_ffi.rs", "content": "//! C ABI wrappers for JSON functions.\n//! Replaces json.c's core functions with calls to Rust json.rs.\n//! Arrow output and json_buf API are also implemented here.\n\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\n\n// ── quoted ─────────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn quoted(input: *const c_char) -> *mut c_char {\n if input.is_null() {\n return ptr::null_mut();\n }\n let s = CStr::from_ptr(input);\n let bytes = s.to_bytes();\n let len = bytes.len();\n // Simple wrapping: \"input\" (matching C behavior — no escaping)\n let buf = libc::calloc(len + 3, 1) as *mut c_char;\n if buf.is_null() {\n return ptr::null_mut();\n }\n *buf = b'\"' as c_char;\n std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf.add(1) as *mut u8, len);\n *buf.add(len + 1) = b'\"' as c_char;\n buf\n}\n\n// ── read_json_with_schema ──────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_json_with_schema(\n dest: *mut u8,\n json_str: *mut c_char,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n if json_str.is_null() || schema.is_null() {\n set_errmsg(errmsg, &MorlocError::NullPointer);\n return ptr::null_mut();\n }\n\n let rs = CSchema::to_rust(schema);\n let json = CStr::from_ptr(json_str).to_string_lossy();\n\n let dest_opt = if dest.is_null() { None } else { Some(dest) };\n match crate::json::read_json_with_schema_dest(dest_opt, &json, &rs) {\n Ok(ptr) => ptr,\n Err(e) => {\n set_errmsg(errmsg, &e);\n ptr::null_mut()\n }\n }\n}\n\n// ── voidstar_to_json_string ────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn voidstar_to_json_string(\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n match crate::json::voidstar_to_json_string(data as *mut u8, &rs) {\n Ok(s) => {\n match CString::new(s) {\n Ok(cs) => cs.into_raw(),\n Err(_) => {\n set_errmsg(errmsg, &MorlocError::Other(\"CString conversion failed\".into()));\n ptr::null_mut()\n }\n }\n }\n Err(e) => {\n set_errmsg(errmsg, &e);\n ptr::null_mut()\n }\n }\n}\n\n// ── print_voidstar ─────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn print_voidstar(\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n match crate::json::print_voidstar(data as *mut u8, &rs) {\n Ok(_) => true,\n Err(e) => {\n set_errmsg(errmsg, &e);\n false\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn pretty_print_voidstar(\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n match crate::json::pretty_print_voidstar(data as *mut u8, &rs) {\n Ok(_) => true,\n Err(e) => {\n set_errmsg(errmsg, &e);\n false\n }\n }\n}\n\n// ── json_buf API (used by daemon.c for discovery JSON) ─────────────────────\n\n/// Dynamic JSON string builder.\npub struct JsonBuf {\n buf: String,\n needs_comma: Vec,\n}\n\n#[no_mangle]\npub extern \"C\" fn json_buf_new() -> *mut JsonBuf {\n Box::into_raw(Box::new(JsonBuf {\n buf: String::with_capacity(256),\n needs_comma: Vec::new(),\n }))\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_buf_free(jb: *mut JsonBuf) {\n if !jb.is_null() {\n let _ = Box::from_raw(jb);\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_buf_finish(jb: *mut JsonBuf) -> *mut c_char {\n if jb.is_null() {\n return ptr::null_mut();\n }\n let jb = Box::from_raw(jb);\n match CString::new(jb.buf) {\n Ok(cs) => cs.into_raw(),\n Err(_) => ptr::null_mut(),\n }\n}\n\nunsafe fn jb_maybe_comma(jb: &mut JsonBuf) {\n if let Some(needs) = jb.needs_comma.last_mut() {\n if *needs {\n jb.buf.push(',');\n }\n *needs = true;\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_obj_start(jb: *mut JsonBuf) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n jb.buf.push('{');\n jb.needs_comma.push(false);\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_obj_end(jb: *mut JsonBuf) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb.needs_comma.pop();\n jb.buf.push('}');\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_arr_start(jb: *mut JsonBuf) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n jb.buf.push('[');\n jb.needs_comma.push(false);\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_arr_end(jb: *mut JsonBuf) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb.needs_comma.pop();\n jb.buf.push(']');\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_key(jb: *mut JsonBuf, key: *const c_char) {\n if jb.is_null() || key.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n let s = CStr::from_ptr(key).to_string_lossy();\n jb.buf.push('\"');\n jb.buf.push_str(&s);\n jb.buf.push_str(\"\\\":\");\n // Don't set needs_comma — the value will follow immediately\n if let Some(needs) = jb.needs_comma.last_mut() {\n *needs = false;\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_string(jb: *mut JsonBuf, val: *const c_char) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n if val.is_null() {\n jb.buf.push_str(\"null\");\n } else {\n let s = CStr::from_ptr(val).to_string_lossy();\n // JSON-escape the string\n jb.buf.push('\"');\n for ch in s.chars() {\n match ch {\n '\"' => jb.buf.push_str(\"\\\\\\\"\"),\n '\\\\' => jb.buf.push_str(\"\\\\\\\\\"),\n '\\n' => jb.buf.push_str(\"\\\\n\"),\n '\\r' => jb.buf.push_str(\"\\\\r\"),\n '\\t' => jb.buf.push_str(\"\\\\t\"),\n c if c < '\\x20' => {\n jb.buf.push_str(&format!(\"\\\\u{:04x}\", c as u32));\n }\n c => jb.buf.push(c),\n }\n }\n jb.buf.push('\"');\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_int(jb: *mut JsonBuf, val: i64) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n jb.buf.push_str(&val.to_string());\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_uint(jb: *mut JsonBuf, val: u64) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n jb.buf.push_str(&val.to_string());\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_bool(jb: *mut JsonBuf, val: bool) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n jb.buf.push_str(if val { \"true\" } else { \"false\" });\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_null(jb: *mut JsonBuf) {\n if jb.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n jb.buf.push_str(\"null\");\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn json_write_raw(jb: *mut JsonBuf, raw: *const c_char) {\n if jb.is_null() || raw.is_null() { return; }\n let jb = &mut *jb;\n jb_maybe_comma(jb);\n let s = CStr::from_ptr(raw).to_string_lossy();\n jb.buf.push_str(&s);\n}\n\n// ── Arrow JSON output ──────────────────────────────────────────────────────\n// Arrow output is complex and depends on the Arrow C Data Interface.\n// These are implemented in C (arrow_json.c) and linked via the hybrid build.\n// The functions below are stubs that will be overridden by the C implementations\n// when we create a separate arrow_json.c file.\n// For now, remove the Rust stubs and let C json.c's implementations be used\n// from a separate compilation unit.\n\n// print_arrow_as_json and print_arrow_as_table are provided by the C\n// arrow_json code (extracted from json.c, kept in build as arrow_json.c)\n\n#[allow(dead_code)]\nunsafe fn _print_arrow_as_json_stub(\n data: *const c_void,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n\n // Use the arrow.c implementation which handles the Arrow C Data Interface\n extern \"C\" {\n fn arrow_column_desc(\n header: *const c_void,\n col_idx: usize,\n ) -> *const c_void;\n fn arrow_column_data(\n header: *const c_void,\n col_idx: usize,\n ) -> *const c_void;\n fn arrow_column_name(\n header: *const c_void,\n col_idx: usize,\n ) -> *const c_char;\n }\n\n // Read arrow_shm_header fields\n // arrow_shm_header_t: { magic: u32, n_columns: u32, n_rows: u64, ... }\n let header = data as *const u8;\n let n_columns = *(header.add(4) as *const u32) as usize;\n let n_rows = *(header.add(8) as *const u64) as usize;\n\n // Build JSON array of objects\n print!(\"[\");\n for row in 0..n_rows {\n if row > 0 { print!(\",\"); }\n print!(\"{{\");\n for col in 0..n_columns {\n if col > 0 { print!(\",\"); }\n let name = arrow_column_name(data, col);\n let name_str = if name.is_null() { \"?\" } else { CStr::from_ptr(name).to_str().unwrap_or(\"?\") };\n print!(\"\\\"{}\\\":\", name_str);\n\n let desc = arrow_column_desc(data, col);\n if desc.is_null() {\n print!(\"null\");\n continue;\n }\n // desc is arrow_column_desc_t: { type: u8, length: u32, null_count: u32, name_offset, data_offset }\n let col_type = *(desc as *const u8);\n let col_data = arrow_column_data(data, col);\n\n // Print value based on type\n // Types: 0=nil, 1=bool, 2=i8, ..., 11=f64, 13=string\n match col_type {\n 4 => { // i32\n let vals = col_data as *const i32;\n print!(\"{}\", *vals.add(row));\n }\n 5 => { // i64\n let vals = col_data as *const i64;\n print!(\"{}\", *vals.add(row));\n }\n 11 => { // f64\n let vals = col_data as *const f64;\n let mut cbuf = [0u8; 64];\n let fmt = b\"%.15g\\0\";\n let n = libc::snprintf(cbuf.as_mut_ptr() as *mut c_char, 64, fmt.as_ptr() as *const c_char, *vals.add(row));\n if n > 0 {\n let s = std::str::from_utf8(&cbuf[..n as usize]).unwrap_or(\"0\");\n print!(\"{}\", s);\n }\n }\n 13 => { // string\n // Arrow strings: offsets array + data buffer\n // For simplicity, use arrow_column_data which gives the data pointer\n // This is a simplified implementation — full Arrow string handling\n // requires offset arrays\n print!(\"\\\"\\\"\");\n }\n _ => {\n print!(\"null\");\n }\n }\n }\n print!(\"}}\");\n }\n println!(\"]\");\n\n true\n}\n\n#[allow(dead_code)]\nunsafe fn _print_arrow_as_table_stub(\n _data: *const c_void,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n // Stub — Arrow table output is rarely used\n // The full implementation would print TSV-formatted columns\n set_errmsg(errmsg, &MorlocError::Other(\"Arrow table output not yet implemented in Rust\".into()));\n false\n}\n" }, { "path": "data/rust/morloc-runtime/src/lib.rs", "content": "pub mod error;\npub mod schema;\npub mod packet;\npub mod shm;\npub mod hash;\npub mod ipc;\npub mod json;\npub mod mpack;\n// FFI and utility modules export #[no_mangle] extern \"C\" symbols.\n// When the \"no-ffi-exports\" feature is active (nexus build), these modules\n// are not compiled, preventing symbol conflicts with libmorloc.so.\n// CSchema type is always available (used by nexus for Rust<->C conversion)\npub mod cschema;\npub mod ffi;\npub mod utility;\npub mod cache;\npub mod intrinsics;\npub mod voidstar;\npub mod json_ffi;\npub mod packet_ffi;\npub mod ipc_ffi;\npub mod http_ffi;\npub mod slurm_ffi;\npub mod manifest_ffi;\npub mod eval_ffi;\npub mod arrow_ffi;\npub mod pool_ffi;\npub mod daemon_ffi;\npub mod router_ffi;\npub mod cli;\n\n/// Shared test SHM initialization. Call from all test modules.\n#[cfg(test)]\npub(crate) fn init_test_shm() {\n use std::sync::Once;\n static INIT: Once = Once::new();\n INIT.call_once(|| {\n let tmpdir = std::env::temp_dir();\n let test_dir = tmpdir.join(format!(\"morloc_test_{}\", std::process::id()));\n let _ = std::fs::create_dir_all(&test_dir);\n shm::shm_set_fallback_dir(test_dir.to_str().unwrap());\n let basename = format!(\"morloc_test_{}\", std::process::id());\n shm::shinit(&basename, 0, 0x100000).unwrap(); // 1MB\n });\n}\n\n// Re-export core types at crate root\npub use error::MorlocError;\npub use schema::{Schema, SerialType};\npub use packet::{PacketHeader, PACKET_MAGIC};\npub use shm::{RelPtr, VolPtr, AbsPtr, Array, Tensor};\n" }, { "path": "data/rust/morloc-runtime/src/manifest_ffi.rs", "content": "//! C ABI wrappers for manifest parsing and discovery JSON.\n//!\n//! This file mirrors the manifest data model into raw `#[repr(C)]`\n//! structs that the daemon and slurm subsystems read via FFI from C\n//! code. It is **not** the canonical Rust deserializer of the manifest\n//! schema -- that lives in\n//! `data/rust/morloc-nexus/src/manifest.rs`, which has full doc\n//! comments describing the v2 manifest schema, every field's purpose,\n//! and which slots are reserved for future expansion.\n//!\n//! The split exists for two reasons:\n//!\n//! 1. **C ABI stability.** The C structs here have the original v1\n//! field layout (flat `arg_schemas`, `return_schema`, `return_type`,\n//! `return_desc`, `build_dir`, `version`) so that downstream C\n//! callers (the daemon, the slurm bridge, any third-party FFI\n//! consumers) don't break when the JSON schema evolves. The\n//! `parse_manifest` function below reads the new v2 JSON shape and\n//! populates these legacy C struct fields, acting as a translation\n//! layer.\n//!\n//! 2. **Build-time decoupling.** The morloc-runtime crate needs to\n//! consume manifests without depending on the morloc-nexus crate.\n//! Sharing a Rust-level data model would create a circular\n//! dependency between the two crates.\n//!\n//! When the v2 schema gains new fields (constraints, metadata, etc.),\n//! the canonical Rust model in `morloc-nexus/src/manifest.rs` is\n//! updated first. This file gets new C struct fields only when a C-side\n//! consumer needs them; otherwise the new JSON keys are silently\n//! ignored here, which is the correct forward-compatible behavior.\n\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\n\n// -- C-compatible types matching eval.h ---------------------------------------\n\n#[repr(C)]\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum MorlocExpressionType {\n Dat = 0,\n App = 1,\n Lam = 2,\n Bnd = 3,\n Pat = 4,\n Fmt = 5,\n Show = 6,\n Read = 7,\n Hash = 8,\n Save = 9,\n Load = 10,\n}\n\n#[repr(C)]\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum MorlocAppExpressionType {\n Pattern = 0,\n Lambda = 1,\n Format = 2,\n}\n\n#[repr(C)]\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum MorlocPatternType {\n ByKey = 0,\n ByIndex = 1,\n End = 2,\n}\n\n#[repr(C)]\npub union PatternFields {\n pub indices: *mut usize,\n pub keys: *mut *mut c_char,\n}\n\n#[repr(C)]\npub struct MorlocPattern {\n pub ptype: MorlocPatternType,\n pub size: usize,\n pub fields: PatternFields,\n pub selectors: *mut *mut MorlocPattern,\n}\n\n#[repr(C)]\npub union Primitive {\n pub s: *mut c_char,\n pub z: u8,\n pub b: bool,\n pub i1: i8,\n pub i2: i16,\n pub i4: i32,\n pub i8_: i64,\n pub u1: u8,\n pub u2: u16,\n pub u4: u32,\n pub u8_: u64,\n pub f4: f32,\n pub f8: f64,\n}\n\n#[repr(C)]\npub struct MorlocDataArray {\n pub schema: *mut CSchema,\n pub size: usize,\n pub values: *mut *mut MorlocExpression,\n}\n\n#[repr(C)]\n// Primitive contains a pointer (s: *mut c_char), so DataUnion uses ManuallyDrop\n#[repr(C)]\npub union DataUnion {\n pub lit_val: std::mem::ManuallyDrop,\n pub tuple_val: *mut *mut MorlocExpression,\n pub array_val: *mut MorlocDataArray,\n pub voidstar: *mut c_void,\n}\n\n#[repr(C)]\npub struct MorlocData {\n pub is_voidstar: bool,\n pub data: DataUnion,\n}\n\n#[repr(C)]\npub union AppFunction {\n pub pattern: *mut MorlocPattern,\n pub lambda: *mut MorlocLamExpression,\n pub fmt: *mut *mut c_char,\n}\n\n#[repr(C)]\npub struct MorlocAppExpression {\n pub atype: MorlocAppExpressionType,\n pub function: AppFunction,\n pub args: *mut *mut MorlocExpression,\n pub nargs: usize,\n}\n\n#[repr(C)]\npub struct MorlocLamExpression {\n pub nargs: usize,\n pub args: *mut *mut c_char,\n pub body: *mut MorlocExpression,\n}\n\n#[repr(C)]\npub struct MorlocSaveExpression {\n pub format: *mut c_char,\n pub value: *mut MorlocExpression,\n pub path: *mut MorlocExpression,\n}\n\n#[repr(C)]\npub union ExprUnion {\n pub app_expr: *mut MorlocAppExpression,\n pub lam_expr: *mut MorlocLamExpression,\n pub bnd_expr: *mut c_char,\n pub interpolation: *mut *mut c_char,\n pub pattern_expr: *mut MorlocPattern,\n pub data_expr: *mut MorlocData,\n pub unary_expr: *mut MorlocExpression,\n pub save_expr: *mut MorlocSaveExpression,\n}\n\n#[repr(C)]\npub struct MorlocExpression {\n pub etype: MorlocExpressionType,\n pub schema: *mut CSchema,\n pub expr: ExprUnion,\n}\n\n// -- C-ABI mirror of morloc-manifest v2 ---------------------------------------\n//\n// These #[repr(C)] structs are the in-memory layout that daemon_ffi /\n// router_ffi / slurm_ffi consume via raw pointers. They mirror the\n// shape of morloc_manifest's Rust types one-for-one (modulo C-string\n// encoding) -- when the Rust schema gains a new field, it's added here\n// too as a parallel C field. There is no longer any \"translation\" or\n// reshape layer; parse_manifest below is a near-1:1 walker.\n//\n// Conventions:\n// - C-string fields are owned by the manifest and freed by free_manifest.\n// - Array fields use a (pointer, count) pair (e.g. desc + n_desc).\n// - The \"constraints\" and \"metadata\" extension slots described in\n// morloc-manifest's docs are mirrored here so daemon-side constraint\n// enforcement can later read them without another C ABI break.\n// - metadata is serialized as JSON text (`metadata_json`) because the\n// C side has no natural map type and the slot is reserved for now.\n\n#[repr(C)]\npub struct ManifestBuild {\n pub path: *mut c_char,\n pub time: i64,\n pub morloc_version: *mut c_char,\n}\n\n#[repr(C)]\npub struct ManifestConstraint {\n /// Constraint discriminator (e.g. \"kind\", \"min\", \"regex\").\n pub ctype: *mut c_char,\n /// JSON-encoded payload for the constraint, or NULL when the\n /// constraint type carries no value (e.g. \"non_empty\").\n pub value_json: *mut c_char,\n}\n\n#[repr(C)]\npub struct ManifestPool {\n pub lang: *mut c_char,\n pub exec: *mut *mut c_char, // NULL-terminated\n pub socket: *mut c_char,\n /// JSON-encoded pool-level metadata. Reserved.\n pub metadata_json: *mut c_char,\n}\n\n#[repr(C)]\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum ManifestArgKind {\n Pos = 0,\n Opt = 1,\n Flag = 2,\n Grp = 3,\n}\n\n#[repr(C)]\npub struct ManifestGrpEntry {\n pub key: *mut c_char,\n pub arg: *mut ManifestArg,\n}\n\n#[repr(C)]\npub struct ManifestArg {\n pub kind: ManifestArgKind,\n /// Per-arg morloc serialization schema. NULL for flags. Group\n /// entries also have NULL here (the group's top-level schema\n /// covers them).\n pub schema: *mut c_char,\n /// User-facing type name. NULL for flags.\n pub type_desc: *mut c_char,\n pub metavar: *mut c_char,\n pub quoted: bool,\n pub short_opt: c_char,\n pub long_opt: *mut c_char,\n pub long_rev: *mut c_char,\n pub default_val: *mut c_char,\n /// NULL-terminated array of description lines.\n pub desc: *mut *mut c_char,\n pub n_desc: usize,\n /// Array of ManifestConstraint owned by this arg.\n pub constraints: *mut ManifestConstraint,\n pub n_constraints: usize,\n /// Group sub-fields (only meaningful when kind == Grp).\n pub grp_short: c_char,\n pub grp_long: *mut c_char,\n pub entries: *mut ManifestGrpEntry,\n pub n_entries: usize,\n /// JSON-encoded per-arg metadata. Reserved.\n pub metadata_json: *mut c_char,\n}\n\n#[repr(C)]\npub struct ManifestReturn {\n pub schema: *mut c_char,\n pub type_desc: *mut c_char,\n pub desc: *mut *mut c_char,\n pub n_desc: usize,\n pub constraints: *mut ManifestConstraint,\n pub n_constraints: usize,\n pub metadata_json: *mut c_char,\n}\n\n#[repr(C)]\npub struct ManifestCmdGroup {\n pub name: *mut c_char,\n pub desc: *mut *mut c_char,\n pub n_desc: usize,\n pub metadata_json: *mut c_char,\n}\n\n#[repr(C)]\npub struct ManifestCommand {\n pub name: *mut c_char,\n pub is_pure: bool,\n pub mid: u32,\n pub pool_index: usize,\n pub needed_pools: *mut usize,\n pub n_needed_pools: usize,\n pub desc: *mut *mut c_char,\n pub n_desc: usize,\n pub args: *mut ManifestArg,\n pub n_args: usize,\n /// Return-value descriptor as a sub-struct (replaces v1's flat\n /// return_schema/return_type/return_desc fields).\n pub ret: ManifestReturn,\n pub constraints: *mut ManifestConstraint,\n pub n_constraints: usize,\n pub expr: *mut MorlocExpression,\n pub group: *mut c_char,\n pub metadata_json: *mut c_char,\n}\n\n#[repr(C)]\npub struct ManifestService {\n pub stype: *mut c_char,\n pub host: *mut c_char,\n pub port: i32,\n pub socket: *mut c_char,\n pub metadata_json: *mut c_char,\n}\n\n#[repr(C)]\npub struct Manifest {\n pub name: *mut c_char,\n pub build: ManifestBuild,\n pub pools: *mut ManifestPool,\n pub n_pools: usize,\n pub commands: *mut ManifestCommand,\n pub n_commands: usize,\n pub groups: *mut ManifestCmdGroup,\n pub n_groups: usize,\n pub service: *mut ManifestService,\n pub metadata_json: *mut c_char,\n}\n\nimpl ManifestCommand {\n /// Build a transient NULL-terminated array of schema strings for\n /// the command's args, in declaration order. The caller owns the\n /// outer array allocation but NOT the inner C strings (they\n /// remain owned by the ManifestArg objects). Use\n /// `libc::free(arr as *mut c_void)` to release the outer array\n /// when done.\n ///\n /// The array has one entry per arg INCLUDING flags. Flags have a\n /// per-arg `schema` field of NULL on the v2 ManifestArg, but the\n /// legacy callers (e.g. make_call_packet_from_cli) expect a slot\n /// per arg position to keep index alignment with the parallel\n /// args array; we substitute \"b\" (the boolean schema) for flags\n /// so dispatch reads the flag value as a Bool, matching v1\n /// behavior.\n pub unsafe fn build_arg_schemas_array(&self) -> *mut *mut c_char {\n let n = self.n_args;\n let arr = libc::calloc(n + 1, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n for i in 0..n {\n let arg = &*self.args.add(i);\n *arr.add(i) = if arg.kind == ManifestArgKind::Flag || arg.schema.is_null() {\n // Flag schema fallback: use the static \"b\" string. This\n // pointer is NOT freed by the caller (it's a global\n // string literal), but since the caller only frees the\n // OUTER array, this is fine. We use a per-call CString\n // leak so each call has a distinct C-string buffer the\n // caller doesn't try to free with the per-arg owners.\n // Cleaner: we just point to the existing per-arg\n // schema slot if non-null, else fall through. Most\n // flag args won't be hit by legacy callers anyway.\n static FLAG_SCHEMA: &[u8] = b\"b\\0\";\n FLAG_SCHEMA.as_ptr() as *mut c_char\n } else {\n arg.schema\n };\n }\n *arr.add(n) = ptr::null_mut();\n arr\n }\n}\n\n// -- Helpers ------------------------------------------------------------------\n\nunsafe fn c_strdup(s: &str) -> *mut c_char {\n match CString::new(s) {\n Ok(cs) => libc::strdup(cs.as_ptr()),\n Err(_) => ptr::null_mut(),\n }\n}\n\nunsafe fn nullable_strdup(s: Option<&str>) -> *mut c_char {\n match s {\n Some(s) => c_strdup(s),\n None => ptr::null_mut(),\n }\n}\n\n// -- Expression builder (using serde_json::Value) -----------------------------\n\nunsafe fn build_pattern(jp: &serde_json::Value) -> Result<*mut MorlocPattern, MorlocError> {\n let ptype = jp.get(\"type\").and_then(|v| v.as_str()).ok_or_else(|| MorlocError::Other(\"Pattern missing 'type' field\".into()))?;\n\n if ptype == \"end\" {\n // make_morloc_pattern_end - call C function\n extern \"C\" {\n fn make_morloc_pattern_end() -> *mut MorlocPattern;\n }\n return Ok(make_morloc_pattern_end());\n }\n\n let sels = jp.get(\"selectors\").and_then(|v| v.as_array());\n let n = sels.map(|a| a.len()).unwrap_or(0);\n\n let pat = libc::calloc(1, std::mem::size_of::()) as *mut MorlocPattern;\n (*pat).size = n;\n (*pat).selectors = libc::calloc(n, std::mem::size_of::<*mut MorlocPattern>()) as *mut *mut MorlocPattern;\n\n if ptype == \"idx\" {\n (*pat).ptype = MorlocPatternType::ByIndex;\n (*pat).fields.indices = libc::calloc(n, std::mem::size_of::()) as *mut usize;\n if let Some(sels) = sels {\n for (i, sel) in sels.iter().enumerate() {\n *(*pat).fields.indices.add(i) = sel.get(\"index\").and_then(|v| v.as_f64()).unwrap_or(0.0) as usize;\n *(*pat).selectors.add(i) = build_pattern(sel.get(\"sub\").unwrap_or(&serde_json::Value::Null))?;\n }\n }\n } else if ptype == \"key\" {\n (*pat).ptype = MorlocPatternType::ByKey;\n (*pat).fields.keys = libc::calloc(n, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n if let Some(sels) = sels {\n for (i, sel) in sels.iter().enumerate() {\n let key = sel.get(\"key\").and_then(|v| v.as_str()).unwrap_or(\"\");\n *(*pat).fields.keys.add(i) = c_strdup(key);\n *(*pat).selectors.add(i) = build_pattern(sel.get(\"sub\").unwrap_or(&serde_json::Value::Null))?;\n }\n }\n } else {\n return Err(MorlocError::Other(format!(\"Unknown pattern type: {}\", ptype)));\n }\n\n Ok(pat)\n}\n\nunsafe fn build_expr(je: &serde_json::Value) -> Result<*mut MorlocExpression, MorlocError> {\n let tag = je.get(\"tag\").and_then(|v| v.as_str()).ok_or_else(|| MorlocError::Other(\"Expression missing 'tag' field\".into()))?;\n\n extern \"C\" {\n fn parse_schema(s: *const c_char, errmsg: *mut *mut c_char) -> *mut CSchema;\n fn make_morloc_literal(schema: *const c_char, prim: Primitive, errmsg: *mut *mut c_char) -> *mut MorlocExpression;\n fn make_morloc_bound_var(schema: *const c_char, var: *mut c_char, errmsg: *mut *mut c_char) -> *mut MorlocExpression;\n fn make_morloc_pattern(schema: *const c_char, pat: *mut MorlocPattern, errmsg: *mut *mut c_char) -> *mut MorlocExpression;\n }\n\n let mut err: *mut c_char = ptr::null_mut();\n\n match tag {\n \"lit\" => {\n let schema = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let lt = je.get(\"lit_type\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let val = je.get(\"value\").and_then(|v| v.as_str()).unwrap_or(\"0\");\n let mut prim: Primitive = std::mem::zeroed();\n\n match lt {\n \"f4\" => prim.f4 = val.parse::().unwrap_or(0.0),\n \"f8\" => prim.f8 = val.parse::().unwrap_or(0.0),\n \"i1\" => prim.i1 = val.parse::().unwrap_or(0),\n \"i2\" => prim.i2 = val.parse::().unwrap_or(0),\n \"i4\" => prim.i4 = val.parse::().unwrap_or(0),\n \"i8\" => prim.i8_ = val.parse::().unwrap_or(0),\n \"u1\" => prim.u1 = val.parse::().unwrap_or(0),\n \"u2\" => prim.u2 = val.parse::().unwrap_or(0),\n \"u4\" => prim.u4 = val.parse::().unwrap_or(0),\n \"u8\" => prim.u8_ = val.parse::().unwrap_or(0),\n \"b\" => prim.b = val != \"0\",\n \"z\" => prim.z = 0,\n _ => return Err(MorlocError::Other(format!(\"Unknown lit_type: {}\", lt))),\n }\n\n let c_schema = CString::new(schema).unwrap_or_default();\n let result = make_morloc_literal(c_schema.as_ptr(), prim, &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n Ok(result)\n }\n\n \"str\" => {\n let schema = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let val = je.get(\"value\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let mut prim: Primitive = std::mem::zeroed();\n prim.s = c_strdup(val);\n let c_schema = CString::new(schema).unwrap_or_default();\n let result = make_morloc_literal(c_schema.as_ptr(), prim, &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n Ok(result)\n }\n\n \"container\" => {\n let schema_str = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let elems = je.get(\"elements\").and_then(|v| v.as_array());\n let n = elems.map(|a| a.len()).unwrap_or(0);\n\n let c_schema_str = CString::new(schema_str).unwrap_or_default();\n let schema = parse_schema(c_schema_str.as_ptr(), &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n\n let values = libc::calloc(n, std::mem::size_of::<*mut MorlocExpression>()) as *mut *mut MorlocExpression;\n if let Some(elems) = elems {\n for (i, elem) in elems.iter().enumerate() {\n *values.add(i) = build_expr(elem)?;\n }\n }\n\n let data = libc::calloc(1, std::mem::size_of::()) as *mut MorlocData;\n (*data).is_voidstar = false;\n\n let schema_type = (*schema).serial_type;\n // MORLOC_ARRAY = 14, MORLOC_TUPLE = 15, MORLOC_MAP = 16\n if schema_type == 14 { // Array\n let arr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocDataArray;\n (*arr).schema = if (*schema).size > 0 && !(*schema).parameters.is_null() { *(*schema).parameters } else { ptr::null_mut() };\n (*arr).size = n;\n (*arr).values = values;\n (*data).data.array_val = arr;\n } else if schema_type == 15 || schema_type == 16 { // Tuple or Map\n (*data).data.tuple_val = values;\n } else {\n libc::free(values as *mut c_void);\n libc::free(data as *mut c_void);\n CSchema::free(schema);\n return Err(MorlocError::Other(\"Container schema is not a container type\".into()));\n }\n\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*expr).etype = MorlocExpressionType::Dat;\n (*expr).schema = schema;\n (*expr).expr.data_expr = data;\n Ok(expr)\n }\n\n \"app\" => {\n let schema_str = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let jargs = je.get(\"args\").and_then(|v| v.as_array());\n let n = jargs.map(|a| a.len()).unwrap_or(0);\n\n let c_schema_str = CString::new(schema_str).unwrap_or_default();\n let schema = parse_schema(c_schema_str.as_ptr(), &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n\n let func = build_expr(je.get(\"func\").unwrap_or(&serde_json::Value::Null))?;\n let args = libc::calloc(n, std::mem::size_of::<*mut MorlocExpression>()) as *mut *mut MorlocExpression;\n if let Some(jargs) = jargs {\n for (i, a) in jargs.iter().enumerate() {\n *args.add(i) = build_expr(a)?;\n }\n }\n\n let app = libc::calloc(1, std::mem::size_of::()) as *mut MorlocAppExpression;\n match (*func).etype {\n MorlocExpressionType::Pat => {\n (*app).atype = MorlocAppExpressionType::Pattern;\n (*app).function.pattern = (*func).expr.pattern_expr;\n }\n MorlocExpressionType::Lam => {\n (*app).atype = MorlocAppExpressionType::Lambda;\n (*app).function.lambda = (*func).expr.lam_expr;\n }\n MorlocExpressionType::Fmt => {\n (*app).atype = MorlocAppExpressionType::Format;\n (*app).function.fmt = (*func).expr.interpolation;\n }\n _ => {\n return Err(MorlocError::Other(format!(\"Invalid function in app expression (type={:?})\", (*func).etype)));\n }\n }\n (*app).args = args;\n (*app).nargs = n;\n\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*expr).etype = MorlocExpressionType::App;\n (*expr).schema = schema;\n (*expr).expr.app_expr = app;\n Ok(expr)\n }\n\n \"lambda\" => {\n let jvars = je.get(\"vars\").and_then(|v| v.as_array());\n let n = jvars.map(|a| a.len()).unwrap_or(0);\n\n let body = build_expr(je.get(\"body\").unwrap_or(&serde_json::Value::Null))?;\n let vars = libc::calloc(n, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n if let Some(jvars) = jvars {\n for (i, v) in jvars.iter().enumerate() {\n *vars.add(i) = c_strdup(v.as_str().unwrap_or(\"\"));\n }\n }\n\n let lam = libc::calloc(1, std::mem::size_of::()) as *mut MorlocLamExpression;\n (*lam).nargs = n;\n (*lam).args = vars;\n (*lam).body = body;\n\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*expr).etype = MorlocExpressionType::Lam;\n (*expr).schema = ptr::null_mut();\n (*expr).expr.lam_expr = lam;\n Ok(expr)\n }\n\n \"bound\" => {\n let schema = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let var = je.get(\"var\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let c_schema = CString::new(schema).unwrap_or_default();\n let c_var = c_strdup(var);\n let result = make_morloc_bound_var(c_schema.as_ptr(), c_var, &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n Ok(result)\n }\n\n \"show\" | \"read\" | \"hash\" | \"load\" => {\n let schema_str = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let c_schema_str = CString::new(schema_str).unwrap_or_default();\n let schema = parse_schema(c_schema_str.as_ptr(), &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n let child = build_expr(je.get(\"child\").unwrap_or(&serde_json::Value::Null))?;\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*expr).etype = match tag {\n \"show\" => MorlocExpressionType::Show,\n \"read\" => MorlocExpressionType::Read,\n \"hash\" => MorlocExpressionType::Hash,\n \"load\" => MorlocExpressionType::Load,\n _ => unreachable!(),\n };\n (*expr).schema = schema;\n (*expr).expr.unary_expr = child;\n Ok(expr)\n }\n\n \"save\" => {\n let schema_str = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let c_schema_str = CString::new(schema_str).unwrap_or_default();\n let schema = parse_schema(c_schema_str.as_ptr(), &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n let fmt_str = je.get(\"format\").and_then(|v| v.as_str()).unwrap_or(\"voidstar\");\n let c_fmt = CString::new(fmt_str).unwrap_or_default();\n let value = build_expr(je.get(\"value\").unwrap_or(&serde_json::Value::Null))?;\n let path = build_expr(je.get(\"path\").unwrap_or(&serde_json::Value::Null))?;\n let save = libc::calloc(1, std::mem::size_of::()) as *mut MorlocSaveExpression;\n (*save).format = c_fmt.into_raw();\n (*save).value = value;\n (*save).path = path;\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*expr).etype = MorlocExpressionType::Save;\n (*expr).schema = schema;\n (*expr).expr.save_expr = save;\n Ok(expr)\n }\n\n \"interpolation\" => {\n let schema_str = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let jstrs = je.get(\"strings\").and_then(|v| v.as_array());\n let n = jstrs.map(|a| a.len()).unwrap_or(0);\n\n let c_schema_str = CString::new(schema_str).unwrap_or_default();\n let schema = parse_schema(c_schema_str.as_ptr(), &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n\n let strings = libc::calloc(n + 1, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n if let Some(jstrs) = jstrs {\n for (i, s) in jstrs.iter().enumerate() {\n *strings.add(i) = c_strdup(s.as_str().unwrap_or(\"\"));\n }\n }\n\n let expr = libc::calloc(1, std::mem::size_of::()) as *mut MorlocExpression;\n (*expr).etype = MorlocExpressionType::Fmt;\n (*expr).schema = schema;\n (*expr).expr.interpolation = strings;\n Ok(expr)\n }\n\n \"pattern\" => {\n let schema_str = je.get(\"schema\").and_then(|v| v.as_str()).unwrap_or(\"\");\n let pat = build_pattern(je.get(\"pattern\").unwrap_or(&serde_json::Value::Null))?;\n let c_schema = CString::new(schema_str).unwrap_or_default();\n let result = make_morloc_pattern(c_schema.as_ptr(), pat, &mut err);\n if !err.is_null() {\n let msg = CStr::from_ptr(err).to_string_lossy().into_owned();\n libc::free(err as *mut c_void);\n return Err(MorlocError::Other(msg));\n }\n Ok(result)\n }\n\n _ => Err(MorlocError::Other(format!(\"Unknown expression tag: {}\", tag))),\n }\n}\n\n// -- build_manifest_expr ------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn build_manifest_expr(\n json_str: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut MorlocExpression {\n clear_errmsg(errmsg);\n let s = CStr::from_ptr(json_str).to_string_lossy();\n let jv: serde_json::Value = match serde_json::from_str(&s) {\n Ok(v) => v,\n Err(e) => {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Failed to parse expression JSON: {}\", e)));\n return ptr::null_mut();\n }\n };\n match build_expr(&jv) {\n Ok(expr) => expr,\n Err(e) => {\n set_errmsg(errmsg, &e);\n ptr::null_mut()\n }\n }\n}\n\n// -- C-ABI population from morloc-manifest Rust types -------------------------\n//\n// parse_manifest is now a thin shell over morloc_manifest::parse_manifest\n// (which does all the JSON parsing, version checking, and serde\n// validation). The walker functions below convert the Rust-native\n// structs into owned C structs, mirroring the v2 schema field-for-field.\n\nunsafe fn populate_constraint(dst: *mut ManifestConstraint, src: &morloc_manifest::Constraint) {\n (*dst).ctype = c_strdup(&src.ctype);\n (*dst).value_json = match &src.value {\n Some(v) => c_strdup(&v.to_string()),\n None => ptr::null_mut(),\n };\n}\n\nunsafe fn populate_constraints(\n src: &[morloc_manifest::Constraint],\n) -> (*mut ManifestConstraint, usize) {\n if src.is_empty() {\n return (ptr::null_mut(), 0);\n }\n let arr =\n libc::calloc(src.len(), std::mem::size_of::()) as *mut ManifestConstraint;\n for (i, c) in src.iter().enumerate() {\n populate_constraint(arr.add(i), c);\n }\n (arr, src.len())\n}\n\nunsafe fn populate_metadata(src: &morloc_manifest::Metadata) -> *mut c_char {\n if src.is_empty() {\n c_strdup(\"{}\")\n } else {\n let s = serde_json::to_string(src).unwrap_or_else(|_| \"{}\".into());\n c_strdup(&s)\n }\n}\n\n/// Convert a Vec to a NULL-terminated array of C strings,\n/// and return (pointer, count). Caller owns the allocation.\nunsafe fn populate_str_vec(src: &[String]) -> (*mut *mut c_char, usize) {\n let n = src.len();\n let arr = libc::calloc(n + 1, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n for (i, s) in src.iter().enumerate() {\n *arr.add(i) = c_strdup(s);\n }\n *arr.add(n) = ptr::null_mut();\n (arr, n)\n}\n\nunsafe fn populate_arg(dst: *mut ManifestArg, src: &morloc_manifest::Arg) {\n use morloc_manifest::Arg;\n match src {\n Arg::Positional {\n schema,\n type_desc,\n metavar,\n quoted,\n desc,\n constraints,\n ..\n } => {\n (*dst).kind = ManifestArgKind::Pos;\n (*dst).schema = nullable_strdup(schema.as_deref());\n (*dst).type_desc = nullable_strdup(type_desc.as_deref());\n (*dst).metavar = nullable_strdup(metavar.as_deref());\n (*dst).quoted = *quoted;\n let (d, n) = populate_str_vec(desc);\n (*dst).desc = d;\n (*dst).n_desc = n;\n let (cs, nc) = populate_constraints(constraints);\n (*dst).constraints = cs;\n (*dst).n_constraints = nc;\n (*dst).metadata_json = c_strdup(\"{}\");\n }\n Arg::Optional {\n schema,\n type_desc,\n metavar,\n quoted,\n short_opt,\n long_opt,\n default_val,\n desc,\n constraints,\n ..\n } => {\n (*dst).kind = ManifestArgKind::Opt;\n (*dst).schema = nullable_strdup(schema.as_deref());\n (*dst).type_desc = nullable_strdup(type_desc.as_deref());\n (*dst).metavar = nullable_strdup(metavar.as_deref());\n (*dst).quoted = *quoted;\n (*dst).short_opt = short_opt\n .as_ref()\n .and_then(|s| s.as_bytes().first().copied())\n .map(|b| b as c_char)\n .unwrap_or(0);\n (*dst).long_opt = nullable_strdup(long_opt.as_deref());\n (*dst).default_val = nullable_strdup(default_val.as_deref());\n let (d, n) = populate_str_vec(desc);\n (*dst).desc = d;\n (*dst).n_desc = n;\n let (cs, nc) = populate_constraints(constraints);\n (*dst).constraints = cs;\n (*dst).n_constraints = nc;\n (*dst).metadata_json = c_strdup(\"{}\");\n }\n Arg::Flag {\n short_opt,\n long_opt,\n long_rev,\n default_val,\n desc,\n ..\n } => {\n (*dst).kind = ManifestArgKind::Flag;\n (*dst).short_opt = short_opt\n .as_ref()\n .and_then(|s| s.as_bytes().first().copied())\n .map(|b| b as c_char)\n .unwrap_or(0);\n (*dst).long_opt = nullable_strdup(long_opt.as_deref());\n (*dst).long_rev = nullable_strdup(long_rev.as_deref());\n (*dst).default_val = nullable_strdup(default_val.as_deref());\n let (d, n) = populate_str_vec(desc);\n (*dst).desc = d;\n (*dst).n_desc = n;\n (*dst).metadata_json = c_strdup(\"{}\");\n }\n Arg::Group {\n schema,\n type_desc,\n metavar,\n desc,\n group_opt,\n entries,\n constraints,\n ..\n } => {\n (*dst).kind = ManifestArgKind::Grp;\n (*dst).schema = nullable_strdup(schema.as_deref());\n (*dst).type_desc = nullable_strdup(type_desc.as_deref());\n (*dst).metavar = nullable_strdup(metavar.as_deref());\n let (d, n) = populate_str_vec(desc);\n (*dst).desc = d;\n (*dst).n_desc = n;\n if let Some(g) = group_opt {\n (*dst).grp_short = g\n .short_opt\n .as_ref()\n .and_then(|s| s.as_bytes().first().copied())\n .map(|b| b as c_char)\n .unwrap_or(0);\n (*dst).grp_long = nullable_strdup(g.long_opt.as_deref());\n }\n if !entries.is_empty() {\n (*dst).n_entries = entries.len();\n (*dst).entries = libc::calloc(\n entries.len(),\n std::mem::size_of::(),\n ) as *mut ManifestGrpEntry;\n for (i, ge) in entries.iter().enumerate() {\n let ge_dst = &mut *(*dst).entries.add(i);\n ge_dst.key = c_strdup(&ge.key);\n let sub_arg = libc::calloc(1, std::mem::size_of::())\n as *mut ManifestArg;\n populate_arg(sub_arg, &ge.arg);\n ge_dst.arg = sub_arg;\n }\n }\n let (cs, nc) = populate_constraints(constraints);\n (*dst).constraints = cs;\n (*dst).n_constraints = nc;\n (*dst).metadata_json = c_strdup(\"{}\");\n }\n }\n}\n\nunsafe fn populate_return(dst: *mut ManifestReturn, src: &morloc_manifest::Return) {\n (*dst).schema = c_strdup(&src.schema);\n (*dst).type_desc = c_strdup(&src.type_desc);\n let (d, n) = populate_str_vec(&src.desc);\n (*dst).desc = d;\n (*dst).n_desc = n;\n let (cs, nc) = populate_constraints(&src.constraints);\n (*dst).constraints = cs;\n (*dst).n_constraints = nc;\n (*dst).metadata_json = populate_metadata(&src.metadata);\n}\n\nunsafe fn populate_command(dst: *mut ManifestCommand, src: &morloc_manifest::Command) -> Result<(), MorlocError> {\n (*dst).name = c_strdup(&src.name);\n (*dst).is_pure = src.is_pure();\n (*dst).mid = src.mid;\n (*dst).pool_index = src.pool_index;\n if !src.needed_pools.is_empty() {\n (*dst).n_needed_pools = src.needed_pools.len();\n (*dst).needed_pools = libc::calloc(\n src.needed_pools.len(),\n std::mem::size_of::(),\n ) as *mut usize;\n for (i, p) in src.needed_pools.iter().enumerate() {\n *(*dst).needed_pools.add(i) = *p;\n }\n }\n let (d, n) = populate_str_vec(&src.desc);\n (*dst).desc = d;\n (*dst).n_desc = n;\n\n if !src.args.is_empty() {\n (*dst).n_args = src.args.len();\n (*dst).args = libc::calloc(\n src.args.len(),\n std::mem::size_of::(),\n ) as *mut ManifestArg;\n for (i, a) in src.args.iter().enumerate() {\n populate_arg((*dst).args.add(i), a);\n }\n }\n\n populate_return(&mut (*dst).ret, &src.ret);\n\n let (cs, nc) = populate_constraints(&src.constraints);\n (*dst).constraints = cs;\n (*dst).n_constraints = nc;\n\n (*dst).metadata_json = populate_metadata(&src.metadata);\n\n if src.is_pure() {\n if let Some(expr_val) = &src.expr {\n match build_expr(expr_val) {\n Ok(e) => (*dst).expr = e,\n Err(e) => return Err(e),\n }\n }\n }\n\n (*dst).group = match &src.group {\n Some(g) => c_strdup(g),\n None => ptr::null_mut(),\n };\n\n Ok(())\n}\n\nunsafe fn populate_pool(dst: *mut ManifestPool, src: &morloc_manifest::Pool) {\n (*dst).lang = c_strdup(&src.lang);\n let n = src.exec.len();\n (*dst).exec = libc::calloc(n + 1, std::mem::size_of::<*mut c_char>()) as *mut *mut c_char;\n for (i, e) in src.exec.iter().enumerate() {\n *(*dst).exec.add(i) = c_strdup(e);\n }\n *(*dst).exec.add(n) = ptr::null_mut();\n (*dst).socket = c_strdup(&src.socket);\n (*dst).metadata_json = populate_metadata(&src.metadata);\n}\n\nunsafe fn populate_cmd_group(dst: *mut ManifestCmdGroup, src: &morloc_manifest::CmdGroup) {\n (*dst).name = c_strdup(&src.name);\n let (d, n) = populate_str_vec(&src.desc);\n (*dst).desc = d;\n (*dst).n_desc = n;\n (*dst).metadata_json = populate_metadata(&src.metadata);\n}\n\nunsafe fn populate_service(dst: *mut ManifestService, src: &morloc_manifest::Service) {\n (*dst).stype = nullable_strdup(src.service_type.as_deref());\n (*dst).host = nullable_strdup(src.host.as_deref());\n (*dst).port = src.port.unwrap_or(0);\n (*dst).socket = nullable_strdup(src.socket.as_deref());\n (*dst).metadata_json = populate_metadata(&src.metadata);\n}\n\n// -- parse_manifest -----------------------------------------------------------\n//\n// Reads a manifest JSON string, parses it via the canonical\n// morloc-manifest crate (which performs the version staleness check),\n// and converts the resulting Rust struct into owned C structs for\n// daemon-side consumers. There is no separate JSON walker here -- the\n// shape and validation rules live in one place (the morloc-manifest\n// crate).\n\n#[no_mangle]\npub unsafe extern \"C\" fn parse_manifest(\n text: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut Manifest {\n clear_errmsg(errmsg);\n let s = CStr::from_ptr(text).to_string_lossy();\n\n let parsed = match morloc_manifest::parse_manifest(&s) {\n Ok(p) => p,\n Err(e) => {\n set_errmsg(errmsg, &MorlocError::Other(e));\n return ptr::null_mut();\n }\n };\n\n let m = libc::calloc(1, std::mem::size_of::()) as *mut Manifest;\n (*m).name = c_strdup(&parsed.name);\n\n // build sub-object\n (*m).build.path = c_strdup(&parsed.build.path);\n (*m).build.time = parsed.build.time;\n (*m).build.morloc_version = c_strdup(&parsed.build.morloc_version);\n\n // pools\n if !parsed.pools.is_empty() {\n (*m).n_pools = parsed.pools.len();\n (*m).pools = libc::calloc(\n parsed.pools.len(),\n std::mem::size_of::(),\n ) as *mut ManifestPool;\n for (i, p) in parsed.pools.iter().enumerate() {\n populate_pool((*m).pools.add(i), p);\n }\n }\n\n // commands\n if !parsed.commands.is_empty() {\n (*m).n_commands = parsed.commands.len();\n (*m).commands = libc::calloc(\n parsed.commands.len(),\n std::mem::size_of::(),\n ) as *mut ManifestCommand;\n for (i, c) in parsed.commands.iter().enumerate() {\n if let Err(e) = populate_command((*m).commands.add(i), c) {\n set_errmsg(errmsg, &e);\n return ptr::null_mut();\n }\n }\n }\n\n // groups\n if !parsed.groups.is_empty() {\n (*m).n_groups = parsed.groups.len();\n (*m).groups = libc::calloc(\n parsed.groups.len(),\n std::mem::size_of::(),\n ) as *mut ManifestCmdGroup;\n for (i, g) in parsed.groups.iter().enumerate() {\n populate_cmd_group((*m).groups.add(i), g);\n }\n }\n\n // service\n if let Some(svc) = parsed.service {\n (*m).service =\n libc::calloc(1, std::mem::size_of::()) as *mut ManifestService;\n populate_service((*m).service, &svc);\n }\n\n (*m).metadata_json = populate_metadata(&parsed.metadata);\n\n m\n}\n\n// -- read_manifest ------------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_manifest(\n path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut Manifest {\n clear_errmsg(errmsg);\n let path_str = CStr::from_ptr(path).to_string_lossy();\n match std::fs::read_to_string(path_str.as_ref()) {\n Ok(text) => {\n let c_text = CString::new(text).unwrap_or_default();\n parse_manifest(c_text.as_ptr(), errmsg)\n }\n Err(e) => {\n set_errmsg(errmsg, &MorlocError::Io(e));\n ptr::null_mut()\n }\n }\n}\n\n// -- free_manifest ------------------------------------------------------------\n//\n// Walks the v2 C structs and releases every owned C string + sub-allocation.\n// Helper functions mirror the populate_* helpers above for symmetry.\n\nunsafe fn free_str_array(arr: *mut *mut c_char) {\n if arr.is_null() {\n return;\n }\n let mut j = 0;\n while !(*arr.add(j)).is_null() {\n libc::free(*arr.add(j) as *mut c_void);\n j += 1;\n }\n libc::free(arr as *mut c_void);\n}\n\nunsafe fn free_constraints(ptr: *mut ManifestConstraint, n: usize) {\n if ptr.is_null() {\n return;\n }\n for i in 0..n {\n let c = &*ptr.add(i);\n if !c.ctype.is_null() {\n libc::free(c.ctype as *mut c_void);\n }\n if !c.value_json.is_null() {\n libc::free(c.value_json as *mut c_void);\n }\n }\n libc::free(ptr as *mut c_void);\n}\n\nunsafe fn free_arg(arg: &ManifestArg) {\n if !arg.schema.is_null() {\n libc::free(arg.schema as *mut c_void);\n }\n if !arg.type_desc.is_null() {\n libc::free(arg.type_desc as *mut c_void);\n }\n if !arg.metavar.is_null() {\n libc::free(arg.metavar as *mut c_void);\n }\n if !arg.long_opt.is_null() {\n libc::free(arg.long_opt as *mut c_void);\n }\n if !arg.long_rev.is_null() {\n libc::free(arg.long_rev as *mut c_void);\n }\n if !arg.default_val.is_null() {\n libc::free(arg.default_val as *mut c_void);\n }\n free_str_array(arg.desc);\n free_constraints(arg.constraints, arg.n_constraints);\n if !arg.grp_long.is_null() {\n libc::free(arg.grp_long as *mut c_void);\n }\n if !arg.entries.is_null() {\n for i in 0..arg.n_entries {\n let ge = &*arg.entries.add(i);\n if !ge.key.is_null() {\n libc::free(ge.key as *mut c_void);\n }\n if !ge.arg.is_null() {\n free_arg(&*ge.arg);\n libc::free(ge.arg as *mut c_void);\n }\n }\n libc::free(arg.entries as *mut c_void);\n }\n if !arg.metadata_json.is_null() {\n libc::free(arg.metadata_json as *mut c_void);\n }\n}\n\nunsafe fn free_return(ret: &ManifestReturn) {\n if !ret.schema.is_null() {\n libc::free(ret.schema as *mut c_void);\n }\n if !ret.type_desc.is_null() {\n libc::free(ret.type_desc as *mut c_void);\n }\n free_str_array(ret.desc);\n free_constraints(ret.constraints, ret.n_constraints);\n if !ret.metadata_json.is_null() {\n libc::free(ret.metadata_json as *mut c_void);\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn free_manifest(manifest: *mut Manifest) {\n if manifest.is_null() {\n return;\n }\n let m = &*manifest;\n if !m.name.is_null() {\n libc::free(m.name as *mut c_void);\n }\n // build sub-object\n if !m.build.path.is_null() {\n libc::free(m.build.path as *mut c_void);\n }\n if !m.build.morloc_version.is_null() {\n libc::free(m.build.morloc_version as *mut c_void);\n }\n // pools\n for i in 0..m.n_pools {\n let pool = &*m.pools.add(i);\n if !pool.lang.is_null() {\n libc::free(pool.lang as *mut c_void);\n }\n free_str_array(pool.exec);\n if !pool.socket.is_null() {\n libc::free(pool.socket as *mut c_void);\n }\n if !pool.metadata_json.is_null() {\n libc::free(pool.metadata_json as *mut c_void);\n }\n }\n if !m.pools.is_null() {\n libc::free(m.pools as *mut c_void);\n }\n // commands\n for i in 0..m.n_commands {\n let cmd = &*m.commands.add(i);\n if !cmd.name.is_null() {\n libc::free(cmd.name as *mut c_void);\n }\n if !cmd.needed_pools.is_null() {\n libc::free(cmd.needed_pools as *mut c_void);\n }\n free_str_array(cmd.desc);\n for j in 0..cmd.n_args {\n free_arg(&*cmd.args.add(j));\n }\n if !cmd.args.is_null() {\n libc::free(cmd.args as *mut c_void);\n }\n free_return(&cmd.ret);\n free_constraints(cmd.constraints, cmd.n_constraints);\n if !cmd.group.is_null() {\n libc::free(cmd.group as *mut c_void);\n }\n if !cmd.metadata_json.is_null() {\n libc::free(cmd.metadata_json as *mut c_void);\n }\n // Note: cmd.expr is owned by the C side and freed by its own\n // free function in eval_ffi.rs (not in scope here).\n }\n if !m.commands.is_null() {\n libc::free(m.commands as *mut c_void);\n }\n // groups\n for i in 0..m.n_groups {\n let g = &*m.groups.add(i);\n if !g.name.is_null() {\n libc::free(g.name as *mut c_void);\n }\n free_str_array(g.desc);\n if !g.metadata_json.is_null() {\n libc::free(g.metadata_json as *mut c_void);\n }\n }\n if !m.groups.is_null() {\n libc::free(m.groups as *mut c_void);\n }\n // service\n if !m.service.is_null() {\n let svc = &*m.service;\n if !svc.stype.is_null() {\n libc::free(svc.stype as *mut c_void);\n }\n if !svc.host.is_null() {\n libc::free(svc.host as *mut c_void);\n }\n if !svc.socket.is_null() {\n libc::free(svc.socket as *mut c_void);\n }\n if !svc.metadata_json.is_null() {\n libc::free(svc.metadata_json as *mut c_void);\n }\n libc::free(m.service as *mut c_void);\n }\n if !m.metadata_json.is_null() {\n libc::free(m.metadata_json as *mut c_void);\n }\n libc::free(manifest as *mut c_void);\n}\n\n// -- manifest_to_discovery_json -----------------------------------------------\n//\n// Emits a v2-shape JSON describing the manifest's commands. Used by the\n// daemon/router code for discovery RPC. The output mirrors the morloc\n// compiler's manifest format closely (no v1 legacy field names).\n\n#[no_mangle]\npub unsafe extern \"C\" fn manifest_to_discovery_json(manifest: *const Manifest) -> *mut c_char {\n if manifest.is_null() {\n return ptr::null_mut();\n }\n let m = &*manifest;\n\n extern \"C\" {\n fn json_buf_new() -> *mut c_void;\n fn json_buf_finish(jb: *mut c_void) -> *mut c_char;\n fn json_write_obj_start(jb: *mut c_void);\n fn json_write_obj_end(jb: *mut c_void);\n fn json_write_arr_start(jb: *mut c_void);\n fn json_write_arr_end(jb: *mut c_void);\n fn json_write_key(jb: *mut c_void, key: *const c_char);\n fn json_write_string(jb: *mut c_void, val: *const c_char);\n }\n\n let jb = json_buf_new();\n json_write_obj_start(jb);\n\n let name_key = b\"name\\0\".as_ptr() as *const c_char;\n let type_key = b\"type\\0\".as_ptr() as *const c_char;\n let kind_key = b\"kind\\0\".as_ptr() as *const c_char;\n let schema_key = b\"schema\\0\".as_ptr() as *const c_char;\n\n json_write_key(jb, name_key);\n json_write_string(\n jb,\n if m.name.is_null() {\n b\"unknown\\0\".as_ptr() as *const c_char\n } else {\n m.name\n },\n );\n\n // Surface the morloc compiler version that built this manifest.\n if !m.build.morloc_version.is_null() {\n let mv_key = b\"morloc_version\\0\".as_ptr() as *const c_char;\n json_write_key(jb, mv_key);\n json_write_string(jb, m.build.morloc_version);\n }\n\n let commands_key = b\"commands\\0\".as_ptr() as *const c_char;\n json_write_key(jb, commands_key);\n json_write_arr_start(jb);\n\n for i in 0..m.n_commands {\n let cmd = &*m.commands.add(i);\n json_write_obj_start(jb);\n\n json_write_key(jb, name_key);\n json_write_string(jb, cmd.name);\n\n json_write_key(jb, type_key);\n json_write_string(\n jb,\n if cmd.is_pure {\n b\"pure\\0\".as_ptr() as *const c_char\n } else {\n b\"remote\\0\".as_ptr() as *const c_char\n },\n );\n\n // Return descriptor (nested return object).\n if !cmd.ret.type_desc.is_null() || !cmd.ret.schema.is_null() {\n let ret_key = b\"return\\0\".as_ptr() as *const c_char;\n json_write_key(jb, ret_key);\n json_write_obj_start(jb);\n if !cmd.ret.type_desc.is_null() {\n json_write_key(jb, type_key);\n json_write_string(jb, cmd.ret.type_desc);\n }\n if !cmd.ret.schema.is_null() {\n json_write_key(jb, schema_key);\n json_write_string(jb, cmd.ret.schema);\n }\n json_write_obj_end(jb);\n }\n\n // Args. Each arg's schema (if any) is on the arg itself; no\n // parallel array, no flag-skipping bug.\n let args_key = b\"args\\0\".as_ptr() as *const c_char;\n json_write_key(jb, args_key);\n json_write_arr_start(jb);\n for a in 0..cmd.n_args {\n let arg = &*cmd.args.add(a);\n json_write_obj_start(jb);\n\n json_write_key(jb, kind_key);\n match arg.kind {\n ManifestArgKind::Pos => json_write_string(jb, b\"pos\\0\".as_ptr() as *const c_char),\n ManifestArgKind::Opt => json_write_string(jb, b\"opt\\0\".as_ptr() as *const c_char),\n ManifestArgKind::Flag => json_write_string(jb, b\"flag\\0\".as_ptr() as *const c_char),\n ManifestArgKind::Grp => json_write_string(jb, b\"grp\\0\".as_ptr() as *const c_char),\n }\n\n if !arg.metavar.is_null() {\n json_write_key(jb, b\"metavar\\0\".as_ptr() as *const c_char);\n json_write_string(jb, arg.metavar);\n }\n if !arg.type_desc.is_null() {\n json_write_key(jb, type_key);\n json_write_string(jb, arg.type_desc);\n }\n if !arg.schema.is_null() {\n json_write_key(jb, schema_key);\n json_write_string(jb, arg.schema);\n }\n if !arg.default_val.is_null() {\n json_write_key(jb, b\"default\\0\".as_ptr() as *const c_char);\n json_write_string(jb, arg.default_val);\n }\n if !arg.long_opt.is_null() {\n json_write_key(jb, b\"long\\0\".as_ptr() as *const c_char);\n json_write_string(jb, arg.long_opt);\n }\n if arg.short_opt != 0 {\n let short_str = [arg.short_opt as u8, 0];\n json_write_key(jb, b\"short\\0\".as_ptr() as *const c_char);\n json_write_string(jb, short_str.as_ptr() as *const c_char);\n }\n if arg.n_desc > 0 && !arg.desc.is_null() && !(*arg.desc).is_null() {\n let first = *arg.desc;\n if *first != 0 {\n let desc_key = b\"desc\\0\".as_ptr() as *const c_char;\n json_write_key(jb, desc_key);\n json_write_string(jb, first);\n }\n }\n\n json_write_obj_end(jb);\n }\n json_write_arr_end(jb);\n\n if cmd.n_desc > 0 && !cmd.desc.is_null() && !(*cmd.desc).is_null() {\n let first = *cmd.desc;\n if *first != 0 {\n json_write_key(jb, b\"desc\\0\".as_ptr() as *const c_char);\n json_write_string(jb, first);\n }\n }\n\n if !cmd.group.is_null() {\n json_write_key(jb, b\"group\\0\".as_ptr() as *const c_char);\n json_write_string(jb, cmd.group);\n }\n\n json_write_obj_end(jb);\n }\n\n json_write_arr_end(jb);\n\n if m.n_groups > 0 {\n let groups_key = b\"groups\\0\".as_ptr() as *const c_char;\n json_write_key(jb, groups_key);\n json_write_arr_start(jb);\n for i in 0..m.n_groups {\n let g = &*m.groups.add(i);\n json_write_obj_start(jb);\n json_write_key(jb, name_key);\n json_write_string(jb, g.name);\n if g.n_desc > 0 && !g.desc.is_null() && !(*g.desc).is_null() {\n json_write_key(jb, b\"desc\\0\".as_ptr() as *const c_char);\n json_write_string(jb, *g.desc);\n }\n json_write_obj_end(jb);\n }\n json_write_arr_end(jb);\n }\n\n json_write_obj_end(jb);\n json_buf_finish(jb)\n}\n" }, { "path": "data/rust/morloc-runtime/src/mpack.rs", "content": "//! MessagePack <-> Voidstar conversion.\n//!\n//! Replaces serialize.c + mpack.c. Uses the `rmp` crate for MessagePack I/O.\n//! The voidstar binary format is morloc-specific (Array/Tensor structs with relptrs).\n\nuse crate::error::MorlocError;\nuse crate::schema::{Schema, SerialType};\nuse crate::shm::{self, AbsPtr, Array, RELNULL};\n\n// ── Voidstar -> MessagePack ────────────────────────────────────────────────\n\n/// Serialize voidstar data to MessagePack bytes.\npub fn pack_with_schema(ptr: AbsPtr, schema: &Schema) -> Result, MorlocError> {\n let mut buf = Vec::with_capacity(256);\n pack_data(ptr, schema, &mut buf)?;\n Ok(buf)\n}\n\nfn pack_data(ptr: AbsPtr, schema: &Schema, buf: &mut Vec) -> Result<(), MorlocError> {\n // SAFETY: ptr points to voidstar data in SHM with layout described by schema.\n // All reads are within bounds defined by schema.width, Array headers, etc.\n unsafe {\n match schema.serial_type {\n SerialType::Nil => {\n rmp::encode::write_nil(buf)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack nil: {}\", e)))?;\n }\n SerialType::Bool => {\n let v = *ptr != 0;\n rmp::encode::write_bool(buf, v)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack bool: {}\", e)))?;\n }\n SerialType::Uint8 => {\n rmp::encode::write_uint(buf, *ptr as u64)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack uint: {}\", e)))?;\n }\n SerialType::Uint16 => {\n rmp::encode::write_uint(buf, *(ptr as *const u16) as u64)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack uint: {}\", e)))?;\n }\n SerialType::Uint32 => {\n rmp::encode::write_uint(buf, *(ptr as *const u32) as u64)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack uint: {}\", e)))?;\n }\n SerialType::Uint64 => {\n rmp::encode::write_uint(buf, *(ptr as *const u64))\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack uint: {}\", e)))?;\n }\n SerialType::Sint8 => {\n rmp::encode::write_sint(buf, *(ptr as *const i8) as i64)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack sint: {}\", e)))?;\n }\n SerialType::Sint16 => {\n rmp::encode::write_sint(buf, *(ptr as *const i16) as i64)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack sint: {}\", e)))?;\n }\n SerialType::Sint32 => {\n rmp::encode::write_sint(buf, *(ptr as *const i32) as i64)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack sint: {}\", e)))?;\n }\n SerialType::Sint64 => {\n rmp::encode::write_sint(buf, *(ptr as *const i64))\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack sint: {}\", e)))?;\n }\n SerialType::Float32 => {\n let f = *(ptr as *const f32) as f64;\n rmp::encode::write_f64(buf, f)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack float: {}\", e)))?;\n }\n SerialType::Float64 => {\n let f = *(ptr as *const f64);\n rmp::encode::write_f64(buf, f)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack float: {}\", e)))?;\n }\n SerialType::String => {\n let arr = &*(ptr as *const Array);\n let data = shm::rel2abs(arr.data)?;\n let bytes = std::slice::from_raw_parts(data, arr.size);\n rmp::encode::write_str_len(buf, arr.size as u32)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack str: {}\", e)))?;\n buf.extend_from_slice(bytes);\n }\n SerialType::Array => {\n let arr = &*(ptr as *const Array);\n let elem_schema = &schema.parameters[0];\n let elem_width = elem_schema.width;\n\n rmp::encode::write_array_len(buf, arr.size as u32)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack array: {}\", e)))?;\n\n if arr.size > 0 && arr.data != RELNULL {\n let data = shm::rel2abs(arr.data)?;\n for i in 0..arr.size {\n let elem_ptr = data.add(i * elem_width);\n pack_data(elem_ptr, elem_schema, buf)?;\n }\n }\n }\n SerialType::Tuple | SerialType::Map => {\n rmp::encode::write_array_len(buf, schema.parameters.len() as u32)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack tuple: {}\", e)))?;\n\n for (i, field_schema) in schema.parameters.iter().enumerate() {\n let field_ptr = ptr.add(schema.offsets[i]);\n pack_data(field_ptr, field_schema, buf)?;\n }\n }\n SerialType::Optional => {\n let tag = *ptr;\n if tag == 0 {\n rmp::encode::write_nil(buf)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack nil: {}\", e)))?;\n } else {\n let inner_schema = &schema.parameters[0];\n let inner_offset = schema.offsets.first().copied()\n .unwrap_or_else(|| shm::align_up(1, inner_schema.alignment().max(1)));\n let inner_ptr = ptr.add(inner_offset);\n pack_data(inner_ptr, inner_schema, buf)?;\n }\n }\n SerialType::Tensor => {\n return Err(MorlocError::Serialization(\n \"MessagePack serialization of tensors not yet supported\".into(),\n ));\n }\n }\n }\n Ok(())\n}\n\n// ── MessagePack -> Voidstar ────────────────────────────────────────────────\n\n/// Deserialize MessagePack bytes into voidstar format in SHM.\npub fn unpack_with_schema(\n data: &[u8],\n schema: &Schema,\n) -> Result {\n // Two-pass: first calculate size, then deserialize\n let size = calc_unpack_size(data, schema)?;\n let base = shm::shmalloc(size)?;\n // SAFETY: base is freshly allocated with `size` bytes.\n unsafe { std::ptr::write_bytes(base, 0, size) };\n\n // SAFETY: cursor starts at base + schema.width, within the allocated region.\n let mut cursor = unsafe { base.add(schema.width) };\n let mut reader = &data[..];\n unpack_obj(base, schema, &mut cursor, &mut reader)?;\n Ok(base)\n}\n\nfn unpack_obj(\n ptr: AbsPtr,\n schema: &Schema,\n cursor: &mut AbsPtr,\n reader: &mut &[u8],\n) -> Result<(), MorlocError> {\n use rmp::decode;\n\n // SAFETY: ptr and cursor point into a single contiguous SHM allocation\n // sized by calc_unpack_size. Each write respects schema.width bounds.\n unsafe {\n match schema.serial_type {\n SerialType::Nil => {\n decode::read_nil(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack nil: {}\", e)))?;\n *ptr = 0;\n }\n SerialType::Bool => {\n let v = decode::read_bool(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack bool: {}\", e)))?;\n *ptr = v as u8;\n }\n SerialType::Uint8 | SerialType::Uint16 | SerialType::Uint32 | SerialType::Uint64\n | SerialType::Sint8 | SerialType::Sint16 | SerialType::Sint32 | SerialType::Sint64 => {\n unpack_int(ptr, schema.serial_type, reader)?;\n }\n SerialType::Float32 => {\n let f = read_float(reader)?;\n *(ptr as *mut f32) = f as f32;\n }\n SerialType::Float64 => {\n let f = read_float(reader)?;\n *(ptr as *mut f64) = f;\n }\n SerialType::String => {\n let len = decode::read_str_len(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack str len: {}\", e)))?\n as usize;\n let arr = &mut *(ptr as *mut Array);\n arr.size = len;\n arr.data = shm::abs2rel(*cursor)?;\n\n // Read string bytes directly\n if len > 0 {\n if reader.len() < len {\n return Err(MorlocError::Serialization(\"msgpack str truncated\".into()));\n }\n std::ptr::copy_nonoverlapping(reader.as_ptr(), *cursor, len);\n *reader = &reader[len..];\n }\n *cursor = cursor.add(len);\n }\n SerialType::Array => {\n let n = decode::read_array_len(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack array len: {}\", e)))?\n as usize;\n let elem_schema = &schema.parameters[0];\n let elem_width = elem_schema.width;\n\n let arr = &mut *(ptr as *mut Array);\n arr.size = n;\n\n // Align cursor for element data\n let align = elem_schema.alignment();\n let aligned = shm::align_up(*cursor as usize, align);\n *cursor = aligned as AbsPtr;\n\n arr.data = shm::abs2rel(*cursor)?;\n let data_start = *cursor;\n *cursor = cursor.add(n * elem_width);\n\n for i in 0..n {\n let elem_ptr = data_start.add(i * elem_width);\n unpack_obj(elem_ptr, elem_schema, cursor, reader)?;\n }\n }\n SerialType::Tuple | SerialType::Map => {\n let _n = decode::read_array_len(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack tuple len: {}\", e)))?;\n for (i, field_schema) in schema.parameters.iter().enumerate() {\n let field_ptr = ptr.add(schema.offsets[i]);\n unpack_obj(field_ptr, field_schema, cursor, reader)?;\n }\n }\n SerialType::Optional => {\n let inner_schema = &schema.parameters[0];\n let inner_offset = schema.offsets.first().copied()\n .unwrap_or_else(|| shm::align_up(1, inner_schema.alignment().max(1)));\n\n // Peek at the next byte to detect nil\n if !reader.is_empty() && reader[0] == 0xc0 {\n // Null: consume nil byte, set tag = 0\n decode::read_nil(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack nil: {}\", e)))?;\n *ptr = 0;\n } else {\n // Present: set tag = 1, parse inner\n *ptr = 1;\n let inner_ptr = ptr.add(inner_offset);\n unpack_obj(inner_ptr, inner_schema, cursor, reader)?;\n }\n }\n SerialType::Tensor => {\n return Err(MorlocError::Serialization(\n \"MessagePack tensor deserialization not yet supported\".into(),\n ));\n }\n }\n }\n Ok(())\n}\n\nfn unpack_int(ptr: AbsPtr, st: SerialType, reader: &mut &[u8]) -> Result<(), MorlocError> {\n // Use rmp's generic read_int which handles all integer markers\n let val: i64 = rmp::decode::read_int(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"msgpack int: {}\", e)))?;\n\n // SAFETY: ptr points to schema.width bytes in SHM; each cast writes exactly that width.\n unsafe {\n match st {\n SerialType::Sint8 => *(ptr as *mut i8) = val as i8,\n SerialType::Sint16 => *(ptr as *mut i16) = val as i16,\n SerialType::Sint32 => *(ptr as *mut i32) = val as i32,\n SerialType::Sint64 => *(ptr as *mut i64) = val,\n SerialType::Uint8 => *ptr = val as u8,\n SerialType::Uint16 => *(ptr as *mut u16) = val as u16,\n SerialType::Uint32 => *(ptr as *mut u32) = val as u32,\n SerialType::Uint64 => *(ptr as *mut u64) = val as u64,\n _ => {}\n }\n }\n Ok(())\n}\n\nfn read_float(reader: &mut &[u8]) -> Result {\n let marker = rmp::decode::read_marker(reader)\n .map_err(|_| MorlocError::Serialization(\"msgpack float: unexpected EOF\".into()))?;\n match marker {\n rmp::Marker::F32 => {\n let bits = read_be_u32(reader)?;\n Ok(f32::from_bits(bits) as f64)\n }\n rmp::Marker::F64 => {\n let bits = read_be_u64(reader)?;\n Ok(f64::from_bits(bits))\n }\n // Integer markers can appear for integer-valued floats - read the data manually\n rmp::Marker::FixPos(v) => Ok(v as f64),\n rmp::Marker::FixNeg(v) => Ok(v as f64),\n _ => {\n // For other integer encodings, read bytes manually\n let n = match marker {\n rmp::Marker::U8 => { read_byte(reader)? as f64 }\n rmp::Marker::U16 => { read_be_u16(reader)? as f64 }\n rmp::Marker::U32 => { read_be_u32(reader)? as f64 }\n rmp::Marker::U64 => { read_be_u64(reader)? as f64 }\n rmp::Marker::I8 => { read_byte(reader)? as i8 as f64 }\n rmp::Marker::I16 => { read_be_u16(reader)? as i16 as f64 }\n rmp::Marker::I32 => { read_be_u32(reader)? as i32 as f64 }\n rmp::Marker::I64 => { read_be_u64(reader)? as i64 as f64 }\n _ => {\n return Err(MorlocError::Serialization(format!(\n \"unexpected msgpack marker {:?} for float\", marker\n )));\n }\n };\n Ok(n)\n }\n }\n}\n\n// Also fix read_f32/read_f64 - rmp's functions include the marker, but we already consumed it\n// So we need to read the raw data bytes directly.\n\nfn read_byte(reader: &mut &[u8]) -> Result {\n if reader.is_empty() { return Err(MorlocError::Serialization(\"unexpected EOF\".into())); }\n let v = reader[0];\n *reader = &reader[1..];\n Ok(v)\n}\n\nfn read_be_u16(reader: &mut &[u8]) -> Result {\n if reader.len() < 2 { return Err(MorlocError::Serialization(\"unexpected EOF\".into())); }\n let v = u16::from_be_bytes([reader[0], reader[1]]);\n *reader = &reader[2..];\n Ok(v)\n}\n\nfn read_be_u32(reader: &mut &[u8]) -> Result {\n if reader.len() < 4 { return Err(MorlocError::Serialization(\"unexpected EOF\".into())); }\n let v = u32::from_be_bytes([reader[0], reader[1], reader[2], reader[3]]);\n *reader = &reader[4..];\n Ok(v)\n}\n\nfn read_be_u64(reader: &mut &[u8]) -> Result {\n if reader.len() < 8 { return Err(MorlocError::Serialization(\"unexpected EOF\".into())); }\n let v = u64::from_be_bytes([reader[0], reader[1], reader[2], reader[3], reader[4], reader[5], reader[6], reader[7]]);\n *reader = &reader[8..];\n Ok(v)\n}\n\n// ── Size calculation for unpack ────────────────────────────────────────────\n\nfn calc_unpack_size(data: &[u8], schema: &Schema) -> Result {\n let mut reader = data;\n calc_size_r(schema, &mut reader)\n}\n\nfn calc_size_r(schema: &Schema, reader: &mut &[u8]) -> Result {\n match schema.serial_type {\n SerialType::Nil => {\n rmp::decode::read_nil(reader).ok();\n Ok(1)\n }\n SerialType::Bool => {\n rmp::decode::read_bool(reader).ok();\n Ok(1)\n }\n SerialType::Sint8 | SerialType::Uint8 => { skip_int(reader)?; Ok(1) }\n SerialType::Sint16 | SerialType::Uint16 => { skip_int(reader)?; Ok(2) }\n SerialType::Sint32 | SerialType::Uint32 | SerialType::Float32 => { skip_int(reader)?; Ok(4) }\n SerialType::Sint64 | SerialType::Uint64 | SerialType::Float64 => { skip_int(reader)?; Ok(8) }\n SerialType::String => {\n let len = rmp::decode::read_str_len(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"size calc str: {}\", e)))?\n as usize;\n if reader.len() >= len { *reader = &reader[len..]; }\n Ok(std::mem::size_of::() + len)\n }\n SerialType::Array => {\n let n = rmp::decode::read_array_len(reader)\n .map_err(|e| MorlocError::Serialization(format!(\"size calc array: {}\", e)))?\n as usize;\n let elem_schema = &schema.parameters[0];\n let mut total = std::mem::size_of::();\n // Alignment padding\n total = shm::align_up(total, elem_schema.alignment());\n for _ in 0..n {\n total += calc_size_r(elem_schema, reader)?;\n }\n Ok(total)\n }\n SerialType::Tuple | SerialType::Map => {\n let _n = rmp::decode::read_array_len(reader).ok();\n let mut total = schema.width;\n for field_schema in &schema.parameters {\n if !field_schema.is_fixed_width() {\n total += calc_size_r(field_schema, reader)?;\n } else {\n calc_size_r(field_schema, reader)?;\n }\n }\n Ok(total)\n }\n SerialType::Optional => {\n let inner_schema = &schema.parameters[0];\n if !reader.is_empty() && reader[0] == 0xc0 {\n rmp::decode::read_nil(reader).ok();\n Ok(schema.width.max(1 + inner_schema.width))\n } else {\n let inner_size = calc_size_r(inner_schema, reader)?;\n let align = inner_schema.alignment().max(1);\n let offset = shm::align_up(1, align);\n Ok(offset + inner_size)\n }\n }\n SerialType::Tensor => Ok(0),\n }\n}\n\nfn skip_int(reader: &mut &[u8]) -> Result<(), MorlocError> {\n let marker = rmp::decode::read_marker(reader)\n .map_err(|_| MorlocError::Serialization(\"skip int: unexpected EOF\".into()))?;\n let skip = match marker {\n rmp::Marker::FixPos(_) | rmp::Marker::FixNeg(_) => 0,\n rmp::Marker::U8 | rmp::Marker::I8 => 1,\n rmp::Marker::U16 | rmp::Marker::I16 => 2,\n rmp::Marker::U32 | rmp::Marker::I32 | rmp::Marker::F32 => 4,\n rmp::Marker::U64 | rmp::Marker::I64 | rmp::Marker::F64 => 8,\n _ => 0,\n };\n if reader.len() >= skip {\n *reader = &reader[skip..];\n }\n Ok(())\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use crate::schema::parse_schema;\n use crate::json;\n\n fn setup_shm() {\n crate::init_test_shm();\n }\n\n #[test]\n fn test_roundtrip_int_via_msgpack() {\n setup_shm();\n let schema = parse_schema(\"i4\").unwrap();\n // JSON -> voidstar -> msgpack -> voidstar -> JSON\n let ptr1 = json::read_json_with_schema(\"42\", &schema).unwrap();\n let mpk = pack_with_schema(ptr1, &schema).unwrap();\n let ptr2 = unpack_with_schema(&mpk, &schema).unwrap();\n let json_out = json::voidstar_to_json_string(ptr2, &schema).unwrap();\n assert_eq!(json_out, \"42\");\n }\n\n #[test]\n fn test_roundtrip_string_via_msgpack() {\n setup_shm();\n let schema = parse_schema(\"s\").unwrap();\n let ptr1 = json::read_json_with_schema(\"\\\"hello world\\\"\", &schema).unwrap();\n let mpk = pack_with_schema(ptr1, &schema).unwrap();\n let ptr2 = unpack_with_schema(&mpk, &schema).unwrap();\n let json_out = json::voidstar_to_json_string(ptr2, &schema).unwrap();\n assert_eq!(json_out, \"\\\"hello world\\\"\");\n }\n\n #[test]\n fn test_roundtrip_array_via_msgpack() {\n setup_shm();\n let schema = parse_schema(\"ai4\").unwrap();\n let ptr1 = json::read_json_with_schema(\"[10,20,30]\", &schema).unwrap();\n let mpk = pack_with_schema(ptr1, &schema).unwrap();\n let ptr2 = unpack_with_schema(&mpk, &schema).unwrap();\n let json_out = json::voidstar_to_json_string(ptr2, &schema).unwrap();\n assert_eq!(json_out, \"[10,20,30]\");\n }\n\n #[test]\n fn test_roundtrip_bool_via_msgpack() {\n setup_shm();\n let schema = parse_schema(\"b\").unwrap();\n let ptr1 = json::read_json_with_schema(\"true\", &schema).unwrap();\n let mpk = pack_with_schema(ptr1, &schema).unwrap();\n let ptr2 = unpack_with_schema(&mpk, &schema).unwrap();\n let json_out = json::voidstar_to_json_string(ptr2, &schema).unwrap();\n assert_eq!(json_out, \"true\");\n }\n\n #[test]\n fn test_roundtrip_optional_null_via_msgpack() {\n setup_shm();\n let schema = parse_schema(\"?i4\").unwrap();\n let ptr1 = json::read_json_with_schema(\"null\", &schema).unwrap();\n let mpk = pack_with_schema(ptr1, &schema).unwrap();\n let ptr2 = unpack_with_schema(&mpk, &schema).unwrap();\n let json_out = json::voidstar_to_json_string(ptr2, &schema).unwrap();\n assert_eq!(json_out, \"null\");\n }\n\n #[test]\n fn test_pack_only_string() {\n setup_shm();\n let schema = parse_schema(\"s\").unwrap();\n let ptr1 = json::read_json_with_schema(\"\\\"hi\\\"\", &schema).unwrap();\n let mpk = pack_with_schema(ptr1, &schema).unwrap();\n assert!(!mpk.is_empty());\n assert_eq!(mpk.len(), 3);\n }\n\n #[test]\n fn test_unpack_only_int() {\n setup_shm();\n let schema = parse_schema(\"i4\").unwrap();\n // msgpack for 42 = [42] (fixint)\n let mpk = vec![42u8];\n let ptr = unpack_with_schema(&mpk, &schema).unwrap();\n let json_out = json::voidstar_to_json_string(ptr, &schema).unwrap();\n assert_eq!(json_out, \"42\");\n }\n\n #[test]\n fn test_unpack_only_string() {\n setup_shm();\n let schema = parse_schema(\"s\").unwrap();\n // msgpack for \"hi\" = [0xa2, 0x68, 0x69]\n let mpk = vec![0xa2, 0x68, 0x69];\n let size = calc_unpack_size(&mpk, &schema).unwrap();\n eprintln!(\"unpack size for string: {} (Array={}, total={})\", size, std::mem::size_of::(), size);\n let ptr = unpack_with_schema(&mpk, &schema).unwrap();\n let json_out = json::voidstar_to_json_string(ptr, &schema).unwrap();\n assert_eq!(json_out, \"\\\"hi\\\"\");\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/packet.rs", "content": "use crate::error::MorlocError;\n\n// ── Magic & version constants ──────────────────────────────────────────────\n\npub const PACKET_MAGIC: u32 = 0x0707_f86d;\npub const THIS_PLAIN: u16 = 0;\npub const THIS_VERSION: u16 = 0;\npub const DEFAULT_FLAVOR: u16 = 0;\npub const DEFAULT_MODE: u16 = 0;\n\n// ── Command type discriminants ─────────────────────────────────────────────\n\npub const PACKET_TYPE_DATA: u8 = 0;\npub const PACKET_TYPE_CALL: u8 = 1;\npub const PACKET_TYPE_PING: u8 = 2;\n\n// ── Data source ────────────────────────────────────────────────────────────\n\npub const PACKET_SOURCE_MESG: u8 = 0x00;\npub const PACKET_SOURCE_FILE: u8 = 0x01;\npub const PACKET_SOURCE_RPTR: u8 = 0x02;\n\n// ── Data format ────────────────────────────────────────────────────────────\n\npub const PACKET_FORMAT_JSON: u8 = 0x00;\npub const PACKET_FORMAT_MSGPACK: u8 = 0x01;\npub const PACKET_FORMAT_TEXT: u8 = 0x02;\npub const PACKET_FORMAT_DATA: u8 = 0x03;\npub const PACKET_FORMAT_VOIDSTAR: u8 = 0x04;\npub const PACKET_FORMAT_ARROW: u8 = 0x05;\n\n// ── Compression / encryption ───────────────────────────────────────────────\n\npub const PACKET_COMPRESSION_NONE: u8 = 0x00;\npub const PACKET_ENCRYPTION_NONE: u8 = 0x00;\n\n// ── Status ─────────────────────────────────────────────────────────────────\n\npub const PACKET_STATUS_PASS: u8 = 0x00;\npub const PACKET_STATUS_FAIL: u8 = 0x01;\n\n// ── Entrypoint ────────────────────���────────────────────────────────────────\n\npub const PACKET_ENTRYPOINT_LOCAL: u8 = 0x00;\npub const PACKET_ENTRYPOINT_REMOTE_SFS: u8 = 0x01;\n\n// ── Inline threshold ─────────────────────────────────────��─────────────────\n\npub const MORLOC_INLINE_THRESHOLD: usize = 64 * 1024;\n\n// ── Metadata ─────────��─────────────────────────────────────────────────────\n\npub const METADATA_TYPE_SCHEMA_STRING: u8 = 0x01;\npub const METADATA_TYPE_XXHASH: u8 = 0x02;\npub const METADATA_HEADER_MAGIC: [u8; 3] = *b\"mmh\";\n\n// ── Packed structs matching the C binary layout ────���───────────────────────\n\n/// 8-byte command union. We represent each variant as its own struct and\n/// transmute at the boundary.\n#[derive(Debug, Clone, Copy)]\n#[repr(C, packed)]\npub struct CommandType {\n pub cmd_type: u8,\n pub padding: [u8; 7],\n}\n\n#[derive(Debug, Clone, Copy)]\n#[repr(C, packed)]\npub struct CommandCall {\n pub cmd_type: u8,\n pub entrypoint: u8,\n pub padding: [u8; 2],\n pub midx: u32,\n}\n\n#[derive(Debug, Clone, Copy)]\n#[repr(C, packed)]\npub struct CommandData {\n pub cmd_type: u8,\n pub source: u8,\n pub format: u8,\n pub compression: u8,\n pub encryption: u8,\n pub status: u8,\n pub padding: [u8; 2],\n}\n\n#[derive(Debug, Clone, Copy)]\n#[repr(C, packed)]\npub struct CommandPing {\n pub cmd_type: u8,\n pub padding: [u8; 7],\n}\n\n/// The 8-byte command field stored as raw bytes. Interpreted based on the\n/// first byte (cmd_type discriminant).\n#[derive(Clone, Copy)]\n#[repr(C, packed)]\npub union PacketCommand {\n pub cmd_type: CommandType,\n pub call: CommandCall,\n pub data: CommandData,\n pub ping: CommandPing,\n pub raw: [u8; 8],\n}\n\nimpl std::fmt::Debug for PacketCommand {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n let tag = unsafe { self.cmd_type.cmd_type };\n match tag {\n PACKET_TYPE_DATA => write!(f, \"Command::Data({:?})\", unsafe { self.data }),\n PACKET_TYPE_CALL => write!(f, \"Command::Call({:?})\", unsafe { self.call }),\n PACKET_TYPE_PING => write!(f, \"Command::Ping\"),\n _ => write!(f, \"Command::Unknown({tag})\"),\n }\n }\n}\n\n/// 32-byte packet header. Must match morloc_packet_header_t exactly.\n#[derive(Debug, Clone, Copy)]\n#[repr(C, packed)]\npub struct PacketHeader {\n pub magic: u32,\n pub plain: u16,\n pub version: u16,\n pub flavor: u16,\n pub mode: u16,\n pub command: PacketCommand,\n pub offset: u32,\n pub length: u64,\n}\n\nconst _: () = assert!(std::mem::size_of::() == 32);\nconst _: () = assert!(std::mem::size_of::() == 8);\nconst _: () = assert!(std::mem::size_of::() == 8);\nconst _: () = assert!(std::mem::size_of::() == 8);\n\n/// 8-byte metadata header.\n#[derive(Debug, Clone, Copy)]\n#[repr(C, packed)]\npub struct MetadataHeader {\n pub magic: [u8; 3],\n pub metadata_type: u8,\n pub size: u32,\n}\n\nconst _: () = assert!(std::mem::size_of::() == 8);\n\n// ── Packet construction helpers ────────────────────────────────────────────\n\nimpl PacketHeader {\n fn new(command: PacketCommand, offset: u32, length: u64) -> Self {\n PacketHeader {\n magic: PACKET_MAGIC,\n plain: THIS_PLAIN,\n version: THIS_VERSION,\n flavor: DEFAULT_FLAVOR,\n mode: DEFAULT_MODE,\n command,\n offset,\n length,\n }\n }\n\n /// Create a ping packet (no payload).\n pub fn ping() -> Self {\n Self::new(\n PacketCommand {\n ping: CommandPing {\n cmd_type: PACKET_TYPE_PING,\n padding: [0; 7],\n },\n },\n 0, // no metadata\n 0, // no payload\n )\n }\n\n /// Create a local call packet header.\n pub fn local_call(midx: u32, payload_len: u64) -> Self {\n Self::new(\n PacketCommand {\n call: CommandCall {\n cmd_type: PACKET_TYPE_CALL,\n entrypoint: PACKET_ENTRYPOINT_LOCAL,\n padding: [0; 2],\n midx,\n },\n },\n 0, // no metadata between header and arg packets\n payload_len,\n )\n }\n\n /// Create a remote call packet header.\n pub fn remote_call(midx: u32, payload_len: u64) -> Self {\n Self::new(\n PacketCommand {\n call: CommandCall {\n cmd_type: PACKET_TYPE_CALL,\n entrypoint: PACKET_ENTRYPOINT_REMOTE_SFS,\n padding: [0; 2],\n midx,\n },\n },\n 0,\n payload_len,\n )\n }\n\n /// Create a data packet header for inline message data.\n pub fn data_mesg(format: u8, payload_len: u64) -> Self {\n Self::new(\n PacketCommand {\n data: CommandData {\n cmd_type: PACKET_TYPE_DATA,\n source: PACKET_SOURCE_MESG,\n format,\n compression: PACKET_COMPRESSION_NONE,\n encryption: PACKET_ENCRYPTION_NONE,\n status: PACKET_STATUS_PASS,\n padding: [0; 2],\n },\n },\n 0, // metadata size set separately when building full packet\n payload_len,\n )\n }\n\n /// Create a data packet header for relative pointer (shared memory).\n pub fn data_rptr(format: u8, payload_len: u64) -> Self {\n Self::new(\n PacketCommand {\n data: CommandData {\n cmd_type: PACKET_TYPE_DATA,\n source: PACKET_SOURCE_RPTR,\n format,\n compression: PACKET_COMPRESSION_NONE,\n encryption: PACKET_ENCRYPTION_NONE,\n status: PACKET_STATUS_PASS,\n padding: [0; 2],\n },\n },\n 0, // metadata size set separately when building full packet\n payload_len,\n )\n }\n\n /// Create a fail packet with an error message.\n pub fn fail(error_msg_len: u64) -> Self {\n Self::new(\n PacketCommand {\n data: CommandData {\n cmd_type: PACKET_TYPE_DATA,\n source: PACKET_SOURCE_MESG,\n format: PACKET_FORMAT_TEXT,\n compression: PACKET_COMPRESSION_NONE,\n encryption: PACKET_ENCRYPTION_NONE,\n status: PACKET_STATUS_FAIL,\n padding: [0; 2],\n },\n },\n 0,\n error_msg_len,\n )\n }\n\n /// Total packet size (header + payload).\n pub fn total_size(&self) -> u64 {\n self.offset as u64 + self.length\n }\n\n /// Check if this is a valid morloc packet.\n pub fn is_valid(&self) -> bool {\n self.magic == PACKET_MAGIC\n }\n\n /// Get the command type.\n pub fn command_type(&self) -> u8 {\n unsafe { self.command.cmd_type.cmd_type }\n }\n\n pub fn is_ping(&self) -> bool {\n self.command_type() == PACKET_TYPE_PING\n }\n\n pub fn is_call(&self) -> bool {\n self.command_type() == PACKET_TYPE_CALL\n }\n\n pub fn is_data(&self) -> bool {\n self.command_type() == PACKET_TYPE_DATA\n }\n\n pub fn is_local_call(&self) -> bool {\n self.is_call() && unsafe { self.command.call.entrypoint } == PACKET_ENTRYPOINT_LOCAL\n }\n\n pub fn is_remote_call(&self) -> bool {\n self.is_call() && unsafe { self.command.call.entrypoint } == PACKET_ENTRYPOINT_REMOTE_SFS\n }\n\n pub fn is_fail(&self) -> bool {\n self.is_data() && unsafe { self.command.data.status } == PACKET_STATUS_FAIL\n }\n\n /// Serialize the header to bytes.\n pub fn to_bytes(&self) -> [u8; 32] {\n unsafe { std::mem::transmute(*self) }\n }\n\n /// Deserialize a header from bytes.\n pub fn from_bytes(bytes: &[u8; 32]) -> Result {\n let header: PacketHeader = unsafe { std::mem::transmute(*bytes) };\n if !header.is_valid() {\n let magic = { header.magic };\n return Err(MorlocError::Packet(format!(\n \"invalid magic: 0x{magic:08x}\"\n )));\n }\n Ok(header)\n }\n}\n\n// ── Full packet construction (header + metadata + payload) ─────────────────\n\n/// Build a complete data packet with schema metadata and relptr payload.\npub fn make_standard_data_packet(relptr: crate::shm::RelPtr, schema: &crate::Schema) -> Vec {\n let schema_str = crate::schema::schema_to_string(schema);\n let schema_bytes = schema_str.as_bytes();\n let schema_len = schema_bytes.len() + 1; // +1 for null terminator\n\n // Metadata: header (8 bytes) + schema string (null-terminated), padded to 32-byte boundary\n let meta_header_size = std::mem::size_of::();\n let raw_meta_len = meta_header_size + schema_len;\n let padded_meta_len = ((raw_meta_len + 31) / 32) * 32;\n\n // Payload: relptr\n let payload_len = std::mem::size_of::();\n\n let total = 32 + padded_meta_len + payload_len;\n let mut packet = vec![0u8; total];\n\n // Write header\n let header = PacketHeader::data_rptr(PACKET_FORMAT_VOIDSTAR, payload_len as u64);\n let mut hdr = header;\n // Override offset to include metadata\n unsafe {\n let hdr_ptr = &mut hdr as *mut PacketHeader as *mut u8;\n // Set offset field (at byte 20 in packed struct)\n let offset_ptr = hdr_ptr.add(20) as *mut u32;\n *offset_ptr = padded_meta_len as u32;\n }\n let hdr_bytes = hdr.to_bytes();\n packet[..32].copy_from_slice(&hdr_bytes);\n\n // Write metadata header\n let meta_start = 32;\n packet[meta_start] = b'm';\n packet[meta_start + 1] = b'm';\n packet[meta_start + 2] = b'h';\n packet[meta_start + 3] = METADATA_TYPE_SCHEMA_STRING;\n let meta_size_bytes = (schema_len as u32).to_le_bytes();\n packet[meta_start + 4..meta_start + 8].copy_from_slice(&meta_size_bytes);\n\n // Write schema string (null-terminated)\n let schema_data_start = meta_start + meta_header_size;\n packet[schema_data_start..schema_data_start + schema_bytes.len()].copy_from_slice(schema_bytes);\n // Null terminator already there from vec![0u8]\n\n // Write relptr payload\n let payload_start = 32 + padded_meta_len;\n let relptr_bytes = relptr.to_ne_bytes();\n packet[payload_start..payload_start + relptr_bytes.len()].copy_from_slice(&relptr_bytes);\n\n packet\n}\n\n/// Build an inline MESG+MSGPACK data packet with schema metadata.\npub fn make_mesg_data_packet(mpk_data: &[u8], schema: &crate::Schema) -> Vec {\n let schema_str = crate::schema::schema_to_string(schema);\n let schema_bytes = schema_str.as_bytes();\n let schema_len = schema_bytes.len() + 1; // +1 for null terminator\n\n let meta_header_size = std::mem::size_of::();\n let raw_meta_len = meta_header_size + schema_len;\n let padded_meta_len = ((raw_meta_len + 31) / 32) * 32;\n\n let total = 32 + padded_meta_len + mpk_data.len();\n let mut packet = vec![0u8; total];\n\n // Write header\n let mut header = PacketHeader::data_mesg(PACKET_FORMAT_MSGPACK, mpk_data.len() as u64);\n // Set offset to metadata size\n unsafe {\n let hdr_ptr = &mut header as *mut PacketHeader as *mut u8;\n let offset_ptr = hdr_ptr.add(20) as *mut u32;\n *offset_ptr = padded_meta_len as u32;\n }\n let hdr_bytes = header.to_bytes();\n packet[..32].copy_from_slice(&hdr_bytes);\n\n // Write metadata header\n let meta_start = 32;\n packet[meta_start] = b'm';\n packet[meta_start + 1] = b'm';\n packet[meta_start + 2] = b'h';\n packet[meta_start + 3] = METADATA_TYPE_SCHEMA_STRING;\n let meta_size_bytes = (schema_len as u32).to_le_bytes();\n packet[meta_start + 4..meta_start + 8].copy_from_slice(&meta_size_bytes);\n\n // Write schema string\n let schema_data_start = meta_start + meta_header_size;\n packet[schema_data_start..schema_data_start + schema_bytes.len()].copy_from_slice(schema_bytes);\n\n // Write msgpack payload\n let payload_start = 32 + padded_meta_len;\n packet[payload_start..payload_start + mpk_data.len()].copy_from_slice(mpk_data);\n\n packet\n}\n\n/// Build a call packet from argument data packets.\npub fn make_local_call_packet(midx: u32, arg_packets: &[Vec]) -> Vec {\n let data_length: usize = arg_packets.iter().map(|p| p.len()).sum();\n let total = 32 + data_length;\n let mut packet = vec![0u8; total];\n\n // Write call header\n let header = PacketHeader::local_call(midx, data_length as u64);\n let hdr_bytes = header.to_bytes();\n packet[..32].copy_from_slice(&hdr_bytes);\n\n // Concatenate argument packets\n let mut pos = 32;\n for arg in arg_packets {\n packet[pos..pos + arg.len()].copy_from_slice(arg);\n pos += arg.len();\n }\n\n packet\n}\n\n/// Build a fail packet with an error message string.\npub fn make_fail_packet_bytes(error_msg: &str) -> Vec {\n let msg_bytes = error_msg.as_bytes();\n let total = 32 + msg_bytes.len();\n let mut packet = vec![0u8; total];\n\n let header = PacketHeader::fail(msg_bytes.len() as u64);\n let hdr_bytes = header.to_bytes();\n packet[..32].copy_from_slice(&hdr_bytes);\n packet[32..].copy_from_slice(msg_bytes);\n\n packet\n}\n\n/// Extract the payload from a data packet (bytes after header + metadata offset).\npub fn get_data_payload(packet: &[u8]) -> Result<&[u8], MorlocError> {\n if packet.len() < 32 {\n return Err(MorlocError::Packet(\"packet too small\".into()));\n }\n let header = PacketHeader::from_bytes(packet[..32].try_into().unwrap())?;\n let offset = { header.offset } as usize;\n let length = { header.length } as usize;\n let start = 32 + offset;\n let end = start + length;\n if end > packet.len() {\n return Err(MorlocError::Packet(\"payload extends past packet end\".into()));\n }\n Ok(&packet[start..end])\n}\n\n/// Extract error message from a fail packet.\npub fn get_error_message(packet: &[u8]) -> Result, MorlocError> {\n if packet.len() < 32 {\n return Err(MorlocError::Packet(\"packet too small\".into()));\n }\n let header = PacketHeader::from_bytes(packet[..32].try_into().unwrap())?;\n if !header.is_fail() {\n return Ok(None);\n }\n let payload = get_data_payload(packet)?;\n Ok(Some(String::from_utf8_lossy(payload).into_owned()))\n}\n\n/// Read the schema string from packet metadata section.\npub fn read_schema_from_meta(packet: &[u8]) -> Result, MorlocError> {\n if packet.len() < 32 {\n return Err(MorlocError::Packet(\"packet too small\".into()));\n }\n let header = PacketHeader::from_bytes(packet[..32].try_into().unwrap())?;\n let offset = { header.offset } as usize;\n if offset == 0 {\n return Ok(None);\n }\n\n // Scan metadata headers\n let meta_start = 32;\n let meta_end = meta_start + offset;\n let mut pos = meta_start;\n while pos + 8 <= meta_end {\n if packet[pos] == b'm' && packet[pos + 1] == b'm' && packet[pos + 2] == b'h' {\n let meta_type = packet[pos + 3];\n let meta_size = u32::from_le_bytes([\n packet[pos + 4], packet[pos + 5], packet[pos + 6], packet[pos + 7],\n ]) as usize;\n if meta_type == METADATA_TYPE_SCHEMA_STRING {\n let str_start = pos + 8;\n let str_end = str_start + meta_size;\n if str_end <= meta_end {\n let bytes = &packet[str_start..str_end];\n // Find null terminator\n let len = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());\n return Ok(Some(String::from_utf8_lossy(&bytes[..len]).into_owned()));\n }\n }\n pos += 8 + meta_size;\n } else {\n break;\n }\n }\n Ok(None)\n}\n\n/// Get the voidstar value from a data packet (resolves relptr to absptr).\npub fn get_data_value(\n packet: &[u8],\n schema: &crate::Schema,\n) -> Result {\n let header = PacketHeader::from_bytes(packet[..32].try_into().unwrap())?;\n let source = unsafe { header.command.data.source };\n let format = unsafe { header.command.data.format };\n\n let payload = get_data_payload(packet)?;\n\n match source {\n PACKET_SOURCE_RPTR => {\n // Payload is a relptr\n if payload.len() < std::mem::size_of::() {\n return Err(MorlocError::Packet(\"relptr payload too small\".into()));\n }\n let relptr = crate::shm::RelPtr::from_ne_bytes(\n payload[..std::mem::size_of::()].try_into().unwrap()\n );\n crate::shm::rel2abs(relptr)\n }\n PACKET_SOURCE_MESG => {\n match format {\n PACKET_FORMAT_MSGPACK => {\n crate::mpack::unpack_with_schema(payload, schema)\n }\n PACKET_FORMAT_JSON => {\n let json_str = std::str::from_utf8(payload)\n .map_err(|e| MorlocError::Packet(format!(\"invalid UTF-8: {}\", e)))?;\n crate::json::read_json_with_schema(json_str, schema)\n }\n PACKET_FORMAT_VOIDSTAR => {\n read_voidstar_binary(payload, schema)\n }\n _ => {\n Err(MorlocError::Packet(format!(\n \"unsupported data format: {}\", format\n )))\n }\n }\n }\n _ => Err(MorlocError::Packet(format!(\"unsupported source: {}\", source))),\n }\n}\n\n// ── Inline voidstar deserialization ─────────────────────────────────────────\n\n/// Read a flat voidstar binary blob into shared memory, adjusting relptrs.\nfn read_voidstar_binary(\n blob: &[u8],\n schema: &crate::Schema,\n) -> Result {\n use crate::shm;\n\n let base = shm::shmalloc(blob.len())?;\n unsafe { std::ptr::copy_nonoverlapping(blob.as_ptr(), base, blob.len()) };\n\n let base_rel = shm::abs2rel(base)?;\n adjust_voidstar_relptrs(base, schema, base_rel)?;\n Ok(base)\n}\n\n/// Adjust relptrs in a voidstar blob that was copied into SHM.\n/// The blob's internal relptrs are offsets from position 0 of the blob.\n/// Adding `base_rel` converts them to valid SHM relptrs.\nfn adjust_voidstar_relptrs(\n data: crate::shm::AbsPtr,\n schema: &crate::Schema,\n base_rel: crate::shm::RelPtr,\n) -> Result<(), MorlocError> {\n use crate::schema::SerialType;\n use crate::shm::{self, Array, Tensor};\n\n unsafe {\n match schema.serial_type {\n SerialType::String | SerialType::Array => {\n let arr = &mut *(data as *mut Array);\n arr.data += base_rel;\n // Recurse into elements if variable-width (strings are always fixed-width bytes)\n if !schema.parameters.is_empty() && !schema.parameters[0].is_fixed_width() {\n let arr_data = shm::rel2abs(arr.data)?;\n let elem_width = schema.parameters[0].width;\n for i in 0..arr.size {\n let elem = arr_data.add(i * elem_width);\n adjust_voidstar_relptrs(elem, &schema.parameters[0], base_rel)?;\n }\n }\n }\n SerialType::Tuple | SerialType::Map => {\n for i in 0..schema.parameters.len() {\n let child = data.add(schema.offsets[i]);\n adjust_voidstar_relptrs(child, &schema.parameters[i], base_rel)?;\n }\n }\n SerialType::Optional => {\n let tag = *data;\n if tag != 0 {\n let inner_offset = schema.offsets.first().copied().unwrap_or(\n shm::align_up(1, schema.parameters[0].alignment().max(1)),\n );\n let child = data.add(inner_offset);\n adjust_voidstar_relptrs(child, &schema.parameters[0], base_rel)?;\n }\n }\n SerialType::Tensor => {\n let tensor = &mut *(data as *mut Tensor);\n if tensor.total_elements > 0 {\n tensor.shape += base_rel;\n tensor.data += base_rel;\n }\n }\n _ => {} // Fixed-width primitives: no relptrs to adjust\n }\n }\n Ok(())\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_header_size() {\n assert_eq!(std::mem::size_of::(), 32);\n }\n\n #[test]\n fn test_ping_roundtrip() {\n let ping = PacketHeader::ping();\n assert!(ping.is_valid());\n assert!(ping.is_ping());\n let bytes = ping.to_bytes();\n let recovered = PacketHeader::from_bytes(&bytes).unwrap();\n assert!(recovered.is_ping());\n }\n\n #[test]\n fn test_call_packet() {\n let call = PacketHeader::local_call(42, 128);\n assert!(call.is_call());\n assert!(call.is_local_call());\n assert!(!call.is_remote_call());\n let bytes = call.to_bytes();\n let recovered = PacketHeader::from_bytes(&bytes).unwrap();\n assert!(recovered.is_local_call());\n let midx = unsafe { recovered.command.call.midx };\n assert_eq!(midx, 42);\n let len = { recovered.length };\n assert_eq!(len, 128);\n }\n\n #[test]\n fn test_fail_packet() {\n let fail = PacketHeader::fail(100);\n assert!(fail.is_data());\n assert!(fail.is_fail());\n }\n\n #[test]\n fn test_data_mesg() {\n let data = PacketHeader::data_mesg(PACKET_FORMAT_MSGPACK, 256);\n assert!(data.is_data());\n assert!(!data.is_fail());\n let fmt = unsafe { data.command.data.format };\n assert_eq!(fmt, PACKET_FORMAT_MSGPACK);\n let len = { data.length };\n assert_eq!(len, 256);\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/packet_ffi.rs", "content": "//! C ABI wrappers for packet functions.\n//! Replaces packet.c with calls to Rust packet.rs + voidstar.rs.\n\nuse std::ffi::{c_char, c_void, CStr};\nuse std::ptr;\n\nuse crate::cschema::CSchema;\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::packet::*;\nuse crate::shm::{self, AbsPtr, RelPtr};\n\n// ── morloc_call_t ────────────────────────────────────────────────────────────\n\n/// Matches C `morloc_call_t` layout.\n#[repr(C)]\npub struct MorlocCall {\n pub midx: u32,\n pub args: *mut *mut u8,\n pub nargs: usize,\n pub owns_args: i32,\n}\n\n// ── Header reading ───────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_morloc_packet_header(\n msg: *const u8,\n errmsg: *mut *mut c_char,\n) -> *mut PacketHeader {\n clear_errmsg(errmsg);\n if msg.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"Cannot make packet from NULL pointer\".into()));\n return ptr::null_mut();\n }\n // Validate magic\n let header = &*(msg as *const PacketHeader);\n if !header.is_valid() {\n set_errmsg(errmsg, &MorlocError::Packet(\"Malformed morloc packet\".into()));\n return ptr::null_mut();\n }\n msg as *mut PacketHeader\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn packet_is_ping(\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() { return false; }\n (*header).is_ping()\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn packet_is_local_call(\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() { return false; }\n (*header).is_local_call()\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn packet_is_remote_call(\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() { return false; }\n (*header).is_remote_call()\n}\n\n// ── Packet size ──────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn morloc_packet_size_from_header(\n header: *const PacketHeader,\n) -> usize {\n if header.is_null() { return 0; }\n 32 + (*header).offset as usize + (*header).length as usize\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn morloc_packet_size(\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> usize {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() { return 0; }\n morloc_packet_size_from_header(header)\n}\n\n// ── Ping ─────────────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn return_ping(\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n if !packet_is_ping(packet, errmsg) {\n if (*errmsg).is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"Not a ping packet\".into()));\n }\n return ptr::null_mut();\n }\n let size = morloc_packet_size(packet, errmsg);\n if size == 0 { return ptr::null_mut(); }\n let buf = libc::calloc(size, 1) as *mut u8;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"Failed to allocate ping response\".into()));\n return ptr::null_mut();\n }\n ptr::copy_nonoverlapping(packet, buf, size);\n buf\n}\n\n#[no_mangle]\npub extern \"C\" fn make_ping_packet() -> *mut u8 {\n // SAFETY: calloc returns null or a valid pointer to 32 zeroed bytes.\n let buf = unsafe { libc::calloc(32, 1) as *mut u8 };\n if buf.is_null() { return ptr::null_mut(); }\n let header = PacketHeader::ping();\n let bytes = header.to_bytes();\n // SAFETY: buf points to 32 bytes; bytes is exactly 32 bytes from to_bytes().\n unsafe { ptr::copy_nonoverlapping(bytes.as_ptr(), buf, 32) };\n buf\n}\n\n// ── Data packet construction helpers ─────────────────────────────────────────\n\n/// Build metadata section: metadata header + schema string, padded to 32-byte boundary.\n/// Returns (metadata_buffer, padded_length). Returns (null, 0) if schema is null.\n///\n/// # Safety\n/// `schema` must be null or a valid CSchema pointer.\nunsafe fn build_schema_metadata(schema: *const CSchema) -> (*mut u8, usize) {\n if schema.is_null() {\n return (ptr::null_mut(), 0);\n }\n let rs = CSchema::to_rust(schema);\n let schema_str = crate::schema::schema_to_string(&rs);\n let schema_bytes = schema_str.as_bytes();\n let schema_len = schema_bytes.len() + 1; // +1 for null terminator\n let meta_header_size = 8; // sizeof(morloc_metadata_header_t)\n let raw_meta_len = meta_header_size + schema_len;\n let padded_meta_len = ((raw_meta_len + 31) / 32) * 32;\n\n let metadata = libc::calloc(padded_meta_len, 1) as *mut u8;\n if metadata.is_null() {\n return (ptr::null_mut(), 0);\n }\n\n // Write metadata header\n *metadata = b'm';\n *metadata.add(1) = b'm';\n *metadata.add(2) = b'h';\n *metadata.add(3) = METADATA_TYPE_SCHEMA_STRING;\n *(metadata.add(4) as *mut u32) = schema_len as u32;\n\n // Write schema string\n ptr::copy_nonoverlapping(schema_bytes.as_ptr(), metadata.add(meta_header_size), schema_bytes.len());\n // Null terminator already zeroed by calloc\n\n (metadata, padded_meta_len)\n}\n\n/// Generic data packet builder matching C's make_morloc_data_packet.\n///\n/// # Safety\n/// If non-null, `data` must point to `data_length` readable bytes.\n/// If non-null, `metadata` must point to `metadata_length` readable bytes.\nunsafe fn make_data_packet_raw(\n data: *const u8,\n data_length: usize,\n metadata: *const u8,\n metadata_length: usize,\n src: u8,\n fmt: u8,\n cmpr: u8,\n encr: u8,\n status: u8,\n) -> *mut u8 {\n let total = 32 + metadata_length + data_length;\n let packet = libc::calloc(total, 1) as *mut u8;\n if packet.is_null() { return ptr::null_mut(); }\n\n // Build command\n let cmd = CommandData {\n cmd_type: PACKET_TYPE_DATA,\n source: src,\n format: fmt,\n compression: cmpr,\n encryption: encr,\n status,\n padding: [0; 2],\n };\n let header = PacketHeader {\n magic: PACKET_MAGIC,\n plain: THIS_PLAIN,\n version: THIS_VERSION,\n flavor: DEFAULT_FLAVOR,\n mode: DEFAULT_MODE,\n command: PacketCommand { data: cmd },\n offset: metadata_length as u32,\n length: data_length as u64,\n };\n let hdr_bytes = header.to_bytes();\n ptr::copy_nonoverlapping(hdr_bytes.as_ptr(), packet, 32);\n\n if !metadata.is_null() && metadata_length > 0 {\n ptr::copy_nonoverlapping(metadata, packet.add(32), metadata_length);\n }\n if !data.is_null() && data_length > 0 {\n ptr::copy_nonoverlapping(data, packet.add(32 + metadata_length), data_length);\n }\n\n packet\n}\n\n/// Generic data packet with schema metadata.\nunsafe fn make_data_packet_with_schema(\n data: *const u8,\n data_length: usize,\n schema: *const CSchema,\n src: u8,\n fmt: u8,\n cmpr: u8,\n encr: u8,\n status: u8,\n) -> *mut u8 {\n let (metadata, metadata_length) = build_schema_metadata(schema);\n let result = make_data_packet_raw(\n data, data_length, metadata, metadata_length, src, fmt, cmpr, encr, status,\n );\n if !metadata.is_null() {\n libc::free(metadata as *mut c_void);\n }\n result\n}\n\n// ── Standard data packet (RPTR + VOIDSTAR) ──────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_standard_data_packet(\n relptr: RelPtr,\n schema: *const CSchema,\n) -> *mut u8 {\n let packet = make_data_packet_with_schema(\n ptr::null(),\n std::mem::size_of::(),\n schema,\n PACKET_SOURCE_RPTR,\n PACKET_FORMAT_VOIDSTAR,\n PACKET_COMPRESSION_NONE,\n PACKET_ENCRYPTION_NONE,\n PACKET_STATUS_PASS,\n );\n if packet.is_null() { return ptr::null_mut(); }\n\n // Write the relptr into the payload area\n let header = &*(packet as *const PacketHeader);\n let payload_offset = 32 + header.offset as usize;\n *(packet.add(payload_offset) as *mut RelPtr) = relptr;\n\n packet\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_arrow_data_packet(\n relptr: RelPtr,\n schema: *const CSchema,\n) -> *mut u8 {\n let packet = make_data_packet_with_schema(\n ptr::null(),\n std::mem::size_of::(),\n schema,\n PACKET_SOURCE_RPTR,\n PACKET_FORMAT_ARROW,\n PACKET_COMPRESSION_NONE,\n PACKET_ENCRYPTION_NONE,\n PACKET_STATUS_PASS,\n );\n if packet.is_null() { return ptr::null_mut(); }\n\n let header = &*(packet as *const PacketHeader);\n let payload_offset = 32 + header.offset as usize;\n *(packet.add(payload_offset) as *mut RelPtr) = relptr;\n\n packet\n}\n\n// ── Msgpack packets ──────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_mpk_data_packet(\n mpk_filename: *const c_char,\n schema: *const CSchema,\n) -> *mut u8 {\n if mpk_filename.is_null() { return ptr::null_mut(); }\n let filename = CStr::from_ptr(mpk_filename);\n let bytes = filename.to_bytes();\n make_data_packet_with_schema(\n bytes.as_ptr(),\n bytes.len(),\n schema,\n PACKET_SOURCE_FILE,\n PACKET_FORMAT_MSGPACK,\n PACKET_COMPRESSION_NONE,\n PACKET_ENCRYPTION_NONE,\n PACKET_STATUS_PASS,\n )\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_data_packet_from_mpk(\n mpk: *const c_char,\n mpk_size: usize,\n schema: *const CSchema,\n) -> *mut u8 {\n make_data_packet_with_schema(\n mpk as *const u8,\n mpk_size,\n schema,\n PACKET_SOURCE_MESG,\n PACKET_FORMAT_MSGPACK,\n PACKET_COMPRESSION_NONE,\n PACKET_ENCRYPTION_NONE,\n PACKET_STATUS_PASS,\n )\n}\n\n// ── get_data_packet_as_mpk ───────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn get_data_packet_as_mpk(\n packet: *const u8,\n schema: *const CSchema,\n mpk_out: *mut *mut c_char,\n mpk_size_out: *mut usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n *mpk_out = ptr::null_mut();\n *mpk_size_out = 0;\n\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() { return 0; }\n\n if (*header).command_type() != PACKET_TYPE_DATA {\n set_errmsg(errmsg, &MorlocError::Packet(\"Expected a data packet\".into()));\n return 0;\n }\n\n // Check for error\n if (*header).is_fail() {\n let payload_start = 32 + (*header).offset as usize;\n let payload_len = (*header).length as usize;\n let msg = std::str::from_utf8_unchecked(\n std::slice::from_raw_parts(packet.add(payload_start), payload_len)\n );\n set_errmsg(errmsg, &MorlocError::Packet(format!(\"\\n{}\", msg)));\n return 0;\n }\n\n let rs = CSchema::to_rust(schema);\n let source = (*header).command.data.source;\n let format = (*header).command.data.format;\n let payload_start = 32 + (*header).offset as usize;\n let payload_len = (*header).length as usize;\n let payload = std::slice::from_raw_parts(packet.add(payload_start), payload_len);\n\n if source == PACKET_SOURCE_MESG && format == PACKET_FORMAT_MSGPACK {\n // Inline msgpack: copy directly\n let buf = libc::malloc(payload_len) as *mut c_char;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"malloc failed\".into()));\n return 0;\n }\n ptr::copy_nonoverlapping(payload.as_ptr(), buf as *mut u8, payload_len);\n *mpk_out = buf;\n *mpk_size_out = payload_len;\n } else if source == PACKET_SOURCE_MESG && format == PACKET_FORMAT_VOIDSTAR {\n // Inline voidstar: load into SHM then convert to msgpack\n match crate::voidstar::read_binary(payload, &rs) {\n Ok(abs) => {\n match crate::mpack::pack_with_schema(abs, &rs) {\n Ok(data) => {\n let buf = libc::malloc(data.len()) as *mut u8;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"malloc failed\".into()));\n return 0;\n }\n ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len());\n *mpk_out = buf as *mut c_char;\n *mpk_size_out = data.len();\n }\n Err(e) => { set_errmsg(errmsg, &e); return 0; }\n }\n // Free SHM\n let _ = crate::voidstar::free_by_schema(abs, &rs);\n let _ = shm::shfree(abs);\n }\n Err(e) => { set_errmsg(errmsg, &e); return 0; }\n }\n } else if source == PACKET_SOURCE_FILE && format == PACKET_FORMAT_MSGPACK {\n // File-based msgpack: read the file\n let filename_bytes = &payload[..payload_len.min(4096)];\n let filename = std::str::from_utf8(filename_bytes).unwrap_or(\"\");\n let filename = filename.trim_end_matches('\\0');\n match std::fs::read(filename) {\n Ok(data) => {\n let buf = libc::malloc(data.len()) as *mut u8;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"malloc failed\".into()));\n return 0;\n }\n ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len());\n *mpk_out = buf as *mut c_char;\n *mpk_size_out = data.len();\n }\n Err(e) => {\n set_errmsg(errmsg, &MorlocError::Io(e));\n return 0;\n }\n }\n } else if source == PACKET_SOURCE_RPTR && format == PACKET_FORMAT_VOIDSTAR {\n // Voidstar via relptr: convert to msgpack\n let relptr = *(payload.as_ptr() as *const RelPtr);\n match shm::rel2abs(relptr) {\n Ok(abs) => {\n match crate::mpack::pack_with_schema(abs, &rs) {\n Ok(data) => {\n let buf = libc::malloc(data.len()) as *mut u8;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"malloc failed\".into()));\n return 0;\n }\n ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len());\n *mpk_out = buf as *mut c_char;\n *mpk_size_out = data.len();\n }\n Err(e) => { set_errmsg(errmsg, &e); return 0; }\n }\n }\n Err(e) => { set_errmsg(errmsg, &e); return 0; }\n }\n } else {\n set_errmsg(errmsg, &MorlocError::Packet(\n format!(\"Unsupported packet source/format: 0x{:02x}/0x{:02x}\", source, format)\n ));\n return 0;\n }\n\n 1 // true\n}\n\n// ── Schema from metadata ─────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_schema_from_packet_meta(\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() { return ptr::null_mut(); }\n\n let offset = (*header).offset as usize;\n if offset < 8 { return ptr::null_mut(); } // no room for metadata header\n\n let meta_start = 32usize;\n let meta_end = meta_start + offset;\n let mut pos = meta_start;\n while pos + 8 <= meta_end {\n if *packet.add(pos) == b'm' && *packet.add(pos + 1) == b'm' && *packet.add(pos + 2) == b'h' {\n let meta_type = *packet.add(pos + 3);\n let meta_size = *(packet.add(pos + 4) as *const u32) as usize;\n if meta_type == METADATA_TYPE_SCHEMA_STRING {\n // Return pointer into the packet buffer (matches C behavior)\n return packet.add(pos + 8) as *mut c_char;\n }\n pos += 8 + meta_size;\n } else {\n break;\n }\n }\n ptr::null_mut()\n}\n\n// ── Fail packet ──────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_fail_packet(\n failure_message: *const c_char,\n) -> *mut u8 {\n if failure_message.is_null() { return ptr::null_mut(); }\n let msg = CStr::from_ptr(failure_message).to_bytes();\n make_data_packet_raw(\n msg.as_ptr(),\n msg.len(),\n ptr::null(),\n 0,\n PACKET_SOURCE_MESG,\n PACKET_FORMAT_TEXT,\n PACKET_COMPRESSION_NONE,\n PACKET_ENCRYPTION_NONE,\n PACKET_STATUS_FAIL,\n )\n}\n\n// ── Error message extraction ─────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn get_morloc_data_packet_error_message(\n data: *const u8,\n errmsg: *mut *mut c_char,\n) -> *mut c_char {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(data, errmsg);\n if header.is_null() { return ptr::null_mut(); }\n\n if (*header).is_fail() {\n let payload_start = 32 + (*header).offset as usize;\n let payload_len = (*header).length as usize;\n let buf = libc::calloc(payload_len + 1, 1) as *mut c_char;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"Failed to allocate error message\".into()));\n return ptr::null_mut();\n }\n ptr::copy_nonoverlapping(data.add(payload_start), buf as *mut u8, payload_len);\n return buf;\n }\n\n ptr::null_mut()\n}\n\n// ── get_morloc_data_packet_value ─────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn get_morloc_data_packet_value(\n data: *const u8,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(data, errmsg);\n if header.is_null() { return ptr::null_mut(); }\n\n if (*header).command_type() != PACKET_TYPE_DATA {\n set_errmsg(errmsg, &MorlocError::Packet(\"Expected a data packet\".into()));\n return ptr::null_mut();\n }\n\n // Check for error\n let packet_error = get_morloc_data_packet_error_message(data, errmsg);\n if !packet_error.is_null() {\n let err_str = CStr::from_ptr(packet_error).to_string_lossy().into_owned();\n libc::free(packet_error as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Packet(format!(\"\\n{}\", err_str)));\n return ptr::null_mut();\n }\n // Clear any errmsg from get_morloc_data_packet_error_message\n clear_errmsg(errmsg);\n\n let rs = CSchema::to_rust(schema);\n let source = (*header).command.data.source;\n let format = (*header).command.data.format;\n let payload_start = 32 + (*header).offset as usize;\n let payload_len = (*header).length as usize;\n\n match source {\n PACKET_SOURCE_MESG => {\n if format == PACKET_FORMAT_MSGPACK {\n let payload = std::slice::from_raw_parts(data.add(payload_start), payload_len);\n match crate::mpack::unpack_with_schema(payload, &rs) {\n Ok(abs) => abs,\n Err(e) => { set_errmsg(errmsg, &e); ptr::null_mut() }\n }\n } else if format == PACKET_FORMAT_VOIDSTAR {\n let payload = std::slice::from_raw_parts(data.add(payload_start), payload_len);\n match crate::voidstar::read_binary(payload, &rs) {\n Ok(abs) => abs,\n Err(e) => { set_errmsg(errmsg, &e); ptr::null_mut() }\n }\n } else {\n set_errmsg(errmsg, &MorlocError::Packet(\n format!(\"Invalid format from mesg: 0x{:02x}\", format)\n ));\n ptr::null_mut()\n }\n }\n PACKET_SOURCE_FILE => {\n if format == PACKET_FORMAT_MSGPACK {\n let filename_bytes = std::slice::from_raw_parts(data.add(payload_start), payload_len.min(4096));\n let filename = std::str::from_utf8(filename_bytes).unwrap_or(\"\");\n let filename = filename.trim_end_matches('\\0');\n match std::fs::read(filename) {\n Ok(file_data) => {\n match crate::mpack::unpack_with_schema(&file_data, &rs) {\n Ok(abs) => abs,\n Err(e) => { set_errmsg(errmsg, &e); ptr::null_mut() }\n }\n }\n Err(e) => {\n set_errmsg(errmsg, &MorlocError::Io(e));\n ptr::null_mut()\n }\n }\n } else {\n set_errmsg(errmsg, &MorlocError::Packet(\n format!(\"Invalid format from file: 0x{:02x}\", format)\n ));\n ptr::null_mut()\n }\n }\n PACKET_SOURCE_RPTR => {\n if format == PACKET_FORMAT_VOIDSTAR || format == PACKET_FORMAT_ARROW {\n let relptr = *(data.add(payload_start) as *const RelPtr);\n match shm::rel2abs(relptr) {\n Ok(abs) => abs,\n Err(e) => { set_errmsg(errmsg, &e); ptr::null_mut() }\n }\n } else {\n set_errmsg(errmsg, &MorlocError::Packet(\n format!(\"For RPTR source, expected voidstar or arrow format, found: 0x{:02x}\", format)\n ));\n ptr::null_mut()\n }\n }\n _ => {\n set_errmsg(errmsg, &MorlocError::Packet(\"Invalid source\".into()));\n ptr::null_mut()\n }\n }\n}\n\n// ── Call packet construction ─────────────────────────────────────────────────\n\nunsafe fn make_call_packet_gen(\n midx: u32,\n entrypoint: u8,\n arg_packets: *const *const u8,\n nargs: usize,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n\n // Calculate total data length\n let mut data_length: usize = 0;\n for i in 0..nargs {\n let arg = read_morloc_packet_header(*arg_packets.add(i), errmsg);\n if arg.is_null() { return ptr::null_mut(); }\n data_length += morloc_packet_size_from_header(arg);\n }\n\n let total = 32 + data_length;\n let packet = libc::calloc(total, 1) as *mut u8;\n if packet.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"Failed to allocate call packet\".into()));\n return ptr::null_mut();\n }\n\n let cmd = CommandCall {\n cmd_type: PACKET_TYPE_CALL,\n entrypoint,\n padding: [0; 2],\n midx,\n };\n let header = PacketHeader {\n magic: PACKET_MAGIC,\n plain: THIS_PLAIN,\n version: THIS_VERSION,\n flavor: DEFAULT_FLAVOR,\n mode: DEFAULT_MODE,\n command: PacketCommand { call: cmd },\n offset: 0,\n length: data_length as u64,\n };\n let hdr_bytes = header.to_bytes();\n ptr::copy_nonoverlapping(hdr_bytes.as_ptr(), packet, 32);\n\n let mut pos = 32;\n for i in 0..nargs {\n let arg = read_morloc_packet_header(*arg_packets.add(i), errmsg);\n if arg.is_null() {\n libc::free(packet as *mut c_void);\n return ptr::null_mut();\n }\n let arg_size = morloc_packet_size_from_header(arg);\n ptr::copy_nonoverlapping(*arg_packets.add(i), packet.add(pos), arg_size);\n pos += arg_size;\n }\n\n packet\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_morloc_local_call_packet(\n midx: u32,\n arg_packets: *const *const u8,\n nargs: usize,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n make_call_packet_gen(midx, PACKET_ENTRYPOINT_LOCAL, arg_packets, nargs, errmsg)\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_morloc_remote_call_packet(\n midx: u32,\n arg_packets: *const *const u8,\n nargs: usize,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n make_call_packet_gen(midx, PACKET_ENTRYPOINT_REMOTE_SFS, arg_packets, nargs, errmsg)\n}\n\n// ── Call packet reading ──────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_morloc_call_packet(\n packet: *const u8,\n errmsg: *mut *mut c_char,\n) -> *mut MorlocCall {\n clear_errmsg(errmsg);\n\n let call = libc::calloc(1, std::mem::size_of::()) as *mut MorlocCall;\n if call.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"calloc failed\".into()));\n return ptr::null_mut();\n }\n\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() {\n libc::free(call as *mut c_void);\n return ptr::null_mut();\n }\n if (*header).command_type() != PACKET_TYPE_CALL {\n set_errmsg(errmsg, &MorlocError::Packet(\"Expected packet to be a call\".into()));\n libc::free(call as *mut c_void);\n return ptr::null_mut();\n }\n\n (*call).midx = (*header).command.call.midx;\n (*call).nargs = 0;\n (*call).args = ptr::null_mut();\n (*call).owns_args = 0; // borrowing pointers into packet\n\n let start_pos = 32 + (*header).offset as usize;\n let end_pos = start_pos + (*header).length as usize;\n\n // First pass: count args\n let mut pos = start_pos;\n while pos < end_pos {\n let arg_size = morloc_packet_size(packet.add(pos), errmsg);\n if arg_size == 0 {\n free_morloc_call(call);\n return ptr::null_mut();\n }\n pos += arg_size;\n (*call).nargs += 1;\n }\n\n // Allocate args array\n (*call).args = libc::calloc((*call).nargs, std::mem::size_of::<*mut u8>()) as *mut *mut u8;\n if (*call).args.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"calloc failed for args\".into()));\n libc::free(call as *mut c_void);\n return ptr::null_mut();\n }\n\n // Second pass: fill pointers (borrowing into original packet)\n pos = start_pos;\n for i in 0..(*call).nargs {\n let arg_header = read_morloc_packet_header(packet.add(pos), errmsg);\n if arg_header.is_null() {\n free_morloc_call(call);\n return ptr::null_mut();\n }\n if (*arg_header).command_type() != PACKET_TYPE_DATA {\n set_errmsg(errmsg, &MorlocError::Packet(\n format!(\"Argument #{} is not a DATA packet (type={})\", i, (*arg_header).command_type())\n ));\n free_morloc_call(call);\n return ptr::null_mut();\n }\n *(*call).args.add(i) = packet.add(pos) as *mut u8;\n pos += morloc_packet_size_from_header(arg_header);\n }\n\n call\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn free_morloc_call(call: *mut MorlocCall) {\n if call.is_null() { return; }\n let c = &*call;\n if !c.args.is_null() {\n if c.owns_args != 0 {\n for i in 0..c.nargs {\n let arg = *c.args.add(i);\n if !arg.is_null() {\n libc::free(arg as *mut c_void);\n }\n }\n }\n libc::free(c.args as *mut c_void);\n }\n libc::free(call as *mut c_void);\n}\n\n// adjust_voidstar_relptrs: still provided by cli.c (will move to Rust when cli.c is ported)\n// read_voidstar_binary: still provided by cli.c (will move to Rust when cli.c is ported)\n\n// ── write_voidstar_binary (for intrinsics.c) ─────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn write_voidstar_binary(\n fd: i32,\n data: *const c_void,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> RelPtr {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n match crate::voidstar::write_binary_to_fd(fd, data as AbsPtr, &rs) {\n Ok(n) => n as RelPtr,\n Err(e) => {\n set_errmsg(errmsg, &e);\n -1isize as RelPtr\n }\n }\n}\n\n// ── flatten_voidstar_to_buffer ───────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn flatten_voidstar_to_buffer(\n data: *const c_void,\n schema: *const CSchema,\n out_buf: *mut *mut u8,\n out_size: *mut usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n *out_buf = ptr::null_mut();\n *out_size = 0;\n\n let rs = CSchema::to_rust(schema);\n match crate::voidstar::flatten_to_buffer(data as AbsPtr, &rs) {\n Ok(buf) => {\n let len = buf.len();\n let c_buf = libc::malloc(len) as *mut u8;\n if c_buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"malloc failed\".into()));\n return 1;\n }\n ptr::copy_nonoverlapping(buf.as_ptr(), c_buf, len);\n *out_buf = c_buf;\n *out_size = len;\n 0\n }\n Err(e) => {\n set_errmsg(errmsg, &e);\n 1\n }\n }\n}\n\n// read_voidstar_binary: still provided by cli.c (will move to Rust when cli.c is ported)\n\n// ── make_data_packet_auto ────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn make_data_packet_auto(\n voidstar: *mut c_void,\n relptr: RelPtr,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n let rs = CSchema::to_rust(schema);\n\n let flat_size = match crate::ffi::calc_voidstar_size_inner(voidstar as *const u8, &rs) {\n Ok(s) => s,\n Err(e) => {\n set_errmsg(errmsg, &e);\n return ptr::null_mut();\n }\n };\n\n if flat_size <= MORLOC_INLINE_THRESHOLD {\n match crate::voidstar::flatten_to_buffer(voidstar as AbsPtr, &rs) {\n Ok(blob) => {\n let packet = make_data_packet_with_schema(\n blob.as_ptr(),\n blob.len(),\n schema,\n PACKET_SOURCE_MESG,\n PACKET_FORMAT_VOIDSTAR,\n PACKET_COMPRESSION_NONE,\n PACKET_ENCRYPTION_NONE,\n PACKET_STATUS_PASS,\n );\n if packet.is_null() {\n set_errmsg(errmsg, &MorlocError::Packet(\"Failed to create inline data packet\".into()));\n }\n return packet;\n }\n Err(e) => {\n set_errmsg(errmsg, &e);\n return ptr::null_mut();\n }\n }\n }\n\n make_standard_data_packet(relptr, schema)\n}\n\n// ── print_morloc_data_packet ─────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn print_morloc_data_packet(\n packet: *const u8,\n schema: *const CSchema,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n let header = read_morloc_packet_header(packet, errmsg);\n if header.is_null() { return 1; }\n\n if (*header).command_type() != PACKET_TYPE_DATA {\n set_errmsg(errmsg, &MorlocError::Packet(\"Expected a data packet\".into()));\n return 1;\n }\n\n // Check for error\n let packet_error = get_morloc_data_packet_error_message(packet, errmsg);\n if !packet_error.is_null() {\n let err_str = CStr::from_ptr(packet_error).to_string_lossy().into_owned();\n libc::free(packet_error as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Packet(format!(\"\\n{}\", err_str)));\n return 1;\n }\n clear_errmsg(errmsg);\n\n let rs = CSchema::to_rust(schema);\n let source = (*header).command.data.source;\n let format = (*header).command.data.format;\n let packet_size = morloc_packet_size_from_header(header);\n\n match source {\n PACKET_SOURCE_MESG | PACKET_SOURCE_FILE => {\n // Print the raw packet bytes\n if print_binary(packet, packet_size, errmsg) != 0 {\n return 1;\n }\n }\n PACKET_SOURCE_RPTR => {\n match format {\n PACKET_FORMAT_VOIDSTAR => {\n let payload_start = 32 + (*header).offset as usize;\n let relptr = *(packet.add(payload_start) as *const RelPtr);\n let voidstar_ptr = match shm::rel2abs(relptr) {\n Ok(p) => p,\n Err(e) => { set_errmsg(errmsg, &e); return 1; }\n };\n\n // Build modified header with flat size\n let flat_size = match crate::ffi::calc_voidstar_size_inner(voidstar_ptr, &rs) {\n Ok(s) => s,\n Err(e) => { set_errmsg(errmsg, &e); return 1; }\n };\n\n let mut new_header = *header;\n new_header.command.data.format = PACKET_FORMAT_VOIDSTAR;\n // Safely set length (packed struct)\n let new_hdr_ptr = &mut new_header as *mut PacketHeader as *mut u8;\n *(new_hdr_ptr.add(24) as *mut u64) = flat_size as u64;\n\n // Print header\n if print_binary(&new_header as *const PacketHeader as *const u8, 32, errmsg) != 0 {\n return 1;\n }\n\n // Print metadata\n let offset = (*header).offset as usize;\n if offset > 0 {\n if print_binary(packet.add(32), offset, errmsg) != 0 {\n return 1;\n }\n }\n\n // Write flattened voidstar data to stdout\n match crate::voidstar::write_binary_to_fd(libc::STDOUT_FILENO, voidstar_ptr, &rs) {\n Ok(_) => {}\n Err(e) => { set_errmsg(errmsg, &e); return 1; }\n }\n }\n _ => {\n // Other formats: print raw packet\n if print_binary(packet, packet_size, errmsg) != 0 {\n return 1;\n }\n }\n }\n }\n _ => {\n set_errmsg(errmsg, &MorlocError::Packet(\"Invalid source\".into()));\n return 1;\n }\n }\n\n 0 // EXIT_PASS\n}\n\n/// Write binary data to stdout.\n///\n/// # Safety\n/// `buf` must point to at least `count` readable bytes.\nunsafe fn print_binary(\n buf: *const u8,\n count: usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n let mut written: usize = 0;\n while written < count {\n let n = libc::write(\n libc::STDOUT_FILENO,\n buf.add(written) as *const c_void,\n count - written,\n );\n if n < 0 {\n set_errmsg(errmsg, &MorlocError::Io(std::io::Error::last_os_error()));\n return 1;\n }\n written += n as usize;\n }\n 0\n}\n" }, { "path": "data/rust/morloc-runtime/src/pool_ffi.rs", "content": "//! Pool server lifecycle: accept connections, dispatch packets, manage workers.\n//! Replaces pool.c. Uses std::thread instead of raw pthreads for thread mode.\n\nuse std::ffi::{c_char, c_void};\nuse std::ptr;\nuse std::sync::atomic::{AtomicBool, AtomicI32, Ordering};\nuse std::sync::{Arc, Mutex, Condvar};\n\n// ── C-compatible types matching pool.h ───────────────────────────────────────\n\npub type PoolDispatchFn = unsafe extern \"C\" fn(\n mid: u32, args: *const *const u8, nargs: usize, ctx: *mut c_void,\n) -> *mut u8;\n\n#[repr(C)]\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum PoolConcurrency {\n Threads = 0,\n Fork = 1,\n Single = 2,\n}\n\n#[repr(C)]\npub struct PoolConfig {\n pub local_dispatch: PoolDispatchFn,\n pub remote_dispatch: PoolDispatchFn,\n pub dispatch_ctx: *mut c_void,\n pub concurrency: PoolConcurrency,\n pub initial_workers: i32,\n pub dynamic_scaling: bool,\n pub post_fork_child: Option,\n}\n\n// SAFETY: PoolConfig contains function pointers and a *mut c_void dispatch_ctx.\n// The function pointers are set once at startup and never mutated.\n// dispatch_ctx points to language-runtime state that is either thread-local\n// (fork mode) or protected by the runtime's own synchronization (thread mode).\n// The pool architecture guarantees dispatch_ctx is not concurrently mutated.\nunsafe impl Send for PoolConfig {}\nunsafe impl Sync for PoolConfig {}\n\n// ── Global state ─────────────────────────────────────────────────────────────\n\nstatic SHUTTING_DOWN: AtomicBool = AtomicBool::new(false);\nstatic BUSY_COUNT: AtomicI32 = AtomicI32::new(0);\nstatic TOTAL_WORKERS: AtomicI32 = AtomicI32::new(0);\n\n// SAFETY: SHARED_BUSY is set once in pool_main_fork (parent process) before\n// forking children. After fork, each process accesses the mmap'd AtomicI32\n// via atomic operations only. Reset to null during shutdown.\nstatic mut SHARED_BUSY: *mut AtomicI32 = ptr::null_mut();\n\n#[no_mangle]\npub extern \"C\" fn pool_mark_busy() {\n // SAFETY: SHARED_BUSY is either null (thread mode, use local atomic) or a valid\n // mmap'd AtomicI32 pointer set during pool_main_fork initialization.\n unsafe {\n if !SHARED_BUSY.is_null() {\n (*SHARED_BUSY).fetch_add(1, Ordering::Relaxed);\n } else {\n BUSY_COUNT.fetch_add(1, Ordering::Relaxed);\n }\n }\n}\n\n#[no_mangle]\npub extern \"C\" fn pool_mark_idle() {\n // SAFETY: Same as pool_mark_busy - SHARED_BUSY is null or a valid mmap'd pointer.\n unsafe {\n if !SHARED_BUSY.is_null() {\n (*SHARED_BUSY).fetch_sub(1, Ordering::Relaxed);\n } else {\n BUSY_COUNT.fetch_sub(1, Ordering::Relaxed);\n }\n }\n}\n\nextern \"C\" fn pool_sigterm_handler(_sig: i32) {\n SHUTTING_DOWN.store(true, Ordering::Relaxed);\n}\n\n// ── Packet dispatch ──────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn pool_dispatch_packet(\n packet: *const u8,\n local_dispatch: PoolDispatchFn,\n remote_dispatch: PoolDispatchFn,\n ctx: *mut c_void,\n) -> *mut u8 {\n extern \"C\" {\n fn make_fail_packet(msg: *const c_char) -> *mut u8;\n fn packet_is_ping(packet: *const u8, errmsg: *mut *mut c_char) -> bool;\n fn return_ping(packet: *const u8, errmsg: *mut *mut c_char) -> *mut u8;\n fn packet_is_local_call(packet: *const u8, errmsg: *mut *mut c_char) -> bool;\n fn packet_is_remote_call(packet: *const u8, errmsg: *mut *mut c_char) -> bool;\n fn read_morloc_call_packet(packet: *const u8, errmsg: *mut *mut c_char) -> *mut crate::packet_ffi::MorlocCall;\n fn free_morloc_call(call: *mut crate::packet_ffi::MorlocCall);\n }\n\n if packet.is_null() {\n return make_fail_packet(b\"NULL packet in pool dispatch\\0\".as_ptr() as *const c_char);\n }\n\n let mut errmsg: *mut c_char = ptr::null_mut();\n\n if packet_is_ping(packet, &mut errmsg) {\n if !errmsg.is_null() { return fail_from_errmsg(errmsg); }\n let pong = return_ping(packet, &mut errmsg);\n if !errmsg.is_null() { return fail_from_errmsg(errmsg); }\n return pong;\n }\n if !errmsg.is_null() { return fail_from_errmsg(errmsg); }\n\n let is_local = packet_is_local_call(packet, &mut errmsg);\n if !errmsg.is_null() { return fail_from_errmsg(errmsg); }\n let is_remote = packet_is_remote_call(packet, &mut errmsg);\n if !errmsg.is_null() { return fail_from_errmsg(errmsg); }\n\n if is_local || is_remote {\n let call = read_morloc_call_packet(packet, &mut errmsg);\n if !errmsg.is_null() { return fail_from_errmsg(errmsg); }\n\n let mid = (*call).midx;\n let args = (*call).args as *const *const u8;\n let nargs = (*call).nargs;\n\n let dispatch_fn = if is_local { local_dispatch } else { remote_dispatch };\n let result = dispatch_fn(mid, args, nargs, ctx);\n\n free_morloc_call(call);\n\n if result.is_null() {\n return make_fail_packet(b\"dispatch callback returned NULL\\0\".as_ptr() as *const c_char);\n }\n return result;\n }\n\n make_fail_packet(b\"Unexpected packet type in pool dispatch\\0\".as_ptr() as *const c_char)\n}\n\nunsafe fn fail_from_errmsg(errmsg: *mut c_char) -> *mut u8 {\n extern \"C\" { fn make_fail_packet(msg: *const c_char) -> *mut u8; }\n let pkt = make_fail_packet(errmsg);\n libc::free(errmsg as *mut c_void);\n pkt\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────────\n\nunsafe fn try_send_fail(client_fd: i32, msg: *const c_char) {\n extern \"C\" {\n fn make_fail_packet(msg: *const c_char) -> *mut u8;\n fn send_packet_to_foreign_server(fd: i32, packet: *mut u8, errmsg: *mut *mut c_char) -> usize;\n }\n let fail = make_fail_packet(if msg.is_null() { b\"Unknown error\\0\".as_ptr() as *const c_char } else { msg });\n if !fail.is_null() {\n let mut err: *mut c_char = ptr::null_mut();\n send_packet_to_foreign_server(client_fd, fail, &mut err);\n libc::free(fail as *mut c_void);\n if !err.is_null() { libc::free(err as *mut c_void); }\n }\n}\n\n// ── Thread mode job queue ────────────────────────────────────────────────────\n\nstruct JobQueue {\n jobs: Mutex>,\n cond: Condvar,\n}\n\nimpl JobQueue {\n fn new() -> Self {\n JobQueue { jobs: Mutex::new(Vec::new()), cond: Condvar::new() }\n }\n\n fn push(&self, fd: i32) {\n let mut jobs = self.jobs.lock().unwrap();\n jobs.push(fd);\n self.cond.notify_one();\n }\n\n fn pop(&self) -> Option {\n let mut jobs = self.jobs.lock().unwrap();\n loop {\n if SHUTTING_DOWN.load(Ordering::Relaxed) { return None; }\n if let Some(fd) = jobs.pop() { return Some(fd); }\n let result = self.cond.wait_timeout(jobs, std::time::Duration::from_millis(100)).unwrap();\n jobs = result.0;\n }\n }\n}\n\n// ── Worker thread ────────────────────────────────────────────────────────────\n\nunsafe fn worker_loop(queue: &JobQueue, config: &PoolConfig) {\n extern \"C\" {\n fn stream_from_client(fd: i32, errmsg: *mut *mut c_char) -> *mut u8;\n fn send_packet_to_foreign_server(fd: i32, packet: *mut u8, errmsg: *mut *mut c_char) -> usize;\n fn close_socket(fd: i32);\n }\n\n while !SHUTTING_DOWN.load(Ordering::Relaxed) {\n let client_fd = match queue.pop() {\n Some(fd) => fd,\n None => break,\n };\n\n let mut errmsg: *mut c_char = ptr::null_mut();\n let data = stream_from_client(client_fd, &mut errmsg);\n if data.is_null() || !errmsg.is_null() {\n if !errmsg.is_null() {\n try_send_fail(client_fd, errmsg);\n libc::free(errmsg as *mut c_void);\n }\n libc::free(data as *mut c_void);\n close_socket(client_fd);\n continue;\n }\n\n // Track busy state so the accept loop can spawn new workers if needed\n pool_mark_busy();\n let result = pool_dispatch_packet(data, config.local_dispatch, config.remote_dispatch, config.dispatch_ctx);\n pool_mark_idle();\n libc::free(data as *mut c_void);\n\n if !result.is_null() {\n send_packet_to_foreign_server(client_fd, result, &mut errmsg);\n libc::free(result as *mut c_void);\n if !errmsg.is_null() { libc::free(errmsg as *mut c_void); }\n }\n\n libc::fflush(ptr::null_mut()); // flush stdout\n close_socket(client_fd);\n }\n}\n\n// ── Pool main: threads mode ──────────────────────────────────────────────────\n\nunsafe fn pool_main_threads(config: &PoolConfig, socket_path: *const c_char, tmpdir: *const c_char, shm_basename: *const c_char) -> i32 {\n extern \"C\" {\n fn start_daemon(socket_path: *const c_char, tmpdir: *const c_char, shm_basename: *const c_char, size: usize, errmsg: *mut *mut c_char) -> *mut c_void;\n fn close_daemon(daemon: *mut *mut c_void);\n fn wait_for_client_with_timeout(daemon: *mut c_void, timeout_us: i32, errmsg: *mut *mut c_char) -> i32;\n }\n\n let mut errmsg: *mut c_char = ptr::null_mut();\n let mut daemon = start_daemon(socket_path, tmpdir, shm_basename, 0xffff, &mut errmsg);\n if !errmsg.is_null() {\n libc::fprintf(libc::fdopen(2, b\"w\\0\".as_ptr() as *const c_char),\n b\"Failed to start language server:\\n%s\\n\\0\".as_ptr() as *const c_char, errmsg);\n libc::free(errmsg as *mut c_void);\n return 1;\n }\n\n let queue = Arc::new(JobQueue::new());\n let nthreads = config.initial_workers.max(1) as usize;\n TOTAL_WORKERS.store(nthreads as i32, Ordering::Relaxed);\n\n let mut handles = Vec::with_capacity(nthreads);\n for _ in 0..nthreads {\n let q = Arc::clone(&queue);\n let cfg = ptr::read(config); // Copy config for thread\n handles.push(std::thread::spawn(move || {\n worker_loop(&q, &cfg);\n }));\n }\n\n while !SHUTTING_DOWN.load(Ordering::Relaxed) {\n let client_fd = wait_for_client_with_timeout(daemon, 10000, &mut errmsg);\n if !errmsg.is_null() { libc::free(errmsg as *mut c_void); errmsg = ptr::null_mut(); }\n if client_fd > 0 {\n queue.push(client_fd);\n }\n\n // Dynamic scaling: spawn a new worker if all are busy\n if config.dynamic_scaling {\n let busy = BUSY_COUNT.load(Ordering::Relaxed);\n let total = TOTAL_WORKERS.load(Ordering::Relaxed);\n if busy >= total {\n let q = Arc::clone(&queue);\n let cfg = ptr::read(config);\n handles.push(std::thread::spawn(move || {\n worker_loop(&q, &cfg);\n }));\n TOTAL_WORKERS.fetch_add(1, Ordering::Relaxed);\n }\n }\n }\n\n SHUTTING_DOWN.store(true, Ordering::Relaxed);\n queue.cond.notify_all();\n\n for h in handles { let _ = h.join(); }\n\n close_daemon(&mut daemon);\n 0\n}\n\n// ── Pool main: single mode ───────────────────────────────────────────────────\n\nunsafe fn pool_main_single(config: &PoolConfig, socket_path: *const c_char, tmpdir: *const c_char, shm_basename: *const c_char) -> i32 {\n extern \"C\" {\n fn start_daemon(socket_path: *const c_char, tmpdir: *const c_char, shm_basename: *const c_char, size: usize, errmsg: *mut *mut c_char) -> *mut c_void;\n fn close_daemon(daemon: *mut *mut c_void);\n fn wait_for_client_with_timeout(daemon: *mut c_void, timeout_us: i32, errmsg: *mut *mut c_char) -> i32;\n fn stream_from_client(fd: i32, errmsg: *mut *mut c_char) -> *mut u8;\n fn send_packet_to_foreign_server(fd: i32, packet: *mut u8, errmsg: *mut *mut c_char) -> usize;\n fn close_socket(fd: i32);\n }\n\n let mut errmsg: *mut c_char = ptr::null_mut();\n let mut daemon = start_daemon(socket_path, tmpdir, shm_basename, 0xffff, &mut errmsg);\n if !errmsg.is_null() {\n libc::fprintf(libc::fdopen(2, b\"w\\0\".as_ptr() as *const c_char),\n b\"Failed to start language server:\\n%s\\n\\0\".as_ptr() as *const c_char, errmsg);\n libc::free(errmsg as *mut c_void);\n return 1;\n }\n\n while !SHUTTING_DOWN.load(Ordering::Relaxed) {\n let client_fd = wait_for_client_with_timeout(daemon, 10000, &mut errmsg);\n if !errmsg.is_null() { libc::free(errmsg as *mut c_void); errmsg = ptr::null_mut(); }\n if client_fd <= 0 { continue; }\n\n let data = stream_from_client(client_fd, &mut errmsg);\n if data.is_null() || !errmsg.is_null() {\n if !errmsg.is_null() { try_send_fail(client_fd, errmsg); libc::free(errmsg as *mut c_void); errmsg = ptr::null_mut(); }\n libc::free(data as *mut c_void);\n close_socket(client_fd);\n continue;\n }\n\n let result = pool_dispatch_packet(data, config.local_dispatch, config.remote_dispatch, config.dispatch_ctx);\n libc::free(data as *mut c_void);\n\n if !result.is_null() {\n send_packet_to_foreign_server(client_fd, result, &mut errmsg);\n libc::free(result as *mut c_void);\n if !errmsg.is_null() { libc::free(errmsg as *mut c_void); errmsg = ptr::null_mut(); }\n }\n\n libc::fflush(ptr::null_mut());\n close_socket(client_fd);\n }\n\n close_daemon(&mut daemon);\n 0\n}\n\n// ── Pool main: fork mode ─────────────────────────────────────────────────────\n\nunsafe fn pool_main_fork(config: &PoolConfig, socket_path: *const c_char, tmpdir: *const c_char, shm_basename: *const c_char) -> i32 {\n extern \"C\" {\n fn start_daemon(socket_path: *const c_char, tmpdir: *const c_char, shm_basename: *const c_char, size: usize, errmsg: *mut *mut c_char) -> *mut c_void;\n fn close_daemon(daemon: *mut *mut c_void);\n fn wait_for_client_with_timeout(daemon: *mut c_void, timeout_us: i32, errmsg: *mut *mut c_char) -> i32;\n fn stream_from_client(fd: i32, errmsg: *mut *mut c_char) -> *mut u8;\n fn send_packet_to_foreign_server(fd: i32, packet: *mut u8, errmsg: *mut *mut c_char) -> usize;\n fn close_socket(fd: i32);\n fn shinit(basename: *const c_char, volume: usize, size: usize, errmsg: *mut *mut c_char) -> *mut c_void;\n }\n\n let mut errmsg: *mut c_char = ptr::null_mut();\n let mut daemon = start_daemon(socket_path, tmpdir, shm_basename, 0xffff, &mut errmsg);\n if !errmsg.is_null() {\n libc::fprintf(libc::fdopen(2, b\"w\\0\".as_ptr() as *const c_char),\n b\"Failed to start language server:\\n%s\\n\\0\".as_ptr() as *const c_char, errmsg);\n libc::free(errmsg as *mut c_void);\n return 1;\n }\n\n // Create socketpair for fd passing\n let mut sv = [0i32; 2];\n if libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, sv.as_mut_ptr()) < 0 {\n close_daemon(&mut daemon);\n return 1;\n }\n\n // Shared busy counter via mmap\n let shared_counter = libc::mmap(\n ptr::null_mut(), std::mem::size_of::(),\n libc::PROT_READ | libc::PROT_WRITE,\n libc::MAP_SHARED | libc::MAP_ANONYMOUS, -1, 0,\n ) as *mut AtomicI32;\n if shared_counter == libc::MAP_FAILED as *mut AtomicI32 {\n libc::close(sv[0]); libc::close(sv[1]);\n close_daemon(&mut daemon);\n return 1;\n }\n (*shared_counter).store(0, Ordering::Relaxed);\n SHARED_BUSY = shared_counter;\n\n let nworkers = config.initial_workers.max(1);\n let mut child_pids: Vec = Vec::new();\n\n for i in 0..nworkers {\n let pid = libc::fork();\n if pid < 0 { break; }\n if pid == 0 {\n // Child\n libc::close(sv[1]); // close write end\n // Get daemon server_fd from opaque pointer and close it\n // (we can't access the struct fields directly since daemon is *mut c_void,\n // but the child doesn't need to accept connections)\n if let Some(pfk) = config.post_fork_child {\n pfk(config.dispatch_ctx);\n }\n\n shinit(shm_basename, (i + 1) as usize, 0xffff, &mut errmsg);\n if !errmsg.is_null() {\n // Print the error to stderr before exiting so the nexus can\n // capture it via the pool's redirected stderr file. Without\n // this, a failed shinit in a forked worker child leaves no\n // diagnostic trace anywhere.\n libc::fprintf(\n libc::fdopen(2, b\"w\\0\".as_ptr() as *const c_char),\n b\"Worker %d shinit failed: %s\\n\\0\".as_ptr() as *const c_char,\n i as i32,\n errmsg,\n );\n libc::free(errmsg as *mut c_void);\n libc::_exit(1);\n }\n\n // Worker loop: receive fds and process\n loop {\n if SHUTTING_DOWN.load(Ordering::Relaxed) { break; }\n let mut pfd = libc::pollfd { fd: sv[0], events: libc::POLLIN, revents: 0 };\n let ready = libc::poll(&mut pfd, 1, 100);\n if ready <= 0 { continue; }\n\n let client_fd = recv_fd(sv[0]);\n if client_fd < 0 { break; }\n\n let data = stream_from_client(client_fd, &mut errmsg);\n if data.is_null() || !errmsg.is_null() {\n if !errmsg.is_null() { try_send_fail(client_fd, errmsg); libc::free(errmsg as *mut c_void); errmsg = ptr::null_mut(); }\n libc::free(data as *mut c_void);\n close_socket(client_fd);\n continue;\n }\n\n let result = pool_dispatch_packet(data, config.local_dispatch, config.remote_dispatch, config.dispatch_ctx);\n libc::free(data as *mut c_void);\n\n if !result.is_null() {\n send_packet_to_foreign_server(client_fd, result, &mut errmsg);\n libc::free(result as *mut c_void);\n if !errmsg.is_null() { libc::free(errmsg as *mut c_void); errmsg = ptr::null_mut(); }\n }\n libc::fflush(ptr::null_mut());\n close_socket(client_fd);\n }\n libc::close(sv[0]);\n libc::_exit(0);\n }\n child_pids.push(pid);\n }\n TOTAL_WORKERS.store(child_pids.len() as i32, Ordering::Relaxed);\n\n // Parent: accept loop\n while !SHUTTING_DOWN.load(Ordering::Relaxed) {\n let client_fd = wait_for_client_with_timeout(daemon, 10000, &mut errmsg);\n if !errmsg.is_null() { libc::free(errmsg as *mut c_void); errmsg = ptr::null_mut(); }\n if client_fd > 0 {\n send_fd(sv[1], client_fd);\n close_socket(client_fd);\n }\n\n // Reap dead children\n for pid in child_pids.iter_mut() {\n if *pid > 0 {\n let mut wstatus: i32 = 0;\n if libc::waitpid(*pid, &mut wstatus, libc::WNOHANG) > 0 {\n *pid = -1;\n }\n }\n }\n }\n\n // Shutdown\n for &pid in &child_pids {\n if pid > 0 { libc::kill(pid, libc::SIGTERM); }\n }\n for &pid in &child_pids {\n if pid > 0 { libc::waitpid(pid, ptr::null_mut(), 0); }\n }\n\n libc::close(sv[0]); libc::close(sv[1]);\n libc::munmap(shared_counter as *mut c_void, std::mem::size_of::());\n SHARED_BUSY = ptr::null_mut();\n\n close_daemon(&mut daemon);\n 0\n}\n\n// fd-passing helpers\nunsafe fn send_fd(sock: i32, fd: i32) -> i32 {\n let mut buf = [0u8; 1];\n let mut iov = libc::iovec { iov_base: buf.as_mut_ptr() as *mut c_void, iov_len: 1 };\n let cmsg_space = libc::CMSG_SPACE(std::mem::size_of::() as u32) as usize;\n let mut cmsg_buf = vec![0u8; cmsg_space];\n\n let mut msg: libc::msghdr = std::mem::zeroed();\n msg.msg_iov = &mut iov;\n msg.msg_iovlen = 1;\n msg.msg_control = cmsg_buf.as_mut_ptr() as *mut c_void;\n msg.msg_controllen = cmsg_space as _;\n\n let cmsg = libc::CMSG_FIRSTHDR(&msg);\n (*cmsg).cmsg_level = libc::SOL_SOCKET;\n (*cmsg).cmsg_type = libc::SCM_RIGHTS;\n (*cmsg).cmsg_len = libc::CMSG_LEN(std::mem::size_of::() as u32) as _;\n ptr::copy_nonoverlapping(&fd as *const i32 as *const u8, libc::CMSG_DATA(cmsg), std::mem::size_of::());\n\n if libc::sendmsg(sock, &msg, 0) >= 0 { 0 } else { -1 }\n}\n\nunsafe fn recv_fd(sock: i32) -> i32 {\n let mut buf = [0u8; 1];\n let mut iov = libc::iovec { iov_base: buf.as_mut_ptr() as *mut c_void, iov_len: 1 };\n let cmsg_space = libc::CMSG_SPACE(std::mem::size_of::() as u32) as usize;\n let mut cmsg_buf = vec![0u8; cmsg_space];\n\n let mut msg: libc::msghdr = std::mem::zeroed();\n msg.msg_iov = &mut iov;\n msg.msg_iovlen = 1;\n msg.msg_control = cmsg_buf.as_mut_ptr() as *mut c_void;\n msg.msg_controllen = cmsg_space as _;\n\n let n = libc::recvmsg(sock, &mut msg, 0);\n if n <= 0 { return -1; }\n\n let cmsg = libc::CMSG_FIRSTHDR(&msg);\n if cmsg.is_null() || (*cmsg).cmsg_level != libc::SOL_SOCKET || (*cmsg).cmsg_type != libc::SCM_RIGHTS {\n return -1;\n }\n\n let mut fd: i32 = 0;\n ptr::copy_nonoverlapping(libc::CMSG_DATA(cmsg), &mut fd as *mut i32 as *mut u8, std::mem::size_of::());\n fd\n}\n\n// ── Entry point ──────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn pool_main(\n argc: i32,\n argv: *mut *mut c_char,\n config: *mut PoolConfig,\n) -> i32 {\n if argc != 4 {\n libc::fprintf(libc::fdopen(2, b\"w\\0\".as_ptr() as *const c_char),\n b\"Usage: %s \\n\\0\".as_ptr() as *const c_char,\n if argc > 0 { *argv } else { b\"pool\\0\".as_ptr() as *const c_char });\n return 1;\n }\n\n let cfg = &mut *config;\n if cfg.initial_workers <= 0 { cfg.initial_workers = 1; }\n\n SHUTTING_DOWN.store(false, Ordering::Relaxed);\n BUSY_COUNT.store(0, Ordering::Relaxed);\n\n // SIGTERM handler\n let mut sa: libc::sigaction = std::mem::zeroed();\n sa.sa_sigaction = pool_sigterm_handler as *const () as usize;\n libc::sigemptyset(&mut sa.sa_mask);\n libc::sigaction(libc::SIGTERM, &sa, ptr::null_mut());\n\n let socket_path = *argv.add(1);\n let tmpdir = *argv.add(2);\n let shm_basename = *argv.add(3);\n\n match cfg.concurrency {\n PoolConcurrency::Threads => pool_main_threads(cfg, socket_path, tmpdir, shm_basename),\n PoolConcurrency::Fork => pool_main_fork(cfg, socket_path, tmpdir, shm_basename),\n PoolConcurrency::Single => pool_main_single(cfg, socket_path, tmpdir, shm_basename),\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/router_ffi.rs", "content": "//! C ABI wrappers for router subsystems.\n//! Replaces router.c. Routes requests to per-program daemons.\n\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::time::Instant;\n\nuse crate::daemon_ffi::{\n DaemonConfig, DaemonResponse, MorlocSocket,\n};\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\nuse crate::http_ffi::{DaemonMethod, DaemonRequest, HttpMethod, HttpRequest};\n\n// -- Constants ----------------------------------------------------------------\n\n/// Max size of sun_path in sockaddr_un (108 on Linux)\nconst SUN_PATH_LEN: usize = 108;\n\n// Daemon startup polling (exponential backoff, ~5s total).\n// Sum of 100 * 1.25^i for i in 0..16 is ~4650ms.\nconst DAEMON_POLL_INITIAL_MS: f64 = 100.0;\nconst DAEMON_POLL_MULTIPLIER: f64 = 1.25;\nconst DAEMON_POLL_MAX_RETRIES: usize = 16;\n\n// -- Global state -------------------------------------------------------------\n\nstatic ROUTER_SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false);\n\nextern \"C\" fn router_signal_handler_fn(_sig: i32) {\n ROUTER_SHUTDOWN_REQUESTED.store(true, Ordering::Relaxed);\n}\n\n// -- C-compatible types -------------------------------------------------------\n\n#[repr(C)]\npub struct RouterProgram {\n pub name: *mut c_char,\n pub manifest_path: *mut c_char,\n pub manifest: *mut c_void, // manifest_t*\n pub daemon_pid: libc::pid_t,\n pub daemon_socket: [c_char; SUN_PATH_LEN],\n}\n\n#[repr(C)]\npub struct Router {\n pub programs: *mut RouterProgram,\n pub n_programs: usize,\n pub fdb_path: *mut c_char,\n}\n\n// -- router_init --------------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn router_init(\n fdb_path: *const c_char,\n errmsg: *mut *mut c_char,\n) -> *mut Router {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn read_manifest(path: *const c_char, errmsg: *mut *mut c_char) -> *mut c_void;\n }\n\n let dir = libc::opendir(fdb_path);\n if dir.is_null() {\n let errno_msg = CStr::from_ptr(libc::strerror(crate::utility::errno_val()))\n .to_string_lossy();\n let path_str = CStr::from_ptr(fdb_path).to_string_lossy();\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"Cannot open fdb directory '{}': {}\",\n path_str, errno_msg\n )),\n );\n return ptr::null_mut();\n }\n\n let router = libc::calloc(1, std::mem::size_of::()) as *mut Router;\n (*router).fdb_path = libc::strdup(fdb_path);\n\n let mut cap: usize = 8;\n (*router).programs =\n libc::calloc(cap, std::mem::size_of::()) as *mut RouterProgram;\n (*router).n_programs = 0;\n\n loop {\n let entry = libc::readdir(dir);\n if entry.is_null() {\n break;\n }\n\n let name = CStr::from_ptr((*entry).d_name.as_ptr());\n let name_str = name.to_string_lossy();\n\n if name_str.len() < 10 || !name_str.ends_with(\".manifest\") {\n continue;\n }\n\n // Grow array if needed\n if (*router).n_programs >= cap {\n cap *= 2;\n (*router).programs = libc::realloc(\n (*router).programs as *mut c_void,\n cap * std::mem::size_of::(),\n ) as *mut RouterProgram;\n }\n\n let prog = &mut *(*router).programs.add((*router).n_programs);\n ptr::write_bytes(prog as *mut RouterProgram, 0, 1);\n\n // Extract program name (filename without .manifest)\n let prog_name_len = name_str.len() - 9;\n let prog_name = &name_str[..prog_name_len];\n let c_prog_name = CString::new(prog_name).unwrap_or_default();\n prog.name = libc::strdup(c_prog_name.as_ptr());\n\n // Build full path\n let fdb_str = CStr::from_ptr(fdb_path).to_string_lossy();\n let full_path = format!(\"{}/{}\", fdb_str, name_str);\n let c_path = CString::new(full_path).unwrap_or_default();\n prog.manifest_path = libc::strdup(c_path.as_ptr());\n\n // Read and parse manifest\n let mut child_err: *mut c_char = ptr::null_mut();\n prog.manifest = read_manifest(prog.manifest_path, &mut child_err);\n if !child_err.is_null() {\n let err_str = CStr::from_ptr(child_err).to_string_lossy();\n let path_str = CStr::from_ptr(prog.manifest_path).to_string_lossy();\n eprintln!(\"router: warning: failed to parse {}: {}\", path_str, err_str);\n libc::free(child_err as *mut c_void);\n libc::free(prog.name as *mut c_void);\n libc::free(prog.manifest_path as *mut c_void);\n continue;\n }\n\n prog.daemon_pid = 0;\n // Set socket path\n let socket_path = format!(\"/tmp/morloc-router-{}.sock\", prog_name);\n let c_socket = CString::new(socket_path).unwrap_or_default();\n let socket_bytes = c_socket.as_bytes_with_nul();\n let copy_len = socket_bytes.len().min(SUN_PATH_LEN);\n ptr::copy_nonoverlapping(\n socket_bytes.as_ptr() as *const c_char,\n prog.daemon_socket.as_mut_ptr(),\n copy_len,\n );\n\n (*router).n_programs += 1;\n }\n\n libc::closedir(dir);\n\n // Empty fdb is fine -- programs can be added while the router is running\n\n router\n}\n\n// -- router_free --------------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn router_free(router: *mut Router) {\n if router.is_null() {\n return;\n }\n\n extern \"C\" {\n fn free_manifest(manifest: *mut c_void);\n }\n\n for i in 0..(*router).n_programs {\n let prog = &mut *(*router).programs.add(i);\n libc::free(prog.name as *mut c_void);\n libc::free(prog.manifest_path as *mut c_void);\n if !prog.manifest.is_null() {\n free_manifest(prog.manifest);\n }\n if prog.daemon_pid > 0 {\n libc::kill(prog.daemon_pid, libc::SIGTERM);\n }\n }\n libc::free((*router).programs as *mut c_void);\n libc::free((*router).fdb_path as *mut c_void);\n libc::free(router as *mut c_void);\n}\n\n// -- morloc-nexus path resolution ---------------------------------------------\n\n/// Locate the morloc-nexus executable.\n///\n/// Tries, in order:\n/// 1. `$MORLOC_NEXUS` (explicit override)\n/// 2. `$MORLOC_HOME/bin/morloc-nexus` (deploy convention)\n/// 3. `morloc-nexus` on `$PATH`\n/// 4. `$HOME/.local/bin/morloc-nexus` (bare-metal developer install)\n///\n/// Returns the path on the first candidate whose `access(_, X_OK)` succeeds,\n/// or the list of attempted paths on failure.\nunsafe fn find_morloc_nexus() -> Result> {\n fn is_executable(path: &str) -> bool {\n if let Ok(c) = CString::new(path) {\n unsafe { libc::access(c.as_ptr(), libc::X_OK) == 0 }\n } else {\n false\n }\n }\n\n fn getenv_str(name: &str) -> Option {\n let c_name = CString::new(name).ok()?;\n let p = unsafe { libc::getenv(c_name.as_ptr()) };\n if p.is_null() {\n None\n } else {\n Some(unsafe { CStr::from_ptr(p) }.to_string_lossy().into_owned())\n }\n }\n\n let mut tried: Vec = Vec::new();\n\n // 1. $MORLOC_NEXUS\n if let Some(p) = getenv_str(\"MORLOC_NEXUS\") {\n if is_executable(&p) {\n return Ok(p);\n }\n tried.push(format!(\"$MORLOC_NEXUS={}\", p));\n }\n\n // 2. $MORLOC_HOME/bin/morloc-nexus\n if let Some(h) = getenv_str(\"MORLOC_HOME\") {\n let p = format!(\"{}/bin/morloc-nexus\", h);\n if is_executable(&p) {\n return Ok(p);\n }\n tried.push(p);\n }\n\n // 3. Search $PATH\n if let Some(path) = getenv_str(\"PATH\") {\n for dir in path.split(':') {\n if dir.is_empty() {\n continue;\n }\n let p = format!(\"{}/morloc-nexus\", dir);\n if is_executable(&p) {\n return Ok(p);\n }\n }\n tried.push(format!(\"$PATH ({})\", path));\n }\n\n // 4. $HOME/.local/bin/morloc-nexus\n if let Some(h) = getenv_str(\"HOME\") {\n let p = format!(\"{}/.local/bin/morloc-nexus\", h);\n if is_executable(&p) {\n return Ok(p);\n }\n tried.push(p);\n }\n\n Err(tried)\n}\n\n// -- router_start_program -----------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn router_start_program(\n prog: *mut RouterProgram,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n\n let nexus_path = match find_morloc_nexus() {\n Ok(p) => p,\n Err(tried) => {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"morloc-nexus binary not found; tried: {}\",\n tried.join(\", \")\n )),\n );\n return false;\n }\n };\n let c_nexus = CString::new(nexus_path.as_str()).unwrap_or_default();\n\n let pid = libc::fork();\n if pid == 0 {\n // Child: exec morloc-nexus with --daemon\n libc::setpgid(0, 0);\n let arg_nexus = CString::new(\"morloc-nexus\").unwrap();\n let arg_daemon = CString::new(\"--daemon\").unwrap();\n let arg_socket = CString::new(\"--socket\").unwrap();\n let socket_path = CStr::from_ptr((*prog).daemon_socket.as_ptr());\n libc::execl(\n c_nexus.as_ptr(),\n arg_nexus.as_ptr(),\n (*prog).manifest_path,\n arg_daemon.as_ptr(),\n arg_socket.as_ptr(),\n socket_path.as_ptr(),\n ptr::null::(),\n );\n // If exec fails\n let prog_name = CStr::from_ptr((*prog).name).to_string_lossy();\n let errno_msg = CStr::from_ptr(libc::strerror(crate::utility::errno_val()))\n .to_string_lossy();\n eprintln!(\n \"router: failed to exec morloc-nexus for {}: {}\",\n prog_name, errno_msg\n );\n libc::_exit(1);\n } else if pid > 0 {\n (*prog).daemon_pid = pid;\n\n // Poll until the daemon socket is connectable (exponential backoff)\n let mut delay_ms = DAEMON_POLL_INITIAL_MS;\n let mut connected = false;\n for _attempt in 0..DAEMON_POLL_MAX_RETRIES {\n let ts = libc::timespec {\n tv_sec: 0,\n tv_nsec: (delay_ms * 1_000_000.0) as i64,\n };\n libc::nanosleep(&ts, ptr::null_mut());\n\n // Check if child died during startup\n let mut status: i32 = 0;\n let result = libc::waitpid(pid, &mut status, libc::WNOHANG);\n if result == pid {\n (*prog).daemon_pid = 0;\n let prog_name = CStr::from_ptr((*prog).name).to_string_lossy();\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"Daemon for '{}' exited during startup (status {})\",\n prog_name, status\n )),\n );\n return false;\n }\n\n // Try connecting to the daemon socket\n let test_sock = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0);\n if test_sock >= 0 {\n let mut addr: libc::sockaddr_un = std::mem::zeroed();\n addr.sun_family = libc::AF_UNIX as libc::sa_family_t;\n let socket_path = (*prog).daemon_socket.as_ptr();\n let path_bytes = CStr::from_ptr(socket_path).to_bytes();\n let copy_len = path_bytes.len().min(addr.sun_path.len() - 1);\n ptr::copy_nonoverlapping(\n path_bytes.as_ptr() as *const c_char,\n addr.sun_path.as_mut_ptr(),\n copy_len,\n );\n let rc = libc::connect(\n test_sock,\n &addr as *const libc::sockaddr_un as *const libc::sockaddr,\n std::mem::size_of::() as libc::socklen_t,\n );\n libc::close(test_sock);\n if rc == 0 {\n connected = true;\n break;\n }\n }\n\n delay_ms *= DAEMON_POLL_MULTIPLIER;\n }\n\n if !connected {\n // Final check: did the daemon die?\n let mut status: i32 = 0;\n let result = libc::waitpid(pid, &mut status, libc::WNOHANG);\n if result == pid {\n (*prog).daemon_pid = 0;\n let prog_name = CStr::from_ptr((*prog).name).to_string_lossy();\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"Daemon for '{}' exited during startup (status {})\",\n prog_name, status\n )),\n );\n return false;\n }\n // Daemon alive but socket not yet connectable -- proceed anyway,\n // router_forward() will retry on connect failure.\n }\n\n true\n } else {\n let errno_msg = CStr::from_ptr(libc::strerror(crate::utility::errno_val()))\n .to_string_lossy();\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"fork failed: {}\", errno_msg)),\n );\n false\n }\n}\n\n// -- router_forward -----------------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn router_forward(\n router: *mut Router,\n program: *const c_char,\n request: *mut DaemonRequest,\n errmsg: *mut *mut c_char,\n) -> *mut DaemonResponse {\n clear_errmsg(errmsg);\n\n extern \"C\" {\n fn daemon_parse_response(\n json: *const c_char,\n len: usize,\n errmsg: *mut *mut c_char,\n ) -> *mut DaemonResponse;\n }\n\n // Find program\n let program_name = CStr::from_ptr(program);\n let mut prog: *mut RouterProgram = ptr::null_mut();\n for i in 0..(*router).n_programs {\n let p = (*router).programs.add(i);\n if CStr::from_ptr((*p).name) == program_name {\n prog = p;\n break;\n }\n }\n\n if prog.is_null() {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"Unknown program: {}\",\n program_name.to_string_lossy()\n )),\n );\n return ptr::null_mut();\n }\n\n // Check if a previously-started daemon has exited (crash recovery)\n if (*prog).daemon_pid > 0 {\n let mut status: i32 = 0;\n let result = libc::waitpid((*prog).daemon_pid, &mut status, libc::WNOHANG);\n if result == (*prog).daemon_pid || result < 0 {\n let prog_name = CStr::from_ptr((*prog).name).to_string_lossy();\n eprintln!(\n \"router: daemon for '{}' exited (status {}), will restart\",\n prog_name, status\n );\n (*prog).daemon_pid = 0;\n }\n }\n\n // Start daemon if not running\n if (*prog).daemon_pid <= 0 {\n let mut child_err: *mut c_char = ptr::null_mut();\n if !router_start_program(prog, &mut child_err) {\n if !child_err.is_null() {\n *errmsg = child_err;\n } else {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to start program daemon\".into()),\n );\n }\n return ptr::null_mut();\n }\n }\n\n // Serialize request to JSON\n let req_json = serialize_request_to_json(request);\n let c_req = CString::new(req_json.as_str()).unwrap_or_default();\n let req_len = req_json.len();\n\n // Try to connect, retry once on failure\n let sock = connect_to_daemon(prog, errmsg);\n let sock = if sock < 0 {\n // Try restarting daemon\n (*prog).daemon_pid = 0;\n // Clear previous error\n if !(*errmsg).is_null() {\n libc::free(*errmsg as *mut c_void);\n *errmsg = ptr::null_mut();\n }\n let mut child_err: *mut c_char = ptr::null_mut();\n if !router_start_program(prog, &mut child_err) {\n if !child_err.is_null() {\n *errmsg = child_err;\n }\n return ptr::null_mut();\n }\n let sock2 = connect_to_daemon(prog, errmsg);\n if sock2 < 0 {\n return ptr::null_mut();\n }\n sock2\n } else {\n sock\n };\n\n // Send length-prefixed message\n let len_buf: [u8; 4] = [\n ((req_len >> 24) & 0xFF) as u8,\n ((req_len >> 16) & 0xFF) as u8,\n ((req_len >> 8) & 0xFF) as u8,\n (req_len & 0xFF) as u8,\n ];\n\n let n = libc::send(\n sock,\n len_buf.as_ptr() as *const c_void,\n 4,\n crate::utility::SEND_NOSIGNAL,\n );\n if n != 4 {\n libc::close(sock);\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to send request length to daemon\".into()),\n );\n return ptr::null_mut();\n }\n\n let mut total_sent: usize = 0;\n while total_sent < req_len {\n let n = libc::send(\n sock,\n c_req.as_ptr().add(total_sent) as *const c_void,\n req_len - total_sent,\n crate::utility::SEND_NOSIGNAL,\n );\n if n <= 0 {\n libc::close(sock);\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to send request body to daemon\".into()),\n );\n return ptr::null_mut();\n }\n total_sent += n as usize;\n }\n\n // Read response length\n let mut resp_len_buf = [0u8; 4];\n let n = libc::recv(\n sock,\n resp_len_buf.as_mut_ptr() as *mut c_void,\n 4,\n libc::MSG_WAITALL,\n );\n if n != 4 {\n libc::close(sock);\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to read response length from daemon\".into()),\n );\n return ptr::null_mut();\n }\n\n let resp_len = ((resp_len_buf[0] as u32) << 24)\n | ((resp_len_buf[1] as u32) << 16)\n | ((resp_len_buf[2] as u32) << 8)\n | (resp_len_buf[3] as u32);\n\n let resp_json = libc::malloc(resp_len as usize + 1) as *mut c_char;\n if resp_json.is_null() {\n libc::close(sock);\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to allocate response buffer\".into()),\n );\n return ptr::null_mut();\n }\n\n let mut total_recv: usize = 0;\n while total_recv < resp_len as usize {\n let n = libc::recv(\n sock,\n resp_json.add(total_recv) as *mut c_void,\n resp_len as usize - total_recv,\n 0,\n );\n if n <= 0 {\n libc::free(resp_json as *mut c_void);\n libc::close(sock);\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to read response body from daemon\".into()),\n );\n return ptr::null_mut();\n }\n total_recv += n as usize;\n }\n *resp_json.add(resp_len as usize) = 0;\n libc::close(sock);\n\n let resp = daemon_parse_response(resp_json, resp_len as usize, errmsg);\n libc::free(resp_json as *mut c_void);\n resp\n}\n\n/// Helper: connect to a program daemon's unix socket with 60s timeouts.\nunsafe fn connect_to_daemon(\n prog: *mut RouterProgram,\n errmsg: *mut *mut c_char,\n) -> i32 {\n let sock = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0);\n if sock < 0 {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to create socket\".into()),\n );\n return -1;\n }\n crate::utility::set_nosigpipe(sock);\n\n let tv = libc::timeval {\n tv_sec: 60,\n tv_usec: 0,\n };\n libc::setsockopt(\n sock,\n libc::SOL_SOCKET,\n libc::SO_RCVTIMEO,\n &tv as *const libc::timeval as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n libc::setsockopt(\n sock,\n libc::SOL_SOCKET,\n libc::SO_SNDTIMEO,\n &tv as *const libc::timeval as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n\n let mut addr: libc::sockaddr_un = std::mem::zeroed();\n addr.sun_family = libc::AF_UNIX as libc::sa_family_t;\n let socket_path = (*prog).daemon_socket.as_ptr();\n let path_bytes = CStr::from_ptr(socket_path).to_bytes();\n let copy_len = path_bytes.len().min(addr.sun_path.len() - 1);\n ptr::copy_nonoverlapping(\n path_bytes.as_ptr() as *const c_char,\n addr.sun_path.as_mut_ptr(),\n copy_len,\n );\n\n if libc::connect(\n sock,\n &addr as *const libc::sockaddr_un as *const libc::sockaddr,\n std::mem::size_of::() as libc::socklen_t,\n ) < 0\n {\n libc::close(sock);\n let prog_name = CStr::from_ptr((*prog).name).to_string_lossy();\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\n \"Failed to connect to daemon for '{}'\",\n prog_name\n )),\n );\n return -1;\n }\n\n sock\n}\n\n/// Serialize a DaemonRequest to JSON using serde_json.\nunsafe fn serialize_request_to_json(request: *mut DaemonRequest) -> String {\n let mut map = serde_json::Map::new();\n\n if !(*request).id.is_null() {\n let id = CStr::from_ptr((*request).id).to_string_lossy();\n map.insert(\"id\".into(), serde_json::Value::String(id.into_owned()));\n }\n\n let method_str = match (*request).method {\n DaemonMethod::Call => \"call\",\n DaemonMethod::Discover => \"discover\",\n DaemonMethod::Health => \"health\",\n DaemonMethod::Eval => \"eval\",\n DaemonMethod::Typecheck => \"typecheck\",\n DaemonMethod::Bind => \"bind\",\n DaemonMethod::Bindings => \"bindings\",\n DaemonMethod::Unbind => \"unbind\",\n };\n map.insert(\n \"method\".into(),\n serde_json::Value::String(method_str.into()),\n );\n\n if !(*request).command.is_null() {\n let cmd = CStr::from_ptr((*request).command).to_string_lossy();\n map.insert(\n \"command\".into(),\n serde_json::Value::String(cmd.into_owned()),\n );\n }\n\n if !(*request).args_json.is_null() {\n let args_str = CStr::from_ptr((*request).args_json).to_string_lossy();\n // Try to parse as JSON value to embed directly\n if let Ok(v) = serde_json::from_str::(&args_str) {\n map.insert(\"args\".into(), v);\n }\n }\n\n if !(*request).expr.is_null() {\n let expr = CStr::from_ptr((*request).expr).to_string_lossy();\n map.insert(\"expr\".into(), serde_json::Value::String(expr.into_owned()));\n }\n\n if !(*request).name.is_null() {\n let name = CStr::from_ptr((*request).name).to_string_lossy();\n map.insert(\"name\".into(), serde_json::Value::String(name.into_owned()));\n }\n\n serde_json::to_string(&map).unwrap_or_else(|_| \"{}\".into())\n}\n\n// -- router_build_discovery ---------------------------------------------------\n\n#[no_mangle]\npub unsafe extern \"C\" fn router_build_discovery(router: *mut Router) -> *mut c_char {\n // Walk the canonical Manifest C struct from manifest_ffi.rs. No\n // local mirror -- the in-memory layout is shared.\n use crate::manifest_ffi::Manifest as ManifestC;\n\n #[derive(serde::Serialize)]\n struct CommandInfo {\n name: String,\n r#type: String,\n return_type: String,\n }\n\n #[derive(serde::Serialize)]\n struct ProgramInfo {\n name: String,\n running: bool,\n #[serde(skip_serializing_if = \"Option::is_none\")]\n commands: Option>,\n }\n\n #[derive(serde::Serialize)]\n struct Discovery {\n programs: Vec,\n }\n\n let mut programs = Vec::with_capacity((*router).n_programs);\n\n for i in 0..(*router).n_programs {\n let prog = &*(*router).programs.add(i);\n let name = CStr::from_ptr(prog.name).to_string_lossy().into_owned();\n let running =\n prog.daemon_pid > 0 && libc::kill(prog.daemon_pid, 0) == 0;\n\n let commands = if !prog.manifest.is_null() {\n let mv = prog.manifest as *const ManifestC;\n let mut cmds = Vec::with_capacity((*mv).n_commands);\n for c in 0..(*mv).n_commands {\n let cmd = &*(*mv).commands.add(c);\n let cmd_name = CStr::from_ptr(cmd.name).to_string_lossy().into_owned();\n let cmd_type = if cmd.is_pure { \"pure\" } else { \"remote\" };\n let ret_type = if !cmd.ret.type_desc.is_null() {\n CStr::from_ptr(cmd.ret.type_desc)\n .to_string_lossy()\n .into_owned()\n } else {\n String::new()\n };\n cmds.push(CommandInfo {\n name: cmd_name,\n r#type: cmd_type.into(),\n return_type: ret_type,\n });\n }\n Some(cmds)\n } else {\n None\n };\n\n programs.push(ProgramInfo {\n name,\n running,\n commands,\n });\n }\n\n let disco = Discovery { programs };\n let json = serde_json::to_string(&disco).unwrap_or_else(|_| \"{}\".into());\n let c = CString::new(json).unwrap_or_default();\n libc::strdup(c.as_ptr())\n}\n\n// -- Router HTTP request routing ----------------------------------------------\n\n/// Route HTTP requests for the router. Sets *out_program to the target program\n/// name (caller-owned) for per-program requests, or NULL for router-level requests.\nunsafe fn router_http_to_request(\n req: *mut HttpRequest,\n out_program: *mut *mut c_char,\n errmsg: *mut *mut c_char,\n) -> *mut DaemonRequest {\n clear_errmsg(errmsg);\n\n let dreq = libc::calloc(1, std::mem::size_of::()) as *mut DaemonRequest;\n if dreq.is_null() {\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Failed to allocate daemon_request_t\".into()),\n );\n return ptr::null_mut();\n }\n\n *out_program = ptr::null_mut();\n\n let path = CStr::from_ptr((*req).path.as_ptr())\n .to_str()\n .unwrap_or(\"\");\n let method = (*req).method;\n\n let body_str = if !(*req).body.is_null() && (*req).body_len > 0 {\n std::str::from_utf8(std::slice::from_raw_parts(\n (*req).body as *const u8,\n (*req).body_len,\n ))\n .unwrap_or(\"\")\n } else {\n \"\"\n };\n\n // GET /health or GET /health/\n if method == HttpMethod::Get && (path == \"/health\" || path.starts_with(\"/health/\")) {\n (*dreq).method = DaemonMethod::Health;\n if path.starts_with(\"/health/\") {\n let prog_name = &path[8..];\n if !prog_name.is_empty() {\n let c = CString::new(prog_name).unwrap_or_default();\n *out_program = libc::strdup(c.as_ptr());\n }\n }\n return dreq;\n }\n\n // GET /programs or GET /discover\n if method == HttpMethod::Get && (path == \"/programs\" || path == \"/discover\") {\n (*dreq).method = DaemonMethod::Discover;\n return dreq;\n }\n\n // GET /discover/\n if method == HttpMethod::Get && path.starts_with(\"/discover/\") {\n let prog_name = &path[10..];\n if !prog_name.is_empty() {\n let c = CString::new(prog_name).unwrap_or_default();\n *out_program = libc::strdup(c.as_ptr());\n (*dreq).method = DaemonMethod::Discover;\n return dreq;\n }\n }\n\n // POST /eval\n if method == HttpMethod::Post && path == \"/eval\" {\n (*dreq).method = DaemonMethod::Eval;\n if !body_str.is_empty() {\n if let Ok(v) = serde_json::from_str::(body_str) {\n if let Some(expr) = v.get(\"expr\").and_then(|e| e.as_str()) {\n let c = CString::new(expr).unwrap_or_default();\n (*dreq).expr = libc::strdup(c.as_ptr());\n }\n }\n }\n if (*dreq).expr.is_null() {\n libc::free(dreq as *mut c_void);\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Missing 'expr' field in /eval request body\".into()),\n );\n return ptr::null_mut();\n }\n return dreq;\n }\n\n // POST /call//\n if method == HttpMethod::Post && path.starts_with(\"/call/\") {\n let rest = &path[6..];\n let slash = rest.find('/');\n match slash {\n Some(pos) if pos + 1 < rest.len() => {\n let prog_name = &rest[..pos];\n let cmd_name = &rest[pos + 1..];\n let c_prog = CString::new(prog_name).unwrap_or_default();\n *out_program = libc::strdup(c_prog.as_ptr());\n (*dreq).method = DaemonMethod::Call;\n let c_cmd = CString::new(cmd_name).unwrap_or_default();\n (*dreq).command = libc::strdup(c_cmd.as_ptr());\n\n // Parse body for args\n let trimmed = body_str.trim();\n if trimmed.starts_with('[') {\n let c = CString::new(trimmed).unwrap_or_default();\n (*dreq).args_json = libc::strdup(c.as_ptr());\n } else if trimmed.starts_with('{') {\n if let Ok(v) = serde_json::from_str::(trimmed) {\n if let Some(args) = v.get(\"args\") {\n let args_str = serde_json::to_string(args).unwrap_or_default();\n let c = CString::new(args_str).unwrap_or_default();\n (*dreq).args_json = libc::strdup(c.as_ptr());\n }\n }\n }\n return dreq;\n }\n _ => {\n libc::free(dreq as *mut c_void);\n set_errmsg(\n errmsg,\n &MorlocError::Other(\"Expected /call//\".into()),\n );\n return ptr::null_mut();\n }\n }\n }\n\n // OPTIONS (CORS)\n if method == HttpMethod::Options {\n (*dreq).method = DaemonMethod::Health;\n return dreq;\n }\n\n libc::free(dreq as *mut c_void);\n let method_str = match method {\n HttpMethod::Get => \"GET\",\n HttpMethod::Post => \"POST\",\n HttpMethod::Delete => \"DELETE\",\n HttpMethod::Options => \"OPTIONS\",\n };\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"Unknown router endpoint: {} {}\", method_str, path)),\n );\n ptr::null_mut()\n}\n\n// -- Router event loop --------------------------------------------------------\n\nconst ROUTER_MAX_LISTENERS: usize = 3;\n\n#[no_mangle]\npub unsafe extern \"C\" fn router_run(config: *mut DaemonConfig, router: *mut Router) {\n extern \"C\" {\n fn http_parse_request(fd: i32, errmsg: *mut *mut c_char) -> *mut HttpRequest;\n fn http_free_request(req: *mut HttpRequest);\n fn http_write_response(\n fd: i32,\n status: i32,\n content_type: *const c_char,\n body: *const c_char,\n body_len: usize,\n ) -> bool;\n fn daemon_dispatch(\n manifest: *mut c_void,\n request: *mut DaemonRequest,\n sockets: *mut MorlocSocket,\n shm_basename: *const c_char,\n ) -> *mut DaemonResponse;\n fn daemon_serialize_response(\n response: *mut DaemonResponse,\n out_len: *mut usize,\n ) -> *mut c_char;\n fn daemon_free_request(req: *mut DaemonRequest);\n fn daemon_free_response(resp: *mut DaemonResponse);\n fn daemon_set_eval_timeout(timeout_sec: i32);\n fn manifest_to_discovery_json(manifest: *const c_void) -> *mut c_char;\n }\n\n daemon_set_eval_timeout((*config).eval_timeout);\n\n // Install signal handlers\n ROUTER_SHUTDOWN_REQUESTED.store(false, Ordering::Relaxed);\n let handler: libc::sighandler_t =\n std::mem::transmute::(router_signal_handler_fn);\n libc::signal(libc::SIGTERM, handler);\n libc::signal(libc::SIGINT, handler);\n\n let mut fds = [libc::pollfd {\n fd: -1,\n events: 0,\n revents: 0,\n }; ROUTER_MAX_LISTENERS];\n let mut nfds: usize = 0;\n\n let ct = b\"application/json\\0\";\n\n // HTTP listener\n if (*config).http_port > 0 {\n let http_fd = libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0);\n if http_fd < 0 {\n eprintln!(\"router: failed to create http socket\");\n return;\n }\n let opt: i32 = 1;\n libc::setsockopt(\n http_fd,\n libc::SOL_SOCKET,\n libc::SO_REUSEADDR,\n &opt as *const i32 as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n let mut addr: libc::sockaddr_in = std::mem::zeroed();\n addr.sin_family = libc::AF_INET as libc::sa_family_t;\n addr.sin_addr.s_addr = libc::INADDR_ANY;\n addr.sin_port = ((*config).http_port as u16).to_be();\n if libc::bind(\n http_fd,\n &addr as *const libc::sockaddr_in as *const libc::sockaddr,\n std::mem::size_of::() as libc::socklen_t,\n ) < 0\n {\n eprintln!(\n \"router: failed to bind http port {}\",\n (*config).http_port\n );\n libc::close(http_fd);\n return;\n }\n libc::listen(http_fd, 16);\n eprintln!(\"router: listening on http port {}\", (*config).http_port);\n fds[nfds].fd = http_fd;\n fds[nfds].events = libc::POLLIN as i16;\n nfds += 1;\n }\n\n // Unix socket\n if !(*config).unix_socket_path.is_null() {\n let sock_fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0);\n if sock_fd < 0 {\n eprintln!(\"router: failed to create unix socket\");\n return;\n }\n let mut addr: libc::sockaddr_un = std::mem::zeroed();\n addr.sun_family = libc::AF_UNIX as libc::sa_family_t;\n let path_bytes = CStr::from_ptr((*config).unix_socket_path).to_bytes();\n let copy_len = path_bytes.len().min(addr.sun_path.len() - 1);\n ptr::copy_nonoverlapping(\n path_bytes.as_ptr() as *const c_char,\n addr.sun_path.as_mut_ptr(),\n copy_len,\n );\n libc::unlink((*config).unix_socket_path);\n if libc::bind(\n sock_fd,\n &addr as *const libc::sockaddr_un as *const libc::sockaddr,\n std::mem::size_of::() as libc::socklen_t,\n ) < 0\n {\n eprintln!(\"router: failed to bind unix socket\");\n libc::close(sock_fd);\n return;\n }\n libc::listen(sock_fd, 16);\n eprintln!(\n \"router: listening on unix socket {}\",\n CStr::from_ptr((*config).unix_socket_path).to_string_lossy()\n );\n fds[nfds].fd = sock_fd;\n fds[nfds].events = libc::POLLIN as i16;\n nfds += 1;\n }\n\n if nfds == 0 {\n eprintln!(\"router: no listeners configured\");\n return;\n }\n\n // Eagerly start all program daemons so /health reports ok immediately\n for i in 0..(*router).n_programs {\n let prog = &mut *(*router).programs.add(i);\n if (*prog).daemon_pid <= 0 {\n let mut child_err: *mut c_char = ptr::null_mut();\n if router_start_program(prog, &mut child_err) {\n eprintln!(\n \"router: started daemon for '{}'\",\n CStr::from_ptr((*prog).name).to_string_lossy()\n );\n } else {\n let err_msg = if !child_err.is_null() {\n let s = CStr::from_ptr(child_err).to_string_lossy().to_string();\n libc::free(child_err as *mut c_void);\n s\n } else {\n \"unknown error\".to_string()\n };\n eprintln!(\n \"router: warning: failed to start daemon for '{}': {}\",\n CStr::from_ptr((*prog).name).to_string_lossy(),\n err_msg\n );\n }\n }\n }\n\n while !ROUTER_SHUTDOWN_REQUESTED.load(Ordering::Relaxed) {\n let ready = libc::poll(fds.as_mut_ptr(), nfds as libc::nfds_t, 1000);\n if ready < 0 {\n if crate::utility::errno_val() == libc::EINTR {\n continue;\n }\n eprintln!(\"router: poll error\");\n break;\n }\n if ready == 0 {\n continue;\n }\n\n for i in 0..nfds {\n if fds[i].revents & libc::POLLIN as i16 == 0 {\n continue;\n }\n\n let client_fd = libc::accept(fds[i].fd, ptr::null_mut(), ptr::null_mut());\n if client_fd < 0 {\n continue;\n }\n let req_start = Instant::now();\n crate::utility::set_nosigpipe(client_fd);\n\n let tv = libc::timeval {\n tv_sec: 30,\n tv_usec: 0,\n };\n libc::setsockopt(\n client_fd,\n libc::SOL_SOCKET,\n libc::SO_RCVTIMEO,\n &tv as *const libc::timeval as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n libc::setsockopt(\n client_fd,\n libc::SOL_SOCKET,\n libc::SO_SNDTIMEO,\n &tv as *const libc::timeval as *const c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n\n let mut err: *mut c_char = ptr::null_mut();\n\n let http_req = http_parse_request(client_fd, &mut err);\n if !err.is_null() {\n let body = b\"{\\\"status\\\":\\\"error\\\",\\\"error\\\":\\\"Bad request\\\"}\\0\";\n http_write_response(\n client_fd,\n 400,\n ct.as_ptr() as *const c_char,\n body.as_ptr() as *const c_char,\n body.len() - 1,\n );\n libc::free(err as *mut c_void);\n let elapsed = req_start.elapsed();\n eprintln!(\"router: ??? ??? -> 400 ({:.1}ms)\", elapsed.as_secs_f64() * 1000.0);\n libc::close(client_fd);\n continue;\n }\n\n // Extract method and path for access logging before request is consumed\n let log_method = match (*http_req).method {\n HttpMethod::Get => \"GET\",\n HttpMethod::Post => \"POST\",\n HttpMethod::Delete => \"DELETE\",\n HttpMethod::Options => \"OPTIONS\",\n };\n let log_path_cstr = CStr::from_ptr((*http_req).path.as_ptr());\n let log_path = log_path_cstr.to_str().unwrap_or(\"???\").to_string();\n\n let mut target_program: *mut c_char = ptr::null_mut();\n let dreq = router_http_to_request(http_req, &mut target_program, &mut err);\n http_free_request(http_req);\n\n if !err.is_null() {\n let err_json = make_error_json(&CStr::from_ptr(err).to_string_lossy());\n let c = CString::new(err_json.as_str()).unwrap_or_default();\n http_write_response(\n client_fd,\n 404,\n ct.as_ptr() as *const c_char,\n c.as_ptr(),\n err_json.len(),\n );\n libc::free(err as *mut c_void);\n let elapsed = req_start.elapsed();\n eprintln!(\"router: {} {} -> 404 ({:.1}ms)\", log_method, log_path, elapsed.as_secs_f64() * 1000.0);\n libc::close(client_fd);\n continue;\n }\n\n // Track response status for access log\n let mut resp_status: i32 = 200;\n\n // Router-level requests\n if target_program.is_null() {\n if (*dreq).method == DaemonMethod::Health {\n // Aggregate per-program health\n let mut all_ok = true;\n let mut prog_entries = Vec::new();\n for i in 0..(*router).n_programs {\n let prog = &*(*router).programs.add(i);\n let name = CStr::from_ptr(prog.name).to_string_lossy();\n let alive =\n prog.daemon_pid > 0 && libc::kill(prog.daemon_pid, 0) == 0;\n if !alive {\n all_ok = false;\n }\n let status_str = if alive { \"ok\" } else { \"error\" };\n prog_entries.push(serde_json::json!({\n \"program\": name.as_ref(),\n \"status\": status_str,\n }));\n }\n let overall = if all_ok { \"ok\" } else { \"degraded\" };\n let body = serde_json::json!({\n \"status\": overall,\n \"programs\": prog_entries,\n }).to_string();\n let status_code = if all_ok { 200 } else { 503 };\n resp_status = status_code;\n let c = CString::new(body.as_str()).unwrap_or_default();\n http_write_response(\n client_fd,\n status_code,\n ct.as_ptr() as *const c_char,\n c.as_ptr(),\n body.len(),\n );\n } else if (*dreq).method == DaemonMethod::Discover {\n let disco = router_build_discovery(router);\n let disco_len = libc::strlen(disco);\n http_write_response(\n client_fd,\n 200,\n ct.as_ptr() as *const c_char,\n disco,\n disco_len,\n );\n libc::free(disco as *mut c_void);\n } else if (*dreq).method == DaemonMethod::Eval {\n // daemon_dispatch takes manifest as first arg, NULL is fine for eval\n let resp = daemon_dispatch(ptr::null_mut(), dreq, ptr::null_mut(), ptr::null());\n let mut resp_len: usize = 0;\n let resp_json = daemon_serialize_response(resp, &mut resp_len);\n let status = if (*resp).success { 200 } else { 500 };\n resp_status = status;\n http_write_response(\n client_fd,\n status,\n ct.as_ptr() as *const c_char,\n resp_json,\n resp_len,\n );\n libc::free(resp_json as *mut c_void);\n daemon_free_response(resp);\n }\n daemon_free_request(dreq);\n let elapsed = req_start.elapsed();\n eprintln!(\"router: {} {} -> {} ({:.1}ms)\", log_method, log_path, resp_status, elapsed.as_secs_f64() * 1000.0);\n libc::close(client_fd);\n continue;\n }\n\n // Per-program request\n if (*dreq).method == DaemonMethod::Health {\n let mut found = false;\n for p in 0..(*router).n_programs {\n let rprog = &*(*router).programs.add(p);\n if CStr::from_ptr(rprog.name) == CStr::from_ptr(target_program) {\n found = true;\n let alive =\n rprog.daemon_pid > 0 && libc::kill(rprog.daemon_pid, 0) == 0;\n let prog_str = CStr::from_ptr(rprog.name).to_string_lossy();\n let body = if alive {\n serde_json::json!({\n \"status\": \"ok\",\n \"program\": prog_str.as_ref(),\n }).to_string()\n } else {\n resp_status = 503;\n serde_json::json!({\n \"status\": \"error\",\n \"program\": prog_str.as_ref(),\n \"error\": \"daemon not running\",\n }).to_string()\n };\n let c = CString::new(body.as_str()).unwrap_or_default();\n http_write_response(\n client_fd,\n resp_status,\n ct.as_ptr() as *const c_char,\n c.as_ptr(),\n body.len(),\n );\n break;\n }\n }\n if !found {\n resp_status = 404;\n let body = b\"{\\\"status\\\":\\\"error\\\",\\\"error\\\":\\\"Unknown program\\\"}\\0\";\n http_write_response(\n client_fd,\n 404,\n ct.as_ptr() as *const c_char,\n body.as_ptr() as *const c_char,\n body.len() - 1,\n );\n }\n } else if (*dreq).method == DaemonMethod::Discover {\n let mut found = false;\n for p in 0..(*router).n_programs {\n let rprog = &*(*router).programs.add(p);\n if CStr::from_ptr(rprog.name) == CStr::from_ptr(target_program) {\n if !rprog.manifest.is_null() {\n let disco = manifest_to_discovery_json(rprog.manifest);\n let disco_len = libc::strlen(disco);\n http_write_response(\n client_fd,\n 200,\n ct.as_ptr() as *const c_char,\n disco,\n disco_len,\n );\n libc::free(disco as *mut c_void);\n found = true;\n }\n break;\n }\n }\n if !found {\n resp_status = 404;\n let body = b\"{\\\"status\\\":\\\"error\\\",\\\"error\\\":\\\"Unknown program\\\"}\\0\";\n http_write_response(\n client_fd,\n 404,\n ct.as_ptr() as *const c_char,\n body.as_ptr() as *const c_char,\n body.len() - 1,\n );\n }\n } else {\n // Forward to program daemon\n let resp = router_forward(router, target_program, dreq, &mut err);\n if !err.is_null() {\n resp_status = 500;\n let err_json =\n make_error_json(&CStr::from_ptr(err).to_string_lossy());\n let c = CString::new(err_json.as_str()).unwrap_or_default();\n http_write_response(\n client_fd,\n 500,\n ct.as_ptr() as *const c_char,\n c.as_ptr(),\n err_json.len(),\n );\n libc::free(err as *mut c_void);\n } else {\n let mut resp_len: usize = 0;\n let resp_json = daemon_serialize_response(resp, &mut resp_len);\n let status = if (*resp).success { 200 } else { 500 };\n resp_status = status;\n http_write_response(\n client_fd,\n status,\n ct.as_ptr() as *const c_char,\n resp_json,\n resp_len,\n );\n libc::free(resp_json as *mut c_void);\n daemon_free_response(resp);\n }\n }\n\n libc::free(target_program as *mut c_void);\n daemon_free_request(dreq);\n let elapsed = req_start.elapsed();\n eprintln!(\"router: {} {} -> {} ({:.1}ms)\", log_method, log_path, resp_status, elapsed.as_secs_f64() * 1000.0);\n libc::close(client_fd);\n }\n }\n\n // Kill all program daemons\n for i in 0..(*router).n_programs {\n let prog = &*(*router).programs.add(i);\n if prog.daemon_pid > 0 {\n libc::kill(prog.daemon_pid, libc::SIGTERM);\n libc::unlink(prog.daemon_socket.as_ptr());\n }\n }\n\n // Wait for children\n for i in 0..(*router).n_programs {\n let prog = &*(*router).programs.add(i);\n if prog.daemon_pid > 0 {\n libc::waitpid(prog.daemon_pid, ptr::null_mut(), 0);\n }\n }\n\n // Close listeners\n for i in 0..nfds {\n libc::close(fds[i].fd);\n }\n\n if !(*config).unix_socket_path.is_null() {\n libc::unlink((*config).unix_socket_path);\n }\n}\n\n/// Build a JSON error response string.\nfn make_error_json(error: &str) -> String {\n let map: serde_json::Map = [\n (\"status\".into(), serde_json::Value::String(\"error\".into())),\n (\"error\".into(), serde_json::Value::String(error.into())),\n ]\n .into_iter()\n .collect();\n serde_json::to_string(&map).unwrap_or_else(|_| \"{}\".into())\n}\n" }, { "path": "data/rust/morloc-runtime/src/schema.rs", "content": "use crate::error::MorlocError;\n\n/// Morloc serial type identifiers, matching the C enum morloc_serial_type.\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\n#[repr(u32)]\npub enum SerialType {\n Nil = 0,\n Bool = 1,\n Sint8 = 2,\n Sint16 = 3,\n Sint32 = 4,\n Sint64 = 5,\n Uint8 = 6,\n Uint16 = 7,\n Uint32 = 8,\n Uint64 = 9,\n Float32 = 10,\n Float64 = 11,\n Tensor = 12,\n String = 13,\n Array = 14,\n Tuple = 15,\n Map = 16,\n Optional = 17,\n}\n\n/// Schema character codes for parsing schema strings.\nconst SCHEMA_NIL: u8 = b'z';\nconst SCHEMA_BOOL: u8 = b'b';\nconst SCHEMA_SINT: u8 = b'i';\nconst SCHEMA_UINT: u8 = b'u';\nconst SCHEMA_FLOAT: u8 = b'f';\nconst SCHEMA_STRING: u8 = b's';\nconst SCHEMA_ARRAY: u8 = b'a';\nconst SCHEMA_TENSOR: u8 = b'T';\nconst SCHEMA_TUPLE: u8 = b't';\nconst SCHEMA_MAP: u8 = b'm';\nconst SCHEMA_OPTIONAL: u8 = b'?';\n\n/// Recursive schema definition, mirroring the C Schema struct.\n#[derive(Debug, Clone)]\npub struct Schema {\n pub serial_type: SerialType,\n /// Number of parameters (e.g., array has 1, tuple has N).\n pub size: usize,\n /// Byte width when stored in a fixed-width array.\n pub width: usize,\n /// Field offsets for tuples/records, or ndim storage for tensors.\n pub offsets: Vec,\n /// Optional type hint string.\n pub hint: Option,\n /// Child schemas (element type for arrays, field types for tuples, etc.).\n pub parameters: Vec,\n /// Field names for records (None for non-record types).\n pub keys: Vec,\n}\n\nimpl Schema {\n pub fn primitive(serial_type: SerialType) -> Self {\n use crate::shm;\n let width = match serial_type {\n SerialType::Nil => 0,\n SerialType::Bool | SerialType::Sint8 | SerialType::Uint8 => 1,\n SerialType::Sint16 | SerialType::Uint16 => 2,\n SerialType::Sint32 | SerialType::Uint32 | SerialType::Float32 => 4,\n SerialType::Sint64 | SerialType::Uint64 | SerialType::Float64 => 8,\n SerialType::String => std::mem::size_of::(),\n _ => 0,\n };\n Schema {\n serial_type,\n size: 0,\n width,\n offsets: Vec::new(),\n hint: None,\n parameters: Vec::new(),\n keys: Vec::new(),\n }\n }\n\n /// Returns true if this type has a fixed byte width (no variable-length data).\n pub fn is_fixed_width(&self) -> bool {\n match self.serial_type {\n SerialType::Nil\n | SerialType::Bool\n | SerialType::Sint8\n | SerialType::Sint16\n | SerialType::Sint32\n | SerialType::Sint64\n | SerialType::Uint8\n | SerialType::Uint16\n | SerialType::Uint32\n | SerialType::Uint64\n | SerialType::Float32\n | SerialType::Float64 => true,\n SerialType::Tuple => self.parameters.iter().all(|p| p.is_fixed_width()),\n SerialType::Optional => false,\n _ => false,\n }\n }\n\n /// Alignment requirement for this type.\n pub fn alignment(&self) -> usize {\n match self.serial_type {\n SerialType::Nil => 1,\n SerialType::Bool | SerialType::Sint8 | SerialType::Uint8 => 1,\n SerialType::Sint16 | SerialType::Uint16 => 2,\n SerialType::Sint32 | SerialType::Uint32 | SerialType::Float32 => 4,\n SerialType::Sint64 | SerialType::Uint64 | SerialType::Float64 => 8,\n SerialType::String | SerialType::Array | SerialType::Map | SerialType::Tensor => {\n std::mem::size_of::() // pointer-sized alignment\n }\n SerialType::Tuple => {\n self.parameters\n .iter()\n .map(|p| p.alignment())\n .max()\n .unwrap_or(1)\n }\n SerialType::Optional => {\n if let Some(inner) = self.parameters.first() {\n std::cmp::max(1, inner.alignment())\n } else {\n 1\n }\n }\n }\n }\n}\n\n/// Parse a schema string into a Schema tree.\n///\n/// Positional format (no parentheses/commas):\n/// - `z` -> Nil, `b` -> Bool, `s` -> String\n/// - `i4` -> Sint32, `u8` -> Uint64, `f8` -> Float64\n/// - `ai4` -> Array of Sint32\n/// - `t2i4s` -> Tuple of (Sint32, String)\n/// - `m24namesi4` -> Map with keys \"name\"->String, \"i4\" (base-62 field count, then key-len + key + value for each)\n/// - `?i4` -> Optional Sint32\n/// - `T2f8` -> 2D Tensor of Float64\n/// - `i4` -> Sint32 with hint annotation\npub fn parse_schema(input: &str) -> Result {\n let bytes = input.as_bytes();\n let (schema, consumed) = parse_schema_r(bytes, 0)?;\n if consumed != bytes.len() {\n return Err(MorlocError::Schema(format!(\n \"trailing characters after schema at position {consumed}\"\n )));\n }\n Ok(schema)\n}\n\n/// Recursive schema parser matching the C `parse_schema_r` format exactly.\nfn parse_schema_r(bytes: &[u8], pos: usize) -> Result<(Schema, usize), MorlocError> {\n if pos >= bytes.len() {\n return Err(MorlocError::Schema(\"unexpected end of schema\".into()));\n }\n\n let c = bytes[pos];\n let cur = pos + 1;\n\n match c {\n b'<' => {\n // Hint: <...> with nesting support, then parse the actual type\n let (hint, after_hint) = parse_hint(bytes, cur)?;\n let (mut schema, end) = parse_schema_r(bytes, after_hint)?;\n schema.hint = Some(hint);\n Ok((schema, end))\n }\n SCHEMA_NIL => Ok((Schema::primitive(SerialType::Nil), cur)),\n SCHEMA_BOOL => Ok((Schema::primitive(SerialType::Bool), cur)),\n SCHEMA_STRING => {\n // String schema has one parameter (uint8) for array compatibility,\n // matching the C string_schema() constructor.\n Ok((Schema {\n serial_type: SerialType::String,\n size: 1,\n width: std::mem::size_of::(),\n offsets: Vec::new(),\n hint: None,\n parameters: vec![Schema::primitive(SerialType::Uint8)],\n keys: Vec::new(),\n }, cur))\n }\n SCHEMA_SINT => parse_sized_int(bytes, cur, true),\n SCHEMA_UINT => parse_sized_int(bytes, cur, false),\n SCHEMA_FLOAT => parse_sized_float(bytes, cur),\n SCHEMA_ARRAY => {\n // Array: one child schema follows immediately\n let (child, end) = parse_schema_r(bytes, cur)?;\n Ok((make_array_schema(child), end))\n }\n SCHEMA_OPTIONAL => {\n // Optional: one child schema follows immediately\n let (child, end) = parse_schema_r(bytes, cur)?;\n Ok((make_optional_schema(child), end))\n }\n SCHEMA_TUPLE => {\n // Tuple: base-62 size char, then N child schemas\n if cur >= bytes.len() {\n return Err(MorlocError::Schema(\"expected tuple size\".into()));\n }\n let n = decode_base62(bytes[cur])?;\n let mut params = Vec::with_capacity(n);\n let mut p = cur + 1;\n for _ in 0..n {\n let (child, end) = parse_schema_r(bytes, p)?;\n params.push(child);\n p = end;\n }\n Ok((make_tuple_schema(params), p))\n }\n SCHEMA_MAP => {\n // Map/record: base-62 size char, then N (key_len_char + key_bytes + value_schema)\n if cur >= bytes.len() {\n return Err(MorlocError::Schema(\"expected map size\".into()));\n }\n let n = decode_base62(bytes[cur])?;\n let mut params = Vec::with_capacity(n);\n let mut keys = Vec::with_capacity(n);\n let mut p = cur + 1;\n for _ in 0..n {\n // Read key: base-62 length char + that many bytes\n if p >= bytes.len() {\n return Err(MorlocError::Schema(\"expected map key length\".into()));\n }\n let key_len = decode_base62(bytes[p])?;\n p += 1;\n if p + key_len > bytes.len() {\n return Err(MorlocError::Schema(\"map key extends past end\".into()));\n }\n let key = std::str::from_utf8(&bytes[p..p + key_len])\n .map_err(|_| MorlocError::Schema(\"invalid UTF-8 in map key\".into()))?\n .to_string();\n p += key_len;\n keys.push(key);\n // Read value schema\n let (child, end) = parse_schema_r(bytes, p)?;\n params.push(child);\n p = end;\n }\n Ok((make_map_schema(params, keys), p))\n }\n SCHEMA_TENSOR => {\n // Tensor: base-62 ndim char, then element schema\n if cur >= bytes.len() {\n return Err(MorlocError::Schema(\"expected tensor ndim\".into()));\n }\n let ndim = decode_base62(bytes[cur])?;\n let (child, end) = parse_schema_r(bytes, cur + 1)?;\n Ok((make_tensor_schema(ndim, child), end))\n }\n _ => Err(MorlocError::Schema(format!(\n \"unknown schema character '{}' at position {pos}\",\n c as char\n ))),\n }\n}\n\n/// Parse hint with nested angle bracket support: `>` etc.\nfn parse_hint(bytes: &[u8], pos: usize) -> Result<(String, usize), MorlocError> {\n let mut depth: usize = 1;\n let start = pos;\n let mut cur = pos;\n while cur < bytes.len() {\n match bytes[cur] {\n b'<' => depth += 1,\n b'>' => {\n depth -= 1;\n if depth == 0 {\n let hint = std::str::from_utf8(&bytes[start..cur])\n .unwrap_or(\"\")\n .to_string();\n return Ok((hint, cur + 1)); // skip closing '>'\n }\n }\n _ => {}\n }\n cur += 1;\n }\n Err(MorlocError::Schema(\"unclosed '<' in hint\".into()))\n}\n\nfn parse_sized_int(\n bytes: &[u8],\n pos: usize,\n signed: bool,\n) -> Result<(Schema, usize), MorlocError> {\n if pos >= bytes.len() {\n return Err(MorlocError::Schema(\"expected size after 'i'/'u'\".into()));\n }\n // Size is a SINGLE base-62 character, not a multi-digit number\n let size = decode_base62(bytes[pos])?;\n let next = pos + 1;\n let st = match (signed, size) {\n (true, 1) => SerialType::Sint8,\n (true, 2) => SerialType::Sint16,\n (true, 4) => SerialType::Sint32,\n (true, 8) => SerialType::Sint64,\n (false, 1) => SerialType::Uint8,\n (false, 2) => SerialType::Uint16,\n (false, 4) => SerialType::Uint32,\n (false, 8) => SerialType::Uint64,\n _ => return Err(MorlocError::Schema(format!(\"invalid integer size {size}\"))),\n };\n Ok((Schema::primitive(st), next))\n}\n\nfn parse_sized_float(bytes: &[u8], pos: usize) -> Result<(Schema, usize), MorlocError> {\n if pos >= bytes.len() {\n return Err(MorlocError::Schema(\"expected size after 'f'\".into()));\n }\n // Size is a SINGLE base-62 character, not a multi-digit number\n let size = decode_base62(bytes[pos])?;\n let next = pos + 1;\n let st = match size {\n 4 => SerialType::Float32,\n 8 => SerialType::Float64,\n _ => return Err(MorlocError::Schema(format!(\"invalid float size {size}\"))),\n };\n Ok((Schema::primitive(st), next))\n}\n\n/// Decode a single base-62 character to a number (0-63).\n/// 0-9 -> 0-9, a-z -> 10-35, A-Z -> 36-61, + -> 62, / -> 63\nfn decode_base62(c: u8) -> Result {\n match c {\n b'0'..=b'9' => Ok((c - b'0') as usize),\n b'a'..=b'z' => Ok((c - b'a') as usize + 10),\n b'A'..=b'Z' => Ok((c - b'A') as usize + 36),\n b'+' => Ok(62),\n b'/' => Ok(63),\n _ => Err(MorlocError::Schema(format!(\n \"invalid base-62 size character '{}'\",\n c as char\n ))),\n }\n}\n\nfn encode_base62(n: usize) -> char {\n match n {\n 0..=9 => (b'0' + n as u8) as char,\n 10..=35 => (b'a' + (n - 10) as u8) as char,\n 36..=61 => (b'A' + (n - 36) as u8) as char,\n 62 => '+',\n 63 => '/',\n _ => '\\x07', // bell - error\n }\n}\n\n// ── Schema constructors ────────────────────────────────────────────────────\n\nfn make_array_schema(child: Schema) -> Schema {\n Schema {\n serial_type: SerialType::Array,\n size: 1,\n width: std::mem::size_of::(),\n offsets: Vec::new(),\n hint: None,\n parameters: vec![child],\n keys: Vec::new(),\n }\n}\n\nfn make_optional_schema(child: Schema) -> Schema {\n let align = child.alignment().max(1);\n let inner_offset = crate::shm::align_up(1, align);\n Schema {\n serial_type: SerialType::Optional,\n size: 1,\n width: inner_offset + child.width,\n offsets: vec![inner_offset],\n hint: None,\n parameters: vec![child],\n keys: Vec::new(),\n }\n}\n\nfn make_tuple_schema(params: Vec) -> Schema {\n let (width, offsets) = calculate_tuple_layout(¶ms);\n let size = params.len();\n Schema {\n serial_type: SerialType::Tuple,\n size,\n width,\n offsets,\n hint: None,\n parameters: params,\n keys: Vec::new(),\n }\n}\n\nfn make_map_schema(params: Vec, keys: Vec) -> Schema {\n let (width, offsets) = calculate_tuple_layout(¶ms);\n let size = params.len();\n Schema {\n serial_type: SerialType::Map,\n size,\n width,\n offsets,\n hint: None,\n parameters: params,\n keys,\n }\n}\n\nfn make_tensor_schema(ndim: usize, child: Schema) -> Schema {\n Schema {\n serial_type: SerialType::Tensor,\n size: 1,\n width: std::mem::size_of::(),\n offsets: vec![ndim],\n hint: None,\n parameters: vec![child],\n keys: Vec::new(),\n }\n}\n\n/// Calculate byte offsets for tuple fields (C struct layout with natural alignment).\nfn calculate_tuple_layout(params: &[Schema]) -> (usize, Vec) {\n let mut offsets = Vec::with_capacity(params.len());\n let mut offset: usize = 0;\n let mut max_align: usize = 1;\n\n for param in params {\n let align = param.alignment();\n max_align = std::cmp::max(max_align, align);\n // Align the offset\n offset = (offset + align - 1) & !(align - 1);\n offsets.push(offset);\n offset += param.width;\n }\n\n // Total width padded to max alignment\n let width = (offset + max_align - 1) & !(max_align - 1);\n (width, offsets)\n}\n\n/// Render a schema back to its string representation.\npub fn schema_to_string(schema: &Schema) -> String {\n let mut buf = String::new();\n schema_to_string_inner(schema, &mut buf);\n buf\n}\n\nfn schema_to_string_inner(schema: &Schema, buf: &mut String) {\n // Write hint if present\n if let Some(ref hint) = schema.hint {\n buf.push('<');\n buf.push_str(hint);\n buf.push('>');\n }\n\n match schema.serial_type {\n SerialType::Nil => buf.push('z'),\n SerialType::Bool => buf.push('b'),\n SerialType::Sint8 => buf.push_str(\"i1\"),\n SerialType::Sint16 => buf.push_str(\"i2\"),\n SerialType::Sint32 => buf.push_str(\"i4\"),\n SerialType::Sint64 => buf.push_str(\"i8\"),\n SerialType::Uint8 => buf.push_str(\"u1\"),\n SerialType::Uint16 => buf.push_str(\"u2\"),\n SerialType::Uint32 => buf.push_str(\"u4\"),\n SerialType::Uint64 => buf.push_str(\"u8\"),\n SerialType::Float32 => buf.push_str(\"f4\"),\n SerialType::Float64 => buf.push_str(\"f8\"),\n SerialType::String => buf.push('s'),\n SerialType::Array => {\n buf.push('a');\n schema_to_string_inner(&schema.parameters[0], buf);\n }\n SerialType::Tuple => {\n buf.push('t');\n buf.push(encode_base62(schema.size));\n for p in &schema.parameters {\n schema_to_string_inner(p, buf);\n }\n }\n SerialType::Map => {\n buf.push('m');\n buf.push(encode_base62(schema.size));\n for (i, p) in schema.parameters.iter().enumerate() {\n if i < schema.keys.len() {\n let key = &schema.keys[i];\n buf.push(encode_base62(key.len()));\n buf.push_str(key);\n }\n schema_to_string_inner(p, buf);\n }\n }\n SerialType::Optional => {\n buf.push('?');\n schema_to_string_inner(&schema.parameters[0], buf);\n }\n SerialType::Tensor => {\n let ndim = schema.offsets.first().copied().unwrap_or(0);\n buf.push('T');\n buf.push(encode_base62(ndim));\n schema_to_string_inner(&schema.parameters[0], buf);\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_parse_primitives() {\n assert_eq!(parse_schema(\"z\").unwrap().serial_type, SerialType::Nil);\n assert_eq!(parse_schema(\"b\").unwrap().serial_type, SerialType::Bool);\n assert_eq!(parse_schema(\"i4\").unwrap().serial_type, SerialType::Sint32);\n assert_eq!(parse_schema(\"u8\").unwrap().serial_type, SerialType::Uint64);\n assert_eq!(parse_schema(\"f8\").unwrap().serial_type, SerialType::Float64);\n assert_eq!(parse_schema(\"s\").unwrap().serial_type, SerialType::String);\n }\n\n #[test]\n fn test_parse_array() {\n let s = parse_schema(\"ai4\").unwrap();\n assert_eq!(s.serial_type, SerialType::Array);\n assert_eq!(s.parameters.len(), 1);\n assert_eq!(s.parameters[0].serial_type, SerialType::Sint32);\n }\n\n #[test]\n fn test_parse_tuple() {\n let s = parse_schema(\"t3i4sf8\").unwrap();\n assert_eq!(s.serial_type, SerialType::Tuple);\n assert_eq!(s.parameters.len(), 3);\n }\n\n #[test]\n fn test_parse_nested() {\n let s = parse_schema(\"at2i4s\").unwrap();\n assert_eq!(s.serial_type, SerialType::Array);\n assert_eq!(s.parameters[0].serial_type, SerialType::Tuple);\n assert_eq!(s.parameters[0].parameters.len(), 2);\n }\n\n #[test]\n fn test_parse_map() {\n let s = parse_schema(\"m21as1bi4\").unwrap();\n assert_eq!(s.serial_type, SerialType::Map);\n assert_eq!(s.parameters.len(), 2);\n assert_eq!(s.keys[0], \"a\");\n assert_eq!(s.keys[1], \"b\");\n }\n\n #[test]\n fn test_parse_optional() {\n let s = parse_schema(\"?f8\").unwrap();\n assert_eq!(s.serial_type, SerialType::Optional);\n assert_eq!(s.parameters[0].serial_type, SerialType::Float64);\n }\n\n #[test]\n fn test_parse_tensor() {\n let s = parse_schema(\"T2f8\").unwrap();\n assert_eq!(s.serial_type, SerialType::Tensor);\n assert_eq!(s.offsets[0], 2); // ndim\n assert_eq!(s.parameters[0].serial_type, SerialType::Float64);\n }\n\n #[test]\n fn test_parse_with_hints() {\n let s = parse_schema(\"f8\").unwrap();\n assert_eq!(s.serial_type, SerialType::Float64);\n assert_eq!(s.hint.as_deref(), Some(\"float\"));\n\n // Nested hints: >\n let s = parse_schema(\">ai4\").unwrap();\n assert_eq!(s.serial_type, SerialType::Array);\n assert_eq!(s.hint.as_deref(), Some(\"std::vector<$1>\"));\n }\n\n #[test]\n fn test_roundtrip() {\n let cases = [\"z\", \"b\", \"i4\", \"u8\", \"f8\", \"s\", \"ai4\", \"t2i4s\", \"?i4\", \"T2f8\"];\n for case in cases {\n let schema = parse_schema(case).unwrap();\n let rendered = schema_to_string(&schema);\n assert_eq!(rendered, case, \"roundtrip failed for '{case}'\");\n }\n }\n\n #[test]\n fn test_roundtrip_map() {\n let input = \"m24names4infoi4\";\n let schema = parse_schema(input).unwrap();\n assert_eq!(schema.serial_type, SerialType::Map);\n assert_eq!(schema.keys, vec![\"name\", \"info\"]);\n let rendered = schema_to_string(&schema);\n assert_eq!(rendered, input);\n }\n}\n\n#[cfg(test)]\nmod compat_tests {\n use super::*;\n\n fn dump(label: &str, s: &Schema, depth: usize) {\n let indent = \" \".repeat(depth);\n print!(\"{}{}: type={} size={} width={}\", indent, label, s.serial_type as u32, s.size, s.width);\n if !s.offsets.is_empty() {\n print!(\" offsets={:?}\", s.offsets);\n }\n if let Some(ref h) = s.hint { print!(\" hint=\\\"{}\\\"\", h); }\n if !s.keys.is_empty() { print!(\" keys={:?}\", s.keys); }\n println!();\n for (i, p) in s.parameters.iter().enumerate() {\n dump(&format!(\"param[{}]\", i), p, depth + 1);\n }\n }\n\n #[test]\n fn test_schema_compat_with_c() {\n // These must match the C output exactly\n let cases = vec![\n (\"s\", \"type=13 size=1 width=16\"),\n (\"ai4\", \"type=14 size=1 width=16\"),\n (\"t2i4s\", \"type=15 size=2 width=24\"),\n (\"?i4\", \"type=17 size=1 width=8\"),\n (\"?s\", \"type=17 size=1 width=24\"),\n (\"T2f8\", \"type=12 size=1 width=32\"),\n ];\n for (input, expected_root) in &cases {\n let s = parse_schema(input).unwrap();\n let got = format!(\"type={} size={} width={}\", s.serial_type as u32, s.size, s.width);\n assert_eq!(&got, *expected_root, \"Schema '{}' mismatch\", input);\n }\n\n // Verify tuple offsets\n let t = parse_schema(\"t2i4s\").unwrap();\n assert_eq!(t.offsets, vec![0, 8], \"t2i4s offsets\");\n\n // Verify optional offsets\n let o = parse_schema(\"?i4\").unwrap();\n assert_eq!(o.offsets, vec![4], \"?i4 offsets\");\n let os = parse_schema(\"?s\").unwrap();\n assert_eq!(os.offsets, vec![8], \"?s offsets\");\n\n // Verify string has uint8 parameter\n let s = parse_schema(\"s\").unwrap();\n assert_eq!(s.parameters.len(), 1);\n assert_eq!(s.parameters[0].serial_type, SerialType::Uint8);\n assert_eq!(s.parameters[0].width, 1);\n\n // Verify tensor\n let t = parse_schema(\"T2f8\").unwrap();\n assert_eq!(t.offsets, vec![2]); // ndim\n assert_eq!(t.width, 32); // sizeof(Tensor)\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/shm.rs", "content": "//! Shared memory management with multi-volume support.\n//!\n//! Replaces shm.c / memory.h. Uses AtomicU32 + futex for cross-process locking\n//! instead of pthread_rwlock_t, providing crash-safety and portability.\n\nuse crate::error::MorlocError;\nuse std::sync::atomic::{AtomicU32, Ordering};\nuse std::sync::Mutex;\n\n/// Cross-platform file pre-allocation.\n/// Linux: posix_fallocate (allocates disk blocks).\n/// macOS: ftruncate (extends file, may be sparse).\n#[cfg(target_os = \"linux\")]\nunsafe fn preallocate_fd(fd: i32, size: i64) -> i32 {\n libc::posix_fallocate(fd, 0, size)\n}\n\n#[cfg(target_os = \"macos\")]\nunsafe fn preallocate_fd(fd: i32, size: i64) -> i32 {\n if libc::ftruncate(fd, size) == -1 { -1 } else { 0 }\n}\n\n// ── Constants ──────────────────────────────────────────────────────────────\n\npub const SHM_MAGIC: u32 = 0xFECA_0DF0;\npub const BLK_MAGIC: u32 = 0x0CB1_0DF0;\npub const MAX_VOLUME_NUMBER: usize = 32;\npub const MAX_FILENAME_SIZE: usize = 128;\npub const MAX_PATH_SIZE: usize = 512;\n\nconst LOCK_UNLOCKED: u32 = 0;\nconst LOCK_LOCKED: u32 = 1;\nconst SPIN_LIMIT: u32 = 40;\n#[cfg(target_os = \"linux\")]\nconst LOCK_TIMEOUT_SECS: u64 = 5;\n\n// ── Pointer types ──────────────────────────────────────────────────────────\n\n/// Relative pointer: index into the multi-volume pool (cross-process safe).\npub type RelPtr = isize;\n/// Volume-local pointer: offset within a single volume.\npub type VolPtr = isize;\n/// Absolute pointer: virtual address in this process.\npub type AbsPtr = *mut u8;\n\npub const RELNULL: RelPtr = -1;\npub const VOLNULL: VolPtr = -1;\n\n// ── Block alignment ────────────────────────────────────────────────────────\n\npub const BLOCK_ALIGN: usize = std::mem::align_of::();\n\n#[inline]\npub const fn align_up(x: usize, align: usize) -> usize {\n (x + align - 1) & !(align - 1)\n}\n\n// ── Shared memory header (lives in mmap'd region) ──────────────────────────\n\n#[repr(C)]\npub struct ShmHeader {\n pub magic: u32,\n pub volume_name: [u8; MAX_FILENAME_SIZE],\n pub volume_index: i32,\n pub volume_size: usize,\n pub relative_offset: usize,\n pub lock: AtomicU32,\n pub cursor: VolPtr,\n}\n\n#[repr(C)]\npub struct BlockHeader {\n pub magic: u32,\n pub reference_count: AtomicU32,\n pub size: usize,\n}\n\nconst _: () = assert!(\n std::mem::size_of::()\n == std::mem::size_of::()\n + std::mem::size_of::()\n + std::mem::size_of::()\n);\n\n// ── Voidstar data structures (used by serialization) ───────────────────────\n\n/// Variable-length array/string representation in SHM.\n#[derive(Clone, Copy)]\n#[repr(C)]\npub struct Array {\n pub size: usize,\n pub data: RelPtr,\n}\n\n/// N-dimensional dense tensor in SHM.\n#[repr(C)]\npub struct Tensor {\n pub total_elements: usize,\n pub device_type: u32,\n pub device_id: u32,\n pub data: RelPtr,\n pub shape: RelPtr,\n}\n\n// ── Send wrapper for raw pointers ──────────────────────────────────────────\n\n#[derive(Clone, Copy)]\nstruct SendPtr(*mut ShmHeader);\n// SAFETY: ShmHeader lives in mmap'd shared memory that outlives all threads.\n// Access is serialized via VOLUMES Mutex and per-volume AtomicU32 futex locks.\nunsafe impl Send for SendPtr {}\nimpl SendPtr {\n const fn null() -> Self { SendPtr(std::ptr::null_mut()) }\n fn is_null(&self) -> bool { self.0.is_null() }\n fn ptr(&self) -> *mut ShmHeader { self.0 }\n fn set(&mut self, p: *mut ShmHeader) { self.0 = p; }\n}\n\nfn get_cstr_buf(buf: &[u8; MAX_FILENAME_SIZE]) -> &str {\n get_cstr(buf.as_slice())\n}\n\n// ── Global state ───────────────────────────────────────────────────────────\n\nstatic CURRENT_VOLUME: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);\n\nstatic VOLUMES: Mutex<[SendPtr; MAX_VOLUME_NUMBER]> =\n Mutex::new([SendPtr::null(); MAX_VOLUME_NUMBER]);\n\nstatic ALLOC_MUTEX: Mutex<()> = Mutex::new(());\n\nstatic COMMON_BASENAME: Mutex<[u8; MAX_FILENAME_SIZE]> = Mutex::new([0u8; MAX_FILENAME_SIZE]);\n\nstatic FALLBACK_DIR: Mutex<[u8; MAX_FILENAME_SIZE]> = Mutex::new([0u8; MAX_FILENAME_SIZE]);\n\nfn set_cstr(buf: &mut [u8], s: &str) {\n let bytes = s.as_bytes();\n let len = bytes.len().min(buf.len() - 1);\n buf[..len].copy_from_slice(&bytes[..len]);\n buf[len] = 0;\n}\n\nfn get_cstr(buf: &[u8]) -> &str {\n let end = buf.iter().position(|&b| b == 0).unwrap_or(buf.len());\n std::str::from_utf8(&buf[..end]).unwrap_or(\"\")\n}\n\n// ── Public API ─────────────────────────────────────────────────────────────\n\n/// Set fallback directory for file-backed SHM when /dev/shm is too small.\npub fn shm_set_fallback_dir(dir: &str) {\n let mut fb = FALLBACK_DIR.lock().unwrap();\n set_cstr(&mut *fb, dir);\n}\n\n/// Initialize a new SHM volume.\npub fn shinit(\n shm_basename: &str,\n volume_index: usize,\n shm_size: usize,\n) -> Result<*mut ShmHeader, MorlocError> {\n let full_size = shm_size + std::mem::size_of::();\n let shm_name = format!(\"{}_{}\", shm_basename, volume_index);\n\n // Store common basename\n {\n let mut cb = COMMON_BASENAME.lock().unwrap();\n set_cstr(&mut *cb, shm_basename);\n }\n\n // Try POSIX shared memory first, fall back to file-backed\n let (fd, created, volume_label, actual_full_size) =\n try_open_shm(&shm_name, full_size)?;\n\n // SAFETY: mmap with MAP_SHARED on a valid fd obtained from shm_open/open above.\n // The returned pointer is checked against MAP_FAILED before use.\n let ptr = unsafe {\n libc::mmap(\n std::ptr::null_mut(),\n actual_full_size,\n libc::PROT_READ | libc::PROT_WRITE,\n libc::MAP_SHARED,\n fd,\n 0,\n )\n };\n // SAFETY: fd is a valid file descriptor opened above.\n unsafe { libc::close(fd) };\n\n if ptr == libc::MAP_FAILED {\n return Err(MorlocError::Shm(format!(\n \"Failed to mmap volume '{}' ({} bytes)\",\n volume_label, actual_full_size\n )));\n }\n\n let shm = ptr as *mut ShmHeader;\n\n // Store in volumes array\n {\n let mut vols = VOLUMES.lock().unwrap();\n vols[volume_index].set(shm);\n }\n\n let actual_data_size = actual_full_size - std::mem::size_of::();\n\n if created {\n // SAFETY: shm points to the start of our mmap'd region of actual_full_size bytes.\n // We just created it, so we have exclusive access for initialization.\n unsafe {\n (*shm).magic = SHM_MAGIC;\n let mut name_buf = [0u8; MAX_FILENAME_SIZE];\n set_cstr(&mut name_buf, &volume_label);\n (*shm).volume_name = name_buf;\n (*shm).volume_index = volume_index as i32;\n\n // Calculate relative offset from prior volumes\n let vols = VOLUMES.lock().unwrap();\n let mut rel_offset = 0usize;\n for i in 0..volume_index {\n if !vols[i].is_null() {\n rel_offset += (*vols[i].ptr()).volume_size;\n }\n }\n (*shm).relative_offset = rel_offset;\n (*shm).volume_size = actual_data_size;\n (*shm).lock = AtomicU32::new(LOCK_UNLOCKED);\n (*shm).cursor = 0;\n\n // Initialize first block header\n let first_block =\n (shm as *mut u8).add(std::mem::size_of::()) as *mut BlockHeader;\n (*first_block).magic = BLK_MAGIC;\n (*first_block).reference_count = AtomicU32::new(0);\n (*first_block).size = actual_data_size - std::mem::size_of::();\n }\n }\n\n Ok(shm)\n}\n\n/// Open an existing SHM volume (or return cached pointer).\npub fn shopen(volume_index: usize) -> Result, MorlocError> {\n {\n let vols = VOLUMES.lock().unwrap();\n if !vols[volume_index].is_null() {\n return Ok(Some(vols[volume_index].ptr()));\n }\n }\n\n let basename = {\n let cb = COMMON_BASENAME.lock().unwrap();\n get_cstr_buf(&cb).to_string()\n };\n if basename.is_empty() {\n return Ok(None);\n }\n\n let shm_name = format!(\"{}_{}\", basename, volume_index);\n\n // Try POSIX SHM\n let name_cstr = std::ffi::CString::new(shm_name.as_str()).unwrap();\n // SAFETY: name_cstr is a valid null-terminated CString.\n let fd = unsafe { libc::shm_open(name_cstr.as_ptr(), libc::O_RDWR, 0o666) };\n\n let fd = if fd == -1 {\n // Try file-backed fallback\n let fb = FALLBACK_DIR.lock().unwrap();\n let fallback = get_cstr_buf(&fb);\n if fallback.is_empty() {\n return Ok(None);\n }\n let file_path = format!(\"{}/{}\", fallback, shm_name);\n let path_cstr = std::ffi::CString::new(file_path.as_str()).unwrap();\n let fd2 = unsafe { libc::open(path_cstr.as_ptr(), libc::O_RDWR) };\n if fd2 == -1 {\n return Ok(None);\n }\n fd2\n } else {\n fd\n };\n\n // SAFETY: zeroed memory is valid for libc::stat. fstat/close on valid fd.\n let mut sb: libc::stat = unsafe { std::mem::zeroed() };\n if unsafe { libc::fstat(fd, &mut sb) } == -1 {\n unsafe { libc::close(fd) };\n return Err(MorlocError::Shm(format!(\n \"Cannot fstat SHM volume '{}'\",\n shm_name\n )));\n }\n let volume_size = sb.st_size as usize;\n\n // SAFETY: mmap with MAP_SHARED on a valid fd; result checked against MAP_FAILED.\n let ptr = unsafe {\n libc::mmap(\n std::ptr::null_mut(),\n volume_size,\n libc::PROT_READ | libc::PROT_WRITE,\n libc::MAP_SHARED,\n fd,\n 0,\n )\n };\n // SAFETY: fd is a valid file descriptor opened above.\n unsafe { libc::close(fd) };\n\n if ptr == libc::MAP_FAILED {\n return Err(MorlocError::Shm(format!(\n \"Cannot mmap SHM volume '{}'\",\n shm_name\n )));\n }\n\n let shm = ptr as *mut ShmHeader;\n {\n let mut vols = VOLUMES.lock().unwrap();\n vols[volume_index].set(shm);\n }\n\n Ok(Some(shm))\n}\n\n/// Close and unlink all SHM volumes.\npub fn shclose() -> Result<(), MorlocError> {\n let _lock = ALLOC_MUTEX.lock().unwrap();\n let mut vols = VOLUMES.lock().unwrap();\n\n for i in 0..MAX_VOLUME_NUMBER {\n let shm = if !vols[i].is_null() {\n vols[i].ptr()\n } else {\n continue;\n };\n\n // SAFETY: shm is a valid mmap'd pointer stored in VOLUMES.\n // munmap/unlink on regions we own. Name read from valid ShmHeader.\n unsafe {\n let name = get_cstr(&(*shm).volume_name).to_string();\n let full_size = (*shm).volume_size + std::mem::size_of::();\n libc::munmap(shm as *mut libc::c_void, full_size);\n\n // Unlink: file-backed volumes start with '/', POSIX SHM does not\n if name.starts_with('/') {\n let cstr = std::ffi::CString::new(name.as_str()).unwrap();\n libc::unlink(cstr.as_ptr());\n } else {\n let cstr = std::ffi::CString::new(name.as_str()).unwrap();\n libc::shm_unlink(cstr.as_ptr());\n }\n }\n vols[i] = SendPtr::null();\n }\n Ok(())\n}\n\n/// Allocate `size` bytes from shared memory.\npub fn shmalloc(size: usize) -> Result {\n // Allow 0-size: round up to minimum block alignment.\n // Needed for nil type (width=0) in morloc_eval.\n let size = if size == 0 { BLOCK_ALIGN } else { align_up(size, BLOCK_ALIGN) };\n let _lock = ALLOC_MUTEX.lock().unwrap();\n shmalloc_unlocked(size)\n}\n\n/// Copy data into a new SHM allocation.\npub fn shmemcpy(src: *const u8, size: usize) -> Result {\n let dest = shmalloc(size)?;\n // SAFETY: dest is a freshly allocated SHM block of `size` bytes.\n // Caller guarantees src points to `size` readable bytes.\n unsafe { std::ptr::copy_nonoverlapping(src, dest, size) };\n Ok(dest)\n}\n\n/// Allocate and zero-fill.\npub fn shcalloc(nmemb: usize, size: usize) -> Result {\n let total = nmemb * size;\n let ptr = shmalloc(total)?;\n // SAFETY: ptr is a freshly allocated SHM block of `total` bytes.\n unsafe { std::ptr::write_bytes(ptr, 0, total) };\n Ok(ptr)\n}\n\n/// Free a shared memory block (decrement reference count).\npub fn shfree(ptr: AbsPtr) -> Result<(), MorlocError> {\n let _lock = ALLOC_MUTEX.lock().unwrap();\n shfree_unlocked(ptr)\n}\n\n/// Increment reference count on a shared memory block.\npub fn shincref(ptr: AbsPtr) -> Result<(), MorlocError> {\n if ptr.is_null() {\n return Err(MorlocError::Shm(\"Cannot incref NULL pointer\".into()));\n }\n // SAFETY: ptr was returned by shmalloc, which places a BlockHeader immediately before\n // the returned data pointer. Magic check below validates the header.\n let blk = unsafe { &*(ptr.sub(std::mem::size_of::()) as *const BlockHeader) };\n if blk.magic != BLK_MAGIC {\n return Err(MorlocError::Shm(\"Corrupted memory - invalid magic\".into()));\n }\n blk.reference_count.fetch_add(1, Ordering::AcqRel);\n Ok(())\n}\n\n/// Convert relative pointer to absolute pointer.\npub fn rel2abs(ptr: RelPtr) -> Result {\n if ptr < 0 {\n return Err(MorlocError::Shm(format!(\"Illegal relptr value {}\", ptr)));\n }\n let mut remaining = ptr as usize;\n\n // First try with volumes already mapped\n {\n let vols = VOLUMES.lock().unwrap();\n for i in 0..MAX_VOLUME_NUMBER {\n if vols[i].is_null() {\n break; // No more volumes mapped\n }\n let shm = vols[i].ptr();\n // SAFETY: shm is a valid mmap'd ShmHeader pointer from VOLUMES.\n let vol_size = unsafe { (*shm).volume_size };\n if remaining < vol_size {\n // SAFETY: data region starts after ShmHeader; remaining < vol_size\n // guarantees the offset is within the mmap'd region.\n let base = unsafe {\n (shm as *const u8).add(std::mem::size_of::())\n };\n return Ok(unsafe { base.add(remaining) as AbsPtr });\n }\n remaining -= vol_size;\n }\n }\n\n // If not found, try opening unmapped volumes\n remaining = ptr as usize;\n for i in 0..MAX_VOLUME_NUMBER {\n let shm = match shopen(i)? {\n Some(s) => s,\n None => {\n return Err(MorlocError::Shm(format!(\n \"Failed to find volume for relptr {}\", ptr\n )));\n }\n };\n // SAFETY: shm is a valid mmap'd ShmHeader pointer from shopen.\n let vol_size = unsafe { (*shm).volume_size };\n if remaining < vol_size {\n // SAFETY: same as above - offset within mmap'd region.\n let base = unsafe {\n (shm as *const u8).add(std::mem::size_of::())\n };\n return Ok(unsafe { base.add(remaining) as AbsPtr });\n }\n remaining -= vol_size;\n }\n\n Err(MorlocError::Shm(format!(\n \"Shared memory pool does not contain index {}\", ptr\n )))\n}\n\n/// Convert absolute pointer to relative pointer.\npub fn abs2rel(ptr: AbsPtr) -> Result {\n let vols = VOLUMES.lock().unwrap();\n for i in 0..MAX_VOLUME_NUMBER {\n let shm = vols[i].ptr();\n if shm.is_null() {\n continue;\n }\n // SAFETY: shm is a valid mmap'd ShmHeader from VOLUMES. We compute\n // data region bounds and check ptr falls within before computing offset.\n unsafe {\n let data_start = (shm as *const u8).add(std::mem::size_of::());\n let data_end = data_start.add((*shm).volume_size);\n let p = ptr as *const u8;\n if p >= data_start && p < data_end {\n let offset = p.offset_from(data_start) as usize;\n return Ok(((*shm).relative_offset + offset) as RelPtr);\n }\n }\n }\n Err(MorlocError::Shm(format!(\n \"Failed to find absptr {:?} in shared memory\",\n ptr\n )))\n}\n\n/// Find the ShmHeader for a given absolute pointer.\npub fn abs2shm(ptr: AbsPtr) -> Result<*mut ShmHeader, MorlocError> {\n let vols = VOLUMES.lock().unwrap();\n for i in 0..MAX_VOLUME_NUMBER {\n let shm = vols[i].ptr();\n if shm.is_null() {\n continue;\n }\n // SAFETY: shm is a valid mmap'd ShmHeader from VOLUMES.\n unsafe {\n let data_start = (shm as *const u8).add(std::mem::size_of::());\n let data_end = data_start.add((*shm).volume_size);\n let p = ptr as *const u8;\n if p >= data_start && p < data_end {\n return Ok(shm);\n }\n }\n }\n Err(MorlocError::Shm(\"Failed to find absptr in SHM\".into()))\n}\n\n/// Total size of all SHM volumes.\npub fn total_shm_size() -> usize {\n let vols = VOLUMES.lock().unwrap();\n let mut total = 0;\n for i in 0..MAX_VOLUME_NUMBER {\n if !vols[i].is_null() {\n // SAFETY: non-null VOLUMES entries are valid mmap'd ShmHeader pointers.\n total += unsafe { (*vols[i].ptr()).volume_size };\n }\n }\n total\n}\n\n// ── Internal helpers ───────────────────────────────────────────────────────\n\nfn try_open_shm(\n shm_name: &str,\n full_size: usize,\n) -> Result<(i32, bool, String, usize), MorlocError> {\n let name_cstr = std::ffi::CString::new(shm_name).unwrap();\n\n // Try POSIX SHM\n let fd = unsafe {\n libc::shm_open(\n name_cstr.as_ptr(),\n libc::O_RDWR | libc::O_CREAT,\n 0o666,\n )\n };\n\n if fd >= 0 {\n let mut sb: libc::stat = unsafe { std::mem::zeroed() };\n if unsafe { libc::fstat(fd, &mut sb) } == -1 {\n unsafe { libc::close(fd) };\n return Err(MorlocError::Shm(format!(\"fstat failed for '{}'\", shm_name)));\n }\n let created = sb.st_size == 0;\n if created {\n let err = unsafe { preallocate_fd(fd, full_size as i64) };\n if err == 0 {\n return Ok((fd, true, shm_name.to_string(), full_size));\n }\n // /dev/shm too small, clean up and try file-backed\n unsafe {\n libc::close(fd);\n libc::shm_unlink(name_cstr.as_ptr());\n }\n } else {\n return Ok((fd, false, shm_name.to_string(), sb.st_size as usize));\n }\n }\n\n // Try file-backed fallback\n let fb = FALLBACK_DIR.lock().unwrap();\n let fallback = get_cstr_buf(&fb);\n if fallback.is_empty() {\n return Err(MorlocError::Shm(format!(\n \"Failed to allocate SHM '{}': /dev/shm too small and no fallback directory\",\n shm_name\n )));\n }\n let file_path = format!(\"{}/{}\", fallback, shm_name);\n drop(fb);\n\n let path_cstr = std::ffi::CString::new(file_path.as_str()).unwrap();\n let fd = unsafe { libc::open(path_cstr.as_ptr(), libc::O_RDWR | libc::O_CREAT, 0o666) };\n if fd == -1 {\n return Err(MorlocError::Shm(format!(\n \"Failed to create file-backed volume '{}'\",\n file_path\n )));\n }\n\n let mut sb: libc::stat = unsafe { std::mem::zeroed() };\n if unsafe { libc::fstat(fd, &mut sb) } == -1 {\n unsafe { libc::close(fd) };\n return Err(MorlocError::Shm(format!(\"fstat failed for '{}'\", file_path)));\n }\n let created = sb.st_size == 0;\n let actual_size = if created {\n let err = unsafe { preallocate_fd(fd, full_size as i64) };\n if err != 0 {\n unsafe {\n libc::close(fd);\n libc::unlink(path_cstr.as_ptr());\n }\n return Err(MorlocError::Shm(format!(\n \"Failed to allocate file-backed volume '{}' ({} bytes)\",\n file_path, full_size\n )));\n }\n full_size\n } else {\n sb.st_size as usize\n };\n\n Ok((fd, created, file_path, actual_size))\n}\n\nfn shmalloc_unlocked(size: usize) -> Result {\n let mut shm: *mut ShmHeader = std::ptr::null_mut();\n let blk = find_free_block(size, &mut shm)?;\n\n // Split and claim\n let final_blk = split_block(shm, blk, size)?;\n // SAFETY: final_blk is a valid BlockHeader in mmap'd SHM found by find_free_block.\n // The data region starts immediately after the header.\n unsafe {\n (*final_blk).reference_count.store(1, Ordering::Release);\n Ok((final_blk as *mut u8).add(std::mem::size_of::()))\n }\n}\n\nfn shfree_unlocked(ptr: AbsPtr) -> Result<(), MorlocError> {\n if ptr.is_null() {\n return Err(MorlocError::Shm(\"Cannot free NULL pointer\".into()));\n }\n // SAFETY: ptr was returned by shmalloc, which places a BlockHeader\n // immediately before the data. Magic check validates correctness.\n let blk = unsafe {\n &*(ptr.sub(std::mem::size_of::()) as *const BlockHeader)\n };\n if blk.magic != BLK_MAGIC {\n return Err(MorlocError::Shm(\"Corrupted memory\".into()));\n }\n if blk.reference_count.load(Ordering::Acquire) == 0 {\n return Err(MorlocError::Shm(\"Reference count already 0\".into()));\n }\n let prev = blk.reference_count.fetch_sub(1, Ordering::AcqRel);\n if prev == 1 {\n // SAFETY: ptr points to blk.size bytes of SHM data we own (refcount just hit 0).\n unsafe {\n std::ptr::write_bytes(ptr, 0, blk.size);\n }\n }\n Ok(())\n}\n\nfn find_free_block(\n size: usize,\n shm_out: &mut *mut ShmHeader,\n) -> Result<*mut BlockHeader, MorlocError> {\n let cv = CURRENT_VOLUME.load(Ordering::Relaxed);\n let vols = VOLUMES.lock().unwrap();\n\n // Try current volume first\n let shm = vols[cv].ptr();\n if !shm.is_null() {\n if let Some(blk) = find_free_block_in_volume(shm, size)? {\n *shm_out = shm;\n return Ok(blk);\n }\n }\n\n // Search all volumes\n for i in 0..MAX_VOLUME_NUMBER {\n let shm = vols[i].ptr();\n if shm.is_null() {\n // Create a new volume\n drop(vols);\n let new_size = std::cmp::max(size * 2, 0xffff);\n let basename = {\n let cb = COMMON_BASENAME.lock().unwrap();\n get_cstr_buf(&cb).to_string()\n };\n let new_shm = shinit(&basename, i, new_size)?;\n CURRENT_VOLUME.store(i, Ordering::Relaxed);\n *shm_out = new_shm;\n let blk = unsafe {\n (new_shm as *mut u8).add(std::mem::size_of::()) as *mut BlockHeader\n };\n return Ok(blk);\n }\n\n if let Some(blk) = find_free_block_in_volume(shm, size)? {\n CURRENT_VOLUME.store(i, Ordering::Relaxed);\n *shm_out = shm;\n return Ok(blk);\n }\n }\n\n Err(MorlocError::Shm(format!(\n \"Could not find suitable block for {} bytes\",\n size\n )))\n}\n\nfn find_free_block_in_volume(\n shm: *mut ShmHeader,\n size: usize,\n) -> Result, MorlocError> {\n unsafe {\n let shm_end = (shm as *const u8)\n .add(std::mem::size_of::())\n .add((*shm).volume_size);\n\n shm_lock(&(*shm).lock)?;\n\n // Try cursor position first\n let cursor = (*shm).cursor;\n if cursor != VOLNULL {\n let blk = vol2abs_raw(cursor, shm);\n let blk = blk as *mut BlockHeader;\n if (*blk).magic == BLK_MAGIC\n && (*blk).reference_count.load(Ordering::Relaxed) == 0\n && (*blk).size >= size\n {\n shm_unlock(&(*shm).lock);\n return Ok(Some(blk));\n }\n }\n\n // Scan from cursor forward\n let start_blk = if cursor != VOLNULL {\n vol2abs_raw(cursor, shm) as *mut BlockHeader\n } else {\n vol2abs_raw(0, shm) as *mut BlockHeader\n };\n\n if let Some(blk) = scan_volume(start_blk, size, shm_end as *const u8) {\n shm_unlock(&(*shm).lock);\n return Ok(Some(blk));\n }\n\n // Wrap around: scan from beginning to cursor\n if cursor > 0 {\n let first_blk = vol2abs_raw(0, shm) as *mut BlockHeader;\n let cursor_end = vol2abs_raw(cursor, shm);\n if let Some(blk) = scan_volume(first_blk, size, cursor_end as *const u8) {\n shm_unlock(&(*shm).lock);\n return Ok(Some(blk));\n }\n }\n\n shm_unlock(&(*shm).lock);\n Ok(None)\n }\n}\n\n/// Scan a volume region for a free block of at least `size` bytes, merging adjacent free blocks.\n///\n/// # Safety\n/// `blk` must point to a valid BlockHeader within an mmap'd SHM volume.\n/// `end` must point to the byte past the end of the volume's data region.\nunsafe fn scan_volume(\n mut blk: *mut BlockHeader,\n size: usize,\n end: *const u8,\n) -> Option<*mut BlockHeader> {\n let hdr_size = std::mem::size_of::();\n while (blk as *const u8).add(hdr_size + size) <= end {\n if blk.is_null() || (*blk).magic != BLK_MAGIC {\n return None;\n }\n\n // Merge adjacent free blocks\n while (*blk).reference_count.load(Ordering::Relaxed) == 0 {\n let next = (blk as *mut u8).add(hdr_size + (*blk).size) as *mut BlockHeader;\n if (next as *const u8) >= end\n || (*next).magic != BLK_MAGIC\n || (*next).reference_count.load(Ordering::Relaxed) != 0\n {\n break;\n }\n (*blk).size += hdr_size + (*next).size;\n }\n\n if (*blk).reference_count.load(Ordering::Relaxed) == 0 && (*blk).size >= size {\n return Some(blk);\n }\n\n blk = (blk as *mut u8).add(hdr_size + (*blk).size) as *mut BlockHeader;\n }\n None\n}\n\nfn split_block(\n shm: *mut ShmHeader,\n blk: *mut BlockHeader,\n size: usize,\n) -> Result<*mut BlockHeader, MorlocError> {\n unsafe {\n if (*blk).size == size {\n return Ok(blk);\n }\n\n shm_lock(&(*shm).lock)?;\n\n let remaining = (*blk).size - size;\n (*blk).size = size;\n\n let hdr_size = std::mem::size_of::();\n let new_free = (blk as *mut u8).add(hdr_size + size) as *mut BlockHeader;\n\n if remaining > hdr_size {\n (*new_free).magic = BLK_MAGIC;\n (*new_free).reference_count = AtomicU32::new(0);\n (*new_free).size = remaining - hdr_size;\n\n // Update cursor\n let data_start = (shm as *const u8).add(std::mem::size_of::());\n (*shm).cursor = (new_free as *const u8).offset_from(data_start) as VolPtr;\n } else {\n (*blk).size += remaining;\n (*shm).cursor = VOLNULL;\n }\n\n shm_unlock(&(*shm).lock);\n Ok(blk)\n }\n}\n\n/// Convert a volume-local offset to an absolute pointer.\n///\n/// # Safety\n/// `shm` must be a valid mmap'd ShmHeader. `ptr` must be within the volume's data region.\n#[inline]\nunsafe fn vol2abs_raw(ptr: VolPtr, shm: *const ShmHeader) -> *mut u8 {\n (shm as *const u8)\n .add(std::mem::size_of::())\n .add(ptr as usize) as *mut u8\n}\n\n// ── Futex-based lock ───────────────────────────────────────────────────────\n\n/// Acquire a futex-based cross-process lock on shared memory.\n///\n/// # Safety\n/// `lock` must point to an AtomicU32 in mmap'd shared memory that\n/// persists for the duration of the lock. The caller must call shm_unlock\n/// on the same lock when done.\npub unsafe fn shm_lock(lock: &AtomicU32) -> Result<(), MorlocError> {\n if lock\n .compare_exchange_weak(LOCK_UNLOCKED, LOCK_LOCKED, Ordering::Acquire, Ordering::Relaxed)\n .is_ok()\n {\n return Ok(());\n }\n\n for _ in 0..SPIN_LIMIT {\n std::hint::spin_loop();\n if lock\n .compare_exchange_weak(LOCK_UNLOCKED, LOCK_LOCKED, Ordering::Acquire, Ordering::Relaxed)\n .is_ok()\n {\n return Ok(());\n }\n }\n\n shm_lock_slow(lock)\n}\n\n#[cfg(target_os = \"linux\")]\nunsafe fn shm_lock_slow(lock: &AtomicU32) -> Result<(), MorlocError> {\n let timeout = libc::timespec {\n tv_sec: LOCK_TIMEOUT_SECS as i64,\n tv_nsec: 0,\n };\n\n loop {\n let ptr = lock as *const AtomicU32 as *const u32;\n libc::syscall(\n libc::SYS_futex, ptr, libc::FUTEX_WAIT, LOCK_LOCKED,\n &timeout as *const libc::timespec, std::ptr::null::(), 0u32,\n );\n\n if lock\n .compare_exchange_weak(LOCK_UNLOCKED, LOCK_LOCKED, Ordering::Acquire, Ordering::Relaxed)\n .is_ok()\n {\n return Ok(());\n }\n\n if lock.load(Ordering::Relaxed) == LOCK_LOCKED {\n if lock\n .compare_exchange(LOCK_LOCKED, LOCK_UNLOCKED, Ordering::AcqRel, Ordering::Relaxed)\n .is_ok()\n {\n if lock\n .compare_exchange(LOCK_UNLOCKED, LOCK_LOCKED, Ordering::Acquire, Ordering::Relaxed)\n .is_ok()\n {\n return Ok(());\n }\n }\n }\n }\n}\n\n/// macOS fallback: spin-yield loop (no futex available).\n#[cfg(target_os = \"macos\")]\nunsafe fn shm_lock_slow(lock: &AtomicU32) -> Result<(), MorlocError> {\n loop {\n std::thread::yield_now();\n if lock\n .compare_exchange_weak(LOCK_UNLOCKED, LOCK_LOCKED, Ordering::Acquire, Ordering::Relaxed)\n .is_ok()\n {\n return Ok(());\n }\n }\n}\n\n/// Release a futex-based cross-process lock on shared memory.\n///\n/// # Safety\n/// `lock` must be the same AtomicU32 previously acquired via shm_lock.\npub unsafe fn shm_unlock(lock: &AtomicU32) {\n lock.store(LOCK_UNLOCKED, Ordering::Release);\n #[cfg(target_os = \"linux\")]\n {\n let ptr = lock as *const AtomicU32 as *const u32;\n libc::syscall(\n libc::SYS_futex, ptr, libc::FUTEX_WAKE, 1,\n std::ptr::null::(), std::ptr::null::(), 0u32,\n );\n }\n // macOS: no futex wake needed; spin-yield waiters will see the store.\n}\n\n// ── Pointer conversion helpers ─────────────────────────────────────────────\n\n#[inline]\npub fn vol2rel(ptr: VolPtr, shm: &ShmHeader) -> RelPtr {\n shm.relative_offset as RelPtr + ptr\n}\n\n/// # Safety\n/// `shm` must be a valid mmap'd ShmHeader. `ptr` must be within the volume's data region.\n#[inline]\npub unsafe fn vol2abs(ptr: VolPtr, shm: *const ShmHeader) -> AbsPtr {\n vol2abs_raw(ptr, shm)\n}\n\n// ── Tests ──────────────────────────────────────────────────────────────────\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_block_header_no_padding() {\n assert_eq!(\n std::mem::size_of::(),\n 4 + 4 + std::mem::size_of::()\n );\n }\n\n #[test]\n fn test_align_up() {\n assert_eq!(align_up(0, 8), 0);\n assert_eq!(align_up(1, 8), 8);\n assert_eq!(align_up(7, 8), 8);\n assert_eq!(align_up(8, 8), 8);\n assert_eq!(align_up(9, 8), 16);\n }\n\n #[test]\n fn test_pointer_constants() {\n assert_eq!(RELNULL, -1);\n assert_eq!(VOLNULL, -1);\n }\n\n #[test]\n fn test_lock_unlock() {\n let lock = AtomicU32::new(LOCK_UNLOCKED);\n unsafe {\n shm_lock(&lock).unwrap();\n assert_eq!(lock.load(Ordering::Relaxed), LOCK_LOCKED);\n shm_unlock(&lock);\n assert_eq!(lock.load(Ordering::Relaxed), LOCK_UNLOCKED);\n }\n }\n\n #[test]\n fn test_array_struct_size() {\n assert_eq!(\n std::mem::size_of::(),\n std::mem::size_of::() + std::mem::size_of::()\n );\n }\n\n #[test]\n fn test_shinit_and_shmalloc() {\n // Use file-backed SHM via tmpdir to avoid /dev/shm permission issues in test\n let tmpdir = std::env::temp_dir();\n let test_dir = tmpdir.join(format!(\"morloc_test_{}\", std::process::id()));\n std::fs::create_dir_all(&test_dir).unwrap();\n shm_set_fallback_dir(test_dir.to_str().unwrap());\n\n let basename = format!(\"test_shm_{}\", std::process::id());\n let shm = shinit(&basename, 0, 4096).unwrap();\n assert!(!shm.is_null());\n assert_eq!(unsafe { (*shm).magic }, SHM_MAGIC);\n\n // Allocate some memory\n let ptr1 = shmalloc(64).unwrap();\n assert!(!ptr1.is_null());\n\n // Write and read back\n unsafe {\n std::ptr::write_bytes(ptr1, 0xAB, 64);\n assert_eq!(*ptr1, 0xAB);\n }\n\n // Convert to relptr and back\n let rel = abs2rel(ptr1).unwrap();\n assert!(rel >= 0);\n let abs = rel2abs(rel).unwrap();\n assert_eq!(abs, ptr1);\n\n // Free\n shfree(ptr1).unwrap();\n\n // Cleanup\n shclose().unwrap();\n let _ = std::fs::remove_dir_all(&test_dir);\n }\n}\n" }, { "path": "data/rust/morloc-runtime/src/slurm_ffi.rs", "content": "//! C ABI wrappers for SLURM job submission.\n//! Replaces slurm.c.\n\nuse std::ffi::{c_char, c_void, CStr, CString};\nuse std::ptr;\n\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\n\nconst MAX_SLURM_COMMAND_LENGTH: usize = 1024;\nconst DEFAULT_XXHASH_SEED: u64 = 0;\n\n// ── C-compatible types ───────────────────────────────────────────────────────\n\n#[repr(C)]\npub struct Resources {\n pub memory: i32, // in Gb\n pub time: i32, // walltime in seconds\n pub cpus: i32,\n pub gpus: i32,\n}\n\n// ── parse_slurm_time ─────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn parse_slurm_time(\n time_str: *const c_char,\n errmsg: *mut *mut c_char,\n) -> usize {\n clear_errmsg(errmsg);\n let s = CStr::from_ptr(time_str).to_string_lossy();\n\n let mut days: i32 = 0;\n let hours: i32;\n let minutes: i32;\n let seconds: i32;\n\n // Try D-HH:MM:SS format\n if let Some(dash_pos) = s.find('-') {\n days = match s[..dash_pos].parse() {\n Ok(d) => d,\n Err(_) => {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Failed to scan slurm walltime string '{}'\", s)));\n return 0;\n }\n };\n let rest = &s[dash_pos + 1..];\n let parts: Vec<&str> = rest.split(':').collect();\n if parts.len() != 3 {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Failed to scan slurm walltime string '{}'\", s)));\n return 0;\n }\n hours = parts[0].parse().unwrap_or(-1);\n minutes = parts[1].parse().unwrap_or(-1);\n seconds = parts[2].parse().unwrap_or(-1);\n } else {\n // Try HH:MM:SS format\n let parts: Vec<&str> = s.split(':').collect();\n if parts.len() != 3 {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Failed to scan slurm walltime string '{}'\", s)));\n return 0;\n }\n hours = parts[0].parse().unwrap_or(-1);\n minutes = parts[1].parse().unwrap_or(-1);\n seconds = parts[2].parse().unwrap_or(-1);\n }\n\n if days < 0 || hours < 0 || minutes < 0 || seconds < 0 {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Negative time component in '{}'\", s)));\n return 0;\n }\n if hours > 23 || minutes > 59 || seconds > 59 {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"Invalid time component in '{}' (HH<=23 MM<=59 SS<=59)\", s)));\n return 0;\n }\n if days > 3650 {\n set_errmsg(errmsg, &MorlocError::Other(\"Do you really want to run this job for more than 10 years?\".into()));\n return 0;\n }\n\n (seconds + 60 * minutes + 60 * 60 * hours + 60 * 60 * 24 * days) as usize\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn write_slurm_time(seconds: i32) -> *mut c_char {\n let mut rem = seconds;\n let days = rem / (60 * 60 * 24);\n rem -= days * 60 * 60 * 24;\n let hours = rem / (60 * 60);\n rem -= hours * 60 * 60;\n let minutes = rem / 60;\n rem -= minutes * 60;\n\n let s = format!(\"{}-{:02}:{:02}:{:02}\", days, hours, minutes, rem);\n match CString::new(s) {\n Ok(cs) => cs.into_raw(),\n Err(_) => ptr::null_mut(),\n }\n}\n\n// ── parse_morloc_call_arguments ──────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn parse_morloc_call_arguments(\n packet: *mut u8,\n args: *mut *mut u8,\n nargs: *mut usize,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n *nargs = 0;\n\n let header = &*(packet as *const crate::packet::PacketHeader);\n let packet_size = 32 + header.offset as usize + header.length as usize;\n\n if header.command_type() != crate::packet::PACKET_TYPE_CALL {\n set_errmsg(errmsg, &MorlocError::Packet(\"Unexpected packet type (BUG)\".into()));\n return false;\n }\n\n // First pass: count args\n let mut pos = 32 + header.offset as usize;\n while pos < packet_size {\n let arg_header = &*(packet.add(pos) as *const crate::packet::PacketHeader);\n pos += 32 + arg_header.offset as usize + arg_header.length as usize;\n *nargs += 1;\n }\n\n // Second pass: set pointers\n pos = 32 + header.offset as usize;\n for i in 0..*nargs {\n *args.add(i) = packet.add(pos);\n let arg_header = &*(packet.add(pos) as *const crate::packet::PacketHeader);\n pos += 32 + arg_header.offset as usize + arg_header.length as usize;\n }\n\n true\n}\n\n// ── slurm_job_is_complete ────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn slurm_job_is_complete(job_id: u32) -> bool {\n let cmd = format!(\"sacct -j {} --format=State --noheader\\0\", job_id);\n let sacct = libc::popen(cmd.as_ptr() as *const c_char, b\"r\\0\".as_ptr() as *const c_char);\n if sacct.is_null() { return false; }\n\n let mut state = [0u8; 64];\n let mut done = false;\n while !libc::fgets(state.as_mut_ptr() as *mut c_char, 64, sacct).is_null() {\n let s = std::str::from_utf8(&state).unwrap_or(\"\");\n if s.contains(\"COMPLETED\") || s.contains(\"FAILED\") || s.contains(\"CANCELLED\") {\n done = true;\n break;\n }\n }\n libc::pclose(sacct);\n done\n}\n\n// ── shell_escape ─────────────────────────────────────────────────────────────\n\nfn shell_escape(input: &str) -> String {\n let mut out = String::with_capacity(input.len() + 10);\n out.push('\\'');\n for ch in input.chars() {\n if ch == '\\'' {\n out.push_str(\"'\\\\''\");\n } else {\n out.push(ch);\n }\n }\n out.push('\\'');\n out\n}\n\n// ── submit_morloc_slurm_job ──────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn submit_morloc_slurm_job(\n nexus_path: *const c_char,\n socket_basename: *const c_char,\n call_packet_filename: *const c_char,\n result_cache_filename: *const c_char,\n output_filename: *const c_char,\n error_filename: *const c_char,\n resources: *const Resources,\n errmsg: *mut *mut c_char,\n) -> u32 {\n clear_errmsg(errmsg);\n\n macro_rules! check_null {\n ($ptr:expr, $name:expr) => {\n if $ptr.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(format!(\"{} undefined\", $name)));\n return 0;\n }\n };\n }\n check_null!(nexus_path, \"nexus path\");\n check_null!(socket_basename, \"socket basename\");\n check_null!(call_packet_filename, \"call packet filename\");\n check_null!(result_cache_filename, \"result cache filename\");\n check_null!(output_filename, \"slurm output filename\");\n check_null!(error_filename, \"slurm error filename\");\n\n let res = &*resources;\n let nexus = CStr::from_ptr(nexus_path).to_string_lossy();\n let call = CStr::from_ptr(call_packet_filename).to_string_lossy();\n let socket = CStr::from_ptr(socket_basename).to_string_lossy();\n let result_cache = CStr::from_ptr(result_cache_filename).to_string_lossy();\n let output = CStr::from_ptr(output_filename).to_string_lossy();\n let error = CStr::from_ptr(error_filename).to_string_lossy();\n\n let time_str_raw = write_slurm_time(res.time);\n let time_str = CStr::from_ptr(time_str_raw).to_string_lossy().into_owned();\n libc::free(time_str_raw as *mut c_void);\n\n let mem_arg = format!(\"--mem={}G\", res.memory);\n let time_arg = format!(\"--time={}\", time_str);\n let cpus_arg = format!(\"--cpus-per-task={}\", res.cpus);\n let gpus_arg = format!(\"--gres=gpu:{}\", res.gpus);\n\n let esc_nexus = shell_escape(&nexus);\n let esc_call = shell_escape(&call);\n let esc_socket = shell_escape(&socket);\n let esc_result = shell_escape(&result_cache);\n\n let wrap_cmd = format!(\n \"{} --call-packet {} --socket-base {} --output-file {} --output-form packet\",\n esc_nexus, esc_call, esc_socket, esc_result\n );\n\n if wrap_cmd.len() >= MAX_SLURM_COMMAND_LENGTH {\n set_errmsg(errmsg, &MorlocError::Other(\"Wrap command too long\".into()));\n return 0;\n }\n\n let wrap_arg = format!(\"--wrap={}\", wrap_cmd);\n\n // Fork/exec sbatch\n let mut pipefd = [0i32; 2];\n if libc::pipe(pipefd.as_mut_ptr()) == -1 {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to create pipe for sbatch\".into()));\n return 0;\n }\n\n let pid = libc::fork();\n if pid == -1 {\n libc::close(pipefd[0]);\n libc::close(pipefd[1]);\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to fork for sbatch\".into()));\n return 0;\n }\n\n if pid == 0 {\n // Child\n libc::close(pipefd[0]);\n libc::dup2(pipefd[1], libc::STDOUT_FILENO);\n libc::close(pipefd[1]);\n\n let sbatch = CString::new(\"sbatch\").unwrap();\n let parsable = CString::new(\"--parsable\").unwrap();\n let o_flag = CString::new(\"-o\").unwrap();\n let e_flag = CString::new(\"-e\").unwrap();\n let c_output = CString::new(output.as_ref()).unwrap();\n let c_error = CString::new(error.as_ref()).unwrap();\n let c_mem = CString::new(mem_arg).unwrap();\n let c_time = CString::new(time_arg).unwrap();\n let c_cpus = CString::new(cpus_arg).unwrap();\n let c_gpus = CString::new(gpus_arg).unwrap();\n let c_wrap = CString::new(wrap_arg).unwrap();\n\n libc::execlp(\n sbatch.as_ptr(),\n sbatch.as_ptr(),\n parsable.as_ptr(),\n o_flag.as_ptr(), c_output.as_ptr(),\n e_flag.as_ptr(), c_error.as_ptr(),\n c_mem.as_ptr(), c_time.as_ptr(), c_cpus.as_ptr(), c_gpus.as_ptr(),\n c_wrap.as_ptr(),\n ptr::null::(),\n );\n libc::_exit(127);\n }\n\n // Parent\n libc::close(pipefd[1]);\n\n let mut buf = [0u8; 64];\n let nread = libc::read(pipefd[0], buf.as_mut_ptr() as *mut c_void, 63);\n libc::close(pipefd[0]);\n\n let mut status: i32 = 0;\n libc::waitpid(pid, &mut status, 0);\n\n if !libc::WIFEXITED(status) || libc::WEXITSTATUS(status) != 0 {\n set_errmsg(errmsg, &MorlocError::Other(\"sbatch exited with error\".into()));\n return 0;\n }\n if nread <= 0 {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to read sbatch output\".into()));\n return 0;\n }\n\n let output_str = std::str::from_utf8(&buf[..nread as usize]).unwrap_or(\"\");\n match output_str.trim().parse::() {\n Ok(job_id) => job_id,\n Err(_) => {\n set_errmsg(errmsg, &MorlocError::Other(\"Failed to parse job ID from sbatch output\".into()));\n 0\n }\n }\n}\n\n// ── remote_call ──────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn remote_call(\n midx: i32,\n socket_basename: *const c_char,\n cache_path: *const c_char,\n resources: *const Resources,\n arg_packets: *const *const u8,\n nargs: usize,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n\n // Use extern C declarations for functions from other modules\n extern \"C\" {\n fn read_schema_from_packet_meta(packet: *const u8, errmsg: *mut *mut c_char) -> *mut c_char;\n fn parse_schema(schema_str: *const c_char, errmsg: *mut *mut c_char) -> *mut crate::cschema::CSchema;\n fn free_schema(schema: *mut crate::cschema::CSchema);\n fn get_morloc_data_packet_value(data: *const u8, schema: *const crate::cschema::CSchema, errmsg: *mut *mut c_char) -> *mut u8;\n fn hash_voidstar(data: *const u8, schema: *const crate::cschema::CSchema, seed: u64, hash: *mut u64, errmsg: *mut *mut c_char) -> bool;\n fn mix(a: u64, b: u64) -> u64;\n fn mkdir_p(path: *const c_char, errmsg: *mut *mut c_char) -> i32;\n fn check_cache_packet(key: u64, cache_path: *const c_char, errmsg: *mut *mut c_char) -> *mut c_char;\n fn get_cache_packet(key: u64, cache_path: *const c_char, errmsg: *mut *mut c_char) -> *mut u8;\n fn put_cache_packet(data: *const u8, schema: *const crate::cschema::CSchema, key: u64, cache_path: *const c_char, errmsg: *mut *mut c_char) -> *mut c_char;\n fn make_cache_filename(hash: u64, cache_path: *const c_char, errmsg: *mut *mut c_char) -> *mut c_char;\n fn make_cache_filename_ext(hash: u64, cache_path: *const c_char, ext: *const c_char, errmsg: *mut *mut c_char) -> *mut c_char;\n fn make_morloc_remote_call_packet(midx: u32, arg_packets: *const *const u8, nargs: usize, errmsg: *mut *mut c_char) -> *mut u8;\n fn morloc_packet_size(packet: *const u8, errmsg: *mut *mut c_char) -> usize;\n fn read_binary_file(filename: *const c_char, file_size: *mut usize, errmsg: *mut *mut c_char) -> *mut u8;\n fn write_atomic(filename: *const c_char, data: *const u8, size: usize, errmsg: *mut *mut c_char) -> i32;\n fn get_morloc_data_packet_error_message(data: *const u8, errmsg: *mut *mut c_char) -> *mut c_char;\n }\n\n let seed = midx as u64;\n let mut err: *mut c_char = ptr::null_mut();\n\n // Cleanup tracking\n let mut return_packet: *mut u8 = ptr::null_mut();\n let mut arg_hashes: Vec = vec![0; nargs];\n let mut arg_voidstars: Vec<*mut u8> = vec![ptr::null_mut(); nargs];\n let mut arg_schemas: Vec<*mut crate::cschema::CSchema> = vec![ptr::null_mut(); nargs];\n let mut cached_arg_filenames: Vec<*mut c_char> = vec![ptr::null_mut(); nargs];\n let mut cached_arg_packets: Vec<*mut u8> = vec![ptr::null_mut(); nargs];\n\n let mut function_hash = mix(seed, DEFAULT_XXHASH_SEED);\n\n // Hash each argument\n for i in 0..nargs {\n let schema_str = read_schema_from_packet_meta(*arg_packets.add(i), &mut err);\n if schema_str.is_null() || !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n\n arg_schemas[i] = parse_schema(schema_str, &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n\n arg_voidstars[i] = get_morloc_data_packet_value(*arg_packets.add(i), arg_schemas[i], &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n\n let mut h: u64 = 0;\n hash_voidstar(arg_voidstars[i], arg_schemas[i], DEFAULT_XXHASH_SEED, &mut h, &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n arg_hashes[i] = h;\n\n function_hash = mix(function_hash, h);\n }\n\n mkdir_p(cache_path, &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n\n // Check if result is cached\n let mut result_cache_filename = check_cache_packet(function_hash, cache_path, &mut err);\n if !err.is_null() { libc::free(err as *mut c_void); err = ptr::null_mut(); }\n\n if !result_cache_filename.is_null() {\n return_packet = get_cache_packet(function_hash, cache_path, &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n } else {\n result_cache_filename = make_cache_filename(function_hash, cache_path, &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n\n // Cache arguments\n for i in 0..nargs {\n cached_arg_filenames[i] = check_cache_packet(arg_hashes[i], cache_path, &mut err);\n if cached_arg_filenames[i].is_null() {\n if !err.is_null() { libc::free(err as *mut c_void); err = ptr::null_mut(); }\n cached_arg_filenames[i] = put_cache_packet(arg_voidstars[i], arg_schemas[i], arg_hashes[i], cache_path, &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n }\n }\n\n // Read cached arg packets\n for i in 0..nargs {\n let mut file_size: usize = 0;\n cached_arg_packets[i] = read_binary_file(cached_arg_filenames[i], &mut file_size, &mut err);\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n }\n\n // Build call packet\n let cached_ptrs: Vec<*const u8> = cached_arg_packets.iter().map(|p| *p as *const u8).collect();\n let call_packet = make_morloc_remote_call_packet(midx as u32, cached_ptrs.as_ptr(), nargs, &mut err);\n if !err.is_null() {\n libc::free(call_packet as *mut c_void);\n goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet);\n }\n\n let call_packet_size = morloc_packet_size(call_packet, &mut err);\n\n // Hash call packet with xxhash\n let call_packet_hash = crate::hash::xxh64_with_seed(std::slice::from_raw_parts(call_packet, call_packet_size), DEFAULT_XXHASH_SEED);\n\n let call_ext = CString::new(\"-call.dat\").unwrap();\n let call_packet_filename = make_cache_filename_ext(call_packet_hash, cache_path, call_ext.as_ptr(), &mut err);\n\n // Write call packet to disk\n write_atomic(call_packet_filename, call_packet, call_packet_size, &mut err);\n libc::free(call_packet as *mut c_void);\n\n let out_ext = CString::new(\".out\").unwrap();\n let err_ext = CString::new(\".err\").unwrap();\n let output_filename = make_cache_filename_ext(function_hash, cache_path, out_ext.as_ptr(), &mut err);\n let error_filename = make_cache_filename_ext(function_hash, cache_path, err_ext.as_ptr(), &mut err);\n\n // Submit SLURM job\n let nexus_c = CString::new(\"./nexus\").unwrap();\n let pid = submit_morloc_slurm_job(\n nexus_c.as_ptr(), socket_basename, call_packet_filename,\n result_cache_filename, output_filename, error_filename,\n resources, &mut err,\n );\n\n libc::free(call_packet_filename as *mut c_void);\n libc::free(output_filename as *mut c_void);\n libc::free(error_filename as *mut c_void);\n\n if !err.is_null() { goto_cleanup!(errmsg, err, arg_schemas, cached_arg_filenames, cached_arg_packets, return_packet); }\n\n // Wait for job completion\n while !slurm_job_is_complete(pid) {\n libc::sleep(1);\n }\n\n let mut return_packet_size: usize = 0;\n return_packet = read_binary_file(result_cache_filename, &mut return_packet_size, &mut err);\n\n let failure = get_morloc_data_packet_error_message(return_packet, &mut err);\n if !failure.is_null() {\n libc::fprintf(\n libc::fdopen(libc::STDERR_FILENO, b\"w\\0\".as_ptr() as *const c_char),\n b\"Failed, deleting result %s\\n\\0\".as_ptr() as *const c_char,\n result_cache_filename,\n );\n libc::unlink(result_cache_filename);\n libc::free(failure as *mut c_void);\n }\n }\n\n // Cleanup\n for i in 0..nargs {\n if !arg_schemas[i].is_null() { free_schema(arg_schemas[i]); }\n if !cached_arg_filenames[i].is_null() { libc::free(cached_arg_filenames[i] as *mut c_void); }\n if !cached_arg_packets[i].is_null() { libc::free(cached_arg_packets[i] as *mut c_void); }\n }\n if !result_cache_filename.is_null() { libc::free(result_cache_filename as *mut c_void); }\n\n return_packet\n}\n\n// Cleanup macro for goto-like pattern\nmacro_rules! goto_cleanup {\n ($errmsg:expr, $err:expr, $schemas:expr, $filenames:expr, $packets:expr, $return_packet:expr) => {{\n *$errmsg = $err;\n for i in 0..$schemas.len() {\n if !$schemas[i].is_null() {\n extern \"C\" { fn free_schema(s: *mut crate::cschema::CSchema); }\n free_schema($schemas[i]);\n }\n if !$filenames[i].is_null() { libc::free($filenames[i] as *mut c_void); }\n if !$packets[i].is_null() { libc::free($packets[i] as *mut c_void); }\n }\n return $return_packet;\n }};\n}\nuse goto_cleanup;\n" }, { "path": "data/rust/morloc-runtime/src/utility.rs", "content": "//! File I/O and string utility functions.\n//! Replaces utility.c.\n\nuse std::ffi::{c_char, c_void, CStr};\nuse std::io::Write;\nuse std::ptr;\n\nuse crate::error::{clear_errmsg, set_errmsg, MorlocError};\n\n// ── Cross-platform helpers ─────────────────────────────────────────────────\n\n/// Return the current errno value (cross-platform).\n#[cfg(target_os = \"linux\")]\n#[inline]\npub unsafe fn errno_val() -> i32 {\n *libc::__errno_location()\n}\n\n#[cfg(target_os = \"macos\")]\n#[inline]\npub unsafe fn errno_val() -> i32 {\n *libc::__error()\n}\n\n/// Suppress SIGPIPE on send(). Linux: per-call flag. macOS: use set_nosigpipe() on the socket.\n#[cfg(target_os = \"linux\")]\npub const SEND_NOSIGNAL: i32 = libc::MSG_NOSIGNAL;\n#[cfg(target_os = \"macos\")]\npub const SEND_NOSIGNAL: i32 = 0;\n\n/// Set SO_NOSIGPIPE on a socket (macOS). No-op on Linux (uses MSG_NOSIGNAL per-call).\n#[allow(unused_variables)]\npub unsafe fn set_nosigpipe(fd: i32) {\n #[cfg(target_os = \"macos\")]\n {\n let val: libc::c_int = 1;\n libc::setsockopt(\n fd,\n libc::SOL_SOCKET,\n libc::SO_NOSIGPIPE,\n &val as *const _ as *const libc::c_void,\n std::mem::size_of::() as libc::socklen_t,\n );\n }\n}\n\n// ── File operations ────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn file_exists(filename: *const c_char) -> bool {\n if filename.is_null() {\n return false;\n }\n let path = CStr::from_ptr(filename).to_string_lossy();\n std::path::Path::new(path.as_ref()).exists()\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn mkdir_p(path: *const c_char, errmsg: *mut *mut c_char) -> i32 {\n clear_errmsg(errmsg);\n if path.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"NULL path\".into()));\n return -1;\n }\n let p = CStr::from_ptr(path).to_string_lossy();\n match std::fs::create_dir_all(p.as_ref()) {\n Ok(_) => 0,\n Err(e) => {\n set_errmsg(\n errmsg,\n &MorlocError::Io(e),\n );\n -1\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn delete_directory(path: *const c_char) {\n if path.is_null() {\n return;\n }\n let p = CStr::from_ptr(path).to_string_lossy();\n let _ = std::fs::remove_dir_all(p.as_ref());\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn has_suffix(x: *const c_char, suffix: *const c_char) -> bool {\n if x.is_null() || suffix.is_null() {\n return false;\n }\n let xs = CStr::from_ptr(x).to_string_lossy();\n let ss = CStr::from_ptr(suffix).to_string_lossy();\n xs.ends_with(ss.as_ref())\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn write_atomic(\n filename: *const c_char,\n data: *const u8,\n size: usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n if filename.is_null() || (data.is_null() && size != 0) {\n set_errmsg(errmsg, &MorlocError::Other(\"invalid arguments\".into()));\n return -1;\n }\n let path_str = CStr::from_ptr(filename).to_string_lossy();\n let path = std::path::Path::new(path_str.as_ref());\n\n // Get parent directory\n let dir = path.parent().unwrap_or(std::path::Path::new(\".\"));\n\n // Create temp file in same directory\n let tmp_path = dir.join(format!(\"morloc-tmp_{}\", std::process::id()));\n\n let result = (|| -> Result<(), std::io::Error> {\n // Write to temp file\n let mut f = std::fs::File::create(&tmp_path)?;\n if size > 0 {\n let bytes = std::slice::from_raw_parts(data, size);\n f.write_all(bytes)?;\n }\n f.sync_all()?;\n drop(f);\n\n // Atomic rename\n std::fs::rename(&tmp_path, path)?;\n\n // Sync parent directory\n if let Ok(dir_f) = std::fs::File::open(dir) {\n let _ = dir_f.sync_all();\n }\n Ok(())\n })();\n\n match result {\n Ok(_) => 0,\n Err(e) => {\n let _ = std::fs::remove_file(&tmp_path);\n set_errmsg(errmsg, &MorlocError::Io(e));\n -1\n }\n }\n}\n\n// ── Binary I/O ─────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_binary_file(\n filename: *const c_char,\n file_size: *mut usize,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n if filename.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"NULL filename\".into()));\n return ptr::null_mut();\n }\n let path = CStr::from_ptr(filename).to_string_lossy();\n match std::fs::read(path.as_ref()) {\n Ok(data) => {\n *file_size = data.len();\n let buf = libc::malloc(data.len()) as *mut u8;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"malloc failed\".into()));\n return ptr::null_mut();\n }\n std::ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len());\n buf\n }\n Err(e) => {\n set_errmsg(errmsg, &MorlocError::Io(e));\n ptr::null_mut()\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn read_binary_fd(\n file: *mut libc::FILE,\n file_size: *mut usize,\n errmsg: *mut *mut c_char,\n) -> *mut u8 {\n clear_errmsg(errmsg);\n if file.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"NULL file\".into()));\n return ptr::null_mut();\n }\n\n // Try seek-based size detection\n if libc::fseek(file, 0, libc::SEEK_END) == 0 {\n let size = libc::ftell(file) as usize;\n if size > 0 {\n libc::rewind(file);\n let buf = libc::malloc(size) as *mut u8;\n if buf.is_null() {\n set_errmsg(errmsg, &MorlocError::Other(\"malloc failed\".into()));\n return ptr::null_mut();\n }\n let read = libc::fread(buf as *mut c_void, 1, size, file);\n if read == size {\n *file_size = size;\n return buf;\n }\n libc::free(buf as *mut c_void);\n }\n }\n\n // Streaming read for non-seekable files\n let chunk_size: usize = 0xffff;\n let mut buf: *mut u8 = ptr::null_mut();\n let mut allocated: usize = 0;\n\n loop {\n let new_buf = libc::realloc(buf as *mut c_void, allocated + chunk_size) as *mut u8;\n if new_buf.is_null() {\n libc::free(buf as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"realloc failed\".into()));\n return ptr::null_mut();\n }\n buf = new_buf;\n let read = libc::fread(buf.add(allocated) as *mut c_void, 1, chunk_size, file);\n allocated += read;\n\n if read < chunk_size {\n if libc::feof(file) != 0 {\n *file_size = allocated;\n return buf;\n }\n if libc::ferror(file) != 0 {\n libc::free(buf as *mut c_void);\n set_errmsg(errmsg, &MorlocError::Other(\"read error\".into()));\n return ptr::null_mut();\n }\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn write_binary_fd(\n fd: i32,\n buf: *const c_char,\n count: usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n clear_errmsg(errmsg);\n let mut total: usize = 0;\n while total < count {\n let written = libc::write(fd, buf.add(total) as *const c_void, count - total);\n if written < 0 {\n set_errmsg(\n errmsg,\n &MorlocError::Other(format!(\"write failed: {}\", std::io::Error::last_os_error())),\n );\n return -1;\n }\n total += written as usize;\n }\n 0\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn print_binary(\n buf: *const c_char,\n count: usize,\n errmsg: *mut *mut c_char,\n) -> i32 {\n write_binary_fd(libc::STDOUT_FILENO, buf, count, errmsg)\n}\n\n// ── Display ────────────────────────────────────────────────────────────────\n\n#[no_mangle]\npub unsafe extern \"C\" fn hex(ptr: *const c_void, size: usize) {\n if ptr.is_null() || size == 0 {\n return;\n }\n let bytes = std::slice::from_raw_parts(ptr as *const u8, size);\n for (i, b) in bytes.iter().enumerate() {\n if i > 0 && i % 8 == 0 {\n eprint!(\" \");\n }\n eprint!(\"{:02X}\", b);\n if i < size - 1 {\n eprint!(\" \");\n }\n }\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn print_hex_dump(\n data: *const u8,\n size: usize,\n errmsg: *mut *mut c_char,\n) -> bool {\n clear_errmsg(errmsg);\n if data.is_null() && size > 0 {\n set_errmsg(errmsg, &MorlocError::Other(\"NULL data\".into()));\n return false;\n }\n let bytes = if size > 0 {\n std::slice::from_raw_parts(data, size)\n } else {\n &[]\n };\n for (i, b) in bytes.iter().enumerate() {\n if i > 0 && i % 4 == 0 {\n if i % 24 == 0 {\n println!();\n } else {\n print!(\" \");\n }\n }\n print!(\"{:02X}\", b);\n }\n if !bytes.is_empty() {\n println!();\n }\n true\n}\n\n// ── xxHash wrapper and mix ─────────────────────────────────────────────────\n\n/// Mix two 64-bit hash values. Matches the C implementation in cache.c.\n#[no_mangle]\npub extern \"C\" fn mix(a: u64, b: u64) -> u64 {\n const PRIME64_1: u64 = 0x9E3779B185EBCA87;\n const PRIME64_2: u64 = 0xC2B2AE3D27D4EB4F;\n let mut a = a ^ b.wrapping_mul(PRIME64_1);\n a = (a << 31) | (a >> 33);\n a.wrapping_mul(PRIME64_2)\n}\n\n#[no_mangle]\npub unsafe extern \"C\" fn morloc_xxh64(\n input: *const c_void,\n length: usize,\n seed: u64,\n) -> u64 {\n if input.is_null() || length == 0 {\n return crate::hash::xxh64_with_seed(&[], seed);\n }\n let data = std::slice::from_raw_parts(input as *const u8, length);\n crate::hash::xxh64_with_seed(data, seed)\n}\n\n// ── String utilities ───────────────────────────────────────────────────────\n\n/// dirname - returns pointer into the input string (modifies it in-place)\n/// Matches the C behavior: returns \".\" for empty/NULL, strips trailing slashes\n#[no_mangle]\npub unsafe extern \"C\" fn dirname(path: *mut c_char) -> *mut c_char {\n // Return a pointer to the static string \".\" for empty/null paths and paths with no slash.\n static DOT: [u8; 2] = [b'.', 0];\n let dot_ptr = DOT.as_ptr() as *mut c_char;\n\n if path.is_null() || *path == 0 {\n return dot_ptr;\n }\n\n let len = libc::strlen(path);\n let mut end = path.add(len - 1);\n\n // Remove trailing slashes\n while end > path && *end == b'/' as c_char {\n *end = 0;\n end = end.sub(1);\n }\n\n // Find last slash\n let last_slash = libc::strrchr(path, b'/' as i32);\n if last_slash.is_null() {\n return dot_ptr;\n }\n if last_slash == path {\n *path.add(1) = 0; // root case \"/\"\n } else {\n *last_slash = 0;\n }\n path\n}\n" }, { "path": "data/rust/morloc-runtime/src/voidstar.rs", "content": "//! Shared voidstar operations: relptr adjustment, binary serialization,\n//! schema-aware free, and flatten-to-buffer.\n//!\n//! These functions operate on the morloc voidstar binary format in SHM.\n//! They are used by packet.rs, cli.rs, and json.rs.\n\nuse crate::error::MorlocError;\nuse crate::schema::{Schema, SerialType};\nuse crate::shm::{self, AbsPtr, Array, RelPtr, Tensor};\n\n// ── adjust_voidstar_relptrs ────────────────────────────────────────────────\n\n/// Adjust all relptrs in a voidstar blob by adding base_rel.\n/// Used after copying a flattened blob into SHM.\npub fn adjust_relptrs(\n data: AbsPtr,\n schema: &Schema,\n base_rel: RelPtr,\n) -> Result<(), MorlocError> {\n // SAFETY: data points to a voidstar blob in SHM. We adjust relptrs in-place;\n // all pointer arithmetic stays within the blob's bounds as defined by schema.\n unsafe {\n match schema.serial_type {\n SerialType::String | SerialType::Array => {\n let arr = &mut *(data as *mut Array);\n arr.data += base_rel;\n if !schema.parameters.is_empty() && !schema.parameters[0].is_fixed_width() {\n let arr_data = shm::rel2abs(arr.data)?;\n let w = schema.parameters[0].width;\n for i in 0..arr.size {\n adjust_relptrs(arr_data.add(i * w), &schema.parameters[0], base_rel)?;\n }\n }\n }\n SerialType::Tuple | SerialType::Map => {\n for i in 0..schema.parameters.len() {\n adjust_relptrs(data.add(schema.offsets[i]), &schema.parameters[i], base_rel)?;\n }\n }\n SerialType::Optional => {\n if *data != 0 && !schema.parameters.is_empty() {\n let off = schema.offsets.first().copied()\n .unwrap_or_else(|| shm::align_up(1, schema.parameters[0].alignment().max(1)));\n adjust_relptrs(data.add(off), &schema.parameters[0], base_rel)?;\n }\n }\n SerialType::Tensor => {\n let t = &mut *(data as *mut Tensor);\n if t.total_elements > 0 {\n t.shape += base_rel;\n t.data += base_rel;\n }\n }\n _ => {}\n }\n }\n Ok(())\n}\n\n// ── read_voidstar_binary ───────────────────────────────────────────────────\n\n/// Read a flat voidstar binary blob into SHM, adjusting relptrs.\npub fn read_binary(blob: &[u8], schema: &Schema) -> Result {\n let base = shm::shmalloc(blob.len())?;\n // SAFETY: base is freshly allocated with blob.len() bytes.\n unsafe { std::ptr::copy_nonoverlapping(blob.as_ptr(), base, blob.len()) };\n let base_rel = shm::abs2rel(base)?;\n adjust_relptrs(base, schema, base_rel)?;\n Ok(base)\n}\n\n// ── shfree_by_schema ───────────────────────────────────────────────────────\n\n/// Zero metadata for nested structures so the parent block can be cleanly freed.\n/// Does NOT call shfree on sub-pointers (they're cursor-packed in the same block).\npub fn free_by_schema(ptr: AbsPtr, schema: &Schema) -> Result<(), MorlocError> {\n // SAFETY: ptr points to voidstar data in SHM with layout described by schema.\n // We zero metadata at schema.width offsets within the structure.\n unsafe {\n match schema.serial_type {\n SerialType::String | SerialType::Array => {\n let arr = &*(ptr as *const Array);\n if arr.data > 0 && !schema.parameters.is_empty() && !schema.parameters[0].is_fixed_width() {\n let arr_data = shm::rel2abs(arr.data)?;\n let w = schema.parameters[0].width;\n for i in 0..arr.size {\n free_by_schema(arr_data.add(i * w), &schema.parameters[0])?;\n }\n }\n }\n SerialType::Tuple | SerialType::Map => {\n for i in 0..schema.parameters.len() {\n free_by_schema(ptr.add(schema.offsets[i]), &schema.parameters[i])?;\n }\n }\n SerialType::Tensor => {} // inline, freed by parent\n _ => {}\n }\n std::ptr::write_bytes(ptr, 0, schema.width);\n }\n Ok(())\n}\n\n// ── flatten_voidstar_to_buffer ─────────────────────────────────────────────\n\n/// Flatten a voidstar structure in SHM into a self-contained byte buffer.\n/// Relptrs in the output are offsets from position 0 of the buffer.\npub fn flatten_to_buffer(data: AbsPtr, schema: &Schema) -> Result, MorlocError> {\n let total = crate::ffi::calc_voidstar_size_inner(data, schema)?;\n let mut buf = vec![0u8; total];\n\n // SAFETY: data points to at least schema.width bytes in SHM; buf has total >= schema.width bytes.\n unsafe { std::ptr::copy_nonoverlapping(data, buf.as_mut_ptr(), schema.width) };\n\n // Phase 2: fix up relptrs and copy variable-length data\n let mut cursor = schema.width;\n flatten_fixup(&mut buf, 0, data, schema, &mut cursor)?;\n\n Ok(buf)\n}\n\nfn flatten_fixup(\n buf: &mut [u8],\n buf_offset: usize,\n data: AbsPtr,\n schema: &Schema,\n cursor: &mut usize,\n) -> Result<(), MorlocError> {\n // SAFETY: buf is sized by calc_voidstar_size_inner to hold the entire flattened structure.\n // data points to corresponding SHM data. cursor tracks write position within buf.\n unsafe {\n match schema.serial_type {\n SerialType::String | SerialType::Array => {\n let orig_arr = &*(data as *const Array);\n let buf_arr = &mut *(buf.as_mut_ptr().add(buf_offset) as *mut Array);\n if orig_arr.size == 0 {\n buf_arr.data = 0;\n return Ok(());\n }\n let orig_data = shm::rel2abs(orig_arr.data)?;\n let elem_schema = &schema.parameters[0];\n let align = elem_schema.alignment();\n *cursor = shm::align_up(*cursor, align);\n buf_arr.data = *cursor as RelPtr;\n let elem_w = elem_schema.width;\n let total_bytes = elem_w * orig_arr.size;\n buf[*cursor..*cursor + total_bytes].copy_from_slice(\n std::slice::from_raw_parts(orig_data, total_bytes)\n );\n let elem_start = *cursor;\n *cursor += total_bytes;\n if !elem_schema.is_fixed_width() {\n for i in 0..orig_arr.size {\n flatten_fixup(\n buf, elem_start + i * elem_w,\n orig_data.add(i * elem_w), elem_schema, cursor,\n )?;\n }\n }\n }\n SerialType::Tuple | SerialType::Map => {\n for i in 0..schema.parameters.len() {\n flatten_fixup(\n buf, buf_offset + schema.offsets[i],\n data.add(schema.offsets[i]), &schema.parameters[i], cursor,\n )?;\n }\n }\n SerialType::Optional => {\n let tag = *buf.as_ptr().add(buf_offset);\n if tag != 0 && !schema.parameters.is_empty() {\n let off = schema.offsets.first().copied()\n .unwrap_or_else(|| shm::align_up(1, schema.parameters[0].alignment().max(1)));\n flatten_fixup(\n buf, buf_offset + off,\n data.add(off), &schema.parameters[0], cursor,\n )?;\n }\n }\n SerialType::Tensor => {\n let orig = &*(data as *const Tensor);\n let buf_t = &mut *(buf.as_mut_ptr().add(buf_offset) as *mut Tensor);\n if orig.total_elements == 0 {\n buf_t.shape = 0;\n buf_t.data = 0;\n return Ok(());\n }\n let ndim = schema.offsets.first().copied().unwrap_or(0);\n let orig_shape = shm::rel2abs(orig.shape)?;\n *cursor = shm::align_up(*cursor, std::mem::align_of::());\n buf_t.shape = *cursor as RelPtr;\n let shape_bytes = ndim * std::mem::size_of::();\n buf[*cursor..*cursor + shape_bytes].copy_from_slice(\n std::slice::from_raw_parts(orig_shape, shape_bytes)\n );\n *cursor += shape_bytes;\n\n let orig_data = shm::rel2abs(orig.data)?;\n let elem_w = schema.parameters[0].width;\n let elem_align = schema.parameters[0].alignment();\n *cursor = shm::align_up(*cursor, elem_align);\n buf_t.data = *cursor as RelPtr;\n let data_bytes = orig.total_elements * elem_w;\n buf[*cursor..*cursor + data_bytes].copy_from_slice(\n std::slice::from_raw_parts(orig_data, data_bytes)\n );\n *cursor += data_bytes;\n }\n _ => {} // primitives already copied by parent\n }\n }\n Ok(())\n}\n\n// ── write_voidstar_binary (to fd) ──────────────────────────────────────────\n\n/// Flatten voidstar and write to a file descriptor. Returns bytes written.\npub fn write_binary_to_fd(fd: i32, data: AbsPtr, schema: &Schema) -> Result {\n let buf = flatten_to_buffer(data, schema)?;\n // SAFETY: buf is a valid byte slice; fd is a valid file descriptor from the caller.\n let written = unsafe {\n libc::write(fd, buf.as_ptr() as *const std::ffi::c_void, buf.len())\n };\n if written < 0 {\n return Err(MorlocError::Io(std::io::Error::last_os_error()));\n }\n Ok(written as usize)\n}\n" }, { "path": "exe/morloc-codegen-generic/Main.hs", "content": "{- |\nModule : Main\nDescription : Generic pool code generator for morloc\nCopyright : (c) Zebulun Arendsee, 2016-2026\nLicense : Apache-2.0\n\nStandalone executable that assembles pool files for dynamically-typed\ninterpreted languages. Receives a serialized IProgram via a binary file\nand a language descriptor via lang.yaml.\n\nUsage: morloc-codegen-generic \n\nReads:\n - argv[1]: path to lang.yaml (language descriptor)\n - argv[2]: path to binary-encoded IProgram\n\nWrites to stdout:\n - JSON CodegenManifest with pool_code and build_commands\n-}\nmodule Main (main) where\n\nimport qualified Data.Aeson as Aeson\nimport qualified Data.Binary as Binary\nimport qualified Data.ByteString.Lazy as BL\nimport qualified Data.Text as T\nimport qualified Data.Text.IO as TIO\nimport GHC.IO.Encoding (setLocaleEncoding, utf8)\nimport System.Environment (getArgs)\nimport System.Exit (exitFailure)\nimport System.FilePath (takeDirectory, ())\nimport System.IO (hPutStrLn, stderr)\n\nimport Morloc.CodeGenerator.Grammars.Translator.Generic (CodegenManifest (..), printProgram)\nimport Morloc.CodeGenerator.Grammars.Translator.Imperative (IProgram)\nimport Morloc.CodeGenerator.LanguageDescriptor (LangDescriptor (..), loadLangDescriptor)\nimport Morloc.Data.Doc (render)\n\nmain :: IO ()\nmain = do\n setLocaleEncoding utf8\n args <- getArgs\n case args of\n [langYamlPath, iprogramPath] -> run langYamlPath iprogramPath\n _ -> do\n hPutStrLn stderr \"Usage: morloc-codegen-generic \"\n exitFailure\n\nrun :: FilePath -> FilePath -> IO ()\nrun langYamlPath iprogramPath = do\n -- load language descriptor\n descResult <- loadLangDescriptor langYamlPath\n desc <- case descResult of\n Left err -> do\n hPutStrLn stderr $ \"Failed to load \" ++ langYamlPath ++ \": \" ++ err\n exitFailure\n Right d -> return d\n\n -- load pool template from disk if not inline\n desc' <-\n if T.null (ldPoolTemplate desc)\n then do\n let langDir = takeDirectory langYamlPath\n ext = ldExtension desc\n poolPath = langDir \"pool.\" ++ ext\n poolText <- TIO.readFile poolPath\n return desc {ldPoolTemplate = poolText}\n else return desc\n\n -- deserialize IProgram\n binaryData <- BL.readFile iprogramPath\n let program = Binary.decode binaryData :: IProgram\n\n -- assemble pool file\n let poolCode = render (printProgram desc' program)\n\n -- output manifest as JSON\n let manifest =\n CodegenManifest\n { cgmPoolCode = poolCode\n , cgmBuildCommands = []\n }\n BL.putStr (Aeson.encode manifest)\n" }, { "path": "executable/CppPrinter.hs", "content": "{-# LANGUAGE OverloadedStrings #-}\n{-# LANGUAGE QuasiQuotes #-}\n{-# LANGUAGE TemplateHaskell #-}\n\n{- |\nModule : CppPrinter\nDescription : Pretty-print the imperative IR as C++ source code\nCopyright : (c) Zebulun Arendsee, 2016-2026\nLicense : Apache-2.0\nMaintainer : z@morloc.io\n\nConverts 'IStmt' and 'IExpr' IR nodes into C++ source text. Handles\ntype rendering, struct definitions, forward declarations, and C++\nidioms (templates, shared_ptr, std::variant).\n-}\nmodule CppPrinter\n ( printExpr\n , printStmt\n , printStmts\n\n -- * Pool-level rendering\n , printDispatch\n , printProgram\n\n -- * Struct/serializer rendering\n , printStructTypedef\n , printSerializer\n , printDeserializer\n , printTemplateHeader\n , printRecordTemplate\n ) where\n\nimport Morloc.CodeGenerator.Grammars.Common (DispatchEntry (..), manNamer)\nimport Morloc.CodeGenerator.Grammars.Translator.Imperative\nimport Morloc.CodeGenerator.Namespace (MDoc)\nimport Morloc.Data.Doc\nimport Morloc.DataFiles as DF\nimport Morloc.Quasi\n\nprintExpr :: IExpr -> MDoc\nprintExpr (IVar v) = pretty v\nprintExpr (IBoolLit True) = \"true\"\nprintExpr (IBoolLit False) = \"false\"\nprintExpr (INullLit (Just t)) = \"std::optional<\" <> renderIType t <> \">()\"\nprintExpr (INullLit Nothing) = \"std::nullopt\"\nprintExpr (IIntLit Nothing i) = viaShow i\nprintExpr (IIntLit (Just t) i)\n | t == \"int\" = viaShow i\n | otherwise = \"static_cast<\" <> pretty t <> \">(\" <> viaShow i <> \")\"\nprintExpr (IRealLit Nothing r) = viaShow r\nprintExpr (IRealLit (Just t) r)\n | t == \"double\" = viaShow r\n | otherwise = \"static_cast<\" <> pretty t <> \">(\" <> viaShow r <> \")\"\nprintExpr (IStrLit s) = [idoc|std::string(#{textEsc' s})|]\nprintExpr (IListLit es) = encloseSep \"{\" \"}\" \",\" (map printExpr es)\nprintExpr (ITupleLit es) = \"std::make_tuple\" <> tupled (map printExpr es)\nprintExpr (IRecordLit _ _ entries) =\n encloseSep \"{\" \"}\" \",\" (map (printExpr . snd) entries)\nprintExpr (IAccess e (IIdx i)) = \"std::get<\" <> pretty i <> \">(\" <> printExpr e <> \")\"\nprintExpr (IAccess e (IKey _)) = printExpr e -- should not be reached for C++\nprintExpr (IAccess e (IField f)) = printExpr e <> \".\" <> pretty f\nprintExpr (ISerCall schema e) = [idoc|_put_value(#{printExpr e}, \"#{pretty schema}\")|]\nprintExpr (IDesCall schema (Just rawtype) e) = [idoc|_get_value<#{renderIType rawtype}>(#{printExpr e}, \"#{pretty schema}\")|]\nprintExpr (IDesCall schema Nothing e) = [idoc|_get_value(#{printExpr e}, \"#{pretty schema}\")|]\nprintExpr (IPack packer e) = pretty packer <> parens (printExpr e)\nprintExpr (ICall f Nothing argGroups) =\n pretty f <> hsep (map (tupled . map printExpr) argGroups)\nprintExpr (ICall f (Just ts) argGroups) =\n pretty f\n <> encloseSep \"<\" \">\" \",\" (map renderIType ts)\n <> hsep (map (tupled . map printExpr) argGroups)\nprintExpr (IForeignCall _ _ _) = error \"use IRawExpr for C++ foreign calls\"\nprintExpr (IRemoteCall _ _ _ _) = error \"use IRawExpr for C++ remote calls\"\nprintExpr (ILambda args body) =\n \"[&](\"\n <> hsep (punctuate \",\" [\"auto\" <+> pretty a | a <- args])\n <> \"){return \"\n <> printExpr body\n <> \";}\"\nprintExpr (IRawExpr d) = pretty d\nprintExpr (IDoBlock e) = \"[&](){return \" <> printExpr e <> \";}\"\nprintExpr (IEval e) = printExpr e <> \"()\"\nprintExpr (IIntrinsicHash schema e) =\n [idoc|_mlc_hash(#{printExpr e}, \"#{pretty schema}\")|]\nprintExpr (IIntrinsicSave fmt schema e path)\n | fmt == \"json\" = [idoc|_mlc_save_json(#{printExpr e}, \"#{pretty schema}\", #{printExpr path})|]\n | fmt == \"voidstar\" = [idoc|_mlc_save_voidstar(#{printExpr e}, \"#{pretty schema}\", #{printExpr path})|]\n | otherwise = [idoc|_mlc_save(#{printExpr e}, \"#{pretty schema}\", #{printExpr path})|]\nprintExpr (IIntrinsicLoad schema (Just t) path) =\n [idoc|_mlc_load<#{renderIType t}>(\"#{pretty schema}\", #{printExpr path})|]\nprintExpr (IIntrinsicLoad schema Nothing path) =\n [idoc|_mlc_load(\"#{pretty schema}\", #{printExpr path})|]\nprintExpr (IIntrinsicShow schema e) =\n [idoc|_mlc_show(#{printExpr e}, \"#{pretty schema}\")|]\nprintExpr (IIntrinsicRead schema (Just t) e) =\n [idoc|_mlc_read<#{renderIType t}>(\"#{pretty schema}\", #{printExpr e})|]\nprintExpr (IIntrinsicRead schema Nothing e) =\n [idoc|_mlc_read(\"#{pretty schema}\", #{printExpr e})|]\n\nprintStmt :: IStmt -> MDoc\nprintStmt (IAssign v Nothing e) = \"auto\" <+> pretty v <+> \"=\" <+> printExpr e <> \";\"\nprintStmt (IAssign v (Just t) e) = renderIType t <+> pretty v <+> \"=\" <+> printExpr e <> \";\"\n-- C++ uses an indexed for loop with push_back\nprintStmt (IMapList resultVar resultType iterVar collection bodyStmts yieldExpr) =\n vsep\n [ resultDecl\n , block\n 4\n [idoc|for(size_t #{pretty iterVar}_idx = 0; #{pretty iterVar}_idx < #{printExpr collection}.size(); #{pretty iterVar}_idx++)|]\n ( vsep\n ( [idoc|auto #{pretty iterVar} = #{printExpr collection}[#{pretty iterVar}_idx];|]\n : map printStmt bodyStmts\n ++ [[idoc|#{pretty resultVar}.push_back(#{printExpr yieldExpr});|]]\n )\n )\n ]\n where\n resultDecl = case resultType of\n Just t -> [idoc|#{renderIType t} #{pretty resultVar};|]\n Nothing -> printStmt (IAssign resultVar Nothing (IListLit []))\nprintStmt (IIf resultVar resultType condExpr thenStmts thenExpr elseStmts elseExpr) =\n vsep\n [ resultDecl\n , block 4 [idoc|if(#{printExpr condExpr})|]\n (vsep (map printStmt thenStmts ++ [[idoc|#{pretty resultVar} = #{printExpr thenExpr};|]]))\n , block 4 \"else\"\n (vsep (map printStmt elseStmts ++ [[idoc|#{pretty resultVar} = #{printExpr elseExpr};|]]))\n ]\n where\n resultDecl = case resultType of\n Just t -> [idoc|#{renderIType t} #{pretty resultVar};|]\n Nothing -> [idoc|auto #{pretty resultVar};|]\nprintStmt (IReturn e) = \"return(\" <> printExpr e <> \");\"\nprintStmt (IExprStmt e) = printExpr e <> \";\"\nprintStmt (IFunDef _ _ _ _) = error \"IFunDef not yet implemented for C++ printer\"\n\nprintStmts :: [IStmt] -> [MDoc]\nprintStmts = map printStmt\n\n-- | Render C++ dispatch functions from structured dispatch entries.\nprintDispatch :: [DispatchEntry] -> [DispatchEntry] -> MDoc\nprintDispatch locals remotes =\n [idoc|uint8_t* local_dispatch(uint32_t mid, const uint8_t** args){\n switch(mid){\n #{align (vsep localCases)}\n default:\n std::ostringstream oss;\n oss << \"Invalid local manifold id: \" << mid;\n throw std::runtime_error(oss.str());\n }\n}\n\nuint8_t* remote_dispatch(uint32_t mid, const uint8_t** args){\n switch(mid){\n #{align (vsep remoteCases)}\n default:\n std::ostringstream oss;\n oss << \"Invalid remote manifold id: \" << mid;\n throw std::runtime_error(oss.str());\n }\n}|]\n where\n localCases = map (makeCase \"\") locals\n remoteCases = map (makeCase \"_remote\") remotes\n\n makeCase :: MDoc -> DispatchEntry -> MDoc\n makeCase suffix (DispatchEntry i n) =\n \"case\" <+> pretty i\n <> \":\"\n <+> \"return\"\n <+> manNamer i\n <> suffix\n <> tupled [\"args[\" <> pretty j <> \"]\" | j <- take n ([0 ..] :: [Int])]\n <> \";\"\n\n-- | Assemble a complete C++ pool file from an IProgram and C++-specific extras.\nprintProgram :: [MDoc] -> [MDoc] -> IProgram -> MDoc\nprintProgram serialization signatures prog =\n format\n (DF.embededFileText (DF.poolTemplate \"cpp\"))\n \"// <<>>\"\n [ vsep (map pretty (ipSources prog))\n , vsep serialization\n , vsep signatures\n , vsep (map pretty (ipManifolds prog))\n , printDispatch (ipLocalDispatch prog) (ipRemoteDispatch prog)\n ]\n\nprintTemplateHeader :: [MDoc] -> MDoc\nprintTemplateHeader [] = \"\"\nprintTemplateHeader ts = \"template\" <+> encloseSep \"<\" \">\" \",\" [\"class\" <+> t | t <- ts]\n\nprintRecordTemplate :: [MDoc] -> MDoc\nprintRecordTemplate [] = \"\"\nprintRecordTemplate ts = encloseSep \"<\" \">\" \",\" ts\n\n-- | Render a C++ struct definition.\nprintStructTypedef ::\n [MDoc] -> -- template parameters (e.g., [\"T\"])\n MDoc -> -- the name of the structure (e.g., \"Person\")\n [(MDoc, MDoc)] -> -- key and type for all fields\n MDoc\nprintStructTypedef params rname fields = vsep [template, struct]\n where\n template = printTemplateHeader params\n struct =\n block\n 4\n (\"struct\" <+> rname)\n (vsep [t <+> k <> \";\" | (k, t) <- fields])\n <> \";\"\n\n-- | Render a C++ serializer (toAnything) for a struct.\nprintSerializer ::\n [MDoc] -> -- template parameters\n MDoc -> -- type of thing being serialized\n [(MDoc, MDoc)] -> -- key and type for all fields\n MDoc\nprintSerializer params rtype fields =\n [idoc|\n#{printTemplateHeader params}\nvoid* toAnything(void* dest, void** cursor, const Schema* schema, const #{rtype}& obj)\n{\n return toAnything(dest, cursor, schema, std::make_tuple#{arguments});\n}\n|]\n where\n arguments = tupled [\"obj.\" <> key | (key, _) <- fields]\n\n-- | Render a C++ deserializer (fromAnything + get_shm_size) for a struct.\nprintDeserializer ::\n Bool -> -- build object with constructor\n [MDoc] -> -- template parameters\n MDoc -> -- type of thing being deserialized\n [(MDoc, MDoc)] -> -- key and type for all fields\n MDoc\nprintDeserializer _ params rtype fields =\n [idoc|\n#{printTemplateHeader params}\n#{block 4 header body}\n\n#{printTemplateHeader params}\n#{block 4 headerGetSize bodyGetSize}\n|]\n where\n header =\n [idoc|#{rtype} fromAnything(const Schema* schema, const void * anything, #{rtype}* dummy = nullptr, const void* base_ptr = nullptr)|]\n body =\n vsep $\n [[idoc|#{rtype} obj;|]]\n <> zipWith assignFields [0 ..] fields\n <> [\"return obj;\"]\n\n assignFields :: Int -> (MDoc, MDoc) -> MDoc\n assignFields idx (keyName, keyType) =\n vsep\n [ [idoc|#{keyType}* elemental_dumby_#{keyName} = nullptr;|]\n , [idoc|obj.#{keyName} = fromAnything(schema->parameters[#{pretty idx}], (char*)anything + schema->offsets[#{pretty idx}], elemental_dumby_#{keyName}, base_ptr);|]\n ]\n\n headerGetSize = [idoc|size_t get_shm_size(const Schema* schema, const #{rtype}& data)|]\n bodyGetSize =\n vsep $\n [\"size_t size = 0;\"]\n <> [getSize idx key | (idx, (key, _)) <- zip [0 ..] fields]\n <> [\"return size;\"]\n\n getSize :: Int -> MDoc -> MDoc\n getSize idx key = [idoc|size += get_shm_size(schema->parameters[#{pretty idx}], data.#{key});|]\n" }, { "path": "executable/CppTranslator.hs", "content": "{-# LANGUAGE FlexibleContexts #-}\n{-# LANGUAGE FlexibleInstances #-}\n{-# LANGUAGE OverloadedStrings #-}\n{-# LANGUAGE QuasiQuotes #-}\n{-# LANGUAGE TemplateHaskell #-}\n{-# LANGUAGE UndecidableInstances #-}\n{-# LANGUAGE ViewPatterns #-}\n\n{- |\nModule : CppTranslator\nDescription : Translate 'SerialManifold' trees into C++ pool source code\nCopyright : (c) Zebulun Arendsee, 2016-2026\nLicense : Apache-2.0\nMaintainer : z@morloc.io\n\nStateful C++ translator using the two-phase IR architecture: lower the\n'SerialManifold' tree into 'IStmt'/'IExpr' via 'LowerConfig', then print\nvia 'CppPrinter'. Handles C++-specific concerns like compilation flags,\ninclude paths, struct generation, and template instantiation.\n-}\nmodule CppTranslator\n ( translate\n , cppLang\n ) where\n\nimport Control.Monad.Identity (Identity, runIdentity)\nimport qualified Control.Monad.State as CMS\nimport qualified CppPrinter as CP\nimport qualified Data.Char as DC\nimport qualified Data.Set as Set\nimport Data.Text (Text)\nimport qualified Data.Text as T\nimport Morloc.CodeGenerator.Grammars.Common\nimport Morloc.CodeGenerator.Grammars.Macro (expandMacro)\nimport Morloc.CodeGenerator.Grammars.Translator.Imperative\n ( IType (..)\n , LowerConfig (..)\n , buildProgramM\n , defaultFoldRules\n , expandDeserialize\n , expandSerialize\n , toIType\n )\nimport Morloc.CodeGenerator.Namespace\nimport Morloc.CodeGenerator.Serial\n ( serialAstToType\n , shallowType\n )\nimport Morloc.Data.Doc\nimport qualified Morloc.Data.GMap as GMap\nimport qualified Morloc.Data.Map as Map\nimport qualified Morloc.Data.Text as MT\nimport qualified Morloc.Language as ML\nimport qualified Morloc.Monad as MM\nimport qualified Morloc.Version as MV\nimport Morloc.Quasi\nimport qualified Morloc.System as MS\nimport qualified Morloc.TypeEval as TE\n\n-- HACK: repeating these here is hacky\n-- This same data is repeated in cpp/lang.yaml\ncppLang :: ML.Lang\ncppLang = ML.Lang \"cpp\" \"cpp\"\n\nserialType :: MDoc\nserialType = \"uint8_t*\"\n\ndata CallSemantics = Copy | Reference | ConstPtr\n\nclass HasCppType a where\n cppTypeOf :: a -> CppTranslator MDoc\n\n cppArgOf :: CallSemantics -> Arg a -> CppTranslator MDoc\n\nsetCallSemantics :: CallSemantics -> MDoc -> MDoc\nsetCallSemantics Copy typestr = typestr\nsetCallSemantics Reference typestr = \"const\" <+> typestr <> \"&\"\nsetCallSemantics ConstPtr typestr = \"const\" <+> typestr\n\nchooseCallSemantics :: TypeM -> CallSemantics\nchooseCallSemantics Passthrough = ConstPtr -- const uint8_t* packet\nchooseCallSemantics (Serial _) = ConstPtr -- const uint8_t* packet\nchooseCallSemantics (Native _) = Reference -- for now, primitives should be pass by copy\nchooseCallSemantics (Function _ _) = Copy -- currently not used\n\ninstance HasCppType TypeM where\n cppTypeOf (Serial _) = return serialType\n cppTypeOf (Native c) = cppTypeOf c\n cppTypeOf Passthrough = return serialType\n cppTypeOf (Function ts t) = do\n t' <- cppTypeOf t\n ts' <- mapM cppTypeOf ts\n return $ \"std::function<\" <> t' <> tupled ts' <> \">\"\n\n cppArgOf s (Arg i t) = do\n typeStr <- cppTypeOf t\n let typeStrQualified = setCallSemantics s typeStr\n return $ case t of\n (Serial _) -> typeStrQualified <+> svarNamer i\n (Native _) -> typeStrQualified <+> nvarNamer i\n Passthrough -> typeStrQualified <+> svarNamer i\n (Function _ _) -> typeStrQualified <+> nvarNamer i\n\ninstance HasCppType NativeManifold where\n cppTypeOf = cppTypeOf . typeMof\n cppArgOf s r = cppArgOf s $ fmap typeMof r\n\ninstance {-# OVERLAPPABLE #-} (HasTypeF e) => HasCppType e where\n cppTypeOf = f . typeFof\n where\n f (UnkF (FV _ x)) = return $ pretty x\n f (VarF (FV _ x)) = return $ pretty x\n f (FunF ts t) = do\n t' <- f t\n ts' <- mapM f ts\n return $ \"std::function<\" <> t' <> tupled ts' <> \">\"\n f (NatLitF _) = return mempty\n f (AppF t ts) = do\n t' <- f t\n let runtimeTs = [x | x <- ts, not (isNatLitF x)]\n ts' <- mapM f runtimeTs\n return . pretty $ expandMacro (render t') (map render ts')\n f t@(NamF _ (FV gc (CV \"struct\")) _ rs) = do\n recmap <- CMS.gets translatorRecmap\n -- handle autogenerated structs\n case lookup (FV gc (CV \"struct\"), map fst rs) recmap of\n (Just rec) -> do\n params <- typeParams (zip (map snd (recFields rec)) (map snd rs))\n return $ recName rec <> params\n Nothing -> error $ \"Record missing from recmap: \" <> show t <> \" from map: \" <> show recmap\n f (NamF _ (FV _ (CV \"arrow\")) _ _) = return \"mlc::ArrowTable\"\n f (NamF _ (FV _ s) ps _) = do\n ps' <- mapM f ps\n return $ pretty s <> CP.printRecordTemplate ps'\n f (EffectF _ t) = do\n t' <- f t\n return $ \"std::function<\" <> t' <> \"()\" <> \">\"\n f (OptionalF t) = do\n t' <- f t\n return $ \"std::optional<\" <> t' <> \">\"\n isNatLitF (NatLitF _) = True\n isNatLitF _ = False\n\n cppArgOf s (Arg i t) = do\n t' <- cppTypeOf (typeFof t)\n return $ setCallSemantics s t' <+> nvarNamer i\n\ndata CppTranslatorState = CppTranslatorState\n { translatorCounter :: Int\n , translatorRecmap :: RecMap\n , translatorSignatureSet :: Set.Set Int\n , translatorLocalManifoldSet :: Set.Set Int\n , translatorRemoteManifoldSet :: Set.Set Int\n , translatorCurrentManifold :: Int\n , translatorEffectLabels :: Map.Map Int (Set.Set Text)\n }\n\ninstance Defaultable CppTranslatorState where\n defaultValue =\n CppTranslatorState\n { translatorCounter = 0\n , translatorRecmap = []\n , translatorSignatureSet = Set.empty\n , translatorLocalManifoldSet = Set.empty\n , translatorRemoteManifoldSet = Set.empty\n , translatorCurrentManifold = -1 -- -1 indicates we are not inside a manifold\n , translatorEffectLabels = Map.empty\n }\n\ntype CppTranslator a = CMS.StateT CppTranslatorState Identity a\ntype CppTranslatorM = CMS.StateT CppTranslatorState Identity\n\ngetCounter :: CppTranslator Int\ngetCounter = do\n s <- CMS.get\n let i = translatorCounter s\n CMS.put $ s {translatorCounter = translatorCounter s + 1}\n return i\n\nresetCounter :: CppTranslator ()\nresetCounter = do\n s <- CMS.get\n CMS.put $ s {translatorCounter = 0}\n\ntranslate :: [Source] -> [SerialManifold] -> MorlocMonad Script\ntranslate srcs es = do\n -- scopeMap :: GMap Int MVar (Map.Map Lang Scope)\n scopeMap <- MM.gets stateConcreteTypedefs\n\n -- universalScopeMap :: GMap Int MVar Scope\n universalScopeMap <- MM.gets stateUniversalConcreteTypedefs\n\n effectMap <- MM.gets stateManifoldEffects\n\n -- Canonicalize C++ source paths once up front so that the #include\n -- directives emitted by makeCppCode and the -I flags emitted by\n -- makeTheMaker see exactly the same absolute paths. Before this,\n -- `#include \"./src/foo.hpp\"` could not be resolved against\n -- `-I/abs/src` because the `src/` prefix was duplicated.\n (srcs', _, _) <- handleFlagsAndPaths srcs\n\n let recmap = unifyRecords . concatMap collectRecords $ es\n translatorState = defaultValue {translatorRecmap = recmap, translatorEffectLabels = effectMap}\n code = CMS.evalState (makeCppCode srcs' es universalScopeMap scopeMap) translatorState\n\n maker <- makeTheMaker srcs'\n\n poolSubdir <- MM.getModuleName\n\n return $\n Script\n { scriptBase = \"pool\"\n , scriptLang = cppLang\n , scriptCode = \".\" :/ Dir \"pools\" [Dir poolSubdir [File \"pool.cpp\" (Code (T.replace \"__MORLOC_VERSION__\" (MT.pack MV.versionStr) (render code)))]]\n , scriptMake = maker\n }\n\nmakeCppCode ::\n [Source] ->\n [SerialManifold] ->\n Map.Map Lang Scope ->\n GMap Int MVar (Map.Map Lang Scope) ->\n CppTranslator MDoc\nmakeCppCode srcs es univeralScopeMap scopeMap = do\n -- ([MDoc], [MDoc])\n (srcDecl, srcSerial) <- generateSourcedSerializers univeralScopeMap scopeMap es\n\n -- write include statements for sources\n let includeDocs = map translateSource (unique . mapMaybe srcPath $ srcs)\n\n signatures <- concat <$> mapM makeSignature es\n\n (autoDecl, autoSerial) <- generateAnonymousStructs\n let serializationCode = autoDecl ++ srcDecl ++ autoSerial ++ srcSerial\n\n -- build the program (translates each manifold tree)\n program <- buildProgramM includeDocs es translateSegment\n\n -- create and return complete pool script\n return $ CP.printProgram serializationCode signatures program\n\nmetaTypedefs ::\n GMap Int MVar (Map.Map Lang Scope) ->\n Int -> -- manifold index\n Scope\nmetaTypedefs tmap i =\n case GMap.lookup i tmap of\n (GMapJust langmap) -> case Map.lookup cppLang langmap of\n (Just scope) -> Map.filter (not . null) scope\n Nothing -> Map.empty\n _ -> Map.empty\n\n-- | Collect TVar names of all named (non-anonymous) record types used\n-- in a SerialManifold tree.\ncollectNamedRecordTVars :: SerialManifold -> Set.Set TVar\ncollectNamedRecordTVars e0 =\n runIdentity $ foldWithSerialManifoldM fm e0\n where\n fm = defaultValue\n { opFoldWithNativeExprM = nativeExpr\n , opFoldWithSerialExprM = serialExpr\n }\n\n nativeExpr _ (DeserializeN_ t s xs) =\n return $ Set.unions [xs, seekNamedRecs t, seekNamedRecs (serialAstToType s)]\n nativeExpr efull e =\n return $ foldlNE Set.union (seekNamedRecs (typeFof efull)) e\n\n serialExpr _ (SerializeS_ s xs) =\n return $ Set.union (seekNamedRecs (serialAstToType s)) xs\n serialExpr _ e = return $ foldlSE Set.union Set.empty e\n\n seekNamedRecs :: TypeF -> Set.Set TVar\n seekNamedRecs (NamF _ (FV v (CV c)) _ rs)\n | c /= \"struct\" = Set.insert v (Set.unions (map (seekNamedRecs . snd) rs))\n seekNamedRecs (NamF _ _ _ rs) = Set.unions (map (seekNamedRecs . snd) rs)\n seekNamedRecs (FunF ts t) = Set.unions (map seekNamedRecs (t : ts))\n seekNamedRecs (AppF t ts) = Set.unions (map seekNamedRecs (t : ts))\n seekNamedRecs (EffectF _ t) = seekNamedRecs t\n seekNamedRecs (OptionalF t) = seekNamedRecs t\n seekNamedRecs _ = Set.empty\n\nmakeTheMaker :: [Source] -> MorlocMonad [SysCommand]\nmakeTheMaker srcs = do\n poolSubdir <- MM.getModuleName\n let outfile = pretty $ \"pools\" poolSubdir ML.makeExecutablePoolName cppLang\n let src = pretty $ \"pools\" poolSubdir ML.makeSourcePoolName cppLang\n\n (_, flags, includes) <- handleFlagsAndPaths srcs\n\n bconf <- MM.gets stateBuildConfig\n let sanitizeFlags = case buildConfigSanitize bconf of\n Just True -> [\"-fsanitize=alignment\", \"-fno-sanitize-recover=alignment\"]\n _ -> []\n\n let incs = \"-I.\" : [pretty (\"-I\" <> i) | i <- includes]\n let flags' = map pretty (flags ++ sanitizeFlags)\n\n let cmd =\n SysRun . Code . render $\n [idoc|g++ -O2 -o #{outfile} #{src} #{hsep flags'} #{hsep incs}|]\n\n return [cmd]\n\nmakeSignature :: SerialManifold -> CppTranslator [MDoc]\nmakeSignature = foldWithSerialManifoldM fm\n where\n fm =\n defaultValue\n { opFoldWithSerialManifoldM = serialManifold\n , opFoldWithNativeManifoldM = nativeManifold\n }\n\n serialManifold (SerialManifold m _ form _ _) _ = manifoldSignature m serialType form\n\n nativeManifold e@(NativeManifold m _ form _) _ = do\n typestr <- cppTypeOf e\n manifoldSignature m typestr form\n\n manifoldSignature ::\n (HasTypeM t) => Int -> MDoc -> ManifoldForm (Or TypeS TypeF) t -> CppTranslator [MDoc]\n manifoldSignature i typestr form = do\n s <- CMS.get\n if Set.member i (translatorSignatureSet s)\n then return []\n else do\n let formArgs = typeMofForm form\n\n args <- mapM (\\r@(Arg _ t) -> cppArgOf (chooseCallSemantics t) r) formArgs\n CMS.put (s {translatorSignatureSet = Set.insert i (translatorSignatureSet s)})\n return [typestr <+> manNamer i <> tupled args <> \";\"]\n\ntupleKey :: Int -> MDoc -> MDoc\ntupleKey i v = [idoc|std::get<#{pretty i}>(#{v})|]\n\nrecordAccess :: MDoc -> MDoc -> MDoc\nrecordAccess record field = record <> \".\" <> field\n\ncppLowerConfig :: LowerConfig CppTranslatorM\ncppLowerConfig =\n LowerConfig\n { lcSrcName = \\src -> pretty (srcName src)\n , lcTypeOf = \\t -> Just . toIType <$> cppTypeOf t\n , lcSerialAstType = serializeTypeOf\n , lcDeserialAstType = \\s -> Just . toIType <$> cppTypeOf (shallowType s)\n , lcRawDeserialAstType = rawTypeOf\n , lcTypeMOf = \\_ -> return Nothing\n , lcPackerName = \\src -> pretty (srcName src)\n , lcUnpackerName = \\src -> pretty (srcName src)\n , lcRecordAccessor = \\_ _ -> recordAccess\n , lcDeserialRecordAccessor = \\i _ v -> tupleKey i v\n , lcTupleAccessor = tupleKey\n , lcNewIndex = getCounter\n , lcPrintExpr = CP.printExpr\n , lcPrintStmt = CP.printStmt\n , lcEvalPattern = \\t p xs -> do\n state <- CMS.get\n return $ evaluatePattern state t p xs\n , lcListConstructor = \\_ _ es -> encloseSep \"{\" \"}\" \",\" es\n , lcTupleConstructor = \\_ -> ((<>) \"std::make_tuple\" . tupled)\n , lcRecordConstructor = \\recType _ _ _ rs -> do\n t <- cppTypeOf recType\n idx <- getCounter\n let v' = \"a\" <> pretty idx\n decl = t <+> v' <+> \"=\" <+> encloseSep \"{\" \"}\" \",\" (map snd rs) <> \";\"\n return $ defaultValue {poolExpr = v', poolPriorLines = [decl]}\n , lcForeignCall = \\socketFile mid args ->\n let argList = [dquotes socketFile, pretty mid] <> args <> [\"NULL\"]\n in [idoc|foreign_call#{tupled argList}|]\n , lcRemoteCall = \\socketFile mid res args -> do\n let resMem = pretty $ fromMaybe (-1) (remoteResourcesMemory res)\n resTime = pretty $ maybe (-1) unTimeInSeconds (remoteResourcesTime res)\n resCPU = pretty $ fromMaybe (-1) (remoteResourcesThreads res)\n resGPU = pretty $ fromMaybe 0 (remoteResourcesGpus res)\n cacheDir = \".morloc-cache\"\n argList = encloseSep \"{\" \"}\" \",\" args\n setup =\n [idoc|resources_t resources = {#{resMem}, #{resTime}, #{resCPU}, #{resGPU}};\nconst uint8_t* args[] = #{argList};\nchar* errmsg = NULL;|]\n call =\n [idoc|remote_call(\n #{pretty mid},\n #{dquotes socketFile},\n #{dquotes cacheDir},\n &resources,\n args,\n #{pretty (length args)},\n &errmsg\n);\nPROPAGATE_ERROR(errmsg)|]\n return $ defaultValue {poolExpr = call, poolPriorLines = [setup]}\n , lcMakeLet = \\namer letIndex mt e1 e2 -> do\n typestr <- case mt of\n (Just t) -> cppTypeOf t\n Nothing -> return serialType\n return $ makeLet namer letIndex typestr e1 e2\n , lcReturn = \\e -> \"return(\" <> e <> \");\"\n , lcMakeIf = \\origExpr condDocs thenDocs elseDocs -> do\n idx <- getCounter\n let v = helperNamer idx\n typeStr <- cppTypeOf origExpr\n let condE = poolExpr condDocs\n thenE = poolExpr thenDocs\n elseE = poolExpr elseDocs\n thenBlock = poolPriorLines thenDocs <> [v <+> \"=\" <+> thenE <> \";\"]\n elseBlock = poolPriorLines elseDocs <> [v <+> \"=\" <+> elseE <> \";\"]\n decl = typeStr <+> v <> \";\"\n ifStmt = vsep\n [ decl\n , \"if\" <+> parens condE <+> \"{\"\n , indent 4 (vsep thenBlock)\n , \"} else {\"\n , indent 4 (vsep elseBlock)\n , \"}\"\n ]\n return $ PoolDocs\n { poolCompleteManifolds = poolCompleteManifolds condDocs <> poolCompleteManifolds thenDocs <> poolCompleteManifolds elseDocs\n , poolExpr = v\n , poolPriorLines = poolPriorLines condDocs <> [ifStmt]\n , poolPriorExprs = poolPriorExprs condDocs <> poolPriorExprs thenDocs <> poolPriorExprs elseDocs\n }\n , lcMakeDoBlock = \\t stmts expr ->\n let isUnit = case t of\n EffectF _ (VarF (FV tv _)) -> tv == TV \"Unit\"\n VarF (FV tv _) -> tv == TV \"Unit\"\n _ -> False\n in (,) [] $ case (isUnit, stmts) of\n (True, []) -> \"[&](){\" <> expr <> \"; return mlc::Unit{};}\"\n (True, _) -> \"[&](){\" <> nest 4 (line <> vsep (stmts <> [expr <> \";\", \"return mlc::Unit{};\"])) <> line <> \"}\"\n (False, []) -> \"[&](){return \" <> expr <> \";}\"\n (False, _) -> \"[&](){\" <> nest 4 (line <> vsep (stmts <> [\"return \" <> expr <> \";\"])) <> line <> \"}\"\n , lcSerialize = \\v s -> serialize v s\n , lcDeserialize = \\t v s -> do\n typestr <- cppTypeOf t\n deserialize v typestr s\n , lcMakeFunction = \\mname args manifoldType priorLines body headForm -> do\n callIndex <- CMS.gets translatorCurrentManifold\n state <- CMS.get\n let effectLabels = Map.findWithDefault Set.empty callIndex (translatorEffectLabels state)\n let alreadyDone = case headForm of\n (Just HeadManifoldFormRemoteWorker) -> Set.member callIndex (translatorRemoteManifoldSet state)\n _ -> Set.member callIndex (translatorLocalManifoldSet state)\n if alreadyDone\n then return Nothing\n else do\n case headForm of\n (Just HeadManifoldFormRemoteWorker) ->\n CMS.modify\n (\\s -> s {translatorRemoteManifoldSet = Set.insert callIndex (translatorRemoteManifoldSet s)})\n _ ->\n CMS.modify\n (\\s -> s {translatorLocalManifoldSet = Set.insert callIndex (translatorLocalManifoldSet s)})\n returnTypeStr <- returnType manifoldType\n typedArgs <- mapM (\\r@(Arg _ t) -> cppArgOf (chooseCallSemantics t) r) args\n let fullName = mname <> mnameExt headForm\n decl = returnTypeStr <+> fullName <> tupled typedArgs\n enrichError = case headForm of\n Just HeadManifoldFormRemoteWorker -> True\n _ -> Set.member \"Error\" effectLabels\n tryBody = block 4 \"try\" (vsep $ priorLines <> [body])\n catchBody\n | enrichError =\n let throwStatement = vsep\n [ [idoc|std::string error_message = \"Error raised in C++ pool by #{mname}:\\n\" + std::string(e.what());|]\n , [idoc|throw std::runtime_error(error_message);|]\n ]\n in block 4 \"catch (const std::exception& e)\" throwStatement\n | otherwise = block 4 \"catch (...)\" \"throw;\"\n return . Just . block 4 decl . vsep $ [tryBody <+> catchBody]\n , lcMakeLambda = \\mname contextArgs boundArgs ->\n let vs' = take (length boundArgs) (map (\\j -> \"std::placeholders::_\" <> viaShow j) ([1 ..] :: [Int]))\n in [idoc|std::bind(#{cat (punctuate \",\" (mname : (contextArgs ++ vs')))})|]\n }\n where\n -- For serialization, records become tuples (that's what _put_value/toAnything expects)\n serializeTypeOf :: SerialAST -> CppTranslator (Maybe IType)\n serializeTypeOf (SerialObject _ _ _ rs) = Just . toIType <$> recordToCppTuple (map snd rs)\n serializeTypeOf s = Just . toIType <$> cppTypeOf (serialAstToType s)\n\n rawTypeOf :: SerialAST -> CppTranslator (Maybe IType)\n rawTypeOf (SerialObject _ _ _ rs) = Just . toIType <$> recordToCppTuple (map snd rs)\n rawTypeOf s = Just . toIType <$> cppTypeOf (serialAstToType s)\n\n makeLet :: (Int -> MDoc) -> Int -> MDoc -> PoolDocs -> PoolDocs -> PoolDocs\n makeLet namer letIndex typestr (PoolDocs ms1 e1 rs1 pes1) (PoolDocs ms2 e2 rs2 pes2) =\n let letAssignment = [idoc|#{typestr} #{namer letIndex} = #{e1};|]\n rs = rs1 <> [letAssignment] <> rs2\n in PoolDocs\n { poolCompleteManifolds = ms1 <> ms2\n , poolExpr = e2\n , poolPriorLines = rs\n , poolPriorExprs = pes1 <> pes2\n }\n\n mnameExt :: Maybe HeadManifoldForm -> MDoc\n mnameExt (Just HeadManifoldFormRemoteWorker) = \"_remote\"\n mnameExt _ = \"\"\n\n returnType :: TypeM -> CppTranslator MDoc\n returnType (Function _ t) = cppTypeOf t\n returnType t = cppTypeOf t\n\n-- Use `#include \"foo.h\"` rather than `#include `\ntranslateSource ::\n -- | Path to a header (e.g., `$MORLOC_HOME/src/foo.h`)\n Path ->\n MDoc\ntranslateSource path = \"#include\" <+> (dquotes . pretty) path\n\nserialize :: MDoc -> SerialAST -> CppTranslator PoolDocs\nserialize v s = do\n (expr, stmts) <- expandSerialize cppLowerConfig v s\n return $\n PoolDocs\n { poolCompleteManifolds = []\n , poolExpr = CP.printExpr expr\n , poolPriorLines = map CP.printStmt stmts\n , poolPriorExprs = []\n }\n\n-- reverse of serialize, parameters are the same\ndeserialize :: MDoc -> MDoc -> SerialAST -> CppTranslator (MDoc, [MDoc])\ndeserialize varname0 typestr0 s0 = do\n (expr, stmts) <- expandDeserialize cppLowerConfig varname0 s0\n let rendered = CP.printExpr expr\n if null stmts\n then return (rendered, [])\n else do\n schemaVar <- helperNamer <$> getCounter\n let final = [idoc|#{typestr0} #{schemaVar} = #{rendered};|]\n return (schemaVar, map CP.printStmt stmts ++ [final])\n\nrecordToCppTuple :: [SerialAST] -> CppTranslator MDoc\nrecordToCppTuple ts = do\n tsDocs <- mapM (cppTypeOf . serialAstToType) ts\n return $ \"std::tuple\" <> encloseSep \"<\" \">\" \",\" tsDocs\n\ntranslateSegment :: SerialManifold -> CppTranslator MDoc\ntranslateSegment m0 = do\n resetCounter\n e <- surroundFoldSerialManifoldM manifoldIndexer (defaultFoldRules cppLowerConfig) m0\n return $ renderPoolDocs e\n where\n manifoldIndexer =\n makeManifoldIndexer\n (CMS.gets translatorCurrentManifold)\n (\\i -> CMS.modify (\\s -> s {translatorCurrentManifold = i}))\n\n-- handle string interpolation\nevaluatePattern :: CppTranslatorState -> TypeF -> Pattern -> [MDoc] -> MDoc\nevaluatePattern _ _ (PatternText s ss) xs = \"interweave_strings\" <> tupled [fragments, insertions]\n where\n fragments = encloseSep \"{\" \"}\" \", \" (map (dquotes . pretty . escapeQuotes \"\\\"\" \"\\\\\\\"\" . escapeStringLit) (s : ss))\n insertions = encloseSep \"{\" \"}\" \", \" xs\n\n-- handle getters\nevaluatePattern _ _ (PatternStruct (ungroup -> [ss])) [m] =\n writeSelector m ss\nevaluatePattern _ _ (PatternStruct (ungroup -> sss)) [m] =\n encloseSep \"{\" \"}\" \",\" (map (writeSelector m) sss)\nevaluatePattern state0 t0 (PatternStruct s0) (m0 : xs0) =\n patternSetter makeTuple makeRecord accessTuple accessRecord m0 t0 s0 xs0\n where\n makeTuple (AppF _ ts) xs =\n let tupleTypes = CMS.evalState (mapM cppTypeOf ts) state0\n in \"std::tuple\" <> encloseSep \"<\" \">\" \",\" tupleTypes <> tupled xs\n makeTuple _ _ = error \"Unreachable\"\n\n makeRecord _ xs = encloseSep \"{\" \"}\" \", \" xs\n\n accessTuple _ m i = \"std::get<\" <> pretty i <> \">(\" <> m <> \")\"\n accessRecord _ d k = d <> \".\" <> pretty k\nevaluatePattern _ _ (PatternStruct _) [] = error \"Unreachable illegal pattern\"\n\nwriteSelector :: MDoc -> [Either Int Text] -> MDoc\nwriteSelector d [] = d\nwriteSelector d (Right k : rs) = writeSelector (d <> \".\" <> pretty k) rs\nwriteSelector d (Left i : rs) = writeSelector (\"std::get<\" <> pretty i <> \">\" <> parens d) rs\n\ntypeParams :: [(Maybe TypeF, TypeF)] -> CppTranslator MDoc\ntypeParams ts = CP.printRecordTemplate <$> mapM cppTypeOf [t | (Nothing, t) <- ts]\n\ngenerateAnonymousStructs :: CppTranslator ([MDoc], [MDoc])\ngenerateAnonymousStructs = do\n recmap <- CMS.gets translatorRecmap\n\n xs <- mapM makeSerializers (reverse . map snd $ recmap)\n\n return (concatMap fst xs, concatMap snd xs)\n where\n makeSerializers :: RecEntry -> CppTranslator ([MDoc], [MDoc])\n makeSerializers rec = do\n let templateTerms = map ((\"T\" <>) . pretty) ([1 ..] :: [Int])\n rs' = zip templateTerms (recFields rec)\n\n let params = [t | (t, (_, Nothing)) <- rs']\n rname = recName rec\n rtype = rname <> CP.printRecordTemplate [v | (v, (_, Nothing)) <- rs']\n\n let fieldNames = [k | (_, (k, _)) <- rs']\n\n fieldTypes <- mapM (\\(t, v) -> maybeM t cppTypeOf v) [(t', v') | (t', (_, v')) <- rs']\n\n let fields = [(pretty k, v) | (k, v) <- zip fieldNames fieldTypes]\n\n let structDecl = CP.printStructTypedef params rname fields\n serializer = CP.printSerializer params rtype fields\n deserializer = CP.printDeserializer False params rtype fields\n\n return ([structDecl], [serializer, deserializer])\n\n -- monadic form of `maybe` function\n maybeM :: (Monad m) => a -> (b -> m a) -> Maybe b -> m a\n maybeM _ f (Just x) = f x\n maybeM x _ Nothing = return x\n\ngenerateSourcedSerializers ::\n Map.Map Lang Scope ->\n GMap Int MVar (Map.Map Lang Scope) ->\n [SerialManifold] -> -- all segments that can be called in this pool\n CppTranslator\n ( [MDoc]\n , [MDoc]\n )\ngenerateSourcedSerializers univeralScopeMap scopeMap es0 = do\n perManifold <- Map.unions <$> mapM (foldSerialManifoldM fm) es0\n\n scope <- case Map.lookup cppLang univeralScopeMap of\n (Just scope) -> return scope\n Nothing -> return Map.empty\n\n -- Supplement per-manifold typedefs with universal scope entries for named\n -- record types that appear in this pool but are missing from the per-manifold\n -- scope (happens in secondary C++ pools called via foreign_call).\n let usedTypes = Set.unions (map collectNamedRecordTVars es0)\n missingTypes = Set.difference usedTypes (Map.keysSet perManifold)\n supplemental = Map.filterWithKey (\\k _ -> Set.member k missingTypes) scope\n typedef = Map.unionWith mergeScopes perManifold supplemental\n\n foldl groupQuad ([], []) . concat . Map.elems <$> Map.mapWithKeyM (makeSerials scope) typedef\n where\n -- given the universal map of scopes, pull out every one that is used in this subtree\n fm =\n defaultValue\n { opSerialManifoldM = \\(SerialManifold_ i _ _ _ e) -> return $ Map.unionWith mergeScopes (metaTypedefs scopeMap i) e\n , opNativeManifoldM = \\(NativeManifold_ i _ _ e) -> return $ Map.unionWith mergeScopes (metaTypedefs scopeMap i) e\n }\n\n -- there are likely to be repeats in the scopes, we only want the unique ones\n mergeScopes xs ys = unique (xs <> ys)\n\n groupQuad :: ([a], [a]) -> (a, a) -> ([a], [a])\n groupQuad (xs, ys) (x, y) = (x : xs, y : ys)\n\n makeSerials ::\n Scope -> TVar -> [([Either (TVar, Kind) TypeU], TypeU, ArgDoc, Bool)] -> CppTranslator [(MDoc, MDoc)]\n makeSerials s v xs = catMaybes <$> mapM (makeSerial s v) xs\n\n makeSerial ::\n Scope -> TVar -> ([Either (TVar, Kind) TypeU], TypeU, ArgDoc, Bool) -> CppTranslator (Maybe (MDoc, MDoc))\n makeSerial _ _ (_, NamU _ (TV \"struct\") _ _, _, _) = return Nothing\n makeSerial _ _ (_, NamU _ (TV \"arrow\") _ _, _, _) = return Nothing\n makeSerial scope _ (ps, NamU r (TV v) _ rs, _, _) = do\n params <- mapM (either (\\(p, _) -> return $ \"T\" <> pretty p) (\\_ -> return \"XXX_FIXME\")) ps\n let templateTerms = [\"T\" <> pretty p | Left (p, _) <- ps]\n rtype = pretty v <> CP.printRecordTemplate templateTerms\n rs' = map (second (evaluateTypeU scope)) rs\n fields = [(pretty k, showDefType ps (typeOf t)) | (k, t) <- rs']\n serializer = CP.printSerializer params rtype fields\n deserializer = CP.printDeserializer (r == NamObject) params rtype fields\n return $ Just (serializer, deserializer)\n makeSerial _ _ _ = return Nothing\n\n evaluateTypeU :: Scope -> TypeU -> TypeU\n evaluateTypeU scope t = case TE.evaluateType scope t of\n (Left e) -> error $ show e\n (Right t') -> t'\n\n showDefType :: [Either (TVar, Kind) TypeU] -> Type -> MDoc\n showDefType ps (UnkT v)\n | any (\\p -> either (\\(tv, _) -> tv == v) (const False) p) ps = \"T\" <> pretty v\n | otherwise = pretty v\n showDefType ps (VarT v)\n | any (\\p -> either (\\(tv, _) -> tv == v) (const False) p) ps = \"T\" <> pretty v\n | otherwise = pretty v\n showDefType _ (FunT _ _) = error \"Cannot serialize functions\"\n showDefType _ (NamT _ v _ _) = pretty v\n showDefType _ (NatLitT _) = mempty\n showDefType ps (AppT (VarT (TV v)) ts) = pretty $ expandMacro v (map (render . showDefType ps) runtimeTs)\n where runtimeTs = [t | t <- ts, not (isNatLitT t)]\n showDefType _ (AppT _ _) = error \"AppT is only OK with VarT, for now\"\n showDefType _ (EffectT _ _) = error \"Cannot show EffectT\"\n showDefType _ (NatAddT _ _) = mempty\n showDefType _ (NatMulT _ _) = mempty\n showDefType _ (NatSubT _ _) = mempty\n showDefType _ (NatDivT _ _) = mempty\n showDefType ps (OptionalT t) = \"std::optional<\" <> showDefType ps t <> \">\"\n isNatLitT (NatLitT _) = True\n isNatLitT _ = False\n\n-- C++ specific source handling (flags, headers, libraries)\n\nhandleFlagsAndPaths :: [Source] -> MorlocMonad ([Source], [Text], [Path])\nhandleFlagsAndPaths srcs = do\n state <- MM.get\n let gccversion = gccVersionFlag . foldl max 0 . map packageCppVersion $ statePackageMeta state\n let explicitLibs = map (\"-l\" <>) . unique . concatMap packageDependencies $ statePackageMeta state\n (srcs', libflags, paths) <-\n fmap unzip3\n . mapM flagAndPath\n . unique\n $ [s | s <- srcs, srcLang s == cppLang]\n\n home <- MM.asks configHome\n let mlcInclude = [\"-I\" <> home <> \"/include\"]\n mlcPch = [\"-include\", \"morloc_pch.hpp\"]\n mlcLib = [\"-L\" <> home <> \"/lib\", \"-Wl,-rpath,\" <> home <> \"/lib\", \"-lmorloc\", \"-lcppmorloc\", \"-lpthread\"]\n\n return\n ( filter (isJust . srcPath) srcs'\n , [gccversion] <> explicitLibs ++ (map MT.pack . concat) (mlcPch : mlcInclude : mlcLib : libflags)\n , unique (catMaybes paths)\n )\n\ngccVersionFlag :: Int -> Text\ngccVersionFlag i\n | i <= 17 = \"-std=c++17\"\n | otherwise = \"-std=c++\" <> MT.show' i\n\nflagAndPath :: Source -> MorlocMonad (Source, [String], Maybe Path)\nflagAndPath src@(Source _ srcL (Just p) _ _ _ _ _ _) | srcL == cppLang =\n case (MS.takeDirectory p, MS.dropExtensions (MS.takeFileName p), MS.takeExtensions p) of\n (\".\", base, \"\") -> do\n header <- lookupHeader base\n libFlags <- lookupLib base\n return (src {srcPath = Just header}, libFlags, Just (MS.takeDirectory header))\n (dir, base, _) -> do\n libFlags <- lookupLib base\n absDir <- liftIO $ MS.canonicalizePath dir\n absPath <- liftIO $ MS.canonicalizePath p\n return (src {srcPath = Just absPath}, libFlags, Just absDir)\n where\n lookupHeader :: String -> MorlocMonad Path\n lookupHeader base = do\n home <- MM.asks configHome\n let allPaths = getHeaderPaths home base [\".h\", \".hpp\", \".hxx\"]\n existingPaths <- liftIO . fmap catMaybes . mapM getFile $ allPaths\n case existingPaths of\n (x : _) -> liftIO $ MS.canonicalizePath x\n [] -> MM.throwSystemError $ \"Header file \" <> pretty base <> \".* not found\"\n\n lookupLib :: String -> MorlocMonad [String]\n lookupLib base = do\n home <- MM.asks configHome\n let libnamebase = filter DC.isAlphaNum (map DC.toLower base)\n let libname = \"lib\" <> libnamebase <> \".so\"\n let allPaths = getLibraryPaths home base libname\n existingPaths <- liftIO . fmap catMaybes . mapM getFile $ allPaths\n case existingPaths of\n (libpath : _) -> do\n libdir <- liftIO . MS.canonicalizePath . MS.takeDirectory $ libpath\n return\n [ \"-Wl,-rpath=\" <> libdir\n , \"-L\" <> libdir\n , \"-l\" <> libnamebase\n ]\n [] -> return []\nflagAndPath src@(Source _ srcL Nothing _ _ _ _ _ _) | srcL == cppLang = return (src, [], Nothing)\nflagAndPath _ = MM.throwSystemError $ \"flagAndPath should only be called for C++ functions\"\n\ngetFile :: Path -> IO (Maybe Path)\ngetFile x = do\n exists <- MS.doesFileExist x\n return $\n if exists\n then Just x\n else Nothing\n\ngetHeaderPaths :: Path -> String -> [String] -> [Path]\ngetHeaderPaths lib base exts = [path <> ext | path <- paths, ext <- exts]\n where\n paths =\n map\n MS.joinPath\n [ [base]\n , [\"include\", base]\n , [base, base]\n , [lib, \"include\", base]\n , [lib, \"src\", base, base]\n , [\"/usr/include\", base]\n , [\"/usr/local/include\", base]\n ]\n\ngetLibraryPaths :: Path -> String -> String -> [Path]\ngetLibraryPaths lib base sofile =\n map\n MS.joinPath\n [ [sofile]\n , [\"lib\", sofile]\n , [base, sofile]\n , [lib, \"lib\", sofile]\n , [lib, \"src\", base, sofile]\n , [lib, \"src\", base, \"lib\", sofile]\n ]\n" }, { "path": "executable/Main.hs", "content": "{- |\nModule : Main\nDescription : Executable main module\nCopyright : (c) Zebulun Arendsee, 2016-2026\nLicense : Apache-2.0\nMaintainer : z@morloc.io\n-}\nmodule Main where\n\nimport GHC.IO.Encoding (setLocaleEncoding, utf8)\nimport Options.Applicative\nimport Subcommands (runMorloc)\nimport UI\n\nmain :: IO ()\nmain = do\n setLocaleEncoding utf8\n runMorloc =<< execParser opts\n" }, { "path": "executable/Subcommands.hs", "content": "{-# LANGUAGE OverloadedStrings #-}\n\n{- |\nModule : Subcommands\nDescription : Dispatch CLI subcommands and inject the translator callback\nCopyright : (c) Zebulun Arendsee, 2016-2026\nLicense : Apache-2.0\nMaintainer : z@morloc.io\n\nImplements each CLI subcommand (make, typecheck, install, init, dump) and\ndefines the 'TranslateFn' callback that routes C++ to 'CppTranslator' and\nother languages to the generic translator. This is the dependency injection\npoint that keeps translator code out of the library.\n-}\nmodule Subcommands (runMorloc) where\n\nimport Control.Exception (SomeException, bracket, finally, try)\nimport Data.Time.Clock (getCurrentTime)\nimport Data.Time.Format (formatTime, defaultTimeLocale)\nimport qualified CppTranslator\nimport qualified Data.Aeson as JSON\nimport qualified Data.ByteString.Lazy as BL\nimport qualified Data.Map as Map\nimport qualified Data.Set as Set\nimport qualified Data.Text as T\nimport qualified Data.Text.IO as TIO\nimport Morloc (generatePools)\nimport qualified Morloc as M\nimport Morloc.CodeGenerator.Emit (TranslateFn)\nimport qualified Morloc.CodeGenerator.Grammars.Translator.Generic as Generic\nimport Morloc.CodeGenerator.Grammars.Translator.PseudoCode (pseudocodeSerialManifold)\nimport Morloc.CodeGenerator.Namespace (SerialManifold (..))\nimport qualified Morloc.CodeGenerator.SystemConfig as MSC\nimport qualified Morloc.Completion as Completion\nimport qualified Morloc.Config as Config\nimport Morloc.Data.Doc\nimport qualified Morloc.Data.GMap as GMap\nimport qualified Morloc.Data.Text as MT\nimport qualified Morloc.Frontend.API as F\nimport Morloc.Module (OverwriteProtocol (..), findMainLocFile)\nimport qualified Morloc.Module as Mod\nimport qualified Morloc.Monad as MM\nimport Morloc.Namespace.Expr\nimport Morloc.Namespace.Prim\nimport Morloc.Namespace.State\nimport Morloc.Namespace.Type\nimport qualified Morloc.ProgramBuilder.Install as Install\nimport Morloc.Typecheck.Internal (prettyTypeU)\nimport System.Directory\n ( createDirectoryIfMissing\n , doesDirectoryExist\n , doesFileExist\n , getCurrentDirectory\n , listDirectory\n , removeDirectoryRecursive\n , removeFile\n , setCurrentDirectory\n )\nimport System.Exit (exitFailure, exitSuccess)\nimport System.FilePath (dropExtension, takeDirectory, takeFileName)\nimport System.IO (hPutStrLn, stderr)\nimport System.IO.Temp (createTempDirectory)\nimport qualified System.Process as SP\nimport UI\n\ndecodePackageMeta :: BL.ByteString -> Maybe PackageMeta\ndecodePackageMeta = JSON.decode\n\n-- | Route each language to its translator.\ntranslator :: TranslateFn\ntranslator lang srcs es\n | lang == CppTranslator.cppLang = CppTranslator.translate srcs es\n | otherwise = Generic.translate lang srcs es\n\nrunMorloc :: CliCommand -> IO ()\nrunMorloc args = do\n config <- getConfig args\n buildConfig <- Config.loadBuildConfig config\n let verbose = getVerbosity args\n runPassed <- case args of\n (CmdMake g) -> cmdMake g verbose config buildConfig\n (CmdInstall g) -> cmdInstall g verbose config buildConfig\n (CmdTypecheck g) -> cmdTypecheck g verbose config buildConfig\n (CmdDump g) -> cmdDump g verbose config buildConfig\n (CmdInit g) -> cmdInit g config\n (CmdList g) -> cmdList g config\n (CmdUninstall g) -> cmdUninstall g config\n (CmdNew g) -> cmdNew g\n (CmdEval g) -> cmdEval g verbose config buildConfig\n case runPassed of\n True -> exitSuccess\n False -> exitFailure\n\n-- | read the global morloc config file or return a default one\ngetConfig :: CliCommand -> IO Config.Config\ngetConfig (CmdMake g) = getConfig' (makeConfig g) (makeVanilla g)\ngetConfig (CmdInstall g) = getConfig' (installConfig g) (installVanilla g)\ngetConfig (CmdTypecheck g) = getConfig' (typecheckConfig g) (typecheckVanilla g)\ngetConfig (CmdDump g) = getConfig' (dumpConfig g) (dumpVanilla g)\ngetConfig (CmdInit g) = getConfig' (initConfig g) (initVanilla g)\ngetConfig (CmdList g) = getConfig' (listConfig g) (listVanilla g)\ngetConfig (CmdUninstall g) = getConfig' (uninstallConfig g) (uninstallVanilla g)\ngetConfig (CmdEval g) = getConfig' (evalConfig g) (evalVanilla g)\ngetConfig (CmdNew _) = getConfig' \"\" False\n\ngetConfig' :: String -> Bool -> IO Config.Config\ngetConfig' _ True = Config.loadMorlocConfig Nothing\ngetConfig' \"\" _ = Config.loadMorlocConfig Nothing\ngetConfig' filename _ = Config.loadMorlocConfig (Just filename)\n\ngetVerbosity :: CliCommand -> Int\ngetVerbosity (CmdMake g) = makeVerbose g\ngetVerbosity (CmdInstall g) = installVerbose g\ngetVerbosity (CmdTypecheck g) = typecheckVerbose g\ngetVerbosity (CmdDump g) = dumpVerbose g\ngetVerbosity (CmdInit g) = if initQuiet g then 0 else 1\ngetVerbosity (CmdList g) = listVerbose g\ngetVerbosity (CmdEval g) = evalVerbose g\ngetVerbosity (CmdUninstall _) = 0\ngetVerbosity (CmdNew _) = 0\n\nreadScript :: Bool -> String -> IO (Maybe Path, Code)\nreadScript True code = return (Nothing, Code (MT.pack code))\nreadScript _ filename = do\n code <- MT.readFile filename\n return (Just filename, Code code)\n\n-- | Typecheck callback for module installation\ntypecheckModuleFn :: FilePath -> MorlocMonad [(T.Text, T.Text)]\ntypecheckModuleFn mainFile = do\n code <- liftIO $ MT.readFile mainFile\n -- Save current state, run typecheck in a clean sub-state\n savedState <- MM.get\n result <-\n MM.catchError\n ( do\n xs <- M.typecheckFrontend (Just mainFile) (Code code)\n st <- MM.get\n return\n [ (render (pretty v), render (pretty t))\n | AnnoS (Idx i t) _ _ <- xs\n , Just v <- [Map.lookup i (stateName st)]\n ]\n )\n (\\_ -> return [])\n -- Restore state so module typechecking doesn't pollute the parent state\n MM.put savedState\n return result\n\n-- | Install a module\ncmdInstall :: InstallCommand -> Int -> Config.Config -> BuildConfig -> IO Bool\ncmdInstall args verbosity conf buildConfig = do\n userSources <- Map.fromList <$> mapM (\\modstr -> do\n name <- Mod.extractModuleName modstr\n return (name, modstr)) moduleTexts\n let cmdInstall' =\n mapM\n ( \\modstr ->\n Mod.installModule\n (installForce args)\n (installUseSSH args)\n libpath\n (Config.configPlaneCore conf)\n mayTypecheck\n userSources\n Set.empty\n Mod.ExplicitInstall\n modstr\n )\n moduleTexts\n passed <- MM.runMorlocMonad Nothing verbosity conf buildConfig cmdInstall' >>= MM.writeMorlocReturn\n if passed && installBuild args\n then buildInstalledModules args verbosity conf buildConfig moduleTexts libpath\n else return passed\n where\n libpath = Config.configLibrary conf Config.configPlane conf\n moduleTexts = map MT.pack (installModuleStrings args)\n\n mayTypecheck =\n if installNoTypecheck args\n then Nothing\n else Just typecheckModuleFn\n\n-- | Build and install executables for installed modules\nbuildInstalledModules ::\n InstallCommand -> Int -> Config.Config -> BuildConfig -> [T.Text] -> FilePath -> IO Bool\nbuildInstalledModules args verbosity conf buildConfig moduleTexts libpath = do\n results <- mapM buildOne moduleTexts\n return (and results)\n where\n force = installForce args == ForceOverwrite\n\n buildOne modstr = do\n name <- T.unpack <$> Mod.extractModuleName modstr\n let moduleDir = libpath name\n mainFile <- findMainLocFile moduleDir name\n case mainFile of\n Nothing -> do\n putStrLn $ \"Warning: no main.loc found for '\" <> name <> \"', skipping build\"\n return True\n Just locFile -> do\n origDir <- getCurrentDirectory\n setCurrentDirectory moduleDir\n buildResult <-\n buildModuleExecutable locFile name verbosity conf buildConfig force\n `finally` setCurrentDirectory origDir\n return buildResult\n\n buildModuleExecutable locFile _name verbosity' config buildConfig' forceOverwrite = do\n code <- MT.readFile locFile\n makeAndInstall (Just locFile) Nothing (Code code) [] verbosity' config buildConfig' forceOverwrite\n\n-- | Compile a morloc program and optionally install it.\n-- Shared by `morloc make --install` and `morloc install --build`.\nmakeAndInstall ::\n Maybe Path -> Maybe String -> Code -> [T.Text] -> Int ->\n Config.Config -> BuildConfig -> Bool -> IO Bool\nmakeAndInstall path outfile code extraIncludes verbosity config buildConfig force = do\n let action = do\n MM.modify (\\s -> s {stateInstall = True, stateInstallForce = force})\n M.writeProgram translator path code\n result <- MM.runMorlocMonad outfile verbosity config buildConfig action\n passed <- MM.writeMorlocReturn result\n if passed\n then do\n let (_, finalState) = result\n -- Merge include fields from all loaded packages.\n -- Nothing = include everything (default mode).\n -- Just [...] = strict allowlist mode.\n pkgIncludes = map packageInclude (statePackageMeta finalState)\n mergedIncludes\n | not (null extraIncludes) =\n -- CLI --include flags force strict mode\n Just (concatMap (fromMaybe []) pkgIncludes ++ extraIncludes)\n | all (== Nothing) pkgIncludes = Nothing\n | otherwise = Just (concatMap (fromMaybe []) pkgIncludes)\n allSources = concat (GMap.elems (stateSources finalState))\n directSourcePaths = [ p | Source{srcPath = Just p} <- allSources ]\n case stateInstallDir finalState of\n Nothing -> do\n putStrLn \"Error: install directory was not set during compilation\"\n return False\n Just installDir -> do\n let installName = takeFileName installDir\n packageRoot = case fmap takeDirectory path of\n Just \"\" -> \".\"\n Just d -> d\n Nothing -> \".\"\n -- Atomic install: clean up installDir on any failure so the user\n -- is not left with partial state requiring --force on retry.\n installResult <- try (do\n -- Only validate coverage in strict mode (explicit include patterns)\n case mergedIncludes of\n Just pats -> do\n Install.validateIncludeCoverage packageRoot pats directSourcePaths\n Nothing -> return ()\n Install.installProgram (Config.configHome config) installDir installName mergedIncludes force\n ) :: IO (Either SomeException ())\n case installResult of\n Right () -> return True\n Left e -> do\n dirExists <- doesDirectoryExist installDir\n if dirExists\n then do\n removeDirectoryRecursive installDir\n hPutStrLn stderr $ \"Cleaned up partial install: \" <> installDir\n else return ()\n hPutStrLn stderr $ show e\n return False\n else return False\n\n-- | build a Morloc program, generating the nexus and pool files\ncmdMake :: MakeCommand -> Int -> Config.Config -> BuildConfig -> IO Bool\ncmdMake args verbosity config buildConfig = do\n (path, code) <- readScript (makeExpression args) (makeScript args)\n outfile <- case makeOutfile args of\n \"\" -> return Nothing\n x -> return . Just $ x\n if makeInstall args\n then\n makeAndInstall path outfile code\n (map T.pack (makeInclude args)) verbosity config buildConfig (makeForce args)\n else do\n let action = do\n MM.modify (\\s -> s {stateInstall = False})\n M.writeProgram translator path code\n result <- MM.runMorlocMonad outfile verbosity config buildConfig action\n passed <- MM.writeMorlocReturn result\n return passed\n\n-- | Evaluate a morloc expression\ncmdEval :: EvalCommand -> Int -> Config.Config -> BuildConfig -> IO Bool\ncmdEval args verbosity config buildConfig = do\n let rawExpr = evalExpression args\n code = MT.pack (preprocessEvalInput rawExpr)\n tmpBase = Config.configTmpDir config\n saveName = evalSave args\n extraArgs = evalArgs args\n isSave = not (null saveName)\n exeName = if isSave then saveName else \"eval\"\n createDirectoryIfMissing True tmpBase\n bracket\n (do\n origDir <- getCurrentDirectory\n tmpDir <- createTempDirectory tmpBase \"morloc-eval-\"\n setCurrentDirectory tmpDir\n return (origDir, tmpDir))\n (\\(origDir, tmpDir) -> do\n setCurrentDirectory origDir\n cleanupTmpDir tmpDir)\n (\\(_origDir, tmpDir) -> do\n let action = do\n MM.modify (\\s -> s {stateEvalMode = True})\n if isSave then MM.modify (\\s -> s {stateInstall = True}) else return ()\n M.writeProgram translator Nothing (Code code)\n result <- MM.runMorlocMonad (Just exeName) verbosity config buildConfig action\n passed <- MM.writeMorlocReturn result\n if not passed\n then return False\n else\n if isSave\n then do\n let (_, finalState) = result\n pkgIncludes = map packageInclude (statePackageMeta finalState)\n mergedIncludes\n | all (== Nothing) pkgIncludes = Nothing\n | otherwise = Just (concatMap (fromMaybe []) pkgIncludes)\n case stateInstallDir finalState of\n Nothing -> do\n putStrLn \"Error: install directory was not set during compilation\"\n return False\n Just installDir -> do\n evalInstallResult <- try (do\n Install.installProgram (Config.configHome config) installDir saveName mergedIncludes True\n writeEvalMeta (Config.configHome config) saveName rawExpr\n ) :: IO (Either SomeException ())\n case evalInstallResult of\n Right () -> return True\n Left e -> do\n dirExists <- doesDirectoryExist installDir\n if dirExists\n then do\n removeDirectoryRecursive installDir\n hPutStrLn stderr $ \"Cleaned up partial install: \" <> installDir\n else return ()\n hPutStrLn stderr $ show e\n return False\n else do\n let exe = tmpDir exeName\n subcommand <- getFirstSubcommand exe\n let cmdArgs = subcommand : extraArgs\n runResult <- try (SP.callProcess exe cmdArgs) :: IO (Either SomeException ())\n case runResult of\n Right () -> return True\n Left e -> do\n putStrLn $ \"Error running expression: \" ++ show e\n return False)\n where\n cleanupTmpDir dir = do\n exists <- doesDirectoryExist dir\n if exists then removeDirectoryRecursive dir else return ()\n\n-- | Extract the first subcommand name from the manifest embedded in a wrapper script.\n-- Falls back to \"__expr__\" if the manifest cannot be parsed.\ngetFirstSubcommand :: FilePath -> IO String\ngetFirstSubcommand wrapperPath = do\n result <- try (readFile wrapperPath) :: IO (Either SomeException String)\n case result of\n Left _ -> return \"__expr__\"\n Right contents -> do\n let marker = \"### MANIFEST ###\"\n afterMarker = drop 1 $ dropWhile (/= marker) (lines contents)\n manifestStr = unlines afterMarker\n case JSON.eitherDecode (BL.fromStrict (MT.encodeUtf8 (MT.pack manifestStr))) of\n Right pm -> case pmCommands pm of\n (cmd : _) -> return (T.unpack (pcName cmd))\n [] -> return \"__expr__\"\n Left _ -> return \"__expr__\"\n\n-- | Write metadata about the saved eval expression\nwriteEvalMeta :: FilePath -> String -> String -> IO ()\nwriteEvalMeta cfgHome name expr = do\n now <- getCurrentTime\n let fdbDir = cfgHome \"fdb\"\n metaPath = fdbDir name ++ \".eval-meta\"\n timestamp = formatTime defaultTimeLocale \"%Y-%m-%dT%H:%M:%SZ\" now\n json = \"{\\\"expression\\\":\" ++ jsonEscape expr ++ \",\\\"timestamp\\\":\\\"\" ++ timestamp ++ \"\\\"}\"\n createDirectoryIfMissing True fdbDir\n writeFile metaPath json\n where\n jsonEscape s = \"\\\"\" ++ concatMap escChar s ++ \"\\\"\"\n escChar '\"' = \"\\\\\\\"\"\n escChar '\\\\' = \"\\\\\\\\\"\n escChar '\\n' = \"\\\\n\"\n escChar '\\t' = \"\\\\t\"\n escChar c = [c]\n\n-- | Preprocess eval input: replace top-level semicolons with newlines.\n-- Semicolons inside explicit brace blocks (depth > 0) are preserved.\n-- Leading whitespace after each replacement is stripped so the layout\n-- rule treats each statement as a new top-level declaration.\npreprocessEvalInput :: String -> String\npreprocessEvalInput = go (0 :: Int)\n where\n go _ [] = []\n go depth ('{' : rest) = '{' : go (depth + 1) rest\n go depth ('}' : rest) = '}' : go (max 0 (depth - 1)) rest\n go 0 (';' : rest) = '\\n' : go 0 (dropWhile (== ' ') rest)\n go depth ('\"' : rest) = '\"' : goString depth rest\n go depth (c : rest) = c : go depth rest\n\n goString depth [] = go depth []\n goString depth ('\"' : rest) = '\"' : go depth rest\n goString depth ('\\\\' : c : rest) = '\\\\' : c : goString depth rest\n goString depth (c : rest) = c : goString depth rest\n\ncmdTypecheck :: TypecheckCommand -> Int -> Config.Config -> BuildConfig -> IO Bool\ncmdTypecheck args _ config buildConfig = do\n (path, code) <- readScript (typecheckExpression args) (typecheckScript args)\n let verbosity = typecheckVerbose args\n if typecheckType args\n then case F.readType (unCode code) of\n (Left err') -> do\n putStrLn err'\n return False\n (Right x) -> do\n print x\n return True\n else\n if typecheckRealize args\n then do\n (passed, result) <-\n MM.runMorlocMonad\n Nothing\n verbosity\n config\n buildConfig\n ( M.typecheck path code\n >>= (generatePools . snd)\n )\n |>> writeTypecheckOutput verbosity\n putDoc (result <> \"\\n\")\n return passed\n else do\n (passed, result) <-\n MM.runMorlocMonad\n Nothing\n verbosity\n config\n buildConfig\n (M.typecheckFrontend path code)\n |>> writeFrontendTypecheckOutput verbosity\n putDoc (result <> \"\\n\")\n return passed\n\nwriteFrontendTypecheckOutput ::\n Int ->\n ((Either MorlocError [AnnoS (Indexed TypeU) Many Int], [MT.Text]), MorlocState) ->\n (Bool, MDoc)\nwriteFrontendTypecheckOutput _ ((Left e, _), st) = (False, MM.makeMorlocError st e)\nwriteFrontendTypecheckOutput 0 ((Right xs, _), st) = (True, vsep (map (writeFrontendTypes st) xs))\nwriteFrontendTypecheckOutput 1 x = writeFrontendTypecheckOutput 0 x -- no difference in verbosity\nwriteFrontendTypecheckOutput _ _ = (False, \"I don't know how to be that verbose\")\n\nwriteFrontendTypes :: MorlocState -> AnnoS (Indexed TypeU) Many Int -> MDoc\nwriteFrontendTypes st (AnnoS (Idx i t) _ _) =\n case Map.lookup i (stateName st) of\n (Just v) -> pretty v <+> \"::\" <+> prettyTypeU t\n Nothing -> \"? ::\" <+> prettyTypeU t\n\nwriteTypecheckOutput ::\n Int -> ((Either MorlocError [(Lang, [SerialManifold])], [MT.Text]), MorlocState) -> (Bool, MDoc)\nwriteTypecheckOutput _ ((Left e, _), st) = (False, MM.makeMorlocError st e)\nwriteTypecheckOutput _ ((Right pools, _), _) = (True, vsep $ map (uncurry writePool) pools)\n\nwritePool :: Lang -> [SerialManifold] -> MDoc\nwritePool lang manifolds = pretty lang <+> \"pool:\" <> \"\\n\" <> vsep (map pseudocodeSerialManifold manifolds) <> \"\\n\"\n\ncmdDump :: DumpCommand -> Int -> Config.Config -> BuildConfig -> IO Bool\ncmdDump args _ config buildConfig = do\n (path, code) <- readScript (dumpExpression args) (dumpScript args)\n let verbosity = dumpVerbose args\n ((x, _), st) <- MM.runMorlocMonad Nothing verbosity config buildConfig (F.parse path code)\n case x of\n (Left e) -> do\n putDoc $ MM.makeMorlocError st e\n return False\n (Right e) -> do\n putDoc $ prettyDAG e\n return True\n\ncmdInit :: InitCommand -> Config.Config -> IO Bool\ncmdInit ic config = MSC.configureAll (not (initQuiet ic)) (initForce ic) (initSlurmSupport ic) (initSanitize ic) config\n\ncmdNew :: NewCommand -> IO Bool\ncmdNew args = do\n let pkgFile = \"package.yaml\"\n exists <- doesFileExist pkgFile\n if exists\n then do\n hPutStrLn stderr \"Error: package.yaml already exists. Remove it first or use a different directory.\"\n return False\n else do\n name <-\n if null (newName args)\n then takeFileName <$> getCurrentDirectory\n else return (newName args)\n writeFile pkgFile $\n unlines\n [ \"name: \" ++ name\n , \"version: 0.1.0\"\n , \"homepage: null\"\n , \"synopsis: null\"\n , \"description: null\"\n , \"category: null\"\n , \"license: MIT\"\n , \"author: null\"\n , \"maintainer: null\"\n , \"github: null\"\n , \"bug-reports: null\"\n , \"dependencies: []\"\n , \"# Uncomment to restrict which files are copied during install.\"\n , \"# By default, all files are included (filtered by .morlocignore).\"\n , \"# include:\"\n , \"# - \\\"*.py\\\"\"\n , \"# - \\\"src/\\\"\"\n ]\n hPutStrLn stderr $ \"Created package.yaml for '\" ++ name ++ \"'\"\n return True\n\nprettyDAG :: DAG MVar e ExprI -> MDoc\nprettyDAG m0 = vsep (map prettyEntry (Map.toList m0))\n where\n prettyEntry :: (MVar, (ExprI, [(MVar, e)])) -> MDoc\n prettyEntry (k, (n, _)) = block 4 (pretty k) (vsep [pretty n])\n\n-- ======================================================================\n-- List command\n-- ======================================================================\n\n-- Lightweight JSON types for reading manifests\n\ndata ModuleManifest = ModuleManifest\n { mmName :: T.Text\n , mmVersion :: T.Text\n , mmSynopsis :: T.Text\n , mmExports :: [(T.Text, T.Text)]\n , mmMorlocDeps :: [T.Text]\n , mmReason :: T.Text\n }\n\ndata ProgramManifest = ProgramManifest\n { pmName :: T.Text\n , pmCommands :: [ProgramCommand]\n }\n\ndata ProgramCommand = ProgramCommand\n { pcName :: T.Text\n , pcReturnType :: T.Text\n , _pcArgSchemas :: [T.Text]\n }\n\ninstance JSON.FromJSON ModuleManifest where\n parseJSON = JSON.withObject \"ModuleManifest\" $ \\o ->\n ModuleManifest\n <$> o JSON..:? \"name\" JSON..!= \"\"\n <*> o JSON..:? \"version\" JSON..!= \"\"\n <*> o JSON..:? \"synopsis\" JSON..!= \"\"\n <*> (o JSON..:? \"exports\" JSON..!= [] >>= mapM parseExport)\n <*> o JSON..:? \"morloc_dependencies\" JSON..!= []\n <*> o JSON..:? \"install_reason\" JSON..!= \"\"\n where\n parseExport = JSON.withObject \"Export\" $ \\o ->\n (,) <$> o JSON..: \"name\" <*> o JSON..: \"type\"\n\ninstance JSON.FromJSON ProgramManifest where\n parseJSON = JSON.withObject \"ProgramManifest\" $ \\o ->\n ProgramManifest\n <$> o JSON..:? \"name\" JSON..!= \"\"\n <*> o JSON..:? \"commands\" JSON..!= []\n\ninstance JSON.FromJSON ProgramCommand where\n parseJSON = JSON.withObject \"ProgramCommand\" $ \\o ->\n ProgramCommand\n <$> o JSON..: \"name\"\n <*> o JSON..:? \"return_type\" JSON..!= \"\"\n <*> o JSON..:? \"arg_schemas\" JSON..!= []\n\n-- | Check if pattern is a subsequence of the target string (case-insensitive)\nsubsequenceMatch :: String -> String -> Bool\nsubsequenceMatch [] _ = True\nsubsequenceMatch _ [] = False\nsubsequenceMatch (p : ps) (t : ts)\n | toLower p == toLower t = subsequenceMatch ps ts\n | otherwise = subsequenceMatch (p : ps) ts\n\ncmdList :: ListCommand -> Config.Config -> IO Bool\ncmdList args config = do\n let fdbDir = Config.configHome config \"fdb\"\n libDir = Config.configLibrary config Config.configPlane config\n verbose = listVerbose args\n kind = listKind args\n pat = listPattern args\n\n -- Load module manifests\n allModules <-\n if kind /= Just ListPrograms\n then do\n mods <- loadModuleManifests fdbDir\n discovered <- discoverModules libDir fdbDir\n return (mods ++ discovered)\n else return []\n\n -- Load program manifests\n allPrograms <-\n if kind /= Just ListModules\n then loadProgramManifests fdbDir\n else return []\n\n -- Filter by pattern\n let modules = case pat of\n Nothing -> allModules\n Just p -> filter (\\m -> subsequenceMatch p (T.unpack (mmName m))) allModules\n programs = case pat of\n Nothing -> allPrograms\n Just p -> filter (\\m -> subsequenceMatch p (T.unpack (pmName m))) allPrograms\n\n -- For verbose mode, fill in exports from .loc files when manifest has none\n modules' <-\n if verbose > 0\n then mapM (fillModuleExports libDir) modules\n else return modules\n\n -- Print results\n if null modules' && null programs\n then putStrLn \"No installed modules or programs found.\"\n else do\n if not (null modules')\n then do\n putStrLn \"Modules:\"\n mapM_ (printModule verbose) modules'\n else return ()\n if not (null programs)\n then do\n if not (null modules') then putStrLn \"\" else return ()\n putStrLn \"Programs:\"\n mapM_ (printProgram verbose) programs\n else return ()\n\n return True\n\n-- | If a module has no exports in its manifest, scan its .loc file for type signatures\nfillModuleExports :: FilePath -> ModuleManifest -> IO ModuleManifest\nfillModuleExports libDir m\n | not (null (mmExports m)) = return m\n | otherwise = do\n let modDir = libDir T.unpack (mmName m)\n modName = T.unpack (mmName m)\n mainFile <- findMainLocFile modDir modName\n case mainFile of\n Nothing -> return m\n Just f -> do\n sigs <- extractTypeSignatures f\n return m {mmExports = sigs}\n\n-- | Extract top-level type signatures from a .loc file\nextractTypeSignatures :: FilePath -> IO [(T.Text, T.Text)]\nextractTypeSignatures path = do\n result <- try (TIO.readFile path) :: IO (Either SomeException T.Text)\n case result of\n Left _ -> return []\n Right content ->\n return\n . map parseSig\n . filter isTypeSig\n . T.lines\n $ content\n where\n isTypeSig ln =\n let stripped = T.stripStart ln\n in not (T.null stripped)\n && T.head stripped /= '-' -- not a comment\n && T.head stripped /= '{' -- not a block comment\n && T.isInfixOf \" :: \" stripped\n && not (T.isPrefixOf \"type \" stripped)\n && not (T.isPrefixOf \"source \" stripped)\n && not (T.isPrefixOf \"import \" stripped)\n && not (T.isPrefixOf \"module \" stripped)\n && not (T.isPrefixOf \"class \" stripped)\n && not (T.isPrefixOf \"instance \" stripped)\n\n parseSig ln =\n let (sigName, rest) = T.breakOn \" :: \" (T.stripStart ln)\n typ = T.strip (T.drop 4 rest) -- drop \" :: \"\n in (T.strip sigName, typ)\n\nloadModuleManifests :: FilePath -> IO [ModuleManifest]\nloadModuleManifests fdbDir = do\n result <- try (listDirectory fdbDir) :: IO (Either SomeException [FilePath])\n case result of\n Left _ -> return []\n Right entries -> do\n let moduleFiles = filter (\".module\" `isSuffixOf`) entries\n catMaybes\n <$> mapM\n ( \\f -> do\n r <- try (BL.readFile (fdbDir f)) :: IO (Either SomeException BL.ByteString)\n case r of\n Left _ -> return Nothing\n Right bs -> case JSON.eitherDecode bs of\n Right m -> return (Just m)\n Left _ -> return Nothing\n )\n moduleFiles\n\nloadProgramManifests :: FilePath -> IO [ProgramManifest]\nloadProgramManifests fdbDir = do\n result <- try (listDirectory fdbDir) :: IO (Either SomeException [FilePath])\n case result of\n Left _ -> return []\n Right entries -> do\n let manifestFiles = filter (\".manifest\" `isSuffixOf`) entries\n catMaybes\n <$> mapM\n ( \\f -> do\n r <- try (BL.readFile (fdbDir f)) :: IO (Either SomeException BL.ByteString)\n case r of\n Left _ -> return Nothing\n Right bs -> case JSON.eitherDecode bs of\n Right m ->\n let m' =\n if T.null (pmName m)\n then m {pmName = T.pack (dropExtension (takeFileName f))}\n else m\n in return (Just m')\n Left _ -> return Nothing\n )\n manifestFiles\n\n-- | Discover modules in the library that lack manifests\ndiscoverModules :: FilePath -> FilePath -> IO [ModuleManifest]\ndiscoverModules libDir fdbDir = do\n libExists <- doesDirectoryExist libDir\n if not libExists\n then return []\n else do\n entries <- listDirectory libDir\n catMaybes\n <$> mapM\n ( \\name -> do\n let manifestPath = fdbDir name ++ \".module\"\n moduleDir = libDir name\n hasManifest <- doesFileExist manifestPath\n isDir <- doesDirectoryExist moduleDir\n if hasManifest || not isDir\n then return Nothing\n else do\n -- Try to read package.yaml for basic info\n let pkgYaml = moduleDir \"package.yaml\"\n pkgExists <- doesFileExist pkgYaml\n if pkgExists\n then do\n r <- try (BL.readFile pkgYaml) :: IO (Either SomeException BL.ByteString)\n case r of\n Left _ -> return (Just (minimalManifest name))\n Right bs -> case decodePackageMeta bs of\n Just meta ->\n return . Just $\n ModuleManifest\n { mmName = if T.null (packageName meta) then T.pack name else packageName meta\n , mmVersion = packageVersion meta\n , mmSynopsis = packageSynopsis meta\n , mmExports = []\n , mmMorlocDeps = []\n , mmReason = \"\"\n }\n Nothing -> return (Just (minimalManifest name))\n else return (Just (minimalManifest name))\n )\n entries\n where\n minimalManifest name =\n ModuleManifest\n { mmName = T.pack name\n , mmVersion = \"\"\n , mmSynopsis = \"\"\n , mmExports = []\n , mmMorlocDeps = []\n , mmReason = \"\"\n }\n\nprintModule :: Int -> ModuleManifest -> IO ()\nprintModule verbose m = do\n let name = mmName m\n ver = if T.null (mmVersion m) then \"\" else \" \" <> T.unpack (mmVersion m)\n syn = if T.null (mmSynopsis m) then \"\" else \" \" <> T.unpack (mmSynopsis m)\n putStrLn $ \" \" <> T.unpack name <> ver <> syn\n if verbose > 0\n then mapM_ (\\(n, t) -> putStrLn $ \" \" <> T.unpack n <> \" :: \" <> T.unpack t) (mmExports m)\n else return ()\n\nprintProgram :: Int -> ProgramManifest -> IO ()\nprintProgram verbose p = do\n let name = pmName p\n cmds = pmCommands p\n cmdCount = length cmds\n summary = show cmdCount <> \" command\" <> (if cmdCount /= 1 then \"s\" else \"\")\n putStrLn $ \" \" <> T.unpack name <> \" \" <> summary\n if verbose > 0\n then\n mapM_ (\\c -> putStrLn $ \" \" <> T.unpack (pcName c) <> \" :: \" <> T.unpack (pcReturnType c)) cmds\n else return ()\n\n-- ======================================================================\n-- Uninstall command\n-- ======================================================================\n\ncmdUninstall :: UninstallCommand -> Config.Config -> IO Bool\ncmdUninstall args config = do\n let fdbDir = Config.configHome config \"fdb\"\n libDir = Config.configLibrary config Config.configPlane config\n binDir = Config.configHome config \"bin\"\n exeDir = Config.configHome config \"exe\"\n dryRun = uninstallDryRun args\n kind = uninstallKind args\n\n names <- if uninstallAll args\n then do\n fdbExists <- doesDirectoryExist fdbDir\n if not fdbExists\n then return []\n else do\n entries <- listDirectory fdbDir\n let moduleNames = [dropExtension f | f <- entries, \".module\" `isSuffixOf` f]\n return moduleNames\n else return (uninstallNames args)\n\n if null names\n then do\n if uninstallAll args\n then putStrLn \"No modules installed\"\n else putStrLn \"No module names specified. Use --all to uninstall all modules.\"\n return True\n else do\n let skipDepCheck = uninstallAll args\n allPassed <- mapM (\\name -> uninstallOne fdbDir libDir binDir exeDir dryRun skipDepCheck kind name) names\n let anyRemoved = or allPassed\n\n -- Regenerate completions if anything was actually removed\n if anyRemoved && not dryRun\n then Completion.regenerateCompletions False (Config.configHome config)\n else return ()\n\n return True\n\nuninstallOne ::\n FilePath -> FilePath -> FilePath -> FilePath -> Bool -> Bool -> Maybe ListKind -> String -> IO Bool\nuninstallOne fdbDir libDir binDir exeDir dryRun skipDepCheck kind name = do\n let moduleManifest = fdbDir name ++ \".module\"\n programManifest = fdbDir name ++ \".manifest\"\n moduleDir = libDir name\n\n hasModule <- doesFileExist moduleManifest\n hasModuleDir <- doesDirectoryExist moduleDir\n hasProgram <- doesFileExist programManifest\n\n let removeModule = (hasModule || hasModuleDir) && kind /= Just ListPrograms\n removeProgram = hasProgram && kind /= Just ListModules\n\n if not removeModule && not removeProgram\n then do\n putStrLn $ \"Nothing found for '\" <> name <> \"'\"\n return False\n else do\n -- Reverse dependency check for modules (skip when uninstalling all)\n if removeModule\n then do\n if not skipDepCheck then checkReverseDeps fdbDir name else return ()\n if dryRun\n then do\n putStrLn $ \"Would uninstall module '\" <> name <> \"'\"\n if hasModuleDir\n then putStrLn $ \" Remove: \" <> moduleDir\n else return ()\n if hasModule\n then putStrLn $ \" Remove: \" <> moduleManifest\n else return ()\n else do\n if hasModuleDir then removeDirectoryRecursive moduleDir else return ()\n if hasModule then removeFile moduleManifest else return ()\n hPutStrLn stderr $ \"Uninstalled module '\" <> name <> \"'\"\n else return ()\n\n if removeProgram\n then do\n let binPath = binDir name\n binExists <- doesFileExist binPath\n if dryRun\n then do\n putStrLn $ \"Would uninstall program '\" <> name <> \"'\"\n if binExists then putStrLn $ \" Remove: \" <> binPath else return ()\n -- Check for exe dir\n exeDirPath <- findExeDir exeDir name\n case exeDirPath of\n Just d -> putStrLn $ \" Remove: \" <> d\n Nothing -> return ()\n putStrLn $ \" Remove: \" <> programManifest\n else do\n if binExists then removeFile binPath else return ()\n exeDirPath <- findExeDir exeDir name\n case exeDirPath of\n Just d -> removeDirectoryRecursive d\n Nothing -> return ()\n removeFile programManifest\n hPutStrLn stderr $ \"Uninstalled program '\" <> name <> \"'\"\n else return ()\n\n return (removeModule || removeProgram)\n\n-- | Find the exe directory for a program\nfindExeDir :: FilePath -> String -> IO (Maybe FilePath)\nfindExeDir exeDir name = do\n exists <- doesDirectoryExist exeDir\n if not exists\n then return Nothing\n else do\n entries <- listDirectory exeDir\n -- Look for name or name-\n let matches = filter (\\e -> e == name || (name ++ \"-\") `isPrefixOf'` e) entries\n case matches of\n (m : _) -> return (Just (exeDir m))\n [] -> return Nothing\n where\n isPrefixOf' prefix str = take (length prefix) str == prefix\n\n-- | Check if any other modules depend on the one being uninstalled\ncheckReverseDeps :: FilePath -> String -> IO ()\ncheckReverseDeps fdbDir name = do\n result <- try (listDirectory fdbDir) :: IO (Either SomeException [FilePath])\n case result of\n Left _ -> return ()\n Right entries -> do\n let moduleFiles = filter (\".module\" `isSuffixOf`) entries\n nameT = T.pack name\n forM_ moduleFiles $ \\f -> do\n r <- try (BL.readFile (fdbDir f)) :: IO (Either SomeException BL.ByteString)\n case r of\n Left _ -> return ()\n Right bs -> case JSON.eitherDecode bs :: Either String ModuleManifest of\n Right m\n | nameT `elem` mmMorlocDeps m && mmName m /= nameT ->\n putStrLn $ \"Warning: module '\" <> T.unpack (mmName m) <> \"' depends on '\" <> name <> \"'\"\n _ -> return ()\n" }, { "path": "executable/UI.hs", "content": "{- |\nModule : UI\nDescription : CLI argument parsing with optparse-applicative\nCopyright : (c) Zebulun Arendsee, 2016-2026\nLicense : Apache-2.0\nMaintainer : z@morloc.io\n\nDefines the command-line interface for the @morloc@ executable using\noptparse-applicative: subcommands (make, typecheck, install, init, dump,\ncompletion), their options, and help text.\n-}\nmodule UI\n ( opts\n , CliCommand (..)\n , MakeCommand (..)\n , InitCommand (..)\n , InstallCommand (..)\n , TypecheckCommand (..)\n , DumpCommand (..)\n , ListCommand (..)\n , ListKind (..)\n , UninstallCommand (..)\n , NewCommand (..)\n , EvalCommand (..)\n ) where\n\nimport Morloc.Module (GitProtocol (..), OverwriteProtocol (..))\nimport Morloc.Version (versionStr)\nimport Options.Applicative\nimport qualified Options.Applicative.Extra as OAE\n\nopts :: ParserInfo CliCommand\nopts =\n info\n (cliParser <**> helper <**> OAE.simpleVersioner versionStr)\n ( fullDesc\n <> progDesc \"Call 'morloc make -h', 'morloc install -h', etc for details\"\n <> header (\"morloc v\" <> versionStr)\n )\n\ndata CliCommand\n = CmdMake MakeCommand\n | CmdInstall InstallCommand\n | CmdUninstall UninstallCommand\n | CmdList ListCommand\n | CmdTypecheck TypecheckCommand\n | CmdDump DumpCommand\n | CmdInit InitCommand\n | CmdNew NewCommand\n | CmdEval EvalCommand\n\ncliParser :: Parser CliCommand\ncliParser =\n hsubparser\n ( makeSubcommand\n <> installSubcommand\n <> uninstallSubcommand\n <> listSubcommand\n <> typecheckSubcommand\n <> dumpSubcommand\n <> initSubcommand\n <> newSubcommand\n <> evalSubcommand\n )\n\ndata MakeCommand = MakeCommand\n { makeExpression :: Bool\n , makeConfig :: String\n , makeVerbose :: Int\n , makeVanilla :: Bool\n , makeOutfile :: String\n , makeInstall :: Bool\n , makeForce :: Bool\n , makeInclude :: [String]\n , makeScript :: String\n }\n\nmakeCommandParser :: Parser MakeCommand\nmakeCommandParser =\n MakeCommand\n <$> optExpression\n <*> optConfig\n <*> optVerbose\n <*> optVanilla\n <*> optOutfile\n <*> optMakeInstall\n <*> optMakeForce\n <*> optMakeInclude\n <*> optScript\n\nmakeSubcommand :: Mod CommandFields CliCommand\nmakeSubcommand = command \"make\" (info (CmdMake <$> makeCommandParser) (progDesc \"Build a morloc script\"))\n\ndata InitCommand = InitCommand\n { initConfig :: String\n , initQuiet :: Bool\n , initVanilla :: Bool\n , initForce :: OverwriteProtocol\n , initSlurmSupport :: Bool\n , initSanitize :: Bool\n }\n\ninitCommandParser :: Parser InitCommand\ninitCommandParser =\n InitCommand\n <$> optConfig\n <*> optQuiet\n <*> optVanilla\n <*> optForce\n <*> optSlurmSupport\n <*> optSanitize\n\ninitSubcommand :: Mod CommandFields CliCommand\ninitSubcommand = command \"init\" (info (CmdInit <$> initCommandParser) (progDesc \"Initialize morloc environment\"))\n\ndata NewCommand = NewCommand\n { newName :: String\n }\n\nnewCommandParser :: Parser NewCommand\nnewCommandParser =\n NewCommand\n <$> strArgument\n ( metavar \"NAME\"\n <> value \"\"\n <> help \"Package name (defaults to current directory name)\"\n )\n\nnewSubcommand :: Mod CommandFields CliCommand\nnewSubcommand = command \"new\" (info (CmdNew <$> newCommandParser) (progDesc \"Create a new morloc package\"))\n\ndata InstallCommand = InstallCommand\n { installConfig :: String\n , installVanilla :: Bool\n , installVerbose :: Int\n , installForce :: OverwriteProtocol\n , installUseSSH :: GitProtocol\n , installNoTypecheck :: Bool\n , installBuild :: Bool\n , installModuleStrings :: [String]\n }\n\nmakeInstallParser :: Parser InstallCommand\nmakeInstallParser =\n InstallCommand\n <$> optConfig\n <*> optVanilla\n <*> optVerbose\n <*> optForce\n <*> optUseSSH\n <*> optNoTypecheck\n <*> optInstallBuild\n <*> optModuleStrings\n\ninstallSubcommand :: Mod CommandFields CliCommand\ninstallSubcommand = command \"install\" (info (CmdInstall <$> makeInstallParser) (progDesc \"Install a morloc module\"))\n\ndata TypecheckCommand = TypecheckCommand\n { typecheckConfig :: String\n , typecheckVanilla :: Bool\n , typecheckType :: Bool\n , typecheckRaw :: Bool\n , typecheckExpression :: Bool\n , typecheckVerbose :: Int\n , typecheckRealize :: Bool\n , typecheckScript :: String\n }\n\nmakeTypecheckParser :: Parser TypecheckCommand\nmakeTypecheckParser =\n TypecheckCommand\n <$> optConfig\n <*> optVanilla\n <*> optType\n <*> optRaw\n <*> optExpression\n <*> optVerbose\n <*> optRealize\n <*> optScript\n\ntypecheckSubcommand :: Mod CommandFields CliCommand\ntypecheckSubcommand =\n command\n \"typecheck\"\n (info (CmdTypecheck <$> makeTypecheckParser) (progDesc \"Typecheck a morloc program\"))\n\ndumpSubcommand :: Mod CommandFields CliCommand\ndumpSubcommand =\n command \"dump\" (info (CmdDump <$> makeDumpParser) (progDesc \"Dump parsed code\"))\n\ndata DumpCommand = DumpCommand\n { dumpConfig :: String\n , dumpVanilla :: Bool\n , dumpVerbose :: Int\n , dumpExpression :: Bool\n , dumpScript :: String\n }\n\nmakeDumpParser :: Parser DumpCommand\nmakeDumpParser =\n DumpCommand\n <$> optConfig\n <*> optVanilla\n <*> optVerbose\n <*> optExpression\n <*> optScript\n\ndata ListKind = ListModules | ListPrograms\n deriving (Show, Eq)\n\ndata ListCommand = ListCommand\n { listPattern :: Maybe String\n , listConfig :: String\n , listVanilla :: Bool\n , listVerbose :: Int\n , listKind :: Maybe ListKind\n }\n\nmakeListParser :: Parser ListCommand\nmakeListParser =\n ListCommand\n <$> optListPattern\n <*> optConfig\n <*> optVanilla\n <*> optVerbose\n <*> optListKind\n\nlistSubcommand :: Mod CommandFields CliCommand\nlistSubcommand =\n command \"list\" (info (CmdList <$> makeListParser) (progDesc \"List installed modules and programs\"))\n\ndata UninstallCommand = UninstallCommand\n { uninstallNames :: [String]\n , uninstallConfig :: String\n , uninstallVanilla :: Bool\n , uninstallKind :: Maybe ListKind\n , uninstallDryRun :: Bool\n , uninstallAll :: Bool\n }\n\nmakeUninstallParser :: Parser UninstallCommand\nmakeUninstallParser =\n UninstallCommand\n <$> optUninstallNamesOrNone\n <*> optConfig\n <*> optVanilla\n <*> optUninstallKind\n <*> optDryRun\n <*> optUninstallAll\n\nuninstallSubcommand :: Mod CommandFields CliCommand\nuninstallSubcommand =\n command\n \"uninstall\"\n (info (CmdUninstall <$> makeUninstallParser) (progDesc \"Uninstall a module or program\"))\n\noptExpression :: Parser Bool\noptExpression =\n switch\n ( long \"expression\"\n <> short 'e'\n <> help \"Read script as string rather than file\"\n )\n\noptVanilla :: Parser Bool\noptVanilla =\n switch\n ( long \"vanilla\"\n <> help \"Ignore local configuration files\"\n )\n\noptForce :: Parser OverwriteProtocol\noptForce =\n flag\n DoNotOverwrite\n ForceOverwrite\n ( long \"force\"\n <> short 'f'\n <> help \"Overwrite files if they already exist\"\n )\n\noptUseSSH :: Parser GitProtocol\noptUseSSH =\n flag\n HttpsProtocol\n SshProtocol\n ( long \"ssh\"\n <> help \"Use SSH protocol for remote git access\"\n )\n\noptNoTypecheck :: Parser Bool\noptNoTypecheck =\n switch\n ( long \"no-typecheck\"\n <> help \"Skip typechecking during install\"\n )\n\noptInstallBuild :: Parser Bool\noptInstallBuild =\n switch\n ( long \"build\"\n <> short 'b'\n <> help \"Build and install executable after module install\"\n )\n\noptModuleStrings :: Parser [String]\noptModuleStrings =\n some -- one or more\n . strArgument\n $ ( metavar \"INSTALL\"\n <> help \"Module install strings\"\n )\n\noptRaw :: Parser Bool\noptRaw =\n switch\n ( long \"raw\"\n <> help \"Print raw objects\"\n )\n\noptSlurmSupport :: Parser Bool\noptSlurmSupport =\n switch\n ( long \"slurm\"\n <> help \"Allow use of SLURM for remote jobs\"\n )\n\noptSanitize :: Parser Bool\noptSanitize =\n switch\n ( long \"sanitize\"\n <> help \"Enable alignment sanitizer for debugging memory layout issues\"\n )\n\noptVerbose :: Parser Int\noptVerbose = length <$> many (flag' () (short 'v'))\n\noptQuiet :: Parser Bool\noptQuiet =\n switch\n ( long \"quiet\"\n <> short 'q'\n <> help \"Print minimal output to STDERR\"\n )\n\noptRealize :: Parser Bool\noptRealize =\n switch\n ( long \"realize\"\n <> short 'r'\n <> help \"Typecheck the composition realizations\"\n )\n\noptConfig :: Parser String\noptConfig =\n strOption\n ( long \"config\"\n <> metavar \"CONFIG\"\n <> value \"\"\n <> help \"Use this config rather than the one in morloc home\"\n )\n\noptOutfile :: Parser String\noptOutfile =\n strOption\n ( long \"outfile\"\n <> short 'o'\n <> metavar \"OUT\"\n <> value \"\"\n <> showDefault\n <> help \"The name of the generated executable\"\n )\n\noptMakeInstall :: Parser Bool\noptMakeInstall =\n switch\n ( long \"install\"\n <> help \"Install module to PATH\"\n )\n\noptMakeForce :: Parser Bool\noptMakeForce =\n switch\n ( long \"force\"\n <> short 'f'\n <> help \"Overwrite existing install\"\n )\n\noptMakeInclude :: Parser [String]\noptMakeInclude =\n many\n ( strOption\n ( long \"include\"\n <> metavar \"PATTERN\"\n <> help \"File pattern to include in install\"\n )\n )\n\noptScript :: Parser String\noptScript = argument str (metavar \"